Add metadata to query, move metatagging of metadata to metatagger, tweaks to the generate caching so that the cached data becomes more stable (and is GIT-friendlier)

This commit is contained in:
pietervdvn 2021-04-25 13:25:03 +02:00
parent 591ab349a6
commit 54f01ba554
7 changed files with 129 additions and 66 deletions

View file

@ -1,22 +1,10 @@
import {UIEventSource} from "../UIEventSource"; import {UIEventSource} from "../UIEventSource";
import {ElementStorage} from "../ElementStorage"; import {ElementStorage} from "../ElementStorage";
import {OsmObject, OsmObjectMeta} from "../Osm/OsmObject"; import {OsmObject, OsmObjectMeta} from "../Osm/OsmObject";
import SimpleMetaTagger from "../SimpleMetaTagger";
export default class UpdateTagsFromOsmAPI { export default class UpdateTagsFromOsmAPI {
public static readonly metaTagger = new SimpleMetaTagger(
["_last_edit:contributor",
"_last_edit:contributor:uid",
"_last_edit:changeset",
"_last_edit:timestamp",
"_version_number"],
"Information about the last edit of this object. \n\nIMPORTANT: this data is _only_ loaded when the popup is added. This means it should _not_ be used to render icons!",
() => {/*Do nothing - this is only added for documentation reasons*/
}
)
/*** /***
* This actor downloads the element from the OSM-API and updates the corresponding tags in the UI-updater. * This actor downloads the element from the OSM-API and updates the corresponding tags in the UI-updater.
*/ */

View file

@ -182,8 +182,8 @@ export default class GeoJsonSource implements FeatureSource {
self.seenids.add(feature.properties.id) self.seenids.add(feature.properties.id)
let freshness: Date = time; let freshness: Date = time;
if (feature["_timestamp"] !== undefined) { if (feature.properties["_last_edit:timestamp"] !== undefined) {
freshness = new Date(feature["_timestamp"]) freshness = new Date(feature["_last_edit:timestamp"])
} }
newFeatures.push({feature: feature, freshness: freshness}) newFeatures.push({feature: feature, freshness: freshness})

View file

@ -3,6 +3,7 @@ import SimpleMetaTagger from "./SimpleMetaTagger";
import {ExtraFunction} from "./ExtraFunction"; import {ExtraFunction} from "./ExtraFunction";
import State from "../State"; import State from "../State";
import {Relation} from "./Osm/ExtractRelations"; import {Relation} from "./Osm/ExtractRelations";
import {meta} from "@turf/turf";
interface Params { interface Params {
@ -23,9 +24,15 @@ export default class MetaTagging {
* The features are a list of geojson-features, with a "properties"-field and geometry * The features are a list of geojson-features, with a "properties"-field and geometry
*/ */
static addMetatags(features: { feature: any; freshness: Date }[], static addMetatags(features: { feature: any; freshness: Date }[],
relations: Map<string, { role: string, relation: Relation }[]>, layers: LayerConfig[]) { relations: Map<string, { role: string, relation: Relation }[]>,
layers: LayerConfig[],
includeDates = true) {
for (const metatag of SimpleMetaTagger.metatags) { for (const metatag of SimpleMetaTagger.metatags) {
if(metatag.includesDates && !includeDates){
// We do not add dated entries
continue;
}
try { try {
metatag.addMetaTags(features); metatag.addMetaTags(features);
} catch (e) { } catch (e) {

View file

@ -11,10 +11,12 @@ export class Overpass {
public static testUrl: string = null public static testUrl: string = null
private _filter: TagsFilter private _filter: TagsFilter
private readonly _extraScripts: string[]; private readonly _extraScripts: string[];
private _includeMeta: boolean;
constructor(filter: TagsFilter, extraScripts: string[]) { constructor(filter: TagsFilter, extraScripts: string[], includeMeta = true) {
this._filter = filter this._filter = filter
this._extraScripts = extraScripts; this._extraScripts = extraScripts;
this._includeMeta = includeMeta;
} }
queryGeoJson(bounds: Bounds, continuation: ((any, date: Date) => void), onFail: ((reason) => void)): void { queryGeoJson(bounds: Bounds, continuation: ((any, date: Date) => void), onFail: ((reason) => void)): void {
@ -58,7 +60,7 @@ export class Overpass {
filter += '(' + extraScript + ');'; filter += '(' + extraScript + ');';
} }
const query = const query =
'[out:json][timeout:25]' + bbox + ';(' + filter + ');out body;>;out skel qt;' `[out:json][timeout:25]${bbox};(${filter});out body;${this._includeMeta ? 'out meta;' : ''}>;out skel qt;`
return "https://overpass-api.de/api/interpreter?data=" + encodeURIComponent(query) return "https://overpass-api.de/api/interpreter?data=" + encodeURIComponent(query)
} }
} }

View file

@ -11,30 +11,36 @@ import UpdateTagsFromOsmAPI from "./Actors/UpdateTagsFromOsmAPI";
export default class SimpleMetaTagger { export default class SimpleMetaTagger {
public readonly keys: string[];
public readonly doc: string;
private readonly _f: (feature: any, index: number, freshness: Date) => void;
constructor(keys: string[], doc: string, f: ((feature: any, index: number, freshness: Date) => void)) {
this.keys = keys;
this.doc = doc;
this._f = f;
for (const key of keys) {
if (!key.startsWith('_')) {
throw `Incorrect metakey ${key}: it should start with underscore (_)`
}
}
}
addMetaTags(features: { feature: any, freshness: Date }[]) {
for (let i = 0; i < features.length; i++) {
let feature = features[i];
this._f(feature.feature, i, feature.freshness);
}
}
static coder: any; static coder: any;
private static latlon = new SimpleMetaTagger(["_lat", "_lon"], "The latitude and longitude of the point (or centerpoint in the case of a way/area)", public static readonly objectMetaInfo = new SimpleMetaTagger(
{
keys: ["_last_edit:contributor",
"_last_edit:contributor:uid",
"_last_edit:changeset",
"_last_edit:timestamp",
"_version_number"],
doc: "Information about the last edit of this object."
},
(feature) => {/*Note: also handled by 'UpdateTagsFromOsmAPI'*/
const tgs = feature.properties;
tgs["_last_edit:contributor"] = tgs["user"]
tgs["_last_edit:contributor:uid"] = tgs["uid"]
tgs["_last_edit:changeset"] = tgs["changeset"]
tgs["_last_edit:timestamp"] = tgs["timestamp"]
tgs["_version_number"] = tgs["version"]
delete tgs["timestamp"]
delete tgs["version"]
delete tgs["changeset"]
delete tgs["user"]
delete tgs["uid"]
}
)
private static latlon = new SimpleMetaTagger({
keys: ["_lat", "_lon"],
doc: "The latitude and longitude of the point (or centerpoint in the case of a way/area)"
},
(feature => { (feature => {
const centerPoint = GeoOperations.centerpoint(feature); const centerPoint = GeoOperations.centerpoint(feature);
const lat = centerPoint.geometry.coordinates[1]; const lat = centerPoint.geometry.coordinates[1];
@ -46,7 +52,10 @@ export default class SimpleMetaTagger {
}) })
); );
private static surfaceArea = new SimpleMetaTagger( private static surfaceArea = new SimpleMetaTagger(
["_surface", "_surface:ha"], "The surface area of the feature, in square meters and in hectare. Not set on points and ways", {
keys: ["_surface", "_surface:ha"],
doc: "The surface area of the feature, in square meters and in hectare. Not set on points and ways"
},
(feature => { (feature => {
const sqMeters = GeoOperations.surfaceAreaInSqMeters(feature); const sqMeters = GeoOperations.surfaceAreaInSqMeters(feature);
feature.properties["_surface"] = "" + sqMeters; feature.properties["_surface"] = "" + sqMeters;
@ -54,20 +63,24 @@ export default class SimpleMetaTagger {
feature.area = sqMeters; feature.area = sqMeters;
}) })
); );
private static lngth = new SimpleMetaTagger( private static lngth = new SimpleMetaTagger(
["_length", "_length:km"], "The total length of a feature in meters (and in kilometers, rounded to one decimal for '_length:km'). For a surface, the length of the perimeter", {
keys: ["_length", "_length:km"],
doc: "The total length of a feature in meters (and in kilometers, rounded to one decimal for '_length:km'). For a surface, the length of the perimeter"
},
(feature => { (feature => {
const l = GeoOperations.lengthInMeters(feature) const l = GeoOperations.lengthInMeters(feature)
feature.properties["_length"] = "" + l feature.properties["_length"] = "" + l
const km = Math.floor(l / 1000) const km = Math.floor(l / 1000)
const kmRest = Math.round((l - km * 1000) / 100) const kmRest = Math.round((l - km * 1000) / 100)
feature.properties["_length:km"] = "" + km+ "." + kmRest feature.properties["_length:km"] = "" + km + "." + kmRest
}) })
) )
private static country = new SimpleMetaTagger( private static country = new SimpleMetaTagger(
["_country"], "The country code of the property (with latlon2country)", {
keys: ["_country"],
doc: "The country code of the property (with latlon2country)"
},
feature => { feature => {
@ -87,10 +100,13 @@ export default class SimpleMetaTagger {
} }
) )
private static isOpen = new SimpleMetaTagger( private static isOpen = new SimpleMetaTagger(
["_isOpen", "_isOpen:description"], {
"If 'opening_hours' is present, it will add the current state of the feature (being 'yes' or 'no')", keys: ["_isOpen", "_isOpen:description"],
doc: "If 'opening_hours' is present, it will add the current state of the feature (being 'yes' or 'no')",
includesDates: true
},
(feature => { (feature => {
if(Utils.runningFromConsole){ if (Utils.runningFromConsole) {
// We are running from console, thus probably creating a cache // We are running from console, thus probably creating a cache
// isOpen is irrelevant // isOpen is irrelevant
return return
@ -155,7 +171,10 @@ export default class SimpleMetaTagger {
}) })
) )
private static directionSimplified = new SimpleMetaTagger( private static directionSimplified = new SimpleMetaTagger(
["_direction:simplified", "_direction:leftright"], "_direction:simplified turns 'camera:direction' and 'direction' into either 0, 45, 90, 135, 180, 225, 270 or 315, whichever is closest. _direction:leftright is either 'left' or 'right', which is left-looking on the map or 'right-looking' on the map", {
keys: ["_direction:simplified", "_direction:leftright"],
doc: "_direction:simplified turns 'camera:direction' and 'direction' into either 0, 45, 90, 135, 180, 225, 270 or 315, whichever is closest. _direction:leftright is either 'left' or 'right', which is left-looking on the map or 'right-looking' on the map"
},
(feature => { (feature => {
const tags = feature.properties; const tags = feature.properties;
const direction = tags["camera:direction"] ?? tags["direction"]; const direction = tags["camera:direction"] ?? tags["direction"];
@ -178,8 +197,10 @@ export default class SimpleMetaTagger {
}) })
) )
private static carriageWayWidth = new SimpleMetaTagger( private static carriageWayWidth = new SimpleMetaTagger(
["_width:needed", "_width:needed:no_pedestrians", "_width:difference"], {
"Legacy for a specific project calculating the needed width for safe traffic on a road. Only activated if 'width:carriageway' is present", keys: ["_width:needed", "_width:needed:no_pedestrians", "_width:difference"],
doc: "Legacy for a specific project calculating the needed width for safe traffic on a road. Only activated if 'width:carriageway' is present"
},
(feature: any, index: number) => { (feature: any, index: number) => {
const properties = feature.properties; const properties = feature.properties;
@ -286,8 +307,11 @@ export default class SimpleMetaTagger {
} }
); );
private static currentTime = new SimpleMetaTagger( private static currentTime = new SimpleMetaTagger(
["_now:date", "_now:datetime", "_loaded:date", "_loaded:_datetime"], {
"Adds the time that the data got loaded - pretty much the time of downloading from overpass. The format is YYYY-MM-DD hh:mm, aka 'sortable' aka ISO-8601-but-not-entirely", keys: ["_now:date", "_now:datetime", "_loaded:date", "_loaded:_datetime"],
doc: "Adds the time that the data got loaded - pretty much the time of downloading from overpass. The format is YYYY-MM-DD hh:mm, aka 'sortable' aka ISO-8601-but-not-entirely",
includesDates: true
},
(feature, _, freshness) => { (feature, _, freshness) => {
const now = new Date(); const now = new Date();
@ -318,9 +342,26 @@ export default class SimpleMetaTagger {
SimpleMetaTagger.isOpen, SimpleMetaTagger.isOpen,
SimpleMetaTagger.carriageWayWidth, SimpleMetaTagger.carriageWayWidth,
SimpleMetaTagger.directionSimplified, SimpleMetaTagger.directionSimplified,
SimpleMetaTagger.currentTime SimpleMetaTagger.currentTime,
SimpleMetaTagger.objectMetaInfo
]; ];
public readonly keys: string[];
public readonly doc: string;
public readonly includesDates: boolean
private readonly _f: (feature: any, index: number, freshness: Date) => void;
constructor(docs: { keys: string[], doc: string, includesDates?: boolean }, f: ((feature: any, index: number, freshness: Date) => void)) {
this.keys = docs.keys;
this.doc = docs.doc;
this._f = f;
this.includesDates = docs.includesDates ?? false;
for (const key of docs.keys) {
if (!key.startsWith('_')) {
throw `Incorrect metakey ${key}: it should start with underscore (_)`
}
}
}
static GetCountryCodeFor(lon: number, lat: number, callback: (country: string) => void) { static GetCountryCodeFor(lon: number, lat: number, callback: (country: string) => void) {
SimpleMetaTagger.coder?.GetCountryCodeFor(lon, lat, callback) SimpleMetaTagger.coder?.GetCountryCodeFor(lon, lat, callback)
@ -337,7 +378,7 @@ export default class SimpleMetaTagger {
]; ];
for (const metatag of SimpleMetaTagger.metatags.concat(UpdateTagsFromOsmAPI.metaTagger)) { for (const metatag of SimpleMetaTagger.metatags) {
subElements.push( subElements.push(
new Combine([ new Combine([
"<h3>", metatag.keys.join(", "), "</h3>", "<h3>", metatag.keys.join(", "), "</h3>",
@ -349,5 +390,12 @@ export default class SimpleMetaTagger {
return new Combine(subElements) return new Combine(subElements)
} }
addMetaTags(features: { feature: any, freshness: Date }[]) {
for (let i = 0; i < features.length; i++) {
let feature = features[i];
this._f(feature.feature, i, feature.freshness);
}
}
} }

View file

@ -26,6 +26,7 @@ export default class Hash {
} }
const hash = new UIEventSource<string>(window.location.hash.substr(1)); const hash = new UIEventSource<string>(window.location.hash.substr(1));
hash.addCallback(h => { hash.addCallback(h => {
console.trace("Hash was changed into ",h)
if (h === "undefined") { if (h === "undefined") {
console.warn("Got a literal 'undefined' as hash, ignoring") console.warn("Got a literal 'undefined' as hash, ignoring")
h = undefined; h = undefined;

View file

@ -14,6 +14,8 @@ import ScriptUtils from "./ScriptUtils";
import ExtractRelations from "../Logic/Osm/ExtractRelations"; import ExtractRelations from "../Logic/Osm/ExtractRelations";
import * as OsmToGeoJson from "osmtogeojson"; import * as OsmToGeoJson from "osmtogeojson";
import MetaTagging from "../Logic/MetaTagging"; import MetaTagging from "../Logic/MetaTagging";
import LayerConfig from "../Customizations/JSON/LayerConfig";
import {GeoOperations} from "../Logic/GeoOperations";
function createOverpassObject(theme: LayoutConfig) { function createOverpassObject(theme: LayoutConfig) {
@ -114,8 +116,6 @@ async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass)/
await ScriptUtils.sleep(1000) await ScriptUtils.sleep(1000)
} }
} }
} }
} }
@ -124,6 +124,7 @@ async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass)/
async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig) { async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig) {
let processed = 0; let processed = 0;
const layerIndex = theme.LayerIndex();
for (let x = r.xstart; x <= r.xend; x++) { for (let x = r.xstart; x <= r.xend; x++) {
for (let y = r.ystart; y <= r.yend; y++) { for (let y = r.ystart; y <= r.yend; y++) {
processed++; processed++;
@ -150,7 +151,6 @@ async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig)
} }
} }
const featuresFreshness = geojson.features.map(feature => { const featuresFreshness = geojson.features.map(feature => {
feature["_timestamp"] = rawOsm.osm3s.timestamp_osm_base;
return ({ return ({
freshness: osmTime, freshness: osmTime,
feature: feature feature: feature
@ -158,7 +158,25 @@ async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig)
}); });
// Extract the relationship information // Extract the relationship information
const relations = ExtractRelations.BuildMembershipTable(ExtractRelations.GetRelationElements(rawOsm)) const relations = ExtractRelations.BuildMembershipTable(ExtractRelations.GetRelationElements(rawOsm))
MetaTagging.addMetatags(featuresFreshness, relations, theme.layers); MetaTagging.addMetatags(featuresFreshness, relations, theme.layers, false);
for (const feature of geojson.features) {
const layer = layerIndex.get(feature["_matching_layer_id"])
if (layer === undefined) {
console.error("Didn't find a layer for " + feature["_matching_layer_id"])
continue
}
if (layer.wayHandling == LayerConfig.WAYHANDLING_CENTER_ONLY) {
const centerpoint = GeoOperations.centerpointCoordinates(feature)
feature.geometry.type = "Point"
feature.geometry["coordinates"] = centerpoint;
}
}
writeFileSync(geoJsonName(targetdir, x, y, r.zoomlevel), JSON.stringify(geojson, null, " ")) writeFileSync(geoJsonName(targetdir, x, y, r.zoomlevel), JSON.stringify(geojson, null, " "))
@ -168,7 +186,6 @@ async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig)
} }
async function splitPerLayer(targetdir: string, r: TileRange, theme: LayoutConfig) { async function splitPerLayer(targetdir: string, r: TileRange, theme: LayoutConfig) {
let processed = 0;
const z = r.zoomlevel; const z = r.zoomlevel;
for (let x = r.xstart; x <= r.xend; x++) { for (let x = r.xstart; x <= r.xend; x++) {
for (let y = r.ystart; y <= r.yend; y++) { for (let y = r.ystart; y <= r.yend; y++) {
@ -177,10 +194,10 @@ async function splitPerLayer(targetdir: string, r: TileRange, theme: LayoutConfi
for (const layer of theme.layers) { for (const layer of theme.layers) {
const geojson = JSON.parse(file) const geojson = JSON.parse(file)
geojson.features = geojson.features.filter(f => f._matching_layer_id === layer.id) geojson.features = geojson.features.filter(f => f._matching_layer_id === layer.id)
if(geojson.features.length == 0){ if (geojson.features.length == 0) {
continue; continue;
} }
const new_path = geoJsonName(targetdir+"_"+layer.id, x, y, z); const new_path = geoJsonName(targetdir + "_" + layer.id, x, y, z);
writeFileSync(new_path, JSON.stringify(geojson, null, " ")) writeFileSync(new_path, JSON.stringify(geojson, null, " "))
} }