From 54f01ba5542985d65f6ab87d5434064f1f8e1234 Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Sun, 25 Apr 2021 13:25:03 +0200 Subject: [PATCH] Add metadata to query, move metatagging of metadata to metatagger, tweaks to the generate caching so that the cached data becomes more stable (and is GIT-friendlier) --- Logic/Actors/UpdateTagsFromOsmAPI.ts | 12 --- Logic/FeatureSource/GeoJsonSource.ts | 4 +- Logic/MetaTagging.ts | 9 +- Logic/Osm/Overpass.ts | 6 +- Logic/SimpleMetaTagger.ts | 128 ++++++++++++++++++--------- Logic/Web/Hash.ts | 1 + scripts/generateCache.ts | 35 ++++++-- 7 files changed, 129 insertions(+), 66 deletions(-) diff --git a/Logic/Actors/UpdateTagsFromOsmAPI.ts b/Logic/Actors/UpdateTagsFromOsmAPI.ts index f9ed6a0..e4ad0a1 100644 --- a/Logic/Actors/UpdateTagsFromOsmAPI.ts +++ b/Logic/Actors/UpdateTagsFromOsmAPI.ts @@ -1,22 +1,10 @@ import {UIEventSource} from "../UIEventSource"; import {ElementStorage} from "../ElementStorage"; import {OsmObject, OsmObjectMeta} from "../Osm/OsmObject"; -import SimpleMetaTagger from "../SimpleMetaTagger"; export default class UpdateTagsFromOsmAPI { - public static readonly metaTagger = new SimpleMetaTagger( - ["_last_edit:contributor", - "_last_edit:contributor:uid", - "_last_edit:changeset", - "_last_edit:timestamp", - "_version_number"], - "Information about the last edit of this object. \n\nIMPORTANT: this data is _only_ loaded when the popup is added. This means it should _not_ be used to render icons!", - () => {/*Do nothing - this is only added for documentation reasons*/ - } - ) - /*** * This actor downloads the element from the OSM-API and updates the corresponding tags in the UI-updater. */ diff --git a/Logic/FeatureSource/GeoJsonSource.ts b/Logic/FeatureSource/GeoJsonSource.ts index a655ca1..ce329b0 100644 --- a/Logic/FeatureSource/GeoJsonSource.ts +++ b/Logic/FeatureSource/GeoJsonSource.ts @@ -182,8 +182,8 @@ export default class GeoJsonSource implements FeatureSource { self.seenids.add(feature.properties.id) let freshness: Date = time; - if (feature["_timestamp"] !== undefined) { - freshness = new Date(feature["_timestamp"]) + if (feature.properties["_last_edit:timestamp"] !== undefined) { + freshness = new Date(feature["_last_edit:timestamp"]) } newFeatures.push({feature: feature, freshness: freshness}) diff --git a/Logic/MetaTagging.ts b/Logic/MetaTagging.ts index 1a646b2..0f1bd71 100644 --- a/Logic/MetaTagging.ts +++ b/Logic/MetaTagging.ts @@ -3,6 +3,7 @@ import SimpleMetaTagger from "./SimpleMetaTagger"; import {ExtraFunction} from "./ExtraFunction"; import State from "../State"; import {Relation} from "./Osm/ExtractRelations"; +import {meta} from "@turf/turf"; interface Params { @@ -23,9 +24,15 @@ export default class MetaTagging { * The features are a list of geojson-features, with a "properties"-field and geometry */ static addMetatags(features: { feature: any; freshness: Date }[], - relations: Map, layers: LayerConfig[]) { + relations: Map, + layers: LayerConfig[], + includeDates = true) { for (const metatag of SimpleMetaTagger.metatags) { + if(metatag.includesDates && !includeDates){ + // We do not add dated entries + continue; + } try { metatag.addMetaTags(features); } catch (e) { diff --git a/Logic/Osm/Overpass.ts b/Logic/Osm/Overpass.ts index 007cf9f..7888661 100644 --- a/Logic/Osm/Overpass.ts +++ b/Logic/Osm/Overpass.ts @@ -11,10 +11,12 @@ export class Overpass { public static testUrl: string = null private _filter: TagsFilter private readonly _extraScripts: string[]; + private _includeMeta: boolean; - constructor(filter: TagsFilter, extraScripts: string[]) { + constructor(filter: TagsFilter, extraScripts: string[], includeMeta = true) { this._filter = filter this._extraScripts = extraScripts; + this._includeMeta = includeMeta; } queryGeoJson(bounds: Bounds, continuation: ((any, date: Date) => void), onFail: ((reason) => void)): void { @@ -58,7 +60,7 @@ export class Overpass { filter += '(' + extraScript + ');'; } const query = - '[out:json][timeout:25]' + bbox + ';(' + filter + ');out body;>;out skel qt;' + `[out:json][timeout:25]${bbox};(${filter});out body;${this._includeMeta ? 'out meta;' : ''}>;out skel qt;` return "https://overpass-api.de/api/interpreter?data=" + encodeURIComponent(query) } } diff --git a/Logic/SimpleMetaTagger.ts b/Logic/SimpleMetaTagger.ts index 6b1acd4..743a294 100644 --- a/Logic/SimpleMetaTagger.ts +++ b/Logic/SimpleMetaTagger.ts @@ -11,30 +11,36 @@ import UpdateTagsFromOsmAPI from "./Actors/UpdateTagsFromOsmAPI"; export default class SimpleMetaTagger { - public readonly keys: string[]; - public readonly doc: string; - private readonly _f: (feature: any, index: number, freshness: Date) => void; - - constructor(keys: string[], doc: string, f: ((feature: any, index: number, freshness: Date) => void)) { - this.keys = keys; - this.doc = doc; - this._f = f; - for (const key of keys) { - if (!key.startsWith('_')) { - throw `Incorrect metakey ${key}: it should start with underscore (_)` - } - } - } - - addMetaTags(features: { feature: any, freshness: Date }[]) { - for (let i = 0; i < features.length; i++) { - let feature = features[i]; - this._f(feature.feature, i, feature.freshness); - } - } - static coder: any; - private static latlon = new SimpleMetaTagger(["_lat", "_lon"], "The latitude and longitude of the point (or centerpoint in the case of a way/area)", + public static readonly objectMetaInfo = new SimpleMetaTagger( + { + keys: ["_last_edit:contributor", + "_last_edit:contributor:uid", + "_last_edit:changeset", + "_last_edit:timestamp", + "_version_number"], + doc: "Information about the last edit of this object." + }, + (feature) => {/*Note: also handled by 'UpdateTagsFromOsmAPI'*/ + + const tgs = feature.properties; + tgs["_last_edit:contributor"] = tgs["user"] + tgs["_last_edit:contributor:uid"] = tgs["uid"] + tgs["_last_edit:changeset"] = tgs["changeset"] + tgs["_last_edit:timestamp"] = tgs["timestamp"] + tgs["_version_number"] = tgs["version"] + + delete tgs["timestamp"] + delete tgs["version"] + delete tgs["changeset"] + delete tgs["user"] + delete tgs["uid"] + } + ) + private static latlon = new SimpleMetaTagger({ + keys: ["_lat", "_lon"], + doc: "The latitude and longitude of the point (or centerpoint in the case of a way/area)" + }, (feature => { const centerPoint = GeoOperations.centerpoint(feature); const lat = centerPoint.geometry.coordinates[1]; @@ -46,7 +52,10 @@ export default class SimpleMetaTagger { }) ); private static surfaceArea = new SimpleMetaTagger( - ["_surface", "_surface:ha"], "The surface area of the feature, in square meters and in hectare. Not set on points and ways", + { + keys: ["_surface", "_surface:ha"], + doc: "The surface area of the feature, in square meters and in hectare. Not set on points and ways" + }, (feature => { const sqMeters = GeoOperations.surfaceAreaInSqMeters(feature); feature.properties["_surface"] = "" + sqMeters; @@ -54,20 +63,24 @@ export default class SimpleMetaTagger { feature.area = sqMeters; }) ); - private static lngth = new SimpleMetaTagger( - ["_length", "_length:km"], "The total length of a feature in meters (and in kilometers, rounded to one decimal for '_length:km'). For a surface, the length of the perimeter", + { + keys: ["_length", "_length:km"], + doc: "The total length of a feature in meters (and in kilometers, rounded to one decimal for '_length:km'). For a surface, the length of the perimeter" + }, (feature => { const l = GeoOperations.lengthInMeters(feature) feature.properties["_length"] = "" + l const km = Math.floor(l / 1000) const kmRest = Math.round((l - km * 1000) / 100) - feature.properties["_length:km"] = "" + km+ "." + kmRest + feature.properties["_length:km"] = "" + km + "." + kmRest }) ) - private static country = new SimpleMetaTagger( - ["_country"], "The country code of the property (with latlon2country)", + { + keys: ["_country"], + doc: "The country code of the property (with latlon2country)" + }, feature => { @@ -87,15 +100,18 @@ export default class SimpleMetaTagger { } ) private static isOpen = new SimpleMetaTagger( - ["_isOpen", "_isOpen:description"], - "If 'opening_hours' is present, it will add the current state of the feature (being 'yes' or 'no')", + { + keys: ["_isOpen", "_isOpen:description"], + doc: "If 'opening_hours' is present, it will add the current state of the feature (being 'yes' or 'no')", + includesDates: true + }, (feature => { - if(Utils.runningFromConsole){ + if (Utils.runningFromConsole) { // We are running from console, thus probably creating a cache // isOpen is irrelevant return } - + const tagsSource = State.state.allElements.getEventSourceById(feature.properties.id); tagsSource.addCallbackAndRun(tags => { if (tags.opening_hours === undefined || tags._country === undefined) { @@ -155,7 +171,10 @@ export default class SimpleMetaTagger { }) ) private static directionSimplified = new SimpleMetaTagger( - ["_direction:simplified", "_direction:leftright"], "_direction:simplified turns 'camera:direction' and 'direction' into either 0, 45, 90, 135, 180, 225, 270 or 315, whichever is closest. _direction:leftright is either 'left' or 'right', which is left-looking on the map or 'right-looking' on the map", + { + keys: ["_direction:simplified", "_direction:leftright"], + doc: "_direction:simplified turns 'camera:direction' and 'direction' into either 0, 45, 90, 135, 180, 225, 270 or 315, whichever is closest. _direction:leftright is either 'left' or 'right', which is left-looking on the map or 'right-looking' on the map" + }, (feature => { const tags = feature.properties; const direction = tags["camera:direction"] ?? tags["direction"]; @@ -178,8 +197,10 @@ export default class SimpleMetaTagger { }) ) private static carriageWayWidth = new SimpleMetaTagger( - ["_width:needed", "_width:needed:no_pedestrians", "_width:difference"], - "Legacy for a specific project calculating the needed width for safe traffic on a road. Only activated if 'width:carriageway' is present", + { + keys: ["_width:needed", "_width:needed:no_pedestrians", "_width:difference"], + doc: "Legacy for a specific project calculating the needed width for safe traffic on a road. Only activated if 'width:carriageway' is present" + }, (feature: any, index: number) => { const properties = feature.properties; @@ -286,8 +307,11 @@ export default class SimpleMetaTagger { } ); private static currentTime = new SimpleMetaTagger( - ["_now:date", "_now:datetime", "_loaded:date", "_loaded:_datetime"], - "Adds the time that the data got loaded - pretty much the time of downloading from overpass. The format is YYYY-MM-DD hh:mm, aka 'sortable' aka ISO-8601-but-not-entirely", + { + keys: ["_now:date", "_now:datetime", "_loaded:date", "_loaded:_datetime"], + doc: "Adds the time that the data got loaded - pretty much the time of downloading from overpass. The format is YYYY-MM-DD hh:mm, aka 'sortable' aka ISO-8601-but-not-entirely", + includesDates: true + }, (feature, _, freshness) => { const now = new Date(); @@ -318,9 +342,26 @@ export default class SimpleMetaTagger { SimpleMetaTagger.isOpen, SimpleMetaTagger.carriageWayWidth, SimpleMetaTagger.directionSimplified, - SimpleMetaTagger.currentTime + SimpleMetaTagger.currentTime, + SimpleMetaTagger.objectMetaInfo ]; + public readonly keys: string[]; + public readonly doc: string; + public readonly includesDates: boolean + private readonly _f: (feature: any, index: number, freshness: Date) => void; + + constructor(docs: { keys: string[], doc: string, includesDates?: boolean }, f: ((feature: any, index: number, freshness: Date) => void)) { + this.keys = docs.keys; + this.doc = docs.doc; + this._f = f; + this.includesDates = docs.includesDates ?? false; + for (const key of docs.keys) { + if (!key.startsWith('_')) { + throw `Incorrect metakey ${key}: it should start with underscore (_)` + } + } + } static GetCountryCodeFor(lon: number, lat: number, callback: (country: string) => void) { SimpleMetaTagger.coder?.GetCountryCodeFor(lon, lat, callback) @@ -337,7 +378,7 @@ export default class SimpleMetaTagger { ]; - for (const metatag of SimpleMetaTagger.metatags.concat(UpdateTagsFromOsmAPI.metaTagger)) { + for (const metatag of SimpleMetaTagger.metatags) { subElements.push( new Combine([ "

", metatag.keys.join(", "), "

", @@ -349,5 +390,12 @@ export default class SimpleMetaTagger { return new Combine(subElements) } + addMetaTags(features: { feature: any, freshness: Date }[]) { + for (let i = 0; i < features.length; i++) { + let feature = features[i]; + this._f(feature.feature, i, feature.freshness); + } + } + } \ No newline at end of file diff --git a/Logic/Web/Hash.ts b/Logic/Web/Hash.ts index 169b923..d1e4cd2 100644 --- a/Logic/Web/Hash.ts +++ b/Logic/Web/Hash.ts @@ -26,6 +26,7 @@ export default class Hash { } const hash = new UIEventSource(window.location.hash.substr(1)); hash.addCallback(h => { + console.trace("Hash was changed into ",h) if (h === "undefined") { console.warn("Got a literal 'undefined' as hash, ignoring") h = undefined; diff --git a/scripts/generateCache.ts b/scripts/generateCache.ts index 6249266..2e2bdae 100644 --- a/scripts/generateCache.ts +++ b/scripts/generateCache.ts @@ -14,6 +14,8 @@ import ScriptUtils from "./ScriptUtils"; import ExtractRelations from "../Logic/Osm/ExtractRelations"; import * as OsmToGeoJson from "osmtogeojson"; import MetaTagging from "../Logic/MetaTagging"; +import LayerConfig from "../Customizations/JSON/LayerConfig"; +import {GeoOperations} from "../Logic/GeoOperations"; function createOverpassObject(theme: LayoutConfig) { @@ -114,8 +116,6 @@ async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass)/ await ScriptUtils.sleep(1000) } } - - } } @@ -124,6 +124,7 @@ async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass)/ async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig) { let processed = 0; + const layerIndex = theme.LayerIndex(); for (let x = r.xstart; x <= r.xend; x++) { for (let y = r.ystart; y <= r.yend; y++) { processed++; @@ -150,7 +151,6 @@ async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig) } } const featuresFreshness = geojson.features.map(feature => { - feature["_timestamp"] = rawOsm.osm3s.timestamp_osm_base; return ({ freshness: osmTime, feature: feature @@ -158,7 +158,25 @@ async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig) }); // Extract the relationship information const relations = ExtractRelations.BuildMembershipTable(ExtractRelations.GetRelationElements(rawOsm)) - MetaTagging.addMetatags(featuresFreshness, relations, theme.layers); + MetaTagging.addMetatags(featuresFreshness, relations, theme.layers, false); + + + for (const feature of geojson.features) { + const layer = layerIndex.get(feature["_matching_layer_id"]) + if (layer === undefined) { + console.error("Didn't find a layer for " + feature["_matching_layer_id"]) + continue + } + + if (layer.wayHandling == LayerConfig.WAYHANDLING_CENTER_ONLY) { + + const centerpoint = GeoOperations.centerpointCoordinates(feature) + + feature.geometry.type = "Point" + feature.geometry["coordinates"] = centerpoint; + + } + } writeFileSync(geoJsonName(targetdir, x, y, r.zoomlevel), JSON.stringify(geojson, null, " ")) @@ -168,7 +186,6 @@ async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig) } async function splitPerLayer(targetdir: string, r: TileRange, theme: LayoutConfig) { - let processed = 0; const z = r.zoomlevel; for (let x = r.xstart; x <= r.xend; x++) { for (let y = r.ystart; y <= r.yend; y++) { @@ -177,14 +194,14 @@ async function splitPerLayer(targetdir: string, r: TileRange, theme: LayoutConfi for (const layer of theme.layers) { const geojson = JSON.parse(file) geojson.features = geojson.features.filter(f => f._matching_layer_id === layer.id) - if(geojson.features.length == 0){ + if (geojson.features.length == 0) { continue; } - const new_path = geoJsonName(targetdir+"_"+layer.id, x, y, z); + const new_path = geoJsonName(targetdir + "_" + layer.id, x, y, z); writeFileSync(new_path, JSON.stringify(geojson, null, " ")) } - - + + } } }