From 0dec1d0f756abc62c106806c8604dcd7d66a331b Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Thu, 22 Apr 2021 13:30:00 +0200 Subject: [PATCH] Refactoring of metatagging and extrafunctions to splice out the relation memberships, add calculatedTags and metatags into cache --- Logic/ExtraFunction.ts | 29 +++++------ .../FeatureDuplicatorPerLayer.ts | 4 ++ Logic/FeatureSource/FeaturePipeline.ts | 2 +- .../FeatureSource/MetaTaggingFeatureSource.ts | 3 +- Logic/MetaTagging.ts | 29 +++++++---- Logic/Osm/ExtractRelations.ts | 3 +- Logic/SimpleMetaTagger.ts | 10 +++- assets/layers/slow_roads/slow_roads.json | 18 ++++--- package.json | 2 +- scripts/generateCache.ts | 52 ++++++++++++++----- 10 files changed, 98 insertions(+), 54 deletions(-) diff --git a/Logic/ExtraFunction.ts b/Logic/ExtraFunction.ts index e7aae02..cfb4ed8 100644 --- a/Logic/ExtraFunction.ts +++ b/Logic/ExtraFunction.ts @@ -2,6 +2,7 @@ import {GeoOperations} from "./GeoOperations"; import {UIElement} from "../UI/UIElement"; import Combine from "../UI/Base/Combine"; import State from "../State"; +import {Relation} from "./Osm/ExtractRelations"; export class ExtraFunction { @@ -40,11 +41,11 @@ Some advanced functions are available on feat as well: "overlapWith", "Gives a list of features from the specified layer which this feature overlaps with, the amount of overlap in m². The returned value is { feat: GeoJSONFeature, overlap: number}", ["...layerIds - one or more layer ids of the layer from which every feature is checked for overlap)"], - (featuresPerLayer, feat) => { + (params, feat) => { return (...layerIds: string[]) => { const result = [] for (const layerId of layerIds) { - const otherLayer = featuresPerLayer.get(layerId); + const otherLayer = params.featuresPerLayer.get(layerId); if (otherLayer === undefined) { continue; } @@ -80,10 +81,10 @@ Some advanced functions are available on feat as well: "closest", "Given either a list of geojson features or a single layer name, gives the single object which is nearest to the feature. In the case of ways/polygons, only the centerpoint is considered.", ["list of features"], - (featuresPerLayer, feature) => { + (params, feature) => { return (features) => { if (typeof features === "string") { - features = featuresPerLayer.get(features) + features = params.featuresPerLayer.get(features) } let closestFeature = undefined; let closestDistance = undefined; @@ -118,11 +119,8 @@ Some advanced functions are available on feat as well: "memberships", "Gives a list of {role: string, relation: Relation}-objects, containing all the relations that this feature is part of. \n\nFor example: `_part_of_walking_routes=feat.memberships().map(r => r.relation.tags.name).join(';')`", [], - (featuresPerLayer, feature) => { - return () => { - return State.state.knownRelations.data?.get(feature.id) ?? []; - } - + (params, feature) => { + return () => params.relations ?? []; } ) @@ -130,9 +128,9 @@ Some advanced functions are available on feat as well: private readonly _name: string; private readonly _args: string[]; private readonly _doc: string; - private readonly _f: (featuresPerLayer: Map, feat: any) => any; + private readonly _f: (params: {featuresPerLayer: Map, relations: {role: string, relation: Relation}[]}, feat: any) => any; - constructor(name: string, doc: string, args: string[], f: ((featuresPerLayer: Map, feat: any) => any)) { + constructor(name: string, doc: string, args: string[], f: ((params: {featuresPerLayer: Map, relations: {role: string, relation: Relation}[]}, feat: any) => any)) { this._name = name; this._doc = doc; this._args = args; @@ -140,9 +138,9 @@ Some advanced functions are available on feat as well: } - public static FullPatchFeature(featuresPerLayer: Map, feature) { + public static FullPatchFeature(featuresPerLayer: Map,relations: {role: string, relation: Relation}[], feature) { for (const func of ExtraFunction.allFuncs) { - func.PatchFeature(featuresPerLayer, feature); + func.PatchFeature(featuresPerLayer, relations, feature); } } @@ -168,7 +166,8 @@ Some advanced functions are available on feat as well: ]); } - public PatchFeature(featuresPerLayer: Map, feature: any) { - feature[this._name] = this._f(featuresPerLayer, feature); + public PatchFeature(featuresPerLayer: Map, relations: {role: string, relation: Relation}[], feature: any) { + + feature[this._name] = this._f({featuresPerLayer: featuresPerLayer, relations: relations}, feature); } } \ No newline at end of file diff --git a/Logic/FeatureSource/FeatureDuplicatorPerLayer.ts b/Logic/FeatureSource/FeatureDuplicatorPerLayer.ts index a58534c..5ca6870 100644 --- a/Logic/FeatureSource/FeatureDuplicatorPerLayer.ts +++ b/Logic/FeatureSource/FeatureDuplicatorPerLayer.ts @@ -11,6 +11,10 @@ import LayerConfig from "../../Customizations/JSON/LayerConfig"; export default class FeatureDuplicatorPerLayer implements FeatureSource { public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>; + + public static GetMatchingLayerId(){ + + } constructor(layers: UIEventSource<{ layerDef: LayerConfig }[]>, upstream: FeatureSource) { this.features = upstream.features.map(features => { diff --git a/Logic/FeatureSource/FeaturePipeline.ts b/Logic/FeatureSource/FeaturePipeline.ts index 6b48e95..81fe3e2 100644 --- a/Logic/FeatureSource/FeaturePipeline.ts +++ b/Logic/FeatureSource/FeaturePipeline.ts @@ -35,7 +35,7 @@ export default class FeaturePipeline implements FeatureSource { const geojsonSources: FeatureSource [] = GeoJsonSource .ConstructMultiSource(flayers.data, locationControl) - .map(geojsonSource => new RegisteringFeatureSource(new FeatureDuplicatorPerLayer(flayers, geojsonSource))); + .map(geojsonSource => new RegisteringFeatureSource(new FeatureDuplicatorPerLayer(flayers, geojsonSource))); const amendedLocalStorageSource = new RememberingSource(new RegisteringFeatureSource(new FeatureDuplicatorPerLayer(flayers, new LocalStorageSource(layout)) diff --git a/Logic/FeatureSource/MetaTaggingFeatureSource.ts b/Logic/FeatureSource/MetaTaggingFeatureSource.ts index 83f91d4..3cc1c28 100644 --- a/Logic/FeatureSource/MetaTaggingFeatureSource.ts +++ b/Logic/FeatureSource/MetaTaggingFeatureSource.ts @@ -3,6 +3,7 @@ import {UIEventSource} from "../UIEventSource"; import State from "../../State"; import Hash from "../Web/Hash"; import MetaTagging from "../MetaTagging"; +import ExtractRelations from "../Osm/ExtractRelations"; export default class MetaTaggingFeatureSource implements FeatureSource { features: UIEventSource<{ feature: any; freshness: Date }[]> = new UIEventSource<{feature: any; freshness: Date}[]>(undefined); @@ -21,7 +22,7 @@ export default class MetaTaggingFeatureSource implements FeatureSource { } }) - MetaTagging.addMetatags(featuresFreshness, State.state.layoutToUse.data.layers); + MetaTagging.addMetatags(featuresFreshness, State.state.knownRelations.data, State.state.layoutToUse.data.layers); self.features.setData(featuresFreshness); }); } diff --git a/Logic/MetaTagging.ts b/Logic/MetaTagging.ts index b405947..1a646b2 100644 --- a/Logic/MetaTagging.ts +++ b/Logic/MetaTagging.ts @@ -1,6 +1,14 @@ import LayerConfig from "../Customizations/JSON/LayerConfig"; import SimpleMetaTagger from "./SimpleMetaTagger"; import {ExtraFunction} from "./ExtraFunction"; +import State from "../State"; +import {Relation} from "./Osm/ExtractRelations"; + + +interface Params { + featuresPerLayer: Map, + memberships: Map +} /** * Metatagging adds various tags to the elements, e.g. lat, lon, surface area, ... @@ -14,7 +22,8 @@ export default class MetaTagging { * An actor which adds metatags on every feature in the given object * The features are a list of geojson-features, with a "properties"-field and geometry */ - static addMetatags(features: { feature: any; freshness: Date }[], layers: LayerConfig[]) { + static addMetatags(features: { feature: any; freshness: Date }[], + relations: Map, layers: LayerConfig[]) { for (const metatag of SimpleMetaTagger.metatags) { try { @@ -26,7 +35,7 @@ export default class MetaTagging { } // The functions - per layer - which add the new keys - const layerFuncs = new Map, feature: any) => void)>(); + const layerFuncs = new Map void)>(); for (const layer of layers) { layerFuncs.set(layer.id, this.createRetaggingFunc(layer)); } @@ -48,27 +57,26 @@ export default class MetaTagging { if (f === undefined) { continue; } - - f(featuresPerLayer, feature.feature) + f({featuresPerLayer: featuresPerLayer, memberships: relations}, feature.feature) } } - private static createRetaggingFunc(layer: LayerConfig): ((featuresPerLayer: Map, feature: any) => void) { + private static createRetaggingFunc(layer: LayerConfig): + ((params: Params, feature: any) => void) { const calculatedTags: [string, string][] = layer.calculatedTags; if (calculatedTags === undefined) { return undefined; } - const functions: ((featuresPerLayer: Map, feature: any) => void)[] = []; + const functions: ((params: Params, feature: any) => void)[] = []; for (const entry of calculatedTags) { const key = entry[0] const code = entry[1]; if (code === undefined) { continue; } - const func = new Function("feat", "return " + code + ";"); const f = (featuresPerLayer, feature: any) => { @@ -76,16 +84,17 @@ export default class MetaTagging { } functions.push(f) } - return (featuresPerLayer: Map, feature) => { + return (params: Params, feature) => { const tags = feature.properties if (tags === undefined) { return; } - ExtraFunction.FullPatchFeature(featuresPerLayer, feature); + const relations = params.memberships.get(feature.properties.id) + ExtraFunction.FullPatchFeature(params.featuresPerLayer, relations, feature); try { for (const f of functions) { - f(featuresPerLayer, feature); + f(params, feature); } } catch (e) { console.error("While calculating a tag value: ", e) diff --git a/Logic/Osm/ExtractRelations.ts b/Logic/Osm/ExtractRelations.ts index feeff4c..24eefef 100644 --- a/Logic/Osm/ExtractRelations.ts +++ b/Logic/Osm/ExtractRelations.ts @@ -17,7 +17,6 @@ export default class ExtractRelations { public static RegisterRelations(overpassJson: any) : void{ const memberships = ExtractRelations.BuildMembershipTable(ExtractRelations.GetRelationElements(overpassJson)) - console.log("Assigned memberships: ", memberships) State.state.knownRelations.setData(memberships) } @@ -40,7 +39,7 @@ export default class ExtractRelations { * @param relations * @constructor */ - private static BuildMembershipTable(relations: Relation[]): Map { + public static BuildMembershipTable(relations: Relation[]): Map { const memberships = new Map() for (const relation of relations) { diff --git a/Logic/SimpleMetaTagger.ts b/Logic/SimpleMetaTagger.ts index 833566e..6b1acd4 100644 --- a/Logic/SimpleMetaTagger.ts +++ b/Logic/SimpleMetaTagger.ts @@ -9,6 +9,7 @@ import {UIElement} from "../UI/UIElement"; import Combine from "../UI/Base/Combine"; import UpdateTagsFromOsmAPI from "./Actors/UpdateTagsFromOsmAPI"; + export default class SimpleMetaTagger { public readonly keys: string[]; public readonly doc: string; @@ -89,7 +90,12 @@ export default class SimpleMetaTagger { ["_isOpen", "_isOpen:description"], "If 'opening_hours' is present, it will add the current state of the feature (being 'yes' or 'no')", (feature => { - + if(Utils.runningFromConsole){ + // We are running from console, thus probably creating a cache + // isOpen is irrelevant + return + } + const tagsSource = State.state.allElements.getEventSourceById(feature.properties.id); tagsSource.addCallbackAndRun(tags => { if (tags.opening_hours === undefined || tags._country === undefined) { @@ -317,7 +323,7 @@ export default class SimpleMetaTagger { ]; static GetCountryCodeFor(lon: number, lat: number, callback: (country: string) => void) { - SimpleMetaTagger.coder.GetCountryCodeFor(lon, lat, callback) + SimpleMetaTagger.coder?.GetCountryCodeFor(lon, lat, callback) } static HelpText(): UIElement { diff --git a/assets/layers/slow_roads/slow_roads.json b/assets/layers/slow_roads/slow_roads.json index 1c72ea0..2d84c1f 100644 --- a/assets/layers/slow_roads/slow_roads.json +++ b/assets/layers/slow_roads/slow_roads.json @@ -159,32 +159,34 @@ "width": { "render": "4" }, - "dashArray": "2 10 6 10", - "color": { - "render": "#bb2", + "dashArray": { + "render": "", "mappings": [ { "if": "highway=cycleway", - "then": "#00c" + "then": "" }, { "if": "highway=path", - "then": "#bb2" + "then": "10 3" }, { "if": "highway=footway", - "then": "#c30" + "then": "10 10" }, { "if": "highway=pedestrian", - "then": "#3c3" + "then": "10 10" }, { "if": "highway=living_street", - "then": "#ccc" + "then": "10 5 3 5" } ] }, + "color": { + "render": "#eaba2a" + }, "presets": [ ] } \ No newline at end of file diff --git a/package.json b/package.json index cc5cf14..e41d844 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "generate:translations": "ts-node scripts/generateTranslations.ts", "generate:layouts": "ts-node scripts/generateLayouts.ts", "generate:docs": "ts-node scripts/generateDocs.ts && ts-node scripts/generateTaginfoProjectFiles.ts", - "generate:cache:speelplekken": "ts-node scripts/generateCache.ts speelplekken 14 ./cache/speelplekken 51.2003 4.3925 51.1058 4.5087", + "generate:cache:speelplekken": "ts-node scripts/generateCache.ts speelplekken 14 ../pietervdvn.github.io/speelplekken_cache/ 51.20 4.37 51.11 4.51", "generate:layeroverview": "ts-node scripts/generateLayerOverview.ts --no-fail", "generate:licenses": "ts-node scripts/generateLicenseInfo.ts --no-fail", "validate:layeroverview": "ts-node scripts/generateLayerOverview.ts --report", diff --git a/scripts/generateCache.ts b/scripts/generateCache.ts index a41d67d..f81d124 100644 --- a/scripts/generateCache.ts +++ b/scripts/generateCache.ts @@ -14,6 +14,8 @@ import ScriptUtils from "./ScriptUtils"; import ExtractRelations from "../Logic/Osm/ExtractRelations"; import * as OsmToGeoJson from "osmtogeojson"; import {Script} from "vm"; +import MetaTagging from "../Logic/MetaTagging"; +import State from "../State"; function createOverpassObject(theme: LayoutConfig) { let filters: TagsFilter[] = []; @@ -26,8 +28,8 @@ function createOverpassObject(theme: LayoutConfig) { continue; } if (layer.source.geojsonSource !== undefined) { - // Not our responsibility to download this layer! - continue; + // We download these anyway - we are building the cache after all! + //continue; } @@ -68,17 +70,17 @@ function metaJsonName(targetDir: string, x: number, y: number, z: number): strin return targetDir + "_" + z + "_" + x + "_" + y + ".meta.json" } -async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass) { +async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass) { let downloaded = 0 for (let x = r.xstart; x <= r.xend; x++) { for (let y = r.ystart; y <= r.yend; y++) { - console.log("x:", (x - r.xstart), "/", (r.xend - r.xstart), "; y:", (y - r.ystart), "/", (r.yend - r.ystart), "; total: ", downloaded, "/", r.total) downloaded++; const filename = rawJsonName(targetdir, x, y, r.zoomlevel) if (existsSync(filename)) { console.log("Already exists: ", filename) continue; } + console.log("x:", (x - r.xstart), "/", (r.xend - r.xstart), "; y:", (y - r.ystart), "/", (r.yend - r.ystart), "; total: ", downloaded, "/", r.total) const boundsArr = Utils.tile_bounds(r.zoomlevel, x, y) const bounds = { @@ -87,28 +89,35 @@ async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass) east: Math.max(boundsArr[0][1], boundsArr[1][1]), west: Math.min(boundsArr[0][1], boundsArr[1][1]) } - console.log("Downloading tile", r.zoomlevel, x, y, "with bounds", bounds) const url = overpass.buildQuery("[bbox:" + bounds.south + "," + bounds.west + "," + bounds.north + "," + bounds.east + "]") + let gotResponse = false ScriptUtils.DownloadJSON(url, chunks => { + gotResponse = true; saveResponse(chunks, filename) }) - await ScriptUtils.sleep(10000) - console.debug("Waking up") + while (!gotResponse) { + await ScriptUtils.sleep(10000) + console.debug("Waking up") + if (!gotResponse) { + console.log("Didn't get an answer yet - waiting more") + } + } + } } } -async function postProcess(targetdir: string, r: TileRange) { +async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig) { let processed = 0; for (let x = r.xstart; x <= r.xend; x++) { for (let y = r.ystart; y <= r.yend; y++) { processed++; const filename = rawJsonName(targetdir, x, y, r.zoomlevel) - console.log(" Post processing", processed, "/",r. total, filename) + console.log(" Post processing", processed, "/", r.total, filename) if (!existsSync(filename)) { throw "Not found - and not downloaded. Run this script again!: " + filename } @@ -118,11 +127,26 @@ async function postProcess(targetdir: string, r: TileRange) { // Create and save the geojson file - which is the main chunk of the data const geojson = OsmToGeoJson.default(rawOsm); + const osmTime = new Date(rawOsm.osm3s.timestamp_osm_base); + + for (const feature of geojson.features) { + + for (const layer of theme.layers) { + if (layer.source.osmTags.matchesProperties(feature.properties)) { + feature["_matching_layer_id"] = layer.id; + break; + } + } + } + const featuresFreshness = geojson.features.map(feature => ({ + freshness: osmTime, + feature: feature + })); + // Extract the relationship information + const relations = ExtractRelations.BuildMembershipTable(ExtractRelations.GetRelationElements(rawOsm)) + MetaTagging.addMetatags(featuresFreshness, relations, theme.layers); writeFileSync(geoJsonName(targetdir, x, y, r.zoomlevel), JSON.stringify(geojson)) - // Extract the relationship information - const relations = ExtractRelations.GetRelationElements(rawOsm) - const osmTime = new Date(rawOsm.osm3s.timestamp_osm_base); const meta = { freshness: osmTime, @@ -145,7 +169,7 @@ async function main(args: string[]) { } const themeName = args[0] const zoomlevel = Number(args[1]) - const targetdir = args[2] + const targetdir = args[2] + "/" + themeName const lat0 = Number(args[3]) const lon0 = Number(args[4]) const lat1 = Number(args[5]) @@ -167,7 +191,7 @@ async function main(args: string[]) { await downloadRaw(targetdir, tileRange, overpass) - await postProcess(targetdir, tileRange) + await postProcess(targetdir, tileRange, theme) }