From 2da52501a3b65518b0f05005f24de11d5cd0cdb4 Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Thu, 22 Apr 2021 03:30:46 +0200 Subject: [PATCH] Add capability to load tiled geojsons, eventually as overpass-cache --- Customizations/JSON/LayerConfig.ts | 5 +- Customizations/JSON/LayerConfigJson.ts | 5 +- Customizations/JSON/LayoutConfig.ts | 2 +- Customizations/JSON/SourceConfig.ts | 5 +- Logic/FeatureSource/FeaturePipeline.ts | 13 +- Logic/FeatureSource/GeoJsonSource.ts | 174 ++++++++++++++++-- Logic/Osm/ExtractRelations.ts | 12 +- Logic/Osm/Overpass.ts | 2 +- Utils.ts | 32 +++- assets/themes/speelplekken/speelplekken.json | 70 ++++++-- package.json | 3 +- scripts/ScriptUtils.ts | 26 +++ scripts/fixGeoJson.ts | 21 --- scripts/generateCache.ts | 176 +++++++++++++++++++ test/TestHelper.ts | 2 +- test/Theme.spec.ts | 48 +++++ 16 files changed, 520 insertions(+), 76 deletions(-) delete mode 100644 scripts/fixGeoJson.ts create mode 100644 scripts/generateCache.ts create mode 100644 test/Theme.spec.ts diff --git a/Customizations/JSON/LayerConfig.ts b/Customizations/JSON/LayerConfig.ts index 1816b8f..e77f310 100644 --- a/Customizations/JSON/LayerConfig.ts +++ b/Customizations/JSON/LayerConfig.ts @@ -85,6 +85,7 @@ export default class LayerConfig { this.source = new SourceConfig({ osmTags: osmTags, geojsonSource: json.source["geoJson"], + geojsonSourceLevel: json.source["geoJsonZoomLevel"], overpassScript: json.source["overpassScript"], }); } else { @@ -159,7 +160,7 @@ export default class LayerConfig { if (renderingJson === "questions") { if (readOnly) { - throw `A tagrendering has a question, but asking a question does not make sense here: is it a title icon or a geojson-layer? ${context}` + throw `A tagrendering has a question, but asking a question does not make sense here: is it a title icon or a geojson-layer? ${context}. The offending tagrendering is ${JSON.stringify(renderingJson)}` } return new TagRenderingConfig("questions", undefined) @@ -176,7 +177,7 @@ export default class LayerConfig { }); } - this.tagRenderings = trs(json.tagRenderings, this.source.geojsonSource !== undefined); + this.tagRenderings = trs(json.tagRenderings, false); const titleIcons = []; diff --git a/Customizations/JSON/LayerConfigJson.ts b/Customizations/JSON/LayerConfigJson.ts index f0f88c3..66af912 100644 --- a/Customizations/JSON/LayerConfigJson.ts +++ b/Customizations/JSON/LayerConfigJson.ts @@ -29,7 +29,8 @@ export interface LayerConfigJson { * There are some options: * * source: {osmTags: "key=value"} will fetch all objects with given tags from OSM. Currently, this will create a query to overpass and fetch the data - in the future this might fetch from the OSM API - * source: {geoJsonSource: "https://my.source.net/some-geo-data.geojson"} to fetch a geojson from a third party source + * source: {geoJson: "https://my.source.net/some-geo-data.geojson"} to fetch a geojson from a third party source + * source: {geoJson: "https://my.source.net/some-tile-geojson-{z}-{x}-{y}.geojson", geoJsonZoomLevel: 14} to use a tiled geojson source. The web server must offer multiple geojsons. {z}, {x} and {y} are substituted * * source: {overpassScript: ""} when you want to do special things. _This should be really rare_. * This means that the data will be pulled from overpass with this script, and will ignore the osmTags for the query @@ -40,7 +41,7 @@ export interface LayerConfigJson { * While still supported, this is considered deprecated */ source: { osmTags: AndOrTagConfigJson | string } | - { osmTags: AndOrTagConfigJson | string, geoJson: string } | + { osmTags: AndOrTagConfigJson | string, geoJson: string, geoJsonZoomLevel?: number } | { osmTags: AndOrTagConfigJson | string, overpassScript: string } /** diff --git a/Customizations/JSON/LayoutConfig.ts b/Customizations/JSON/LayoutConfig.ts index 6966101..734f254 100644 --- a/Customizations/JSON/LayoutConfig.ts +++ b/Customizations/JSON/LayoutConfig.ts @@ -108,7 +108,7 @@ export default class LayoutConfig { throw "Unkown fixed layer " + name; } // @ts-ignore - layer = Utils.Merge(layer.override, shared); + layer = Utils.Merge(layer.override, JSON.parse(JSON.stringify(shared))); // We make a deep copy of the shared layer, in order to protect it from changes } // @ts-ignore diff --git a/Customizations/JSON/SourceConfig.ts b/Customizations/JSON/SourceConfig.ts index f354ed3..3368262 100644 --- a/Customizations/JSON/SourceConfig.ts +++ b/Customizations/JSON/SourceConfig.ts @@ -5,11 +5,13 @@ export default class SourceConfig { osmTags?: TagsFilter; overpassScript?: string; geojsonSource?: string; + geojsonZoomLevel?: number; constructor(params: { osmTags?: TagsFilter, overpassScript?: string, - geojsonSource?: string + geojsonSource?: string, + geojsonSourceLevel?: number }) { let defined = 0; @@ -28,5 +30,6 @@ export default class SourceConfig { this.osmTags = params.osmTags; this.overpassScript = params.overpassScript; this.geojsonSource = params.geojsonSource; + this.geojsonZoomLevel = params.geojsonSourceLevel; } } \ No newline at end of file diff --git a/Logic/FeatureSource/FeaturePipeline.ts b/Logic/FeatureSource/FeaturePipeline.ts index 4b79cfb..6b48e95 100644 --- a/Logic/FeatureSource/FeaturePipeline.ts +++ b/Logic/FeatureSource/FeaturePipeline.ts @@ -33,15 +33,10 @@ export default class FeaturePipeline implements FeatureSource { updater) )), layout)); - const geojsonSources: GeoJsonSource [] = [] - for (const flayer of flayers.data) { - const sourceUrl = flayer.layerDef.source.geojsonSource - if (sourceUrl !== undefined) { - geojsonSources.push(new RegisteringFeatureSource(new FeatureDuplicatorPerLayer(flayers, - new GeoJsonSource(flayer.layerDef.id, sourceUrl)))) - } - } - + const geojsonSources: FeatureSource [] = GeoJsonSource + .ConstructMultiSource(flayers.data, locationControl) + .map(geojsonSource => new RegisteringFeatureSource(new FeatureDuplicatorPerLayer(flayers, geojsonSource))); + const amendedLocalStorageSource = new RememberingSource(new RegisteringFeatureSource(new FeatureDuplicatorPerLayer(flayers, new LocalStorageSource(layout)) )); diff --git a/Logic/FeatureSource/GeoJsonSource.ts b/Logic/FeatureSource/GeoJsonSource.ts index dea7356..7f17fd8 100644 --- a/Logic/FeatureSource/GeoJsonSource.ts +++ b/Logic/FeatureSource/GeoJsonSource.ts @@ -1,51 +1,195 @@ import FeatureSource from "./FeatureSource"; import {UIEventSource} from "../UIEventSource"; import * as $ from "jquery"; +import {control} from "leaflet"; +import zoom = control.zoom; +import Loc from "../../Models/Loc"; +import State from "../../State"; +import {Utils} from "../../Utils"; +import LayerConfig from "../../Customizations/JSON/LayerConfig"; + /** * Fetches a geojson file somewhere and passes it along */ export default class GeoJsonSource implements FeatureSource { + features: UIEventSource<{ feature: any; freshness: Date }[]>; - constructor(layerId: string, url: string, onFail: ((errorMsg: any) => void) = undefined) { - if (onFail === undefined) { - onFail = errorMsg => { - console.warn(`Could not load geojson layer from`, url, "due to", errorMsg) + private readonly onFail: ((errorMsg: any, url: string) => void) = undefined; + + private readonly layerId: string; + + private readonly seenids: Set = new Set() + + constructor(locationControl: UIEventSource, + flayer: { isDisplayed: UIEventSource, layerDef: LayerConfig }, + onFail?: ((errorMsg: any) => void)) { + this.layerId = flayer.layerDef.id; + let url = flayer.layerDef.source.geojsonSource; + const zoomLevel = flayer.layerDef.source.geojsonZoomLevel; + + this.features = new UIEventSource<{ feature: any; freshness: Date }[]>([]) + + if (zoomLevel === undefined) { + // This is a classic, static geojson layer + if (onFail === undefined) { + onFail = errorMsg => { + console.warn(`Could not load geojson layer from`, url, "due to", errorMsg) + } } + this.onFail = onFail; + + this.LoadJSONFrom(url) + } else { + // This is a dynamic template with a fixed zoom level + url = url.replace("{z}", "" + zoomLevel) + const loadedTiles = new Set(); + const self = this; + this.onFail = (msg, url) => { + console.warn(`Could not load geojson layer from`, url, "due to", msg) + loadedTiles.delete(url) + } + + const neededTiles = locationControl.map( + location => { + + if (!flayer.isDisplayed.data) { + return undefined; + } + + // Yup, this is cheating to just get the bounds here + const bounds = State.state.leafletMap.data.getBounds() + const tileRange = Utils.TileRangeBetween(zoomLevel, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest()) + const needed = new Set(); + for (let x = tileRange.xstart; x <= tileRange.xend; x++) { + for (let y = tileRange.ystart; y <= tileRange.yend; y++) { + let neededUrl = url.replace("{x}", "" + x).replace("{y}", "" + y); + needed.add(neededUrl) + } + } + return needed; + } + ); + neededTiles.stabilized(250).addCallback((needed: Set) => { + if (needed === undefined) { + return; + } + needed.forEach(neededTile => { + if (loadedTiles.has(neededTile)) { + return; + } + + loadedTiles.add(neededTile) + self.LoadJSONFrom(neededTile) + + }) + }) + } - this.features = new UIEventSource<{ feature: any; freshness: Date }[]>(undefined) + } + + /** + * Merges together the layers which have the same source + * @param flayers + * @param locationControl + * @constructor + */ + public static ConstructMultiSource(flayers: { isDisplayed: UIEventSource, layerDef: LayerConfig }[], locationControl: UIEventSource): GeoJsonSource[] { + + const flayersPerSource = new Map, layerDef: LayerConfig }[]>(); + for (const flayer of flayers) { + const url = flayer.layerDef.source.geojsonSource + if (url === undefined) { + continue; + } + + if (!flayersPerSource.has(url)) { + flayersPerSource.set(url, []) + } + flayersPerSource.get(url).push(flayer) + } + + console.log("SOURCES", flayersPerSource) + + const sources: GeoJsonSource[] = [] + + flayersPerSource.forEach((flayers, key) => { + if (flayers.length == 1) { + sources.push(new GeoJsonSource(locationControl, flayers[0])); + return; + } + + const zoomlevels = Utils.Dedup(flayers.map(flayer => "" + (flayer.layerDef.source.geojsonZoomLevel ?? ""))) + if (zoomlevels.length > 1) { + throw "Multiple zoomlevels defined for same geojson source " + key + } + + let isShown = new UIEventSource(true, "IsShown for multiple layers: or of multiple values"); + for (const flayer of flayers) { + flayer.isDisplayed.addCallbackAndRun(() => { + let value = false; + for (const flayer of flayers) { + value = flayer.isDisplayed.data || value; + } + isShown.setData(value); + }); + + } + + const source = new GeoJsonSource(locationControl, { + isDisplayed: isShown, + layerDef: flayers[0].layerDef // We only care about the source info here + }) + sources.push(source) + + }) + return sources; + + } + + private LoadJSONFrom(url: string) { const eventSource = this.features; + const self = this; $.getJSON(url, function (json, status) { if (status !== "success") { console.log("Fetching geojson failed failed") - onFail(status); + self.onFail(status, url); return; } if (json.elements === [] && json.remarks.indexOf("runtime error") > 0) { console.log("Timeout or other runtime error"); - onFail("Runtime error (timeout)") + self.onFail("Runtime error (timeout)", url) return; } const time = new Date(); - const features: { feature: any, freshness: Date } [] = [] + const newFeatures: { feature: any, freshness: Date } [] = [] let i = 0; + let skipped = 0; for (const feature of json.features) { if (feature.properties.id === undefined) { feature.properties.id = url + "/" + i; feature.id = url + "/" + i; i++; } - feature._matching_layer_id = layerId; - features.push({feature: feature, freshness: time}) + if (self.seenids.has(feature.properties.id)) { + skipped++; + continue; + } + self.seenids.add(feature.properties.id) + + newFeatures.push({feature: feature, freshness: time}) } - console.log("Loaded features are", features) - eventSource.setData(features) - - }).fail(onFail) + console.log("Downloaded "+newFeatures.length+" new features and "+skipped+" already seen features from "+ url); + + if(newFeatures.length == 0){ + return; + } + + eventSource.setData(eventSource.data.concat(newFeatures)) + }).fail(msg => self.onFail(msg, url)) } - } \ No newline at end of file diff --git a/Logic/Osm/ExtractRelations.ts b/Logic/Osm/ExtractRelations.ts index 4c64d63..feeff4c 100644 --- a/Logic/Osm/ExtractRelations.ts +++ b/Logic/Osm/ExtractRelations.ts @@ -20,9 +20,15 @@ export default class ExtractRelations { console.log("Assigned memberships: ", memberships) State.state.knownRelations.setData(memberships) } - - private static GetRelationElements(overpassJson: any): Relation[] { - const relations = overpassJson.elements.filter(element => element.type === "relation") + + /** + * Gets an overview of the relations - except for multipolygons. We don't care about those + * @param overpassJson + * @constructor + */ + public static GetRelationElements(overpassJson: any): Relation[] { + const relations = overpassJson.elements + .filter(element => element.type === "relation" && element.tags.type !== "multipolygon") for (const relation of relations) { relation.properties = relation.tags } diff --git a/Logic/Osm/Overpass.ts b/Logic/Osm/Overpass.ts index b62749e..007cf9f 100644 --- a/Logic/Osm/Overpass.ts +++ b/Logic/Osm/Overpass.ts @@ -48,7 +48,7 @@ export class Overpass { }).fail(onFail) } - private buildQuery(bbox: string): string { + buildQuery(bbox: string): string { const filters = this._filter.asOverpass() let filter = "" for (const filterOr of filters) { diff --git a/Utils.ts b/Utils.ts index 4ac8d4c..aca8abc 100644 --- a/Utils.ts +++ b/Utils.ts @@ -156,8 +156,6 @@ export class Utils { } static Merge(source: any, target: any) { - target = JSON.parse(JSON.stringify(target)); - source = JSON.parse(JSON.stringify(source)); for (const key in source) { const sourceV = source[key]; const targetV = target[key] @@ -203,6 +201,26 @@ export class Utils { static embedded_tile(lat: number, lon: number, z: number): { x: number, y: number, z: number } { return {x: Utils.lon2tile(lon, z), y: Utils.lat2tile(lat, z), z: z} } + + static TileRangeBetween(zoomlevel: number, lat0: number, lon0: number, lat1:number, lon1: number) : TileRange{ + const t0 = Utils.embedded_tile(lat0, lon0, zoomlevel) + const t1 = Utils.embedded_tile(lat1, lon1, zoomlevel) + + const xstart = Math.min(t0.x, t1.x) + const xend = Math.max(t0.x, t1.x) + const ystart = Math.min(t0.y, t1.y) + const yend = Math.max(t0.y, t1.y) + const total = (1 + xend - xstart) * (1 + yend - ystart) + + return { + xstart: xstart, + xend: xend, + ystart: ystart, + yend: yend, + total: total, + zoomlevel: zoomlevel + } + } public static MinifyJSON(stringified: string): string { stringified = stringified.replace(/\|/g, "||"); @@ -257,3 +275,13 @@ export class Utils { return (Math.floor((1 - Math.log(Math.tan(lat * Math.PI / 180) + 1 / Math.cos(lat * Math.PI / 180)) / Math.PI) / 2 * Math.pow(2, zoom))); } } + + +export interface TileRange{ + xstart: number, + ystart: number, + xend: number, + yend: number, + total: number, + zoomlevel: number +} \ No newline at end of file diff --git a/assets/themes/speelplekken/speelplekken.json b/assets/themes/speelplekken/speelplekken.json index 5a9fb8b..5e7217b 100644 --- a/assets/themes/speelplekken/speelplekken.json +++ b/assets/themes/speelplekken/speelplekken.json @@ -24,19 +24,64 @@ "socialImage": "", "defaultBackgroundId": "CartoDB.Positron", "layers": [ - "play_forest", - "playground", - "sport_pitch", + { + "builtin": "play_forest", + "override": { + "source": { + "geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson", + "geoJsonZoomLevel": 14 + } + } + }, + { + "builtin": "playground", + "override": { + "source": { + "geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson", + "geoJsonZoomLevel": 14 + } + } + }, + { + "builtin": "sport_pitch", + "override": { + "minzoom": 15, + "source": { + "geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson", + "geoJsonZoomLevel": 14 + } + } + }, { "builtin": "slow_roads", "override": { + "source": { + "geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson", + "geoJsonZoomLevel": 14 + }, "calculatedTags": [ "_part_of_walking_routes=feat.memberships().map(r => \"\" + r.relation.tags.name + \"\").join(', ')" ] } }, - "grass_in_parks", - "village_green", + { + "builtin": "grass_in_parks", + "override": { + "source": { + "geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson", + "geoJsonZoomLevel": 14 + } + } + }, + { + "builtin": "village_green", + "override": { + "source": { + "geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson", + "geoJsonZoomLevel": 14 + } + } + }, { "id": "walking_routes", "name": { @@ -50,7 +95,9 @@ "route=foot", "operator=provincie Antwerpen" ] - } + }, + "geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson", + "geoJsonZoomLevel": 14 }, "title": { "render": "Wandeling {name}", @@ -141,17 +188,6 @@ "width": { "render": "3" } - }, - { - "id": "speelplekken-cache", - "name": "", - "source": { - "osmTags": { - "or": [] - }, - "geoJson": "https://pietervdvn.github.io/speelplekken-cache.geojson" - }, - "passAllFeatures": true } ], "roamingRenderings": [ diff --git a/package.json b/package.json index 26ac357..cc5cf14 100644 --- a/package.json +++ b/package.json @@ -9,12 +9,13 @@ "scripts": { "increase-memory": "export NODE_OPTIONS=--max_old_space_size=4096", "start": "ts-node scripts/generateLayerOverview.ts --no-fail && npm run increase-memory && parcel *.html UI/** Logic/** assets/** assets/**/** assets/**/**/** vendor/* vendor/*/*", - "test": "ts-node test/Tag.spec.ts && ts-node test/TagQuestion.spec.ts && ts-node test/ImageSearcher.spec.ts && ts-node test/ImageAttribution.spec.ts", + "test": "ts-node test/Tag.spec.ts && ts-node test/TagQuestion.spec.ts && ts-node test/ImageSearcher.spec.ts && ts-node test/ImageAttribution.spec.ts && ts-node test/Theme.spec.ts", "generate:editor-layer-index": "cd assets/ && wget https://osmlab.github.io/editor-layer-index/imagery.geojson --output-document=editor-layer-index.json", "generate:images": "ts-node scripts/generateIncludedImages.ts", "generate:translations": "ts-node scripts/generateTranslations.ts", "generate:layouts": "ts-node scripts/generateLayouts.ts", "generate:docs": "ts-node scripts/generateDocs.ts && ts-node scripts/generateTaginfoProjectFiles.ts", + "generate:cache:speelplekken": "ts-node scripts/generateCache.ts speelplekken 14 ./cache/speelplekken 51.2003 4.3925 51.1058 4.5087", "generate:layeroverview": "ts-node scripts/generateLayerOverview.ts --no-fail", "generate:licenses": "ts-node scripts/generateLicenseInfo.ts --no-fail", "validate:layeroverview": "ts-node scripts/generateLayerOverview.ts --report", diff --git a/scripts/ScriptUtils.ts b/scripts/ScriptUtils.ts index 36ecc24..e725cf0 100644 --- a/scripts/ScriptUtils.ts +++ b/scripts/ScriptUtils.ts @@ -1,4 +1,5 @@ import {lstatSync, readdirSync} from "fs"; +import * as https from "https"; export default class ScriptUtils { public static readDirRecSync(path): string[] { @@ -16,5 +17,30 @@ export default class ScriptUtils { } return result; } + + public static DownloadJSON(url, continuation : (parts : string []) => void){ + https.get(url, (res) => { + console.log("Got response!") + const parts : string[] = [] + res.setEncoding('utf8'); + res.on('data', function (chunk) { + // @ts-ignore + parts.push(chunk) + }); + + res.addListener('end', function () { + continuation(parts) + }); + }) + } + + public static sleep(ms) { + return new Promise((resolve) => { + console.debug("Sleeping for", ms) + setTimeout(resolve, ms); + + }); + } + } diff --git a/scripts/fixGeoJson.ts b/scripts/fixGeoJson.ts deleted file mode 100644 index d6c8e5e..0000000 --- a/scripts/fixGeoJson.ts +++ /dev/null @@ -1,21 +0,0 @@ - - -// Loads a geojson file downloaded from overpass, renames "@id" to "id" and deletes "@relations" - -import {readFileSync, writeFileSync} from "fs"; - -const source = process.argv[2] ?? "~/Downloads/export.json" -console.log("Fixing up ", source) -const contents = readFileSync(source, "UTF8"); -const f = JSON.parse(contents); -let i = 0 -for (const feature of f.features) { - if(feature.properties == undefined){ - continue - } - feature.properties["id"] = feature.properties["@id"] - feature.properties["@id"] = undefined - feature.properties["@relations"] = undefined -} - -writeFileSync(source+".fixed", JSON.stringify(f, null, " ")) \ No newline at end of file diff --git a/scripts/generateCache.ts b/scripts/generateCache.ts new file mode 100644 index 0000000..a41d67d --- /dev/null +++ b/scripts/generateCache.ts @@ -0,0 +1,176 @@ +/** + * Generates a collection of geojson files based on an overpass query for a given theme + */ +import {TileRange, Utils} from "../Utils"; + +Utils.runningFromConsole = true +import {Overpass} from "../Logic/Osm/Overpass"; +import {writeFileSync, existsSync, readFileSync} from "fs"; +import {TagsFilter} from "../Logic/Tags/TagsFilter"; +import {Or} from "../Logic/Tags/Or"; +import LayoutConfig from "../Customizations/JSON/LayoutConfig"; +import {AllKnownLayouts} from "../Customizations/AllKnownLayouts"; +import ScriptUtils from "./ScriptUtils"; +import ExtractRelations from "../Logic/Osm/ExtractRelations"; +import * as OsmToGeoJson from "osmtogeojson"; +import {Script} from "vm"; + +function createOverpassObject(theme: LayoutConfig) { + let filters: TagsFilter[] = []; + let extraScripts: string[] = []; + for (const layer of theme.layers) { + if (typeof (layer) === "string") { + throw "A layer was not expanded!" + } + if (layer.doNotDownload) { + continue; + } + if (layer.source.geojsonSource !== undefined) { + // Not our responsibility to download this layer! + continue; + } + + + // Check if data for this layer has already been loaded + if (layer.source.overpassScript !== undefined) { + extraScripts.push(layer.source.overpassScript) + } else { + filters.push(layer.source.osmTags); + } + } + filters = Utils.NoNull(filters) + extraScripts = Utils.NoNull(extraScripts) + if (filters.length + extraScripts.length === 0) { + throw "Nothing to download! The theme doesn't declare anything to download" + } + return new Overpass(new Or(filters), extraScripts); +} + +function saveResponse(chunks: string[], targetDir: string) { + const contents = chunks.join("") + if (contents.startsWith(" { + saveResponse(chunks, filename) + }) + + await ScriptUtils.sleep(10000) + console.debug("Waking up") + + } + } +} + +async function postProcess(targetdir: string, r: TileRange) { + let processed = 0; + for (let x = r.xstart; x <= r.xend; x++) { + for (let y = r.ystart; y <= r.yend; y++) { + processed++; + const filename = rawJsonName(targetdir, x, y, r.zoomlevel) + console.log(" Post processing", processed, "/",r. total, filename) + if (!existsSync(filename)) { + throw "Not found - and not downloaded. Run this script again!: " + filename + } + + // We read the raw OSM-file and convert it to a geojson + const rawOsm = JSON.parse(readFileSync(filename, "UTF8")) + + // Create and save the geojson file - which is the main chunk of the data + const geojson = OsmToGeoJson.default(rawOsm); + writeFileSync(geoJsonName(targetdir, x, y, r.zoomlevel), JSON.stringify(geojson)) + + // Extract the relationship information + const relations = ExtractRelations.GetRelationElements(rawOsm) + const osmTime = new Date(rawOsm.osm3s.timestamp_osm_base); + + const meta = { + freshness: osmTime, + relations: relations + } + + writeFileSync( + metaJsonName(targetdir, x, y, r.zoomlevel), + JSON.stringify(meta) + ) + } + } +} + +async function main(args: string[]) { + + if (args.length == 0) { + console.error("Expected arguments are: theme zoomlevel targetdirectory lat0 lon0 lat1 lon1") + return; + } + const themeName = args[0] + const zoomlevel = Number(args[1]) + const targetdir = args[2] + const lat0 = Number(args[3]) + const lon0 = Number(args[4]) + const lat1 = Number(args[5]) + const lon1 = Number(args[6]) + + const tileRange = Utils.TileRangeBetween(zoomlevel, lat0, lon0, lat1, lon1) + + const theme = AllKnownLayouts.allKnownLayouts.get(themeName) + if (theme === undefined) { + const keys = [] + AllKnownLayouts.allKnownLayouts.forEach((_, key) => { + keys.push(key) + }) + console.error("The theme " + theme + " was not found; try one of ", keys); + return + } + + const overpass = createOverpassObject(theme) + + + await downloadRaw(targetdir, tileRange, overpass) + await postProcess(targetdir, tileRange) +} + + +let args = [...process.argv] +args.splice(0, 2) +main(args); \ No newline at end of file diff --git a/test/TestHelper.ts b/test/TestHelper.ts index 4232f2c..1018761 100644 --- a/test/TestHelper.ts +++ b/test/TestHelper.ts @@ -1,6 +1,6 @@ export default class T { - + constructor(testsuite: string, tests: [string, () => void ][]) { let failures : string []= []; for (const [name, test] of tests) { diff --git a/test/Theme.spec.ts b/test/Theme.spec.ts new file mode 100644 index 0000000..ea8d182 --- /dev/null +++ b/test/Theme.spec.ts @@ -0,0 +1,48 @@ +import T from "./TestHelper"; +import {Utils} from "../Utils"; + +Utils.runningFromConsole = true; +import TagRenderingQuestion from "../UI/Popup/TagRenderingQuestion"; +import {UIEventSource} from "../Logic/UIEventSource"; +import TagRenderingConfig from "../Customizations/JSON/TagRenderingConfig"; +import LayoutConfig from "../Customizations/JSON/LayoutConfig"; +import {LayoutConfigJson} from "../Customizations/JSON/LayoutConfigJson"; +import * as assert from "assert"; + + +new T("Theme tests", + [ + ["Nested overrides work", () => { + + const themeConfigJson : LayoutConfigJson = { + description: "Descr", + icon: "", + language: ["en"], + layers: [ + { + builtin: "public_bookcase", + override: { + source:{ + geoJson: "xyz" + } + } + } + ], + maintainer: "", + startLat: 0, + startLon: 0, + startZoom: 0, + title: { + en: "Title" + }, + version: "", + id: "test" + } + + const themeConfig = new LayoutConfig(themeConfigJson); + assert.equal("xyz", themeConfig.layers[0].source.geojsonSource) + + + }] + ] +); \ No newline at end of file