Add capability to load tiled geojsons, eventually as overpass-cache

Customizations/JSON/LayerConfig.ts

@@ -85,6 +85,7 @@ export default class LayerConfig {
             this.source = new SourceConfig({
                 osmTags: osmTags,
                 geojsonSource: json.source["geoJson"],
+                geojsonSourceLevel: json.source["geoJsonZoomLevel"],
                 overpassScript: json.source["overpassScript"],
             });
         } else {
@@ -159,7 +160,7 @@ export default class LayerConfig {
 
                         if (renderingJson === "questions") {
                             if (readOnly) {
-                                throw `A tagrendering has a question, but asking a question does not make sense here: is it a title icon or a geojson-layer? ${context}`
+                                throw `A tagrendering has a question, but asking a question does not make sense here: is it a title icon or a geojson-layer? ${context}. The offending tagrendering is ${JSON.stringify(renderingJson)}`
                             }
 
                             return new TagRenderingConfig("questions", undefined)
@@ -176,7 +177,7 @@ export default class LayerConfig {
                 });
         }
 
-        this.tagRenderings = trs(json.tagRenderings, this.source.geojsonSource !== undefined);
+        this.tagRenderings = trs(json.tagRenderings, false);
 
 
         const titleIcons = [];

Customizations/JSON/LayerConfigJson.ts

@@ -29,7 +29,8 @@ export interface LayerConfigJson {
      * There are some options:
      *
      * source: {osmTags: "key=value"} will fetch all objects with given tags from OSM. Currently, this will create a query to overpass and fetch the data - in the future this might fetch from the OSM API
-     * source: {geoJsonSource: "https://my.source.net/some-geo-data.geojson"} to fetch a geojson from a third party source
+     * source: {geoJson: "https://my.source.net/some-geo-data.geojson"} to fetch a geojson from a third party source
+     * source: {geoJson: "https://my.source.net/some-tile-geojson-{z}-{x}-{y}.geojson", geoJsonZoomLevel: 14} to use a tiled geojson source. The web server must offer multiple geojsons. {z}, {x} and {y} are substituted
      *
      * source: {overpassScript: "<custom overpass tags>"} when you want to do special things. _This should be really rare_.
      *      This means that the data will be pulled from overpass with this script, and will ignore the osmTags for the query
@@ -40,7 +41,7 @@ export interface LayerConfigJson {
      *  While still supported, this is considered deprecated
      */
     source: { osmTags: AndOrTagConfigJson | string } |
-        { osmTags: AndOrTagConfigJson | string, geoJson: string } |
+        { osmTags: AndOrTagConfigJson | string, geoJson: string, geoJsonZoomLevel?: number } |
         { osmTags: AndOrTagConfigJson | string, overpassScript: string }
 
     /**

Customizations/JSON/LayoutConfig.ts

@@ -108,7 +108,7 @@ export default class LayoutConfig {
                     throw "Unkown fixed layer " + name;
                 }
                 // @ts-ignore
-                layer = Utils.Merge(layer.override, shared);
+                layer = Utils.Merge(layer.override, JSON.parse(JSON.stringify(shared))); // We make a deep copy of the shared layer, in order to protect it from changes
             }
 
             // @ts-ignore

Customizations/JSON/SourceConfig.ts

@@ -5,11 +5,13 @@ export default class SourceConfig {
     osmTags?: TagsFilter;
     overpassScript?: string;
     geojsonSource?: string;
+    geojsonZoomLevel?: number;
 
     constructor(params: {
         osmTags?: TagsFilter,
         overpassScript?: string,
-        geojsonSource?: string
+        geojsonSource?: string,
+        geojsonSourceLevel?: number
     }) {
 
         let defined = 0;
@@ -28,5 +30,6 @@ export default class SourceConfig {
         this.osmTags = params.osmTags;
         this.overpassScript = params.overpassScript;
         this.geojsonSource = params.geojsonSource;
+        this.geojsonZoomLevel = params.geojsonSourceLevel;
     }
 }

Logic/FeatureSource/FeaturePipeline.ts

@@ -33,14 +33,9 @@ export default class FeaturePipeline implements FeatureSource {
                                 updater)
                         )), layout));
 
-        const geojsonSources: GeoJsonSource [] = []
+        const geojsonSources: FeatureSource [] = GeoJsonSource
-        for (const flayer of flayers.data) {
+            .ConstructMultiSource(flayers.data, locationControl)
-            const sourceUrl = flayer.layerDef.source.geojsonSource
+            .map(geojsonSource => new RegisteringFeatureSource(new FeatureDuplicatorPerLayer(flayers, geojsonSource)));
-            if (sourceUrl !== undefined) {
-                geojsonSources.push(new RegisteringFeatureSource(new FeatureDuplicatorPerLayer(flayers,
-                    new GeoJsonSource(flayer.layerDef.id, sourceUrl))))
-            }
-        }
         
         const amendedLocalStorageSource =
             new RememberingSource(new RegisteringFeatureSource(new FeatureDuplicatorPerLayer(flayers, new LocalStorageSource(layout))

Logic/FeatureSource/GeoJsonSource.ts

@@ -1,51 +1,195 @@
 import FeatureSource from "./FeatureSource";
 import {UIEventSource} from "../UIEventSource";
 import * as $ from "jquery";
+import {control} from "leaflet";
+import zoom = control.zoom;
+import Loc from "../../Models/Loc";
+import State from "../../State";
+import {Utils} from "../../Utils";
+import LayerConfig from "../../Customizations/JSON/LayerConfig";
+
 
 /**
  * Fetches a geojson file somewhere and passes it along
  */
 export default class GeoJsonSource implements FeatureSource {
+
     features: UIEventSource<{ feature: any; freshness: Date }[]>;
 
-    constructor(layerId: string, url: string, onFail: ((errorMsg: any) => void) = undefined) {
+    private readonly onFail: ((errorMsg: any, url: string) => void) = undefined;
+
+    private readonly layerId: string;
+
+    private readonly seenids: Set<string> = new Set<string>()
+
+    constructor(locationControl: UIEventSource<Loc>,
+                flayer: { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig },
+                onFail?: ((errorMsg: any) => void)) {
+        this.layerId = flayer.layerDef.id;
+        let url = flayer.layerDef.source.geojsonSource;
+        const zoomLevel = flayer.layerDef.source.geojsonZoomLevel;
+
+        this.features = new UIEventSource<{ feature: any; freshness: Date }[]>([])
+
+        if (zoomLevel === undefined) {
+            // This is a classic, static geojson layer
             if (onFail === undefined) {
                 onFail = errorMsg => {
                     console.warn(`Could not load geojson layer from`, url, "due to", errorMsg)
                 }
             }
-        this.features = new UIEventSource<{ feature: any; freshness: Date }[]>(undefined)
+            this.onFail = onFail;
+
+            this.LoadJSONFrom(url)
+        } else {
+            // This is a dynamic template with a fixed zoom level
+            url = url.replace("{z}", "" + zoomLevel)
+            const loadedTiles = new Set<string>();
+            const self = this;
+            this.onFail = (msg, url) => {
+                console.warn(`Could not load geojson layer from`, url, "due to", msg)
+                loadedTiles.delete(url)
+            }
+
+            const neededTiles = locationControl.map(
+                location => {
+
+                    if (!flayer.isDisplayed.data) {
+                        return undefined;
+                    }
+
+                    // Yup, this is cheating to just get the bounds here
+                    const bounds = State.state.leafletMap.data.getBounds()
+                    const tileRange = Utils.TileRangeBetween(zoomLevel, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
+                    const needed = new Set<string>();
+                    for (let x = tileRange.xstart; x <= tileRange.xend; x++) {
+                        for (let y = tileRange.ystart; y <= tileRange.yend; y++) {
+                            let neededUrl = url.replace("{x}", "" + x).replace("{y}", "" + y);
+                            needed.add(neededUrl)
+                        }
+                    }
+                    return needed;
+                }
+            );
+            neededTiles.stabilized(250).addCallback((needed: Set<string>) => {
+                if (needed === undefined) {
+                    return;
+                }
+                needed.forEach(neededTile => {
+                    if (loadedTiles.has(neededTile)) {
+                        return;
+                    }
+
+                    loadedTiles.add(neededTile)
+                    self.LoadJSONFrom(neededTile)
+
+                })
+            })
+
+        }
+    }
+
+    /**
+     * Merges together the layers which have the same source
+     * @param flayers
+     * @param locationControl
+     * @constructor
+     */
+    public static ConstructMultiSource(flayers: { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig }[], locationControl: UIEventSource<Loc>): GeoJsonSource[] {
+
+        const flayersPerSource = new Map<string, { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig }[]>();
+        for (const flayer of flayers) {
+            const url = flayer.layerDef.source.geojsonSource
+            if (url === undefined) {
+                continue;
+            }
+
+            if (!flayersPerSource.has(url)) {
+                flayersPerSource.set(url, [])
+            }
+            flayersPerSource.get(url).push(flayer)
+        }
+
+        console.log("SOURCES", flayersPerSource)
+
+        const sources: GeoJsonSource[] = []
+
+        flayersPerSource.forEach((flayers, key) => {
+            if (flayers.length == 1) {
+                sources.push(new GeoJsonSource(locationControl, flayers[0]));
+                return;
+            }
+
+            const zoomlevels = Utils.Dedup(flayers.map(flayer => "" + (flayer.layerDef.source.geojsonZoomLevel ?? "")))
+            if (zoomlevels.length > 1) {
+                throw "Multiple zoomlevels defined for same geojson source " + key
+            }
+
+            let isShown = new UIEventSource<boolean>(true, "IsShown for multiple layers: or of multiple values");
+            for (const flayer of flayers) {
+                flayer.isDisplayed.addCallbackAndRun(() => {
+                    let value = false;
+                    for (const flayer of flayers) {
+                        value = flayer.isDisplayed.data || value;
+                    }
+                    isShown.setData(value);
+                });
+
+            }
+
+            const source = new GeoJsonSource(locationControl, {
+                isDisplayed: isShown,
+                layerDef: flayers[0].layerDef // We only care about the source info here
+            })
+            sources.push(source)
+
+        })
+        return sources;
+
+    }
+
+    private LoadJSONFrom(url: string) {
         const eventSource = this.features;
+        const self = this;
         $.getJSON(url, function (json, status) {
             if (status !== "success") {
                 console.log("Fetching geojson failed failed")
-                onFail(status);
+                self.onFail(status, url);
                 return;
             }
 
             if (json.elements === [] && json.remarks.indexOf("runtime error") > 0) {
                 console.log("Timeout or other runtime error");
-                onFail("Runtime error (timeout)")
+                self.onFail("Runtime error (timeout)", url)
                 return;
             }
             const time = new Date();
-            const features: { feature: any, freshness: Date } [] = []
+            const newFeatures: { feature: any, freshness: Date } [] = []
             let i = 0;
+            let skipped = 0;
             for (const feature of json.features) {
                 if (feature.properties.id === undefined) {
                     feature.properties.id = url + "/" + i;
                     feature.id = url + "/" + i;
                     i++;
                 }
-                feature._matching_layer_id = layerId;
+                if (self.seenids.has(feature.properties.id)) {
-                features.push({feature: feature, freshness: time})
+                    skipped++;
+                    continue;
                 }
-            console.log("Loaded features are", features)
+                self.seenids.add(feature.properties.id)
-            eventSource.setData(features)
 
-        }).fail(onFail)
+                newFeatures.push({feature: feature, freshness: time})
+            }
+            console.log("Downloaded "+newFeatures.length+" new features and "+skipped+" already seen features from "+ url);
             
+            if(newFeatures.length == 0){
+                return;
             }
             
+            eventSource.setData(eventSource.data.concat(newFeatures))
+
+        }).fail(msg => self.onFail(msg, url))
+    }
 
 }

Logic/Osm/ExtractRelations.ts

@@ -21,8 +21,14 @@ export default class ExtractRelations {
         State.state.knownRelations.setData(memberships)
     }
 
-    private static GetRelationElements(overpassJson: any): Relation[] {
+    /**
-        const relations = overpassJson.elements.filter(element => element.type === "relation")
+     * Gets an overview of the relations - except for multipolygons. We don't care about those
+     * @param overpassJson
+     * @constructor
+     */
+    public static GetRelationElements(overpassJson: any): Relation[] {
+        const relations = overpassJson.elements
+            .filter(element => element.type === "relation" && element.tags.type !== "multipolygon")
         for (const relation of relations) {
             relation.properties = relation.tags
         }

Logic/Osm/Overpass.ts

@@ -48,7 +48,7 @@ export class Overpass {
             }).fail(onFail)
     }
 
-    private buildQuery(bbox: string): string {
+    buildQuery(bbox: string): string {
         const filters = this._filter.asOverpass()
         let filter = ""
         for (const filterOr of filters) {

Utils.ts

@@ -156,8 +156,6 @@ export class Utils {
     }
 
     static Merge(source: any, target: any) {
-        target = JSON.parse(JSON.stringify(target));
-        source = JSON.parse(JSON.stringify(source));
         for (const key in source) {
             const sourceV = source[key];
             const targetV = target[key]
@@ -204,6 +202,26 @@ export class Utils {
         return {x: Utils.lon2tile(lon, z), y: Utils.lat2tile(lat, z), z: z}
     }
     
+    static TileRangeBetween(zoomlevel: number, lat0: number, lon0: number, lat1:number, lon1: number) : TileRange{
+        const t0 = Utils.embedded_tile(lat0, lon0, zoomlevel)
+        const t1 = Utils.embedded_tile(lat1, lon1, zoomlevel)
+
+        const xstart = Math.min(t0.x, t1.x)
+        const xend = Math.max(t0.x, t1.x)
+        const ystart = Math.min(t0.y, t1.y)
+        const yend = Math.max(t0.y, t1.y)
+        const total = (1 + xend - xstart) * (1 + yend - ystart)
+        
+        return {
+            xstart: xstart,
+            xend: xend,
+            ystart: ystart,
+            yend: yend, 
+            total: total,
+            zoomlevel: zoomlevel
+        }
+    }
+
     public static MinifyJSON(stringified: string): string {
         stringified = stringified.replace(/\|/g, "||");
 
@@ -257,3 +275,13 @@ export class Utils {
         return (Math.floor((1 - Math.log(Math.tan(lat * Math.PI / 180) + 1 / Math.cos(lat * Math.PI / 180)) / Math.PI) / 2 * Math.pow(2, zoom)));
     }
 }
+
+
+export interface TileRange{
+    xstart: number,
+    ystart: number,
+    xend: number,
+    yend: number,
+    total: number,
+    zoomlevel: number
+}

assets/themes/speelplekken/speelplekken.json

@@ -24,19 +24,64 @@
   "socialImage": "",
   "defaultBackgroundId": "CartoDB.Positron",
   "layers": [
-    "play_forest",
+    {
-    "playground",
+      "builtin": "play_forest",
-    "sport_pitch",
+      "override": {
+        "source": {
+          "geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson",
+          "geoJsonZoomLevel": 14
+        }
+      }
+    },
+    {
+      "builtin": "playground",
+      "override": {
+        "source": {
+          "geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson",
+          "geoJsonZoomLevel": 14
+        }
+      }
+    },
+    {
+      "builtin": "sport_pitch",
+      "override": {
+        "minzoom": 15,
+        "source": {
+          "geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson",
+          "geoJsonZoomLevel": 14
+        }
+      }
+    },
     {
       "builtin": "slow_roads",
       "override": {
+        "source": {
+          "geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson",
+          "geoJsonZoomLevel": 14
+        },
         "calculatedTags": [
           "_part_of_walking_routes=feat.memberships().map(r => \"<a href='#relation/\"+r.relation.id+\"'>\" + r.relation.tags.name + \"</a>\").join(', ')"
         ]
       }
     },
-    "grass_in_parks",
+    {
-    "village_green",
+      "builtin": "grass_in_parks",
+      "override": {
+        "source": {
+          "geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson",
+          "geoJsonZoomLevel": 14
+        }
+      }
+    },
+    {
+      "builtin": "village_green",
+      "override": {
+        "source": {
+          "geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson",
+          "geoJsonZoomLevel": 14
+        }
+      }
+    },
     {
       "id": "walking_routes",
       "name": {
@@ -50,7 +95,9 @@
             "route=foot",
             "operator=provincie Antwerpen"
           ]
-        }
+        },
+        "geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson",
+        "geoJsonZoomLevel": 14
       },
       "title": {
         "render": "Wandeling <i>{name}</i>",
@@ -141,17 +188,6 @@
       "width": {
         "render": "3"
       }
-    },
-    {
-      "id": "speelplekken-cache",
-      "name": "",
-      "source": {
-        "osmTags": {
-          "or": []
-        },
-        "geoJson": "https://pietervdvn.github.io/speelplekken-cache.geojson"
-      },
-      "passAllFeatures": true
     }
   ],
   "roamingRenderings": [

package.json

@@ -9,12 +9,13 @@
   "scripts": {
     "increase-memory": "export NODE_OPTIONS=--max_old_space_size=4096",
     "start": "ts-node scripts/generateLayerOverview.ts --no-fail && npm run increase-memory && parcel *.html UI/** Logic/** assets/** assets/**/** assets/**/**/** vendor/* vendor/*/*",
-    "test": "ts-node test/Tag.spec.ts && ts-node test/TagQuestion.spec.ts && ts-node test/ImageSearcher.spec.ts && ts-node test/ImageAttribution.spec.ts",
+    "test": "ts-node test/Tag.spec.ts && ts-node test/TagQuestion.spec.ts && ts-node test/ImageSearcher.spec.ts && ts-node test/ImageAttribution.spec.ts && ts-node test/Theme.spec.ts",
     "generate:editor-layer-index": "cd assets/ && wget https://osmlab.github.io/editor-layer-index/imagery.geojson --output-document=editor-layer-index.json",
     "generate:images": "ts-node scripts/generateIncludedImages.ts",
     "generate:translations": "ts-node scripts/generateTranslations.ts",
     "generate:layouts": "ts-node scripts/generateLayouts.ts",
     "generate:docs": "ts-node scripts/generateDocs.ts && ts-node scripts/generateTaginfoProjectFiles.ts",
+    "generate:cache:speelplekken": "ts-node scripts/generateCache.ts speelplekken 14 ./cache/speelplekken 51.2003 4.3925 51.1058 4.5087",
     "generate:layeroverview": "ts-node scripts/generateLayerOverview.ts --no-fail",
     "generate:licenses": "ts-node scripts/generateLicenseInfo.ts --no-fail",
     "validate:layeroverview": "ts-node scripts/generateLayerOverview.ts --report",

scripts/ScriptUtils.ts

@@ -1,4 +1,5 @@
 import {lstatSync, readdirSync} from "fs";
+import * as https from "https";
 
 export default class ScriptUtils {
     public static readDirRecSync(path): string[] {
@@ -17,4 +18,29 @@ export default class ScriptUtils {
         return result;
     }
     
+    public static DownloadJSON(url, continuation : (parts : string []) => void){
+        https.get(url, (res) => {
+            console.log("Got response!")
+            const parts : string[] = []
+            res.setEncoding('utf8');
+            res.on('data', function (chunk) {
+                // @ts-ignore
+                parts.push(chunk)
+            });
+
+            res.addListener('end', function () {
+                continuation(parts)
+            });
+        })
+    }
+
+    public static sleep(ms) {
+        return new Promise((resolve) => {
+            console.debug("Sleeping for", ms)
+            setTimeout(resolve, ms);
+           
+        });
+    }
+
+
 }

scripts/fixGeoJson.ts

@@ -1,21 +0,0 @@
-
-
-// Loads a geojson file downloaded from overpass, renames "@id" to "id" and deletes "@relations"
-
-import {readFileSync, writeFileSync} from "fs";
-
-const source = process.argv[2] ?? "~/Downloads/export.json"
-console.log("Fixing up ", source)
-const contents = readFileSync(source, "UTF8");
-const f = JSON.parse(contents);
-let i = 0
-for (const feature of f.features) {
-    if(feature.properties == undefined){
-        continue
-    }
-    feature.properties["id"] = feature.properties["@id"]
-    feature.properties["@id"] = undefined
-    feature.properties["@relations"] = undefined
-}
-
-writeFileSync(source+".fixed", JSON.stringify(f, null, "  "))

scripts/generateCache.ts

@@ -0,0 +1,176 @@
+/**
+ * Generates a collection of geojson files based on an overpass query for a given theme
+ */
+import {TileRange, Utils} from "../Utils";
+
+Utils.runningFromConsole = true
+import {Overpass} from "../Logic/Osm/Overpass";
+import {writeFileSync, existsSync, readFileSync} from "fs";
+import {TagsFilter} from "../Logic/Tags/TagsFilter";
+import {Or} from "../Logic/Tags/Or";
+import LayoutConfig from "../Customizations/JSON/LayoutConfig";
+import {AllKnownLayouts} from "../Customizations/AllKnownLayouts";
+import ScriptUtils from "./ScriptUtils";
+import ExtractRelations from "../Logic/Osm/ExtractRelations";
+import * as OsmToGeoJson from "osmtogeojson";
+import {Script} from "vm";
+
+function createOverpassObject(theme: LayoutConfig) {
+    let filters: TagsFilter[] = [];
+    let extraScripts: string[] = [];
+    for (const layer of theme.layers) {
+        if (typeof (layer) === "string") {
+            throw "A layer was not expanded!"
+        }
+        if (layer.doNotDownload) {
+            continue;
+        }
+        if (layer.source.geojsonSource !== undefined) {
+            // Not our responsibility to download this layer!
+            continue;
+        }
+
+
+        // Check if data for this layer has already been loaded
+        if (layer.source.overpassScript !== undefined) {
+            extraScripts.push(layer.source.overpassScript)
+        } else {
+            filters.push(layer.source.osmTags);
+        }
+    }
+    filters = Utils.NoNull(filters)
+    extraScripts = Utils.NoNull(extraScripts)
+    if (filters.length + extraScripts.length === 0) {
+        throw "Nothing to download! The theme doesn't declare anything to download"
+    }
+    return new Overpass(new Or(filters), extraScripts);
+}
+
+function saveResponse(chunks: string[], targetDir: string) {
+    const contents = chunks.join("")
+    if (contents.startsWith("<?xml")) {
+        // THis is an error message
+        console.error("Failed to create ", targetDir, "probably over quota")
+        return;
+    }
+    writeFileSync(targetDir, contents)
+}
+
+function rawJsonName(targetDir: string, x: number, y: number, z: number): string {
+    return targetDir + "_" + z + "_" + x + "_" + y + ".json"
+}
+
+function geoJsonName(targetDir: string, x: number, y: number, z: number): string {
+    return targetDir + "_" + z + "_" + x + "_" + y + ".geojson"
+}
+
+function metaJsonName(targetDir: string, x: number, y: number, z: number): string {
+    return targetDir + "_" + z + "_" + x + "_" + y + ".meta.json"
+}
+
+async function downloadRaw(targetdir: string,  r: TileRange, overpass: Overpass) {
+    let downloaded = 0
+    for (let x = r.xstart; x <= r.xend; x++) {
+        for (let y = r.ystart; y <= r.yend; y++) {
+            console.log("x:", (x - r.xstart), "/", (r.xend - r.xstart), "; y:", (y - r.ystart), "/", (r.yend - r.ystart), "; total: ", downloaded, "/", r.total)
+            downloaded++;
+            const filename = rawJsonName(targetdir, x, y, r.zoomlevel)
+            if (existsSync(filename)) {
+                console.log("Already exists: ", filename)
+                continue;
+            }
+
+            const boundsArr = Utils.tile_bounds(r.zoomlevel, x, y)
+            const bounds = {
+                north: Math.max(boundsArr[0][0], boundsArr[1][0]),
+                south: Math.min(boundsArr[0][0], boundsArr[1][0]),
+                east: Math.max(boundsArr[0][1], boundsArr[1][1]),
+                west: Math.min(boundsArr[0][1], boundsArr[1][1])
+            }
+            console.log("Downloading tile", r.zoomlevel, x, y, "with bounds", bounds)
+            const url = overpass.buildQuery("[bbox:" + bounds.south + "," + bounds.west + "," + bounds.north + "," + bounds.east + "]")
+
+            ScriptUtils.DownloadJSON(url,
+                chunks => {
+                    saveResponse(chunks, filename)
+                })
+
+            await ScriptUtils.sleep(10000)
+            console.debug("Waking up")
+
+        }
+    }
+}
+
+async function postProcess(targetdir: string, r: TileRange) {
+    let processed = 0;
+    for (let x = r.xstart; x <= r.xend; x++) {
+        for (let y = r.ystart; y <= r.yend; y++) {
+            processed++;
+            const filename = rawJsonName(targetdir, x, y, r.zoomlevel)
+            console.log(" Post processing", processed, "/",r. total, filename)
+            if (!existsSync(filename)) {
+                throw "Not found - and not downloaded. Run this script again!: " + filename
+            }
+
+            // We read the raw OSM-file and convert it to a geojson
+            const rawOsm = JSON.parse(readFileSync(filename, "UTF8"))
+
+            // Create and save the geojson file - which is the main chunk of the data
+            const geojson = OsmToGeoJson.default(rawOsm);
+            writeFileSync(geoJsonName(targetdir, x, y, r.zoomlevel), JSON.stringify(geojson))
+
+            // Extract the relationship information
+            const relations = ExtractRelations.GetRelationElements(rawOsm)
+            const osmTime = new Date(rawOsm.osm3s.timestamp_osm_base);
+
+            const meta = {
+                freshness: osmTime,
+                relations: relations
+            }
+
+            writeFileSync(
+                metaJsonName(targetdir, x, y, r.zoomlevel),
+                JSON.stringify(meta)
+            )
+        }
+    }
+}
+
+async function main(args: string[]) {
+
+    if (args.length == 0) {
+        console.error("Expected arguments are: theme zoomlevel targetdirectory lat0 lon0 lat1 lon1")
+        return;
+    }
+    const themeName = args[0]
+    const zoomlevel = Number(args[1])
+    const targetdir = args[2]
+    const lat0 = Number(args[3])
+    const lon0 = Number(args[4])
+    const lat1 = Number(args[5])
+    const lon1 = Number(args[6])
+
+    const tileRange = Utils.TileRangeBetween(zoomlevel, lat0, lon0, lat1, lon1)
+
+    const theme = AllKnownLayouts.allKnownLayouts.get(themeName)
+    if (theme === undefined) {
+        const keys = []
+        AllKnownLayouts.allKnownLayouts.forEach((_, key) => {
+            keys.push(key)
+        })
+        console.error("The theme " + theme + " was not found; try one of ", keys);
+        return
+    }
+
+    const overpass = createOverpassObject(theme)
+
+
+    await downloadRaw(targetdir, tileRange, overpass)
+    await postProcess(targetdir, tileRange)
+}
+
+
+let args = [...process.argv]
+args.splice(0, 2)
+main(args);

test/Theme.spec.ts

@@ -0,0 +1,48 @@
+import T from "./TestHelper";
+import {Utils} from "../Utils";
+
+Utils.runningFromConsole = true;
+import TagRenderingQuestion from "../UI/Popup/TagRenderingQuestion";
+import {UIEventSource} from "../Logic/UIEventSource";
+import TagRenderingConfig from "../Customizations/JSON/TagRenderingConfig";
+import LayoutConfig from "../Customizations/JSON/LayoutConfig";
+import {LayoutConfigJson} from "../Customizations/JSON/LayoutConfigJson";
+import * as assert from "assert";
+
+
+new T("Theme tests",
+    [
+        ["Nested overrides work", () => {
+    
+            const themeConfigJson : LayoutConfigJson = {
+                description: "Descr",
+                icon: "",
+                language: ["en"],
+                layers: [
+                    {
+                        builtin: "public_bookcase",
+                        override: {
+                            source:{
+                                geoJson: "xyz"
+                            }
+                        }
+                    }
+                ],
+                maintainer: "",
+                startLat: 0,
+                startLon: 0,
+                startZoom: 0,
+                title: {
+                    en: "Title"
+                },
+                version: "",
+                id: "test"
+            }
+            
+            const themeConfig = new LayoutConfig(themeConfigJson);
+            assert.equal("xyz", themeConfig.layers[0].source.geojsonSource)
+    
+            
+        }]
+    ]
+);