Fix multilayer geojson source

Customizations/JSON/LayerConfigJson.ts

@@ -30,7 +30,7 @@ export interface LayerConfigJson {
      *
      * source: {osmTags: "key=value"} will fetch all objects with given tags from OSM. Currently, this will create a query to overpass and fetch the data - in the future this might fetch from the OSM API
      * source: {geoJson: "https://my.source.net/some-geo-data.geojson"} to fetch a geojson from a third party source
-     * source: {geoJson: "https://my.source.net/some-tile-geojson-{z}-{x}-{y}.geojson", geoJsonZoomLevel: 14} to use a tiled geojson source. The web server must offer multiple geojsons. {z}, {x} and {y} are substituted
+     * source: {geoJson: "https://my.source.net/some-tile-geojson-{layer}-{z}-{x}-{y}.geojson", geoJsonZoomLevel: 14} to use a tiled geojson source. The web server must offer multiple geojsons. {z}, {x} and {y} are substituted by the location; {layer} is substituted with the id of the loaded layer
      *
      * source: {overpassScript: "<custom overpass tags>"} when you want to do special things. _This should be really rare_.
      *      This means that the data will be pulled from overpass with this script, and will ignore the osmTags for the query

Logic/FeatureSource/GeoJsonSource.ts

@@ -16,15 +16,15 @@ export default class GeoJsonSource implements FeatureSource {
 
     public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>;
     public readonly name;
-    private readonly onFail: ((errorMsg: any, url: string) => void) = undefined;
+    private onFail: ((errorMsg: any, url: string) => void) = undefined;
     private readonly layerId: string;
     private readonly seenids: Set<string> = new Set<string>()
 
-    constructor(locationControl: UIEventSource<Loc>,
+   private constructor(locationControl: UIEventSource<Loc>,
                 flayer: { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig },
                 onFail?: ((errorMsg: any) => void)) {
         this.layerId = flayer.layerDef.id;
-        let url = flayer.layerDef.source.geojsonSource;
+        let url = flayer.layerDef.source.geojsonSource.replace("{layer}", flayer.layerDef.id);
         this.name = "GeoJsonSource of " + url;
         const zoomLevel = flayer.layerDef.source.geojsonZoomLevel;
 
@@ -34,58 +34,66 @@ export default class GeoJsonSource implements FeatureSource {
             // This is a classic, static geojson layer
             if (onFail === undefined) {
                 onFail = errorMsg => {
-                    console.warn(`Could not load geojson layer from`, url, "due to", errorMsg)
                 }
             }
             this.onFail = onFail;
 
             this.LoadJSONFrom(url)
         } else {
-            // This is a dynamic template with a fixed zoom level
+            this.ConfigureDynamicLayer(url, zoomLevel, locationControl, flayer)
-            url = url.replace("{z}", "" + zoomLevel)
+        }
-            const loadedTiles = new Set<string>();
+    }
-            const self = this;
+    
-            this.onFail = (msg, url) => {
+    private ConfigureDynamicLayer(url: string, zoomLevel: number, locationControl: UIEventSource<Loc>, flayer:  { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig }){
-                console.warn(`Could not load geojson layer from`, url, "due to", msg)
+        // This is a dynamic template with a fixed zoom level
-                loadedTiles.delete(url)
+        url = url.replace("{z}", "" + zoomLevel)
-            }
+        const loadedTiles = new Set<string>();
+        const self = this;
+        this.onFail = (msg, url) => {
+            console.warn(`Could not load geojson layer from`, url, "due to", msg)
+            loadedTiles.add(url); // We add the url to the 'loadedTiles' in order to not reload it in the future
+        }
 
-            const neededTiles = locationControl.map(
+        const neededTiles = locationControl.map(
-                location => {
+            location => {
-
+                // Yup, this is cheating to just get the bounds here
-                    if (!flayer.isDisplayed.data) {
+                const bounds = State.state.leafletMap.data.getBounds()
-                        return undefined;
+                const tileRange = Utils.TileRangeBetween(zoomLevel, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
+                const needed = new Set<string>();
+                for (let x = tileRange.xstart; x <= tileRange.xend; x++) {
+                    for (let y = tileRange.ystart; y <= tileRange.yend; y++) {
+                        let neededUrl = url.replace("{x}", "" + x).replace("{y}", "" + y);
+                        needed.add(neededUrl)
                     }
-
-                    // Yup, this is cheating to just get the bounds here
-                    const bounds = State.state.leafletMap.data.getBounds()
-                    const tileRange = Utils.TileRangeBetween(zoomLevel, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
-                    const needed = new Set<string>();
-                    for (let x = tileRange.xstart; x <= tileRange.xend; x++) {
-                        for (let y = tileRange.ystart; y <= tileRange.yend; y++) {
-                            let neededUrl = url.replace("{x}", "" + x).replace("{y}", "" + y);
-                            needed.add(neededUrl)
-                        }
-                    }
-                    return needed;
                 }
-            );
+                return needed;
-            neededTiles.stabilized(250).addCallback((needed: Set<string>) => {
+            }
-                if (needed === undefined) {
+        , [flayer.isDisplayed]);
+        neededTiles.stabilized(250).addCallback((needed: Set<string>) => {
+            if (needed === undefined) {
+                return;
+            }
+            if (!flayer.isDisplayed.data) {
+                // No need to download! - the layer is disabled
+                return;
+            }
+        console.log("???", locationControl, flayer.layerDef)
+           if(locationControl.data.zoom < flayer.layerDef.minzoom){
+               console.log("Not downloading ", needed, "not sufficiently zoomed")
+               return;
+           }
+            
+            needed.forEach(neededTile => {
+                if (loadedTiles.has(neededTile)) {
                     return;
                 }
-                needed.forEach(neededTile => {
-                    if (loadedTiles.has(neededTile)) {
-                        return;
-                    }
 
-                    loadedTiles.add(neededTile)
+                loadedTiles.add(neededTile)
-                    self.LoadJSONFrom(neededTile)
+                self.LoadJSONFrom(neededTile)
 
-                })
             })
+        })
 
-        }
     }
 
     /**
@@ -98,7 +106,7 @@ export default class GeoJsonSource implements FeatureSource {
 
         const flayersPerSource = new Map<string, { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig }[]>();
         for (const flayer of flayers) {
-            const url = flayer.layerDef.source.geojsonSource
+            const url = flayer.layerDef.source.geojsonSource?.replace(/{layer}/g, flayer.layerDef.id)
             if (url === undefined) {
                 continue;
             }

Logic/UIEventSource.ts

@@ -98,7 +98,8 @@ export class UIEventSource<T> {
         const self = this;
 
         const newSource = new UIEventSource<J>(
-            f(this.data)
+            f(this.data),
+            "map("+this.tag+")"
         );
 
         const update = function () {

Logic/Web/QueryParameters.ts

@@ -79,7 +79,7 @@ export class QueryParameters {
             return QueryParameters.knownSources[key];
         }
         QueryParameters.addOrder(key);
-        const source = new UIEventSource<string>(deflt);
+        const source = new UIEventSource<string>(deflt, "&"+key);
         QueryParameters.knownSources[key] = source;
         source.addCallback(() => QueryParameters.Serialize())
         return source;

Models/Constants.ts

@@ -2,7 +2,7 @@ import { Utils } from "../Utils";
 
 export default class Constants {
     
-    public static vNumber = "0.6.11b";
+    public static vNumber = "0.6.11c";
 
     // The user journey states thresholds when a new feature gets unlocked
     public static userJourney = {

assets/themes/speelplekken/speelplekken.json

@@ -55,7 +55,12 @@
         "minzoom":14
       }
     },
-     "sport_pitch",
+    {
+      "builtin":   "sport_pitch",
+      "override": {
+        "minzoom": 14
+      }
+    },
     
     {
       "builtin": "slow_roads",
@@ -174,9 +179,14 @@
       }
     }
   ],
+  "clustering": {
+    "maxZoom": 16,
+    "minNeededElements": 100
+  },
   "overrideAll": {
     "source": {
-      "geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson",
+      "geoJsonLocal": "http://127.0.0.1:8080/speelplekken_{layer}_{z}_{x}_{y}.geojson",
+      "geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{layer}_{z}_{x}_{y}.geojson",
       "geoJsonZoomLevel": 14
     }
   },

scripts/generateCache.ts

@@ -2,6 +2,7 @@
  * Generates a collection of geojson files based on an overpass query for a given theme
  */
 import {TileRange, Utils} from "../Utils";
+
 Utils.runningFromConsole = true
 import {Overpass} from "../Logic/Osm/Overpass";
 import {existsSync, readFileSync, writeFileSync} from "fs";
@@ -15,7 +16,6 @@ import * as OsmToGeoJson from "osmtogeojson";
 import MetaTagging from "../Logic/MetaTagging";
 
 
-
 function createOverpassObject(theme: LayoutConfig) {
     let filters: TagsFilter[] = [];
     let extraScripts: string[] = [];
@@ -105,8 +105,8 @@ async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass)/
                     console.log("Didn't get an answer yet - waiting more")
                 }
             }
-            
+
-            if(!success){
+            if (!success) {
                 failed++;
                 console.log("Hit the rate limit - waiting 90s")
                 for (let i = 0; i < 90; i++) {
@@ -159,12 +159,37 @@ async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig)
             // Extract the relationship information
             const relations = ExtractRelations.BuildMembershipTable(ExtractRelations.GetRelationElements(rawOsm))
             MetaTagging.addMetatags(featuresFreshness, relations, theme.layers);
-            writeFileSync(geoJsonName(targetdir, x, y, r.zoomlevel), JSON.stringify(geojson))
+
-         
+
+            writeFileSync(geoJsonName(targetdir, x, y, r.zoomlevel), JSON.stringify(geojson, null, " "))
+
         }
     }
 }
 
+async function splitPerLayer(targetdir: string, r: TileRange, theme: LayoutConfig) {
+    let processed = 0;
+    const z = r.zoomlevel;
+    for (let x = r.xstart; x <= r.xend; x++) {
+        for (let y = r.ystart; y <= r.yend; y++) {
+            const file = readFileSync(geoJsonName(targetdir, x, y, z), "UTF8")
+
+            for (const layer of theme.layers) {
+                const geojson = JSON.parse(file)
+                geojson.features = geojson.features.filter(f => f._matching_layer_id === layer.id)
+                if(geojson.features.length == 0){
+                    continue;
+                }
+                const new_path = geoJsonName(targetdir+"_"+layer.id, x, y, z);
+                writeFileSync(new_path, JSON.stringify(geojson, null, " "))
+            }
+        
+            
+        }
+    }
+}
+
+
 async function main(args: string[]) {
 
     if (args.length == 0) {
@@ -203,6 +228,7 @@ async function main(args: string[]) {
     } while (failed > 0)
 
     await postProcess(targetdir, tileRange, theme)
+    await splitPerLayer(targetdir, tileRange, theme)
 }