From f4dacab9ef9cb4f8bf064c3fcad01b05deae438c Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Fri, 23 Apr 2021 20:09:27 +0200 Subject: [PATCH] Fix multilayer geojson source --- Customizations/JSON/LayerConfigJson.ts | 2 +- Logic/FeatureSource/GeoJsonSource.ts | 90 +++++++++++--------- Logic/UIEventSource.ts | 3 +- Logic/Web/QueryParameters.ts | 2 +- Models/Constants.ts | 2 +- assets/themes/speelplekken/speelplekken.json | 14 ++- scripts/generateCache.ts | 36 ++++++-- 7 files changed, 97 insertions(+), 52 deletions(-) diff --git a/Customizations/JSON/LayerConfigJson.ts b/Customizations/JSON/LayerConfigJson.ts index 66af912..f532c0d 100644 --- a/Customizations/JSON/LayerConfigJson.ts +++ b/Customizations/JSON/LayerConfigJson.ts @@ -30,7 +30,7 @@ export interface LayerConfigJson { * * source: {osmTags: "key=value"} will fetch all objects with given tags from OSM. Currently, this will create a query to overpass and fetch the data - in the future this might fetch from the OSM API * source: {geoJson: "https://my.source.net/some-geo-data.geojson"} to fetch a geojson from a third party source - * source: {geoJson: "https://my.source.net/some-tile-geojson-{z}-{x}-{y}.geojson", geoJsonZoomLevel: 14} to use a tiled geojson source. The web server must offer multiple geojsons. {z}, {x} and {y} are substituted + * source: {geoJson: "https://my.source.net/some-tile-geojson-{layer}-{z}-{x}-{y}.geojson", geoJsonZoomLevel: 14} to use a tiled geojson source. The web server must offer multiple geojsons. {z}, {x} and {y} are substituted by the location; {layer} is substituted with the id of the loaded layer * * source: {overpassScript: ""} when you want to do special things. _This should be really rare_. * This means that the data will be pulled from overpass with this script, and will ignore the osmTags for the query diff --git a/Logic/FeatureSource/GeoJsonSource.ts b/Logic/FeatureSource/GeoJsonSource.ts index e6bdfcb..20cdb6c 100644 --- a/Logic/FeatureSource/GeoJsonSource.ts +++ b/Logic/FeatureSource/GeoJsonSource.ts @@ -16,15 +16,15 @@ export default class GeoJsonSource implements FeatureSource { public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>; public readonly name; - private readonly onFail: ((errorMsg: any, url: string) => void) = undefined; + private onFail: ((errorMsg: any, url: string) => void) = undefined; private readonly layerId: string; private readonly seenids: Set = new Set() - constructor(locationControl: UIEventSource, + private constructor(locationControl: UIEventSource, flayer: { isDisplayed: UIEventSource, layerDef: LayerConfig }, onFail?: ((errorMsg: any) => void)) { this.layerId = flayer.layerDef.id; - let url = flayer.layerDef.source.geojsonSource; + let url = flayer.layerDef.source.geojsonSource.replace("{layer}", flayer.layerDef.id); this.name = "GeoJsonSource of " + url; const zoomLevel = flayer.layerDef.source.geojsonZoomLevel; @@ -34,58 +34,66 @@ export default class GeoJsonSource implements FeatureSource { // This is a classic, static geojson layer if (onFail === undefined) { onFail = errorMsg => { - console.warn(`Could not load geojson layer from`, url, "due to", errorMsg) } } this.onFail = onFail; this.LoadJSONFrom(url) } else { - // This is a dynamic template with a fixed zoom level - url = url.replace("{z}", "" + zoomLevel) - const loadedTiles = new Set(); - const self = this; - this.onFail = (msg, url) => { - console.warn(`Could not load geojson layer from`, url, "due to", msg) - loadedTiles.delete(url) - } + this.ConfigureDynamicLayer(url, zoomLevel, locationControl, flayer) + } + } + + private ConfigureDynamicLayer(url: string, zoomLevel: number, locationControl: UIEventSource, flayer: { isDisplayed: UIEventSource, layerDef: LayerConfig }){ + // This is a dynamic template with a fixed zoom level + url = url.replace("{z}", "" + zoomLevel) + const loadedTiles = new Set(); + const self = this; + this.onFail = (msg, url) => { + console.warn(`Could not load geojson layer from`, url, "due to", msg) + loadedTiles.add(url); // We add the url to the 'loadedTiles' in order to not reload it in the future + } - const neededTiles = locationControl.map( - location => { - - if (!flayer.isDisplayed.data) { - return undefined; + const neededTiles = locationControl.map( + location => { + // Yup, this is cheating to just get the bounds here + const bounds = State.state.leafletMap.data.getBounds() + const tileRange = Utils.TileRangeBetween(zoomLevel, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest()) + const needed = new Set(); + for (let x = tileRange.xstart; x <= tileRange.xend; x++) { + for (let y = tileRange.ystart; y <= tileRange.yend; y++) { + let neededUrl = url.replace("{x}", "" + x).replace("{y}", "" + y); + needed.add(neededUrl) } - - // Yup, this is cheating to just get the bounds here - const bounds = State.state.leafletMap.data.getBounds() - const tileRange = Utils.TileRangeBetween(zoomLevel, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest()) - const needed = new Set(); - for (let x = tileRange.xstart; x <= tileRange.xend; x++) { - for (let y = tileRange.ystart; y <= tileRange.yend; y++) { - let neededUrl = url.replace("{x}", "" + x).replace("{y}", "" + y); - needed.add(neededUrl) - } - } - return needed; } - ); - neededTiles.stabilized(250).addCallback((needed: Set) => { - if (needed === undefined) { + return needed; + } + , [flayer.isDisplayed]); + neededTiles.stabilized(250).addCallback((needed: Set) => { + if (needed === undefined) { + return; + } + if (!flayer.isDisplayed.data) { + // No need to download! - the layer is disabled + return; + } + console.log("???", locationControl, flayer.layerDef) + if(locationControl.data.zoom < flayer.layerDef.minzoom){ + console.log("Not downloading ", needed, "not sufficiently zoomed") + return; + } + + needed.forEach(neededTile => { + if (loadedTiles.has(neededTile)) { return; } - needed.forEach(neededTile => { - if (loadedTiles.has(neededTile)) { - return; - } - loadedTiles.add(neededTile) - self.LoadJSONFrom(neededTile) + loadedTiles.add(neededTile) + self.LoadJSONFrom(neededTile) - }) }) + }) - } } /** @@ -98,7 +106,7 @@ export default class GeoJsonSource implements FeatureSource { const flayersPerSource = new Map, layerDef: LayerConfig }[]>(); for (const flayer of flayers) { - const url = flayer.layerDef.source.geojsonSource + const url = flayer.layerDef.source.geojsonSource?.replace(/{layer}/g, flayer.layerDef.id) if (url === undefined) { continue; } diff --git a/Logic/UIEventSource.ts b/Logic/UIEventSource.ts index 0c92d31..63ed45d 100644 --- a/Logic/UIEventSource.ts +++ b/Logic/UIEventSource.ts @@ -98,7 +98,8 @@ export class UIEventSource { const self = this; const newSource = new UIEventSource( - f(this.data) + f(this.data), + "map("+this.tag+")" ); const update = function () { diff --git a/Logic/Web/QueryParameters.ts b/Logic/Web/QueryParameters.ts index b1adfcd..d08df0f 100644 --- a/Logic/Web/QueryParameters.ts +++ b/Logic/Web/QueryParameters.ts @@ -79,7 +79,7 @@ export class QueryParameters { return QueryParameters.knownSources[key]; } QueryParameters.addOrder(key); - const source = new UIEventSource(deflt); + const source = new UIEventSource(deflt, "&"+key); QueryParameters.knownSources[key] = source; source.addCallback(() => QueryParameters.Serialize()) return source; diff --git a/Models/Constants.ts b/Models/Constants.ts index 156f871..3f65c84 100644 --- a/Models/Constants.ts +++ b/Models/Constants.ts @@ -2,7 +2,7 @@ import { Utils } from "../Utils"; export default class Constants { - public static vNumber = "0.6.11b"; + public static vNumber = "0.6.11c"; // The user journey states thresholds when a new feature gets unlocked public static userJourney = { diff --git a/assets/themes/speelplekken/speelplekken.json b/assets/themes/speelplekken/speelplekken.json index 0e76add..5ab7bd4 100644 --- a/assets/themes/speelplekken/speelplekken.json +++ b/assets/themes/speelplekken/speelplekken.json @@ -55,7 +55,12 @@ "minzoom":14 } }, - "sport_pitch", + { + "builtin": "sport_pitch", + "override": { + "minzoom": 14 + } + }, { "builtin": "slow_roads", @@ -174,9 +179,14 @@ } } ], + "clustering": { + "maxZoom": 16, + "minNeededElements": 100 + }, "overrideAll": { "source": { - "geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson", + "geoJsonLocal": "http://127.0.0.1:8080/speelplekken_{layer}_{z}_{x}_{y}.geojson", + "geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{layer}_{z}_{x}_{y}.geojson", "geoJsonZoomLevel": 14 } }, diff --git a/scripts/generateCache.ts b/scripts/generateCache.ts index 3f8ef71..6249266 100644 --- a/scripts/generateCache.ts +++ b/scripts/generateCache.ts @@ -2,6 +2,7 @@ * Generates a collection of geojson files based on an overpass query for a given theme */ import {TileRange, Utils} from "../Utils"; + Utils.runningFromConsole = true import {Overpass} from "../Logic/Osm/Overpass"; import {existsSync, readFileSync, writeFileSync} from "fs"; @@ -15,7 +16,6 @@ import * as OsmToGeoJson from "osmtogeojson"; import MetaTagging from "../Logic/MetaTagging"; - function createOverpassObject(theme: LayoutConfig) { let filters: TagsFilter[] = []; let extraScripts: string[] = []; @@ -105,8 +105,8 @@ async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass)/ console.log("Didn't get an answer yet - waiting more") } } - - if(!success){ + + if (!success) { failed++; console.log("Hit the rate limit - waiting 90s") for (let i = 0; i < 90; i++) { @@ -159,12 +159,37 @@ async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig) // Extract the relationship information const relations = ExtractRelations.BuildMembershipTable(ExtractRelations.GetRelationElements(rawOsm)) MetaTagging.addMetatags(featuresFreshness, relations, theme.layers); - writeFileSync(geoJsonName(targetdir, x, y, r.zoomlevel), JSON.stringify(geojson)) - + + + writeFileSync(geoJsonName(targetdir, x, y, r.zoomlevel), JSON.stringify(geojson, null, " ")) + } } } +async function splitPerLayer(targetdir: string, r: TileRange, theme: LayoutConfig) { + let processed = 0; + const z = r.zoomlevel; + for (let x = r.xstart; x <= r.xend; x++) { + for (let y = r.ystart; y <= r.yend; y++) { + const file = readFileSync(geoJsonName(targetdir, x, y, z), "UTF8") + + for (const layer of theme.layers) { + const geojson = JSON.parse(file) + geojson.features = geojson.features.filter(f => f._matching_layer_id === layer.id) + if(geojson.features.length == 0){ + continue; + } + const new_path = geoJsonName(targetdir+"_"+layer.id, x, y, z); + writeFileSync(new_path, JSON.stringify(geojson, null, " ")) + } + + + } + } +} + + async function main(args: string[]) { if (args.length == 0) { @@ -203,6 +228,7 @@ async function main(args: string[]) { } while (failed > 0) await postProcess(targetdir, tileRange, theme) + await splitPerLayer(targetdir, tileRange, theme) }