From e530eba55ab9c278d35a7f283d099cd556fc6efc Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Tue, 12 Oct 2021 16:39:36 +0200 Subject: [PATCH 1/4] Quickfix: disable optimazation which blocks loading dynamic tiles --- .../TiledFeatureSource/DynamicGeoJsonTileSource.ts | 3 ++- assets/themes/natuurpunt/natuurpunt.json | 2 +- index.ts | 3 +++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts b/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts index fcfdbea5a..c2ccf518d 100644 --- a/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts +++ b/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts @@ -46,7 +46,8 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource { if(whitelist !== undefined){ const isWhiteListed = whitelist.get(zxy[1])?.has(zxy[2]) if(!isWhiteListed){ - return undefined; + console.log("Not whitelisted:",zxy, isWhiteListed, whitelist) + // return undefined; } } diff --git a/assets/themes/natuurpunt/natuurpunt.json b/assets/themes/natuurpunt/natuurpunt.json index 4b49e7465..fb45742ca 100644 --- a/assets/themes/natuurpunt/natuurpunt.json +++ b/assets/themes/natuurpunt/natuurpunt.json @@ -34,7 +34,7 @@ "hideFromOverview": true, "clustering": { "#": "Disable clustering for this theme", - "maxZoom": 0 + "maxZoom": 1 }, "layers": [ { diff --git a/index.ts b/index.ts index e5e58a6f0..dc738eaf4 100644 --- a/index.ts +++ b/index.ts @@ -63,6 +63,9 @@ if (path !== "index.html" && path !== "") { defaultLayout = QueryParameters.GetQueryParameter("layout", defaultLayout, "The layout to load into MapComplete").data; let layoutToUse: LayoutConfig = AllKnownLayouts.allKnownLayouts.get(defaultLayout.toLowerCase()); +if(layoutToUse.id === "natuurpunt"){ + localStorage.clear() +} const userLayoutParam = QueryParameters.GetQueryParameter("userlayout", "false", "If not 'false', a custom (non-official) theme is loaded. This custom layout can be done in multiple ways: \n\n- The hash of the URL contains a base64-encoded .json-file containing the theme definition\n- The hash of the URL contains a lz-compressed .json-file, as generated by the custom theme generator\n- The parameter itself is an URL, in which case that URL will be downloaded. It should point to a .json of a theme"); From e5a24ff499e04e874edd32b8da07359ac5f81294 Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Tue, 12 Oct 2021 19:11:48 +0200 Subject: [PATCH 2/4] Fix build --- index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.ts b/index.ts index dc738eaf4..3bfa4f28b 100644 --- a/index.ts +++ b/index.ts @@ -63,7 +63,7 @@ if (path !== "index.html" && path !== "") { defaultLayout = QueryParameters.GetQueryParameter("layout", defaultLayout, "The layout to load into MapComplete").data; let layoutToUse: LayoutConfig = AllKnownLayouts.allKnownLayouts.get(defaultLayout.toLowerCase()); -if(layoutToUse.id === "natuurpunt"){ +if(layoutToUse?.id === "natuurpunt"){ localStorage.clear() } From 2bcc573f2a148952b9c997ec7683956682da371f Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Tue, 12 Oct 2021 19:11:59 +0200 Subject: [PATCH 3/4] Fix build --- index.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/index.ts b/index.ts index 3bfa4f28b..7126a0a7c 100644 --- a/index.ts +++ b/index.ts @@ -63,10 +63,6 @@ if (path !== "index.html" && path !== "") { defaultLayout = QueryParameters.GetQueryParameter("layout", defaultLayout, "The layout to load into MapComplete").data; let layoutToUse: LayoutConfig = AllKnownLayouts.allKnownLayouts.get(defaultLayout.toLowerCase()); -if(layoutToUse?.id === "natuurpunt"){ - localStorage.clear() -} - const userLayoutParam = QueryParameters.GetQueryParameter("userlayout", "false", "If not 'false', a custom (non-official) theme is loaded. This custom layout can be done in multiple ways: \n\n- The hash of the URL contains a base64-encoded .json-file containing the theme definition\n- The hash of the URL contains a lz-compressed .json-file, as generated by the custom theme generator\n- The parameter itself is an URL, in which case that URL will be downloaded. It should point to a .json of a theme"); // Workaround/legacy to keep the old paramters working as I renamed some of them From 8bdabe5d73447a2e32208230e90917f588762af9 Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Wed, 13 Oct 2021 00:08:41 +0200 Subject: [PATCH 4/4] Fixes to caching --- .../DynamicGeoJsonTileSource.ts | 4 +- scripts/generateCache.ts | 40 +++++++++++++------ 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts b/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts index c2ccf518d..fc110e1c2 100644 --- a/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts +++ b/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts @@ -21,7 +21,9 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource { throw "Invalid layer: geojsonSource expected" } - const whitelistUrl = source.geojsonSource.replace("{z}_{x}_{y}.geojson", "overview.json") + const whitelistUrl = source.geojsonSource + .replace("{z}", ""+source.geojsonZoomLevel) + .replace("{x}_{y}.geojson", "overview.json") .replace("{layer}",layer.layerDef.id) let whitelist = undefined diff --git a/scripts/generateCache.ts b/scripts/generateCache.ts index 1377b3153..d92fc8628 100644 --- a/scripts/generateCache.ts +++ b/scripts/generateCache.ts @@ -20,6 +20,7 @@ import FeatureSource, {FeatureSourceForLayer} from "../Logic/FeatureSource/Featu import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSource"; import TiledFeatureSource from "../Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource"; import Constants from "../Models/Constants"; +import {GeoOperations} from "../Logic/GeoOperations"; ScriptUtils.fixUtils() @@ -170,11 +171,11 @@ function loadAllTiles(targetdir: string, r: TileRange, theme: LayoutConfig, extr /** * Load all the tiles into memory from disk */ -function postProcess(allFeatures: FeatureSource, theme: LayoutConfig, relationsTracker: RelationsTracker, targetdir: string) { - - +function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relationsTracker: RelationsTracker, targetdir: string, pointsOnlyLayers: string[]) { function handleLayer(source: FeatureSourceForLayer) { const layer = source.layer.layerDef; + const targetZoomLevel = layer.source.geojsonZoomLevel ?? 0 + const layerId = layer.id if (layer.source.isOsmCacheLayer !== true) { return; @@ -200,13 +201,10 @@ function postProcess(allFeatures: FeatureSource, theme: LayoutConfig, relationsT // At this point, we have all the features of the entire area. // However, we want to export them per tile of a fixed size, so we use a dynamicTileSOurce to split it up TiledFeatureSource.createHierarchy(source, { - minZoomLevel: 14, - maxZoomLevel: 14, + minZoomLevel: targetZoomLevel, + maxZoomLevel: targetZoomLevel, maxFeatureCount: undefined, registerTile: tile => { - if (tile.z < 12) { - return; - } if (tile.features.data.length === 0) { return } @@ -229,7 +227,7 @@ function postProcess(allFeatures: FeatureSource, theme: LayoutConfig, relationsT // All the tiles are written at this point // Only thing left to do is to create the index - const path = targetdir + "_" + layerId + "_overview.json" + const path = targetdir + "_" + layerId + "_" + targetZoomLevel + "_overview.json" const perX = {} createdTiles.map(i => Tiles.tile_from_index(i)).forEach(([z, x, y]) => { const key = "" + x @@ -240,7 +238,18 @@ function postProcess(allFeatures: FeatureSource, theme: LayoutConfig, relationsT }) writeFileSync(path, JSON.stringify(perX)) - + // And, if needed, to create a points-only layer + if(pointsOnlyLayers.indexOf(layer.id) >= 0){ + const features = source.features.data.map(f => f.feature) + const points = features.map(feature => GeoOperations.centerpoint(feature)) + console.log("Writing points overview for ", layerId) + const targetPath = targetdir+"_"+layerId+"_points.geojson" + // This is the geojson file containing all features for this tile + writeFileSync(targetPath, JSON.stringify({ + type: "FeatureCollection", + features: points + }, null, " ")) + } } new PerLayerFeatureSourceSplitter( @@ -255,10 +264,11 @@ function postProcess(allFeatures: FeatureSource, theme: LayoutConfig, relationsT } + async function main(args: string[]) { if (args.length == 0) { - console.error("Expected arguments are: theme zoomlevel targetdirectory lat0 lon0 lat1 lon1 [--generate-point-overview layer-name]") + console.error("Expected arguments are: theme zoomlevel targetdirectory lat0 lon0 lat1 lon1 [--generate-point-overview layer-name,layer-name,...]") return; } const themeName = args[0] @@ -268,6 +278,12 @@ async function main(args: string[]) { const lon0 = Number(args[4]) const lat1 = Number(args[5]) const lon1 = Number(args[6]) + + let generatePointLayersFor = [] + if(args[7] == "--generate-point-overview"){ + generatePointLayersFor = args[8].split(",") + } + const tileRange = Tiles.TileRangeBetween(zoomlevel, lat0, lon0, lat1, lon1) @@ -293,7 +309,7 @@ async function main(args: string[]) { const extraFeatures = await downloadExtraData(theme); const allFeaturesSource = loadAllTiles(targetdir, tileRange, theme, extraFeatures) - postProcess(allFeaturesSource, theme, relationTracker, targetdir) + sliceToTiles(allFeaturesSource, theme, relationTracker, targetdir, generatePointLayersFor) }