Merge branch 'develop'

This commit is contained in:
pietervdvn 2021-10-27 19:58:27 +02:00
commit f5f4bc7fde
17 changed files with 751 additions and 562 deletions

View file

@ -1,5 +1,6 @@
import * as turf from "@turf/turf"; import * as turf from "@turf/turf";
import {TileRange, Tiles} from "../Models/TileRange"; import {TileRange, Tiles} from "../Models/TileRange";
import {GeoOperations} from "./GeoOperations";
export class BBox { export class BBox {
@ -119,7 +120,7 @@ export class BBox {
pad(factor: number, maxIncrease = 2): BBox { pad(factor: number, maxIncrease = 2): BBox {
const latDiff = Math.min(maxIncrease / 2, Math.abs(this.maxLat - this.minLat) * factor) const latDiff = Math.min(maxIncrease / 2, Math.abs(this.maxLat - this.minLat) * factor)
const lonDiff =Math.min(maxIncrease / 2, Math.abs(this.maxLon - this.minLon) * factor) const lonDiff = Math.min(maxIncrease / 2, Math.abs(this.maxLon - this.minLon) * factor)
return new BBox([[ return new BBox([[
this.minLon - lonDiff, this.minLon - lonDiff,
this.minLat - latDiff this.minLat - latDiff
@ -161,4 +162,16 @@ export class BBox {
const boundslr = Tiles.tile_bounds_lon_lat(lr.z, lr.x, lr.y) const boundslr = Tiles.tile_bounds_lon_lat(lr.z, lr.x, lr.y)
return new BBox([].concat(boundsul, boundslr)) return new BBox([].concat(boundsul, boundslr))
} }
toMercator(): { minLat: number, maxLat: number, minLon: number, maxLon: number } {
const [minLon, minLat] = GeoOperations.ConvertWgs84To900913([this.minLon, this.minLat])
const [maxLon, maxLat] = GeoOperations.ConvertWgs84To900913([this.maxLon, this.maxLat])
return {
minLon, maxLon,
minLat, maxLat
}
}
} }

View file

@ -18,7 +18,6 @@ export default class DetermineLayout {
*/ */
public static async GetLayout(): Promise<[LayoutConfig, string]> { public static async GetLayout(): Promise<[LayoutConfig, string]> {
const loadCustomThemeParam = QueryParameters.GetQueryParameter("userlayout", "false", "If not 'false', a custom (non-official) theme is loaded. This custom layout can be done in multiple ways: \n\n- The hash of the URL contains a base64-encoded .json-file containing the theme definition\n- The hash of the URL contains a lz-compressed .json-file, as generated by the custom theme generator\n- The parameter itself is an URL, in which case that URL will be downloaded. It should point to a .json of a theme") const loadCustomThemeParam = QueryParameters.GetQueryParameter("userlayout", "false", "If not 'false', a custom (non-official) theme is loaded. This custom layout can be done in multiple ways: \n\n- The hash of the URL contains a base64-encoded .json-file containing the theme definition\n- The hash of the URL contains a lz-compressed .json-file, as generated by the custom theme generator\n- The parameter itself is an URL, in which case that URL will be downloaded. It should point to a .json of a theme")
const layoutFromBase64 = decodeURIComponent(loadCustomThemeParam.data); const layoutFromBase64 = decodeURIComponent(loadCustomThemeParam.data);
@ -73,17 +72,13 @@ export default class DetermineLayout {
try { try {
const data = await Utils.downloadJson(link) const parsed = await Utils.downloadJson(link)
console.log("Got ", parsed)
try { try {
let parsed = data;
if (typeof parsed == "string") {
parsed = JSON.parse(parsed);
}
// Overwrite the id to the url
parsed.id = link; parsed.id = link;
return new LayoutConfig(parsed, false).patchImages(link, data); return new LayoutConfig(parsed, false).patchImages(link, JSON.stringify(parsed));
} catch (e) { } catch (e) {
console.error(e)
DetermineLayout.ShowErrorOnCustomTheme( DetermineLayout.ShowErrorOnCustomTheme(
`<a href="${link}">${link}</a> is invalid:`, `<a href="${link}">${link}</a> is invalid:`,
new FixedUiElement(e) new FixedUiElement(e)
@ -92,6 +87,7 @@ export default class DetermineLayout {
} }
} catch (e) { } catch (e) {
console.erorr(e)
DetermineLayout.ShowErrorOnCustomTheme( DetermineLayout.ShowErrorOnCustomTheme(
`<a href="${link}">${link}</a> is invalid - probably not found or invalid JSON:`, `<a href="${link}">${link}</a> is invalid - probably not found or invalid JSON:`,
new FixedUiElement(e) new FixedUiElement(e)
@ -107,7 +103,7 @@ export default class DetermineLayout {
try { try {
// layoutFromBase64 contains the name of the theme. This is partly to do tracking with goat counter // layoutFromBase64 contains the name of the theme. This is partly to do tracking with goat counter
const dedicatedHashFromLocalStorage = LocalStorageSource.Get( const dedicatedHashFromLocalStorage = LocalStorageSource.Get(
"user-layout-" + userLayoutParam.data.replace(" ", "_") "user-layout-" + userLayoutParam.data?.replace(" ", "_")
); );
if (dedicatedHashFromLocalStorage.data?.length < 10) { if (dedicatedHashFromLocalStorage.data?.length < 10) {
dedicatedHashFromLocalStorage.setData(undefined); dedicatedHashFromLocalStorage.setData(undefined);
@ -134,6 +130,7 @@ export default class DetermineLayout {
try { try {
json = JSON.parse(Utils.UnMinify(LZString.decompressFromBase64(hash))) json = JSON.parse(Utils.UnMinify(LZString.decompressFromBase64(hash)))
} catch (e) { } catch (e) {
console.error(e)
DetermineLayout.ShowErrorOnCustomTheme("Could not decode the hash", new FixedUiElement("Not a valid (LZ-compressed) JSON")) DetermineLayout.ShowErrorOnCustomTheme("Could not decode the hash", new FixedUiElement("Not a valid (LZ-compressed) JSON"))
return null; return null;
} }
@ -143,6 +140,7 @@ export default class DetermineLayout {
userLayoutParam.setData(layoutToUse.id); userLayoutParam.setData(layoutToUse.id);
return [layoutToUse, btoa(Utils.MinifyJSON(JSON.stringify(json)))]; return [layoutToUse, btoa(Utils.MinifyJSON(JSON.stringify(json)))];
} catch (e) { } catch (e) {
console.error(e)
if (hash === undefined || hash.length < 10) { if (hash === undefined || hash.length < 10) {
DetermineLayout.ShowErrorOnCustomTheme("Could not load a theme from the hash", new FixedUiElement("Hash does not contain data")) DetermineLayout.ShowErrorOnCustomTheme("Could not load a theme from the hash", new FixedUiElement("Hash does not contain data"))
} }

View file

@ -205,7 +205,9 @@ export default class FeaturePipeline {
neededTiles: neededTilesFromOsm, neededTiles: neededTilesFromOsm,
handleTile: tile => { handleTile: tile => {
new RegisteringAllFromFeatureSourceActor(tile) new RegisteringAllFromFeatureSourceActor(tile)
if (tile.layer.layerDef.maxAgeOfCache > 0) {
new SaveTileToLocalStorageActor(tile, tile.tileIndex) new SaveTileToLocalStorageActor(tile, tile.tileIndex)
}
perLayerHierarchy.get(tile.layer.layerDef.id).registerTile(tile) perLayerHierarchy.get(tile.layer.layerDef.id).registerTile(tile)
tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile)) tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
@ -213,7 +215,9 @@ export default class FeaturePipeline {
state: state, state: state,
markTileVisited: (tileId) => markTileVisited: (tileId) =>
state.filteredLayers.data.forEach(flayer => { state.filteredLayers.data.forEach(flayer => {
if (flayer.layerDef.maxAgeOfCache > 0) {
SaveTileToLocalStorageActor.MarkVisited(flayer.layerDef.id, tileId, new Date()) SaveTileToLocalStorageActor.MarkVisited(flayer.layerDef.id, tileId, new Date())
}
self.freshnesses.get(flayer.layerDef.id).addTileLoad(tileId, new Date()) self.freshnesses.get(flayer.layerDef.id).addTileLoad(tileId, new Date())
}) })
}) })

View file

@ -7,6 +7,7 @@ import {Utils} from "../../../Utils";
import {FeatureSourceForLayer, Tiled} from "../FeatureSource"; import {FeatureSourceForLayer, Tiled} from "../FeatureSource";
import {Tiles} from "../../../Models/TileRange"; import {Tiles} from "../../../Models/TileRange";
import {BBox} from "../../BBox"; import {BBox} from "../../BBox";
import {GeoOperations} from "../../GeoOperations";
export default class GeoJsonSource implements FeatureSourceForLayer, Tiled { export default class GeoJsonSource implements FeatureSourceForLayer, Tiled {
@ -14,7 +15,6 @@ export default class GeoJsonSource implements FeatureSourceForLayer, Tiled {
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>; public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>;
public readonly name; public readonly name;
public readonly isOsmCache: boolean public readonly isOsmCache: boolean
private onFail: ((errorMsg: any, url: string) => void) = undefined;
private readonly seenids: Set<string> = new Set<string>() private readonly seenids: Set<string> = new Set<string>()
public readonly layer: FilteredLayer; public readonly layer: FilteredLayer;
@ -44,10 +44,20 @@ export default class GeoJsonSource implements FeatureSourceForLayer, Tiled {
let url = flayer.layerDef.source.geojsonSource.replace("{layer}", flayer.layerDef.id); let url = flayer.layerDef.source.geojsonSource.replace("{layer}", flayer.layerDef.id);
if (zxy !== undefined) { if (zxy !== undefined) {
const [z, x, y] = zxy; const [z, x, y] = zxy;
let tile_bbox = BBox.fromTile(z, x, y)
let bounds : { minLat: number, maxLat: number, minLon: number, maxLon: number } = tile_bbox
if(this.layer.layerDef.source.mercatorCrs){
bounds = tile_bbox.toMercator()
}
url = url url = url
.replace('{z}', "" + z) .replace('{z}', "" + z)
.replace('{x}', "" + x) .replace('{x}', "" + x)
.replace('{y}', "" + y) .replace('{y}', "" + y)
.replace('{y_min}',""+bounds.minLat)
.replace('{y_max}',""+bounds.maxLat)
.replace('{x_min}',""+bounds.minLon)
.replace('{x_max}',""+bounds.maxLon)
this.tileIndex = Tiles.tile_index(z, x, y) this.tileIndex = Tiles.tile_index(z, x, y)
this.bbox = BBox.fromTile(z, x, y) this.bbox = BBox.fromTile(z, x, y)
} else { } else {
@ -72,6 +82,10 @@ export default class GeoJsonSource implements FeatureSourceForLayer, Tiled {
return; return;
} }
if(self.layer.layerDef.source.mercatorCrs){
json = GeoOperations.GeoJsonToWGS84(json)
}
const time = new Date(); const time = new Date();
const newFeatures: { feature: any, freshness: Date } [] = [] const newFeatures: { feature: any, freshness: Date } [] = []
let i = 0; let i = 0;

View file

@ -21,23 +21,27 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource {
throw "Invalid layer: geojsonSource expected" throw "Invalid layer: geojsonSource expected"
} }
const whitelistUrl = source.geojsonSource
.replace("{z}", ""+source.geojsonZoomLevel)
.replace("{x}_{y}.geojson", "overview.json")
.replace("{layer}",layer.layerDef.id)
let whitelist = undefined let whitelist = undefined
if (source.geojsonSource.indexOf("{x}_{y}.geojson") > 0) {
const whitelistUrl = source.geojsonSource
.replace("{z}", "" + source.geojsonZoomLevel)
.replace("{x}_{y}.geojson", "overview.json")
.replace("{layer}", layer.layerDef.id)
Utils.downloadJson(whitelistUrl).then( Utils.downloadJson(whitelistUrl).then(
json => { json => {
const data = new Map<number, Set<number>>(); const data = new Map<number, Set<number>>();
for (const x in json) { for (const x in json) {
data.set(Number(x), new Set(json[x])) data.set(Number(x), new Set(json[x]))
} }
console.log("The whitelist is", data, "based on ", json, "from", whitelistUrl)
whitelist = data whitelist = data
} }
).catch(err => { ).catch(err => {
console.warn("No whitelist found for ", layer.layerDef.id, err) console.warn("No whitelist found for ", layer.layerDef.id, err)
}) })
}
const seenIds = new Set<string>(); const seenIds = new Set<string>();
const blackList = new UIEventSource(seenIds) const blackList = new UIEventSource(seenIds)
@ -45,9 +49,9 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource {
layer, layer,
source.geojsonZoomLevel, source.geojsonZoomLevel,
(zxy) => { (zxy) => {
if(whitelist !== undefined){ if (whitelist !== undefined) {
const isWhiteListed = whitelist.get(zxy[1])?.has(zxy[2]) const isWhiteListed = whitelist.get(zxy[1])?.has(zxy[2])
if(!isWhiteListed){ if (!isWhiteListed) {
console.log("Not downloading tile", ...zxy, "as it is not on the whitelist") console.log("Not downloading tile", ...zxy, "as it is not on the whitelist")
return undefined; return undefined;
} }

View file

@ -283,6 +283,34 @@ export class GeoOperations {
return headerValuesOrdered.map(v => JSON.stringify(v)).join(",") + "\n" + lines.join("\n") return headerValuesOrdered.map(v => JSON.stringify(v)).join(",") + "\n" + lines.join("\n")
} }
private static readonly _earthRadius = 6378137;
private static readonly _originShift = 2 * Math.PI * GeoOperations._earthRadius / 2;
//Converts given lat/lon in WGS84 Datum to XY in Spherical Mercator EPSG:900913
public static ConvertWgs84To900913(lonLat: [number, number]): [number, number] {
const lon = lonLat[0];
const lat = lonLat[1];
const x = lon * GeoOperations._originShift / 180;
let y = Math.log(Math.tan((90 + lat) * Math.PI / 360)) / (Math.PI / 180);
y = y * GeoOperations._originShift / 180;
return [x, y];
}
//Converts XY point from (Spherical) Web Mercator EPSG:3785 (unofficially EPSG:900913) to lat/lon in WGS84 Datum
public static Convert900913ToWgs84(lonLat: [number, number]): [number, number] {
const lon = lonLat[0]
const lat = lonLat[1]
const x = 180 * lon / GeoOperations._originShift;
let y = 180 * lat / GeoOperations._originShift;
y = 180 / Math.PI * (2 * Math.atan(Math.exp(y * Math.PI / 180)) - Math.PI / 2);
return [x, y];
}
public static GeoJsonToWGS84(geojson){
return turf.toWgs84(geojson)
}
/** /**
* Calculates the intersection between two features. * Calculates the intersection between two features.
* Returns the length if intersecting a linestring and a (multi)polygon (in meters), returns a surface area (in m²) if intersecting two (multi)polygons * Returns the length if intersecting a linestring and a (multi)polygon (in meters), returns a surface area (in m²) if intersecting two (multi)polygons

View file

@ -53,6 +53,8 @@ export interface LayerConfigJson {
* source: {geoJson: "https://my.source.net/some-tile-geojson-{layer}-{z}-{x}-{y}.geojson", geoJsonZoomLevel: 14} * source: {geoJson: "https://my.source.net/some-tile-geojson-{layer}-{z}-{x}-{y}.geojson", geoJsonZoomLevel: 14}
* to use a tiled geojson source. The web server must offer multiple geojsons. {z}, {x} and {y} are substituted by the location; {layer} is substituted with the id of the loaded layer * to use a tiled geojson source. The web server must offer multiple geojsons. {z}, {x} and {y} are substituted by the location; {layer} is substituted with the id of the loaded layer
* *
* Some API's use a BBOX instead of a tile, this can be used by specifying {y_min}, {y_max}, {x_min} and {x_max}
* Some API's use a mercator-projection (EPSG:900913) instead of WGS84. Set the flag `mercatorCrs: true` in the source for this
* *
* Note that both geojson-options might set a flag 'isOsmCache' indicating that the data originally comes from OSM too * Note that both geojson-options might set a flag 'isOsmCache' indicating that the data originally comes from OSM too
* *
@ -61,7 +63,7 @@ export interface LayerConfigJson {
* While still supported, this is considered deprecated * While still supported, this is considered deprecated
*/ */
source: ({ osmTags: AndOrTagConfigJson | string, overpassScript?: string } | source: ({ osmTags: AndOrTagConfigJson | string, overpassScript?: string } |
{ osmTags: AndOrTagConfigJson | string, geoJson: string, geoJsonZoomLevel?: number, isOsmCache?: boolean }) & ({ { osmTags: AndOrTagConfigJson | string, geoJson: string, geoJsonZoomLevel?: number, isOsmCache?: boolean, mercatorCrs?: boolean }) & ({
/** /**
* The maximum amount of seconds that a tile is allowed to linger in the cache * The maximum amount of seconds that a tile is allowed to linger in the cache
*/ */

View file

@ -124,6 +124,7 @@ export default class LayerConfig {
geojsonSourceLevel: json.source["geoJsonZoomLevel"], geojsonSourceLevel: json.source["geoJsonZoomLevel"],
overpassScript: json.source["overpassScript"], overpassScript: json.source["overpassScript"],
isOsmCache: json.source["isOsmCache"], isOsmCache: json.source["isOsmCache"],
mercatorCrs: json.source["mercatorCrs"]
}, },
this.id this.id
); );

View file

@ -304,7 +304,6 @@ export default class LayoutConfig {
} }
rewriting.forEach((value, key) => { rewriting.forEach((value, key) => {
console.log("Rewriting", key, "==>", value) console.log("Rewriting", key, "==>", value)
originalJson = originalJson.replace(new RegExp(key, "g"), value) originalJson = originalJson.replace(new RegExp(key, "g"), value)
}) })
return new LayoutConfig(JSON.parse(originalJson), false, "Layout rewriting") return new LayoutConfig(JSON.parse(originalJson), false, "Layout rewriting")

View file

@ -1,4 +1,5 @@
import {TagsFilter} from "../../Logic/Tags/TagsFilter"; import {TagsFilter} from "../../Logic/Tags/TagsFilter";
import {RegexTag} from "../../Logic/Tags/RegexTag";
export default class SourceConfig { export default class SourceConfig {
@ -7,8 +8,10 @@ export default class SourceConfig {
public readonly geojsonSource?: string; public readonly geojsonSource?: string;
public readonly geojsonZoomLevel?: number; public readonly geojsonZoomLevel?: number;
public readonly isOsmCacheLayer: boolean; public readonly isOsmCacheLayer: boolean;
public readonly mercatorCrs: boolean;
constructor(params: { constructor(params: {
mercatorCrs?: boolean;
osmTags?: TagsFilter, osmTags?: TagsFilter,
overpassScript?: string, overpassScript?: string,
geojsonSource?: string, geojsonSource?: string,
@ -33,10 +36,15 @@ export default class SourceConfig {
console.error(params) console.error(params)
throw `Source said it is a OSM-cached layer, but didn't define the actual source of the cache (in context ${context})` throw `Source said it is a OSM-cached layer, but didn't define the actual source of the cache (in context ${context})`
} }
this.osmTags = params.osmTags; if(params.geojsonSource !== undefined && params.geojsonSourceLevel !== undefined){
if(! ["x","y","x_min","x_max","y_min","Y_max"].some(toSearch => params.geojsonSource.indexOf(toSearch) > 0)){
throw `Source defines a geojson-zoomLevel, but does not specify {x} nor {y} (or equivalent), this is probably a bug (in context ${context})`
}}
this.osmTags = params.osmTags ?? new RegexTag("id",/.*/);
this.overpassScript = params.overpassScript; this.overpassScript = params.overpassScript;
this.geojsonSource = params.geojsonSource; this.geojsonSource = params.geojsonSource;
this.geojsonZoomLevel = params.geojsonSourceLevel; this.geojsonZoomLevel = params.geojsonSourceLevel;
this.isOsmCacheLayer = params.isOsmCache ?? false; this.isOsmCacheLayer = params.isOsmCache ?? false;
this.mercatorCrs = params.mercatorCrs ?? false;
} }
} }

View file

@ -83,7 +83,7 @@ export default class FeatureInfoBox extends ScrollableFullScreen {
layerConfig.allowMove layerConfig.allowMove
); );
}) })
) ).SetClass("text-base")
); );
} }
@ -94,14 +94,14 @@ export default class FeatureInfoBox extends ScrollableFullScreen {
id, id,
layerConfig.deletion layerConfig.deletion
)) ))
)) ).SetClass("text-base"))
} }
if (layerConfig.allowSplit) { if (layerConfig.allowSplit) {
editElements.push( editElements.push(
new VariableUiElement(tags.map(tags => tags.id).map(id => new VariableUiElement(tags.map(tags => tags.id).map(id =>
new SplitRoadWizard(id)) new SplitRoadWizard(id))
)) ).SetClass("text-base"))
} }

View file

@ -457,7 +457,7 @@ There are also some technicalities in your theme to keep in mind:
return new Combine([new FixedUiElement("The import button is disabled for unofficial themes to prevent accidents.").SetClass("alert"), return new Combine([new FixedUiElement("The import button is disabled for unofficial themes to prevent accidents.").SetClass("alert"),
new FixedUiElement("To test, add 'test=true' to the URL. The changeset will be printed in the console. Please open a PR to officialize this theme to actually enable the import button.")]) new FixedUiElement("To test, add 'test=true' to the URL. The changeset will be printed in the console. Please open a PR to officialize this theme to actually enable the import button.")])
} }
const tgsSpec = args[0].split(",").map(spec => { const tgsSpec = args[0].split(";").map(spec => {
const kv = spec.split("=").map(s => s.trim()); const kv = spec.split("=").map(s => s.trim());
if (kv.length != 2) { if (kv.length != 2) {
throw "Invalid key spec: multiple '=' found in " + spec throw "Invalid key spec: multiple '=' found in " + spec

View file

@ -433,6 +433,7 @@
"id": "public_bookcase-website" "id": "public_bookcase-website"
} }
], ],
"allowMove": true,
"deletion": { "deletion": {
"softDeletionTags": { "softDeletionTags": {
"and": [ "and": [

View file

@ -0,0 +1,20 @@
GRB Import helper
===================
Preparing the CRAB dataset
--------------------------
````
# The original data is downloaded from https://download.vlaanderen.be/Producten/Detail?id=447&title=CRAB_Adressenlijst# (the GML-file here )
wget https://downloadagiv.blob.core.windows.net/crab-adressenlijst/GML/CRAB_Adressenlijst_GML.zip
# Extract the zip file
unzip CRAB_Adressenlijst_GML.zip
# convert the pesky GML file into geojson
ogr2ogr -progress -t_srs WGS84 -f \"GeoJson\" CRAB.geojson CrabAdr.gml
# When done, this big file is sliced into tiles with the slicer script
node --max_old_space_size=8000 $(which ts-node) ~/git/MapComplete/scripts/slice.ts CRAB.geojson 18 ~/git/pietervdvn.github.io/CRAB_2021_10_26
````

View file

@ -22,7 +22,7 @@
"socialImage": "", "socialImage": "",
"layers": [ "layers": [
{ {
"id": "grb-fixmes", "id": "osm-fixmes",
"name": { "name": {
"nl": "Fixmes op gebouwen" "nl": "Fixmes op gebouwen"
}, },
@ -198,6 +198,43 @@
}, },
"wayHandling": 2, "wayHandling": 2,
"presets": [] "presets": []
},
{
"id": "crab-addresses 2021-10-26",
"source": {
"osmTags": "HUISNR~*",
"geoJson": "https://raw.githubusercontent.com/pietervdvn/pietervdvn.github.io/master/CRAB_2021_10_26/tile_{z}_{x}_{y}.geojson",
"#geoJson": "https://pietervdvn.github.io/CRAB_2021_10_26/tile_{z}_{x}_{y}.geojson",
"geoJsonZoomLevel": 18,
"maxCacheAge": 0
},
"minzoom": 19,
"name": "CRAB-addressen",
"title": "CRAB-adres",
"icon": "circle:#bb3322",
"iconSize": "15,15,center",
"tagRenderings": [
"all_tags",
{
"id": "import-button",
"render": "{import_button(addr:street=$STRAATNM; addr:housenumber=$HUISNR)}"
}
]
},
{
"id": "GRB",
"source": {
"geoJson": "https://betadata.grbosm.site/grb?bbox={x_min},{y_min},{x_max},{y_max}",
"geoJsonZoomLevel": 18,
"mercatorCrs": true,
"maxCacheAge": 0
},
"name": "GRB geometries",
"title": "GRB outline",
"minzoom": 19,
"tagRenderings": [
"all_tags"
]
} }
], ],
"hideFromOverview": true, "hideFromOverview": true,

View file

@ -33,8 +33,6 @@ if (location.href.startsWith("http://buurtnatuur.be")) {
class Init { class Init {
public static Init(layoutToUse: LayoutConfig, encoded: string) { public static Init(layoutToUse: LayoutConfig, encoded: string) {
if(layoutToUse === null){ if(layoutToUse === null){

View file

@ -4,11 +4,84 @@ import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSou
import * as readline from "readline"; import * as readline from "readline";
import ScriptUtils from "./ScriptUtils"; import ScriptUtils from "./ScriptUtils";
async function readFeaturesFromLineDelimitedJsonFile(inputFile: string): Promise<any[]> {
const fileStream = fs.createReadStream(inputFile);
const rl = readline.createInterface({
input: fileStream,
crlfDelay: Infinity
});
// Note: we use the crlfDelay option to recognize all instances of CR LF
// ('\r\n') in input.txt as a single line break.
const allFeatures: any[] = []
// @ts-ignore
for await (const line of rl) {
try {
allFeatures.push(JSON.parse(line))
} catch (e) {
console.error("Could not parse", line)
break
}
if (allFeatures.length % 10000 === 0) {
ScriptUtils.erasableLog("Loaded ", allFeatures.length, "features up till now")
}
}
return allFeatures
}
async function readGeojsonLineByLine(inputFile: string): Promise<any[]> {
const fileStream = fs.createReadStream(inputFile);
const rl = readline.createInterface({
input: fileStream,
crlfDelay: Infinity
});
// Note: we use the crlfDelay option to recognize all instances of CR LF
// ('\r\n') in input.txt as a single line break.
const allFeatures: any[] = []
let featuresSeen = false
// @ts-ignore
for await (let line: string of rl) {
if (!featuresSeen && line.startsWith("\"features\":")) {
featuresSeen = true;
continue;
}
if (!featuresSeen) {
continue
}
if (line.endsWith(",")) {
line = line.substring(0, line.length - 1)
}
try {
allFeatures.push(JSON.parse(line))
} catch (e) {
console.error("Could not parse", line)
break
}
if (allFeatures.length % 10000 === 0) {
ScriptUtils.erasableLog("Loaded ", allFeatures.length, "features up till now")
}
}
return allFeatures
}
async function readFeaturesFromGeoJson(inputFile: string): Promise<any[]> {
try {
return JSON.parse(fs.readFileSync(inputFile, "UTF-8")).features
} catch (e) {
// We retry, but with a line-by-line approach
return await readGeojsonLineByLine(inputFile)
}
}
async function main(args: string[]) { async function main(args: string[]) {
console.log("GeoJSON slicer") console.log("GeoJSON slicer")
if (args.length < 3) { if (args.length < 3) {
console.log("USAGE: <input-file.line-delimited-geojson> <target-zoom-level> <output-directory>") console.log("USAGE: <input-file.geojson> <target-zoom-level> <output-directory>")
return return
} }
@ -23,37 +96,22 @@ async function main(args: string[]) {
console.log("Using directory ", outputDirectory) console.log("Using directory ", outputDirectory)
const fileStream = fs.createReadStream(inputFile); let allFeatures: any [];
if (inputFile.endsWith(".geojson")) {
allFeatures = await readFeaturesFromGeoJson(inputFile)
} else {
allFeatures = await readFeaturesFromLineDelimitedJsonFile(inputFile)
}
const rl = readline.createInterface({
input: fileStream,
crlfDelay: Infinity
});
// Note: we use the crlfDelay option to recognize all instances of CR LF
// ('\r\n') in input.txt as a single line break.
const allFeatures = []
// @ts-ignore
for await (const line of rl) {
// Each line in input.txt will be successively available here as `line`.
try{
allFeatures.push(JSON.parse(line))
}catch (e) {
console.error("Could not parse", line)
break
}
if(allFeatures.length % 10000 === 0){
ScriptUtils.erasableLog("Loaded ", allFeatures.length, "features up till now")
}
}
console.log("Loaded all", allFeatures.length, "points") console.log("Loaded all", allFeatures.length, "points")
const keysToRemove = ["ID","STRAATNMID","NISCODE","GEMEENTE","POSTCODE","HERKOMST","APPTNR"] const keysToRemove = ["ID", "STRAATNMID", "NISCODE", "GEMEENTE", "POSTCODE", "HERKOMST"]
for (const f of allFeatures) { for (const f of allFeatures) {
for (const keyToRm of keysToRemove) { for (const keyToRm of keysToRemove) {
delete f.properties[keyToRm] delete f.properties[keyToRm]
} }
delete f.bbox
} }
//const knownKeys = Utils.Dedup([].concat(...allFeatures.map(f => Object.keys(f.properties)))) //const knownKeys = Utils.Dedup([].concat(...allFeatures.map(f => Object.keys(f.properties))))
@ -67,11 +125,15 @@ async function main(args: string[]) {
maxFeatureCount: Number.MAX_VALUE, maxFeatureCount: Number.MAX_VALUE,
registerTile: tile => { registerTile: tile => {
const path = `${outputDirectory}/tile_${tile.z}_${tile.x}_${tile.y}.geojson` const path = `${outputDirectory}/tile_${tile.z}_${tile.x}_${tile.y}.geojson`
const features = tile.features.data.map(ff => ff.feature)
features.forEach(f => {
delete f.bbox
})
fs.writeFileSync(path, JSON.stringify({ fs.writeFileSync(path, JSON.stringify({
"type": "FeatureCollection", "type": "FeatureCollection",
"features": tile.features.data.map(ff => ff.feature) "features": features
}, null, " ")) }, null, " "))
console.log("Written ", path, "which has ", tile.features.data.length, "features") ScriptUtils.erasableLog("Written ", path, "which has ", tile.features.data.length, "features")
} }
} }
) )