Fix multilayer geojson source
This commit is contained in:
pietervdvn
parent
a6e3b41b1d
commit
f4dacab9ef
7 changed files with 97 additions and 52 deletions
|
|
@ -30,7 +30,7 @@ export interface LayerConfigJson {
|
|||
*
|
||||
* source: {osmTags: "key=value"} will fetch all objects with given tags from OSM. Currently, this will create a query to overpass and fetch the data - in the future this might fetch from the OSM API
|
||||
* source: {geoJson: "https://my.source.net/some-geo-data.geojson"} to fetch a geojson from a third party source
|
||||
* source: {geoJson: "https://my.source.net/some-tile-geojson-{z}-{x}-{y}.geojson", geoJsonZoomLevel: 14} to use a tiled geojson source. The web server must offer multiple geojsons. {z}, {x} and {y} are substituted
|
||||
* source: {geoJson: "https://my.source.net/some-tile-geojson-{layer}-{z}-{x}-{y}.geojson", geoJsonZoomLevel: 14} to use a tiled geojson source. The web server must offer multiple geojsons. {z}, {x} and {y} are substituted by the location; {layer} is substituted with the id of the loaded layer
|
||||
*
|
||||
* source: {overpassScript: "<custom overpass tags>"} when you want to do special things. _This should be really rare_.
|
||||
* This means that the data will be pulled from overpass with this script, and will ignore the osmTags for the query
|
||||
|
|
|
|||
|
|
@ -16,15 +16,15 @@ export default class GeoJsonSource implements FeatureSource {
|
|||
|
||||
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>;
|
||||
public readonly name;
|
||||
private readonly onFail: ((errorMsg: any, url: string) => void) = undefined;
|
||||
private onFail: ((errorMsg: any, url: string) => void) = undefined;
|
||||
private readonly layerId: string;
|
||||
private readonly seenids: Set<string> = new Set<string>()
|
||||
|
||||
constructor(locationControl: UIEventSource<Loc>,
|
||||
private constructor(locationControl: UIEventSource<Loc>,
|
||||
flayer: { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig },
|
||||
onFail?: ((errorMsg: any) => void)) {
|
||||
this.layerId = flayer.layerDef.id;
|
||||
let url = flayer.layerDef.source.geojsonSource;
|
||||
let url = flayer.layerDef.source.geojsonSource.replace("{layer}", flayer.layerDef.id);
|
||||
this.name = "GeoJsonSource of " + url;
|
||||
const zoomLevel = flayer.layerDef.source.geojsonZoomLevel;
|
||||
|
||||
|
|
@ -34,29 +34,28 @@ export default class GeoJsonSource implements FeatureSource {
|
|||
// This is a classic, static geojson layer
|
||||
if (onFail === undefined) {
|
||||
onFail = errorMsg => {
|
||||
console.warn(`Could not load geojson layer from`, url, "due to", errorMsg)
|
||||
}
|
||||
}
|
||||
this.onFail = onFail;
|
||||
|
||||
this.LoadJSONFrom(url)
|
||||
} else {
|
||||
this.ConfigureDynamicLayer(url, zoomLevel, locationControl, flayer)
|
||||
}
|
||||
}
|
||||
|
||||
private ConfigureDynamicLayer(url: string, zoomLevel: number, locationControl: UIEventSource<Loc>, flayer: { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig }){
|
||||
// This is a dynamic template with a fixed zoom level
|
||||
url = url.replace("{z}", "" + zoomLevel)
|
||||
const loadedTiles = new Set<string>();
|
||||
const self = this;
|
||||
this.onFail = (msg, url) => {
|
||||
console.warn(`Could not load geojson layer from`, url, "due to", msg)
|
||||
loadedTiles.delete(url)
|
||||
loadedTiles.add(url); // We add the url to the 'loadedTiles' in order to not reload it in the future
|
||||
}
|
||||
|
||||
const neededTiles = locationControl.map(
|
||||
location => {
|
||||
|
||||
if (!flayer.isDisplayed.data) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Yup, this is cheating to just get the bounds here
|
||||
const bounds = State.state.leafletMap.data.getBounds()
|
||||
const tileRange = Utils.TileRangeBetween(zoomLevel, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
|
||||
|
|
@ -69,11 +68,21 @@ export default class GeoJsonSource implements FeatureSource {
|
|||
}
|
||||
return needed;
|
||||
}
|
||||
);
|
||||
, [flayer.isDisplayed]);
|
||||
neededTiles.stabilized(250).addCallback((needed: Set<string>) => {
|
||||
if (needed === undefined) {
|
||||
return;
|
||||
}
|
||||
if (!flayer.isDisplayed.data) {
|
||||
// No need to download! - the layer is disabled
|
||||
return;
|
||||
}
|
||||
console.log("???", locationControl, flayer.layerDef)
|
||||
if(locationControl.data.zoom < flayer.layerDef.minzoom){
|
||||
console.log("Not downloading ", needed, "not sufficiently zoomed")
|
||||
return;
|
||||
}
|
||||
|
||||
needed.forEach(neededTile => {
|
||||
if (loadedTiles.has(neededTile)) {
|
||||
return;
|
||||
|
|
@ -86,7 +95,6 @@ export default class GeoJsonSource implements FeatureSource {
|
|||
})
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Merges together the layers which have the same source
|
||||
|
|
@ -98,7 +106,7 @@ export default class GeoJsonSource implements FeatureSource {
|
|||
|
||||
const flayersPerSource = new Map<string, { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig }[]>();
|
||||
for (const flayer of flayers) {
|
||||
const url = flayer.layerDef.source.geojsonSource
|
||||
const url = flayer.layerDef.source.geojsonSource?.replace(/{layer}/g, flayer.layerDef.id)
|
||||
if (url === undefined) {
|
||||
continue;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -98,7 +98,8 @@ export class UIEventSource<T> {
|
|||
const self = this;
|
||||
|
||||
const newSource = new UIEventSource<J>(
|
||||
f(this.data)
|
||||
f(this.data),
|
||||
"map("+this.tag+")"
|
||||
);
|
||||
|
||||
const update = function () {
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ export class QueryParameters {
|
|||
return QueryParameters.knownSources[key];
|
||||
}
|
||||
QueryParameters.addOrder(key);
|
||||
const source = new UIEventSource<string>(deflt);
|
||||
const source = new UIEventSource<string>(deflt, "&"+key);
|
||||
QueryParameters.knownSources[key] = source;
|
||||
source.addCallback(() => QueryParameters.Serialize())
|
||||
return source;
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { Utils } from "../Utils";
|
|||
|
||||
export default class Constants {
|
||||
|
||||
public static vNumber = "0.6.11b";
|
||||
public static vNumber = "0.6.11c";
|
||||
|
||||
// The user journey states thresholds when a new feature gets unlocked
|
||||
public static userJourney = {
|
||||
|
|
|
|||
|
|
@ -55,7 +55,12 @@
|
|||
"minzoom":14
|
||||
}
|
||||
},
|
||||
"sport_pitch",
|
||||
{
|
||||
"builtin": "sport_pitch",
|
||||
"override": {
|
||||
"minzoom": 14
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
"builtin": "slow_roads",
|
||||
|
|
@ -174,9 +179,14 @@
|
|||
}
|
||||
}
|
||||
],
|
||||
"clustering": {
|
||||
"maxZoom": 16,
|
||||
"minNeededElements": 100
|
||||
},
|
||||
"overrideAll": {
|
||||
"source": {
|
||||
"geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson",
|
||||
"geoJsonLocal": "http://127.0.0.1:8080/speelplekken_{layer}_{z}_{x}_{y}.geojson",
|
||||
"geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{layer}_{z}_{x}_{y}.geojson",
|
||||
"geoJsonZoomLevel": 14
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
* Generates a collection of geojson files based on an overpass query for a given theme
|
||||
*/
|
||||
import {TileRange, Utils} from "../Utils";
|
||||
|
||||
Utils.runningFromConsole = true
|
||||
import {Overpass} from "../Logic/Osm/Overpass";
|
||||
import {existsSync, readFileSync, writeFileSync} from "fs";
|
||||
|
|
@ -15,7 +16,6 @@ import * as OsmToGeoJson from "osmtogeojson";
|
|||
import MetaTagging from "../Logic/MetaTagging";
|
||||
|
||||
|
||||
|
||||
function createOverpassObject(theme: LayoutConfig) {
|
||||
let filters: TagsFilter[] = [];
|
||||
let extraScripts: string[] = [];
|
||||
|
|
@ -106,7 +106,7 @@ async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass)/
|
|||
}
|
||||
}
|
||||
|
||||
if(!success){
|
||||
if (!success) {
|
||||
failed++;
|
||||
console.log("Hit the rate limit - waiting 90s")
|
||||
for (let i = 0; i < 90; i++) {
|
||||
|
|
@ -159,12 +159,37 @@ async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig)
|
|||
// Extract the relationship information
|
||||
const relations = ExtractRelations.BuildMembershipTable(ExtractRelations.GetRelationElements(rawOsm))
|
||||
MetaTagging.addMetatags(featuresFreshness, relations, theme.layers);
|
||||
writeFileSync(geoJsonName(targetdir, x, y, r.zoomlevel), JSON.stringify(geojson))
|
||||
|
||||
|
||||
writeFileSync(geoJsonName(targetdir, x, y, r.zoomlevel), JSON.stringify(geojson, null, " "))
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function splitPerLayer(targetdir: string, r: TileRange, theme: LayoutConfig) {
|
||||
let processed = 0;
|
||||
const z = r.zoomlevel;
|
||||
for (let x = r.xstart; x <= r.xend; x++) {
|
||||
for (let y = r.ystart; y <= r.yend; y++) {
|
||||
const file = readFileSync(geoJsonName(targetdir, x, y, z), "UTF8")
|
||||
|
||||
for (const layer of theme.layers) {
|
||||
const geojson = JSON.parse(file)
|
||||
geojson.features = geojson.features.filter(f => f._matching_layer_id === layer.id)
|
||||
if(geojson.features.length == 0){
|
||||
continue;
|
||||
}
|
||||
const new_path = geoJsonName(targetdir+"_"+layer.id, x, y, z);
|
||||
writeFileSync(new_path, JSON.stringify(geojson, null, " "))
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
async function main(args: string[]) {
|
||||
|
||||
if (args.length == 0) {
|
||||
|
|
@ -203,6 +228,7 @@ async function main(args: string[]) {
|
|||
} while (failed > 0)
|
||||
|
||||
await postProcess(targetdir, tileRange, theme)
|
||||
await splitPerLayer(targetdir, tileRange, theme)
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue