Fix multilayer geojson source
This commit is contained in:
parent
a6e3b41b1d
commit
f4dacab9ef
7 changed files with 97 additions and 52 deletions
|
@ -30,7 +30,7 @@ export interface LayerConfigJson {
|
||||||
*
|
*
|
||||||
* source: {osmTags: "key=value"} will fetch all objects with given tags from OSM. Currently, this will create a query to overpass and fetch the data - in the future this might fetch from the OSM API
|
* source: {osmTags: "key=value"} will fetch all objects with given tags from OSM. Currently, this will create a query to overpass and fetch the data - in the future this might fetch from the OSM API
|
||||||
* source: {geoJson: "https://my.source.net/some-geo-data.geojson"} to fetch a geojson from a third party source
|
* source: {geoJson: "https://my.source.net/some-geo-data.geojson"} to fetch a geojson from a third party source
|
||||||
* source: {geoJson: "https://my.source.net/some-tile-geojson-{z}-{x}-{y}.geojson", geoJsonZoomLevel: 14} to use a tiled geojson source. The web server must offer multiple geojsons. {z}, {x} and {y} are substituted
|
* source: {geoJson: "https://my.source.net/some-tile-geojson-{layer}-{z}-{x}-{y}.geojson", geoJsonZoomLevel: 14} to use a tiled geojson source. The web server must offer multiple geojsons. {z}, {x} and {y} are substituted by the location; {layer} is substituted with the id of the loaded layer
|
||||||
*
|
*
|
||||||
* source: {overpassScript: "<custom overpass tags>"} when you want to do special things. _This should be really rare_.
|
* source: {overpassScript: "<custom overpass tags>"} when you want to do special things. _This should be really rare_.
|
||||||
* This means that the data will be pulled from overpass with this script, and will ignore the osmTags for the query
|
* This means that the data will be pulled from overpass with this script, and will ignore the osmTags for the query
|
||||||
|
|
|
@ -16,15 +16,15 @@ export default class GeoJsonSource implements FeatureSource {
|
||||||
|
|
||||||
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>;
|
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>;
|
||||||
public readonly name;
|
public readonly name;
|
||||||
private readonly onFail: ((errorMsg: any, url: string) => void) = undefined;
|
private onFail: ((errorMsg: any, url: string) => void) = undefined;
|
||||||
private readonly layerId: string;
|
private readonly layerId: string;
|
||||||
private readonly seenids: Set<string> = new Set<string>()
|
private readonly seenids: Set<string> = new Set<string>()
|
||||||
|
|
||||||
constructor(locationControl: UIEventSource<Loc>,
|
private constructor(locationControl: UIEventSource<Loc>,
|
||||||
flayer: { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig },
|
flayer: { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig },
|
||||||
onFail?: ((errorMsg: any) => void)) {
|
onFail?: ((errorMsg: any) => void)) {
|
||||||
this.layerId = flayer.layerDef.id;
|
this.layerId = flayer.layerDef.id;
|
||||||
let url = flayer.layerDef.source.geojsonSource;
|
let url = flayer.layerDef.source.geojsonSource.replace("{layer}", flayer.layerDef.id);
|
||||||
this.name = "GeoJsonSource of " + url;
|
this.name = "GeoJsonSource of " + url;
|
||||||
const zoomLevel = flayer.layerDef.source.geojsonZoomLevel;
|
const zoomLevel = flayer.layerDef.source.geojsonZoomLevel;
|
||||||
|
|
||||||
|
@ -34,29 +34,28 @@ export default class GeoJsonSource implements FeatureSource {
|
||||||
// This is a classic, static geojson layer
|
// This is a classic, static geojson layer
|
||||||
if (onFail === undefined) {
|
if (onFail === undefined) {
|
||||||
onFail = errorMsg => {
|
onFail = errorMsg => {
|
||||||
console.warn(`Could not load geojson layer from`, url, "due to", errorMsg)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
this.onFail = onFail;
|
this.onFail = onFail;
|
||||||
|
|
||||||
this.LoadJSONFrom(url)
|
this.LoadJSONFrom(url)
|
||||||
} else {
|
} else {
|
||||||
|
this.ConfigureDynamicLayer(url, zoomLevel, locationControl, flayer)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private ConfigureDynamicLayer(url: string, zoomLevel: number, locationControl: UIEventSource<Loc>, flayer: { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig }){
|
||||||
// This is a dynamic template with a fixed zoom level
|
// This is a dynamic template with a fixed zoom level
|
||||||
url = url.replace("{z}", "" + zoomLevel)
|
url = url.replace("{z}", "" + zoomLevel)
|
||||||
const loadedTiles = new Set<string>();
|
const loadedTiles = new Set<string>();
|
||||||
const self = this;
|
const self = this;
|
||||||
this.onFail = (msg, url) => {
|
this.onFail = (msg, url) => {
|
||||||
console.warn(`Could not load geojson layer from`, url, "due to", msg)
|
console.warn(`Could not load geojson layer from`, url, "due to", msg)
|
||||||
loadedTiles.delete(url)
|
loadedTiles.add(url); // We add the url to the 'loadedTiles' in order to not reload it in the future
|
||||||
}
|
}
|
||||||
|
|
||||||
const neededTiles = locationControl.map(
|
const neededTiles = locationControl.map(
|
||||||
location => {
|
location => {
|
||||||
|
|
||||||
if (!flayer.isDisplayed.data) {
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Yup, this is cheating to just get the bounds here
|
// Yup, this is cheating to just get the bounds here
|
||||||
const bounds = State.state.leafletMap.data.getBounds()
|
const bounds = State.state.leafletMap.data.getBounds()
|
||||||
const tileRange = Utils.TileRangeBetween(zoomLevel, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
|
const tileRange = Utils.TileRangeBetween(zoomLevel, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
|
||||||
|
@ -69,11 +68,21 @@ export default class GeoJsonSource implements FeatureSource {
|
||||||
}
|
}
|
||||||
return needed;
|
return needed;
|
||||||
}
|
}
|
||||||
);
|
, [flayer.isDisplayed]);
|
||||||
neededTiles.stabilized(250).addCallback((needed: Set<string>) => {
|
neededTiles.stabilized(250).addCallback((needed: Set<string>) => {
|
||||||
if (needed === undefined) {
|
if (needed === undefined) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
if (!flayer.isDisplayed.data) {
|
||||||
|
// No need to download! - the layer is disabled
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
console.log("???", locationControl, flayer.layerDef)
|
||||||
|
if(locationControl.data.zoom < flayer.layerDef.minzoom){
|
||||||
|
console.log("Not downloading ", needed, "not sufficiently zoomed")
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
needed.forEach(neededTile => {
|
needed.forEach(neededTile => {
|
||||||
if (loadedTiles.has(neededTile)) {
|
if (loadedTiles.has(neededTile)) {
|
||||||
return;
|
return;
|
||||||
|
@ -86,7 +95,6 @@ export default class GeoJsonSource implements FeatureSource {
|
||||||
})
|
})
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Merges together the layers which have the same source
|
* Merges together the layers which have the same source
|
||||||
|
@ -98,7 +106,7 @@ export default class GeoJsonSource implements FeatureSource {
|
||||||
|
|
||||||
const flayersPerSource = new Map<string, { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig }[]>();
|
const flayersPerSource = new Map<string, { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig }[]>();
|
||||||
for (const flayer of flayers) {
|
for (const flayer of flayers) {
|
||||||
const url = flayer.layerDef.source.geojsonSource
|
const url = flayer.layerDef.source.geojsonSource?.replace(/{layer}/g, flayer.layerDef.id)
|
||||||
if (url === undefined) {
|
if (url === undefined) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
|
@ -98,7 +98,8 @@ export class UIEventSource<T> {
|
||||||
const self = this;
|
const self = this;
|
||||||
|
|
||||||
const newSource = new UIEventSource<J>(
|
const newSource = new UIEventSource<J>(
|
||||||
f(this.data)
|
f(this.data),
|
||||||
|
"map("+this.tag+")"
|
||||||
);
|
);
|
||||||
|
|
||||||
const update = function () {
|
const update = function () {
|
||||||
|
|
|
@ -79,7 +79,7 @@ export class QueryParameters {
|
||||||
return QueryParameters.knownSources[key];
|
return QueryParameters.knownSources[key];
|
||||||
}
|
}
|
||||||
QueryParameters.addOrder(key);
|
QueryParameters.addOrder(key);
|
||||||
const source = new UIEventSource<string>(deflt);
|
const source = new UIEventSource<string>(deflt, "&"+key);
|
||||||
QueryParameters.knownSources[key] = source;
|
QueryParameters.knownSources[key] = source;
|
||||||
source.addCallback(() => QueryParameters.Serialize())
|
source.addCallback(() => QueryParameters.Serialize())
|
||||||
return source;
|
return source;
|
||||||
|
|
|
@ -2,7 +2,7 @@ import { Utils } from "../Utils";
|
||||||
|
|
||||||
export default class Constants {
|
export default class Constants {
|
||||||
|
|
||||||
public static vNumber = "0.6.11b";
|
public static vNumber = "0.6.11c";
|
||||||
|
|
||||||
// The user journey states thresholds when a new feature gets unlocked
|
// The user journey states thresholds when a new feature gets unlocked
|
||||||
public static userJourney = {
|
public static userJourney = {
|
||||||
|
|
|
@ -55,7 +55,12 @@
|
||||||
"minzoom":14
|
"minzoom":14
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"sport_pitch",
|
{
|
||||||
|
"builtin": "sport_pitch",
|
||||||
|
"override": {
|
||||||
|
"minzoom": 14
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"builtin": "slow_roads",
|
"builtin": "slow_roads",
|
||||||
|
@ -174,9 +179,14 @@
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
|
"clustering": {
|
||||||
|
"maxZoom": 16,
|
||||||
|
"minNeededElements": 100
|
||||||
|
},
|
||||||
"overrideAll": {
|
"overrideAll": {
|
||||||
"source": {
|
"source": {
|
||||||
"geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson",
|
"geoJsonLocal": "http://127.0.0.1:8080/speelplekken_{layer}_{z}_{x}_{y}.geojson",
|
||||||
|
"geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{layer}_{z}_{x}_{y}.geojson",
|
||||||
"geoJsonZoomLevel": 14
|
"geoJsonZoomLevel": 14
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
|
@ -2,6 +2,7 @@
|
||||||
* Generates a collection of geojson files based on an overpass query for a given theme
|
* Generates a collection of geojson files based on an overpass query for a given theme
|
||||||
*/
|
*/
|
||||||
import {TileRange, Utils} from "../Utils";
|
import {TileRange, Utils} from "../Utils";
|
||||||
|
|
||||||
Utils.runningFromConsole = true
|
Utils.runningFromConsole = true
|
||||||
import {Overpass} from "../Logic/Osm/Overpass";
|
import {Overpass} from "../Logic/Osm/Overpass";
|
||||||
import {existsSync, readFileSync, writeFileSync} from "fs";
|
import {existsSync, readFileSync, writeFileSync} from "fs";
|
||||||
|
@ -15,7 +16,6 @@ import * as OsmToGeoJson from "osmtogeojson";
|
||||||
import MetaTagging from "../Logic/MetaTagging";
|
import MetaTagging from "../Logic/MetaTagging";
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
function createOverpassObject(theme: LayoutConfig) {
|
function createOverpassObject(theme: LayoutConfig) {
|
||||||
let filters: TagsFilter[] = [];
|
let filters: TagsFilter[] = [];
|
||||||
let extraScripts: string[] = [];
|
let extraScripts: string[] = [];
|
||||||
|
@ -106,7 +106,7 @@ async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass)/
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if(!success){
|
if (!success) {
|
||||||
failed++;
|
failed++;
|
||||||
console.log("Hit the rate limit - waiting 90s")
|
console.log("Hit the rate limit - waiting 90s")
|
||||||
for (let i = 0; i < 90; i++) {
|
for (let i = 0; i < 90; i++) {
|
||||||
|
@ -159,12 +159,37 @@ async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig)
|
||||||
// Extract the relationship information
|
// Extract the relationship information
|
||||||
const relations = ExtractRelations.BuildMembershipTable(ExtractRelations.GetRelationElements(rawOsm))
|
const relations = ExtractRelations.BuildMembershipTable(ExtractRelations.GetRelationElements(rawOsm))
|
||||||
MetaTagging.addMetatags(featuresFreshness, relations, theme.layers);
|
MetaTagging.addMetatags(featuresFreshness, relations, theme.layers);
|
||||||
writeFileSync(geoJsonName(targetdir, x, y, r.zoomlevel), JSON.stringify(geojson))
|
|
||||||
|
|
||||||
|
writeFileSync(geoJsonName(targetdir, x, y, r.zoomlevel), JSON.stringify(geojson, null, " "))
|
||||||
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function splitPerLayer(targetdir: string, r: TileRange, theme: LayoutConfig) {
|
||||||
|
let processed = 0;
|
||||||
|
const z = r.zoomlevel;
|
||||||
|
for (let x = r.xstart; x <= r.xend; x++) {
|
||||||
|
for (let y = r.ystart; y <= r.yend; y++) {
|
||||||
|
const file = readFileSync(geoJsonName(targetdir, x, y, z), "UTF8")
|
||||||
|
|
||||||
|
for (const layer of theme.layers) {
|
||||||
|
const geojson = JSON.parse(file)
|
||||||
|
geojson.features = geojson.features.filter(f => f._matching_layer_id === layer.id)
|
||||||
|
if(geojson.features.length == 0){
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const new_path = geoJsonName(targetdir+"_"+layer.id, x, y, z);
|
||||||
|
writeFileSync(new_path, JSON.stringify(geojson, null, " "))
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
async function main(args: string[]) {
|
async function main(args: string[]) {
|
||||||
|
|
||||||
if (args.length == 0) {
|
if (args.length == 0) {
|
||||||
|
@ -203,6 +228,7 @@ async function main(args: string[]) {
|
||||||
} while (failed > 0)
|
} while (failed > 0)
|
||||||
|
|
||||||
await postProcess(targetdir, tileRange, theme)
|
await postProcess(targetdir, tileRange, theme)
|
||||||
|
await splitPerLayer(targetdir, tileRange, theme)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue