Add capability to load tiled geojsons, eventually as overpass-cache
This commit is contained in:
pietervdvn
parent
475cdae19f
commit
2da52501a3
16 changed files with 520 additions and 76 deletions
|
|
@ -85,6 +85,7 @@ export default class LayerConfig {
|
|||
this.source = new SourceConfig({
|
||||
osmTags: osmTags,
|
||||
geojsonSource: json.source["geoJson"],
|
||||
geojsonSourceLevel: json.source["geoJsonZoomLevel"],
|
||||
overpassScript: json.source["overpassScript"],
|
||||
});
|
||||
} else {
|
||||
|
|
@ -159,7 +160,7 @@ export default class LayerConfig {
|
|||
|
||||
if (renderingJson === "questions") {
|
||||
if (readOnly) {
|
||||
throw `A tagrendering has a question, but asking a question does not make sense here: is it a title icon or a geojson-layer? ${context}`
|
||||
throw `A tagrendering has a question, but asking a question does not make sense here: is it a title icon or a geojson-layer? ${context}. The offending tagrendering is ${JSON.stringify(renderingJson)}`
|
||||
}
|
||||
|
||||
return new TagRenderingConfig("questions", undefined)
|
||||
|
|
@ -176,7 +177,7 @@ export default class LayerConfig {
|
|||
});
|
||||
}
|
||||
|
||||
this.tagRenderings = trs(json.tagRenderings, this.source.geojsonSource !== undefined);
|
||||
this.tagRenderings = trs(json.tagRenderings, false);
|
||||
|
||||
|
||||
const titleIcons = [];
|
||||
|
|
|
|||
|
|
@ -29,7 +29,8 @@ export interface LayerConfigJson {
|
|||
* There are some options:
|
||||
*
|
||||
* source: {osmTags: "key=value"} will fetch all objects with given tags from OSM. Currently, this will create a query to overpass and fetch the data - in the future this might fetch from the OSM API
|
||||
* source: {geoJsonSource: "https://my.source.net/some-geo-data.geojson"} to fetch a geojson from a third party source
|
||||
* source: {geoJson: "https://my.source.net/some-geo-data.geojson"} to fetch a geojson from a third party source
|
||||
* source: {geoJson: "https://my.source.net/some-tile-geojson-{z}-{x}-{y}.geojson", geoJsonZoomLevel: 14} to use a tiled geojson source. The web server must offer multiple geojsons. {z}, {x} and {y} are substituted
|
||||
*
|
||||
* source: {overpassScript: "<custom overpass tags>"} when you want to do special things. _This should be really rare_.
|
||||
* This means that the data will be pulled from overpass with this script, and will ignore the osmTags for the query
|
||||
|
|
@ -40,7 +41,7 @@ export interface LayerConfigJson {
|
|||
* While still supported, this is considered deprecated
|
||||
*/
|
||||
source: { osmTags: AndOrTagConfigJson | string } |
|
||||
{ osmTags: AndOrTagConfigJson | string, geoJson: string } |
|
||||
{ osmTags: AndOrTagConfigJson | string, geoJson: string, geoJsonZoomLevel?: number } |
|
||||
{ osmTags: AndOrTagConfigJson | string, overpassScript: string }
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -108,7 +108,7 @@ export default class LayoutConfig {
|
|||
throw "Unkown fixed layer " + name;
|
||||
}
|
||||
// @ts-ignore
|
||||
layer = Utils.Merge(layer.override, shared);
|
||||
layer = Utils.Merge(layer.override, JSON.parse(JSON.stringify(shared))); // We make a deep copy of the shared layer, in order to protect it from changes
|
||||
}
|
||||
|
||||
// @ts-ignore
|
||||
|
|
|
|||
|
|
@ -5,11 +5,13 @@ export default class SourceConfig {
|
|||
osmTags?: TagsFilter;
|
||||
overpassScript?: string;
|
||||
geojsonSource?: string;
|
||||
geojsonZoomLevel?: number;
|
||||
|
||||
constructor(params: {
|
||||
osmTags?: TagsFilter,
|
||||
overpassScript?: string,
|
||||
geojsonSource?: string
|
||||
geojsonSource?: string,
|
||||
geojsonSourceLevel?: number
|
||||
}) {
|
||||
|
||||
let defined = 0;
|
||||
|
|
@ -28,5 +30,6 @@ export default class SourceConfig {
|
|||
this.osmTags = params.osmTags;
|
||||
this.overpassScript = params.overpassScript;
|
||||
this.geojsonSource = params.geojsonSource;
|
||||
this.geojsonZoomLevel = params.geojsonSourceLevel;
|
||||
}
|
||||
}
|
||||
|
|
@ -33,14 +33,9 @@ export default class FeaturePipeline implements FeatureSource {
|
|||
updater)
|
||||
)), layout));
|
||||
|
||||
const geojsonSources: GeoJsonSource [] = []
|
||||
for (const flayer of flayers.data) {
|
||||
const sourceUrl = flayer.layerDef.source.geojsonSource
|
||||
if (sourceUrl !== undefined) {
|
||||
geojsonSources.push(new RegisteringFeatureSource(new FeatureDuplicatorPerLayer(flayers,
|
||||
new GeoJsonSource(flayer.layerDef.id, sourceUrl))))
|
||||
}
|
||||
}
|
||||
const geojsonSources: FeatureSource [] = GeoJsonSource
|
||||
.ConstructMultiSource(flayers.data, locationControl)
|
||||
.map(geojsonSource => new RegisteringFeatureSource(new FeatureDuplicatorPerLayer(flayers, geojsonSource)));
|
||||
|
||||
const amendedLocalStorageSource =
|
||||
new RememberingSource(new RegisteringFeatureSource(new FeatureDuplicatorPerLayer(flayers, new LocalStorageSource(layout))
|
||||
|
|
|
|||
|
|
@ -1,51 +1,195 @@
|
|||
import FeatureSource from "./FeatureSource";
|
||||
import {UIEventSource} from "../UIEventSource";
|
||||
import * as $ from "jquery";
|
||||
import {control} from "leaflet";
|
||||
import zoom = control.zoom;
|
||||
import Loc from "../../Models/Loc";
|
||||
import State from "../../State";
|
||||
import {Utils} from "../../Utils";
|
||||
import LayerConfig from "../../Customizations/JSON/LayerConfig";
|
||||
|
||||
|
||||
/**
|
||||
* Fetches a geojson file somewhere and passes it along
|
||||
*/
|
||||
export default class GeoJsonSource implements FeatureSource {
|
||||
|
||||
features: UIEventSource<{ feature: any; freshness: Date }[]>;
|
||||
|
||||
constructor(layerId: string, url: string, onFail: ((errorMsg: any) => void) = undefined) {
|
||||
private readonly onFail: ((errorMsg: any, url: string) => void) = undefined;
|
||||
|
||||
private readonly layerId: string;
|
||||
|
||||
private readonly seenids: Set<string> = new Set<string>()
|
||||
|
||||
constructor(locationControl: UIEventSource<Loc>,
|
||||
flayer: { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig },
|
||||
onFail?: ((errorMsg: any) => void)) {
|
||||
this.layerId = flayer.layerDef.id;
|
||||
let url = flayer.layerDef.source.geojsonSource;
|
||||
const zoomLevel = flayer.layerDef.source.geojsonZoomLevel;
|
||||
|
||||
this.features = new UIEventSource<{ feature: any; freshness: Date }[]>([])
|
||||
|
||||
if (zoomLevel === undefined) {
|
||||
// This is a classic, static geojson layer
|
||||
if (onFail === undefined) {
|
||||
onFail = errorMsg => {
|
||||
console.warn(`Could not load geojson layer from`, url, "due to", errorMsg)
|
||||
}
|
||||
}
|
||||
this.features = new UIEventSource<{ feature: any; freshness: Date }[]>(undefined)
|
||||
this.onFail = onFail;
|
||||
|
||||
this.LoadJSONFrom(url)
|
||||
} else {
|
||||
// This is a dynamic template with a fixed zoom level
|
||||
url = url.replace("{z}", "" + zoomLevel)
|
||||
const loadedTiles = new Set<string>();
|
||||
const self = this;
|
||||
this.onFail = (msg, url) => {
|
||||
console.warn(`Could not load geojson layer from`, url, "due to", msg)
|
||||
loadedTiles.delete(url)
|
||||
}
|
||||
|
||||
const neededTiles = locationControl.map(
|
||||
location => {
|
||||
|
||||
if (!flayer.isDisplayed.data) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Yup, this is cheating to just get the bounds here
|
||||
const bounds = State.state.leafletMap.data.getBounds()
|
||||
const tileRange = Utils.TileRangeBetween(zoomLevel, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
|
||||
const needed = new Set<string>();
|
||||
for (let x = tileRange.xstart; x <= tileRange.xend; x++) {
|
||||
for (let y = tileRange.ystart; y <= tileRange.yend; y++) {
|
||||
let neededUrl = url.replace("{x}", "" + x).replace("{y}", "" + y);
|
||||
needed.add(neededUrl)
|
||||
}
|
||||
}
|
||||
return needed;
|
||||
}
|
||||
);
|
||||
neededTiles.stabilized(250).addCallback((needed: Set<string>) => {
|
||||
if (needed === undefined) {
|
||||
return;
|
||||
}
|
||||
needed.forEach(neededTile => {
|
||||
if (loadedTiles.has(neededTile)) {
|
||||
return;
|
||||
}
|
||||
|
||||
loadedTiles.add(neededTile)
|
||||
self.LoadJSONFrom(neededTile)
|
||||
|
||||
})
|
||||
})
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Merges together the layers which have the same source
|
||||
* @param flayers
|
||||
* @param locationControl
|
||||
* @constructor
|
||||
*/
|
||||
public static ConstructMultiSource(flayers: { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig }[], locationControl: UIEventSource<Loc>): GeoJsonSource[] {
|
||||
|
||||
const flayersPerSource = new Map<string, { isDisplayed: UIEventSource<boolean>, layerDef: LayerConfig }[]>();
|
||||
for (const flayer of flayers) {
|
||||
const url = flayer.layerDef.source.geojsonSource
|
||||
if (url === undefined) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!flayersPerSource.has(url)) {
|
||||
flayersPerSource.set(url, [])
|
||||
}
|
||||
flayersPerSource.get(url).push(flayer)
|
||||
}
|
||||
|
||||
console.log("SOURCES", flayersPerSource)
|
||||
|
||||
const sources: GeoJsonSource[] = []
|
||||
|
||||
flayersPerSource.forEach((flayers, key) => {
|
||||
if (flayers.length == 1) {
|
||||
sources.push(new GeoJsonSource(locationControl, flayers[0]));
|
||||
return;
|
||||
}
|
||||
|
||||
const zoomlevels = Utils.Dedup(flayers.map(flayer => "" + (flayer.layerDef.source.geojsonZoomLevel ?? "")))
|
||||
if (zoomlevels.length > 1) {
|
||||
throw "Multiple zoomlevels defined for same geojson source " + key
|
||||
}
|
||||
|
||||
let isShown = new UIEventSource<boolean>(true, "IsShown for multiple layers: or of multiple values");
|
||||
for (const flayer of flayers) {
|
||||
flayer.isDisplayed.addCallbackAndRun(() => {
|
||||
let value = false;
|
||||
for (const flayer of flayers) {
|
||||
value = flayer.isDisplayed.data || value;
|
||||
}
|
||||
isShown.setData(value);
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
const source = new GeoJsonSource(locationControl, {
|
||||
isDisplayed: isShown,
|
||||
layerDef: flayers[0].layerDef // We only care about the source info here
|
||||
})
|
||||
sources.push(source)
|
||||
|
||||
})
|
||||
return sources;
|
||||
|
||||
}
|
||||
|
||||
private LoadJSONFrom(url: string) {
|
||||
const eventSource = this.features;
|
||||
const self = this;
|
||||
$.getJSON(url, function (json, status) {
|
||||
if (status !== "success") {
|
||||
console.log("Fetching geojson failed failed")
|
||||
onFail(status);
|
||||
self.onFail(status, url);
|
||||
return;
|
||||
}
|
||||
|
||||
if (json.elements === [] && json.remarks.indexOf("runtime error") > 0) {
|
||||
console.log("Timeout or other runtime error");
|
||||
onFail("Runtime error (timeout)")
|
||||
self.onFail("Runtime error (timeout)", url)
|
||||
return;
|
||||
}
|
||||
const time = new Date();
|
||||
const features: { feature: any, freshness: Date } [] = []
|
||||
const newFeatures: { feature: any, freshness: Date } [] = []
|
||||
let i = 0;
|
||||
let skipped = 0;
|
||||
for (const feature of json.features) {
|
||||
if (feature.properties.id === undefined) {
|
||||
feature.properties.id = url + "/" + i;
|
||||
feature.id = url + "/" + i;
|
||||
i++;
|
||||
}
|
||||
feature._matching_layer_id = layerId;
|
||||
features.push({feature: feature, freshness: time})
|
||||
if (self.seenids.has(feature.properties.id)) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
console.log("Loaded features are", features)
|
||||
eventSource.setData(features)
|
||||
self.seenids.add(feature.properties.id)
|
||||
|
||||
}).fail(onFail)
|
||||
newFeatures.push({feature: feature, freshness: time})
|
||||
}
|
||||
console.log("Downloaded "+newFeatures.length+" new features and "+skipped+" already seen features from "+ url);
|
||||
|
||||
if(newFeatures.length == 0){
|
||||
return;
|
||||
}
|
||||
|
||||
eventSource.setData(eventSource.data.concat(newFeatures))
|
||||
|
||||
}).fail(msg => self.onFail(msg, url))
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -21,8 +21,14 @@ export default class ExtractRelations {
|
|||
State.state.knownRelations.setData(memberships)
|
||||
}
|
||||
|
||||
private static GetRelationElements(overpassJson: any): Relation[] {
|
||||
const relations = overpassJson.elements.filter(element => element.type === "relation")
|
||||
/**
|
||||
* Gets an overview of the relations - except for multipolygons. We don't care about those
|
||||
* @param overpassJson
|
||||
* @constructor
|
||||
*/
|
||||
public static GetRelationElements(overpassJson: any): Relation[] {
|
||||
const relations = overpassJson.elements
|
||||
.filter(element => element.type === "relation" && element.tags.type !== "multipolygon")
|
||||
for (const relation of relations) {
|
||||
relation.properties = relation.tags
|
||||
}
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ export class Overpass {
|
|||
}).fail(onFail)
|
||||
}
|
||||
|
||||
private buildQuery(bbox: string): string {
|
||||
buildQuery(bbox: string): string {
|
||||
const filters = this._filter.asOverpass()
|
||||
let filter = ""
|
||||
for (const filterOr of filters) {
|
||||
|
|
|
|||
32
Utils.ts
32
Utils.ts
|
|
@ -156,8 +156,6 @@ export class Utils {
|
|||
}
|
||||
|
||||
static Merge(source: any, target: any) {
|
||||
target = JSON.parse(JSON.stringify(target));
|
||||
source = JSON.parse(JSON.stringify(source));
|
||||
for (const key in source) {
|
||||
const sourceV = source[key];
|
||||
const targetV = target[key]
|
||||
|
|
@ -204,6 +202,26 @@ export class Utils {
|
|||
return {x: Utils.lon2tile(lon, z), y: Utils.lat2tile(lat, z), z: z}
|
||||
}
|
||||
|
||||
static TileRangeBetween(zoomlevel: number, lat0: number, lon0: number, lat1:number, lon1: number) : TileRange{
|
||||
const t0 = Utils.embedded_tile(lat0, lon0, zoomlevel)
|
||||
const t1 = Utils.embedded_tile(lat1, lon1, zoomlevel)
|
||||
|
||||
const xstart = Math.min(t0.x, t1.x)
|
||||
const xend = Math.max(t0.x, t1.x)
|
||||
const ystart = Math.min(t0.y, t1.y)
|
||||
const yend = Math.max(t0.y, t1.y)
|
||||
const total = (1 + xend - xstart) * (1 + yend - ystart)
|
||||
|
||||
return {
|
||||
xstart: xstart,
|
||||
xend: xend,
|
||||
ystart: ystart,
|
||||
yend: yend,
|
||||
total: total,
|
||||
zoomlevel: zoomlevel
|
||||
}
|
||||
}
|
||||
|
||||
public static MinifyJSON(stringified: string): string {
|
||||
stringified = stringified.replace(/\|/g, "||");
|
||||
|
||||
|
|
@ -257,3 +275,13 @@ export class Utils {
|
|||
return (Math.floor((1 - Math.log(Math.tan(lat * Math.PI / 180) + 1 / Math.cos(lat * Math.PI / 180)) / Math.PI) / 2 * Math.pow(2, zoom)));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
export interface TileRange{
|
||||
xstart: number,
|
||||
ystart: number,
|
||||
xend: number,
|
||||
yend: number,
|
||||
total: number,
|
||||
zoomlevel: number
|
||||
}
|
||||
|
|
@ -24,19 +24,64 @@
|
|||
"socialImage": "",
|
||||
"defaultBackgroundId": "CartoDB.Positron",
|
||||
"layers": [
|
||||
"play_forest",
|
||||
"playground",
|
||||
"sport_pitch",
|
||||
{
|
||||
"builtin": "play_forest",
|
||||
"override": {
|
||||
"source": {
|
||||
"geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson",
|
||||
"geoJsonZoomLevel": 14
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"builtin": "playground",
|
||||
"override": {
|
||||
"source": {
|
||||
"geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson",
|
||||
"geoJsonZoomLevel": 14
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"builtin": "sport_pitch",
|
||||
"override": {
|
||||
"minzoom": 15,
|
||||
"source": {
|
||||
"geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson",
|
||||
"geoJsonZoomLevel": 14
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"builtin": "slow_roads",
|
||||
"override": {
|
||||
"source": {
|
||||
"geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson",
|
||||
"geoJsonZoomLevel": 14
|
||||
},
|
||||
"calculatedTags": [
|
||||
"_part_of_walking_routes=feat.memberships().map(r => \"<a href='#relation/\"+r.relation.id+\"'>\" + r.relation.tags.name + \"</a>\").join(', ')"
|
||||
]
|
||||
}
|
||||
},
|
||||
"grass_in_parks",
|
||||
"village_green",
|
||||
{
|
||||
"builtin": "grass_in_parks",
|
||||
"override": {
|
||||
"source": {
|
||||
"geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson",
|
||||
"geoJsonZoomLevel": 14
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"builtin": "village_green",
|
||||
"override": {
|
||||
"source": {
|
||||
"geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson",
|
||||
"geoJsonZoomLevel": 14
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "walking_routes",
|
||||
"name": {
|
||||
|
|
@ -50,7 +95,9 @@
|
|||
"route=foot",
|
||||
"operator=provincie Antwerpen"
|
||||
]
|
||||
}
|
||||
},
|
||||
"geoJson": "https://pietervdvn.github.io/speelplekken_cache/speelplekken_{z}_{x}_{y}.geojson",
|
||||
"geoJsonZoomLevel": 14
|
||||
},
|
||||
"title": {
|
||||
"render": "Wandeling <i>{name}</i>",
|
||||
|
|
@ -141,17 +188,6 @@
|
|||
"width": {
|
||||
"render": "3"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "speelplekken-cache",
|
||||
"name": "",
|
||||
"source": {
|
||||
"osmTags": {
|
||||
"or": []
|
||||
},
|
||||
"geoJson": "https://pietervdvn.github.io/speelplekken-cache.geojson"
|
||||
},
|
||||
"passAllFeatures": true
|
||||
}
|
||||
],
|
||||
"roamingRenderings": [
|
||||
|
|
|
|||
|
|
@ -9,12 +9,13 @@
|
|||
"scripts": {
|
||||
"increase-memory": "export NODE_OPTIONS=--max_old_space_size=4096",
|
||||
"start": "ts-node scripts/generateLayerOverview.ts --no-fail && npm run increase-memory && parcel *.html UI/** Logic/** assets/** assets/**/** assets/**/**/** vendor/* vendor/*/*",
|
||||
"test": "ts-node test/Tag.spec.ts && ts-node test/TagQuestion.spec.ts && ts-node test/ImageSearcher.spec.ts && ts-node test/ImageAttribution.spec.ts",
|
||||
"test": "ts-node test/Tag.spec.ts && ts-node test/TagQuestion.spec.ts && ts-node test/ImageSearcher.spec.ts && ts-node test/ImageAttribution.spec.ts && ts-node test/Theme.spec.ts",
|
||||
"generate:editor-layer-index": "cd assets/ && wget https://osmlab.github.io/editor-layer-index/imagery.geojson --output-document=editor-layer-index.json",
|
||||
"generate:images": "ts-node scripts/generateIncludedImages.ts",
|
||||
"generate:translations": "ts-node scripts/generateTranslations.ts",
|
||||
"generate:layouts": "ts-node scripts/generateLayouts.ts",
|
||||
"generate:docs": "ts-node scripts/generateDocs.ts && ts-node scripts/generateTaginfoProjectFiles.ts",
|
||||
"generate:cache:speelplekken": "ts-node scripts/generateCache.ts speelplekken 14 ./cache/speelplekken 51.2003 4.3925 51.1058 4.5087",
|
||||
"generate:layeroverview": "ts-node scripts/generateLayerOverview.ts --no-fail",
|
||||
"generate:licenses": "ts-node scripts/generateLicenseInfo.ts --no-fail",
|
||||
"validate:layeroverview": "ts-node scripts/generateLayerOverview.ts --report",
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import {lstatSync, readdirSync} from "fs";
|
||||
import * as https from "https";
|
||||
|
||||
export default class ScriptUtils {
|
||||
public static readDirRecSync(path): string[] {
|
||||
|
|
@ -17,4 +18,29 @@ export default class ScriptUtils {
|
|||
return result;
|
||||
}
|
||||
|
||||
public static DownloadJSON(url, continuation : (parts : string []) => void){
|
||||
https.get(url, (res) => {
|
||||
console.log("Got response!")
|
||||
const parts : string[] = []
|
||||
res.setEncoding('utf8');
|
||||
res.on('data', function (chunk) {
|
||||
// @ts-ignore
|
||||
parts.push(chunk)
|
||||
});
|
||||
|
||||
res.addListener('end', function () {
|
||||
continuation(parts)
|
||||
});
|
||||
})
|
||||
}
|
||||
|
||||
public static sleep(ms) {
|
||||
return new Promise((resolve) => {
|
||||
console.debug("Sleeping for", ms)
|
||||
setTimeout(resolve, ms);
|
||||
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,21 +0,0 @@
|
|||
|
||||
|
||||
// Loads a geojson file downloaded from overpass, renames "@id" to "id" and deletes "@relations"
|
||||
|
||||
import {readFileSync, writeFileSync} from "fs";
|
||||
|
||||
const source = process.argv[2] ?? "~/Downloads/export.json"
|
||||
console.log("Fixing up ", source)
|
||||
const contents = readFileSync(source, "UTF8");
|
||||
const f = JSON.parse(contents);
|
||||
let i = 0
|
||||
for (const feature of f.features) {
|
||||
if(feature.properties == undefined){
|
||||
continue
|
||||
}
|
||||
feature.properties["id"] = feature.properties["@id"]
|
||||
feature.properties["@id"] = undefined
|
||||
feature.properties["@relations"] = undefined
|
||||
}
|
||||
|
||||
writeFileSync(source+".fixed", JSON.stringify(f, null, " "))
|
||||
176
scripts/generateCache.ts
Normal file
176
scripts/generateCache.ts
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
/**
|
||||
* Generates a collection of geojson files based on an overpass query for a given theme
|
||||
*/
|
||||
import {TileRange, Utils} from "../Utils";
|
||||
|
||||
Utils.runningFromConsole = true
|
||||
import {Overpass} from "../Logic/Osm/Overpass";
|
||||
import {writeFileSync, existsSync, readFileSync} from "fs";
|
||||
import {TagsFilter} from "../Logic/Tags/TagsFilter";
|
||||
import {Or} from "../Logic/Tags/Or";
|
||||
import LayoutConfig from "../Customizations/JSON/LayoutConfig";
|
||||
import {AllKnownLayouts} from "../Customizations/AllKnownLayouts";
|
||||
import ScriptUtils from "./ScriptUtils";
|
||||
import ExtractRelations from "../Logic/Osm/ExtractRelations";
|
||||
import * as OsmToGeoJson from "osmtogeojson";
|
||||
import {Script} from "vm";
|
||||
|
||||
function createOverpassObject(theme: LayoutConfig) {
|
||||
let filters: TagsFilter[] = [];
|
||||
let extraScripts: string[] = [];
|
||||
for (const layer of theme.layers) {
|
||||
if (typeof (layer) === "string") {
|
||||
throw "A layer was not expanded!"
|
||||
}
|
||||
if (layer.doNotDownload) {
|
||||
continue;
|
||||
}
|
||||
if (layer.source.geojsonSource !== undefined) {
|
||||
// Not our responsibility to download this layer!
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// Check if data for this layer has already been loaded
|
||||
if (layer.source.overpassScript !== undefined) {
|
||||
extraScripts.push(layer.source.overpassScript)
|
||||
} else {
|
||||
filters.push(layer.source.osmTags);
|
||||
}
|
||||
}
|
||||
filters = Utils.NoNull(filters)
|
||||
extraScripts = Utils.NoNull(extraScripts)
|
||||
if (filters.length + extraScripts.length === 0) {
|
||||
throw "Nothing to download! The theme doesn't declare anything to download"
|
||||
}
|
||||
return new Overpass(new Or(filters), extraScripts);
|
||||
}
|
||||
|
||||
function saveResponse(chunks: string[], targetDir: string) {
|
||||
const contents = chunks.join("")
|
||||
if (contents.startsWith("<?xml")) {
|
||||
// THis is an error message
|
||||
console.error("Failed to create ", targetDir, "probably over quota")
|
||||
return;
|
||||
}
|
||||
writeFileSync(targetDir, contents)
|
||||
}
|
||||
|
||||
function rawJsonName(targetDir: string, x: number, y: number, z: number): string {
|
||||
return targetDir + "_" + z + "_" + x + "_" + y + ".json"
|
||||
}
|
||||
|
||||
function geoJsonName(targetDir: string, x: number, y: number, z: number): string {
|
||||
return targetDir + "_" + z + "_" + x + "_" + y + ".geojson"
|
||||
}
|
||||
|
||||
function metaJsonName(targetDir: string, x: number, y: number, z: number): string {
|
||||
return targetDir + "_" + z + "_" + x + "_" + y + ".meta.json"
|
||||
}
|
||||
|
||||
async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass) {
|
||||
let downloaded = 0
|
||||
for (let x = r.xstart; x <= r.xend; x++) {
|
||||
for (let y = r.ystart; y <= r.yend; y++) {
|
||||
console.log("x:", (x - r.xstart), "/", (r.xend - r.xstart), "; y:", (y - r.ystart), "/", (r.yend - r.ystart), "; total: ", downloaded, "/", r.total)
|
||||
downloaded++;
|
||||
const filename = rawJsonName(targetdir, x, y, r.zoomlevel)
|
||||
if (existsSync(filename)) {
|
||||
console.log("Already exists: ", filename)
|
||||
continue;
|
||||
}
|
||||
|
||||
const boundsArr = Utils.tile_bounds(r.zoomlevel, x, y)
|
||||
const bounds = {
|
||||
north: Math.max(boundsArr[0][0], boundsArr[1][0]),
|
||||
south: Math.min(boundsArr[0][0], boundsArr[1][0]),
|
||||
east: Math.max(boundsArr[0][1], boundsArr[1][1]),
|
||||
west: Math.min(boundsArr[0][1], boundsArr[1][1])
|
||||
}
|
||||
console.log("Downloading tile", r.zoomlevel, x, y, "with bounds", bounds)
|
||||
const url = overpass.buildQuery("[bbox:" + bounds.south + "," + bounds.west + "," + bounds.north + "," + bounds.east + "]")
|
||||
|
||||
ScriptUtils.DownloadJSON(url,
|
||||
chunks => {
|
||||
saveResponse(chunks, filename)
|
||||
})
|
||||
|
||||
await ScriptUtils.sleep(10000)
|
||||
console.debug("Waking up")
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function postProcess(targetdir: string, r: TileRange) {
|
||||
let processed = 0;
|
||||
for (let x = r.xstart; x <= r.xend; x++) {
|
||||
for (let y = r.ystart; y <= r.yend; y++) {
|
||||
processed++;
|
||||
const filename = rawJsonName(targetdir, x, y, r.zoomlevel)
|
||||
console.log(" Post processing", processed, "/",r. total, filename)
|
||||
if (!existsSync(filename)) {
|
||||
throw "Not found - and not downloaded. Run this script again!: " + filename
|
||||
}
|
||||
|
||||
// We read the raw OSM-file and convert it to a geojson
|
||||
const rawOsm = JSON.parse(readFileSync(filename, "UTF8"))
|
||||
|
||||
// Create and save the geojson file - which is the main chunk of the data
|
||||
const geojson = OsmToGeoJson.default(rawOsm);
|
||||
writeFileSync(geoJsonName(targetdir, x, y, r.zoomlevel), JSON.stringify(geojson))
|
||||
|
||||
// Extract the relationship information
|
||||
const relations = ExtractRelations.GetRelationElements(rawOsm)
|
||||
const osmTime = new Date(rawOsm.osm3s.timestamp_osm_base);
|
||||
|
||||
const meta = {
|
||||
freshness: osmTime,
|
||||
relations: relations
|
||||
}
|
||||
|
||||
writeFileSync(
|
||||
metaJsonName(targetdir, x, y, r.zoomlevel),
|
||||
JSON.stringify(meta)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function main(args: string[]) {
|
||||
|
||||
if (args.length == 0) {
|
||||
console.error("Expected arguments are: theme zoomlevel targetdirectory lat0 lon0 lat1 lon1")
|
||||
return;
|
||||
}
|
||||
const themeName = args[0]
|
||||
const zoomlevel = Number(args[1])
|
||||
const targetdir = args[2]
|
||||
const lat0 = Number(args[3])
|
||||
const lon0 = Number(args[4])
|
||||
const lat1 = Number(args[5])
|
||||
const lon1 = Number(args[6])
|
||||
|
||||
const tileRange = Utils.TileRangeBetween(zoomlevel, lat0, lon0, lat1, lon1)
|
||||
|
||||
const theme = AllKnownLayouts.allKnownLayouts.get(themeName)
|
||||
if (theme === undefined) {
|
||||
const keys = []
|
||||
AllKnownLayouts.allKnownLayouts.forEach((_, key) => {
|
||||
keys.push(key)
|
||||
})
|
||||
console.error("The theme " + theme + " was not found; try one of ", keys);
|
||||
return
|
||||
}
|
||||
|
||||
const overpass = createOverpassObject(theme)
|
||||
|
||||
|
||||
await downloadRaw(targetdir, tileRange, overpass)
|
||||
await postProcess(targetdir, tileRange)
|
||||
}
|
||||
|
||||
|
||||
let args = [...process.argv]
|
||||
args.splice(0, 2)
|
||||
main(args);
|
||||
48
test/Theme.spec.ts
Normal file
48
test/Theme.spec.ts
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
import T from "./TestHelper";
|
||||
import {Utils} from "../Utils";
|
||||
|
||||
Utils.runningFromConsole = true;
|
||||
import TagRenderingQuestion from "../UI/Popup/TagRenderingQuestion";
|
||||
import {UIEventSource} from "../Logic/UIEventSource";
|
||||
import TagRenderingConfig from "../Customizations/JSON/TagRenderingConfig";
|
||||
import LayoutConfig from "../Customizations/JSON/LayoutConfig";
|
||||
import {LayoutConfigJson} from "../Customizations/JSON/LayoutConfigJson";
|
||||
import * as assert from "assert";
|
||||
|
||||
|
||||
new T("Theme tests",
|
||||
[
|
||||
["Nested overrides work", () => {
|
||||
|
||||
const themeConfigJson : LayoutConfigJson = {
|
||||
description: "Descr",
|
||||
icon: "",
|
||||
language: ["en"],
|
||||
layers: [
|
||||
{
|
||||
builtin: "public_bookcase",
|
||||
override: {
|
||||
source:{
|
||||
geoJson: "xyz"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
maintainer: "",
|
||||
startLat: 0,
|
||||
startLon: 0,
|
||||
startZoom: 0,
|
||||
title: {
|
||||
en: "Title"
|
||||
},
|
||||
version: "",
|
||||
id: "test"
|
||||
}
|
||||
|
||||
const themeConfig = new LayoutConfig(themeConfigJson);
|
||||
assert.equal("xyz", themeConfig.layers[0].source.geojsonSource)
|
||||
|
||||
|
||||
}]
|
||||
]
|
||||
);
|
||||
Loading…
Reference in a new issue