midgard/mapcomplete
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Enable caching for some natuurpunt layers

Browse source
This commit is contained in:
pietervdvn 2021-07-18 19:08:14 +02:00
parent 2ff279566d
commit 6d73330f6b
2 changed files with 18 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
package.json
View file

@ -20,7 +20,7 @@
"generate:layouts": "ts-node scripts/generateLayouts.ts", "generate:layouts": "ts-node scripts/generateLayouts.ts",
"generate:docs": "ts-node scripts/generateDocs.ts && ts-node scripts/generateTaginfoProjectFiles.ts", "generate:docs": "ts-node scripts/generateDocs.ts && ts-node scripts/generateTaginfoProjectFiles.ts",
"generate:cache:speelplekken": "npm run generate:layeroverview && ts-node scripts/generateCache.ts speelplekken 14 ../pietervdvn.github.io/speelplekken_cache/ 51.20 4.35 51.09 4.56", "generate:cache:speelplekken": "npm run generate:layeroverview && ts-node scripts/generateCache.ts speelplekken 14 ../pietervdvn.github.io/speelplekken_cache/ 51.20 4.35 51.09 4.56",
"generate:cache:natuurpunt": "npm run generate:layeroverview && ts-node scripts/generateCache.ts natuurpunt 6 ../pietervdvn.github.io/natuurpunt_cache/ 50.40 2.1 51.54 6.4", "generate:cache:natuurpunt": "npm run generate:layeroverview && ts-node scripts/generateCache.ts natuurpunt 12 ../pietervdvn.github.io/natuurpunt_cache/ 50.40 2.1 51.54 6.4",
"generate:layeroverview": "npm run generate:licenses && echo {\\\"layers\\\":[], \\\"themes\\\":[]} > ./assets/generated/known_layers_and_themes.json && ts-node scripts/generateLayerOverview.ts --no-fail", "generate:layeroverview": "npm run generate:licenses && echo {\\\"layers\\\":[], \\\"themes\\\":[]} > ./assets/generated/known_layers_and_themes.json && ts-node scripts/generateLayerOverview.ts --no-fail",
"generate:licenses": "ts-node scripts/generateLicenseInfo.ts --no-fail", "generate:licenses": "ts-node scripts/generateLicenseInfo.ts --no-fail",
"generate:report": "cd Docs/Tools && ./compileStats.sh && git commit . -m 'New statistics ands graphs' && git push", "generate:report": "cd Docs/Tools && ./compileStats.sh && git commit . -m 'New statistics ands graphs' && git push",

21
scripts/generateCache.ts
View file

@ -88,13 +88,24 @@ async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass)/
await ScriptUtils.DownloadJSON(url) await ScriptUtils.DownloadJSON(url)
.then(json => { .then(json => {
if (json.elements.length === 0) {
console.log("Got an empty response!")
if ((<string>json.remark ?? "").startsWith("runtime error")) {
console.error("Got a runtime error: ", json.remark)
failed++;
return
}
}
console.log("Got the response - writing to ", filename) console.log("Got the response - writing to ", filename)
writeFileSync(filename, JSON.stringify(json, null, " ")); writeFileSync(filename, JSON.stringify(json, null, " "));
} }
) )
.catch(err => { .catch(err => {
console.log(url) console.log(url)
console.log("Could not download - probably hit the rate limit; waiting a bit. ("+err+")") console.log("Could not download - probably hit the rate limit; waiting a bit. (" + err + ")")
failed++; failed++;
return ScriptUtils.sleep(60000).then(() => console.log("Waiting is done")) return ScriptUtils.sleep(60000).then(() => console.log("Waiting is done"))
}) })
@ -168,7 +179,7 @@ async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig,
// Extract the relationship information // Extract the relationship information
const relations = ExtractRelations.BuildMembershipTable(ExtractRelations.GetRelationElements(rawOsm)) const relations = ExtractRelations.BuildMembershipTable(ExtractRelations.GetRelationElements(rawOsm))
MetaTagging.addMetatags(featuresFreshness, new UIEventSource<{feature: any; freshness: Date}[]>(featuresFreshness) , relations, theme.layers, false); MetaTagging.addMetatags(featuresFreshness, new UIEventSource<{ feature: any; freshness: Date }[]>(featuresFreshness), relations, theme.layers, false);
for (const feature of geojson.features) { for (const feature of geojson.features) {
@ -192,7 +203,7 @@ async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig,
delete feature["bbox"] delete feature["bbox"]
} }
const targetPath = geoJsonName(targetdir+".unfiltered", x, y, r.zoomlevel) const targetPath = geoJsonName(targetdir + ".unfiltered", x, y, r.zoomlevel)
// This is the geojson file containing all features // This is the geojson file containing all features
writeFileSync(targetPath, JSON.stringify(geojson, null, " ")) writeFileSync(targetPath, JSON.stringify(geojson, null, " "))
@ -204,7 +215,7 @@ async function splitPerLayer(targetdir: string, r: TileRange, theme: LayoutConfi
const z = r.zoomlevel; const z = r.zoomlevel;
for (let x = r.xstart; x <= r.xend; x++) { for (let x = r.xstart; x <= r.xend; x++) {
for (let y = r.ystart; y <= r.yend; y++) { for (let y = r.ystart; y <= r.yend; y++) {
const file = readFileSync(geoJsonName(targetdir+".unfiltered", x, y, z), "UTF8") const file = readFileSync(geoJsonName(targetdir + ".unfiltered", x, y, z), "UTF8")
for (const layer of theme.layers) { for (const layer of theme.layers) {
if (!layer.source.isOsmCacheLayer) { if (!layer.source.isOsmCacheLayer) {
@ -222,7 +233,7 @@ async function splitPerLayer(targetdir: string, r: TileRange, theme: LayoutConfi
return true; return true;
}) })
const new_path = geoJsonName(targetdir + "_" + layer.id, x, y, z); const new_path = geoJsonName(targetdir + "_" + layer.id, x, y, z);
console.log(new_path, " has ", geojson.features.length, " features after filtering (dropped ", oldLength - geojson.features.length,")" ) console.log(new_path, " has ", geojson.features.length, " features after filtering (dropped ", oldLength - geojson.features.length, ")")
if (geojson.features.length == 0) { if (geojson.features.length == 0) {
console.log("Not writing geojson file as it is empty", new_path) console.log("Not writing geojson file as it is empty", new_path)
continue; continue;