Improve slice script, formatting
This commit is contained in:
parent
cce6a9e832
commit
ccb548816f
3 changed files with 88 additions and 73 deletions
|
|
@ -24,6 +24,7 @@ import {GeoOperations} from "../Logic/GeoOperations";
|
|||
import SimpleMetaTaggers from "../Logic/SimpleMetaTagger";
|
||||
import FilteringFeatureSource from "../Logic/FeatureSource/Sources/FilteringFeatureSource";
|
||||
import Loc from "../Models/Loc";
|
||||
|
||||
ScriptUtils.fixUtils()
|
||||
|
||||
function createOverpassObject(theme: LayoutConfig, relationTracker: RelationsTracker, backend: string) {
|
||||
|
|
@ -166,7 +167,7 @@ function loadAllTiles(targetdir: string, r: TileRange, theme: LayoutConfig, extr
|
|||
|
||||
// Create and save the geojson file - which is the main chunk of the data
|
||||
const geojson = OsmToGeoJson.default(rawOsm);
|
||||
console.log(" which as",geojson.features.length, "features")
|
||||
console.log(" which as", geojson.features.length, "features")
|
||||
|
||||
allFeatures.push(...geojson.features)
|
||||
}
|
||||
|
|
@ -180,9 +181,10 @@ function loadAllTiles(targetdir: string, r: TileRange, theme: LayoutConfig, extr
|
|||
function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relationsTracker: RelationsTracker, targetdir: string, pointsOnlyLayers: string[]) {
|
||||
const skippedLayers = new Set<string>()
|
||||
|
||||
const indexedFeatures : Map<string, any> = new Map<string, any>()
|
||||
const indexedFeatures: Map<string, any> = new Map<string, any>()
|
||||
let indexisBuilt = false;
|
||||
function buildIndex(){
|
||||
|
||||
function buildIndex() {
|
||||
for (const ff of allFeatures.features.data) {
|
||||
const f = ff.feature
|
||||
indexedFeatures.set(f.properties.id, f)
|
||||
|
|
@ -190,8 +192,8 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations
|
|||
indexisBuilt = true;
|
||||
}
|
||||
|
||||
function getFeatureById(id){
|
||||
if(!indexisBuilt){
|
||||
function getFeatureById(id) {
|
||||
if (!indexisBuilt) {
|
||||
buildIndex()
|
||||
}
|
||||
return indexedFeatures.get(id)
|
||||
|
|
@ -227,9 +229,8 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations
|
|||
});
|
||||
|
||||
|
||||
|
||||
while (SimpleMetaTaggers.country.runningTasks.size > 0) {
|
||||
console.log("Still waiting for ", SimpleMetaTaggers.country.runningTasks.size," features which don't have a country yet")
|
||||
console.log("Still waiting for ", SimpleMetaTaggers.country.runningTasks.size, " features which don't have a country yet")
|
||||
await ScriptUtils.sleep(1)
|
||||
}
|
||||
|
||||
|
|
@ -248,16 +249,16 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations
|
|||
}
|
||||
|
||||
const filteredTile = new FilteringFeatureSource({
|
||||
locationControl: new UIEventSource<Loc>(undefined),
|
||||
allElements: undefined,
|
||||
selectedElement: new UIEventSource<any>(undefined)
|
||||
},
|
||||
locationControl: new UIEventSource<Loc>(undefined),
|
||||
allElements: undefined,
|
||||
selectedElement: new UIEventSource<any>(undefined)
|
||||
},
|
||||
tileIndex,
|
||||
tile,
|
||||
new UIEventSource<any>(undefined)
|
||||
)
|
||||
)
|
||||
|
||||
console.log("Tile "+layer.id+"."+tileIndex+" contains "+filteredTile.features.data.length+" features after filtering ("+tile.features.data.length+") features before")
|
||||
console.log("Tile " + layer.id + "." + tileIndex + " contains " + filteredTile.features.data.length + " features after filtering (" + tile.features.data.length + ") features before")
|
||||
if (filteredTile.features.data.length === 0) {
|
||||
return
|
||||
}
|
||||
|
|
@ -267,20 +268,20 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations
|
|||
// Some cleanup
|
||||
delete feature.feature["bbox"]
|
||||
|
||||
if(tile.layer.layerDef.calculatedTags !== undefined){
|
||||
if (tile.layer.layerDef.calculatedTags !== undefined) {
|
||||
|
||||
// Evaluate all the calculated tags strictly
|
||||
const calculatedTagKeys = tile.layer.layerDef.calculatedTags.map(ct => ct[0])
|
||||
featureCount++
|
||||
for (const calculatedTagKey of calculatedTagKeys) {
|
||||
const strict = feature.feature.properties[calculatedTagKey]
|
||||
feature.feature.properties[calculatedTagKey] =strict
|
||||
strictlyCalculated ++;
|
||||
if(strictlyCalculated % 100 === 0){
|
||||
console.log("Strictly calculated ", strictlyCalculated, "values for tile",tileIndex,": now at ", featureCount,"/",filteredTile.features.data.length, "examle value: ", strict)
|
||||
// Evaluate all the calculated tags strictly
|
||||
const calculatedTagKeys = tile.layer.layerDef.calculatedTags.map(ct => ct[0])
|
||||
featureCount++
|
||||
for (const calculatedTagKey of calculatedTagKeys) {
|
||||
const strict = feature.feature.properties[calculatedTagKey]
|
||||
feature.feature.properties[calculatedTagKey] = strict
|
||||
strictlyCalculated++;
|
||||
if (strictlyCalculated % 100 === 0) {
|
||||
console.log("Strictly calculated ", strictlyCalculated, "values for tile", tileIndex, ": now at ", featureCount, "/", filteredTile.features.data.length, "examle value: ", strict)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
// Lets save this tile!
|
||||
|
|
@ -293,7 +294,7 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations
|
|||
type: "FeatureCollection",
|
||||
features: filteredTile.features.data.map(f => f.feature)
|
||||
}, null, " "))
|
||||
console.log("Written tile", targetPath,"with", filteredTile.features.data.length)
|
||||
console.log("Written tile", targetPath, "with", filteredTile.features.data.length)
|
||||
}
|
||||
})
|
||||
|
||||
|
|
@ -315,11 +316,11 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations
|
|||
if (pointsOnlyLayers.indexOf(layer.id) >= 0) {
|
||||
|
||||
const filtered = new FilteringFeatureSource({
|
||||
locationControl: new UIEventSource<Loc>(undefined),
|
||||
locationControl: new UIEventSource<Loc>(undefined),
|
||||
allElements: undefined,
|
||||
selectedElement: new UIEventSource<any>(undefined)
|
||||
},
|
||||
Tiles.tile_index(0,0,0),
|
||||
Tiles.tile_index(0, 0, 0),
|
||||
source,
|
||||
new UIEventSource<any>(undefined)
|
||||
)
|
||||
|
|
@ -376,8 +377,6 @@ async function main(args: string[]) {
|
|||
const lon1 = Number(args[6])
|
||||
|
||||
|
||||
|
||||
|
||||
const tileRange = Tiles.TileRangeBetween(zoomlevel, lat0, lon0, lat1, lon1)
|
||||
|
||||
if (tileRange.total === 0) {
|
||||
|
|
@ -397,27 +396,27 @@ async function main(args: string[]) {
|
|||
|
||||
let generatePointLayersFor = []
|
||||
if (args[7] == "--generate-point-overview") {
|
||||
if(args[8] === undefined){
|
||||
if (args[8] === undefined) {
|
||||
throw "--generate-point-overview needs a list of layers to generate the overview for (or * for all)"
|
||||
}else if (args[8] === '*'){
|
||||
} else if (args[8] === '*') {
|
||||
generatePointLayersFor = theme.layers.map(l => l.id)
|
||||
}else{
|
||||
} else {
|
||||
generatePointLayersFor = args[8].split(",")
|
||||
}
|
||||
console.log("Also generating a point overview for layers ", generatePointLayersFor.join(","))
|
||||
}
|
||||
{
|
||||
|
||||
const index = args.indexOf("--force-zoom-level")
|
||||
if(index >= 0){
|
||||
const forcedZoomLevel = Number(args[index + 1])
|
||||
for (const layer of theme.layers) {
|
||||
layer.source.geojsonSource = "https://127.0.0.1/cache_{layer}_{z}_{x}_{y}.geojson"
|
||||
layer.source.isOsmCacheLayer = true
|
||||
layer.source.geojsonZoomLevel = forcedZoomLevel
|
||||
const index = args.indexOf("--force-zoom-level")
|
||||
if (index >= 0) {
|
||||
const forcedZoomLevel = Number(args[index + 1])
|
||||
for (const layer of theme.layers) {
|
||||
layer.source.geojsonSource = "https://127.0.0.1/cache_{layer}_{z}_{x}_{y}.geojson"
|
||||
layer.source.isOsmCacheLayer = true
|
||||
layer.source.geojsonZoomLevel = forcedZoomLevel
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
const relationTracker = new RelationsTracker()
|
||||
|
|
|
|||
|
|
@ -6,7 +6,8 @@ import {LayerConfigJson} from "../Models/ThemeConfig/Json/LayerConfigJson";
|
|||
import Constants from "../Models/Constants";
|
||||
import {
|
||||
DesugaringContext,
|
||||
PrepareLayer, PrepareTheme,
|
||||
PrepareLayer,
|
||||
PrepareTheme,
|
||||
ValidateLayer,
|
||||
ValidateThemeAndLayers
|
||||
} from "../Models/ThemeConfig/Conversion/LegacyJsonConvert";
|
||||
|
|
@ -80,7 +81,7 @@ class LayerOverviewUtils {
|
|||
dict.set(key, <TagRenderingConfigJson>questions[key])
|
||||
}
|
||||
for (const key in icons["default"]) {
|
||||
if(typeof icons[key] !== "object"){
|
||||
if (typeof icons[key] !== "object") {
|
||||
continue
|
||||
}
|
||||
icons[key].id = key;
|
||||
|
|
@ -94,6 +95,32 @@ class LayerOverviewUtils {
|
|||
return dict;
|
||||
}
|
||||
|
||||
main(_: string[]) {
|
||||
|
||||
const licensePaths = new Set<string>()
|
||||
for (const i in licenses) {
|
||||
licensePaths.add(licenses[i].path)
|
||||
}
|
||||
|
||||
const sharedLayers = this.buildLayerIndex(licensePaths);
|
||||
const sharedThemes = this.buildThemeIndex(licensePaths, sharedLayers)
|
||||
|
||||
writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({
|
||||
"layers": Array.from(sharedLayers.values()),
|
||||
"themes": Array.from(sharedThemes.values())
|
||||
}))
|
||||
|
||||
writeFileSync("./assets/generated/known_layers.json", JSON.stringify(Array.from(sharedLayers.values())))
|
||||
|
||||
writeFileSync('./assets/themes/mapcomplete-changes/icons-mapping.txt', JSON.stringify(
|
||||
Array.from(sharedThemes.values()).map(th => ({
|
||||
if: "theme=" + th.id,
|
||||
then: th.icon
|
||||
}))
|
||||
))
|
||||
|
||||
|
||||
}
|
||||
|
||||
private buildLayerIndex(knownImagePaths: Set<string>): Map<string, LayerConfigJson> {
|
||||
// First, we expand and validate all builtin layers. These are written to assets/generated/layers
|
||||
|
|
@ -126,7 +153,6 @@ class LayerOverviewUtils {
|
|||
return sharedLayers;
|
||||
}
|
||||
|
||||
|
||||
private buildThemeIndex(knownImagePaths: Set<string>, sharedLayers: Map<string, LayerConfigJson>): Map<string, LayoutConfigJson> {
|
||||
console.log(" ---------- VALIDATING BUILTIN THEMES ---------")
|
||||
const themeFiles = ScriptUtils.getThemeFiles();
|
||||
|
|
@ -160,24 +186,6 @@ class LayerOverviewUtils {
|
|||
return fixed;
|
||||
|
||||
}
|
||||
|
||||
main(_: string[]) {
|
||||
|
||||
const licensePaths = new Set<string>()
|
||||
for (const i in licenses) {
|
||||
licensePaths.add(licenses[i].path)
|
||||
}
|
||||
|
||||
const sharedLayers = this.buildLayerIndex(licensePaths);
|
||||
const sharedThemes = this.buildThemeIndex(licensePaths, sharedLayers)
|
||||
|
||||
writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({
|
||||
"layers": Array.from(sharedLayers.values()),
|
||||
"themes": Array.from(sharedThemes.values())
|
||||
}))
|
||||
|
||||
writeFileSync("./assets/generated/known_layers.json", JSON.stringify(Array.from(sharedLayers.values())))
|
||||
}
|
||||
}
|
||||
|
||||
new LayerOverviewUtils().main(process.argv)
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import TiledFeatureSource from "../Logic/FeatureSource/TiledFeatureSource/TiledF
|
|||
import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSource";
|
||||
import * as readline from "readline";
|
||||
import ScriptUtils from "./ScriptUtils";
|
||||
import {Utils} from "../Utils";
|
||||
|
||||
/**
|
||||
* This script slices a big newline-delimeted geojson file into tiled geojson
|
||||
|
|
@ -103,16 +104,23 @@ async function main(args: string[]) {
|
|||
|
||||
let allFeatures: any [];
|
||||
if (inputFile.endsWith(".geojson")) {
|
||||
console.log("Detected geojson")
|
||||
allFeatures = await readFeaturesFromGeoJson(inputFile)
|
||||
} else {
|
||||
console.log("Loading as newline-delimited features")
|
||||
allFeatures = await readFeaturesFromLineDelimitedJsonFile(inputFile)
|
||||
}
|
||||
allFeatures = Utils.NoNull(allFeatures)
|
||||
|
||||
|
||||
console.log("Loaded all", allFeatures.length, "points")
|
||||
|
||||
const keysToRemove = ["STRAATNMID", "GEMEENTE", "POSTCODE"]
|
||||
for (const f of allFeatures) {
|
||||
if(f.properties === null){
|
||||
console.log("Got a feature without properties!", f)
|
||||
continue
|
||||
}
|
||||
for (const keyToRm of keysToRemove) {
|
||||
delete f.properties[keyToRm]
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue