midgard/mapcomplete
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Improve slice script, formatting

Browse source
This commit is contained in:
pietervdvn 2022-01-16 01:59:06 +01:00
parent cce6a9e832
commit ccb548816f
3 changed files with 88 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

101
scripts/generateCache.ts
View file

@ -24,6 +24,7 @@ import {GeoOperations} from "../Logic/GeoOperations";
import SimpleMetaTaggers from "../Logic/SimpleMetaTagger";
import FilteringFeatureSource from "../Logic/FeatureSource/Sources/FilteringFeatureSource";
import Loc from "../Models/Loc";
ScriptUtils.fixUtils()
function createOverpassObject(theme: LayoutConfig, relationTracker: RelationsTracker, backend: string) {
@ -166,7 +167,7 @@ function loadAllTiles(targetdir: string, r: TileRange, theme: LayoutConfig, extr
// Create and save the geojson file - which is the main chunk of the data
const geojson = OsmToGeoJson.default(rawOsm);
console.log(" which as",geojson.features.length, "features")
console.log(" which as", geojson.features.length, "features")
allFeatures.push(...geojson.features)
}
@ -180,23 +181,24 @@ function loadAllTiles(targetdir: string, r: TileRange, theme: LayoutConfig, extr
function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relationsTracker: RelationsTracker, targetdir: string, pointsOnlyLayers: string[]) {
const skippedLayers = new Set<string>()
const indexedFeatures : Map<string, any> = new Map<string, any>()
const indexedFeatures: Map<string, any> = new Map<string, any>()
let indexisBuilt = false;
function buildIndex(){
function buildIndex() {
for (const ff of allFeatures.features.data) {
const f = ff.feature
indexedFeatures.set(f.properties.id, f)
}
indexisBuilt = true;
}
function getFeatureById(id){
if(!indexisBuilt){
function getFeatureById(id) {
if (!indexisBuilt) {
buildIndex()
}
return indexedFeatures.get(id)
}
async function handleLayer(source: FeatureSourceForLayer) {
const layer = source.layer.layerDef;
const targetZoomLevel = layer.source.geojsonZoomLevel ?? 0
@ -225,11 +227,10 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations
includeDates: false,
includeNonDates: true
});
while (SimpleMetaTaggers.country.runningTasks.size > 0) {
console.log("Still waiting for ", SimpleMetaTaggers.country.runningTasks.size," features which don't have a country yet")
console.log("Still waiting for ", SimpleMetaTaggers.country.runningTasks.size, " features which don't have a country yet")
await ScriptUtils.sleep(1)
}
@ -246,18 +247,18 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations
if (tile.features.data.length === 0) {
return
}
const filteredTile = new FilteringFeatureSource({
locationControl: new UIEventSource<Loc>(undefined),
allElements: undefined,
selectedElement: new UIEventSource<any>(undefined)
},
locationControl: new UIEventSource<Loc>(undefined),
allElements: undefined,
selectedElement: new UIEventSource<any>(undefined)
},
tileIndex,
tile,
new UIEventSource<any>(undefined)
)
)
console.log("Tile "+layer.id+"."+tileIndex+" contains "+filteredTile.features.data.length+" features after filtering ("+tile.features.data.length+") features before")
console.log("Tile " + layer.id + "." + tileIndex + " contains " + filteredTile.features.data.length + " features after filtering (" + tile.features.data.length + ") features before")
if (filteredTile.features.data.length === 0) {
return
}
@ -266,22 +267,22 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations
for (const feature of filteredTile.features.data) {
// Some cleanup
delete feature.feature["bbox"]
if(tile.layer.layerDef.calculatedTags !== undefined){
// Evaluate all the calculated tags strictly
const calculatedTagKeys = tile.layer.layerDef.calculatedTags.map(ct => ct[0])
featureCount++
for (const calculatedTagKey of calculatedTagKeys) {
const strict = feature.feature.properties[calculatedTagKey]
feature.feature.properties[calculatedTagKey] =strict
strictlyCalculated ++;
if(strictlyCalculated % 100 === 0){
console.log("Strictly calculated ", strictlyCalculated, "values for tile",tileIndex,": now at ", featureCount,"/",filteredTile.features.data.length, "examle value: ", strict)
if (tile.layer.layerDef.calculatedTags !== undefined) {
// Evaluate all the calculated tags strictly
const calculatedTagKeys = tile.layer.layerDef.calculatedTags.map(ct => ct[0])
featureCount++
for (const calculatedTagKey of calculatedTagKeys) {
const strict = feature.feature.properties[calculatedTagKey]
feature.feature.properties[calculatedTagKey] = strict
strictlyCalculated++;
if (strictlyCalculated % 100 === 0) {
console.log("Strictly calculated ", strictlyCalculated, "values for tile", tileIndex, ": now at ", featureCount, "/", filteredTile.features.data.length, "examle value: ", strict)
}
}
}
}
}
// Lets save this tile!
const [z, x, y] = Tiles.tile_from_index(tileIndex)
@ -293,7 +294,7 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations
type: "FeatureCollection",
features: filteredTile.features.data.map(f => f.feature)
}, null, " "))
console.log("Written tile", targetPath,"with", filteredTile.features.data.length)
console.log("Written tile", targetPath, "with", filteredTile.features.data.length)
}
})
@ -315,16 +316,16 @@ function sliceToTiles(allFeatures: FeatureSource, theme: LayoutConfig, relations
if (pointsOnlyLayers.indexOf(layer.id) >= 0) {
const filtered = new FilteringFeatureSource({
locationControl: new UIEventSource<Loc>(undefined),
locationControl: new UIEventSource<Loc>(undefined),
allElements: undefined,
selectedElement: new UIEventSource<any>(undefined)
},
Tiles.tile_index(0,0,0),
Tiles.tile_index(0, 0, 0),
source,
new UIEventSource<any>(undefined)
)
const features = filtered.features.data.map(f => f.feature)
const points = features.map(feature => GeoOperations.centerpoint(feature))
console.log("Writing points overview for ", layerId)
const targetPath = targetdir + "_" + layerId + "_points.geojson"
@ -375,8 +376,6 @@ async function main(args: string[]) {
const lat1 = Number(args[5])
const lon1 = Number(args[6])
const tileRange = Tiles.TileRangeBetween(zoomlevel, lat0, lon0, lat1, lon1)
@ -397,29 +396,29 @@ async function main(args: string[]) {
let generatePointLayersFor = []
if (args[7] == "--generate-point-overview") {
if(args[8] === undefined){
if (args[8] === undefined) {
throw "--generate-point-overview needs a list of layers to generate the overview for (or * for all)"
}else if (args[8] === '*'){
} else if (args[8] === '*') {
generatePointLayersFor = theme.layers.map(l => l.id)
}else{
} else {
generatePointLayersFor = args[8].split(",")
}
console.log("Also generating a point overview for layers ", generatePointLayersFor.join(","))
}
{
const index = args.indexOf("--force-zoom-level")
if(index >= 0){
const forcedZoomLevel = Number(args[index + 1])
for (const layer of theme.layers) {
layer.source.geojsonSource = "https://127.0.0.1/cache_{layer}_{z}_{x}_{y}.geojson"
layer.source.isOsmCacheLayer = true
layer.source.geojsonZoomLevel = forcedZoomLevel
const index = args.indexOf("--force-zoom-level")
if (index >= 0) {
const forcedZoomLevel = Number(args[index + 1])
for (const layer of theme.layers) {
layer.source.geojsonSource = "https://127.0.0.1/cache_{layer}_{z}_{x}_{y}.geojson"
layer.source.isOsmCacheLayer = true
layer.source.geojsonZoomLevel = forcedZoomLevel
}
}
}
}
const relationTracker = new RelationsTracker()
let failed = 0;

52
scripts/generateLayerOverview.ts
View file

@ -6,7 +6,8 @@ import {LayerConfigJson} from "../Models/ThemeConfig/Json/LayerConfigJson";
import Constants from "../Models/Constants";
import {
DesugaringContext,
PrepareLayer, PrepareTheme,
PrepareLayer,
PrepareTheme,
ValidateLayer,
ValidateThemeAndLayers
} from "../Models/ThemeConfig/Conversion/LegacyJsonConvert";
@ -80,7 +81,7 @@ class LayerOverviewUtils {
dict.set(key, <TagRenderingConfigJson>questions[key])
}
for (const key in icons["default"]) {
if(typeof icons[key] !== "object"){
if (typeof icons[key] !== "object") {
continue
}
icons[key].id = key;
@ -94,6 +95,32 @@ class LayerOverviewUtils {
return dict;
}
main(_: string[]) {
const licensePaths = new Set<string>()
for (const i in licenses) {
licensePaths.add(licenses[i].path)
}
const sharedLayers = this.buildLayerIndex(licensePaths);
const sharedThemes = this.buildThemeIndex(licensePaths, sharedLayers)
writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({
"layers": Array.from(sharedLayers.values()),
"themes": Array.from(sharedThemes.values())
}))
writeFileSync("./assets/generated/known_layers.json", JSON.stringify(Array.from(sharedLayers.values())))
writeFileSync('./assets/themes/mapcomplete-changes/icons-mapping.txt', JSON.stringify(
Array.from(sharedThemes.values()).map(th => ({
if: "theme=" + th.id,
then: th.icon
}))
))
}
private buildLayerIndex(knownImagePaths: Set<string>): Map<string, LayerConfigJson> {
// First, we expand and validate all builtin layers. These are written to assets/generated/layers
@ -126,7 +153,6 @@ class LayerOverviewUtils {
return sharedLayers;
}
private buildThemeIndex(knownImagePaths: Set<string>, sharedLayers: Map<string, LayerConfigJson>): Map<string, LayoutConfigJson> {
console.log(" ---------- VALIDATING BUILTIN THEMES ---------")
const themeFiles = ScriptUtils.getThemeFiles();
@ -139,7 +165,7 @@ class LayerOverviewUtils {
for (const themeInfo of themeFiles) {
let themeFile = themeInfo.parsed
const themePath = themeInfo.path
themeFile = new PrepareTheme().convertStrict(convertState, themeFile, themePath)
new ValidateThemeAndLayers(knownImagePaths, themePath, true)
@ -160,24 +186,6 @@ class LayerOverviewUtils {
return fixed;
}
main(_: string[]) {
const licensePaths = new Set<string>()
for (const i in licenses) {
licensePaths.add(licenses[i].path)
}
const sharedLayers = this.buildLayerIndex(licensePaths);
const sharedThemes = this.buildThemeIndex(licensePaths, sharedLayers)
writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({
"layers": Array.from(sharedLayers.values()),
"themes": Array.from(sharedThemes.values())
}))
writeFileSync("./assets/generated/known_layers.json", JSON.stringify(Array.from(sharedLayers.values())))
}
}
new LayerOverviewUtils().main(process.argv)

8
scripts/slice.ts
View file

@ -3,6 +3,7 @@ import TiledFeatureSource from "../Logic/FeatureSource/TiledFeatureSource/TiledF
import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSource";
import * as readline from "readline";
import ScriptUtils from "./ScriptUtils";
import {Utils} from "../Utils";
/**
* This script slices a big newline-delimeted geojson file into tiled geojson
@ -103,16 +104,23 @@ async function main(args: string[]) {
let allFeatures: any [];
if (inputFile.endsWith(".geojson")) {
console.log("Detected geojson")
allFeatures = await readFeaturesFromGeoJson(inputFile)
} else {
console.log("Loading as newline-delimited features")
allFeatures = await readFeaturesFromLineDelimitedJsonFile(inputFile)
}
allFeatures = Utils.NoNull(allFeatures)
console.log("Loaded all", allFeatures.length, "points")
const keysToRemove = ["STRAATNMID", "GEMEENTE", "POSTCODE"]
for (const f of allFeatures) {
if(f.properties === null){
console.log("Got a feature without properties!", f)
continue
}
for (const keyToRm of keysToRemove) {
delete f.properties[keyToRm]
}