midgard/mapcomplete
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Improve script to generate series

Browse source
This commit is contained in:
Pieter Vander Vennet 2023-01-09 20:38:05 +01:00
parent e8ae315d1d
commit 0a5f56d57a
6 changed files with 140 additions and 429039 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
Docs/Tools/cache_centerpoints.sh
View file

@ -1,13 +0,0 @@
#! /usr/bin/env bash
ts-node GenerateSeries.ts
# Move to the root of the repo
cd ../..
cd ../MapComplete-data
git pull
cd -
ts-node scripts/slice.ts ./Docs/Tools/centerpoints.geojson 8 ../MapComplete-data/mapcomplete-changes/
cd -
git add mapcomplete-changes/*
git commit -am "New changeset data"
git push

428969
Docs/Tools/centerpoints.geojson
View file

File diff suppressed because it is too large Load diff

2
UI/StatisticsGUI.ts
View file

@ -184,7 +184,7 @@ class StatisticsForOverviewFile extends Combine {
export default class StatisticsGUI extends VariableUiElement {
private static readonly homeUrl =
"https://raw.githubusercontent.com/pietervdvn/MapComplete/develop/Docs/Tools/stats/"
"https://raw.githubusercontent.com/pietervdvn/MapComplete-data/main/changeset-metadata/"
private static readonly stats_files = "file-overview.json"
constructor() {

3
package.json
View file

@ -31,12 +31,11 @@
"generate:layeroverview": "ts-node scripts/generateLayerOverview.ts",
"generate:licenses": "ts-node scripts/generateLicenseInfo.ts --no-fail",
"query:licenses": "ts-node scripts/generateLicenseInfo.ts --query",
"generate:report": "cd Docs/Tools && ./compileStats.sh && git commit . -m 'New statistics ands graphs' && git push",
"generate:contributor-list": "ts-node scripts/generateContributors.ts",
"generate:schemas": "ts2json-schema -p Models/ThemeConfig/Json/ -o Docs/Schemas/ -t tsconfig.json -R . -m \".*ConfigJson\" && ts-node scripts/fixSchemas.ts ",
"generate:service-worker": "tsc service-worker.ts && git_hash=$(git rev-parse HEAD) && sed -i'.bkp' \"s/GITHUB-COMMIT/$git_hash/\" service-worker.js && rm service-worker.js.bkp",
"optimize-images": "cd assets/generated/ && find -name '*.png' -exec optipng '{}' \\; && echo 'PNGs are optimized'",
"generate:graphs": "ts-node Docs/Tools/GenerateSeries.ts",
"generate:stats": "ts-node scripts/GenerateSeries.ts",
"reset:layeroverview": "echo {\\\"layers\\\":[], \\\"themes\\\":[]} > ./assets/generated/known_layers_and_themes.json && echo {\\\"layers\\\": []} > ./assets/generated/known_layers.json && rm -f ./assets/generated/layers/*.json && rm -f ./assets/generated/themes/*.json && npm run generate:layeroverview && ts-node scripts/generateLayerOverview.ts --force",
"generate": "mkdir -p ./assets/generated; npm run generate:licenses; npm run generate:images; npm run generate:charging-stations; npm run generate:translations; npm run reset:layeroverview; npm run generate:service-worker",
"generate:charging-stations": "cd ./assets/layers/charging_station && ts-node csvToJson.ts && cd -",

181
Docs/Tools/GenerateSeries.ts → scripts/GenerateSeries.ts
View file

@ -1,8 +1,11 @@
import { existsSync, mkdirSync, readdirSync, readFileSync, unlinkSync, writeFileSync } from "fs"
import ScriptUtils from "../../scripts/ScriptUtils"
import { Utils } from "../../Utils"
ScriptUtils.fixUtils()
import fs, { existsSync, readdirSync, readFileSync, unlinkSync, writeFileSync } from "fs"
import ScriptUtils from "./ScriptUtils"
import { Utils } from "../Utils"
import Script from "./Script"
import TiledFeatureSource from "../Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource"
import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSource"
import { GeoOperations } from "../Logic/GeoOperations"
import { Feature, Polygon } from "geojson"
class StatsDownloader {
private readonly urlTemplate =
@ -158,7 +161,7 @@ class StatsDownloader {
}
}
interface ChangeSetData {
interface ChangeSetData extends Feature<Polygon> {
id: number
type: "Feature"
geometry: {
@ -196,56 +199,130 @@ interface ChangeSetData {
}
}
async function main(): Promise<void> {
if (!existsSync("graphs")) {
mkdirSync("graphs")
class GenerateSeries extends Script {
constructor() {
super("Downloads metadata about changesets made by MapComplete from OsmCha")
}
const targetDir = "Docs/Tools/stats"
let year = 2020
let month = 5
let day = 1
if (!isNaN(Number(process.argv[2]))) {
year = Number(process.argv[2])
}
if (!isNaN(Number(process.argv[3]))) {
month = Number(process.argv[3])
}
async main(args: string[]): Promise<void> {
const targetDir = args[0] ?? "../MapComplete-data"
if (!isNaN(Number(process.argv[4]))) {
day = Number(process.argv[4])
}
do {
try {
await new StatsDownloader(targetDir).DownloadStats(year, month, day)
break
} catch (e) {
console.log(e)
}
} while (true)
const allPaths = readdirSync(targetDir).filter(
(p) => p.startsWith("stats.") && p.endsWith(".json")
)
let allFeatures: ChangeSetData[] = [].concat(
...allPaths.map(
(path) => JSON.parse(readFileSync("Docs/Tools/stats/" + path, "utf-8")).features
await this.downloadStatistics(targetDir + "/changeset-metadata")
await this.generateCenterPoints(
targetDir + "/changeset-metadata",
targetDir + "/mapcomplete-changes/",
{
zoomlevel: 8,
}
)
)
allFeatures = allFeatures.filter(
(f) =>
f?.properties !== undefined &&
(f.properties.editor === null ||
f.properties.editor.toLowerCase().startsWith("mapcomplete"))
)
allFeatures = allFeatures.filter((f) => f.properties.metadata?.theme !== "EMPTY CS")
if (process.argv.indexOf("--no-graphs") >= 0) {
return
}
const allFiles = readdirSync("Docs/Tools/stats").filter((p) => p.endsWith(".json"))
writeFileSync("Docs/Tools/stats/file-overview.json", JSON.stringify(allFiles))
private async downloadStatistics(targetDir: string) {
let year = 2020
let month = 5
let day = 1
if (!isNaN(Number(process.argv[2]))) {
year = Number(process.argv[2])
}
if (!isNaN(Number(process.argv[3]))) {
month = Number(process.argv[3])
}
if (!isNaN(Number(process.argv[4]))) {
day = Number(process.argv[4])
}
do {
try {
await new StatsDownloader(targetDir).DownloadStats(year, month, day)
break
} catch (e) {
console.log(e)
}
} while (true)
const allFiles = readdirSync(targetDir).filter((p) => p.endsWith(".json"))
writeFileSync(targetDir + "file-overview.json", JSON.stringify(allFiles))
}
private generateCenterPoints(
sourceDir: string,
targetDir: string,
options: {
zoomlevel: number
}
) {
const allPaths = readdirSync(sourceDir).filter(
(p) => p.startsWith("stats.") && p.endsWith(".json")
)
let allFeatures: ChangeSetData[] = [].concat(
...allPaths.map(
(path) => JSON.parse(readFileSync(sourceDir + "/" + path, "utf-8")).features
)
)
allFeatures = allFeatures.filter(
(f) =>
f?.properties !== undefined &&
(f.properties.editor === null ||
f.properties.editor.toLowerCase().startsWith("mapcomplete"))
)
allFeatures = allFeatures.filter(
(f) => f.geometry !== null && f.properties.metadata?.theme !== "EMPTY CS"
)
allFeatures = allFeatures.filter(
(f) =>
f?.properties !== undefined &&
(f.properties.editor === null ||
f.properties.editor.toLowerCase().startsWith("mapcomplete"))
)
allFeatures = allFeatures.filter((f) => f.properties.metadata?.theme !== "EMPTY CS")
const centerpoints = allFeatures.map((f) => GeoOperations.centerpoint(f))
console.log("Found", centerpoints.length, " changesets in total")
const path = `${targetDir}/all_centerpoints.geojson`
/*fs.writeFileSync(
path,
JSON.stringify(
{
type: "FeatureCollection",
features: centerpoints,
},
null,
" "
)
)//*/
TiledFeatureSource.createHierarchy(StaticFeatureSource.fromGeojson(centerpoints), {
minZoomLevel: options.zoomlevel,
maxZoomLevel: options.zoomlevel,
maxFeatureCount: Number.MAX_VALUE,
registerTile: (tile) => {
const path = `${targetDir}/tile_${tile.z}_${tile.x}_${tile.y}.geojson`
const features = tile.features.data.map((ff) => ff.feature)
features.forEach((f) => {
delete f.bbox
})
fs.writeFileSync(
path,
JSON.stringify(
{
type: "FeatureCollection",
features: features,
},
null,
" "
)
)
ScriptUtils.erasableLog(
"Written ",
path,
"which has ",
tile.features.data.length,
"features"
)
},
})
}
}
main().then((_) => console.log("All done!"))
new GenerateSeries().run()

11
scripts/ScriptUtils.ts
View file

@ -11,6 +11,13 @@ export default class ScriptUtils {
Utils.externalDownloadFunction = ScriptUtils.Download
}
/**
* Returns all files in a directory, recursively reads subdirectories.
* The returned paths include the path given and subdirectories.
*
* @param path
* @param maxDepth
*/
public static readDirRecSync(path, maxDepth = 999): string[] {
const result = []
if (maxDepth <= 0) {
@ -46,13 +53,13 @@ export default class ScriptUtils {
process.stdout.write("\r " + text.join(" ") + " \r")
}
public static sleep(ms) {
public static sleep(ms: number, text?: string) {
if (ms <= 0) {
process.stdout.write("\r \r")
return
}
return new Promise((resolve) => {
process.stdout.write("\r Sleeping for " + ms / 1000 + "s \r")
process.stdout.write("\r" + (text ?? "") + " Sleeping for " + ms / 1000 + "s \r")
setTimeout(resolve, 1000)
}).then(() => ScriptUtils.sleep(ms - 1000))
}