Improve script to generate series
This commit is contained in:
parent
e8ae315d1d
commit
0a5f56d57a
6 changed files with 140 additions and 429039 deletions
|
|
@ -1,13 +0,0 @@
|
|||
#! /usr/bin/env bash
|
||||
|
||||
ts-node GenerateSeries.ts
|
||||
# Move to the root of the repo
|
||||
cd ../..
|
||||
cd ../MapComplete-data
|
||||
git pull
|
||||
cd -
|
||||
ts-node scripts/slice.ts ./Docs/Tools/centerpoints.geojson 8 ../MapComplete-data/mapcomplete-changes/
|
||||
cd -
|
||||
git add mapcomplete-changes/*
|
||||
git commit -am "New changeset data"
|
||||
git push
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -184,7 +184,7 @@ class StatisticsForOverviewFile extends Combine {
|
|||
|
||||
export default class StatisticsGUI extends VariableUiElement {
|
||||
private static readonly homeUrl =
|
||||
"https://raw.githubusercontent.com/pietervdvn/MapComplete/develop/Docs/Tools/stats/"
|
||||
"https://raw.githubusercontent.com/pietervdvn/MapComplete-data/main/changeset-metadata/"
|
||||
private static readonly stats_files = "file-overview.json"
|
||||
|
||||
constructor() {
|
||||
|
|
|
|||
|
|
@ -31,12 +31,11 @@
|
|||
"generate:layeroverview": "ts-node scripts/generateLayerOverview.ts",
|
||||
"generate:licenses": "ts-node scripts/generateLicenseInfo.ts --no-fail",
|
||||
"query:licenses": "ts-node scripts/generateLicenseInfo.ts --query",
|
||||
"generate:report": "cd Docs/Tools && ./compileStats.sh && git commit . -m 'New statistics ands graphs' && git push",
|
||||
"generate:contributor-list": "ts-node scripts/generateContributors.ts",
|
||||
"generate:schemas": "ts2json-schema -p Models/ThemeConfig/Json/ -o Docs/Schemas/ -t tsconfig.json -R . -m \".*ConfigJson\" && ts-node scripts/fixSchemas.ts ",
|
||||
"generate:service-worker": "tsc service-worker.ts && git_hash=$(git rev-parse HEAD) && sed -i'.bkp' \"s/GITHUB-COMMIT/$git_hash/\" service-worker.js && rm service-worker.js.bkp",
|
||||
"optimize-images": "cd assets/generated/ && find -name '*.png' -exec optipng '{}' \\; && echo 'PNGs are optimized'",
|
||||
"generate:graphs": "ts-node Docs/Tools/GenerateSeries.ts",
|
||||
"generate:stats": "ts-node scripts/GenerateSeries.ts",
|
||||
"reset:layeroverview": "echo {\\\"layers\\\":[], \\\"themes\\\":[]} > ./assets/generated/known_layers_and_themes.json && echo {\\\"layers\\\": []} > ./assets/generated/known_layers.json && rm -f ./assets/generated/layers/*.json && rm -f ./assets/generated/themes/*.json && npm run generate:layeroverview && ts-node scripts/generateLayerOverview.ts --force",
|
||||
"generate": "mkdir -p ./assets/generated; npm run generate:licenses; npm run generate:images; npm run generate:charging-stations; npm run generate:translations; npm run reset:layeroverview; npm run generate:service-worker",
|
||||
"generate:charging-stations": "cd ./assets/layers/charging_station && ts-node csvToJson.ts && cd -",
|
||||
|
|
|
|||
|
|
@ -1,8 +1,11 @@
|
|||
import { existsSync, mkdirSync, readdirSync, readFileSync, unlinkSync, writeFileSync } from "fs"
|
||||
import ScriptUtils from "../../scripts/ScriptUtils"
|
||||
import { Utils } from "../../Utils"
|
||||
|
||||
ScriptUtils.fixUtils()
|
||||
import fs, { existsSync, readdirSync, readFileSync, unlinkSync, writeFileSync } from "fs"
|
||||
import ScriptUtils from "./ScriptUtils"
|
||||
import { Utils } from "../Utils"
|
||||
import Script from "./Script"
|
||||
import TiledFeatureSource from "../Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource"
|
||||
import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSource"
|
||||
import { GeoOperations } from "../Logic/GeoOperations"
|
||||
import { Feature, Polygon } from "geojson"
|
||||
|
||||
class StatsDownloader {
|
||||
private readonly urlTemplate =
|
||||
|
|
@ -158,7 +161,7 @@ class StatsDownloader {
|
|||
}
|
||||
}
|
||||
|
||||
interface ChangeSetData {
|
||||
interface ChangeSetData extends Feature<Polygon> {
|
||||
id: number
|
||||
type: "Feature"
|
||||
geometry: {
|
||||
|
|
@ -196,12 +199,25 @@ interface ChangeSetData {
|
|||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
if (!existsSync("graphs")) {
|
||||
mkdirSync("graphs")
|
||||
class GenerateSeries extends Script {
|
||||
constructor() {
|
||||
super("Downloads metadata about changesets made by MapComplete from OsmCha")
|
||||
}
|
||||
|
||||
const targetDir = "Docs/Tools/stats"
|
||||
async main(args: string[]): Promise<void> {
|
||||
const targetDir = args[0] ?? "../MapComplete-data"
|
||||
|
||||
await this.downloadStatistics(targetDir + "/changeset-metadata")
|
||||
await this.generateCenterPoints(
|
||||
targetDir + "/changeset-metadata",
|
||||
targetDir + "/mapcomplete-changes/",
|
||||
{
|
||||
zoomlevel: 8,
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
private async downloadStatistics(targetDir: string) {
|
||||
let year = 2020
|
||||
let month = 5
|
||||
let day = 1
|
||||
|
|
@ -224,12 +240,24 @@ async function main(): Promise<void> {
|
|||
console.log(e)
|
||||
}
|
||||
} while (true)
|
||||
const allPaths = readdirSync(targetDir).filter(
|
||||
|
||||
const allFiles = readdirSync(targetDir).filter((p) => p.endsWith(".json"))
|
||||
writeFileSync(targetDir + "file-overview.json", JSON.stringify(allFiles))
|
||||
}
|
||||
|
||||
private generateCenterPoints(
|
||||
sourceDir: string,
|
||||
targetDir: string,
|
||||
options: {
|
||||
zoomlevel: number
|
||||
}
|
||||
) {
|
||||
const allPaths = readdirSync(sourceDir).filter(
|
||||
(p) => p.startsWith("stats.") && p.endsWith(".json")
|
||||
)
|
||||
let allFeatures: ChangeSetData[] = [].concat(
|
||||
...allPaths.map(
|
||||
(path) => JSON.parse(readFileSync("Docs/Tools/stats/" + path, "utf-8")).features
|
||||
(path) => JSON.parse(readFileSync(sourceDir + "/" + path, "utf-8")).features
|
||||
)
|
||||
)
|
||||
allFeatures = allFeatures.filter(
|
||||
|
|
@ -239,13 +267,62 @@ async function main(): Promise<void> {
|
|||
f.properties.editor.toLowerCase().startsWith("mapcomplete"))
|
||||
)
|
||||
|
||||
allFeatures = allFeatures.filter(
|
||||
(f) => f.geometry !== null && f.properties.metadata?.theme !== "EMPTY CS"
|
||||
)
|
||||
allFeatures = allFeatures.filter(
|
||||
(f) =>
|
||||
f?.properties !== undefined &&
|
||||
(f.properties.editor === null ||
|
||||
f.properties.editor.toLowerCase().startsWith("mapcomplete"))
|
||||
)
|
||||
|
||||
allFeatures = allFeatures.filter((f) => f.properties.metadata?.theme !== "EMPTY CS")
|
||||
|
||||
if (process.argv.indexOf("--no-graphs") >= 0) {
|
||||
return
|
||||
const centerpoints = allFeatures.map((f) => GeoOperations.centerpoint(f))
|
||||
console.log("Found", centerpoints.length, " changesets in total")
|
||||
const path = `${targetDir}/all_centerpoints.geojson`
|
||||
/*fs.writeFileSync(
|
||||
path,
|
||||
JSON.stringify(
|
||||
{
|
||||
type: "FeatureCollection",
|
||||
features: centerpoints,
|
||||
},
|
||||
null,
|
||||
" "
|
||||
)
|
||||
)//*/
|
||||
TiledFeatureSource.createHierarchy(StaticFeatureSource.fromGeojson(centerpoints), {
|
||||
minZoomLevel: options.zoomlevel,
|
||||
maxZoomLevel: options.zoomlevel,
|
||||
maxFeatureCount: Number.MAX_VALUE,
|
||||
registerTile: (tile) => {
|
||||
const path = `${targetDir}/tile_${tile.z}_${tile.x}_${tile.y}.geojson`
|
||||
const features = tile.features.data.map((ff) => ff.feature)
|
||||
features.forEach((f) => {
|
||||
delete f.bbox
|
||||
})
|
||||
fs.writeFileSync(
|
||||
path,
|
||||
JSON.stringify(
|
||||
{
|
||||
type: "FeatureCollection",
|
||||
features: features,
|
||||
},
|
||||
null,
|
||||
" "
|
||||
)
|
||||
)
|
||||
ScriptUtils.erasableLog(
|
||||
"Written ",
|
||||
path,
|
||||
"which has ",
|
||||
tile.features.data.length,
|
||||
"features"
|
||||
)
|
||||
},
|
||||
})
|
||||
}
|
||||
const allFiles = readdirSync("Docs/Tools/stats").filter((p) => p.endsWith(".json"))
|
||||
writeFileSync("Docs/Tools/stats/file-overview.json", JSON.stringify(allFiles))
|
||||
}
|
||||
|
||||
main().then((_) => console.log("All done!"))
|
||||
new GenerateSeries().run()
|
||||
|
|
@ -11,6 +11,13 @@ export default class ScriptUtils {
|
|||
Utils.externalDownloadFunction = ScriptUtils.Download
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns all files in a directory, recursively reads subdirectories.
|
||||
* The returned paths include the path given and subdirectories.
|
||||
*
|
||||
* @param path
|
||||
* @param maxDepth
|
||||
*/
|
||||
public static readDirRecSync(path, maxDepth = 999): string[] {
|
||||
const result = []
|
||||
if (maxDepth <= 0) {
|
||||
|
|
@ -46,13 +53,13 @@ export default class ScriptUtils {
|
|||
process.stdout.write("\r " + text.join(" ") + " \r")
|
||||
}
|
||||
|
||||
public static sleep(ms) {
|
||||
public static sleep(ms: number, text?: string) {
|
||||
if (ms <= 0) {
|
||||
process.stdout.write("\r \r")
|
||||
return
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
process.stdout.write("\r Sleeping for " + ms / 1000 + "s \r")
|
||||
process.stdout.write("\r" + (text ?? "") + " Sleeping for " + ms / 1000 + "s \r")
|
||||
setTimeout(resolve, 1000)
|
||||
}).then(() => ScriptUtils.sleep(ms - 1000))
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue