midgard/mapcomplete
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Velopark: some improvements to the scripts

Browse source
This commit is contained in:
Pieter Vander Vennet 2024-06-14 01:01:41 +02:00
parent 996a08e8d8
commit 48b35f5616
4 changed files with 104 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
scripts/velopark/compare.ts
View file

@ -65,6 +65,10 @@ class Compare extends Script {
async main(args: string[]): Promise<void> {
let [velopark, osm, key] = args
if(velopark === undefined || osm === undefined){
console.log("Needed argument: velopark.geojson osm.geojson [key]\nThe key is optional and will be `ref:velopark` by default\nUse overpass to get a geojson with ref:velopark")
return
}
key ??= "ref:velopark"
const veloparkData: FeatureCollection = JSON.parse(fs.readFileSync(velopark, "utf-8"))
const osmData: FeatureCollection = JSON.parse(fs.readFileSync(osm, "utf-8"))
@ -82,6 +86,7 @@ class Compare extends Script {
console.error("No velopark entry found for", veloId)
continue
}
console.log("Veloparking is", veloparking)
diffs.push(this.compare(veloId, parking, veloparking))
}
console.log(

63
scripts/velopark/diffToCsv.ts Normal file
View file

@ -0,0 +1,63 @@
import Script from "../Script"
import { readFileSync, writeFileSync } from "fs"
import { OsmId } from "../../src/Models/OsmFeature"
import { Utils } from "../../src/Utils"
interface DiffItem {
/**
* Velopark-id
*/
"ref": string,
"osmid": OsmId,
"distance": number,
"diffs": {
key: string,
/**
* The value in OpenStreetMap
* Might be undefined if OSM doesn't have an appropriate value
*/
osm?: string,
velopark: string | number } []
}
export class DiffToCsv extends Script {
constructor() {
super("Converts a 'report.diff' to a CSV file for people who prefer LibreOffice Calc (or other Spreadsheet Software)")
}
async main(args: string[]): Promise<void> {
const file = args[0] ?? "report_diff.json"
const json = <{diffs:DiffItem[], distanceBinds: number[]}> JSON.parse(readFileSync(file, "utf8"))
const diffs = json.diffs
const allKeys = Utils.Dedup(diffs.flatMap(item => item.diffs.map(d => d.key)))
allKeys.sort()
const header = ["osm_id","velopark_id", "distance",...allKeys.flatMap(k => ["osm:"+k, "velopark:"+k])]
const lines = [header]
for (const diffItem of diffs) {
const line = []
lines.push(line)
line.push(diffItem.osmid)
line.push(diffItem.ref)
line.push(diffItem.distance)
const d = diffItem.diffs
for (const k of allKeys) {
const found = d.find(i => i.key === k)
if(!found){
line.push("","")
continue
}
line.push(found.osm, found.velopark)
}
}
const path = "report_diff.csv"
writeFileSync(path,
lines.map(l => l.join(",")).join("\n")
,"utf8")
console.log("Written", path)
}
}
new DiffToCsv().run()

82
scripts/velopark/veloParkToGeojson.ts
View file

@ -23,9 +23,9 @@ class VeloParkToGeojson extends Script {
JSON.stringify(
extension === ".geojson"
? {
type: "FeatureCollection",
features,
}
type: "FeatureCollection",
features
}
: features,
null,
" "
@ -34,34 +34,14 @@ class VeloParkToGeojson extends Script {
console.log("Written", file, "(" + features.length, " features)")
}
public static sumProperties(data: object, addTo: Record<string, Set<string>>) {
delete data["@context"]
for (const k in data) {
if (k === "@graph") {
for (const obj of data["@graph"]) {
this.sumProperties(obj, addTo)
}
continue
}
if (addTo[k] === undefined) {
addTo[k] = new Set<string>()
}
addTo[k].add(data[k])
}
}
private static async downloadDataFor(
url: string,
allProperties: Record<string, Set<string>>
url: string
): Promise<Feature[]> {
const cachePath = "/home/pietervdvn/data/velopark_cache/" + url.replace(/[/:.]/g, "_")
if (!fs.existsSync(cachePath)) {
const data = await Utils.downloadJson(url)
fs.writeFileSync(cachePath, JSON.stringify(data), "utf-8")
console.log("Saved a backup to", cachePath)
const cachePath = "/home/pietervdvn/data/velopark_cache_refined/" + url.replace(/[/:.]/g, "_")
if (fs.existsSync(cachePath)) {
return JSON.parse(fs.readFileSync(cachePath, "utf8"))
}
this.sumProperties(JSON.parse(fs.readFileSync(cachePath, "utf-8")), allProperties)
console.log("Fetching data for", url)
const linkedData = await LinkedDataLoader.fetchVeloparkEntry(url)
const allVelopark: Feature[] = []
@ -70,9 +50,12 @@ class VeloParkToGeojson extends Script {
if (Object.keys(sectionInfo).length === 0) {
console.warn("No result for", url)
}
sectionInfo["ref:velopark"] = [sectionId ?? url]
if(!sectionInfo.geometry?.coordinates){
throw "Invalid properties!"
}
allVelopark.push(sectionInfo)
}
fs.writeFileSync(cachePath, JSON.stringify(allVelopark), "utf8")
return allVelopark
}
@ -85,20 +68,24 @@ class VeloParkToGeojson extends Script {
let failed = 0
console.log("Got", allVeloparkRaw.length, "items")
const allVelopark: Feature[] = []
const allProperties = {}
for (let i = 0; i < allVeloparkRaw.length; i++) {
const f = allVeloparkRaw[i]
console.log("Handling", i + "/" + allVeloparkRaw.length)
try {
const sections: Feature[] = await VeloParkToGeojson.downloadDataFor(
f.url,
allProperties
)
allVelopark.push(...sections)
} catch (e) {
console.error("Loading ", f.url, " failed due to", e)
failed++
}
const batchSize = 50
for (let i = 0; i < allVeloparkRaw.length; i += batchSize) {
await Promise.all(Utils.TimesT(batchSize, j => j).map(
async j => {
const f = allVeloparkRaw[i + j]
if (!f) {
return
}
try {
const sections: Feature[] = await VeloParkToGeojson.downloadDataFor(f.url)
allVelopark.push(...sections)
} catch (e) {
console.error("Loading ", f.url, " failed due to", e)
failed++
}
}
))
}
console.log(
"Fetching data done, got ",
@ -107,11 +94,6 @@ class VeloParkToGeojson extends Script {
failed
)
VeloParkToGeojson.exportGeojsonTo("velopark_all", allVelopark)
for (const k in allProperties) {
allProperties[k] = Array.from(allProperties[k])
}
fs.writeFileSync("all_properties_mashup.json", JSON.stringify(allProperties, null, " "))
return allVelopark
}
@ -158,7 +140,7 @@ class VeloParkToGeojson extends Script {
private static async createDiff(allVelopark: Feature[]) {
const bboxBelgium = new BBox([
[2.51357303225, 49.5294835476],
[6.15665815596, 51.4750237087],
[6.15665815596, 51.4750237087]
])
const alreadyLinkedQuery = new Overpass(
@ -201,7 +183,7 @@ class VeloParkToGeojson extends Script {
VeloParkToGeojson.exportExtraAmenities(allVelopark)
await VeloParkToGeojson.createDiff(allVelopark)
console.log(
"Use vite-node script/velopark/compare to compare the results and generate a diff file"
"Use vite-node scripts/velopark/compare.ts to compare the results and generate a diff file"
)
}
}

7
src/Logic/Web/LinkedDataLoader.ts
View file

@ -316,14 +316,16 @@ export default class LinkedDataLoader {
input: Record<string, Set<string>>
): Record<string, string[]> {
const output: Record<string, string[]> = {}
console.log("Input for patchVelopark:", input)
for (const k in input) {
output[k] = Array.from(input[k])
}
if (output["type"][0] === "https://data.velopark.be/openvelopark/terms#BicycleLocker") {
if (output["type"]?.[0] === "https://data.velopark.be/openvelopark/terms#BicycleLocker") {
output["bicycle_parking"] = ["lockers"]
}
if(output["type"] === undefined){
console.error("No type given for", output)
}
delete output["type"]
function on(key: string, applyF: (s: string) => string) {
@ -506,7 +508,6 @@ export default class LinkedDataLoader {
" ?parking a <http://schema.mobivoc.org/BicycleParkingStation>",
"?parking " + property + " " + (variable ?? "")
)
console.log("Fetching a velopark property gave", property, results)
return results
}