Velopark: some improvements to the scripts

scripts/velopark/compare.ts

@@ -65,6 +65,10 @@ class Compare extends Script {
 
     async main(args: string[]): Promise<void> {
         let [velopark, osm, key] = args
+        if(velopark === undefined || osm === undefined){
+            console.log("Needed argument: velopark.geojson osm.geojson [key]\nThe key is optional and will be `ref:velopark` by default\nUse overpass to get a geojson with ref:velopark")
+            return
+        }
         key ??= "ref:velopark"
         const veloparkData: FeatureCollection = JSON.parse(fs.readFileSync(velopark, "utf-8"))
         const osmData: FeatureCollection = JSON.parse(fs.readFileSync(osm, "utf-8"))
@@ -82,6 +86,7 @@ class Compare extends Script {
                 console.error("No velopark entry found for", veloId)
                 continue
             }
+            console.log("Veloparking is", veloparking)
             diffs.push(this.compare(veloId, parking, veloparking))
         }
         console.log(

scripts/velopark/diffToCsv.ts

@@ -0,0 +1,63 @@
+import Script from "../Script"
+import { readFileSync, writeFileSync } from "fs"
+import { OsmId } from "../../src/Models/OsmFeature"
+import { Utils } from "../../src/Utils"
+
+interface DiffItem {
+    /**
+     * Velopark-id
+     */
+    "ref": string,
+    "osmid": OsmId,
+    "distance": number,
+    "diffs": {
+        key: string,
+        /**
+         * The value in OpenStreetMap
+         * Might be undefined if OSM doesn't have an appropriate value
+         */
+        osm?: string,
+        velopark: string | number } []
+}
+
+export class DiffToCsv extends Script {
+    constructor() {
+        super("Converts a 'report.diff' to a CSV file for people who prefer LibreOffice Calc (or other Spreadsheet Software)")
+    }
+
+    async main(args: string[]): Promise<void> {
+        const file = args[0] ?? "report_diff.json"
+        const json = <{diffs:DiffItem[], distanceBinds: number[]}> JSON.parse(readFileSync(file, "utf8"))
+        const diffs = json.diffs
+        const allKeys = Utils.Dedup(diffs.flatMap(item => item.diffs.map(d => d.key)))
+        allKeys.sort()
+
+        const header = ["osm_id","velopark_id", "distance",...allKeys.flatMap(k => ["osm:"+k, "velopark:"+k])]
+        const lines = [header]
+        for (const diffItem of diffs) {
+            const line = []
+            lines.push(line)
+            line.push(diffItem.osmid)
+            line.push(diffItem.ref)
+            line.push(diffItem.distance)
+
+            const d = diffItem.diffs
+            for (const k of allKeys) {
+                const found = d.find(i => i.key === k)
+                if(!found){
+                    line.push("","")
+                    continue
+                }
+                line.push(found.osm, found.velopark)
+            }
+
+        }
+        const path = "report_diff.csv"
+        writeFileSync(path,
+            lines.map(l => l.join(",")).join("\n")
+            ,"utf8")
+        console.log("Written", path)
+    }
+}
+
+new DiffToCsv().run()

scripts/velopark/veloParkToGeojson.ts

@@ -24,7 +24,7 @@ class VeloParkToGeojson extends Script {
                 extension === ".geojson"
                     ? {
                         type: "FeatureCollection",
-                          features,
+                        features
                     }
                     : features,
                 null,
@@ -34,34 +34,14 @@ class VeloParkToGeojson extends Script {
         console.log("Written", file, "(" + features.length, " features)")
     }
 
-    public static sumProperties(data: object, addTo: Record<string, Set<string>>) {
-        delete data["@context"]
-        for (const k in data) {
-            if (k === "@graph") {
-                for (const obj of data["@graph"]) {
-                    this.sumProperties(obj, addTo)
-                }
-                continue
-            }
-            if (addTo[k] === undefined) {
-                addTo[k] = new Set<string>()
-            }
-            addTo[k].add(data[k])
-        }
-    }
-
     private static async downloadDataFor(
-        url: string,
+        url: string
-        allProperties: Record<string, Set<string>>
     ): Promise<Feature[]> {
-        const cachePath = "/home/pietervdvn/data/velopark_cache/" + url.replace(/[/:.]/g, "_")
+        const cachePath = "/home/pietervdvn/data/velopark_cache_refined/" + url.replace(/[/:.]/g, "_")
-        if (!fs.existsSync(cachePath)) {
+        if (fs.existsSync(cachePath)) {
-            const data = await Utils.downloadJson(url)
+            return JSON.parse(fs.readFileSync(cachePath, "utf8"))
-            fs.writeFileSync(cachePath, JSON.stringify(data), "utf-8")
-            console.log("Saved a backup to", cachePath)
         }
-
+        console.log("Fetching data for", url)
-        this.sumProperties(JSON.parse(fs.readFileSync(cachePath, "utf-8")), allProperties)
 
         const linkedData = await LinkedDataLoader.fetchVeloparkEntry(url)
         const allVelopark: Feature[] = []
@@ -70,9 +50,12 @@ class VeloParkToGeojson extends Script {
             if (Object.keys(sectionInfo).length === 0) {
                 console.warn("No result for", url)
             }
-            sectionInfo["ref:velopark"] = [sectionId ?? url]
+            if(!sectionInfo.geometry?.coordinates){
+                throw "Invalid properties!"
+            }
             allVelopark.push(sectionInfo)
         }
+        fs.writeFileSync(cachePath, JSON.stringify(allVelopark), "utf8")
         return allVelopark
     }
 
@@ -85,21 +68,25 @@ class VeloParkToGeojson extends Script {
         let failed = 0
         console.log("Got", allVeloparkRaw.length, "items")
         const allVelopark: Feature[] = []
-        const allProperties = {}
+        const batchSize = 50
-        for (let i = 0; i < allVeloparkRaw.length; i++) {
+        for (let i = 0; i < allVeloparkRaw.length; i += batchSize) {
-            const f = allVeloparkRaw[i]
+            await Promise.all(Utils.TimesT(batchSize, j => j).map(
-            console.log("Handling", i + "/" + allVeloparkRaw.length)
+                async j => {
+                    const f = allVeloparkRaw[i + j]
+                    if (!f) {
+                        return
+                    }
                     try {
-                const sections: Feature[] = await VeloParkToGeojson.downloadDataFor(
+                        const sections: Feature[] = await VeloParkToGeojson.downloadDataFor(f.url)
-                    f.url,
-                    allProperties
-                )
                         allVelopark.push(...sections)
                     } catch (e) {
                         console.error("Loading ", f.url, " failed due to", e)
                         failed++
                     }
                 }
+            ))
+
+        }
         console.log(
             "Fetching data done, got ",
             allVelopark.length + "/" + allVeloparkRaw.length,
@@ -107,11 +94,6 @@ class VeloParkToGeojson extends Script {
             failed
         )
         VeloParkToGeojson.exportGeojsonTo("velopark_all", allVelopark)
-        for (const k in allProperties) {
-            allProperties[k] = Array.from(allProperties[k])
-        }
-
-        fs.writeFileSync("all_properties_mashup.json", JSON.stringify(allProperties, null, "  "))
 
         return allVelopark
     }
@@ -158,7 +140,7 @@ class VeloParkToGeojson extends Script {
     private static async createDiff(allVelopark: Feature[]) {
         const bboxBelgium = new BBox([
             [2.51357303225, 49.5294835476],
-            [6.15665815596, 51.4750237087],
+            [6.15665815596, 51.4750237087]
         ])
 
         const alreadyLinkedQuery = new Overpass(
@@ -201,7 +183,7 @@ class VeloParkToGeojson extends Script {
         VeloParkToGeojson.exportExtraAmenities(allVelopark)
         await VeloParkToGeojson.createDiff(allVelopark)
         console.log(
-            "Use vite-node script/velopark/compare to compare the results and generate a diff file"
+            "Use vite-node scripts/velopark/compare.ts to compare the results and generate a diff file"
         )
     }
 }

src/Logic/Web/LinkedDataLoader.ts

@@ -316,14 +316,16 @@ export default class LinkedDataLoader {
         input: Record<string, Set<string>>
     ): Record<string, string[]> {
         const output: Record<string, string[]> = {}
-        console.log("Input for patchVelopark:", input)
         for (const k in input) {
             output[k] = Array.from(input[k])
         }
 
-        if (output["type"][0] === "https://data.velopark.be/openvelopark/terms#BicycleLocker") {
+        if (output["type"]?.[0] === "https://data.velopark.be/openvelopark/terms#BicycleLocker") {
             output["bicycle_parking"] = ["lockers"]
         }
+        if(output["type"] === undefined){
+            console.error("No type given for", output)
+        }
         delete output["type"]
 
         function on(key: string, applyF: (s: string) => string) {
@@ -506,7 +508,6 @@ export default class LinkedDataLoader {
             "  ?parking a <http://schema.mobivoc.org/BicycleParkingStation>",
             "?parking " + property + " " + (variable ?? "")
         )
-        console.log("Fetching a velopark property gave", property, results)
         return results
     }