mapcomplete/scripts/velopark/veloParkToGeojson.ts

import Script from "../Script"
import fs from "fs"
import LinkedDataLoader from "../../src/Logic/Web/LinkedDataLoader"
import { Utils } from "../../src/Utils"
import { Feature } from "geojson"
import { BBox } from "../../src/Logic/BBox"
import { Overpass } from "../../src/Logic/Osm/Overpass"
import { RegexTag } from "../../src/Logic/Tags/RegexTag"
import { ImmutableStore } from "../../src/Logic/UIEventSource"
import Constants from "../../src/Models/Constants"

class VeloParkToGeojson extends Script {
    constructor() {
        super(
            "Downloads the latest Velopark data and converts it to a geojson, which will be saved at the current directory"
        )
    }

    private static exportGeojsonTo(filename: string, features: Feature[], extension = ".geojson") {
        const file = filename + "_" + /*new Date().toISOString() + */extension
        fs.writeFileSync(file,
            JSON.stringify(
                extension === ".geojson" ? {
                    type: "FeatureCollection",
                    features
                } : features,
                null,
                "    "
            )
        )
        console.log("Written", file, "(" + features.length, " features)")
    }

    public static sumProperties(data: object, addTo: Record<string, Set<string>>) {
        delete data["@context"]
        for (const k in data) {
            if (k === "@graph") {
                for (const obj of data["@graph"]) {
                    this.sumProperties(obj, addTo)
                }
                continue
            }
            if (addTo[k] === undefined) {
                addTo[k] = new Set<string>()
            }
            addTo[k].add(data[k])
        }

    }

    private static async downloadDataFor(url: string, allProperties: Record<string, Set<string>>): Promise<Feature[]> {
        const cachePath = "/home/pietervdvn/data/velopark_cache/" + url.replace(/[/:.]/g, "_")
        if (!fs.existsSync(cachePath)) {
            const data = await Utils.downloadJson(url)
            fs.writeFileSync(cachePath, JSON.stringify(data), "utf-8")
            console.log("Saved a backup to", cachePath)
        }

        this.sumProperties(JSON.parse(fs.readFileSync(cachePath, "utf-8")), allProperties)

        const linkedData = await LinkedDataLoader.fetchVeloparkEntry(url)
        const allVelopark: Feature[] = []
        for (const sectionId in linkedData) {
            const sectionInfo = linkedData[sectionId]
            if (Object.keys(sectionInfo).length === 0) {
                console.warn("No result for", url)
            }
            sectionInfo["ref:velopark"] = [sectionId ?? url]
            allVelopark.push(sectionInfo)
        }
        return allVelopark
    }

    private static async downloadData() {
        console.log("Downloading velopark data")
        // Download data for NIS-code 1000. 1000 means: all of belgium
        const url = "https://www.velopark.be/api/parkings/1000"
        const allVeloparkRaw: { url: string }[] = <{url: string}[]> await Utils.downloadJson(url)

        let failed = 0
        console.log("Got", allVeloparkRaw.length, "items")
        const allVelopark: Feature[] = []
        const allProperties = {}
        for (let i = 0; i < allVeloparkRaw.length; i++) {
            const f = allVeloparkRaw[i]
            console.log("Handling", i + "/" + allVeloparkRaw.length)
            try {
                const sections: Feature[] = await VeloParkToGeojson.downloadDataFor(f.url, allProperties)
                allVelopark.push(...sections)
            } catch (e) {
                console.error("Loading ", f.url, " failed due to", e)
                failed++

            }
        }
        console.log("Fetching data done, got ", allVelopark.length + "/" + allVeloparkRaw.length, "failed:", failed)
        VeloParkToGeojson.exportGeojsonTo("velopark_all", allVelopark)
        for (const k in allProperties) {
            allProperties[k] = Array.from(allProperties[k])
        }

        fs.writeFileSync("all_properties_mashup.json", JSON.stringify(allProperties, null, "  "))

        return allVelopark
    }

    private static loadFromFile(maxCacheAgeSeconds = 24*60*60): Feature[] | null {
        const path = "velopark_all.geojson"
        if(!fs.existsSync(path)){
            return null
        }
        // Millis since epoch
        const mtime : number = fs.statSync(path).mtime.getTime()
        const stalenessSeconds = (new Date().getTime() - mtime) / 1000
        if(stalenessSeconds > maxCacheAgeSeconds){
            return null
        }

        return JSON.parse(fs.readFileSync(path, "utf-8")).features
    }

    private static exportExtraAmenities(allVelopark: Feature[]) {
        const amenities: Record<string, Feature[]> = {}

        for (const bikeparking of allVelopark) {
            const props = bikeparking.properties
            if (!props["fixme_nearby_amenity"]) {
                continue
            }
            if (props["fixme_nearby_amenity"]?.endsWith("CameraSurveillance")) {
                delete props["fixme_nearby_amenity"]
                continue
            }
            const amenity = props["fixme_nearby_amenity"].split("#")[1]
            if (!amenities[amenity]) {
                amenities[amenity] = []
            }
            amenities[amenity].push(bikeparking)
        }

        for (const k in amenities) {
            this.exportGeojsonTo("velopark_amenity_" + k + ".geojson", amenities[k])
        }
    }

    private static async createDiff(allVelopark: Feature[]) {
        const bboxBelgium = new BBox([
            [2.51357303225, 49.5294835476],
            [6.15665815596, 51.4750237087]
        ])

        const alreadyLinkedQuery = new Overpass(
            new RegexTag("ref:velopark", /.+/),
            [],
            Constants.defaultOverpassUrls[0],
            new ImmutableStore(60 * 5),
            false
        )
        const alreadyLinkedFeatures = (await alreadyLinkedQuery.queryGeoJson(bboxBelgium))[0]
        const seenIds = new Set<string>(
            alreadyLinkedFeatures.features.map((f) => f.properties?.["ref:velopark"])
        )
        this.exportGeojsonTo("osm_with_velopark_link", <Feature[]>alreadyLinkedFeatures.features)
        console.log("OpenStreetMap contains", seenIds.size, "bicycle parkings with a velopark ref")

        const features: Feature[] = allVelopark.filter((f) => !seenIds.has(f.properties["ref:velopark"]))
        VeloParkToGeojson.exportGeojsonTo("velopark_nonsynced", features)

        const allProperties = new Set<string>()
        for (const feature of features) {
            Object.keys(feature).forEach((k) => allProperties.add(k))
        }
        allProperties.delete("ref:velopark")
        for (const feature of features) {
            allProperties.forEach((k) => {
                delete feature[k]
            })
        }

        this.exportGeojsonTo("velopark_nonsynced_id_only", features)


    }

    async main(): Promise<void> {
        const allVelopark =VeloParkToGeojson.loadFromFile() ?? await VeloParkToGeojson.downloadData()
        console.log("Got", allVelopark.length, " items")
        VeloParkToGeojson.exportExtraAmenities(allVelopark)
        await VeloParkToGeojson.createDiff(allVelopark)
        console.log("Use vite-node script/velopark/compare to compare the results and generate a diff file")
    }
}

new VeloParkToGeojson().run()