mapcomplete/scripts/postal_code_tools/openaddresses2geojson.ts

import * as fs from "fs";
import * as readline from "readline";
import ScriptUtils from "./ScriptUtils";
import {existsSync, writeFileSync} from "fs";
import {GeoOperations} from "../Logic/GeoOperations";
import {Utils} from "../Utils";
import {post} from "jquery";
import {createECDH} from "crypto";

/**
 * Converts an open-address CSV file into a big geojson file
 */

async function main(args: string[]) {

    const inputFile = args[0]
    const outputFile = args[1]
    const fileStream = fs.createReadStream(inputFile);
    const perPostalCode = args[2] == "--per-postal-code"
    const rl = readline.createInterface({
        input: fileStream,
        crlfDelay: Infinity
    });
    // Note: we use the crlfDelay option to recognize all instances of CR LF
    // ('\r\n') in input.txt as a single line break.

    const fields = [
        "EPSG:31370_x", "EPSG:31370_y", "EPSG:4326_lat", "EPSG:4326_lon",
        "address_id", "box_number",
        "house_number", "municipality_id", "municipality_name_de", "municipality_name_fr", "municipality_name_nl", "postcode", "postname_fr",
        "postname_nl", "street_id", "streetname_de", "streetname_fr", "streetname_nl", "region_code", "status"
    ]

    let i = 0;
    let failed = 0
    
    let createdFiles : string [] = []
    fs.writeFileSync(outputFile, "")
    // @ts-ignore
    for await (const line of rl) {
        i++;
        if (i % 10000 == 0) {
            ScriptUtils.erasableLog("Converted ", i, "features (of which ", failed, "features don't have a coordinate)")
        }
        const data = line.split(",")
        const parsed: any = {}
        for (let i = 0; i < fields.length; i++) {
            const field = fields[i];
            parsed[field] = data[i]
        }
        const lat = Number(parsed["EPSG:4326_lat"])
        const lon = Number(parsed["EPSG:4326_lon"])

        if (parsed["EPSG:31370_x"] === "0.0") {
            failed++
            continue
        }

        delete parsed["EPSG:4326_lat"]
        delete parsed["EPSG:4326_lon"]
        delete parsed["EPSG:31370_x"]
        delete parsed["EPSG:31370_y"]
        delete parsed["EPSG:4326_lat"]
        delete parsed["EPSG:4326_lat"]

        let targetFile = outputFile
        if (perPostalCode) {
            targetFile = outputFile + "-" + parsed["postcode"] + ".geojson"
            let isFirst = false
            if(!existsSync(targetFile)){
                writeFileSync(targetFile, '{ "type":"FeatureCollection", "features":[')
                createdFiles.push(targetFile)
                isFirst = true
            }
            
            if(!isFirst){
                fs.appendFileSync(targetFile, ",\n")
            }

            fs.appendFileSync(targetFile, JSON.stringify({
                type: "Feature",
                properties: parsed,
                geometry: {
                    type: "Point",
                    coordinates: [lon, lat]
                }
            }))
            
        }else{
            

        fs.appendFileSync(outputFile, JSON.stringify({
            type: "Feature",
            properties: parsed,
            geometry: {
                type: "Point",
                coordinates: [lon, lat]
            }
        }) + "\n")
        }
        

    }

    console.log("Closing files...")
    createdFiles.sort()
    for (const createdFile of createdFiles) {
        ScriptUtils.erasableLog("Closing ", createdFile, "and creating convex hull")
        fs.appendFileSync(createdFile, "]}")
    }
    
    console.log("Done! Converted ", i, "features (of which ", failed, "features don't have a coordinate)")

}

let args = [...process.argv]
args.splice(0, 2)

if (args.length == 0) {
    console.log("USAGE: input-csv-file output.newline-delimited-geojson.json [--per-postal-code]")
} else {
    main(args).catch(e => console.error(e))
}