mapcomplete/scripts/generateLayouts.ts

import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFile, writeFileSync } from "fs"
import Locale from "../src/UI/i18n/Locale"
import Translations from "../src/UI/i18n/Translations"
import { Translation } from "../src/UI/i18n/Translation"
import all_known_layouts from "../src/assets/generated/known_themes.json"
import { LayoutConfigJson } from "../src/Models/ThemeConfig/Json/LayoutConfigJson"
import LayoutConfig from "../src/Models/ThemeConfig/LayoutConfig"
import xml2js from "xml2js"
import ScriptUtils from "./ScriptUtils"
import { Utils } from "../src/Utils"
import SpecialVisualizations from "../src/UI/SpecialVisualizations"
import Constants from "../src/Models/Constants"
import { AvailableRasterLayers, RasterLayerPolygon } from "../src/Models/RasterLayers"
import { ImmutableStore } from "../src/Logic/UIEventSource"
import * as eli from "../src/assets/editor-layer-index.json"
import * as eli_global from "../src/assets/global-raster-layers.json"
import ValidationUtils from "../src/Models/ThemeConfig/Conversion/ValidationUtils"
import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson"
import { QuestionableTagRenderingConfigJson } from "../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
import Script from "./Script"
import crypto from "crypto"

const sharp = require("sharp")

class GenerateLayouts extends Script {
    private readonly template = readFileSync("theme.html", "utf8")
    private readonly codeTemplate = readFileSync("src/index_theme.ts.template", "utf8")
    private readonly removeOtherLanguages = readFileSync("src/UI/RemoveOtherLanguages.ts", "utf8")
        .split("\n")
        .slice(1)
        .map((s) => s.trim())
        .filter((s) => s !== "")
        .join("\n")
    private readonly removeOtherLanguagesHash =
        "sha256-" + crypto.createHash("sha256").update(this.removeOtherLanguages).digest("base64")
    private previousSrc: Set<string> = new Set<string>()
    private eliUrlsCached: string[]
    private date = new Date().toISOString()
    private branchName: string = undefined

    constructor() {
        super("Generates an '<theme>.html' and 'index_<theme>.ts' for every theme")
    }

    enc(str: string): string {
        return encodeURIComponent(str.toLowerCase())
    }

    getBranchName(): Promise<string> {
        if (this.branchName) {
            return Promise.resolve(this.branchName)
        }
        const { exec } = require("child_process")
        return new Promise<string>((resolve, reject) => {
            exec("git rev-parse --abbrev-ref HEAD", (err, stdout, stderr) => {
                if (err) {
                    reject(err)
                    return
                }

                if (typeof stdout === "string") {
                    this.branchName = stdout.trim()
                    resolve(stdout.trim())
                }
                reject("Did not get output")
            })
        })
    }

    async createIcon(iconPath: string, size: number, alreadyWritten: string[]) {
        let name = iconPath.split(".").slice(0, -1).join(".") // drop svg suffix
        if (name.startsWith("./")) {
            name = name.substring(2)
        }

        const newname = `assets/generated/images/${name.replace(/\//g, "_")}${size}.png`
        const targetpath = `public/${newname}`
        if (alreadyWritten.indexOf(newname) >= 0) {
            return newname
        }
        alreadyWritten.push(newname)
        if (existsSync(targetpath)) {
            return newname
        }

        if (!existsSync(iconPath)) {
            throw "No file at " + iconPath
        }

        try {
            // We already read to file, in order to crash here if the file is not found
            const img = await sharp(iconPath)
            const resized = await img.resize(size)
            await resized.toFile(targetpath)
            console.log("Created png version at ", newname)
        } catch (e) {
            console.error("Could not read icon", iconPath, " to create a PNG due to", e)
        }

        return newname
    }

    async createSocialImage(layout: LayoutConfig, template: "" | "Wide"): Promise<string> {
        if (!layout.icon.endsWith(".svg")) {
            console.warn(
                "Not creating a social image for " +
                    layout.id +
                    " as it is _not_ a .svg: " +
                    layout.icon
            )
            return undefined
        }
        const path = `./public/assets/generated/images/social_image_${layout.id}_${template}.svg`
        if (existsSync(path)) {
            return path
        }
        const svg = await ScriptUtils.ReadSvg(layout.icon)
        let width: string = svg.$.width
        if (width === undefined) {
            throw "The logo at " + layout.icon + " does not have a defined width"
        }
        if (width?.endsWith("px")) {
            width = width.substring(0, width.length - 2)
        }
        if (width?.endsWith("%")) {
            throw "The logo at " + layout.icon + " has a relative width; this is not supported"
        }
        delete svg["defs"]
        delete svg["$"]
        let templateSvg = await ScriptUtils.ReadSvg(
            "./public/assets/SocialImageTemplate" + template + ".svg"
        )
        templateSvg = Utils.WalkJson(
            templateSvg,
            (leaf) => {
                const { cx, cy, r } = leaf["circle"][0].$
                return {
                    $: {
                        id: "icon",
                        transform: `translate(${cx - r},${cy - r}) scale(${
                            (r * 2) / Number(width)
                        }) `,
                    },
                    g: [svg],
                }
            },
            (mightBeTokenToReplace) => {
                if (mightBeTokenToReplace?.circle === undefined) {
                    return false
                }
                return mightBeTokenToReplace.circle[0]?.$?.style?.indexOf("fill:#ff00ff") >= 0
            }
        )

        const builder = new xml2js.Builder()
        const xml = builder.buildObject({ svg: templateSvg })
        writeFileSync(path, xml)
        console.log("Created social image at ", path)
        return path
    }

    async createManifest(
        layout: LayoutConfig,
        alreadyWritten: string[]
    ): Promise<{
        manifest: any
        whiteIcons: string[]
    }> {
        Translation.forcedLanguage = "en"
        const icons = []

        const whiteIcons: string[] = []
        let icon = layout.icon
        if (icon.endsWith(".svg") || icon.startsWith("<svg") || icon.startsWith("<?xml")) {
            // This is an svg. Lets create the needed pngs and do some checkes!

            const whiteBackgroundPath =
                "./public/assets/generated/images/theme_" + layout.id + "_white_background.svg"
            {
                const svg = await ScriptUtils.ReadSvg(icon)
                const width: string = svg.$.width
                const height: string = svg.$.height

                const builder = new xml2js.Builder()
                const withRect = { rect: { $: { width, height, style: "fill:#ffffff;" } }, ...svg }
                const xml = builder.buildObject({ svg: withRect })
                writeFileSync(whiteBackgroundPath, xml)
            }

            let path = layout.icon
            if (layout.icon.startsWith("<")) {
                // THis is already the svg
                path = "./public/assets/generated/images/" + layout.id + "_logo.svg"
                writeFileSync(path, layout.icon)
            }

            const sizes = [72, 96, 120, 128, 144, 152, 180, 192, 384, 512]
            for (const size of sizes) {
                const name = await this.createIcon(path, size, alreadyWritten)
                const whiteIcon = await this.createIcon(whiteBackgroundPath, size, alreadyWritten)
                whiteIcons.push(whiteIcon)
                icons.push({
                    src: name,
                    sizes: size + "x" + size,
                    type: "image/png",
                })
            }
            icons.push({
                src: path,
                sizes: "513x513",
                type: "image/svg",
            })
        } else if (icon.endsWith(".png")) {
            icons.push({
                src: icon,
                sizes: "513x513",
                type: "image/png",
            })
        } else {
            console.log(icon)
            throw "Icon is not an svg for " + layout.id
        }
        const ogTitle = Translations.T(layout.title).txt
        const ogDescr = Translations.T(layout.description ?? "").txt

        const manifest = {
            name: ogTitle,
            short_name: ogTitle,
            start_url: `${layout.id.toLowerCase()}.html`,
            lang: "en",
            display: "standalone",
            background_color: "#fff",
            description: ogDescr,
            orientation: "portrait-primary, landscape-primary",
            icons: icons,
            categories: ["map", "navigation"],
        }
        return {
            manifest,
            whiteIcons,
        }
    }

    asLangSpan(t: Translation, tag = "span"): string {
        const values: string[] = []
        let defaultSet = false
        for (const lang in t.translations) {
            if (lang === "_context") {
                continue
            }
            let display = ' style="display: none"'
            if (!defaultSet) {
                display = ""
                defaultSet = true
            }
            values.push(`<${tag} lang="${lang}"${display}>${t.translations[lang]}</${tag}>`)
        }
        return values.join("\n")
    }

    async eliUrls(): Promise<string[]> {
        if (this.eliUrlsCached) {
            return this.eliUrlsCached
        }
        const urls: string[] = []
        const regex = /{switch:([^}]+)}/
        const rasterLayers = [
            AvailableRasterLayers.defaultBackgroundLayer,
            ...eli.features,
            ...eli_global.layers.map((properties) => ({ properties })),
        ]
        for (const feature of rasterLayers) {
            const f = <RasterLayerPolygon>feature
            const url = f.properties.url
            const match = url.match(regex)
            if (match) {
                const domains = match[1].split(",")
                const subpart = match[0]
                urls.push(...domains.map((d) => url.replace(subpart, d)))
            } else {
                urls.push(url)
            }

            if (f.properties.type === "vector") {
                // We also need to whitelist eventual sources
                let url = f.properties.url
                if (url.startsWith("pmtiles://")) {
                    url = url.substring("pmtiles://".length)
                }
                const styleSpec = await Utils.downloadJsonCached(url, 1000 * 120, {
                    Origin: "https://mapcomplete.org",
                })
                urls.push(...(f.properties["connect-src"] ?? []))
                for (const key of Object.keys(styleSpec?.sources ?? {})) {
                    const url = styleSpec.sources[key].url
                    if (!url) {
                        continue
                    }
                    let urlClipped = url
                    if (url.indexOf("?") > 0) {
                        urlClipped = url?.substring(0, url.indexOf("?"))
                    }
                    console.log("Source url ", key, url)
                    urls.push(url)
                    if (urlClipped.endsWith(".json")) {
                        const tileInfo = await Utils.downloadJsonCached(url, 1000 * 120, {
                            Origin: "https://mapcomplete.org",
                        })
                        urls.push(tileInfo["tiles"] ?? [])
                    }
                }
                urls.push(...(styleSpec["tiles"] ?? []))
                urls.push(styleSpec["sprite"])
                urls.push(styleSpec["glyphs"])
            }
        }
        this.eliUrlsCached = urls
        return Utils.NoNull(urls).sort()
    }

    async generateCsp(
        layout: LayoutConfig,
        layoutJson: LayoutConfigJson,
        options: {
            scriptSrcs: string[]
        }
    ): Promise<string> {
        const apiUrls: string[] = [
            ...Constants.allServers,
            Constants.countryCoderEndpoint,
            Constants.nominatimEndpoint,
            "https://www.openstreetmap.org",
            "https://api.openstreetmap.org",
            "https://pietervdvn.goatcounter.com",
        ].concat(...(await this.eliUrls()))

        SpecialVisualizations.specialVisualizations.forEach((sv) => {
            if (typeof sv.needsUrls === "function") {
                // Handled below
                return
            }
            apiUrls.push(...(sv.needsUrls ?? []))
        })

        const usedSpecialVisualisations = [].concat(
            ...layoutJson.layers.map((l) =>
                ValidationUtils.getAllSpecialVisualisations(
                    <QuestionableTagRenderingConfigJson[]>(<LayerConfigJson>l).tagRenderings ?? []
                )
            )
        )
        for (const usedSpecialVisualisation of usedSpecialVisualisations) {
            if (typeof usedSpecialVisualisation === "string") {
                continue
            }
            const neededUrls = usedSpecialVisualisation.func.needsUrls ?? []
            if (typeof neededUrls === "function") {
                let needed: string | string[] = neededUrls(usedSpecialVisualisation.args)
                if (typeof needed === "string") {
                    needed = [needed]
                }
                apiUrls.push(...needed)
            }
        }

        const geojsonSources: string[] = layout.layers.map((l) => l.source?.geojsonSource)
        const hosts = new Set<string>()
        hosts.add("https://schema.org")
        const eliLayers: RasterLayerPolygon[] = AvailableRasterLayers.layersAvailableAt(
            new ImmutableStore({ lon: 0, lat: 0 })
        ).data
        {
            const vectorLayers = eliLayers.filter((l) => l.properties.type === "vector")
            const vectorSources = vectorLayers.map((l) => l.properties.url)
            vectorSources.push(...vectorLayers.map((l) => l.properties.style))
            apiUrls.push(
                ...vectorSources.map((url) => {
                    if (url?.startsWith("pmtiles://")) {
                        return url.substring("pmtiles://".length)
                    }
                    return url
                })
            )
        }
        for (let connectSource of apiUrls.concat(geojsonSources)) {
            if (!connectSource) {
                continue
            }
            try {
                if (!connectSource.startsWith("http")) {
                    connectSource = "https://" + connectSource
                }
                const url = new URL(connectSource)
                hosts.add("https://" + url.host)
            } catch (e) {
                hosts.add(connectSource)
            }
        }

        hosts.add("http://www.schema.org") // Schema.org is _not_ encrypted and thus needs an exception

        if (hosts.has("*")) {
            throw "* is not allowed as connect-src"
        }

        const connectSrc = Array.from(hosts).sort()

        const newSrcs = connectSrc.filter((newItem) => !this.previousSrc.has(newItem))

        console.log(
            "Got",
            hosts.size,
            "connect-src items for theme",
            layout.id,
            newSrcs.length > 0 ? "(extra sources: " + newSrcs.join(" ") + ")" : ""
        )
        this.previousSrc = hosts

        const csp: Record<string, string> = {
            "default-src": "'self'",
            "child-src": "'self' blob: ",
            "img-src": "* data:", // maplibre depends on 'data:' to load
            "connect-src": "'self' " + connectSrc.join(" "),
            "report-to": "https://report.mapcomplete.org/csp",
            "worker-src": "'self' blob:", // Vite somehow loads the worker via a 'blob'
            "style-src": "'self' 'unsafe-inline'", // unsafe-inline is needed to change the default background pin colours
            "script-src": [
                "'self'",
                "https://gc.zgo.at/count.js",
                ...(options?.scriptSrcs?.map((s) => "'" + s + "'") ?? []),
            ].join(" "),
        }
        const content = Object.keys(csp)
            .map((k) => k + " " + csp[k])
            .join(" ; ")

        return [
            `<meta http-equiv ="Report-To" content='{"group":"csp-endpoint", "max_age": 86400,"endpoints": [\{"url": "https://report.mapcomplete.org/csp"}], "include_subdomains": true}'>`,
            `<meta http-equiv="Content-Security-Policy" content="${content}">`,
        ].join("\n")
    }

    async createLandingPage(
        layout: LayoutConfig,
        layoutJson: LayoutConfigJson,
        whiteIcons,
        alreadyWritten
    ) {
        Locale.language.setData(layout.language[0])
        const targetLanguage = layout.language[0]
        const ogTitle = Translations.T(layout.title).textFor(targetLanguage).replace(/"/g, '\\"')
        const ogDescr = Translations.T(
            layout.shortDescription ?? "Easily add and edit geodata with OpenStreetMap"
        )
            .textFor(targetLanguage)
            .replace(/"/g, '\\"')
        let ogImage = layout.socialImage
        let twitterImage = ogImage
        if (ogImage === LayoutConfig.defaultSocialImage && layout.official) {
            try{
                ogImage = (await this.createSocialImage(layout, "")) ?? layout.socialImage
                twitterImage = (await this.createSocialImage(layout, "Wide")) ?? layout.socialImage
            }catch (e) {
                console.error("Could not generate image:", e)
            }
        }
        if (twitterImage.endsWith(".svg")) {
            try{

            // svgs are badly supported as social image, we use a generated svg instead
            twitterImage = await this.createIcon(twitterImage, 512, alreadyWritten)
            }catch (e) {
                console.error("Could not generate image:", e)
            }
        }

        if (ogImage.endsWith(".svg")) {
            try{
                ogImage = await this.createIcon(ogImage, 512, alreadyWritten)
            }catch (e) {
                console.error("Could not generate image:", e)
            }
        }

        let customCss = ""
        if (layout.customCss !== undefined && layout.customCss !== "") {
            try {
                const cssContent = readFileSync(layout.customCss)
                customCss = "<style>" + cssContent + "</style>"
            } catch (e) {
                customCss = `<link rel="stylesheet" href="${layout.customCss}"/>`
            }
        }

        const og = `
    <meta property="og:image" content="${ogImage ?? "assets/SocialImage.png"}">
    <meta property="og:title" content="${ogTitle}">
    <meta property="og:description" content="${ogDescr}">
    <meta name="twitter:card" content="summary_large_image">
    <meta name="twitter:site" content="@mapcomplete.org">
    <meta name="twitter:creator" content="@pietervdvn">
    <meta name="twitter:title" content="${ogTitle}">
    <meta name="twitter:description" content="${ogDescr}">
    <meta name="twitter:image" content="${twitterImage}">`

        let icon = layout.icon
        if (icon.startsWith("<?xml") || icon.startsWith("<svg")) {
            // This already is an svg
            icon = `./public/assets/generated/images/${layout.id}_icon.svg`
            writeFileSync(icon, layout.icon)
        }

        const apple_icons = []
        for (const icon of whiteIcons) {
            if (!existsSync(icon)) {
                continue
            }
            const size = icon.replace(/[^0-9]/g, "")
            apple_icons.push(`<link rel="apple-touch-icon" sizes="${size}x${size}" href="${icon}">`)
        }

        const themeSpecific = [
            `<title>${ogTitle}</title>`,
            `<link rel="manifest" href="${this.enc(layout.id)}.webmanifest">`,
            og,
            customCss,
            `<link rel="icon" href="${icon}" sizes="any" type="image/svg+xml">`,
            ...apple_icons,
        ].join("\n")

        let branchname = await this.getBranchName()
        if (branchname === "master" || branchname === "main") {
            branchname = ""
        } else {
            branchname = "<div class='text-xs'>" + branchname + "</div>"
        }

        const loadingText = Translations.t.general.loadingTheme.Subs({ theme: layout.title })
        // const templateLines: string[] = this.template.split("\n").slice(1) // Slice to remove the 'export {}'-line

        return this.template
            .replace("Loading MapComplete, hang on...", this.asLangSpan(loadingText, "h1"))
            .replace(
                "Made with OpenStreetMap",
                Translations.t.general.poweredByOsm.textFor(targetLanguage)
            )
            .replace(/<!-- THEME-SPECIFIC -->.*<!-- THEME-SPECIFIC-END-->/s, themeSpecific)
            .replace(
                /<!-- CSP -->/,
                await this.generateCsp(layout, layoutJson, {
                    scriptSrcs: [this.removeOtherLanguagesHash],
                })
            )
            .replace(
                /<!-- DESCRIPTION START -->.*<!-- DESCRIPTION END -->/s,
                this.asLangSpan(layout.shortDescription)
            )
            .replace(
                /<!-- IMAGE-START -->.*<!-- IMAGE-END -->/s,
                "<img class='p-0 h-32 w-32 self-start' src='" + icon + "' />"
            )
            .replace(
                /.*\/src\/index\.ts.*/,
                `<script type="module" src="./index_${layout.id}.ts"></script>`
            )

            .replace(
                /\n.*RemoveOtherLanguages.*\n/i,
                "\n<script>" + this.removeOtherLanguages + "</script>\n"
            )
            .replace(
                "Version",
                `${Constants.vNumber} <div class='text-xs'>${this.date}</div>${branchname}`
            )
    }

    async createIndexFor(theme: LayoutConfig) {
        const filename = "index_" + theme.id + ".ts"

        const imports = [
            `import layout from "./src/assets/generated/themes/${theme.id}.json"`,
            `import { ThemeMetaTagging } from "./src/assets/generated/metatagging/${theme.id}"`,
        ]
        for (const layerName of Constants.added_by_default) {
            imports.push(
                `import ${layerName} from "./src/assets/generated/layers/${layerName}.json"`
            )
        }
        writeFileSync(filename, imports.join("\n") + "\n")

        const addLayers = []

        for (const layerName of Constants.added_by_default) {
            addLayers.push(`    layout.layers.push(<any> ${layerName})`)
        }

        let codeTemplate = this.codeTemplate.replace(
            "    // LAYOUT.ADD_LAYERS",
            addLayers.join("\n")
        )

        appendFileSync(filename, codeTemplate)
    }

    createDir(path) {
        if (!existsSync(path)) {
            mkdirSync(path)
        }
    }

    async main(): Promise<void> {
        const alreadyWritten = []
        this.createDir("./public/assets/")
        this.createDir("./public/assets/generated")
        this.createDir("./public/assets/generated/images")

        const blacklist = [
            "",
            "test",
            ".",
            "..",
            "manifest",
            "index",
            "land",
            "preferences",
            "account",
            "openstreetmap",
            "custom",
            "theme",
        ]
        // @ts-ignore
        const all: LayoutConfigJson[] = all_known_layouts.themes
        const args = process.argv
        const theme = args[2]
        if (theme !== undefined) {
            console.warn("Only generating layout " + theme)
        }
        for (const i in all) {
            const layoutConfigJson: LayoutConfigJson = all[i]
            if (theme !== undefined && layoutConfigJson.id !== theme) {
                continue
            }
            const layout = new LayoutConfig(layoutConfigJson, true)
            const layoutName = layout.id
            if (blacklist.indexOf(layoutName.toLowerCase()) >= 0) {
                console.log(`Skipping a layout with name ${layoutName}, it is on the blacklist`)
                continue
            }
            const err = (err) => {
                if (err !== null) {
                    console.log("Could not write manifest for ", layoutName, " because ", err)
                }
            }
            const { manifest, whiteIcons } = await this.createManifest(layout, alreadyWritten)
            const manif = JSON.stringify(manifest, undefined, 2)
            const manifestLocation = encodeURIComponent(layout.id.toLowerCase()) + ".webmanifest"
            writeFile("public/" + manifestLocation, manif, err)

            // Create a landing page for the given theme
            const landing = await this.createLandingPage(
                layout,
                layoutConfigJson,
                whiteIcons,
                alreadyWritten
            )

            writeFile(this.enc(layout.id) + ".html", landing, err)
            await this.createIndexFor(layout)
        }

        const { manifest } = await this.createManifest(
            new LayoutConfig({
                icon: "./assets/svg/mapcomplete_logo.svg",
                id: "index",
                layers: [],
                socialImage: "assets/SocialImage.png",
                startLat: 0,
                startLon: 0,
                startZoom: 0,
                title: { en: "MapComplete" },
                description: { en: "A thematic map viewer and editor based on OpenStreetMap" },
            }),
            alreadyWritten
        )

        const manif = JSON.stringify(manifest, undefined, 2)
        writeFileSync("public/index.webmanifest", manif)
    }
}

new GenerateLayouts().run()