import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFile, writeFileSync } from "fs" import Locale from "../src/UI/i18n/Locale" import Translations from "../src/UI/i18n/Translations" import { Translation } from "../src/UI/i18n/Translation" import all_known_layouts from "../src/assets/generated/known_themes.json" import { ThemeConfigJson } from "../src/Models/ThemeConfig/Json/ThemeConfigJson" import ThemeConfig from "../src/Models/ThemeConfig/ThemeConfig" import xml2js from "xml2js" import ScriptUtils from "./ScriptUtils" import { Utils } from "../src/Utils" import SpecialVisualizations from "../src/UI/SpecialVisualizations" import Constants from "../src/Models/Constants" import { AvailableRasterLayers, EditorLayerIndexProperties, RasterLayerPolygon } from "../src/Models/RasterLayers" import { ImmutableStore } from "../src/Logic/UIEventSource" import * as eli from "../public/assets/data/editor-layer-index.json" import * as layers_global from "../src/assets/global-raster-layers.json" import eli_global from "../src/assets/generated/editor-layer-index-global.json" import bing from "../src/assets/bing.json" import ValidationUtils from "../src/Models/ThemeConfig/Conversion/ValidationUtils" import { LayerConfigJson } from "../src/Models/ThemeConfig/Json/LayerConfigJson" import { QuestionableTagRenderingConfigJson } from "../src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson" import Script from "./Script" import crypto from "crypto" import { RasterLayerProperties } from "../src/Models/RasterLayerProperties" const sharp = require("sharp") class GenerateLayouts extends Script { private readonly template = readFileSync("theme.html", "utf8") private readonly codeTemplate = readFileSync("src/index_theme.ts.template", "utf8") private readonly removeOtherLanguages = readFileSync("src/UI/RemoveOtherLanguages.ts", "utf8") .split("\n") .slice(1) .map((s) => s.trim()) .filter((s) => s !== "") .join("\n") private readonly removeOtherLanguagesHash = "sha256-" + crypto.createHash("sha256").update(this.removeOtherLanguages).digest("base64") private previousSrc: Set = new Set() private eliUrlsCached: string[] private date = new Date().toISOString() private branchName: string = undefined constructor() { super("Generates an '.html' and 'index_.ts' for every theme") } enc(str: string): string { return encodeURIComponent(str.toLowerCase()) } getBranchName(): Promise { if (this.branchName) { return Promise.resolve(this.branchName) } const { exec } = require("child_process") return new Promise((resolve, reject) => { exec("git rev-parse --abbrev-ref HEAD", (err, stdout, stderr) => { if (err) { reject(err) return } if (typeof stdout === "string") { this.branchName = stdout.trim() resolve(stdout.trim()) } reject("Did not get output") }) }) } async createIcon(iconPath: string, size: number, alreadyWritten: string[]) { let name = iconPath.split(".").slice(0, -1).join(".") // drop svg suffix if (name.startsWith("./")) { name = name.substring(2) } const newname = `assets/generated/images/${name.replace(/\//g, "_")}${size}.png` const targetpath = `public/${newname}` if (alreadyWritten.indexOf(newname) >= 0) { return newname } alreadyWritten.push(newname) if (existsSync(targetpath)) { return newname } if (!existsSync(iconPath)) { throw "No file at " + iconPath } try { // We already read to file, in order to crash here if the file is not found const img = await sharp(iconPath) const resized = await img.resize(size) await resized.toFile(targetpath) console.log("Created png version at ", newname) } catch (e) { console.error("Could not read icon", iconPath, " to create a PNG due to", e) } return newname } async createSocialImage(layout: ThemeConfig, template: "" | "Wide"): Promise { if (!layout.icon.endsWith(".svg")) { console.warn( "Not creating a social image for " + layout.id + " as it is _not_ a .svg: " + layout.icon ) return undefined } const path = `./public/assets/generated/images/social_image_${layout.id}_${template}.svg` if (existsSync(path)) { return path } const svg = await ScriptUtils.ReadSvg(layout.icon) let width: string = svg.$.width if (width === undefined) { throw "The logo at " + layout.icon + " does not have a defined width" } if (width?.endsWith("px")) { width = width.substring(0, width.length - 2) } if (width?.endsWith("%")) { throw "The logo at " + layout.icon + " has a relative width; this is not supported" } delete svg["defs"] delete svg["$"] let templateSvg = await ScriptUtils.ReadSvg( "./public/assets/SocialImageTemplate" + template + ".svg" ) templateSvg = Utils.WalkJson( templateSvg, (leaf) => { const { cx, cy, r } = leaf["circle"][0].$ return { $: { id: "icon", transform: `translate(${cx - r},${cy - r}) scale(${ (r * 2) / Number(width) }) `, }, g: [svg], } }, (mightBeTokenToReplace) => { if (mightBeTokenToReplace?.circle === undefined) { return false } return mightBeTokenToReplace.circle[0]?.$?.style?.indexOf("fill:#ff00ff") >= 0 } ) const builder = new xml2js.Builder() const xml = builder.buildObject({ svg: templateSvg }) writeFileSync(path, xml) console.log("Created social image at ", path) return path } async createManifest( layout: ThemeConfig, alreadyWritten: string[] ): Promise<{ manifest: any whiteIcons: string[] }> { Translation.forcedLanguage = "en" const icons = [] const whiteIcons: string[] = [] let icon = layout.icon if (icon.endsWith(".svg") || icon.startsWith("${t.translations[lang]}`) } return values.join("\n") } async eliUrls(): Promise { if (this.eliUrlsCached) { return this.eliUrlsCached } const urls: string[] = [] const regex = /{switch:([^}]+)}/ const rasterLayers: {properties: RasterLayerProperties}[] = [ AvailableRasterLayers.defaultBackgroundLayer, ...eli.features, bing, ...eli_global.map((properties) => ({ properties })), ...layers_global.layers.map((properties) => ({ properties })), ] for (const feature of rasterLayers) { const f = feature const url = f.properties.url const match = url.match(regex) if (match) { const domains = match[1].split(",") const subpart = match[0] urls.push(...domains.map((d) => url.replace(subpart, d))) } else { urls.push(url) } if (f.properties.type === "vector") { // We also need to whitelist eventual sources let url = f.properties.url if (url.startsWith("pmtiles://")) { url = url.substring("pmtiles://".length) } const styleSpec = await Utils.downloadJsonCached(url, 1000 * 120, { Origin: "https://mapcomplete.org", }) urls.push(...(f.properties["connect-src"] ?? [])) for (const key of Object.keys(styleSpec?.sources ?? {})) { const url = styleSpec.sources[key].url if (!url) { continue } let urlClipped = url if (url.indexOf("?") > 0) { urlClipped = url?.substring(0, url.indexOf("?")) } console.log("Source url ", key, url) urls.push(url) if (urlClipped.endsWith(".json")) { const tileInfo = await Utils.downloadJsonCached(url, 1000 * 120, { Origin: "https://mapcomplete.org", }) urls.push(tileInfo["tiles"] ?? []) } } urls.push(...(styleSpec["tiles"] ?? [])) urls.push(styleSpec["sprite"]) urls.push(styleSpec["glyphs"]) } } this.eliUrlsCached = urls return Utils.NoNull(urls).sort() } async generateCsp( layout: ThemeConfig, layoutJson: ThemeConfigJson, options: { scriptSrcs: string[] } ): Promise { const apiUrls: string[] = [ ...Constants.allServers, "https://www.openstreetmap.org", "https://api.openstreetmap.org", "https://pietervdvn.goatcounter.com", "https://api.panoramax.xyz", "https://panoramax.mapcomplete.org", ].concat(...(await this.eliUrls())) SpecialVisualizations.specialVisualizations.forEach((sv) => { if (typeof sv.needsUrls === "function") { // Handled below return } apiUrls.push(...(sv.needsUrls ?? [])) }) const usedSpecialVisualisations = [].concat( ...layoutJson.layers.map((l) => ValidationUtils.getAllSpecialVisualisations( (l).tagRenderings ?? [] ) ) ) for (const usedSpecialVisualisation of usedSpecialVisualisations) { if (typeof usedSpecialVisualisation === "string") { continue } const neededUrls = usedSpecialVisualisation.func.needsUrls ?? [] if (typeof neededUrls === "function") { let needed: string | string[] = neededUrls(usedSpecialVisualisation.args) if (typeof needed === "string") { needed = [needed] } apiUrls.push(...needed) } } const geojsonSources: string[] = layout.layers.map((l) => l.source?.geojsonSource) const hosts = new Set() hosts.add("https://schema.org") const eliLayers: RasterLayerPolygon[] = AvailableRasterLayers.layersAvailableAt( new ImmutableStore({ lon: 0, lat: 0 }) ).store.data { const vectorLayers = eliLayers.filter((l) => l.properties.type === "vector") const vectorSources = vectorLayers.map((l) => l.properties.url) vectorSources.push(...vectorLayers.map((l) => l.properties.style)) apiUrls.push( ...vectorSources.map((url) => { if (url?.startsWith("pmtiles://")) { return url.substring("pmtiles://".length) } return url }) ) } for (let connectSource of apiUrls.concat(geojsonSources)) { if (!connectSource) { continue } try { if (!connectSource.startsWith("http")) { connectSource = "https://" + connectSource } const url = new URL(connectSource) hosts.add("https://" + url.host) } catch (e) { hosts.add(connectSource) } } hosts.add("http://www.schema.org") // Schema.org is _not_ encrypted and thus needs an exception if (hosts.has("*")) { throw "* is not allowed as connect-src" } const connectSrc = Array.from(hosts).sort() const newSrcs = connectSrc.filter((newItem) => !this.previousSrc.has(newItem)) console.log( "Got", hosts.size, "connect-src items for theme", layout.id, newSrcs.length > 0 ? "(extra sources: " + newSrcs.join(" ") + ")" : "" ) this.previousSrc = hosts const csp: Record = { "default-src": "'self'", "child-src": "'self' blob: ", "img-src": "* data:", // maplibre depends on 'data:' to load "connect-src": "'self' " + connectSrc.join(" "), "report-to": "https://report.mapcomplete.org/csp", "worker-src": "'self' blob:", // Vite somehow loads the worker via a 'blob' "style-src": "'self' 'unsafe-inline'", // unsafe-inline is needed to change the default background pin colours "script-src": [ "'self'", "https://gc.zgo.at/count.js", ...(options?.scriptSrcs?.map((s) => "'" + s + "'") ?? []), ].join(" "), } const content = Object.keys(csp) .map((k) => k + " " + csp[k]) .join(" ; ") return [ ``, ``, ].join("\n") } async createLandingPage( layout: ThemeConfig, layoutJson: ThemeConfigJson, whiteIcons, alreadyWritten ) { Locale.language.setData(layout.language[0]) const targetLanguage = layout.language[0] const ogTitle = Translations.T(layout.title).textFor(targetLanguage).replace(/"/g, '\\"') const ogDescr = Translations.T( layout.shortDescription ?? "Easily add and edit geodata with OpenStreetMap" ) .textFor(targetLanguage) .replace(/"/g, '\\"') let ogImage = layout.socialImage let twitterImage = ogImage if (ogImage === ThemeConfig.defaultSocialImage && layout.official) { try { ogImage = (await this.createSocialImage(layout, "")) ?? layout.socialImage twitterImage = (await this.createSocialImage(layout, "Wide")) ?? layout.socialImage } catch (e) { console.error("Could not generate image:", e) } } if (twitterImage.endsWith(".svg")) { try { // svgs are badly supported as social image, we use a generated svg instead twitterImage = await this.createIcon(twitterImage, 512, alreadyWritten) } catch (e) { console.error("Could not generate image:", e) } } if (ogImage.endsWith(".svg")) { try { ogImage = await this.createIcon(ogImage, 512, alreadyWritten) } catch (e) { console.error("Could not generate image:", e) } } let customCss = "" if (layout.customCss !== undefined && layout.customCss !== "") { try { const cssContent = readFileSync(layout.customCss) customCss = "" } catch (e) { customCss = `` } } const og = ` ` let icon = layout.icon if (icon.startsWith("`) } const themeSpecific = [ `${ogTitle}`, ``, og, customCss, ``, ...apple_icons, ].join("\n") let branchname = await this.getBranchName() if (branchname === "master" || branchname === "main") { branchname = "" } else { branchname = "
" + branchname + "
" } const loadingText = Translations.t.general.loadingTheme.Subs({ theme: layout.title }) // const templateLines: string[] = this.template.split("\n").slice(1) // Slice to remove the 'export {}'-line return this.template .replace("Loading MapComplete, hang on...", this.asLangSpan(loadingText, "h1")) .replace( "Made with OpenStreetMap", Translations.t.general.poweredByOsm.textFor(targetLanguage) ) .replace(/.*/s, themeSpecific) .replace( //, await this.generateCsp(layout, layoutJson, { scriptSrcs: [this.removeOtherLanguagesHash], }) ) .replace( /.*/s, this.asLangSpan(layout.shortDescription) ) .replace( /.*/s, "" ) .replace( /.*\/src\/index\.ts.*/, `` ) .replace( /\n.*RemoveOtherLanguages.*\n/i, "\n\n" ) .replace( "Version", `${Constants.vNumber}
${this.date}
${branchname}` ) } async createIndexFor(theme: ThemeConfig) { const filename = "index_" + theme.id + ".ts" const imports = [ `import layout from "./src/assets/generated/themes/${theme.id}.json"`, `import { ThemeMetaTagging } from "./src/assets/generated/metatagging/${theme.id}"`, ] for (const layerName of Constants.added_by_default) { imports.push( `import ${layerName} from "./src/assets/generated/layers/${layerName}.json"` ) } writeFileSync(filename, imports.join("\n") + "\n") const addLayers = [] for (const layerName of Constants.added_by_default) { addLayers.push(` layout.layers.push( ${layerName})`) } let codeTemplate = this.codeTemplate.replace( " // LAYOUT.ADD_LAYERS", addLayers.join("\n") ) appendFileSync(filename, codeTemplate) } createDir(path) { if (!existsSync(path)) { mkdirSync(path) } } async main(): Promise { const alreadyWritten = [] this.createDir("./public/assets/") this.createDir("./public/assets/generated") this.createDir("./public/assets/generated/images") const blacklist = [ "", "test", ".", "..", "manifest", "index", "land", "preferences", "account", "openstreetmap", "custom", "theme", ] // @ts-ignore const all: ThemeConfigJson[] = all_known_layouts.themes const args = process.argv const theme = args[2] if (theme !== undefined) { console.warn("Only generating layout " + theme) } for (const i in all) { const layoutConfigJson: ThemeConfigJson = all[i] if (theme !== undefined && layoutConfigJson.id !== theme) { continue } const layout = new ThemeConfig(layoutConfigJson, true) const layoutName = layout.id if (blacklist.indexOf(layoutName.toLowerCase()) >= 0) { console.log(`Skipping a layout with name ${layoutName}, it is on the blacklist`) continue } const err = (err) => { if (err !== null) { console.log("Could not write manifest for ", layoutName, " because ", err) } } const { manifest, whiteIcons } = await this.createManifest(layout, alreadyWritten) const manif = JSON.stringify(manifest, undefined, 2) const manifestLocation = encodeURIComponent(layout.id.toLowerCase()) + ".webmanifest" writeFile("public/" + manifestLocation, manif, err) // Create a landing page for the given theme const landing = await this.createLandingPage( layout, layoutConfigJson, whiteIcons, alreadyWritten ) writeFile(this.enc(layout.id) + ".html", landing, err) await this.createIndexFor(layout) } const { manifest } = await this.createManifest( new ThemeConfig({ icon: "./assets/svg/mapcomplete_logo.svg", id: "index", layers: [], socialImage: "assets/SocialImage.png", startLat: 0, startLon: 0, startZoom: 0, title: { en: "MapComplete" }, description: { en: "A thematic map viewer and editor based on OpenStreetMap" }, }), alreadyWritten ) const manif = JSON.stringify(manifest, undefined, 2) writeFileSync("public/index.webmanifest", manif) } } new GenerateLayouts().run()