import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFile, writeFileSync } from "fs" import Locale from "../src/UI/i18n/Locale" import Translations from "../src/UI/i18n/Translations" import { Translation } from "../src/UI/i18n/Translation" import all_known_layouts from "../src/assets/generated/known_themes.json" import { LayoutConfigJson } from "../src/Models/ThemeConfig/Json/LayoutConfigJson" import LayoutConfig from "../src/Models/ThemeConfig/LayoutConfig" import xml2js from "xml2js" import ScriptUtils from "./ScriptUtils" import { Utils } from "../src/Utils" import SpecialVisualizations from "../src/UI/SpecialVisualizations" import Constants from "../src/Models/Constants" import { AvailableRasterLayers, RasterLayerPolygon } from "../src/Models/RasterLayers" import { ImmutableStore } from "../src/Logic/UIEventSource" import * as crypto from "crypto" import * as eli from "../src/assets/editor-layer-index.json" import * as eli_global from "../src/assets/global-raster-layers.json" import ValidationUtils from "../src/Models/ThemeConfig/Conversion/ValidationUtils" const sharp = require("sharp") const template = readFileSync("theme.html", "utf8") const codeTemplate = readFileSync("src/index_theme.ts.template", "utf8") function enc(str: string): string { return encodeURIComponent(str.toLowerCase()) } async function createIcon(iconPath: string, size: number, alreadyWritten: string[]) { let name = iconPath.split(".").slice(0, -1).join(".") // drop svg suffix if (name.startsWith("./")) { name = name.substr(2) } const newname = `assets/generated/images/${name.replace(/\//g, "_")}${size}.png` const targetpath = `public/${newname}` if (alreadyWritten.indexOf(newname) >= 0) { return newname } alreadyWritten.push(newname) if (existsSync(targetpath)) { return newname } if (!existsSync(iconPath)) { throw "No file at " + iconPath } try { // We already read to file, in order to crash here if the file is not found let img = await sharp(iconPath) let resized = await img.resize(size) await resized.toFile(targetpath) console.log("Created png version at ", newname) } catch (e) { console.error("Could not read icon", iconPath, " to create a PNG due to", e) } return newname } async function createSocialImage(layout: LayoutConfig, template: "" | "Wide"): Promise { if (!layout.icon.endsWith(".svg")) { console.warn( "Not creating a social image for " + layout.id + " as it is _not_ a .svg: " + layout.icon ) return undefined } const path = `./public/assets/generated/images/social_image_${layout.id}_${template}.svg` if (existsSync(path)) { return path } const svg = await ScriptUtils.ReadSvg(layout.icon) let width: string = svg.$.width if (width === undefined) { throw "The logo at " + layout.icon + " does not have a defined width" } if (width?.endsWith("px")) { width = width.substring(0, width.length - 2) } if (width?.endsWith("%")) { throw "The logo at " + layout.icon + " has a relative width; this is not supported" } delete svg["defs"] delete svg["$"] let templateSvg = await ScriptUtils.ReadSvg( "./public/assets/SocialImageTemplate" + template + ".svg" ) templateSvg = Utils.WalkJson( templateSvg, (leaf) => { const { cx, cy, r } = leaf["circle"][0].$ return { $: { id: "icon", transform: `translate(${cx - r},${cy - r}) scale(${(r * 2) / Number(width)}) `, }, g: [svg], } }, (mightBeTokenToReplace) => { if (mightBeTokenToReplace?.circle === undefined) { return false } return mightBeTokenToReplace.circle[0]?.$?.style?.indexOf("fill:#ff00ff") >= 0 } ) const builder = new xml2js.Builder() const xml = builder.buildObject({ svg: templateSvg }) writeFileSync(path, xml) console.log("Created social image at ", path) return path } async function createManifest( layout: LayoutConfig, alreadyWritten: string[] ): Promise<{ manifest: any whiteIcons: string[] }> { Translation.forcedLanguage = "en" const icons = [] const whiteIcons: string[] = [] let icon = layout.icon if (icon.endsWith(".svg") || icon.startsWith("${t.translations[lang]}`) } return values.join("\n") } let previousSrc: Set = new Set() let eliUrlsCached: string[] async function eliUrls(): Promise { if (eliUrlsCached) { return eliUrlsCached } const urls: string[] = [] const regex = /{switch:([^}]+)}/ const rasterLayers = [ AvailableRasterLayers.maptilerDefaultLayer, ...eli.features, ...eli_global.layers.map((properties) => ({ properties })), ] for (const feature of rasterLayers) { const f = feature const url = f.properties.url const match = url.match(regex) if (match) { const domains = match[1].split(",") const subpart = match[0] urls.push(...domains.map((d) => url.replace(subpart, d))) } else { urls.push(url) } if (f.properties.type === "vector") { // We also need to whitelist eventual sources const styleSpec = await Utils.downloadJsonCached(f.properties.url, 1000 * 120) for (const key of Object.keys(styleSpec.sources)) { const url = styleSpec.sources[key].url if (!url) { continue } let urlClipped = url if (url.indexOf("?") > 0) { urlClipped = url?.substring(0, url.indexOf("?")) } console.log("Source url ", key, url) urls.push(url) if (urlClipped.endsWith(".json")) { const tileInfo = await Utils.downloadJsonCached(url, 1000 * 120) urls.push(tileInfo["tiles"] ?? []) } } urls.push(...(styleSpec["tiles"] ?? [])) urls.push(styleSpec["sprite"]) urls.push(styleSpec["glyphs"]) } } eliUrlsCached = urls return Utils.NoNull(urls).sort() } async function generateCsp( layout: LayoutConfig, layoutJson: LayoutConfigJson, options: { scriptSrcs: string[] } ): Promise { const apiUrls: string[] = [ "'self'", ...Constants.defaultOverpassUrls, Constants.countryCoderEndpoint, Constants.nominatimEndpoint, "https://www.openstreetmap.org", "https://api.openstreetmap.org", "https://pietervdvn.goatcounter.com", ].concat(...(await eliUrls())) SpecialVisualizations.specialVisualizations.forEach(sv => { if(typeof sv.needsUrls === "function"){ return } apiUrls.push(...sv.needsUrls) }) const usedSpecialVisualisations = ValidationUtils.getSpecialVisualisationsWithArgs(layoutJson) for (const usedSpecialVisualisation of usedSpecialVisualisations) { if (typeof usedSpecialVisualisation === "string") { continue } const neededUrls = usedSpecialVisualisation.func.needsUrls if (typeof neededUrls === "function") { apiUrls.push(...neededUrls(usedSpecialVisualisation.args)) } } const geojsonSources: string[] = layout.layers.map((l) => l.source?.geojsonSource) const hosts = new Set() const eliLayers: RasterLayerPolygon[] = AvailableRasterLayers.layersAvailableAt( new ImmutableStore({ lon: 0, lat: 0 }) ).data const vectorLayers = eliLayers.filter((l) => l.properties.type === "vector") const vectorSources = vectorLayers.map((l) => l.properties.url) apiUrls.push(...vectorSources) for (const connectSource of apiUrls.concat(geojsonSources)) { if (!connectSource) { continue } try { const url = new URL(connectSource) hosts.add("https://" + url.host) } catch (e) { hosts.add(connectSource) } } if (hosts.has("*")) { throw "* is not allowed as connect-src" } const connectSrc = Array.from(hosts).sort() const newSrcs = connectSrc.filter((newItem) => !previousSrc.has(newItem)) console.log( "Got", hosts.size, "connect-src items for theme", layout.id, "(extra sources: ", newSrcs.join(" ") + ")" ) previousSrc = hosts const csp: Record = { "default-src": "'self'", "child-src": "'self' blob: ", "img-src": "* data:", // maplibre depends on 'data:' to load "connect-src": connectSrc.join(" "), "report-to": "https://report.mapcomplete.org/csp", "worker-src": "'self' blob:", // Vite somehow loads the worker via a 'blob' "style-src": "'self' 'unsafe-inline'", // unsafe-inline is needed to change the default background pin colours "script-src": ["'self'", "https://gc.zgo.at/count.js", ...(options?.scriptSrcs ?? [])].join( " " ), } const content = Object.keys(csp) .map((k) => k + " " + csp[k]) .join(" ; ") return [ ``, ``, ].join("\n") } const removeOtherLanguages = readFileSync("./src/UI/RemoveOtherLanguages.js", "utf8") .split("\n") .map((s) => s.trim()) .join("\n") const removeOtherLanguagesHash = crypto .createHash("sha256") .update(removeOtherLanguages) .digest("base64") async function createLandingPage( layout: LayoutConfig, layoutJson: LayoutConfigJson, manifest, whiteIcons, alreadyWritten ) { Locale.language.setData(layout.language[0]) const targetLanguage = layout.language[0] const ogTitle = Translations.T(layout.title).textFor(targetLanguage).replace(/"/g, '\\"') const ogDescr = Translations.T( layout.shortDescription ?? "Easily add and edit geodata with OpenStreetMap" ) .textFor(targetLanguage) .replace(/"/g, '\\"') let ogImage = layout.socialImage let twitterImage = ogImage if (ogImage === LayoutConfig.defaultSocialImage && layout.official) { ogImage = (await createSocialImage(layout, "")) ?? layout.socialImage twitterImage = (await createSocialImage(layout, "Wide")) ?? layout.socialImage } if (twitterImage.endsWith(".svg")) { // svgs are badly supported as social image, we use a generated svg instead twitterImage = await createIcon(twitterImage, 512, alreadyWritten) } if (ogImage.endsWith(".svg")) { ogImage = await createIcon(ogImage, 512, alreadyWritten) } let customCss = "" if (layout.customCss !== undefined && layout.customCss !== "") { try { const cssContent = readFileSync(layout.customCss) customCss = "" } catch (e) { customCss = `` } } const og = ` ` let icon = layout.icon if (icon.startsWith("`) } let themeSpecific = [ `${ogTitle}`, ``, og, customCss, ``, ...apple_icons, ].join("\n") const loadingText = Translations.t.general.loadingTheme.Subs({ theme: layout.title }) const templateLines = template.split("\n") const removeOtherLanguagesReference = templateLines.find( (line) => line.indexOf("./src/UI/RemoveOtherLanguages.js") >= 0 ) let output = template .replace("Loading MapComplete, hang on...", asLangSpan(loadingText, "h1")) .replace( "Made with OpenStreetMap", Translations.t.general.poweredByOsm.textFor(targetLanguage) ) .replace(/.*/s, themeSpecific) .replace( //, await generateCsp(layout, layoutJson, { scriptSrcs: [`'sha256-${removeOtherLanguagesHash}'`], }) ) .replace(removeOtherLanguagesReference, "") .replace( /.*/s, asLangSpan(layout.shortDescription) ) .replace( /.*/s, "" ) .replace( /.*\/src\/index\.ts.*/, `` ) .replace("Version", Constants.vNumber) return output } async function createIndexFor(theme: LayoutConfig) { const filename = "index_" + theme.id + ".ts" const imports = [ `import layout from "./src/assets/generated/themes/${theme.id}.json"`, `import { ThemeMetaTagging } from "./src/assets/generated/metatagging/${theme.id}"`, ] writeFileSync(filename, imports.join("\n") + "\n") appendFileSync(filename, codeTemplate) } function createDir(path) { if (!existsSync(path)) { mkdirSync(path) } } async function main(): Promise { const alreadyWritten = [] createDir("./public/assets/") createDir("./public/assets/generated") createDir("./public/assets/generated/images") const blacklist = [ "", "test", ".", "..", "manifest", "index", "land", "preferences", "account", "openstreetmap", "custom", "theme", ] // @ts-ignore const all: LayoutConfigJson[] = all_known_layouts.themes const args = process.argv const theme = args[2] if (theme !== undefined) { console.warn("Only generating layout " + theme) } for (const i in all) { const layoutConfigJson: LayoutConfigJson = all[i] if (theme !== undefined && layoutConfigJson.id !== theme) { continue } const layout = new LayoutConfig(layoutConfigJson, true) const layoutName = layout.id if (blacklist.indexOf(layoutName.toLowerCase()) >= 0) { console.log(`Skipping a layout with name${layoutName}, it is on the blacklist`) continue } const err = (err) => { if (err !== null) { console.log("Could not write manifest for ", layoutName, " because ", err) } } const { manifest, whiteIcons } = await createManifest(layout, alreadyWritten) const manif = JSON.stringify(manifest, undefined, 2) const manifestLocation = encodeURIComponent(layout.id.toLowerCase()) + ".webmanifest" writeFile("public/" + manifestLocation, manif, err) // Create a landing page for the given theme const landing = await createLandingPage( layout, layoutConfigJson, manifest, whiteIcons, alreadyWritten ) writeFile(enc(layout.id) + ".html", landing, err) await createIndexFor(layout) } const { manifest } = await createManifest( new LayoutConfig({ icon: "./assets/svg/mapcomplete_logo.svg", id: "index", layers: [], socialImage: "assets/SocialImage.png", startLat: 0, startLon: 0, startZoom: 0, title: { en: "MapComplete" }, description: { en: "A thematic map viewer and editor based on OpenStreetMap" }, }), alreadyWritten ) const manif = JSON.stringify(manifest, undefined, 2) writeFileSync("public/index.webmanifest", manif) } ScriptUtils.fixUtils() main().then(() => { console.log("All done!") })