Add some more missing translations, small tweaks to scripts

assets/layers/direction/direction.json

@@ -1,7 +1,8 @@
 {
   "id": "direction",
   "name": {
-    "en": "Direction visualization"
+    "en": "Direction visualization",
+    "nl": "Richtingsvisualisatie"
   },
   "minzoom": 16,
   "source": {
@@ -16,7 +17,8 @@
   "passAllFeatures": true,
   "title": null,
   "description": {
-    "en": "This layer visualizes directions"
+    "en": "This layer visualizes directions",
+    "nl": "Deze laag toont de oriëntatie van een object"
   },
   "tagRenderings": [],
   "icon": {

assets/layers/sport_pitch/sport_pitch.json

@@ -160,24 +160,37 @@
     },
     {
       "question": {
-        "nl": "Is dit sportterrein publiek toegankelijk?"
+        "nl": "Is dit sportterrein publiek toegankelijk?",
+        "en": "Is this sport pitch publicly accessible?"
       },
       "mappings": [
         {
           "if": "access=public",
-          "then": "Publiek toegankelijk"
+          "then": {
+            "nl": "Publiek toegankelijk",
+            "en": "Public access"
+          }
         },
         {
           "if": "access=limited",
-          "then": "Beperkt toegankelijk (enkel na reservatie, tijdens bepaalde uren, ...)"
+          "then": {
+            "nl": "Beperkt toegankelijk (enkel na reservatie, tijdens bepaalde uren, ...)",
+            "en": "Limited access (e.g. only with an appointment, during certain hours, ...)"
+          }
         },
         {
           "if": "access=members",
-          "then": "Enkel toegankelijk voor leden van de bijhorende sportclub"
+          "then": {
+            "nl": "Enkel toegankelijk voor leden van de bijhorende sportclub",
+            "en": "Only accessible for members of the club"
+          }
         },
         {
           "if": "access=private",
-          "then": "Privaat en niet toegankelijk"
+          "then": {
+            "nl": "Privaat en niet toegankelijk",
+            "en": "Private - not accessible to the public"
+          }
         }
       ]
     },
@@ -197,29 +210,28 @@
         {
           "if": "reservation=required",
           "then": {
-            "nl":             "Reserveren is verplicht om gebruik te maken van dit sportterrein",
+            "nl": "Reserveren is verplicht om gebruik te maken van dit sportterrein",
             "en": "Making an appointment is obligatory to use this sport pitch"
           }
         },
         {
           "if": "reservation=recommended",
           "then": {
-            "nl": 
+            "nl": "Reserveren is sterk aangeraden om gebruik te maken van dit sportterrein",
-            "Reserveren is sterk aangeraden om gebruik te maken van dit sportterrein",
             "en": "Making an appointment is recommended when using this sport pitch"
           }
         },
         {
           "if": "reservation=yes",
           "then": {
-            "nl":"Reserveren is mogelijk, maar geen voorwaarde",
+            "nl": "Reserveren is mogelijk, maar geen voorwaarde",
             "en": "Making an appointment is possible, but not necessary to use this sport pitch"
           }
         },
         {
           "if": "reservation=no",
           "then": {
-        "nl":    "Reserveren is niet mogelijk",
+            "nl": "Reserveren is niet mogelijk",
             "en": "Making an appointment is not possible"
           }
         }
@@ -227,7 +239,7 @@
     },
     {
       "question": {
-      "nl":  "Wat is het telefoonnummer van de bevoegde dienst of uitbater?",
+        "nl": "Wat is het telefoonnummer van de bevoegde dienst of uitbater?",
         "en": "What is the phone number of the operator?"
       },
       "freeform": {
@@ -306,7 +318,8 @@
   "presets": [
     {
       "title": {
-        "nl": "Ping-pong tafel"
+        "nl": "Ping-pong tafel",
+        "en": "Tabletennis table"
       },
       "tags": [
         "leisure=pitch",
@@ -315,7 +328,8 @@
     },
     {
       "title": {
-        "nl": "Sportterrein"
+        "nl": "Sportterrein",
+        "en": "Sport pitch"
       },
       "tags": [
         "leisure=pitch",

scripts/generateLayerOverview.ts

@@ -1,24 +1,22 @@
 import ScriptUtils from "./ScriptUtils";
 import {Utils} from "../Utils";
-import {lstatSync, readdirSync, readFileSync, writeFileSync} from "fs";
+import {readFileSync, writeFileSync} from "fs";
 
 Utils.runningFromConsole = true
 import LayerConfig from "../Customizations/JSON/LayerConfig";
-import {error} from "util";
 import * as licenses from "../assets/generated/license_info.json"
-import SmallLicense from "../Models/smallLicense";
 import LayoutConfig from "../Customizations/JSON/LayoutConfig";
 import {LayerConfigJson} from "../Customizations/JSON/LayerConfigJson";
-import {Layer} from "leaflet";
+import {Translation} from "../UI/i18n/Translation";
 // This scripts scans 'assets/layers/*.json' for layer definition files and 'assets/themes/*.json' for theme definition files.
 // It spits out an overview of those to be used to load them
 
+interface LayersAndThemes {
+    themes: any[] , layers: {parsed: any, path: string}[]
+}
+
+function loadThemesAndLayers():LayersAndThemes{
     
-// First, remove the old file. It might be buggy!
-writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({
-    "layers": [],
-    "themes": []
-}))
 const layerFiles = ScriptUtils.readDirRecSync("./assets/layers")
     .filter(path => path.indexOf(".json") > 0)
     .filter(path => path.indexOf("license_info.json") < 0)
@@ -36,22 +34,22 @@ const themeFiles: any[] = ScriptUtils.readDirRecSync("./assets/themes")
     .map(path => {
         return JSON.parse(readFileSync(path, "UTF8"));
     })
-writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({
-    "layers": layerFiles.map(l => l.parsed),
-    "themes": themeFiles
-}))
-
 
 console.log("Discovered", layerFiles.length, "layers and", themeFiles.length, "themes\n")
-console.log("   ---------- VALIDATING ---------")
+return {
-// ------------- VALIDATION --------------
+    layers: layerFiles,
-const licensePaths = []
+    themes: themeFiles
-for (const i in licenses) {
+}
-    licensePaths.push(licenses[i].path)
 }
-const knownPaths = new Set<string>(licensePaths)
 
-function validateLayer(layerJson: LayerConfigJson, path: string, context?: string): string[] {
+function writeFiles(lt: LayersAndThemes){
+    writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({
+        "layers": lt.layers.map(l => l.parsed),
+        "themes": lt.themes
+    }))
+}
+
+function validateLayer(layerJson: LayerConfigJson, path: string, knownPaths: Set<string>, context?: string): string[] {
     let errorCount = [];
     if (layerJson["overpassTags"] !== undefined) {
         errorCount.push("Layer " + layerJson.id + "still uses the old 'overpassTags'-format. Please use \"source\": {\"osmTags\": <tags>}' instead of \"overpassTags\": <tags> (note: this isn't your fault, the custom theme generator still spits out the old format)")
@@ -61,8 +59,7 @@ function validateLayer(layerJson: LayerConfigJson, path: string, context?: strin
         const images = Array.from(layer.ExtractImages())
         const remoteImages = images.filter(img => img.indexOf("http") == 0)
         for (const remoteImage of remoteImages) {
-            errorCount.push("Found a remote image: " + remoteImage + " in layer " + layer.id + ", please download it.")
+            errorCount.push("Found a remote image: " + remoteImage + " in layer " + layer.id + ", please download it. You can use the fixTheme script to automate this")
-            const path = remoteImage.substring(remoteImage.lastIndexOf("/") + 1)
         }
         const expected: string = `assets/layers/${layer.id}/${layer.id}.json`
         if (path != undefined && path.indexOf(expected) < 0) {
@@ -83,60 +80,151 @@ function validateLayer(layerJson: LayerConfigJson, path: string, context?: strin
     return errorCount
 }
 
-let layerErrorCount = []
+function validateTranslationCompletenessOfObject(object: any, expectedLanguages: string[], context: string) {
-const knownLayerIds = new Set<string>();
+    const missingTranlations = []
-for (const layerFile of layerFiles) {
+    const translations: {tr: Translation, context: string}[] = [];
-    knownLayerIds.add(layerFile.parsed.id)
+    const queue: {object: any, context: string}[] = [{object: object, context: context}]
-    layerErrorCount.push(...validateLayer(layerFile.parsed, layerFile.path))
+
+    while (queue.length > 0) {
+        const item = queue.pop();
+        const o = item.object
+        for (const key in o) {
+            const v = o[key];
+            if (v === undefined) {
+                continue;
+            }
+            if (v instanceof Translation || v?.translations !== undefined) {
+                translations.push({tr: v, context: item.context});
+            } else if (
+                ["string", "function", "boolean", "number"].indexOf(typeof (v)) < 0) {
+                queue.push({object: v, context: item.context + "." + key})
+            }
+        }
+    }
+
+    const missing = {}
+    const present = {}
+    for (const ln of expectedLanguages) {
+        missing[ln] = 0;
+        present[ln] = 0;
+        for (const translation of translations) {
+            if (translation.tr.translations["*"] !== undefined) {
+                continue;
+            }
+            const txt = translation.tr.translations[ln];
+            const isMissing = txt === undefined || txt === "" || txt.toLowerCase().indexOf("todo") >= 0;
+            if (isMissing) {
+                missingTranlations.push(`${translation.context},${ln},${translation.tr.txt}`)
+                missing[ln]++
+            } else {
+                present[ln]++;
+            }
+        }
+    }
+
+    let message = `Translation completeness for ${context}`
+    let isComplete = true;
+    for (const ln of expectedLanguages) {
+        const amiss = missing[ln];
+        const ok = present[ln];
+        const total = amiss + ok;
+        message += ` ${ln}: ${ok}/${total}`
+        if (ok !== total) {
+            isComplete = false;
+        }
+    }
+    if (!isComplete) {
+        console.log(message)
+    }
+    return missingTranlations
+
 }
 
-let themeErrorCount = []
+function main(args: string[]) {
-for (const themeFile of themeFiles) {
+    
+    const lt = loadThemesAndLayers();
+    const layerFiles = lt.layers;
+    const themeFiles = lt.themes;
+    
+    console.log("   ---------- VALIDATING ---------")
+    const licensePaths = []
+    for (const i in licenses) {
+        licensePaths.push(licenses[i].path)
+    }
+    const knownPaths = new Set<string>(licensePaths)
+
+    let layerErrorCount = []
+    const knownLayerIds = new Map<string, LayerConfig>();
+    for (const layerFile of layerFiles) {
+
+        layerErrorCount.push(...validateLayer(layerFile.parsed, layerFile.path, knownPaths))
+        knownLayerIds.set(layerFile.parsed.id, new LayerConfig(layerFile.parsed))
+    }
+
+    let themeErrorCount = []
+    let missingTranslations = []
+    for (const themeFile of themeFiles) {
+        if (typeof themeFile.language === "string") {
+            themeErrorCount.push("The theme " + themeFile.id + " has a string as language. Please use a list of strings")
+        }
+        for (const layer of themeFile.layers) {
+            if (typeof layer === "string") {
+                if (!knownLayerIds.has(layer)) {
+                    themeErrorCount.push(`Unknown layer id: ${layer} in theme ${themeFile.id}`)
+                } else {
+                    const layerConfig = knownLayerIds.get(layer);
+                    missingTranslations.push(...validateTranslationCompletenessOfObject(layerConfig, themeFile.language, "Layer " + layer))
 
-    for (const layer of themeFile.layers) {
-        if (typeof layer === "string") {
-            if (!knownLayerIds.has(layer)) {
-                themeErrorCount.push(`Unknown layer id: ${layer} in theme ${themeFile.id}`)
-            }
-        } else {
-            if (layer.builtin !== undefined) {
-                if (!knownLayerIds.has(layer.builtin)) {
-                    themeErrorCount.push("Unknown layer id: " + layer.builtin + "(which uses inheritance)")
                 }
             } else {
-                // layer.builtin contains layer overrides - we can skip those
+                if (layer.builtin !== undefined) {
-                layerErrorCount.push(...validateLayer(layer, undefined, themeFile.id))
+                    if (!knownLayerIds.has(layer.builtin)) {
+                        themeErrorCount.push("Unknown layer id: " + layer.builtin + "(which uses inheritance)")
+                    }
+                } else {
+                    // layer.builtin contains layer overrides - we can skip those
+                    layerErrorCount.push(...validateLayer(layer, undefined, knownPaths, themeFile.id))
+                }
             }
         }
-    }
 
-    themeFile.layers = themeFile.layers
+        themeFile.layers = themeFile.layers
-        .filter(l => typeof l != "string") // We remove all the builtin layer references as they don't work with ts-node for some weird reason
+            .filter(l => typeof l != "string") // We remove all the builtin layer references as they don't work with ts-node for some weird reason
-        .filter(l => l.builtin === undefined)
+            .filter(l => l.builtin === undefined)
 
-    try {
+        missingTranslations.push(...validateTranslationCompletenessOfObject(themeFile, themeFile.language, "Theme " + themeFile.id))
-        const theme = new LayoutConfig(themeFile, true, "test")
+
-        if (theme.id !== theme.id.toLowerCase()) {
+        try {
-            themeErrorCount.push("Theme ids should be in lowercase, but it is " + theme.id)
+            const theme = new LayoutConfig(themeFile, true, "test")
+            if (theme.id !== theme.id.toLowerCase()) {
+                themeErrorCount.push("Theme ids should be in lowercase, but it is " + theme.id)
+            }
+        } catch (e) {
+            themeErrorCount.push("Could not parse theme " + themeFile["id"] + "due to", e)
+        }
+    }
+
+    if(missingTranslations.length > 0){
+        writeFileSync("missing_translations.txt", missingTranslations.join("\n"))
+    }
+    
+    if (layerErrorCount.length + themeErrorCount.length == 0) {
+        console.log("All good!")
+        writeFiles(lt);
+    } else {
+        const errors = layerErrorCount.concat(themeErrorCount).join("\n")
+        console.log(errors)
+        const msg = (`Found ${layerErrorCount.length} errors in the layers; ${themeErrorCount.length} errors in the themes`)
+        console.log(msg)
+        if (process.argv.indexOf("--report") >= 0) {
+            console.log("Writing report!")
+            writeFileSync("layer_report.txt", errors)
+        }
+
+        if (process.argv.indexOf("--no-fail") < 0) {
+            throw msg;
         }
-    } catch (e) {
-        themeErrorCount.push("Could not parse theme " + themeFile["id"] + "due to", e)
     }
 }
 
-if (layerErrorCount.length + themeErrorCount.length == 0) {
+main(process.argv)
-    console.log("All good!")
-} else {
-    const errors = layerErrorCount.concat(themeErrorCount).join("\n")
-    console.log(errors)
-    const msg = (`Found ${layerErrorCount.length} errors in the layers; ${themeErrorCount.length} errors in the themes`)
-    console.log(msg)
-    if (process.argv.indexOf("--report") >= 0) {
-        console.log("Writing report!")
-        writeFileSync("layer_report.txt", errors)
-    }
-
-    if (process.argv.indexOf("--no-fail") < 0) {
-        throw msg;
-    }
-}

scripts/generateLayouts.ts

@@ -17,69 +17,9 @@ function enc(str: string): string {
     return encodeURIComponent(str.toLowerCase());
 }
 
-function validate(layout: LayoutConfig) {
-    const translations: Translation[] = [];
-    const queue: any[] = [layout]
-
-    while (queue.length > 0) {
-        const item = queue.pop();
-        for (const key in item) {
-            const v = item[key];
-            if (v === undefined) {
-                continue;
-            }
-            if (v instanceof Translation || v?.translations !== undefined) {
-                translations.push(v);
-            } else if (
-                ["string", "function", "boolean", "number"].indexOf(typeof (v)) < 0) {
-                queue.push(v)
-            }
-        }
-    }
-
-    const missing = {}
-    const present = {}
-    for (const ln of layout.language) {
-        missing[ln] = 0;
-        present[ln] = 0;
-        for (const translation of translations) {
-            if (translation.translations["*"] !== undefined) {
-                continue;
-            }
-            const txt = translation.translations[ln];
-            const isMissing = txt === undefined || txt === "" || txt.toLowerCase().indexOf("todo") >= 0;
-            if (isMissing) {
-                console.log(`   ${layout.id}: No translation for`, ln, "in", translation.translations, "got:", txt)
-                missing[ln]++
-            } else {
-                present[ln]++;
-            }
-        }
-    }
-
-    let message = `Translation completeness for theme ${layout.id}`
-    let isComplete = true;
-    for (const ln of layout.language) {
-        const amiss = missing[ln];
-        const ok = present[ln];
-        const total = amiss + ok;
-        message += `\n${ln}: ${ok}/${total}`
-        if (ok !== total) {
-            isComplete = false;
-        }
-    }
-    if (isComplete) {
-        console.log(`${layout.id} is fully translated!`)
-    } else {
-        console.log(message)
-    }
-
-}
-
-
 const alreadyWritten = []
 
-async function createIcon(iconPath: string, size: number, layout: LayoutConfig) {
+async function createIcon(iconPath: string, size: number) {
     let name = iconPath.split(".").slice(0, -1).join(".");
     if (name.startsWith("./")) {
         name = name.substr(2)
@@ -131,7 +71,7 @@ async function createManifest(layout: LayoutConfig, relativePath: string) {
 
         const sizes = [72, 96, 120, 128, 144, 152, 180, 192, 384, 512];
         for (const size of sizes) {
-            const name = await createIcon(path, size, layout);
+            const name = await createIcon(path, size);
             icons.push({
                 src: name,
                 sizes: size + "x" + size,
@@ -250,7 +190,6 @@ for (const i in all) {
             console.log("Could not write manifest for ", layoutName, " because ", err)
         }
     };
-    validate(layout)
     createManifest(layout, "").then(manifObj => {
         const manif = JSON.stringify(manifObj, undefined, 2);
         const manifestLocation = encodeURIComponent(layout.id.toLowerCase()) + ".webmanifest";