From b2213751c398be02eda592f0ccf2584ca07380b6 Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Fri, 23 Apr 2021 13:56:16 +0200 Subject: [PATCH] Add some more missing translations, small tweaks to scripts --- assets/layers/direction/direction.json | 6 +- assets/layers/sport_pitch/sport_pitch.json | 40 ++-- scripts/generateLayerOverview.ts | 222 ++++++++++++++------- scripts/generateLayouts.ts | 65 +----- 4 files changed, 188 insertions(+), 145 deletions(-) diff --git a/assets/layers/direction/direction.json b/assets/layers/direction/direction.json index f4dd164..da9b20a 100644 --- a/assets/layers/direction/direction.json +++ b/assets/layers/direction/direction.json @@ -1,7 +1,8 @@ { "id": "direction", "name": { - "en": "Direction visualization" + "en": "Direction visualization", + "nl": "Richtingsvisualisatie" }, "minzoom": 16, "source": { @@ -16,7 +17,8 @@ "passAllFeatures": true, "title": null, "description": { - "en": "This layer visualizes directions" + "en": "This layer visualizes directions", + "nl": "Deze laag toont de oriëntatie van een object" }, "tagRenderings": [], "icon": { diff --git a/assets/layers/sport_pitch/sport_pitch.json b/assets/layers/sport_pitch/sport_pitch.json index 3bdfac5..51e6995 100644 --- a/assets/layers/sport_pitch/sport_pitch.json +++ b/assets/layers/sport_pitch/sport_pitch.json @@ -160,24 +160,37 @@ }, { "question": { - "nl": "Is dit sportterrein publiek toegankelijk?" + "nl": "Is dit sportterrein publiek toegankelijk?", + "en": "Is this sport pitch publicly accessible?" }, "mappings": [ { "if": "access=public", - "then": "Publiek toegankelijk" + "then": { + "nl": "Publiek toegankelijk", + "en": "Public access" + } }, { "if": "access=limited", - "then": "Beperkt toegankelijk (enkel na reservatie, tijdens bepaalde uren, ...)" + "then": { + "nl": "Beperkt toegankelijk (enkel na reservatie, tijdens bepaalde uren, ...)", + "en": "Limited access (e.g. only with an appointment, during certain hours, ...)" + } }, { "if": "access=members", - "then": "Enkel toegankelijk voor leden van de bijhorende sportclub" + "then": { + "nl": "Enkel toegankelijk voor leden van de bijhorende sportclub", + "en": "Only accessible for members of the club" + } }, { "if": "access=private", - "then": "Privaat en niet toegankelijk" + "then": { + "nl": "Privaat en niet toegankelijk", + "en": "Private - not accessible to the public" + } } ] }, @@ -197,29 +210,28 @@ { "if": "reservation=required", "then": { - "nl": "Reserveren is verplicht om gebruik te maken van dit sportterrein", + "nl": "Reserveren is verplicht om gebruik te maken van dit sportterrein", "en": "Making an appointment is obligatory to use this sport pitch" } }, { "if": "reservation=recommended", "then": { - "nl": - "Reserveren is sterk aangeraden om gebruik te maken van dit sportterrein", + "nl": "Reserveren is sterk aangeraden om gebruik te maken van dit sportterrein", "en": "Making an appointment is recommended when using this sport pitch" } }, { "if": "reservation=yes", "then": { - "nl":"Reserveren is mogelijk, maar geen voorwaarde", + "nl": "Reserveren is mogelijk, maar geen voorwaarde", "en": "Making an appointment is possible, but not necessary to use this sport pitch" } }, { "if": "reservation=no", "then": { - "nl": "Reserveren is niet mogelijk", + "nl": "Reserveren is niet mogelijk", "en": "Making an appointment is not possible" } } @@ -227,7 +239,7 @@ }, { "question": { - "nl": "Wat is het telefoonnummer van de bevoegde dienst of uitbater?", + "nl": "Wat is het telefoonnummer van de bevoegde dienst of uitbater?", "en": "What is the phone number of the operator?" }, "freeform": { @@ -306,7 +318,8 @@ "presets": [ { "title": { - "nl": "Ping-pong tafel" + "nl": "Ping-pong tafel", + "en": "Tabletennis table" }, "tags": [ "leisure=pitch", @@ -315,7 +328,8 @@ }, { "title": { - "nl": "Sportterrein" + "nl": "Sportterrein", + "en": "Sport pitch" }, "tags": [ "leisure=pitch", diff --git a/scripts/generateLayerOverview.ts b/scripts/generateLayerOverview.ts index 313717e..f959d4e 100644 --- a/scripts/generateLayerOverview.ts +++ b/scripts/generateLayerOverview.ts @@ -1,24 +1,22 @@ import ScriptUtils from "./ScriptUtils"; import {Utils} from "../Utils"; -import {lstatSync, readdirSync, readFileSync, writeFileSync} from "fs"; +import {readFileSync, writeFileSync} from "fs"; Utils.runningFromConsole = true import LayerConfig from "../Customizations/JSON/LayerConfig"; -import {error} from "util"; import * as licenses from "../assets/generated/license_info.json" -import SmallLicense from "../Models/smallLicense"; import LayoutConfig from "../Customizations/JSON/LayoutConfig"; import {LayerConfigJson} from "../Customizations/JSON/LayerConfigJson"; -import {Layer} from "leaflet"; +import {Translation} from "../UI/i18n/Translation"; // This scripts scans 'assets/layers/*.json' for layer definition files and 'assets/themes/*.json' for theme definition files. // It spits out an overview of those to be used to load them +interface LayersAndThemes { + themes: any[] , layers: {parsed: any, path: string}[] +} -// First, remove the old file. It might be buggy! -writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({ - "layers": [], - "themes": [] -})) +function loadThemesAndLayers():LayersAndThemes{ + const layerFiles = ScriptUtils.readDirRecSync("./assets/layers") .filter(path => path.indexOf(".json") > 0) .filter(path => path.indexOf("license_info.json") < 0) @@ -36,22 +34,22 @@ const themeFiles: any[] = ScriptUtils.readDirRecSync("./assets/themes") .map(path => { return JSON.parse(readFileSync(path, "UTF8")); }) -writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({ - "layers": layerFiles.map(l => l.parsed), - "themes": themeFiles -})) - console.log("Discovered", layerFiles.length, "layers and", themeFiles.length, "themes\n") -console.log(" ---------- VALIDATING ---------") -// ------------- VALIDATION -------------- -const licensePaths = [] -for (const i in licenses) { - licensePaths.push(licenses[i].path) +return { + layers: layerFiles, + themes: themeFiles +} } -const knownPaths = new Set(licensePaths) -function validateLayer(layerJson: LayerConfigJson, path: string, context?: string): string[] { +function writeFiles(lt: LayersAndThemes){ + writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({ + "layers": lt.layers.map(l => l.parsed), + "themes": lt.themes + })) +} + +function validateLayer(layerJson: LayerConfigJson, path: string, knownPaths: Set, context?: string): string[] { let errorCount = []; if (layerJson["overpassTags"] !== undefined) { errorCount.push("Layer " + layerJson.id + "still uses the old 'overpassTags'-format. Please use \"source\": {\"osmTags\": }' instead of \"overpassTags\": (note: this isn't your fault, the custom theme generator still spits out the old format)") @@ -61,8 +59,7 @@ function validateLayer(layerJson: LayerConfigJson, path: string, context?: strin const images = Array.from(layer.ExtractImages()) const remoteImages = images.filter(img => img.indexOf("http") == 0) for (const remoteImage of remoteImages) { - errorCount.push("Found a remote image: " + remoteImage + " in layer " + layer.id + ", please download it.") - const path = remoteImage.substring(remoteImage.lastIndexOf("/") + 1) + errorCount.push("Found a remote image: " + remoteImage + " in layer " + layer.id + ", please download it. You can use the fixTheme script to automate this") } const expected: string = `assets/layers/${layer.id}/${layer.id}.json` if (path != undefined && path.indexOf(expected) < 0) { @@ -83,60 +80,151 @@ function validateLayer(layerJson: LayerConfigJson, path: string, context?: strin return errorCount } -let layerErrorCount = [] -const knownLayerIds = new Set(); -for (const layerFile of layerFiles) { - knownLayerIds.add(layerFile.parsed.id) - layerErrorCount.push(...validateLayer(layerFile.parsed, layerFile.path)) +function validateTranslationCompletenessOfObject(object: any, expectedLanguages: string[], context: string) { + const missingTranlations = [] + const translations: {tr: Translation, context: string}[] = []; + const queue: {object: any, context: string}[] = [{object: object, context: context}] + + while (queue.length > 0) { + const item = queue.pop(); + const o = item.object + for (const key in o) { + const v = o[key]; + if (v === undefined) { + continue; + } + if (v instanceof Translation || v?.translations !== undefined) { + translations.push({tr: v, context: item.context}); + } else if ( + ["string", "function", "boolean", "number"].indexOf(typeof (v)) < 0) { + queue.push({object: v, context: item.context + "." + key}) + } + } + } + + const missing = {} + const present = {} + for (const ln of expectedLanguages) { + missing[ln] = 0; + present[ln] = 0; + for (const translation of translations) { + if (translation.tr.translations["*"] !== undefined) { + continue; + } + const txt = translation.tr.translations[ln]; + const isMissing = txt === undefined || txt === "" || txt.toLowerCase().indexOf("todo") >= 0; + if (isMissing) { + missingTranlations.push(`${translation.context},${ln},${translation.tr.txt}`) + missing[ln]++ + } else { + present[ln]++; + } + } + } + + let message = `Translation completeness for ${context}` + let isComplete = true; + for (const ln of expectedLanguages) { + const amiss = missing[ln]; + const ok = present[ln]; + const total = amiss + ok; + message += ` ${ln}: ${ok}/${total}` + if (ok !== total) { + isComplete = false; + } + } + if (!isComplete) { + console.log(message) + } + return missingTranlations + } -let themeErrorCount = [] -for (const themeFile of themeFiles) { +function main(args: string[]) { + + const lt = loadThemesAndLayers(); + const layerFiles = lt.layers; + const themeFiles = lt.themes; + + console.log(" ---------- VALIDATING ---------") + const licensePaths = [] + for (const i in licenses) { + licensePaths.push(licenses[i].path) + } + const knownPaths = new Set(licensePaths) + + let layerErrorCount = [] + const knownLayerIds = new Map(); + for (const layerFile of layerFiles) { + + layerErrorCount.push(...validateLayer(layerFile.parsed, layerFile.path, knownPaths)) + knownLayerIds.set(layerFile.parsed.id, new LayerConfig(layerFile.parsed)) + } + + let themeErrorCount = [] + let missingTranslations = [] + for (const themeFile of themeFiles) { + if (typeof themeFile.language === "string") { + themeErrorCount.push("The theme " + themeFile.id + " has a string as language. Please use a list of strings") + } + for (const layer of themeFile.layers) { + if (typeof layer === "string") { + if (!knownLayerIds.has(layer)) { + themeErrorCount.push(`Unknown layer id: ${layer} in theme ${themeFile.id}`) + } else { + const layerConfig = knownLayerIds.get(layer); + missingTranslations.push(...validateTranslationCompletenessOfObject(layerConfig, themeFile.language, "Layer " + layer)) - for (const layer of themeFile.layers) { - if (typeof layer === "string") { - if (!knownLayerIds.has(layer)) { - themeErrorCount.push(`Unknown layer id: ${layer} in theme ${themeFile.id}`) - } - } else { - if (layer.builtin !== undefined) { - if (!knownLayerIds.has(layer.builtin)) { - themeErrorCount.push("Unknown layer id: " + layer.builtin + "(which uses inheritance)") } } else { - // layer.builtin contains layer overrides - we can skip those - layerErrorCount.push(...validateLayer(layer, undefined, themeFile.id)) + if (layer.builtin !== undefined) { + if (!knownLayerIds.has(layer.builtin)) { + themeErrorCount.push("Unknown layer id: " + layer.builtin + "(which uses inheritance)") + } + } else { + // layer.builtin contains layer overrides - we can skip those + layerErrorCount.push(...validateLayer(layer, undefined, knownPaths, themeFile.id)) + } } } - } - themeFile.layers = themeFile.layers - .filter(l => typeof l != "string") // We remove all the builtin layer references as they don't work with ts-node for some weird reason - .filter(l => l.builtin === undefined) + themeFile.layers = themeFile.layers + .filter(l => typeof l != "string") // We remove all the builtin layer references as they don't work with ts-node for some weird reason + .filter(l => l.builtin === undefined) - try { - const theme = new LayoutConfig(themeFile, true, "test") - if (theme.id !== theme.id.toLowerCase()) { - themeErrorCount.push("Theme ids should be in lowercase, but it is " + theme.id) + missingTranslations.push(...validateTranslationCompletenessOfObject(themeFile, themeFile.language, "Theme " + themeFile.id)) + + try { + const theme = new LayoutConfig(themeFile, true, "test") + if (theme.id !== theme.id.toLowerCase()) { + themeErrorCount.push("Theme ids should be in lowercase, but it is " + theme.id) + } + } catch (e) { + themeErrorCount.push("Could not parse theme " + themeFile["id"] + "due to", e) + } + } + + if(missingTranslations.length > 0){ + writeFileSync("missing_translations.txt", missingTranslations.join("\n")) + } + + if (layerErrorCount.length + themeErrorCount.length == 0) { + console.log("All good!") + writeFiles(lt); + } else { + const errors = layerErrorCount.concat(themeErrorCount).join("\n") + console.log(errors) + const msg = (`Found ${layerErrorCount.length} errors in the layers; ${themeErrorCount.length} errors in the themes`) + console.log(msg) + if (process.argv.indexOf("--report") >= 0) { + console.log("Writing report!") + writeFileSync("layer_report.txt", errors) + } + + if (process.argv.indexOf("--no-fail") < 0) { + throw msg; } - } catch (e) { - themeErrorCount.push("Could not parse theme " + themeFile["id"] + "due to", e) } } -if (layerErrorCount.length + themeErrorCount.length == 0) { - console.log("All good!") -} else { - const errors = layerErrorCount.concat(themeErrorCount).join("\n") - console.log(errors) - const msg = (`Found ${layerErrorCount.length} errors in the layers; ${themeErrorCount.length} errors in the themes`) - console.log(msg) - if (process.argv.indexOf("--report") >= 0) { - console.log("Writing report!") - writeFileSync("layer_report.txt", errors) - } - - if (process.argv.indexOf("--no-fail") < 0) { - throw msg; - } -} +main(process.argv) \ No newline at end of file diff --git a/scripts/generateLayouts.ts b/scripts/generateLayouts.ts index 39ce0fd..86540c2 100644 --- a/scripts/generateLayouts.ts +++ b/scripts/generateLayouts.ts @@ -17,69 +17,9 @@ function enc(str: string): string { return encodeURIComponent(str.toLowerCase()); } -function validate(layout: LayoutConfig) { - const translations: Translation[] = []; - const queue: any[] = [layout] - - while (queue.length > 0) { - const item = queue.pop(); - for (const key in item) { - const v = item[key]; - if (v === undefined) { - continue; - } - if (v instanceof Translation || v?.translations !== undefined) { - translations.push(v); - } else if ( - ["string", "function", "boolean", "number"].indexOf(typeof (v)) < 0) { - queue.push(v) - } - } - } - - const missing = {} - const present = {} - for (const ln of layout.language) { - missing[ln] = 0; - present[ln] = 0; - for (const translation of translations) { - if (translation.translations["*"] !== undefined) { - continue; - } - const txt = translation.translations[ln]; - const isMissing = txt === undefined || txt === "" || txt.toLowerCase().indexOf("todo") >= 0; - if (isMissing) { - console.log(` ${layout.id}: No translation for`, ln, "in", translation.translations, "got:", txt) - missing[ln]++ - } else { - present[ln]++; - } - } - } - - let message = `Translation completeness for theme ${layout.id}` - let isComplete = true; - for (const ln of layout.language) { - const amiss = missing[ln]; - const ok = present[ln]; - const total = amiss + ok; - message += `\n${ln}: ${ok}/${total}` - if (ok !== total) { - isComplete = false; - } - } - if (isComplete) { - console.log(`${layout.id} is fully translated!`) - } else { - console.log(message) - } - -} - - const alreadyWritten = [] -async function createIcon(iconPath: string, size: number, layout: LayoutConfig) { +async function createIcon(iconPath: string, size: number) { let name = iconPath.split(".").slice(0, -1).join("."); if (name.startsWith("./")) { name = name.substr(2) @@ -131,7 +71,7 @@ async function createManifest(layout: LayoutConfig, relativePath: string) { const sizes = [72, 96, 120, 128, 144, 152, 180, 192, 384, 512]; for (const size of sizes) { - const name = await createIcon(path, size, layout); + const name = await createIcon(path, size); icons.push({ src: name, sizes: size + "x" + size, @@ -250,7 +190,6 @@ for (const i in all) { console.log("Could not write manifest for ", layoutName, " because ", err) } }; - validate(layout) createManifest(layout, "").then(manifObj => { const manif = JSON.stringify(manifObj, undefined, 2); const manifestLocation = encodeURIComponent(layout.id.toLowerCase()) + ".webmanifest";