Add some more missing translations, small tweaks to scripts

This commit is contained in:
pietervdvn 2021-04-23 13:56:16 +02:00
parent 9fc8a7136f
commit b2213751c3
4 changed files with 188 additions and 145 deletions

View file

@ -1,7 +1,8 @@
{ {
"id": "direction", "id": "direction",
"name": { "name": {
"en": "Direction visualization" "en": "Direction visualization",
"nl": "Richtingsvisualisatie"
}, },
"minzoom": 16, "minzoom": 16,
"source": { "source": {
@ -16,7 +17,8 @@
"passAllFeatures": true, "passAllFeatures": true,
"title": null, "title": null,
"description": { "description": {
"en": "This layer visualizes directions" "en": "This layer visualizes directions",
"nl": "Deze laag toont de oriëntatie van een object"
}, },
"tagRenderings": [], "tagRenderings": [],
"icon": { "icon": {

View file

@ -160,24 +160,37 @@
}, },
{ {
"question": { "question": {
"nl": "Is dit sportterrein publiek toegankelijk?" "nl": "Is dit sportterrein publiek toegankelijk?",
"en": "Is this sport pitch publicly accessible?"
}, },
"mappings": [ "mappings": [
{ {
"if": "access=public", "if": "access=public",
"then": "Publiek toegankelijk" "then": {
"nl": "Publiek toegankelijk",
"en": "Public access"
}
}, },
{ {
"if": "access=limited", "if": "access=limited",
"then": "Beperkt toegankelijk (enkel na reservatie, tijdens bepaalde uren, ...)" "then": {
"nl": "Beperkt toegankelijk (enkel na reservatie, tijdens bepaalde uren, ...)",
"en": "Limited access (e.g. only with an appointment, during certain hours, ...)"
}
}, },
{ {
"if": "access=members", "if": "access=members",
"then": "Enkel toegankelijk voor leden van de bijhorende sportclub" "then": {
"nl": "Enkel toegankelijk voor leden van de bijhorende sportclub",
"en": "Only accessible for members of the club"
}
}, },
{ {
"if": "access=private", "if": "access=private",
"then": "Privaat en niet toegankelijk" "then": {
"nl": "Privaat en niet toegankelijk",
"en": "Private - not accessible to the public"
}
} }
] ]
}, },
@ -197,29 +210,28 @@
{ {
"if": "reservation=required", "if": "reservation=required",
"then": { "then": {
"nl": "Reserveren is verplicht om gebruik te maken van dit sportterrein", "nl": "Reserveren is verplicht om gebruik te maken van dit sportterrein",
"en": "Making an appointment is obligatory to use this sport pitch" "en": "Making an appointment is obligatory to use this sport pitch"
} }
}, },
{ {
"if": "reservation=recommended", "if": "reservation=recommended",
"then": { "then": {
"nl": "nl": "Reserveren is sterk aangeraden om gebruik te maken van dit sportterrein",
"Reserveren is sterk aangeraden om gebruik te maken van dit sportterrein",
"en": "Making an appointment is recommended when using this sport pitch" "en": "Making an appointment is recommended when using this sport pitch"
} }
}, },
{ {
"if": "reservation=yes", "if": "reservation=yes",
"then": { "then": {
"nl":"Reserveren is mogelijk, maar geen voorwaarde", "nl": "Reserveren is mogelijk, maar geen voorwaarde",
"en": "Making an appointment is possible, but not necessary to use this sport pitch" "en": "Making an appointment is possible, but not necessary to use this sport pitch"
} }
}, },
{ {
"if": "reservation=no", "if": "reservation=no",
"then": { "then": {
"nl": "Reserveren is niet mogelijk", "nl": "Reserveren is niet mogelijk",
"en": "Making an appointment is not possible" "en": "Making an appointment is not possible"
} }
} }
@ -227,7 +239,7 @@
}, },
{ {
"question": { "question": {
"nl": "Wat is het telefoonnummer van de bevoegde dienst of uitbater?", "nl": "Wat is het telefoonnummer van de bevoegde dienst of uitbater?",
"en": "What is the phone number of the operator?" "en": "What is the phone number of the operator?"
}, },
"freeform": { "freeform": {
@ -306,7 +318,8 @@
"presets": [ "presets": [
{ {
"title": { "title": {
"nl": "Ping-pong tafel" "nl": "Ping-pong tafel",
"en": "Tabletennis table"
}, },
"tags": [ "tags": [
"leisure=pitch", "leisure=pitch",
@ -315,7 +328,8 @@
}, },
{ {
"title": { "title": {
"nl": "Sportterrein" "nl": "Sportterrein",
"en": "Sport pitch"
}, },
"tags": [ "tags": [
"leisure=pitch", "leisure=pitch",

View file

@ -1,24 +1,22 @@
import ScriptUtils from "./ScriptUtils"; import ScriptUtils from "./ScriptUtils";
import {Utils} from "../Utils"; import {Utils} from "../Utils";
import {lstatSync, readdirSync, readFileSync, writeFileSync} from "fs"; import {readFileSync, writeFileSync} from "fs";
Utils.runningFromConsole = true Utils.runningFromConsole = true
import LayerConfig from "../Customizations/JSON/LayerConfig"; import LayerConfig from "../Customizations/JSON/LayerConfig";
import {error} from "util";
import * as licenses from "../assets/generated/license_info.json" import * as licenses from "../assets/generated/license_info.json"
import SmallLicense from "../Models/smallLicense";
import LayoutConfig from "../Customizations/JSON/LayoutConfig"; import LayoutConfig from "../Customizations/JSON/LayoutConfig";
import {LayerConfigJson} from "../Customizations/JSON/LayerConfigJson"; import {LayerConfigJson} from "../Customizations/JSON/LayerConfigJson";
import {Layer} from "leaflet"; import {Translation} from "../UI/i18n/Translation";
// This scripts scans 'assets/layers/*.json' for layer definition files and 'assets/themes/*.json' for theme definition files. // This scripts scans 'assets/layers/*.json' for layer definition files and 'assets/themes/*.json' for theme definition files.
// It spits out an overview of those to be used to load them // It spits out an overview of those to be used to load them
interface LayersAndThemes {
themes: any[] , layers: {parsed: any, path: string}[]
}
function loadThemesAndLayers():LayersAndThemes{
// First, remove the old file. It might be buggy!
writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({
"layers": [],
"themes": []
}))
const layerFiles = ScriptUtils.readDirRecSync("./assets/layers") const layerFiles = ScriptUtils.readDirRecSync("./assets/layers")
.filter(path => path.indexOf(".json") > 0) .filter(path => path.indexOf(".json") > 0)
.filter(path => path.indexOf("license_info.json") < 0) .filter(path => path.indexOf("license_info.json") < 0)
@ -36,22 +34,22 @@ const themeFiles: any[] = ScriptUtils.readDirRecSync("./assets/themes")
.map(path => { .map(path => {
return JSON.parse(readFileSync(path, "UTF8")); return JSON.parse(readFileSync(path, "UTF8"));
}) })
writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({
"layers": layerFiles.map(l => l.parsed),
"themes": themeFiles
}))
console.log("Discovered", layerFiles.length, "layers and", themeFiles.length, "themes\n") console.log("Discovered", layerFiles.length, "layers and", themeFiles.length, "themes\n")
console.log(" ---------- VALIDATING ---------") return {
// ------------- VALIDATION -------------- layers: layerFiles,
const licensePaths = [] themes: themeFiles
for (const i in licenses) { }
licensePaths.push(licenses[i].path)
} }
const knownPaths = new Set<string>(licensePaths)
function validateLayer(layerJson: LayerConfigJson, path: string, context?: string): string[] { function writeFiles(lt: LayersAndThemes){
writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({
"layers": lt.layers.map(l => l.parsed),
"themes": lt.themes
}))
}
function validateLayer(layerJson: LayerConfigJson, path: string, knownPaths: Set<string>, context?: string): string[] {
let errorCount = []; let errorCount = [];
if (layerJson["overpassTags"] !== undefined) { if (layerJson["overpassTags"] !== undefined) {
errorCount.push("Layer " + layerJson.id + "still uses the old 'overpassTags'-format. Please use \"source\": {\"osmTags\": <tags>}' instead of \"overpassTags\": <tags> (note: this isn't your fault, the custom theme generator still spits out the old format)") errorCount.push("Layer " + layerJson.id + "still uses the old 'overpassTags'-format. Please use \"source\": {\"osmTags\": <tags>}' instead of \"overpassTags\": <tags> (note: this isn't your fault, the custom theme generator still spits out the old format)")
@ -61,8 +59,7 @@ function validateLayer(layerJson: LayerConfigJson, path: string, context?: strin
const images = Array.from(layer.ExtractImages()) const images = Array.from(layer.ExtractImages())
const remoteImages = images.filter(img => img.indexOf("http") == 0) const remoteImages = images.filter(img => img.indexOf("http") == 0)
for (const remoteImage of remoteImages) { for (const remoteImage of remoteImages) {
errorCount.push("Found a remote image: " + remoteImage + " in layer " + layer.id + ", please download it.") errorCount.push("Found a remote image: " + remoteImage + " in layer " + layer.id + ", please download it. You can use the fixTheme script to automate this")
const path = remoteImage.substring(remoteImage.lastIndexOf("/") + 1)
} }
const expected: string = `assets/layers/${layer.id}/${layer.id}.json` const expected: string = `assets/layers/${layer.id}/${layer.id}.json`
if (path != undefined && path.indexOf(expected) < 0) { if (path != undefined && path.indexOf(expected) < 0) {
@ -83,60 +80,151 @@ function validateLayer(layerJson: LayerConfigJson, path: string, context?: strin
return errorCount return errorCount
} }
let layerErrorCount = [] function validateTranslationCompletenessOfObject(object: any, expectedLanguages: string[], context: string) {
const knownLayerIds = new Set<string>(); const missingTranlations = []
for (const layerFile of layerFiles) { const translations: {tr: Translation, context: string}[] = [];
knownLayerIds.add(layerFile.parsed.id) const queue: {object: any, context: string}[] = [{object: object, context: context}]
layerErrorCount.push(...validateLayer(layerFile.parsed, layerFile.path))
while (queue.length > 0) {
const item = queue.pop();
const o = item.object
for (const key in o) {
const v = o[key];
if (v === undefined) {
continue;
}
if (v instanceof Translation || v?.translations !== undefined) {
translations.push({tr: v, context: item.context});
} else if (
["string", "function", "boolean", "number"].indexOf(typeof (v)) < 0) {
queue.push({object: v, context: item.context + "." + key})
}
}
}
const missing = {}
const present = {}
for (const ln of expectedLanguages) {
missing[ln] = 0;
present[ln] = 0;
for (const translation of translations) {
if (translation.tr.translations["*"] !== undefined) {
continue;
}
const txt = translation.tr.translations[ln];
const isMissing = txt === undefined || txt === "" || txt.toLowerCase().indexOf("todo") >= 0;
if (isMissing) {
missingTranlations.push(`${translation.context},${ln},${translation.tr.txt}`)
missing[ln]++
} else {
present[ln]++;
}
}
}
let message = `Translation completeness for ${context}`
let isComplete = true;
for (const ln of expectedLanguages) {
const amiss = missing[ln];
const ok = present[ln];
const total = amiss + ok;
message += ` ${ln}: ${ok}/${total}`
if (ok !== total) {
isComplete = false;
}
}
if (!isComplete) {
console.log(message)
}
return missingTranlations
} }
let themeErrorCount = [] function main(args: string[]) {
for (const themeFile of themeFiles) {
const lt = loadThemesAndLayers();
const layerFiles = lt.layers;
const themeFiles = lt.themes;
console.log(" ---------- VALIDATING ---------")
const licensePaths = []
for (const i in licenses) {
licensePaths.push(licenses[i].path)
}
const knownPaths = new Set<string>(licensePaths)
let layerErrorCount = []
const knownLayerIds = new Map<string, LayerConfig>();
for (const layerFile of layerFiles) {
layerErrorCount.push(...validateLayer(layerFile.parsed, layerFile.path, knownPaths))
knownLayerIds.set(layerFile.parsed.id, new LayerConfig(layerFile.parsed))
}
let themeErrorCount = []
let missingTranslations = []
for (const themeFile of themeFiles) {
if (typeof themeFile.language === "string") {
themeErrorCount.push("The theme " + themeFile.id + " has a string as language. Please use a list of strings")
}
for (const layer of themeFile.layers) {
if (typeof layer === "string") {
if (!knownLayerIds.has(layer)) {
themeErrorCount.push(`Unknown layer id: ${layer} in theme ${themeFile.id}`)
} else {
const layerConfig = knownLayerIds.get(layer);
missingTranslations.push(...validateTranslationCompletenessOfObject(layerConfig, themeFile.language, "Layer " + layer))
for (const layer of themeFile.layers) {
if (typeof layer === "string") {
if (!knownLayerIds.has(layer)) {
themeErrorCount.push(`Unknown layer id: ${layer} in theme ${themeFile.id}`)
}
} else {
if (layer.builtin !== undefined) {
if (!knownLayerIds.has(layer.builtin)) {
themeErrorCount.push("Unknown layer id: " + layer.builtin + "(which uses inheritance)")
} }
} else { } else {
// layer.builtin contains layer overrides - we can skip those if (layer.builtin !== undefined) {
layerErrorCount.push(...validateLayer(layer, undefined, themeFile.id)) if (!knownLayerIds.has(layer.builtin)) {
themeErrorCount.push("Unknown layer id: " + layer.builtin + "(which uses inheritance)")
}
} else {
// layer.builtin contains layer overrides - we can skip those
layerErrorCount.push(...validateLayer(layer, undefined, knownPaths, themeFile.id))
}
} }
} }
}
themeFile.layers = themeFile.layers themeFile.layers = themeFile.layers
.filter(l => typeof l != "string") // We remove all the builtin layer references as they don't work with ts-node for some weird reason .filter(l => typeof l != "string") // We remove all the builtin layer references as they don't work with ts-node for some weird reason
.filter(l => l.builtin === undefined) .filter(l => l.builtin === undefined)
try { missingTranslations.push(...validateTranslationCompletenessOfObject(themeFile, themeFile.language, "Theme " + themeFile.id))
const theme = new LayoutConfig(themeFile, true, "test")
if (theme.id !== theme.id.toLowerCase()) { try {
themeErrorCount.push("Theme ids should be in lowercase, but it is " + theme.id) const theme = new LayoutConfig(themeFile, true, "test")
if (theme.id !== theme.id.toLowerCase()) {
themeErrorCount.push("Theme ids should be in lowercase, but it is " + theme.id)
}
} catch (e) {
themeErrorCount.push("Could not parse theme " + themeFile["id"] + "due to", e)
}
}
if(missingTranslations.length > 0){
writeFileSync("missing_translations.txt", missingTranslations.join("\n"))
}
if (layerErrorCount.length + themeErrorCount.length == 0) {
console.log("All good!")
writeFiles(lt);
} else {
const errors = layerErrorCount.concat(themeErrorCount).join("\n")
console.log(errors)
const msg = (`Found ${layerErrorCount.length} errors in the layers; ${themeErrorCount.length} errors in the themes`)
console.log(msg)
if (process.argv.indexOf("--report") >= 0) {
console.log("Writing report!")
writeFileSync("layer_report.txt", errors)
}
if (process.argv.indexOf("--no-fail") < 0) {
throw msg;
} }
} catch (e) {
themeErrorCount.push("Could not parse theme " + themeFile["id"] + "due to", e)
} }
} }
if (layerErrorCount.length + themeErrorCount.length == 0) { main(process.argv)
console.log("All good!")
} else {
const errors = layerErrorCount.concat(themeErrorCount).join("\n")
console.log(errors)
const msg = (`Found ${layerErrorCount.length} errors in the layers; ${themeErrorCount.length} errors in the themes`)
console.log(msg)
if (process.argv.indexOf("--report") >= 0) {
console.log("Writing report!")
writeFileSync("layer_report.txt", errors)
}
if (process.argv.indexOf("--no-fail") < 0) {
throw msg;
}
}

View file

@ -17,69 +17,9 @@ function enc(str: string): string {
return encodeURIComponent(str.toLowerCase()); return encodeURIComponent(str.toLowerCase());
} }
function validate(layout: LayoutConfig) {
const translations: Translation[] = [];
const queue: any[] = [layout]
while (queue.length > 0) {
const item = queue.pop();
for (const key in item) {
const v = item[key];
if (v === undefined) {
continue;
}
if (v instanceof Translation || v?.translations !== undefined) {
translations.push(v);
} else if (
["string", "function", "boolean", "number"].indexOf(typeof (v)) < 0) {
queue.push(v)
}
}
}
const missing = {}
const present = {}
for (const ln of layout.language) {
missing[ln] = 0;
present[ln] = 0;
for (const translation of translations) {
if (translation.translations["*"] !== undefined) {
continue;
}
const txt = translation.translations[ln];
const isMissing = txt === undefined || txt === "" || txt.toLowerCase().indexOf("todo") >= 0;
if (isMissing) {
console.log(` ${layout.id}: No translation for`, ln, "in", translation.translations, "got:", txt)
missing[ln]++
} else {
present[ln]++;
}
}
}
let message = `Translation completeness for theme ${layout.id}`
let isComplete = true;
for (const ln of layout.language) {
const amiss = missing[ln];
const ok = present[ln];
const total = amiss + ok;
message += `\n${ln}: ${ok}/${total}`
if (ok !== total) {
isComplete = false;
}
}
if (isComplete) {
console.log(`${layout.id} is fully translated!`)
} else {
console.log(message)
}
}
const alreadyWritten = [] const alreadyWritten = []
async function createIcon(iconPath: string, size: number, layout: LayoutConfig) { async function createIcon(iconPath: string, size: number) {
let name = iconPath.split(".").slice(0, -1).join("."); let name = iconPath.split(".").slice(0, -1).join(".");
if (name.startsWith("./")) { if (name.startsWith("./")) {
name = name.substr(2) name = name.substr(2)
@ -131,7 +71,7 @@ async function createManifest(layout: LayoutConfig, relativePath: string) {
const sizes = [72, 96, 120, 128, 144, 152, 180, 192, 384, 512]; const sizes = [72, 96, 120, 128, 144, 152, 180, 192, 384, 512];
for (const size of sizes) { for (const size of sizes) {
const name = await createIcon(path, size, layout); const name = await createIcon(path, size);
icons.push({ icons.push({
src: name, src: name,
sizes: size + "x" + size, sizes: size + "x" + size,
@ -250,7 +190,6 @@ for (const i in all) {
console.log("Could not write manifest for ", layoutName, " because ", err) console.log("Could not write manifest for ", layoutName, " because ", err)
} }
}; };
validate(layout)
createManifest(layout, "").then(manifObj => { createManifest(layout, "").then(manifObj => {
const manif = JSON.stringify(manifObj, undefined, 2); const manif = JSON.stringify(manifObj, undefined, 2);
const manifestLocation = encodeURIComponent(layout.id.toLowerCase()) + ".webmanifest"; const manifestLocation = encodeURIComponent(layout.id.toLowerCase()) + ".webmanifest";