import * as fs from "fs" import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs" import { Utils } from "../src/Utils" import ScriptUtils from "./ScriptUtils" import Script from "./Script" const knownLanguages = ["en", "nl", "de", "fr", "es", "gl", "ca"] const ignoreTerms = ["searchTerms"] class TranslationPart { contents: Map = new Map() static fromDirectory(path): TranslationPart { const files = ScriptUtils.readDirRecSync(path, 1).filter((file) => file.endsWith(".json")) const rootTranslation = new TranslationPart() for (const file of files) { const content = JSON.parse(readFileSync(file, { encoding: "utf8" })) const language = file.substr(0, file.length - ".json".length) rootTranslation.addTranslation(language, content) } return rootTranslation } /** * Add a leaf object * @param language * @param obj */ add(language: string, obj: any) { for (const key in obj) { const v = obj[key] if (!this.contents.has(key)) { this.contents.set(key, new TranslationPart()) } const subpart = this.contents.get(key) as TranslationPart if (typeof v === "string") { subpart.contents.set(language, v) } else { subpart.add(language, v) } } } addTranslationObject(translations: any, context?: string) { if (translations["#"] === "no-translations") { console.log("Ignoring object at ", context, "which has '#':'no-translations'") return } for (const translationsKey in translations) { const v = translations[translationsKey] if (typeof v != "string") { console.error( `Non-string object at ${context} in translation while trying to add the translation ` + JSON.stringify(v) + ` to '` + translationsKey + "'. The offending object which _should_ be a translation is: ", v, "\n\nThe current object is (only showing en):", this.toJson(), "and has translations for", Array.from(this.contents.keys()) ) throw ( "Error in an object depicting a translation: a non-string object was found. (" + context + ")\n You probably put some other section accidentally in the translation" ) } this.contents.set(translationsKey, v) } } recursiveAdd(object: any, context: string) { const isProbablyTranslationObject = knownLanguages.some((l) => object.hasOwnProperty(l)) // TODO FIXME ID if (isProbablyTranslationObject) { this.addTranslationObject(object, context) return } let dontTranslateKeys: string[] = undefined { const noTranslate = object["#dont-translate"] if (noTranslate === "*") { console.log("Ignoring translations for " + context) return } else if (typeof noTranslate === "string") { dontTranslateKeys = [noTranslate] } else { dontTranslateKeys = noTranslate } if (noTranslate !== undefined) { console.log( "Ignoring some translations for " + context + ": " + dontTranslateKeys.join(", ") ) } } for (let key in object) { if (ignoreTerms.indexOf(key) >= 0) { continue } if (dontTranslateKeys?.indexOf(key) >= 0) { continue } const v = object[key] if (v == null) { continue } if (typeof v !== "object") { continue } if (context.endsWith(".tagRenderings")) { if (v["id"] === undefined) { if (v["builtin"] !== undefined && typeof v["builtin"] === "string") { key = v["builtin"] } else { throw ( "At " + context + ": every object within a tagRenderings-list should have an id. " + JSON.stringify(v) + " has no id" ) } } else { // We use the embedded id as key instead of the index as this is more stable // Note: indonesian is shortened as 'id' as well! if (v["en"] !== undefined || v["nl"] !== undefined) { // This is probably a translation already! // pass } else { key = v["id"] if (typeof key !== "string") { throw "Panic: found a non-string ID at" + context } } } } if (!this.contents.get(key)) { this.contents.set(key, new TranslationPart()) } ;(this.contents.get(key) as TranslationPart).recursiveAdd(v, context + "." + key) } } knownLanguages(): string[] { const languages = [] for (const key of Array.from(this.contents.keys())) { const value = this.contents.get(key) if (typeof value === "string") { if (key === "#") { continue } languages.push(key) } else { languages.push(...(value as TranslationPart).knownLanguages()) } } return Utils.Dedup(languages) } toJson(neededLanguage?: string): string { const parts = [] let keys = Array.from(this.contents.keys()) keys = keys.sort() for (const key of keys) { let value = this.contents.get(key) if (typeof value === "string") { value = value.replace(/"/g, '\\"').replace(/\n/g, "\\n") if (neededLanguage === undefined) { parts.push(`"${key}": "${value}"`) } else if (key === neededLanguage) { return `"${value}"` } } else { const sub = (value as TranslationPart).toJson(neededLanguage) if (sub !== "") { parts.push(`"${key}": ${sub}`) } } } if (parts.length === 0) { return "" } return `{${parts.join(",")}}` } validateStrict(ctx?: string): void { const errors = this.validate() for (const err of errors) { console.error( "ERROR in " + (ctx ?? "") + " " + err.path.join(".") + "\n " + err.error ) } if (errors.length > 0) { throw ctx + " has " + errors.length + " inconsistencies in the translation" } } /** * Checks the leaf objects: special values must be present and identical in every leaf */ validate(path = []): { error: string; path: string[] }[] { const errors: { error: string; path: string[] }[] = [] const neededSubparts = new Set<{ part: string; usedByLanguage: string }>() let isLeaf: boolean = undefined this.contents.forEach((value, key) => { if (typeof value !== "string") { const recErrors = value.validate([...path, key]) errors.push(...recErrors) return } if (isLeaf === undefined) { isLeaf = true } else if (!isLeaf) { errors.push({ error: "Mixed node: non-leaf node has translation strings", path: path, }) } let subparts: string[] = value.match(/{[^}]*}/g) if (subparts !== null) { let [_, __, weblatepart, lang] = key.split("/") if (lang === undefined) { // This is a core translation, it has one less path segment lang = weblatepart } subparts = subparts.map((p) => p.split(/\(.*\)/)[0]) for (const subpart of subparts) { neededSubparts.add({ part: subpart, usedByLanguage: lang }) } } }) // Actually check for the needed sub-parts, e.g. that {key} isn't translated into {sleutel} this.contents.forEach((value, key) => { neededSubparts.forEach(({ part, usedByLanguage }) => { if (typeof value !== "string") { return } let [_, __, weblatepart, lang] = key.split("/") if (lang === undefined) { // This is a core translation, it has one less path segment lang = weblatepart weblatepart = "core" } const fixLink = `Fix it on https://hosted.weblate.org/translate/mapcomplete/${weblatepart}/${lang}/?offset=1&q=context%3A%3D%22${encodeURIComponent( path.join(".") )}%22` let subparts: string[] = value.match(/{[^}]*}/g) if (subparts === null) { if (neededSubparts.size > 0) { errors.push({ error: "The translation for " + key + " does not have any subparts, but expected " + Array.from(neededSubparts) .map( (part) => part.part + " (used in " + part.usedByLanguage + ")" ) .join(",") + " . The full translation is " + value + "\n" + fixLink, path: path, }) } return } subparts = subparts.map((p) => p.split(/\(.*\)/)[0]) if (subparts.indexOf(part) < 0) { if (lang === "en" || usedByLanguage === "en") { errors.push({ error: `The translation for ${key} does not have the required subpart ${part} (in ${usedByLanguage}). \tThe full translation is ${value} \t${fixLink}`, path: path, }) } } }) }) return errors } /** * Recursively adds a translation object, the inverse of 'toJson' * @param language * @param object * @private */ private addTranslation(language: string, object: any) { for (const key in object) { const v = object[key] if (v === "") { delete object[key] continue } let subpart = this.contents.get(key) if (subpart === undefined) { subpart = new TranslationPart() this.contents.set(key, subpart) } if (typeof v === "string") { subpart.contents.set(language, v) } else { subpart.addTranslation(language, v) } } } } /** * Converts a translation object into something that can be added to the 'generated translations'. * * To debug the 'compiledTranslations', add a languageWhiteList to only generate a single language */ function transformTranslation( obj: any, path: string[] = [], languageWhitelist: string[] = undefined, shortNotation = false ) { if (GenerateTranslations.isTranslation(obj)) { return `new Translation( ${JSON.stringify(obj)} )` } const values: string[] = [] const spaces = Utils.Times((_) => " ", path.length + 1) for (const key in obj) { if (key === "#") { continue } if (key.match("^[a-zA-Z0-9_]*$") === null) { throw "Invalid character in key: " + key } let value = obj[key] if (GenerateTranslations.isTranslation(value)) { if (languageWhitelist !== undefined) { const nv = {} for (const ln of languageWhitelist) { nv[ln] = value[ln] } value = nv } if (value["en"] === undefined) { throw `At ${path.join(".")}: Missing 'en' translation at path ${path.join( "." )}.${key}\n\tThe translations in other languages are ${JSON.stringify(value)}` } const subParts: string[] = value["en"].match(/{[^}]*}/g) let expr = `new Translation(${JSON.stringify(value)}, "core:${path.join(".")}.${key}")` if (subParts !== null) { // convert '{to_substitute}' into 'to_substitute' const types = Utils.Dedup(subParts.map((tp) => tp.substring(1, tp.length - 1))) const invalid = types.filter( (part) => part.match(/^[a-z0-9A-Z_]+(\(.*\))?$/) == null ) if (invalid.length > 0) { throw `At ${path.join( "." )}: A subpart contains invalid characters: ${subParts.join(", ")}` } expr = `new TypedTranslation<{ ${types.join(", ")} }>(${JSON.stringify( value )}, "core:${path.join(".")}.${key}")` } if (shortNotation) { values.push(`${spaces} ${key}: ${expr}`) } else { values.push(`${spaces}get ${key}() { return ${expr} }`) } } else { values.push( spaces + key + ": " + transformTranslation(value, [...path, key], languageWhitelist) ) } } return `{${values.join(",\n")}}` } /** * * const result = stringifySorted({"b": 43, "a": 42}) * result // => '{"a": 42,"b": 43}' * * // Should have correct newlines * const result = stringifySorted({"b": {"x": "y"}, "a": 42}, " ") * result // => '{\n "a": 42,\n "b": {\n "x": "y"\n }\n}' * * // Should sort like weblate does * const result = stringifySorted({"1": "abc", "2": "def", "9": "ghi", "10": "xyz", "11": "uvw"}) * result // => '{"1": "abc","10": "xyz","11": "uvw","2": "def","9": "ghi"}' */ function stringifySorted(o: object, space: string = undefined, depth = 0): string { const keys = Object.keys(o) let obj = "{" obj += keys .sort() .map((key) => { const v = o[key] let r = "" if (space !== undefined) { r += "\n" for (let i = 0; i <= depth; i++) { r += space } } r += JSON.stringify("" + key) + ": " if (typeof v === "object") { r += stringifySorted(v, space, depth + 1) } else if (Array.isArray(v)) { r += "[" + v.map((v_) => stringifySorted(v_, space, depth + 1)).join(",") + "]" } else { r += JSON.stringify(v) } return r }) .join(",") if (space !== undefined) { obj += "\n" for (let i = 0; i < depth; i++) { obj += space } } obj += "}" return obj } function removeEmptyString(object: object) { for (const k in object) { if (object[k] === "") { delete object[k] continue } if (typeof object[k] === "object") { removeEmptyString(object[k]) } } return object } /** * Formats the specified file, helps to prevent merge conflicts * */ function formatFile(path) { const original = readFileSync(path, "utf8") let contents = JSON.parse(original) contents = removeEmptyString(contents) contents = stringifySorted(contents, " ") writeFileSync(path, contents) } /** * Reads 'lang/*.json', writes them into to 'assets/generated/translations.json'. * This is only for the core translations */ /** * Get all the strings out of the layers; writes them onto the weblate paths * @param objects * @param target */ function generateTranslationsObjectFrom( objects: { path: string; parsed: { id: string } }[], target: string ): string[] { const tr = new TranslationPart() for (const layerFile of objects) { const config: { id: string } = layerFile.parsed const layerTr = new TranslationPart() if (config === undefined) { throw "Got something not parsed! Path is " + layerFile.path } layerTr.recursiveAdd(config, layerFile.path) tr.contents.set(config.id, layerTr) } const langs = tr.knownLanguages() for (const lang of langs) { if (lang === "#" || lang === "*") { // Lets not export our comments or non-translated stuff continue } let json = tr.toJson(lang) try { json = JSON.stringify(JSON.parse(json), null, " ") // MUST BE FOUR SPACES } catch (e) { console.error(e) } writeFileSync(`langs/${target}/${lang}.json`, json) } return langs } /** * Merge two objects together * @param source: where the translations come from * @param target: the object in which the translations should be merged * @param language: the language code * @param context: context for error handling * @constructor */ function MergeTranslation(source: any, target: any, language: string, context: string = "") { let keyRemapping: Map = undefined if (context.endsWith(".tagRenderings")) { keyRemapping = new Map() for (const key in target) { keyRemapping.set(target[key].id ?? target[key].builtin, key) } } for (const key in source) { const sourceV = source[key] const targetV = target[keyRemapping?.get(key) ?? key] if (typeof sourceV === "string") { // Add the translation if (targetV === undefined) { if (typeof target === "string") { throw `Trying to merge a translation for ${language} into a fixed string at ${context} for key ${key}` } target[key] = source[key] continue } if (targetV[language] === sourceV) { // Already the same continue } if (sourceV === "") { console.log("Ignoring empty string in the translations") } if (typeof targetV === "string") { throw `At context ${context}: Could not add a translation in language ${language}. The target object has a string at the given path, whereas the translation contains an object.\n String at target: ${targetV}\n Object at translation source: ${JSON.stringify( sourceV )}` } targetV[language] = sourceV let was = "" if (targetV[language] !== undefined && targetV[language] !== sourceV) { was = " (overwritten " + targetV[language] + ")" } // console.log(" + ", context + "." + language, "-->", sourceV, was) continue } if (typeof sourceV === "object") { if (targetV === undefined) { try { target[language] = sourceV } catch (e) { throw `At context${context}: Could not add a translation in language ${language} due to ${e}` } } else { MergeTranslation(sourceV, targetV, language, context + "." + key) } continue } throw "Case fallthrough" } return target } function mergeLayerTranslation( layerConfig: { id: string }, path: string, translationFiles: Map ) { const id = layerConfig.id translationFiles.forEach((translations, lang) => { const translationsForLayer = translations[id] MergeTranslation(translationsForLayer, layerConfig, lang, path + ":" + id) }) } function loadTranslationFilesFrom(target: string): Map { const translationFilePaths = ScriptUtils.readDirRecSync("./langs/" + target).filter((path) => path.endsWith(".json") ) const translationFiles = new Map() for (const translationFilePath of translationFilePaths) { let language = translationFilePath.substr(translationFilePath.lastIndexOf("/") + 1) language = language.substr(0, language.length - 5) try { translationFiles.set(language, JSON.parse(readFileSync(translationFilePath, "utf8"))) } catch (e) { console.error("Invalid JSON file or file does not exist", translationFilePath) throw e } } return translationFiles } /** * Load the translations from the weblate files back into the layers */ function mergeLayerTranslations(englishOnly: boolean = false) { const layerFiles = ScriptUtils.getLayerFiles() for (const layerFile of layerFiles) { mergeLayerTranslation(layerFile.parsed, layerFile.path, loadTranslationFilesFrom("layers")) const endsWithNewline = readFileSync(layerFile.path, { encoding: "utf8" })?.endsWith("\n") ?? true let config = layerFile.parsed if (englishOnly) { config = Utils.Clone(config) removeNonEnglishTranslations(config) } writeFileSync( layerFile.path, JSON.stringify(config, null, " ") + (endsWithNewline ? "\n" : "") ) // layers use 2 spaces } } function removeNonEnglishTranslations(object: any) { Utils.WalkObject( object, (leaf: any) => { const en = leaf["en"] if (!en) { return } for (const key in leaf) { if (key.startsWith("#")) { continue } delete leaf[key] } leaf["en"] = en }, (possibleLeaf) => possibleLeaf !== null && typeof possibleLeaf === "object" && GenerateTranslations.isTranslation(possibleLeaf) ) } /** * Load the translations into the theme files */ function mergeThemeTranslations(englishOnly: boolean = false) { const themeFiles = ScriptUtils.getThemeFiles(true) for (const themeFile of themeFiles) { let config = themeFile.parsed mergeLayerTranslation(config, themeFile.path, loadTranslationFilesFrom("themes")) const allTranslations = new TranslationPart() allTranslations.recursiveAdd(config, themeFile.path) const endsWithNewline = readFileSync(themeFile.path, { encoding: "utf8" })?.endsWith("\n") ?? true if (englishOnly) { config = Utils.Clone(config) removeNonEnglishTranslations(config) } writeFileSync( themeFile.path, JSON.stringify(config, null, " ") + (endsWithNewline ? "\n" : "") ) // Themefiles use 2 spaces } } class GenerateTranslations extends Script { constructor() { super("Syncs translations from/to the theme and layer files") } /** * Checks that the given object only contains string-values * @param tr */ static isTranslation(tr: Record): boolean { if (tr["#"] === "no-translations") { return false } if (tr["special"]) { return false } for (const key in tr) { if (typeof tr[key] !== "string") { return false } } return true } /** * OUtputs the 'used_languages.json'-file */ detectUsedLanguages() { { const l1 = generateTranslationsObjectFrom(ScriptUtils.getLayerFiles(), "layers") const l2 = generateTranslationsObjectFrom( ScriptUtils.getThemeFiles(true).filter( (th) => th.parsed.mustHaveLanguage === undefined ), "themes" ) const usedLanguages: string[] = Utils.Dedup(l1.concat(l2)).filter((v) => v !== "*") usedLanguages.sort() fs.writeFileSync( "./src/assets/used_languages.json", JSON.stringify({ languages: usedLanguages }) ) } } /** * Generates the big compiledTranslations file based on 'translations.json' */ genTranslations(englishOnly?: boolean) { if (!fs.existsSync("./src/assets/generated/")) { fs.mkdirSync("./src/assets/generated/") } const translations = JSON.parse( fs.readFileSync("./src/assets/generated/translations.json", "utf-8") ) const transformed = transformTranslation( translations, undefined, englishOnly ? ["en"] : undefined, englishOnly ) let module = `import {Translation, TypedTranslation} from "../../UI/i18n/Translation"\n\nexport default class CompiledTranslations {\n\n` module += " public static t = " + transformed module += "\n }" fs.writeFileSync("./src/assets/generated/CompiledTranslations.ts", module) } compileTranslationsFromWeblate(englishOnly: boolean) { const translations = ScriptUtils.readDirRecSync("./langs", 1).filter( (path) => path.indexOf(".json") > 0 ) const allTranslations = new TranslationPart() allTranslations.validateStrict() for (const translationFile of translations) { try { const contents = JSON.parse(readFileSync(translationFile, "utf-8")) let language = translationFile.substring(translationFile.lastIndexOf("/") + 1) language = language.substring(0, language.length - 5) if (englishOnly && language !== "en") { continue } allTranslations.add(language, contents) } catch (e) { throw "Could not read file " + translationFile + " due to " + e } } writeFileSync( "./src/assets/generated/translations.json", JSON.stringify(JSON.parse(allTranslations.toJson()), null, " ") ) } async main(args: string[]): Promise { if (!existsSync("./langs/themes")) { mkdirSync("./langs/themes") } const themeOverwritesWeblate = args[0] === "--ignore-weblate" const englishOnly = args[0] === "--english-only" if (englishOnly) { console.log("ENGLISH ONLY") } if (!themeOverwritesWeblate) { mergeLayerTranslations(englishOnly) mergeThemeTranslations(englishOnly) this.compileTranslationsFromWeblate(englishOnly) } else { console.log("Ignore weblate") } this.detectUsedLanguages() this.genTranslations(englishOnly) { const allTranslationFiles = ScriptUtils.readDirRecSync("langs").filter((path) => path.endsWith(".json") ) for (const path of allTranslationFiles) { formatFile(path) } } // Some validation TranslationPart.fromDirectory("./langs").validateStrict("./langs") TranslationPart.fromDirectory("./langs/layers").validateStrict("layers") TranslationPart.fromDirectory("./langs/themes").validateStrict("themes") if (englishOnly) { mergeLayerTranslations(true) mergeThemeTranslations(true) } console.log("All done!") } } new GenerateTranslations().run()