midgard/mapcomplete
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add small script to steal shop presets from the iD-project

Browse source
This commit is contained in:
pietervdvn 2022-07-02 01:59:26 +02:00
parent 5cf52690ee
commit ef66b1931f
5 changed files with 4037 additions and 166 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
UI/Popup/TagRenderingQuestion.ts
View file

@ -181,7 +181,7 @@ export default class TagRenderingQuestion extends Combine {
}
if (applicableMappings.length < 8 || configuration.multiAnswer || hasImages || ifNotsPresent) {
if (applicableMappings.length < 8 || configuration.multiAnswer || (hasImages && applicableMappings.length < 16) || ifNotsPresent) {
inputEls = (applicableMappings ?? []).map((mapping, i) => TagRenderingQuestion.GenerateMappingElement(state, tagsSource, mapping, allIfNotsExcept(i)));
inputEls = Utils.NoNull(inputEls);
} else {

3835
assets/layers/shops/shops.json
View file

File diff suppressed because it is too large Load diff

3
package.json
View file

@ -51,7 +51,8 @@
"weblate-fix": "git remote update weblate-github ; git merge weblate-github/weblate-mapcomplete-core; git merge weblate-github/weblate-mapcomplete-layers ; git merge weblate-github/weblate-mapcomplete-layer-translations",
"weblate-fix-heavy": "git remote rm weblate-layers; git remote add weblate-layers https://hosted.weblate.org/git/mapcomplete/layers/; git remote update weblate-layers; git merge weblate-layers/master",
"housekeeping": "npm run generate && npm run generate:docs && npm run generate:contributor-list && git add Docs/* && git commit assets/ langs/ Docs/ -m 'Housekeeping...'",
"parseSchools": "ts-node scripts/schools/amendSchoolData.ts"
"parseSchools": "ts-node scripts/schools/amendSchoolData.ts",
"steal": "ts-node scripts/readIdPresets.ts"
},
"keywords": [
"OpenStreetMap",

160
scripts/generateTranslations.ts
View file

@ -9,6 +9,16 @@ class TranslationPart {
contents: Map<string, TranslationPart | string> = new Map<string, TranslationPart | string>()
static fromDirectory(path): TranslationPart {
const files = ScriptUtils.readDirRecSync(path, 1).filter(file => file.endsWith(".json"))
const rootTranslation = new TranslationPart()
for (const file of files) {
const content = JSON.parse(readFileSync(file, "UTF8"))
rootTranslation.addTranslation(file.substr(0, file.length - ".json".length), content)
}
return rootTranslation
}
/**
* Add a leaf object
* @param language
@ -32,8 +42,8 @@ class TranslationPart {
}
addTranslationObject(translations: any, context?: string) {
if(translations["#"] === "no-translations"){
console.log("Ignoring object at ",context,"which has '#':'no-translations'")
if (translations["#"] === "no-translations") {
console.log("Ignoring object at ", context, "which has '#':'no-translations'")
return;
}
for (const translationsKey in translations) {
@ -59,11 +69,31 @@ class TranslationPart {
return;
}
let dontTranslateKeys: string[] = undefined
{
const noTranslate = <string | string[]>object["#dont-translate"]
if (noTranslate === "*") {
console.log("Ignoring translations for " + context)
return
} else if (typeof noTranslate === "string") {
dontTranslateKeys = [noTranslate]
} else {
dontTranslateKeys = noTranslate
}
if (noTranslate !== undefined) {
console.log("Ignoring some translations for " + context + ": " + dontTranslateKeys.join(", "))
}
}
for (let key in object) {
if (!object.hasOwnProperty(key)) {
continue;
}
if (dontTranslateKeys?.indexOf(key) >= 0) {
continue
}
const v = object[key]
if (v == null) {
@ -145,57 +175,24 @@ class TranslationPart {
return `{${parts.join(",")}}`;
}
/**
* Recursively adds a translation object, the inverse of 'toJson'
* @param language
* @param object
* @private
*/
private addTranslation(language: string, object: any){
for (const key in object) {
const v = object[key]
let subpart = <TranslationPart>this.contents.get(key)
if(subpart === undefined){
subpart = new TranslationPart()
this.contents.set(key, subpart)
}
if(typeof v === "string"){
subpart.contents.set(language, v)
}else{
subpart.addTranslation(language, v)
}
validateStrict(ctx?: string): void {
const errors = this.validate()
for (const err of errors) {
console.error("ERROR in " + (ctx ?? "") + " " + err.path.join(".") + "\n " + err.error)
}
}
static fromDirectory(path): TranslationPart{
const files = ScriptUtils.readDirRecSync(path, 1).filter(file => file.endsWith(".json"))
const rootTranslation = new TranslationPart()
for (const file of files) {
const content = JSON.parse(readFileSync(file, "UTF8"))
rootTranslation.addTranslation(file.substr(0, file.length - ".json".length), content)
if (errors.length > 0) {
throw ctx + " has " + errors.length + " inconsistencies in the translation"
}
return rootTranslation
}
validateStrict(ctx?:string): void {
const errors = this.validate()
for (const err of errors) {
console.error("ERROR in "+(ctx ?? "")+ " " +err.path.join(".")+"\n "+err.error)
}
if(errors.length > 0){
throw ctx+" has "+errors.length+" inconsistencies in the translation"
}
}
/**
* Checks the leaf objects: special values must be present and identical in every leaf
*/
validate(path = []): {error: string, path: string[]} [] {
const errors : {error: string, path: string[]} []= []
validate(path = []): { error: string, path: string[] } [] {
const errors: { error: string, path: string[] } [] = []
const neededSubparts = new Set<{ part: string, usedByLanguage: string }>()
let isLeaf : boolean = undefined
let isLeaf: boolean = undefined
this.contents.forEach((value, key) => {
if (typeof value !== "string") {
const recErrors = value.validate([...path, key])
@ -235,12 +232,12 @@ class TranslationPart {
lang = weblatepart
weblatepart = "core"
}
const fixLink = `Fix it on https://hosted.weblate.org/translate/mapcomplete/${weblatepart}/${lang}/?offset=1&q=context%3A%3D%22${encodeURIComponent( path.join("."))}%22`;
const fixLink = `Fix it on https://hosted.weblate.org/translate/mapcomplete/${weblatepart}/${lang}/?offset=1&q=context%3A%3D%22${encodeURIComponent(path.join("."))}%22`;
let subparts: string[] = value.match(/{[^}]*}/g)
if (subparts === null) {
if (neededSubparts.size > 0) {
errors.push({
error: "The translation for " + key + " does not have any subparts, but expected " + Array.from(neededSubparts).map(part => part.part +" (used in "+part.usedByLanguage+")").join(",") + " . The full translation is " + value+"\n"+fixLink,
error: "The translation for " + key + " does not have any subparts, but expected " + Array.from(neededSubparts).map(part => part.part + " (used in " + part.usedByLanguage + ")").join(",") + " . The full translation is " + value + "\n" + fixLink,
path: path
})
}
@ -248,8 +245,8 @@ class TranslationPart {
}
subparts = subparts.map(p => p.split(/\(.*\)/)[0])
if (subparts.indexOf(part) < 0) {
if(lang === "en" || usedByLanguage === "en"){
if (lang === "en" || usedByLanguage === "en") {
errors.push({
error: `The translation for ${key} does not have the required subpart ${part}.
\tThe full translation is ${value}
@ -261,9 +258,32 @@ class TranslationPart {
})
})
return errors
return errors
}
/**
* Recursively adds a translation object, the inverse of 'toJson'
* @param language
* @param object
* @private
*/
private addTranslation(language: string, object: any) {
for (const key in object) {
const v = object[key]
let subpart = <TranslationPart>this.contents.get(key)
if (subpart === undefined) {
subpart = new TranslationPart()
this.contents.set(key, subpart)
}
if (typeof v === "string") {
subpart.contents.set(language, v)
} else {
subpart.addTranslation(language, v)
}
}
}
}
/**
@ -271,7 +291,7 @@ class TranslationPart {
* @param tr
*/
function isTranslation(tr: any): boolean {
if(tr["#"] === "no-translations") {
if (tr["#"] === "no-translations") {
return false
}
for (const key in tr) {
@ -284,10 +304,10 @@ function isTranslation(tr: any): boolean {
/**
* Converts a translation object into something that can be added to the 'generated translations'.
*
*
* To debug the 'compiledTranslations', add a languageWhiteList to only generate a single language
*/
function transformTranslation(obj: any, path: string[] = [], languageWhitelist : string[] = undefined) {
function transformTranslation(obj: any, path: string[] = [], languageWhitelist: string[] = undefined) {
if (isTranslation(obj)) {
return `new Translation( ${JSON.stringify(obj)} )`
@ -305,30 +325,30 @@ function transformTranslation(obj: any, path: string[] = [], languageWhitelist :
let value = obj[key]
if (isTranslation(value)) {
if(languageWhitelist !== undefined){
if (languageWhitelist !== undefined) {
const nv = {}
for (const ln of languageWhitelist) {
nv[ln] = value[ln]
}
value = nv;
}
if(value["en"] === undefined){
if (value["en"] === undefined) {
throw `At ${path.join(".")}: Missing 'en' translation at path ${path.join(".")}.${key}\n\tThe translations in other languages are ${JSON.stringify(value)}`
}
const subParts : string[] = value["en"].match(/{[^}]*}/g)
const subParts: string[] = value["en"].match(/{[^}]*}/g)
let expr = `return new Translation(${JSON.stringify(value)}, "core:${path.join(".")}.${key}")`
if(subParts !== null){
if (subParts !== null) {
// convert '{to_substitute}' into 'to_substitute'
const types = Utils.Dedup( subParts.map(tp => tp.substring(1, tp.length - 1)))
const types = Utils.Dedup(subParts.map(tp => tp.substring(1, tp.length - 1)))
const invalid = types.filter(part => part.match(/^[a-z0-9A-Z_]+(\(.*\))?$/) == null)
if(invalid.length > 0){
if (invalid.length > 0) {
throw `At ${path.join(".")}: A subpart contains invalid characters: ${subParts.join(', ')}`
}
expr = `return new TypedTranslation<{ ${types.join(", ")} }>(${JSON.stringify(value)}, "core:${path.join(".")}.${key}")`
}
values += `${Utils.Times((_) => " ", path.length + 1)}get ${key}() { ${expr} },
`
} else {
@ -339,15 +359,15 @@ function transformTranslation(obj: any, path: string[] = [], languageWhitelist :
}
function sortKeys(o: object): object{
function sortKeys(o: object): object {
const keys = Object.keys(o)
keys.sort()
const nw = {}
for (const key of keys) {
const v = o[key]
if(typeof v === "object"){
if (typeof v === "object") {
nw[key] = sortKeys(v)
}else{
} else {
nw[key] = v
}
}
@ -370,7 +390,7 @@ function formatFile(path) {
*/
function genTranslations() {
const translations = JSON.parse(fs.readFileSync("./assets/generated/translations.json", "utf-8"))
const transformed = transformTranslation(translations);
const transformed = transformTranslation(translations);
let module = `import {Translation, TypedTranslation} from "../../UI/i18n/Translation"\n\nexport default class CompiledTranslations {\n\n`;
module += " public static t = " + transformed;
@ -389,10 +409,10 @@ function compileTranslationsFromWeblate() {
.filter(path => path.indexOf(".json") > 0)
const allTranslations = new TranslationPart()
allTranslations.validateStrict()
allTranslations.validateStrict()
for (const translationFile of translations) {
try {

203
scripts/readIdPresets.ts Normal file
View file

@ -0,0 +1,203 @@
/***
* Parses presets from the iD repository and extracts some usefull tags from them
*/
import {TagRenderingConfigJson} from "../Models/ThemeConfig/Json/TagRenderingConfigJson";
import ScriptUtils from "./ScriptUtils";
import {existsSync, readFileSync, writeFileSync} from "fs";
import {TagsFilter} from "../Logic/Tags/TagsFilter";
import * as known_languages from "../assets/language_native.json"
import {LayerConfigJson} from "../Models/ThemeConfig/Json/LayerConfigJson";
import {QuestionableTagRenderingConfigJson} from "../Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson";
import SmallLicense from "../Models/smallLicense";
import {icon} from "leaflet";
interface IconThief {
steal(iconName: string): boolean
}
interface IdPreset {
icon: string,
geometry: ("point" | "line" | "area")[]
/**
* Extra search terms
*/
terms: string []
tags: Record<string, string>
name: string,
searchable?: false,
}
class MakiThief implements IconThief{
private readonly _directory: string;
private readonly _targetDir: string;
constructor(directory: string, targetDir: string) {
this._directory = directory;
this._targetDir = targetDir;
}
public steal(iconName: string): boolean{
const target = this._targetDir+iconName+".svg"
if(existsSync(target)){
// return true
}
const file = readFileSync(this._directory+"/icons/"+iconName+".svg", "utf8")
writeFileSync(target, file,'utf8')
writeFileSync(target+".license_info.json",
JSON.stringify(<SmallLicense>{
authors:['Maki icon set'],
license: 'CC0',
path: 'maki-'+iconName+".svg",
sources: ["https://github.com/mapbox/maki"]
}), 'utf8')
console.log("Successfully stolen "+iconName)
return true
}
}
class AggregateIconThief implements IconThief{
private readonly _maki: MakiThief;
constructor(maki: MakiThief) {
this._maki = maki;
}
public steal(iconName: string): boolean{
if(iconName.startsWith('maki-')){
this._maki.steal(iconName.substr('maki-'.length))
return true
}
return false
}
}
class IdThief {
private readonly _idPresetsRepository: string;
private readonly _tranlationFiles: Record<string, object> = {}
private readonly _knownLanguages: string[]
private readonly _iconThief: IconThief;
public constructor(idPresetsRepository: string, iconThief: IconThief) {
this._idPresetsRepository = idPresetsRepository;
this._iconThief = iconThief;
const knownById = ScriptUtils.readDirRecSync(`${this._idPresetsRepository}/dist/translations/`)
.map(pth => pth.substring(pth.lastIndexOf('/') + 1, pth.length - '.json'.length))
.filter(lng => !lng.endsWith('.min'));
const missing = Object.keys(known_languages).filter(lng => knownById.indexOf(lng.replace('-','_')) < 0)
this._knownLanguages = knownById.filter(lng => known_languages[lng] !== undefined)
console.log("Id knows following languages:", this._knownLanguages.join(", "), "missing:", missing)
}
public getTranslation(language: string, ...path: string[]) {
let obj = this.loadTranslationFile(language)[language]
for (const p of path) {
obj = obj[p]
if (obj === undefined) {
return undefined;
}
}
return obj
}
/**
* Creates a tagRenderingConfigJson for the 'shop' theme
*/
public readShopPresets(): {if, then, hideInAnswer?: string | boolean}[] {
const dir = this._idPresetsRepository + "/data/presets/shop"
const mappings:
{
if: string | {and: string[]},
then: Record<string, string>,
hideInAnswer?: string | boolean
icon?: {
path: string,
/**
* Size of the image
*/
class: "small" | "medium" | "large" | string
}
}[] = []
const files = ScriptUtils.readDirRecSync(dir, 1);
for (const file of files) {
const name = file.substring(file.lastIndexOf('/')+1, file.length - '.json'.length)
const preset = <IdPreset>JSON.parse(readFileSync(file, 'utf8'))
if(preset.searchable === false){
continue
}
console.log(` ${name} (shop=${preset.tags["shop"]}), ${preset.icon}` )
const thenClause : Record<string, string> = {
en: preset.name
}
for (const lng of this._knownLanguages) {
const tr = this.getTranslation(lng, "presets", "presets", "shop/"+name, "name")
if(tr === undefined){
continue
}
thenClause[lng.replace('-','_')] = tr
}
let tag : string | {and: string[]}
const tagKeys = Object.keys(preset.tags)
if(tagKeys.length === 1){
tag = tagKeys[0]+"="+preset.tags[tagKeys[0]]
}else{
tag = {
and: tagKeys.map(key => key+"="+preset.tags[key])
}
}
const mapping = {
if: tag,
then: thenClause
}
if(preset.tags["shop"] == "yes"){
mapping["hideInAnswer"] = true
mapping.if["en"] = "Unspecified shop"
}
if(this._iconThief.steal(preset.icon)){
mapping["icon"] = {
path: "./assets/layers/shops/"+preset.icon+".svg",
size: "medium"
}
}
mappings.push(mapping)
}
return mappings
}
private loadTranslationFile(language: string): object {
const cached = this._tranlationFiles[language]
if (cached) {
return cached
}
return this._tranlationFiles[language] = JSON.parse(readFileSync(`${this._idPresetsRepository}/dist/translations/${language}.json`, 'utf8'))
}
}
const targetDir = "./assets/layers/shops/"
const iconThief = new AggregateIconThief(
new MakiThief('../maki', targetDir+"maki-")
)
const shopOptions = new IdThief("../id-tagging-schema/",iconThief ).readShopPresets()
const shopLayerPath =targetDir+"shops.json"
const shopLayer = <LayerConfigJson> JSON.parse(readFileSync(shopLayerPath,'utf8'))
const type = <QuestionableTagRenderingConfigJson> shopLayer.tagRenderings.find(tr => tr["id"] == "shops-type-from-id")
type.mappings = shopOptions
writeFileSync(shopLayerPath, JSON.stringify(shopLayer, null, " "),'utf8')