midgard/mapcomplete
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Improve metadata in language translations, remove wikidata-downloading-script

Browse source
This commit is contained in:
Pieter Vander Vennet 2022-10-29 03:01:24 +02:00
parent cba1a4340e
commit 30a835a232
4 changed files with 66 additions and 197 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
assets/language_in_country.json
View file

@ -132,6 +132,7 @@
"en"
],
"CN": [
"zh",
"zh"
],
"CO": [
@ -148,6 +149,7 @@
],
"CY": [
"tr",
"el",
"el"
],
"CZ": [
@ -247,6 +249,9 @@
"es",
"pt"
],
"GR": [
"el"
],
"GT": [
"es"
],
@ -453,8 +458,7 @@
"fr"
],
"NG": [
"en",
"yo"
"en"
],
"NI": [
"es"
@ -502,9 +506,7 @@
"en"
],
"PK": [
"ur",
"en",
"ar"
"ur"
],
"PL": [
"pl",
@ -559,7 +561,6 @@
"ar"
],
"SE": [
"sv",
"sv"
],
"SG": [
@ -648,6 +649,9 @@
"en",
"en"
],
"TW": [
"zh"
],
"TZ": [
"en",
"sw"
@ -693,16 +697,16 @@
"ar"
],
"ZA": [
"en",
"zu",
"xh",
"af",
"ve",
"ss",
"tn",
"ts",
"st",
"nr",
"en",
"zu",
"xh"
"nr"
],
"ZM": [
"en"

2
package.json
View file

@ -54,7 +54,7 @@
"weblate-fix-heavy": "git remote rm weblate-layers; git remote add weblate-layers https://hosted.weblate.org/git/mapcomplete/layers/; git remote update weblate-layers; git merge weblate-layers/master",
"housekeeping": "npm run generate && npm run generate:docs && npm run generate:contributor-list && npm run format && git add assets/ langs/ Docs/ **/*.ts Docs/* && git commit -m 'Housekeeping...'",
"parseSchools": "ts-node scripts/schools/amendSchoolData.ts",
"steal": "ts-node scripts/thieves/stealLanguages.ts"
"steal": "ts-node scripts/fetchLanguages.ts"
},
"keywords": [
"OpenStreetMap",

110
scripts/fetchLanguages.ts
View file

@ -1,5 +1,6 @@
/**
* Fetches all 'modern languages' from wikidata, then exports their names in every language
* Fetches all 'modern languages' from wikidata, then exports their names in every language.
* Some meta-info (e.g. RTL) is exported too
*/
import * as wds from "wikidata-sdk"
@ -21,12 +22,15 @@ async function fetchRegularLanguages() {
console.log("Fetching languages")
const sparql =
"SELECT ?lang ?label ?code \n" +
"SELECT ?lang ?label ?code ?directionalityLabel \n" +
"WHERE \n" +
"{ \n" +
" ?lang wdt:P31 wd:Q1288568. \n" + // language instanceOf (p31) modern language(Q1288568)
" ?lang rdfs:label ?label. \n" +
" ?lang wdt:P424 ?code" + // Wikimedia language code seems to be close to the weblate entries
" ?lang wdt:P282 ?writing_system. \n"+
" ?writing_system wdt:P1406 ?directionality. \n" +
" ?lang wdt:P424 ?code. \n" +// Wikimedia language code seems to be close to the weblate entries
" SERVICE wikibase:label { bd:serviceParam wikibase:language \"en\". } \n" +
"} "
const url = wds.sparqlQuery(sparql)
@ -67,16 +71,19 @@ async function fetchSpecial(id: number, code: string) {
return bindings
}
function getNativeList(langs: Map<string, Map<string, string>>) {
function getNativeList(langs: Map<string, { translations: Map<string, string> }>) {
const native = {}
const keys: string[] = Array.from(langs.keys())
keys.sort()
for (const key of keys) {
const translations: Map<string, string> = langs.get(key)
const translations: Map<string, string> = langs.get(key).translations
if (!LanguageUtils.usedLanguages.has(key)) {
continue
}
native[key] = translations.get(key)
if(native[key] === undefined){
console.log("No native translation found for "+key)
}
}
return native
}
@ -108,33 +115,7 @@ async function getOfficialLanguagesPerCountry(): Promise<Map<string, string[]>>
return lngs
}
async function main(wipeCache = false) {
const cacheFile = "./assets/generated/languages-wd.json"
if (wipeCache || !existsSync(cacheFile)) {
console.log("Refreshing cache")
await fetch(cacheFile)
} else {
console.log("Reusing the cached file")
}
const data = JSON.parse(readFileSync(cacheFile, "UTF8"))
const perId = WikidataUtils.extractLanguageData(data, WikidataUtils.languageRemapping)
const nativeList = getNativeList(perId)
writeFileSync("./assets/language_native.json", JSON.stringify(nativeList, null, " "))
const translations = Utils.MapToObj(perId, (value, key) => {
if (!LanguageUtils.usedLanguages.has(key)) {
return undefined // Remove unused languages
}
return Utils.MapToObj(value, (v, k) => {
if (!LanguageUtils.usedLanguages.has(k)) {
return undefined
}
return v
})
})
writeFileSync("./assets/language_translations.json", JSON.stringify(translations, null, " "))
async function getOfficialLanguagesPerCountryCached(wipeCache: boolean): Promise<Record<string /*Country code*/, string[] /*Language codes*/>>{
let officialLanguages: Record<string, string[]>
const officialLanguagesPath = "./assets/language_in_country.json"
if (existsSync("./assets/languages_in_country.json") && !wipeCache) {
@ -143,37 +124,48 @@ async function main(wipeCache = false) {
officialLanguages = Utils.MapToObj(await getOfficialLanguagesPerCountry(), (t) => t)
writeFileSync(officialLanguagesPath, JSON.stringify(officialLanguages, null, " "))
}
return officialLanguages
}
const perLanguage = Utils.TransposeMap(officialLanguages)
console.log(JSON.stringify(perLanguage, null, " "))
const mappings: { if: string; then: Record<string, string>; hideInAnswer: string }[] = []
for (const language of Object.keys(perLanguage)) {
const countries = Utils.Dedup(perLanguage[language].map((c) => c.toLowerCase()))
mappings.push({
if: "language=" + language,
then: translations[language],
hideInAnswer: "_country=" + countries.join("|"),
async function main(wipeCache = false) {
const cacheFile = "./assets/generated/languages-wd.json"
if (wipeCache || !existsSync(cacheFile)) {
console.log("Refreshing cache")
await fetch(cacheFile)
} else {
console.log("Reusing the cached file")
}
const data = JSON.parse(readFileSync(cacheFile, "UTF8"))
const perId = WikidataUtils.extractLanguageData(data, WikidataUtils.languageRemapping)
const nativeList = getNativeList(perId)
writeFileSync("./assets/language_native.json", JSON.stringify(nativeList, null, " "))
const languagesPerCountry = Utils.TransposeMap(await getOfficialLanguagesPerCountryCached(wipeCache))
const translations = Utils.MapToObj(perId, (value, key) => {
// We keep all language codes in the list...
const translatedForId : Record<string, string | {countries?: string[], dir: string[]}> = Utils.MapToObj(value.translations, (v, k) => {
if (!LanguageUtils.usedLanguages.has(k)) {
// ... but don't keep translations if we don't have a displayed language for them
return undefined
}
return v
})
}
const tagRenderings = <QuestionableTagRenderingConfigJson>{
id: "official-language",
mappings,
question: "What languages are spoken here?",
}
translatedForId["_meta"] = {
countries : Utils.Dedup( languagesPerCountry[key]),
dir: value.directionality
}
return translatedForId
})
writeFileSync("./assets/language_translations.json", JSON.stringify(translations, null, " "))
writeFileSync(
"./assets/layers/language/language.json",
JSON.stringify(
<LayerConfigJson>{
id: "language",
description: "Various tagRenderings to help language tooling",
tagRenderings,
},
null,
" "
)
)
}
const forceRefresh = process.argv[2] === "--force-refresh"

127
scripts/thieves/stealLanguages.ts
View file

@ -1,127 +0,0 @@
/*
* Uses the languages in and to every translation from wikidata to generate a language question in wikidata/wikidata
* */
import WikidataUtils from "../../Utils/WikidataUtils"
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"
import { LayerConfigJson } from "../../Models/ThemeConfig/Json/LayerConfigJson"
import { MappingConfigJson } from "../../Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson"
import LanguageUtils from "../../Utils/LanguageUtils"
import * as perCountry from "../../assets/language_in_country.json"
import { Utils } from "../../Utils"
function main() {
const sourcepath = "assets/generated/languages-wd.json"
console.log(`Converting language data file '${sourcepath}' into a tagMapping`)
const languages = WikidataUtils.extractLanguageData(
JSON.parse(readFileSync(sourcepath, "utf8")),
{}
)
const mappings: MappingConfigJson[] = []
const schoolmappings: MappingConfigJson[] = []
const brailemappings: MappingConfigJson[] = []
const countryToLanguage: Record<string, string[]> = perCountry
const officialLanguagesPerCountry = Utils.TransposeMap(countryToLanguage)
languages.forEach((l, code) => {
const then: Record<string, string> = {}
l.forEach((tr, lng) => {
const languageCodeWeblate = WikidataUtils.languageRemapping[lng] ?? lng
if (!LanguageUtils.usedLanguages.has(languageCodeWeblate)) {
return
}
then[languageCodeWeblate] = tr
})
const officialCountries = Utils.Dedup(
officialLanguagesPerCountry[code]?.map((s) => s.toLowerCase()) ?? []
)
const prioritySearch =
officialCountries.length > 0
? "_country~" + officialCountries.map((c) => "((^|;)" + c + "($|;))").join("|")
: undefined
mappings.push(<MappingConfigJson>{
if: "language:" + code + "=yes",
ifnot: "language:" + code + "=",
searchTerms: {
"*": [code],
},
then,
priorityIf: prioritySearch,
})
schoolmappings.push(<MappingConfigJson>{
if: "school:language=" + code,
then,
priorityIf: prioritySearch,
searchTerms: {
"*": [code],
},
})
brailemappings.push(<MappingConfigJson>{
if: "tactile_writing:braille:" + code + "=yes",
ifnot: "tactile_writing:braille:" + code + "=",
searchTerms: {
"*": [code],
},
then,
priorityIf: prioritySearch,
})
})
const wikidataLayer = <LayerConfigJson>{
id: "wikidata",
description: {
en: "Various tagrenderings which are generated from Wikidata. Automatically generated with a script, don't edit manually",
},
"#dont-translate": "*",
source: {
osmTags: "id~*",
},
title: null,
mapRendering: null,
tagRenderings: [
{
id: "language",
// @ts-ignore
description: "Enables to pick *a single* 'language:<lng>=yes' within the mappings",
mappings,
},
{
builtin: "wikidata.language",
override: {
id: "language-multi",
// @ts-ignore
description:
"Enables to pick *multiple* 'language:<lng>=yes' within the mappings",
multiAnswer: true,
},
},
{
id: "school-language",
// @ts-ignore
description: "Enables to pick a single 'school:language=<lng>' within the mappings",
multiAnswer: true,
mappings: schoolmappings,
},
{
id: "tactile_writing-braille",
// @ts-ignore
description:
"Enables to pick *multiple* 'tactile_writing:braille=<lng>' within the mappings",
multiAnswer: true,
mappings: brailemappings,
},
],
}
const dir = "./assets/layers/wikidata/"
if (!existsSync(dir)) {
mkdirSync(dir)
}
const path = dir + "wikidata.json"
writeFileSync(path, JSON.stringify(wikidataLayer, null, " "))
console.log("Written " + path)
}
main()