midgard/mapcomplete
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Fix: actually search for keywords in theme view

Browse source
This commit is contained in:
Pieter Vander Vennet 2024-08-27 21:33:47 +02:00
parent 821c1fabd7
commit cdc1e05499
9 changed files with 193 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
assets/layers/toilet/toilet.json
View file

@ -25,6 +25,31 @@
"cs": "Vrstva zobrazující (veřejné) toalety",
"sl": "Prikaz (javnih) stranišč"
},
"searchTerms": {
"en": [
"Toilets",
"Bathroom",
"Lavatory",
"Water Closet",
"outhouse",
"privy",
"head",
"latrine",
"WC",
"W.C."
],
"nl": [
"WC",
"WCs",
"plee",
"gemak",
"opschik",
"kabinet",
"latrine",
"retirade",
"piesemopsantee"
]
},
"source": {
"osmTags": "amenity=toilets"
},

152
scripts/generateLayerOverview.ts
View file

@ -34,6 +34,10 @@ import Translations from "../src/UI/i18n/Translations"
import { Translatable } from "../src/Models/ThemeConfig/Json/Translatable"
import { ValidateThemeAndLayers } from "../src/Models/ThemeConfig/Conversion/ValidateThemeAndLayers"
import { ExtractImages } from "../src/Models/ThemeConfig/Conversion/FixImages"
import {
MinimalTagRenderingConfigJson,
TagRenderingConfigJson,
} from "../src/Models/ThemeConfig/Json/TagRenderingConfigJson"
// This scripts scans 'src/assets/layers/*.json' for layer definition files and 'src/assets/themes/*.json' for theme definition files.
// It spits out an overview of those to be used to load them
@ -56,7 +60,7 @@ class ParseLayer extends Conversion<
convert(
path: string,
context: ConversionContext
context: ConversionContext,
): {
parsed: LayerConfig
raw: LayerConfigJson
@ -85,7 +89,7 @@ class ParseLayer extends Conversion<
context
.enter("source")
.err(
"No source is configured. (Tags might be automatically derived if presets are given)"
"No source is configured. (Tags might be automatically derived if presets are given)",
)
return undefined
}
@ -116,7 +120,7 @@ class AddIconSummary extends DesugaringStep<{ raw: LayerConfigJson; parsed: Laye
const fixed = json.raw
const layerConfig = json.parsed
const pointRendering: PointRenderingConfig = layerConfig.mapRendering.find((pr) =>
pr.location.has("point")
pr.location.has("point"),
)
const defaultTags = layerConfig.GetBaseTags()
fixed["_layerIcon"] = Utils.NoNull(
@ -131,7 +135,7 @@ class AddIconSummary extends DesugaringStep<{ raw: LayerConfigJson; parsed: Laye
result["color"] = c
}
return result
})
}),
)
return { raw: fixed, parsed: layerConfig }
}
@ -153,7 +157,7 @@ class LayerOverviewUtils extends Script {
private static extractLayerIdsFrom(
themeFile: LayoutConfigJson,
includeInlineLayers = true
includeInlineLayers = true,
): string[] {
const publicLayerIds: string[] = []
if (!Array.isArray(themeFile.layers)) {
@ -223,16 +227,60 @@ class LayerOverviewUtils extends Script {
builtin
}
)[]
}[]
}[],
) {
const perId = new Map<string, any>()
for (const theme of themes) {
const keywords: {}[] = []
const keywords: Record<string, string[]> = {}
function addWord(language: string, word: string | string[]) {
if(Array.isArray(word)){
word.forEach(w => addWord(language, w))
return
}
word = Utils.SubstituteKeys(word, {}).trim()
if(!word){
return
}
console.log(language, "--->", word)
if (!keywords[language]) {
keywords[language] = []
}
keywords[language].push(word)
}
function addWords(tr: string | Record<string, string> | Record<string, string[]> | TagRenderingConfigJson) {
if(!tr){
return
}
if (typeof tr === "string") {
addWord("*", tr)
return
}
if (tr["render"] !== undefined || tr["mappings"] !== undefined) {
tr = <TagRenderingConfigJson>tr
addWords(<Translatable>tr.render)
for (let mapping of tr.mappings ?? []) {
if (typeof mapping === "string") {
addWords(mapping)
continue
}
addWords(mapping.then)
}
return
}
for (const lang in tr) {
addWord(lang, tr[lang])
}
}
for (const layer of theme.layers ?? []) {
const l = <LayerConfigJson>layer
keywords.push({ "*": l.id })
keywords.push(l.title)
keywords.push(l.description)
addWord("*", l.id)
addWords(l.title)
addWords(l.description)
addWords(l.searchTerms)
}
const data = {
@ -242,7 +290,7 @@ class LayerOverviewUtils extends Script {
icon: theme.icon,
hideFromOverview: theme.hideFromOverview,
mustHaveLanguage: theme.mustHaveLanguage,
keywords: Utils.NoNull(keywords),
keywords,
}
perId.set(theme.id, data)
}
@ -264,7 +312,7 @@ class LayerOverviewUtils extends Script {
writeFileSync(
"./src/assets/generated/theme_overview.json",
JSON.stringify(sorted, null, " "),
{ encoding: "utf8" }
{ encoding: "utf8" },
)
}
@ -276,7 +324,7 @@ class LayerOverviewUtils extends Script {
writeFileSync(
`${LayerOverviewUtils.themePath}${theme.id}.json`,
JSON.stringify(theme, null, " "),
{ encoding: "utf8" }
{ encoding: "utf8" },
)
}
@ -287,12 +335,12 @@ class LayerOverviewUtils extends Script {
writeFileSync(
`${LayerOverviewUtils.layerPath}${layer.id}.json`,
JSON.stringify(layer, null, " "),
{ encoding: "utf8" }
{ encoding: "utf8" },
)
}
static asDict(
trs: QuestionableTagRenderingConfigJson[]
trs: QuestionableTagRenderingConfigJson[],
): Map<string, QuestionableTagRenderingConfigJson> {
const d = new Map<string, QuestionableTagRenderingConfigJson>()
for (const tr of trs) {
@ -305,12 +353,12 @@ class LayerOverviewUtils extends Script {
getSharedTagRenderings(
doesImageExist: DoesImageExist,
bootstrapTagRenderings: Map<string, QuestionableTagRenderingConfigJson>,
bootstrapTagRenderingsOrder: string[]
bootstrapTagRenderingsOrder: string[],
): QuestionableTagRenderingConfigJson[]
getSharedTagRenderings(
doesImageExist: DoesImageExist,
bootstrapTagRenderings: Map<string, QuestionableTagRenderingConfigJson> = null,
bootstrapTagRenderingsOrder: string[] = []
bootstrapTagRenderingsOrder: string[] = [],
): QuestionableTagRenderingConfigJson[] {
const prepareLayer = new PrepareLayer(
{
@ -321,7 +369,7 @@ class LayerOverviewUtils extends Script {
},
{
addTagRenderingsToContext: true,
}
},
)
const path = "assets/layers/questions/questions.json"
@ -341,7 +389,7 @@ class LayerOverviewUtils extends Script {
return this.getSharedTagRenderings(
doesImageExist,
dict,
sharedQuestions.tagRenderings.map((tr) => tr["id"])
sharedQuestions.tagRenderings.map((tr) => tr["id"]),
)
}
@ -382,7 +430,7 @@ class LayerOverviewUtils extends Script {
console.warn(
"The SVG at " +
path +
" contains a `text`-tag. This is highly discouraged. Every machine viewing your theme has their own font libary, and the font you choose might not be present, resulting in a different font being rendered. Solution: open your .svg in inkscape (or another program), select the text and convert it to a path"
" contains a `text`-tag. This is highly discouraged. Every machine viewing your theme has their own font libary, and the font you choose might not be present, resulting in a different font being rendered. Solution: open your .svg in inkscape (or another program), select the text and convert it to a path",
)
errCount++
}
@ -398,14 +446,14 @@ class LayerOverviewUtils extends Script {
args
.find((a) => a.startsWith("--themes="))
?.substring("--themes=".length)
?.split(",") ?? []
?.split(",") ?? [],
)
const layerWhitelist = new Set(
args
.find((a) => a.startsWith("--layers="))
?.substring("--layers=".length)
?.split(",") ?? []
?.split(",") ?? [],
)
const forceReload = args.some((a) => a == "--force")
@ -440,11 +488,11 @@ class LayerOverviewUtils extends Script {
sharedLayers,
recompiledThemes,
forceReload,
themeWhitelist
themeWhitelist,
)
new ValidateThemeEnsemble().convertStrict(
Array.from(sharedThemes.values()).map((th) => new LayoutConfig(th, true))
Array.from(sharedThemes.values()).map((th) => new LayoutConfig(th, true)),
)
if (recompiledThemes.length > 0) {
@ -452,7 +500,7 @@ class LayerOverviewUtils extends Script {
"./src/assets/generated/known_layers.json",
JSON.stringify({
layers: Array.from(sharedLayers.values()).filter((l) => l.id !== "favourite"),
})
}),
)
}
@ -473,7 +521,7 @@ class LayerOverviewUtils extends Script {
const proto: LayoutConfigJson = JSON.parse(
readFileSync("./assets/themes/mapcomplete-changes/mapcomplete-changes.proto.json", {
encoding: "utf8",
})
}),
)
const protolayer = <LayerConfigJson>(
proto.layers.filter((l) => l["id"] === "mapcomplete-changes")[0]
@ -490,12 +538,12 @@ class LayerOverviewUtils extends Script {
layers: ScriptUtils.getLayerFiles().map((f) => f.parsed),
themes: ScriptUtils.getThemeFiles().map((f) => f.parsed),
},
ConversionContext.construct([], [])
ConversionContext.construct([], []),
)
for (const [_, theme] of sharedThemes) {
theme.layers = theme.layers.filter(
(l) => Constants.added_by_default.indexOf(l["id"]) < 0
(l) => Constants.added_by_default.indexOf(l["id"]) < 0,
)
}
@ -504,7 +552,7 @@ class LayerOverviewUtils extends Script {
"./src/assets/generated/known_themes.json",
JSON.stringify({
themes: Array.from(sharedThemes.values()),
})
}),
)
}
@ -516,7 +564,7 @@ class LayerOverviewUtils extends Script {
private parseLayer(
doesImageExist: DoesImageExist,
prepLayer: PrepareLayer,
sharedLayerPath: string
sharedLayerPath: string,
): {
raw: LayerConfigJson
parsed: LayerConfig
@ -527,7 +575,7 @@ class LayerOverviewUtils extends Script {
const parsed = parser.convertStrict(sharedLayerPath, context)
const result = AddIconSummary.singleton.convertStrict(
parsed,
context.inOperation("AddIconSummary")
context.inOperation("AddIconSummary"),
)
return { ...result, context }
}
@ -535,7 +583,7 @@ class LayerOverviewUtils extends Script {
private buildLayerIndex(
doesImageExist: DoesImageExist,
forceReload: boolean,
whitelist: Set<string>
whitelist: Set<string>,
): Map<string, LayerConfigJson> {
// First, we expand and validate all builtin layers. These are written to src/assets/generated/layers
// At the same time, an index of available layers is built.
@ -595,12 +643,12 @@ class LayerOverviewUtils extends Script {
skippedLayers.length +
" layers. Detected " +
warningCount +
" warnings"
" warnings",
)
// We always need the calculated tags of 'usersettings', so we export them separately
this.extractJavascriptCodeForLayer(
state.sharedLayers.get("usersettings"),
"./src/Logic/State/UserSettingsMetaTagging.ts"
"./src/Logic/State/UserSettingsMetaTagging.ts",
)
return sharedLayers
@ -617,8 +665,8 @@ class LayerOverviewUtils extends Script {
private extractJavascriptCode(themeFile: LayoutConfigJson) {
const allCode = [
"import {Feature} from 'geojson'",
'import { ExtraFuncType } from "../../../Logic/ExtraFunctions";',
'import { Utils } from "../../../Utils"',
"import { ExtraFuncType } from \"../../../Logic/ExtraFunctions\";",
"import { Utils } from \"../../../Utils\"",
"export class ThemeMetaTagging {",
" public static readonly themeName = " + JSON.stringify(themeFile.id),
"",
@ -631,7 +679,7 @@ class LayerOverviewUtils extends Script {
allCode.push(
" public metaTaggging_for_" +
id +
"(feat: Feature, helperFunctions: Record<ExtraFuncType, (feature: Feature) => Function>) {"
"(feat: Feature, helperFunctions: Record<ExtraFuncType, (feature: Feature) => Function>) {",
)
allCode.push(" const {" + ExtraFunctions.types.join(", ") + "} = helperFunctions")
for (const line of code) {
@ -645,7 +693,7 @@ class LayerOverviewUtils extends Script {
attributeName +
"', () => " +
expression +
" ) "
" ) ",
)
} else {
attributeName = attributeName.substring(0, attributeName.length - 1).trim()
@ -690,7 +738,7 @@ class LayerOverviewUtils extends Script {
const code = l.calculatedTags ?? []
allCode.push(
" public metaTaggging_for_" + l.id + "(feat: {properties: Record<string, string>}) {"
" public metaTaggging_for_" + l.id + "(feat: {properties: Record<string, string>}) {",
)
for (const line of code) {
const firstEq = line.indexOf("=")
@ -703,7 +751,7 @@ class LayerOverviewUtils extends Script {
attributeName +
"', () => " +
expression +
" ) "
" ) ",
)
} else {
attributeName = attributeName.substring(0, attributeName.length - 2).trim()
@ -728,14 +776,14 @@ class LayerOverviewUtils extends Script {
sharedLayers: Map<string, LayerConfigJson>,
recompiledThemes: string[],
forceReload: boolean,
whitelist: Set<string>
whitelist: Set<string>,
): Map<string, LayoutConfigJson> {
console.log(" ---------- VALIDATING BUILTIN THEMES ---------")
const themeFiles = ScriptUtils.getThemeFiles()
const fixed = new Map<string, LayoutConfigJson>()
const publicLayers = LayerOverviewUtils.publicLayerIdsFrom(
themeFiles.map((th) => th.parsed)
themeFiles.map((th) => th.parsed),
)
const trs = this.getSharedTagRenderings(new DoesImageExist(licensePaths, existsSync))
@ -775,15 +823,15 @@ class LayerOverviewUtils extends Script {
LayerOverviewUtils.themePath + "/" + themePath.substring(themePath.lastIndexOf("/"))
const usedLayers = Array.from(
LayerOverviewUtils.extractLayerIdsFrom(themeFile, false)
LayerOverviewUtils.extractLayerIdsFrom(themeFile, false),
).map((id) => LayerOverviewUtils.layerPath + id + ".json")
if (!forceReload && !this.shouldBeUpdated([themePath, ...usedLayers], targetPath)) {
fixed.set(
themeFile.id,
JSON.parse(
readFileSync(LayerOverviewUtils.themePath + themeFile.id + ".json", "utf8")
)
readFileSync(LayerOverviewUtils.themePath + themeFile.id + ".json", "utf8"),
),
)
ScriptUtils.erasableLog("Skipping", themeFile.id)
skippedThemes.push(themeFile.id)
@ -794,23 +842,23 @@ class LayerOverviewUtils extends Script {
new PrevalidateTheme().convertStrict(
themeFile,
ConversionContext.construct([themePath], ["PrepareLayer"])
ConversionContext.construct([themePath], ["PrepareLayer"]),
)
try {
themeFile = new PrepareTheme(convertState, {
skipDefaultLayers: true,
}).convertStrict(
themeFile,
ConversionContext.construct([themePath], ["PrepareLayer"])
ConversionContext.construct([themePath], ["PrepareLayer"]),
)
new ValidateThemeAndLayers(
new DoesImageExist(licensePaths, existsSync, knownTagRenderings),
themePath,
true,
knownTagRenderings
knownTagRenderings,
).convertStrict(
themeFile,
ConversionContext.construct([themePath], ["PrepareLayer"])
ConversionContext.construct([themePath], ["PrepareLayer"]),
)
if (themeFile.icon.endsWith(".svg")) {
@ -860,7 +908,7 @@ class LayerOverviewUtils extends Script {
const usedImages = Utils.Dedup(
new ExtractImages(true, knownTagRenderings)
.convertStrict(themeFile)
.map((x) => x.path)
.map((x) => x.path),
)
usedImages.sort()
@ -886,7 +934,7 @@ class LayerOverviewUtils extends Script {
t.shortDescription ?? new Translation(t.description).FirstSentence(),
mustHaveLanguage: t.mustHaveLanguage?.length > 0,
}
})
}),
)
}
@ -895,7 +943,7 @@ class LayerOverviewUtils extends Script {
recompiledThemes.join(", ") +
" and skipped " +
skippedThemes.length +
" themes"
" themes",
)
return fixed

10
src/Models/ThemeConfig/Json/LayerConfigJson.ts
View file

@ -41,14 +41,22 @@ export interface LayerConfigJson {
name?: Translatable
/**
* question: How would you describe the features that are shown on this layer?
*
* A description for the features shown in this layer.
* This often resembles the introduction of the wiki.osm.org-page for this feature.
*
* group: Basic
* question: How would you describe the features that are shown on this layer?
*/
description?: Translatable
/**
* question: What are some other terms used to describe these objects?
*
* This is used in the search functionality
*/
searchTerms?: Record<string, string[]>
/**
* Question: Where should the data be fetched from?
* title: Data Source

3
src/Models/ThemeConfig/LayerConfig.ts
View file

@ -29,6 +29,7 @@ export default class LayerConfig extends WithContextLoader {
public readonly id: string
public readonly name: Translation
public readonly description: Translation
public readonly searchTerms: Record<string, string[]>
/**
* Only 'null' for special, privileged layers
*/
@ -113,8 +114,8 @@ export default class LayerConfig extends WithContextLoader {
json.description = undefined
}
}
this.description = Translations.T(json.description, translationContext + ".description")
this.searchTerms = json.searchTerms ?? {}
this.calculatedTags = undefined
if (json.calculatedTags !== undefined) {

2
src/Models/ThemeConfig/LayoutConfig.ts
View file

@ -25,7 +25,7 @@ export class MinimalLayoutInformation {
definition?: Translatable
mustHaveLanguage?: boolean
hideFromOverview?: boolean
keywords?: (Translatable | TagRenderingConfigJson)[]
keywords?: Record<string, string[]>
}
/**
* Minimal information about a theme

44
src/UI/BigComponents/MoreScreen.ts
View file

@ -15,6 +15,7 @@ export default class MoreScreen {
MoreScreen.officialThemesById.set(th.id, th)
}
}
public static applySearch(searchTerm: string) {
searchTerm = searchTerm.toLowerCase()
if (!searchTerm) {
@ -43,13 +44,13 @@ export default class MoreScreen {
(th) =>
th.hideFromOverview == false &&
th.id !== "personal" &&
MoreScreen.MatchesLayout(th, searchTerm)
MoreScreen.MatchesLayout(th, searchTerm),
)
if (publicTheme !== undefined) {
window.location.href = MoreScreen.createUrlFor(publicTheme, false)
}
const hiddenTheme = MoreScreen.officialThemes.find(
(th) => th.id !== "personal" && MoreScreen.MatchesLayout(th, searchTerm)
(th) => th.id !== "personal" && MoreScreen.MatchesLayout(th, searchTerm),
)
if (hiddenTheme !== undefined) {
window.location.href = MoreScreen.createUrlFor(hiddenTheme, false)
@ -57,18 +58,14 @@ export default class MoreScreen {
}
public static MatchesLayout(
layout: {
id: string
title: Translatable
shortDescription: Translatable
keywords?: (Translatable | TagRenderingConfigJson)[]
},
search: string
layout: MinimalLayoutInformation,
search: string,
language?: string,
): boolean {
if (search === undefined) {
return true
}
search = Utils.RemoveDiacritics(search.toLocaleLowerCase()) // See #1729
search = Utils.simplifyStringForSearch(search.toLocaleLowerCase()) // See #1729
if (search.length > 3 && layout.id.toLowerCase().indexOf(search) >= 0) {
return true
}
@ -78,13 +75,30 @@ export default class MoreScreen {
if (Utils.simplifyStringForSearch(layout.id) === Utils.simplifyStringForSearch(search)) {
return true
}
const entitiesToSearch = [layout.shortDescription, layout.title, ...(layout.keywords ?? [])]
language ??= Locale.language.data
const entitiesToSearch: (string | Record<string, string> | Record<string, string[]>)[] = [layout.shortDescription, layout.title, layout.keywords]
for (const entity of entitiesToSearch) {
if (entity === undefined) {
continue
}
const term: string = entity["*"] ?? entity[Locale.language.data]
if (Utils.RemoveDiacritics(term?.toLowerCase())?.indexOf(search) >= 0) {
let term: string[]
if (typeof entity === "string") {
term = [entity]
} else {
const terms = [].concat(entity["*"], entity[language])
if (Array.isArray(terms)) {
term = terms
} else {
term = [terms]
}
}
const minLevehnstein = Math.min(...Utils.NoNull(term).map(t => Utils.levenshteinDistance(search,
Utils.simplifyStringForSearch(t).slice(0, search.length))))
if (minLevehnstein < 1 || minLevehnstein / search.length < 0.2) {
return true
}
}
@ -95,7 +109,7 @@ export default class MoreScreen {
public static createUrlFor(
layout: { id: string },
isCustom: boolean,
state?: { layoutToUse?: { id } }
state?: { layoutToUse?: { id } },
): string {
if (layout === undefined) {
return undefined
@ -141,7 +155,7 @@ export default class MoreScreen {
new Set<string>(
Object.keys(preferences)
.filter((key) => key.startsWith(prefix))
.map((key) => key.substring(prefix.length, key.length - "-enabled".length))
.map((key) => key.substring(prefix.length, key.length - "-enabled".length)),
))
}
}

4
src/UI/BigComponents/ThemeButton.svelte
View file

@ -3,13 +3,13 @@
import { ImmutableStore, Store, UIEventSource } from "../../Logic/UIEventSource"
import UserDetails, { OsmConnection } from "../../Logic/Osm/OsmConnection"
import Constants from "../../Models/Constants"
import type { LayoutInformation } from "../../Models/ThemeConfig/LayoutConfig"
import type { MinimalLayoutInformation } from "../../Models/ThemeConfig/LayoutConfig"
import Tr from "../Base/Tr.svelte"
import Translations from "../i18n/Translations"
import { LocalStorageSource } from "../../Logic/Web/LocalStorageSource"
import Marker from "../Map/Marker.svelte"
export let theme: LayoutInformation
export let theme: MinimalLayoutInformation
export let isCustom: boolean = false
export let userDetails: UIEventSource<UserDetails>
export let state: { layoutToUse?: { id: string }; osmConnection: OsmConnection }

4
src/UI/BigComponents/ThemesList.svelte
View file

@ -4,12 +4,12 @@
import { OsmConnection } from "../../Logic/Osm/OsmConnection"
import { UIEventSource } from "../../Logic/UIEventSource"
import ThemeButton from "./ThemeButton.svelte"
import { LayoutInformation } from "../../Models/ThemeConfig/LayoutConfig"
import { LayoutInformation, MinimalLayoutInformation } from "../../Models/ThemeConfig/LayoutConfig"
import MoreScreen from "./MoreScreen"
import themeOverview from "../../assets/generated/theme_overview.json"
export let search: UIEventSource<string>
export let themes: LayoutInformation[]
export let themes: MinimalLayoutInformation[]
export let state: { osmConnection: OsmConnection }
export let isCustom: boolean = false
export let hideThemes: boolean = true

4
src/Utils.ts
View file

@ -1602,6 +1602,7 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
* @constructor
*
* Utils.RemoveDiacritics("bâtiments") // => "batiments"
* Utils.RemoveDiacritics(undefined) // => undefined
*/
public static RemoveDiacritics(str?: string): string {
// See #1729
@ -1616,9 +1617,10 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
* @param str
* Utils.simplifyStringForSearch("abc def; ghi 564") // => "abcdefghi564"
* Utils.simplifyStringForSearch("âbc déf; ghi 564") // => "abcdefghi564"
* Utils.simplifyStringForSearch(undefined) // => undefined
*/
public static simplifyStringForSearch(str: string): string {
return Utils.RemoveDiacritics(str).toLowerCase().replace(/[^a-z0-9]/g, "")
return Utils.RemoveDiacritics(str)?.toLowerCase()?.replace(/[^a-z0-9]/g, "")
}
public static randomString(length: number): string {