midgard/mapcomplete
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Cleanup of NSI feature breanch

Browse source
This commit is contained in:
Pieter Vander Vennet 2024-05-23 04:42:26 +02:00
parent f6d93a6ab1
commit 3146fa0d26
14 changed files with 143 additions and 537 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

398
public/assets/data/nsi/stats/brand.BE.json
View file

File diff suppressed because one or more lines are too long

15
scripts/generateStats.ts
View file

@ -20,6 +20,7 @@ class Utilities {
}
}
class GenerateStats extends Script {
async createOptimizationFile(includeTags = true) {
@ -134,14 +135,14 @@ class GenerateStats extends Script {
}
async createNameSuggestionIndexFile(basepath: string,type: "brand" | "operator") {
const path = basepath+type+'.json'
async createNameSuggestionIndexFile(basepath: string, type: "brand" | "operator" | string) {
const path = basepath + type + ".json"
let allBrands = <Record<string, Record<string, number>>>{}
if (existsSync(path)) {
allBrands = JSON.parse(readFileSync(path, "utf8"))
console.log("Loaded", Object.keys(allBrands).length, " previously loaded brands")
}
let lastWrite = new Date()
const lastWrite = new Date()
let skipped = 0
const allBrandNames: string[] = Utils.Dedup(NameSuggestionIndex.allPossible(type).map(item => item.tags[type]))
for (let i = 0; i < allBrandNames.length; i++) {
@ -179,11 +180,13 @@ class GenerateStats extends Script {
}
async main(_: string[]) {
await this.createOptimizationFile()
const type = "brand"
const basepath = "./src/assets/generated/stats/"
for (const type of ["operator","brand"]) {
await this.createNameSuggestionIndexFile(basepath, type)
this.summarizeNSI(basepath+type+".json", "./public/assets/data/stats/"+type)
this.summarizeNSI(basepath + type + ".json", "./public/assets/data/nsi/stats/" + type)
}
await this.createOptimizationFile()
}

54
src/Logic/Web/NameSuggestionIndex.ts
View file

@ -58,6 +58,10 @@ export interface NSIItem {
export default class NameSuggestionIndex {
private static readonly nsiFile: Readonly<NSIFile> = <any>nsi
private static readonly nsiWdFile: Readonly<Record<string, {
logos: { wikidata?: string, facebook?: string }
}>> = <any>nsiWD["wikidata"]
private static loco = new LocationConflation(nsiFeatures) // Some additional boundaries
private static _supportedTypes: string[]
@ -68,7 +72,12 @@ export default class NameSuggestionIndex {
}
const keys = Object.keys(NameSuggestionIndex.nsiFile.nsi)
const all = keys.map(k => NameSuggestionIndex.nsiFile.nsi[k].properties.path.split("/")[0])
this._supportedTypes = Utils.Dedup(all)
this._supportedTypes = Utils.Dedup(all).map(s => {
if(s.endsWith("s")){
s = s.substring(0, s.length - 1)
}
return s
})
return this._supportedTypes
}
@ -82,7 +91,6 @@ export default class NameSuggestionIndex {
private static async fetchFrequenciesFor(type: string, countries: string[]) {
let stats = await Promise.all(countries.map(c => {
try {
return Utils.downloadJsonCached<Record<string, number>>(`./assets/data/nsi/stats/${type}.${c.toUpperCase()}.json`, 24 * 60 * 60 * 1000)
} catch (e) {
console.error("Could not fetch " + type + " statistics due to", e)
@ -93,6 +101,9 @@ export default class NameSuggestionIndex {
if (stats.length === 1) {
return stats[0]
}
if(stats.length === 0){
return {}
}
const merged = stats[0]
for (let i = 1; i < stats.length; i++) {
for (const countryCode in stats[i]) {
@ -103,10 +114,7 @@ export default class NameSuggestionIndex {
}
public static isSvg(nsiItem: NSIItem, type: string): boolean | undefined {
const logos = nsiWD["wikidata"][nsiItem?.tags?.[type + ":wikidata"]]?.logos
if(nsiItem.id === "axa-2f6feb"){
console.trace(">>> HI")
}
const logos = this.nsiWdFile[nsiItem?.tags?.[type + ":wikidata"]]?.logos
if (!logos) {
return undefined
}
@ -120,29 +128,36 @@ export default class NameSuggestionIndex {
return false
}
public static async generateMappings(type: string, key: string, value: string, country: string[], location?: [number, number]) {
public static async generateMappings(type: string, tags: Record<string, string>, country: string[], location?: [number, number]): Promise<Mapping[]> {
const mappings: Mapping[] = []
const frequencies = await NameSuggestionIndex.fetchFrequenciesFor(type, country)
const actualBrands = NameSuggestionIndex.getSuggestionsFor(type, key, value, country.join(";"), location)
for (const key in tags) {
if (key.startsWith("_")) {
continue
}
const value = tags[key]
const actualBrands = NameSuggestionIndex.getSuggestionsForKV(type, key, value, country.join(";"), location)
if(!actualBrands){
continue
}
for (const nsiItem of actualBrands) {
const tags = nsiItem.tags
const frequency = frequencies[nsiItem.displayName]
const logos = nsiWD["wikidata"][nsiItem.tags[type + ":wikidata"]]?.logos
let iconUrl = logos?.facebook ?? logos?.wikidata
const logos = this.nsiWdFile[nsiItem.tags[type + ":wikidata"]]?.logos
const iconUrl = logos?.facebook ?? logos?.wikidata
const hasIcon = iconUrl !== undefined
let icon = undefined
if (hasIcon) {
// Using <img src=...> works fine without an extension for JPG and PNG, but _not_ svg :(
icon = "./assets/data/nsi/logos/" + nsiItem.id
if (NameSuggestionIndex.isSvg(nsiItem, type)) {
console.log("Is svg:", nsiItem.displayName)
icon = icon + ".svg"
}
}
mappings.push({
if: new Tag(type, tags[type]),
addExtraTags: Object.keys(tags).filter(k => k !== type).map(k => new Tag(k, tags[k])),
then: new TypedTranslation<{}>({ "*": nsiItem.displayName }),
then: new TypedTranslation<Record<string, never>>({ "*": nsiItem.displayName }),
hideInAnswer: false,
ifnot: undefined,
alsoShowIf: undefined,
@ -152,6 +167,7 @@ export default class NameSuggestionIndex {
searchTerms: { "*": [nsiItem.displayName, nsiItem.id] }
})
}
}
return mappings
}
@ -184,7 +200,7 @@ export default class NameSuggestionIndex {
for (const osmKey in tags) {
const values = tags[osmKey]
for (const osmValue of values) {
const suggestions = this.getSuggestionsFor(type, osmKey, osmValue)
const suggestions = this.getSuggestionsForKV(type, osmKey, osmValue)
options.push(...suggestions)
}
}
@ -193,11 +209,19 @@ export default class NameSuggestionIndex {
/**
*
* @param path
* @param country: a string containing one or more country codes, separated by ";"
* @param location: center point of the feature, should be [lon, lat]
*/
public static getSuggestionsFor(type: string, key: string, value: string, country: string = undefined, location: [number, number] = undefined): NSIItem[] {
public static getSuggestionsFor(type: string, tags: {key: string, value: string}[], country: string = undefined, location: [number, number] = undefined): NSIItem[] {
return tags.flatMap(tag => this.getSuggestionsForKV(type, tag.key, tag.value, country, location))
}
/**
*
* @param country: a string containing one or more country codes, separated by ";"
* @param location: center point of the feature, should be [lon, lat]
*/
public static getSuggestionsForKV(type: string, key: string, value: string, country: string = undefined, location: [number, number] = undefined): NSIItem[] {
const path = `${type}s/${key}/${value}`
const entry = NameSuggestionIndex.nsiFile.nsi[path]
return entry?.items?.filter(i => {

19
src/Models/ThemeConfig/Conversion/Validation.ts
View file

@ -927,8 +927,10 @@ class CheckTranslation extends DesugaringStep<Translatable> {
}
class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJson> {
constructor() {
private readonly _layerConfig: LayerConfigJson
constructor(layerConfig?: LayerConfigJson) {
super("Miscellaneous checks on the tagrendering", ["special"], "MiscTagRenderingChecks")
this._layerConfig = layerConfig
}
convert(
@ -1092,13 +1094,14 @@ class MiscTagRenderingChecks extends DesugaringStep<TagRenderingConfigJson> {
)
}
}
if(json.freeform.type === "nsi"){
const [key, value] = json.freeform.helperArgs[0].split("=")
const path = `${json.freeform.key}s/${key}/${value}`
const suggestions = NameSuggestionIndex.getSuggestionsFor(json.freeform.key, key, value)
if(this._layerConfig?.source?.osmTags && NameSuggestionIndex.supportedTypes().indexOf(json.freeform.key) >= 0){
const tags= TagUtils.TagD(this._layerConfig?.source?.osmTags)?.usedTags()
const suggestions = NameSuggestionIndex.getSuggestionsFor(json.freeform.key, tags)
if(suggestions === undefined){
context.enters("freeform","type").err("No entry found in the 'Name Suggestion Index' for "+path)
context.enters("freeform","type").err("No entry found in the 'Name Suggestion Index'. None of the 'osmSource'-tags match an entry in the NSI.\n\tOsmSource-tags are "+tags.map(t => t.asHumanString()).join(" ; "))
}
}else if(json.freeform.type === "nsi"){
context.enters("freeform","type").warn("No need to explicitly set type to 'NSI', autodetected based on freeform type")
}
}
if (json.render && json["question"] && json.freeform === undefined) {
@ -1145,7 +1148,7 @@ export class ValidateTagRenderings extends Fuse<TagRenderingConfigJson> {
constructor(layerConfig?: LayerConfigJson, doesImageExist?: DoesImageExist) {
super(
"Various validation on tagRenderingConfigs",
new MiscTagRenderingChecks(),
new MiscTagRenderingChecks(layerConfig),
new DetectShadowedMappings(layerConfig),
new DetectMappingsShadowedByCondition(),
@ -1156,7 +1159,7 @@ export class ValidateTagRenderings extends Fuse<TagRenderingConfigJson> {
new On("question", new ValidatePossibleLinks()),
new On("questionHint", new ValidatePossibleLinks()),
new On("mappings", new Each(new On("then", new ValidatePossibleLinks()))),
new MiscTagRenderingChecks()
new MiscTagRenderingChecks(layerConfig)
)
}
}

6
src/Models/ThemeConfig/Json/QuestionableTagRenderingConfigJson.ts
View file

@ -233,12 +233,6 @@ export interface QuestionableTagRenderingConfigJson extends TagRenderingConfigJs
*/
placeholder?: Translatable
/**
* Extra parameters to initialize the input helper arguments.
* For semantics, see the 'SpecialInputElements.md'
* group: expert
*/
helperArgs?: (string | number | boolean | any)[]
/**
* If a value is added with the textfield, these extra tag is addded.
* Useful to add a 'fixme=freeform textfield used - to be checked'

35
src/Models/ThemeConfig/TagRenderingConfig.ts
View file

@ -72,12 +72,11 @@ export default class TagRenderingConfig {
readonly addExtraTags: UploadableTag[]
readonly inline: boolean
readonly default?: string
readonly helperArgs?: (string | number | boolean)[]
}
public readonly multiAnswer: boolean
public readonly mappings: Mapping[]
public mappings: Mapping[]
public readonly editButtonAriaLabel?: Translation
public readonly labels: string[]
public readonly classes: string[] | undefined
@ -202,8 +201,7 @@ export default class TagRenderingConfig {
TagUtils.ParseUploadableTag(tg, `${context}.extratag[${i}]`)
) ?? [],
inline: json.freeform.inline ?? false,
default: json.freeform.default,
helperArgs: json.freeform.helperArgs
default: json.freeform.default
}
if (json.freeform["extraTags"] !== undefined) {
throw `Freeform.extraTags is defined. This should probably be 'freeform.addExtraTag' (at ${context})`
@ -371,8 +369,8 @@ export default class TagRenderingConfig {
let icon = undefined
let iconClass = commonSize
if (!!mapping.icon) {
if (typeof mapping.icon === "string" && mapping.icon !== "") {
if (mapping.icon) {
if (typeof mapping.icon === "string") {
icon = mapping.icon.trim()
} else if (mapping.icon["path"]) {
icon = mapping.icon["path"].trim()
@ -467,8 +465,8 @@ export default class TagRenderingConfig {
// A flag to check that the freeform key isn't matched multiple times
// If it is undefined, it is "used" already, or at least we don't have to check for it anymore
let freeformKeyDefined = this.freeform?.key !== undefined
let usedFreeformValues = new Set<string>()
const freeformKeyDefined = this.freeform?.key !== undefined
const usedFreeformValues = new Set<string>()
// We run over all the mappings first, to check if the mapping matches
const applicableMappings: {
then: TypedTranslation<Record<string, string>>
@ -556,9 +554,6 @@ export default class TagRenderingConfig {
EnumerateTranslations(): Translation[] {
const translations: Translation[] = []
for (const key in this) {
if (!this.hasOwnProperty(key)) {
continue
}
const o = this[key]
if (o instanceof Translation) {
translations.push(o)
@ -572,7 +567,7 @@ export default class TagRenderingConfig {
const key = this.freeform?.key
const answerMappings = this.mappings?.filter((m) => m.hideInAnswer !== true)
if (key === undefined) {
let values: { k: string; v: string }[][] = Utils.NoNull(
const values: { k: string; v: string }[][] = Utils.NoNull(
answerMappings?.map((m) => m.if.asChange({})) ?? []
)
if (values.length === 0) {
@ -870,21 +865,25 @@ export default class TagRenderingConfig {
export class TagRenderingConfigUtils {
public static withNameSuggestionIndex(config: TagRenderingConfig, tags: UIEventSource<Record<string, string>>, feature?: Feature): Store<TagRenderingConfig> {
if(config.freeform?.type !== "nsi"){
const isNSI = NameSuggestionIndex.supportedTypes().indexOf(config.freeform?.key) >= 0
if (!isNSI) {
return new ImmutableStore(config)
}
const extraMappings = tags.mapD(tags => tags._country).bindD(country => {
const [k, v] = ("" + config.freeform.helperArgs[0]).split("=")
const extraMappings = tags
.bindD(tags => {
const country = tags._country
if(country === undefined){
return undefined
}
const center = GeoOperations.centerpointCoordinates(feature)
return UIEventSource.FromPromise(NameSuggestionIndex.generateMappings(config.freeform.key, k, v, country.split(";"), center))
return UIEventSource.FromPromise(NameSuggestionIndex.generateMappings(config.freeform.key, tags, country.split(";"), center))
})
return extraMappings.map(extraMappings => {
if (!extraMappings || extraMappings.length == 0) {
return config
}
const clone: TagRenderingConfig = Object.create(config)
/// SHHHTTT, this is not cheating at all!
clone.mappings.splice(clone.mappings.length, 0, ...extraMappings)
clone.mappings = [...clone.mappings ?? [], ...extraMappings]
return clone
})
}

1
src/Models/ThemeViewState.ts
View file

@ -391,7 +391,6 @@ export default class ThemeViewState implements SpecialVisualizationState {
if (fs.layer.layerDef.source.geojsonSource !== undefined) {
return // We don't cache external data layers
}
console.log("Setting up a local store feature sink for", layerId)
const storage = new SaveFeatureSourceToLocalStorage(
this.osmConnection.Backend(),
fs.layer.layerDef.id,

1
src/UI/Base/CloseAnimation.svelte
View file

@ -18,7 +18,6 @@
function animate(opened: boolean) {
const moveToElem = moveTo.data
console.log("Animating", debug," to", opened)
if (opened) {
copySizeOf(targetOuter)
elem.style.background = "var(--background-color)"

3
src/UI/BigComponents/SelectedElementView.svelte
View file

@ -11,7 +11,6 @@
import UserRelatedState from "../../Logic/State/UserRelatedState"
import Delete_icon from "../../assets/svg/Delete_icon.svelte"
import BackButton from "../Base/BackButton.svelte"
import TagRenderingEditableDynamic from "../Popup/TagRendering/TagRenderingEditableDynamic.svelte"
export let state: SpecialVisualizationState
export let selectedElement: Feature
@ -69,7 +68,7 @@
tabindex="-1"
>
{#each $knownTagRenderings as config (config.id)}
<TagRenderingEditableDynamic
<TagRenderingEditable
{tags}
{config}
{state}

22
src/UI/Popup/TagRendering/TagRenderingAnswerDynamic.svelte Normal file
View file

@ -0,0 +1,22 @@
<script lang="ts">
import TagRenderingConfig, { TagRenderingConfigUtils } from "../../../Models/ThemeConfig/TagRenderingConfig"
import type { SpecialVisualizationState } from "../../SpecialVisualization"
import type { Feature } from "geojson"
import { UIEventSource } from "../../../Logic/UIEventSource"
import LayerConfig from "../../../Models/ThemeConfig/LayerConfig"
import TagRenderingAnswer from "./TagRenderingAnswer.svelte"
export let tags: UIEventSource<Record<string, string> | undefined>
export let state: SpecialVisualizationState
export let selectedElement: Feature
export let layer: LayerConfig
export let config: TagRenderingConfig
export let extraClasses: string | undefined = undefined
export let id: string = undefined
let dynamicConfig = TagRenderingConfigUtils.withNameSuggestionIndex(config, tags, selectedElement)
</script>
<TagRenderingAnswer {selectedElement} {layer} config={$dynamicConfig} {extraClasses} {id} {tags} {state} />

9
src/UI/Popup/TagRendering/TagRenderingEditable.svelte
View file

@ -3,8 +3,6 @@
import { Store, UIEventSource } from "../../../Logic/UIEventSource"
import type { Feature } from "geojson"
import type { SpecialVisualizationState } from "../../SpecialVisualization"
import TagRenderingAnswer from "./TagRenderingAnswer.svelte"
import { PencilAltIcon } from "@rgossiaux/svelte-heroicons/solid"
import TagRenderingQuestion from "./TagRenderingQuestion.svelte"
import { onDestroy } from "svelte"
import Tr from "../../Base/Tr.svelte"
@ -12,9 +10,8 @@
import LayerConfig from "../../../Models/ThemeConfig/LayerConfig"
import { Utils } from "../../../Utils"
import { twMerge } from "tailwind-merge"
import { ariaLabel } from "../../../Utils/ariaLabel"
import EditButton from "./EditButton.svelte"
import EditItemButton from "../../Studio/EditItemButton.svelte"
import TagRenderingAnswerDynamic from "./TagRenderingAnswerDynamic.svelte"
export let config: TagRenderingConfig
export let tags: UIEventSource<Record<string, string>>
@ -103,7 +100,7 @@
</TagRenderingQuestion>
{:else}
<div class="low-interaction flex items-center justify-between overflow-hidden rounded px-2">
<TagRenderingAnswer id={answerId} {config} {tags} {selectedElement} {state} {layer} />
<TagRenderingAnswerDynamic id={answerId} {config} {tags} {selectedElement} {state} {layer} />
<EditButton
arialabel={config.editButtonAriaLabel}
ariaLabelledBy={answerId}
@ -115,7 +112,7 @@
{/if}
{:else}
<div class="h-full w-full overflow-hidden">
<TagRenderingAnswer {config} {tags} {selectedElement} {state} {layer} />
<TagRenderingAnswerDynamic {config} {tags} {selectedElement} {state} {layer} />
</div>
{/if}
</div>

32
src/UI/Popup/TagRendering/TagRenderingEditableDynamic.svelte
View file

@ -1,32 +0,0 @@
<script lang="ts">/**
* Wrapper around 'tagRenderingEditable' but might add mappings dynamically
*/
import TagRenderingConfig, { TagRenderingConfigUtils } from "../../../Models/ThemeConfig/TagRenderingConfig"
import { UIEventSource } from "../../../Logic/UIEventSource"
import type { Feature } from "geojson"
import type { SpecialVisualizationState } from "../../SpecialVisualization"
import LayerConfig from "../../../Models/ThemeConfig/LayerConfig"
import TagRenderingEditable from "./TagRenderingEditable.svelte"
export let config: TagRenderingConfig
export let tags: UIEventSource<Record<string, string>>
export let selectedElement: Feature | undefined
export let state: SpecialVisualizationState
export let layer: LayerConfig = undefined
export let highlightedRendering: UIEventSource<string> = undefined
export let clss = undefined
let dynamicConfig = TagRenderingConfigUtils.withNameSuggestionIndex(config, tags, selectedElement)
</script>
<TagRenderingEditable
{tags}
config={$dynamicConfig}
{state}
{selectedElement}
{layer}
{highlightedRendering}
{clss}
/>

6
src/UI/Popup/TagRendering/TagRenderingQuestion.svelte
View file

@ -72,7 +72,6 @@
/**
* Prepares and fills the checkedMappings
*/
console.log("Initing ", config.id)
function initialize(tgs: Record<string, string>, confg: TagRenderingConfig): void {
mappings = confg.mappings?.filter((m) => {
@ -160,7 +159,6 @@
}
}
if (somethingChanged) {
console.log("Updating minimal tags to", newMinimal, "of", config.id)
minimalTags.setData(newMinimal)
}
})
@ -224,7 +222,7 @@
}
}
if (extraTags.data) {
if (extraTags.data && selectedTags !== undefined) {
// Map the extraTags into an array of Tag objects
const extraTagsArray = Object.entries(extraTags.data).map(([k, v]) => new Tag(k, v))
@ -236,7 +234,7 @@
// Add the extraTags to the existing And
selectedTags = new And([...selectedTags.and, ...extraTagsArray])
} else {
console.error("selectedTags is not of type Tag or And")
console.error("selectedTags is not of type Tag or And, it is a "+JSON.stringify(selectedTags))
}
}
}

3
src/test.ts
View file

@ -2,7 +2,4 @@ import SvelteUIElement from "./UI/Base/SvelteUIElement"
import Test from "./UI/Test.svelte"
import NameSuggestionIndex from "./Logic/Web/NameSuggestionIndex"
const nsi = NameSuggestionIndex.get()
const secondhandshops = nsi.getSuggestionsFor("brands/shop/second_hand", ["be"])
console.log(secondhandshops)
new SvelteUIElement(Test).AttachTo("maindiv")