midgard/mapcomplete
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add check for possible duplicate filters, consolidate duplicate filters

Browse source
This commit is contained in:
pietervdvn 2022-09-24 03:33:09 +02:00
parent f0c7c65d8d
commit dadba86faf
9 changed files with 236 additions and 154 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

162
Models/ThemeConfig/Conversion/Validation.ts
View file

@ -1,19 +1,23 @@
import { DesugaringStep, Each, Fuse, On } from "./Conversion"
import { LayerConfigJson } from "../Json/LayerConfigJson"
import {DesugaringStep, Each, Fuse, On} from "./Conversion"
import {LayerConfigJson} from "../Json/LayerConfigJson"
import LayerConfig from "../LayerConfig"
import { Utils } from "../../../Utils"
import {Utils} from "../../../Utils"
import Constants from "../../Constants"
import { Translation } from "../../../UI/i18n/Translation"
import { LayoutConfigJson } from "../Json/LayoutConfigJson"
import {Translation} from "../../../UI/i18n/Translation"
import {LayoutConfigJson} from "../Json/LayoutConfigJson"
import LayoutConfig from "../LayoutConfig"
import { TagRenderingConfigJson } from "../Json/TagRenderingConfigJson"
import { TagUtils } from "../../../Logic/Tags/TagUtils"
import { ExtractImages } from "./FixImages"
import {TagRenderingConfigJson} from "../Json/TagRenderingConfigJson"
import {TagUtils} from "../../../Logic/Tags/TagUtils"
import {ExtractImages} from "./FixImages"
import ScriptUtils from "../../../scripts/ScriptUtils"
import { And } from "../../../Logic/Tags/And"
import {And} from "../../../Logic/Tags/And"
import Translations from "../../../UI/i18n/Translations"
import Svg from "../../../Svg"
import { QuestionableTagRenderingConfigJson } from "../Json/QuestionableTagRenderingConfigJson"
import {QuestionableTagRenderingConfigJson} from "../Json/QuestionableTagRenderingConfigJson"
import FilterConfigJson from "../Json/FilterConfigJson";
import {control} from "leaflet";
import layers = control.layers;
import DeleteConfig from "../DeleteConfig";
class ValidateLanguageCompleteness extends DesugaringStep<any> {
private readonly _languages: string[]
@ -79,16 +83,16 @@ export class DoesImageExist extends DesugaringStep<string> {
const information = []
if (image.indexOf("{") >= 0) {
information.push("Ignoring image with { in the path: " + image)
return { result: image }
return {result: image}
}
if (image === "assets/SocialImage.png") {
return { result: image }
return {result: image}
}
if (image.match(/[a-z]*/)) {
if (Svg.All[image + ".svg"] !== undefined) {
// This is a builtin img, e.g. 'checkmark' or 'crosshair'
return { result: image }
return {result: image}
}
}
@ -168,7 +172,7 @@ class ValidateTheme extends DesugaringStep<LayoutConfigJson> {
}
}
}
if(this._isBuiltin) {
if (this._isBuiltin) {
// Check images: are they local, are the licenses there, is the theme icon square, ...
const images = new ExtractImages(
this._isBuiltin,
@ -224,7 +228,7 @@ class ValidateTheme extends DesugaringStep<LayoutConfigJson> {
}
try {
if(this._isBuiltin){
if (this._isBuiltin) {
if (theme.id !== theme.id.toLowerCase()) {
errors.push("Theme ids should be in lowercase, but it is " + theme.id)
@ -321,7 +325,7 @@ class OverrideShadowingCheck extends DesugaringStep<LayoutConfigJson> {
): { result: LayoutConfigJson; errors?: string[]; warnings?: string[] } {
const overrideAll = json.overrideAll
if (overrideAll === undefined) {
return { result: json }
return {result: json}
}
const errors = []
@ -348,7 +352,7 @@ class OverrideShadowingCheck extends DesugaringStep<LayoutConfigJson> {
}
}
return { result: json, errors }
return {result: json, errors}
}
}
@ -458,7 +462,7 @@ export class DetectShadowedMappings extends DesugaringStep<QuestionableTagRender
const errors = []
const warnings = []
if (json.mappings === undefined || json.mappings.length === 0) {
return { result: json }
return {result: json}
}
const defaultProperties = {}
for (const calculatedTagName of this._calculatedTagNames) {
@ -487,7 +491,7 @@ export class DetectShadowedMappings extends DesugaringStep<QuestionableTagRender
}
const keyValues = parsedConditions[i].asChange(defaultProperties)
const properties = {}
keyValues.forEach(({ k, v }) => {
keyValues.forEach(({k, v}) => {
properties[k] = v
})
for (let j = 0; j < i; j++) {
@ -576,7 +580,7 @@ export class DetectMappingsWithImages extends DesugaringStep<TagRenderingConfigJ
const warnings: string[] = []
const information: string[] = []
if (json.mappings === undefined || json.mappings.length === 0) {
return { result: json }
return {result: json}
}
const ignoreToken = "ignore-image-in-then"
for (let i = 0; i < json.mappings.length; i++) {
@ -702,8 +706,14 @@ export class ValidateLayer extends DesugaringStep<LayerConfigJson> {
}
}
if(json.deletion !== undefined && json.deletion instanceof DeleteConfig){
if(json.deletion.softDeletionTags === undefined){
warnings.push("No soft-deletion tags in deletion block for layer "+json.id)
}
}
try {
if(this._isBuiltin) {
if (this._isBuiltin) {
// Some checks for legacy elements
if (json["overpassTags"] !== undefined) {
@ -750,7 +760,7 @@ export class ValidateLayer extends DesugaringStep<LayerConfigJson> {
warnings.push(context + " has a tagRendering as `isShown`")
}
}
if(this._isBuiltin) {
if (this._isBuiltin) {
// Check location of layer file
const expected: string = `assets/layers/${json.id}/${json.id}.json`
if (this._path != undefined && this._path.indexOf(expected) < 0) {
@ -830,7 +840,7 @@ export class ValidateLayer extends DesugaringStep<LayerConfigJson> {
const preset = json.presets[i]
const tags: { k: string; v: string }[] = new And(
preset.tags.map((t) => TagUtils.Tag(t))
).asChange({ id: "node/-1" })
).asChange({id: "node/-1"})
const properties = {}
for (const tag of tags) {
properties[tag.k] = tag.v
@ -861,3 +871,109 @@ export class ValidateLayer extends DesugaringStep<LayerConfigJson> {
}
}
}
export class DetectDuplicateFilters extends DesugaringStep<{ layers: LayerConfigJson[], themes: LayoutConfigJson[]}> {
constructor() {
super("Tries to detect layers where a shared filter can be used (or where similar filters occur)", [], "DetectDuplicateFilters");
}
/**
* Add all filter options into 'perOsmTag'
*/
private addLayerFilters(layer: LayerConfigJson, perOsmTag: Map<string, {
layer: LayerConfigJson,
layout: LayoutConfigJson | undefined,
filter: FilterConfigJson
}[]>, layout?: LayoutConfigJson | undefined): void {
if (layer.filter === undefined || layer.filter === null) {
return;
}
if (layer.filter["sameAs"] !== undefined) {
return;
}
for (const filter of (<(string | FilterConfigJson) []>layer.filter)) {
if (typeof filter === "string") {
continue
}
if(filter["#"]?.indexOf("ignore-possible-duplicate")>=0){
continue
}
for (const option of filter.options) {
if (option.osmTags === undefined) {
continue
}
const key = JSON.stringify(option.osmTags)
if (!perOsmTag.has(key)) {
perOsmTag.set(key, [])
}
perOsmTag.get(key).push({
layer, filter, layout
})
}
}
}
convert(json: { layers: LayerConfigJson[]; themes: LayoutConfigJson[] }, context: string): { result: { layers: LayerConfigJson[]; themes: LayoutConfigJson[] }; errors?: string[]; warnings?: string[]; information?: string[] } {
const errors: string[] = []
const warnings: string[] = []
const information: string[] = []
const {layers, themes} = json
const perOsmTag = new Map<string, {
layer: LayerConfigJson,
layout: LayoutConfigJson | undefined,
filter: FilterConfigJson
}[]>()
for (const layer of layers) {
this.addLayerFilters(layer, perOsmTag)
}
for (const theme of themes) {
if(theme.id === "personal"){
continue
}
for (const layer of theme.layers) {
if(typeof layer === "string"){
continue
}
if(layer["builtin"] !== undefined){
continue
}
this.addLayerFilters(<LayerConfigJson> layer, perOsmTag, theme)
}
}
// At this point, we have gathered all filters per tag - time to find duplicates
perOsmTag.forEach((value, key) => {
if(value.length <= 1){
// Seen this key just once, it is unique
return;
}
let msg = "Possible duplicate filter: "+ key
for (const {filter, layer, layout} of value) {
let id = ""
if(layout !== undefined){
id = layout.id + ":"
}
msg += `\n - ${id}${layer.id}.${filter.id}`
}
warnings.push(msg)
})
return {
result: json,
errors,
warnings,
information
};
}
}

2
assets/layers/climbing/climbing.json
View file

@ -6,7 +6,7 @@
"nl": "Een dummy-laag die tagrenderings bevat, gedeeld over de verschillende klimsport lagen",
"de": "Eine Dummy-Ebene, die Tagrenderings enthält, die von den Kletterebenen gemeinsam genutzt werden"
},
"minzoom": 25,
"minzoom": 19,
"source": {
"osmTags": "sport=climbing"
},

5
assets/layers/filters/filters.json
View file

@ -18,7 +18,10 @@
"es": "Abierta ahora",
"fr": "Ouvert maintenant",
"hu": "Most nyitva van",
"da": "Åbent nu"
"da": "Åbent nu",
"zh_Hant": "目前開放",
"id": "Saat ini buka",
"it": "Aperto ora"
},
"osmTags": "_isOpen=yes"
}

20
assets/layers/parcel_lockers/parcel_lockers.json
View file

@ -212,25 +212,7 @@
}
],
"filter": [
{
"id": "is_open",
"options": [
{
"question": {
"en": "Currently open",
"de": "Aktuell geöffnet",
"zh_Hant": "目前開放",
"id": "Saat ini buka",
"hu": "Most nyitva",
"nl": "Momenteel geopend",
"ca": "Actualment obert",
"es": "Actualmente abierta",
"fr": "Ouvert actuellement"
},
"osmTags": "_isOpen=yes"
}
]
}
"open_now"
],
"allowMove": {
"enableImproveAccuracy": true

2
assets/layers/parking_spaces/parking_spaces.json
View file

@ -8,7 +8,7 @@
"en": "Layer showing individual parking spaces.",
"de": "Ebene mit den einzelnen PKW Stellplätzen."
},
"minzoom": 20,
"minzoom": 19,
"source": {
"osmTags": "amenity=parking_space"
},

20
assets/layers/postoffices/postoffices.json
View file

@ -121,25 +121,7 @@
}
],
"filter": [
{
"id": "is_open",
"options": [
{
"question": {
"en": "Currently open",
"de": "Aktuell geöffnet",
"zh_Hant": "目前開放",
"id": "Saat ini buka",
"hu": "Most nyitva",
"nl": "Momenteel geopend",
"ca": "Actualment obert",
"es": "Actualmente abierta",
"fr": "Ouvert actuellement"
},
"osmTags": "_isOpen=yes"
}
]
}
"open_now"
],
"mapRendering": [
{

16
assets/layers/recycling/recycling.json
View file

@ -990,21 +990,7 @@
}
],
"filter": [
{
"id": "isOpen",
"options": [
{
"question": {
"en": "Currently open",
"nl": "Op dit moment open",
"de": "Derzeit geöffnet",
"es": "Actualmente abierto",
"it": "Aperto ora"
},
"osmTags": "_isOpen=yes"
}
]
},
"open_now",
{
"id": "recyclingType",
"options": [

1
assets/layers/waste_disposal/waste_disposal.json
View file

@ -170,6 +170,7 @@
],
"filter": [
{
"#": "ignore-possible-duplicate",
"id": "public-access",
"options": [
{

16
scripts/generateLayerOverview.ts
View file

@ -5,6 +5,7 @@ import { LayoutConfigJson } from "../Models/ThemeConfig/Json/LayoutConfigJson"
import { LayerConfigJson } from "../Models/ThemeConfig/Json/LayerConfigJson"
import Constants from "../Models/Constants"
import {
DetectDuplicateFilters,
DoesImageExist,
PrevalidateTheme,
ValidateLayer,
@ -21,6 +22,7 @@ import { PrepareTheme } from "../Models/ThemeConfig/Conversion/PrepareTheme"
import { DesugaringContext } from "../Models/ThemeConfig/Conversion/Conversion"
import { Utils } from "../Utils"
import { AllKnownLayouts } from "../Customizations/AllKnownLayouts"
import {Script} from "vm";
// This scripts scans 'assets/layers/*.json' for layer definition files and 'assets/themes/*.json' for theme definition files.
// It spits out an overview of those to be used to load them
@ -202,10 +204,15 @@ class LayerOverviewUtils {
"assets/SocialImageTemplateWide.svg",
"assets/SocialImageBanner.svg",
"assets/svg/osm-logo.svg",
"assets/templates/"
"assets/templates/*"
]
for (const path of allSvgs) {
if (exempt.some((p) => "./" + p === path)) {
if (exempt.some((p) => {
if(p.endsWith("*") && path.startsWith("./"+p.substring(0, p.length - 1))){
return true
}
return "./" + p === path;
})) {
continue
}
@ -289,6 +296,11 @@ class LayerOverviewUtils {
this.checkAllSvgs()
new DetectDuplicateFilters().convertStrict({
layers: ScriptUtils.getLayerFiles().map(f => f.parsed),
themes: ScriptUtils.getThemeFiles().map(f => f.parsed)
}, "GenerateLayerOverview:")
if (AllKnownLayouts.getSharedLayersConfigs().size == 0) {
console.error("This was a bootstrapping-run. Run generate layeroverview again!")
} else {