Fix regression and add tests, add overpass link in layer documentation

Logic/Osm/Overpass.ts

@@ -4,6 +4,8 @@ import {Utils} from "../../Utils";
 import {UIEventSource} from "../UIEventSource";
 import {BBox} from "../BBox";
 import * as osmtogeojson from "osmtogeojson";
+// @ts-ignore
+import {Tag} from "../Tags/Tag"; // used in doctest
 
 /**
  * Interfaces overpass to get all the latest data
@@ -16,21 +18,19 @@ export class Overpass {
     private _includeMeta: boolean;
     private _relationTracker: RelationsTracker;
 
-
     constructor(filter: TagsFilter,
                 extraScripts: string[],
                 interpreterUrl: string,
-                timeout: UIEventSource<number>,
+                timeout?: UIEventSource<number>,
-                relationTracker: RelationsTracker,
+                relationTracker?: RelationsTracker,
                 includeMeta = true) {
-        this._timeout = timeout;
+        this._timeout = timeout ?? new UIEventSource<number>(90);
         this._interpreterUrl = interpreterUrl;
         const optimized = filter.optimize()
         if(optimized === true || optimized === false){
             throw "Invalid filter: optimizes to true of false"
         }
         this._filter = optimized
-        console.log("Overpass filter is",this._filter)
         this._extraScripts = extraScripts;
         this._includeMeta = includeMeta;
         this._relationTracker = relationTracker
@@ -51,23 +51,45 @@ export class Overpass {
             console.warn("No features for", json)
         }
 
-        self._relationTracker.RegisterRelations(json)
+        self._relationTracker?.RegisterRelations(json)
         const geojson = osmtogeojson.default(json);
         const osmTime = new Date(json.osm3s.timestamp_osm_base);
         return [geojson, osmTime];
     }
 
-    buildQuery(bbox: string): string {
+    /**
+     * new Overpass(new Tag("key","value"), [], "").buildScript("{{bbox}}") // => `[out:json][timeout:90]{{bbox}};(nwr["key"="value"];);out body;out meta;>;out skel qt;`
+     */
+    public buildScript(bbox: string, postCall: string = "", pretty = false): string {
         const filters = this._filter.asOverpass()
         let filter = ""
         for (const filterOr of filters) {
-            filter += 'nwr' + filterOr + ';'
+            if(pretty){
+                filter += "    "
+            }
+            filter += 'nwr' + filterOr + postCall + ';'
+            if(pretty){
+                filter+="\n"
+            }
         }
         for (const extraScript of this._extraScripts) {
             filter += '(' + extraScript + ');';
         }
-        const query =
+        return`[out:json][timeout:${this._timeout.data}]${bbox};(${filter});out body;${this._includeMeta ? 'out meta;' : ''}>;out skel qt;`
-            `[out:json][timeout:${this._timeout.data}]${bbox};(${filter});out body;${this._includeMeta ? 'out meta;' : ''}>;out skel qt;`
+    }
+    
+    public buildQuery(bbox: string): string {
+        const query = this.buildScript(bbox)
         return `${this._interpreterUrl}?data=${encodeURIComponent(query)}`
     }
+
+    /**
+     * Little helper method to quickly open overpass-turbo in the browser
+     */
+    public static AsOverpassTurboLink(tags: TagsFilter){
+        const overpass = new Overpass(tags, [], "", undefined, undefined, false)
+        const script = overpass.buildScript("","({{bbox}})", true)
+        const url = "http://overpass-turbo.eu/?Q="
+        return url + encodeURIComponent(script)
+    }
 }

Logic/Tags/RegexTag.ts

@@ -47,6 +47,9 @@ export class RegexTag extends TagsFilter {
      * 
      * // A wildcard regextag should only give the key
      * new RegexTag("a", /^..*$/).asOverpass() // => [ `["a"]` ]
+     * 
+     * // A regextag with a regex key should give correct output
+     * new RegexTag(/a.*x/, /^..*$/).asOverpass() // => [ `[~"a.*x"~\"^..*$\"]` ]
      */
     asOverpass(): string[] {
         const inv =this.invert ? "!" : ""

Logic/Tags/TagUtils.ts

@@ -181,6 +181,7 @@ export class TagUtils {
      * TagUtils.Tag("survey:date:={_date:now}") // => new SubstitutingTag("survey:date", "{_date:now}")
      * TagUtils.Tag("xyz!~\\[\\]") // => new RegexTag("xyz", /^\[\]$/, true)
      * TagUtils.Tag("tags~(^|.*;)amenity=public_bookcase($|;.*)") // => new RegexTag("tags", /(^|.*;)amenity=public_bookcase($|;.*)/)
+     * TagUtils.Tag("service:bicycle:.*~~*") // => new RegexTag(/^service:bicycle:.*$/, /^..*$/)
      */
     public static Tag(json: AndOrTagConfigJson | string, context: string = ""): TagsFilter {
         try {
@@ -219,126 +220,125 @@ export class TagUtils {
         if (json === undefined) {
             throw `Error while parsing a tag: 'json' is undefined in ${context}. Make sure all the tags are defined and at least one tag is present in a complex expression`
         }
-        if (typeof (json) == "string") {
+        if (typeof (json) != "string") {
-            const tag = json as string;
+            if (json.and !== undefined && json.or !== undefined) {
+                throw `Error while parsing a TagConfig: got an object where both 'and' and 'or' are defined`
+            }
+            if (json.and !== undefined) {
+                return new And(json.and.map(t => TagUtils.Tag(t, context)));
+            }
+            if (json.or !== undefined) {
+                return new Or(json.or.map(t => TagUtils.Tag(t, context)));
+            }
+            throw "At " + context + ": unrecognized tag"
+        }
+        
+        
+        const tag = json as string;
+        for (const [operator, comparator] of TagUtils.comparators) {
+            if (tag.indexOf(operator) >= 0) {
+                const split = Utils.SplitFirst(tag, operator);
 
-            for (const [operator, comparator] of TagUtils.comparators) {
+                let val = Number(split[1].trim())
-                if (tag.indexOf(operator) >= 0) {
+                if (isNaN(val)) {
-                    const split = Utils.SplitFirst(tag, operator);
+                    val = new Date(split[1].trim()).getTime()
+                }
 
-                    let val = Number(split[1].trim())
+                const f = (value: string | undefined) => {
-                    if (isNaN(val)) {
+                    if (value === undefined) {
-                        val = new Date(split[1].trim()).getTime()
+                        return false;
                     }
-
+                    let b = Number(value?.trim())
-                    const f = (value: string | undefined) => {
+                    if (isNaN(b)) {
-                        if (value === undefined) {
+                        b = Utils.ParseDate(value).getTime()
-                            return false;
-                        }
-                        let b = Number(value?.trim())
                         if (isNaN(b)) {
-                            b = Utils.ParseDate(value).getTime()
+                            return false
-                            if (isNaN(b)) {
-                                return false
-                            }
                         }
-                        return comparator(b, val)
                     }
-                    return new ComparingTag(split[0], f, operator + val)
+                    return comparator(b, val)
                 }
+                return new ComparingTag(split[0], f, operator + val)
             }
-
-            if (tag.indexOf("!~") >= 0) {
-                const split = Utils.SplitFirst(tag, "!~");
-                if (split[1] === "*") {
-                    throw `Don't use 'key!~*' - use 'key=' instead (empty string as value (in the tag ${tag} while parsing ${context})`
-                }
-                return new RegexTag(
-                    split[0],
-                    split[1],
-                    true
-                );
-            }
-            if (tag.indexOf("~~") >= 0) {
-                const split = Utils.SplitFirst(tag, "~~");
-                if (split[1] === "*") {
-                    split[1] = "..*"
-                }
-                return new RegexTag(
-                    split[0],
-                    split[1]
-                );
-            }
-            if (tag.indexOf("!:=") >= 0) {
-                const split = Utils.SplitFirst(tag, "!:=");
-                return new SubstitutingTag(split[0], split[1], true);
-            }
-            if (tag.indexOf(":=") >= 0) {
-                const split = Utils.SplitFirst(tag, ":=");
-                return new SubstitutingTag(split[0], split[1]);
-            }
-
-            if (tag.indexOf("!=") >= 0) {
-                const split = Utils.SplitFirst(tag, "!=");
-                if (split[1] === "*") {
-                    throw "At "+context+": invalid tag "+tag+". To indicate a missing tag, use '"+split[0]+"!=' instead"
-                }
-                if(split[1] === "") {
-                    split[1] = "..*"
-                }
-                return new RegexTag(
-                    split[0],
-                    new RegExp("^" + split[1] + "$"),
-                    true
-                );
-            }
-            if (tag.indexOf("!~") >= 0) {
-                const split = Utils.SplitFirst(tag, "!~");
-                if (split[1] === "*") {
-                    split[1] = "..*"
-                }
-                return new RegexTag(
-                    split[0],
-                    split[1],
-                    true
-                );
-            }
-            if (tag.indexOf("~") >= 0) {
-                const split = Utils.SplitFirst(tag, "~");
-                if (split[1] === "") {
-                    throw "Detected a regextag with an empty regex; this is not allowed. Use '" + split[0] + "='instead (at " + context + ")"
-                }
-                if (split[1] === "*") {
-                    split[1] = "..*"
-                }
-                return new RegexTag(
-                    split[0],
-                    split[1]
-                );
-            }
-            if (tag.indexOf("=") >= 0) {
-
-
-                const split = Utils.SplitFirst(tag, "=");
-                if (split[1] == "*") {
-                    throw `Error while parsing tag '${tag}' in ${context}: detected a wildcard on a normal value. Use a regex pattern instead`
-                }
-                return new Tag(split[0], split[1])
-            }
-            throw `Error while parsing tag '${tag}' in ${context}: no key part and value part were found`
-
         }
 
-        if (json.and !== undefined && json.or !== undefined) {
+        if (tag.indexOf("!~") >= 0) {
-            throw `Error while parsing a TagConfig: got an object where both 'and' and 'or' are defined`
+            const split = Utils.SplitFirst(tag, "!~");
+            if (split[1] === "*") {
+                throw `Don't use 'key!~*' - use 'key=' instead (empty string as value (in the tag ${tag} while parsing ${context})`
+            }
+            return new RegexTag(
+                split[0],
+                split[1],
+                true
+            );
+        }
+        if (tag.indexOf("~~") >= 0) {
+            const split = Utils.SplitFirst(tag, "~~");
+            if (split[1] === "*") {
+                split[1] = "..*"
+            }
+            return new RegexTag(
+                new RegExp("^"+split[0]+"$"),
+                new RegExp("^"+ split[1]+"$")
+            );
+        }
+        if (tag.indexOf("!:=") >= 0) {
+            const split = Utils.SplitFirst(tag, "!:=");
+            return new SubstitutingTag(split[0], split[1], true);
+        }
+        if (tag.indexOf(":=") >= 0) {
+            const split = Utils.SplitFirst(tag, ":=");
+            return new SubstitutingTag(split[0], split[1]);
         }
 
-        if (json.and !== undefined) {
+        if (tag.indexOf("!=") >= 0) {
-            return new And(json.and.map(t => TagUtils.Tag(t, context)));
+            const split = Utils.SplitFirst(tag, "!=");
+            if (split[1] === "*") {
+                throw "At " + context + ": invalid tag " + tag + ". To indicate a missing tag, use '" + split[0] + "!=' instead"
+            }
+            if (split[1] === "") {
+                split[1] = "..*"
+            }
+            return new RegexTag(
+                split[0],
+                new RegExp("^" + split[1] + "$"),
+                true
+            );
         }
-        if (json.or !== undefined) {
+        if (tag.indexOf("!~") >= 0) {
-            return new Or(json.or.map(t => TagUtils.Tag(t, context)));
+            const split = Utils.SplitFirst(tag, "!~");
+            if (split[1] === "*") {
+                split[1] = "..*"
+            }
+            return new RegexTag(
+                split[0],
+                split[1],
+                true
+            );
         }
+        if (tag.indexOf("~") >= 0) {
+            const split = Utils.SplitFirst(tag, "~");
+            if (split[1] === "") {
+                throw "Detected a regextag with an empty regex; this is not allowed. Use '" + split[0] + "='instead (at " + context + ")"
+            }
+            if (split[1] === "*") {
+                split[1] = "..*"
+            }
+            return new RegexTag(
+                split[0],
+                split[1]
+            );
+        }
+        if (tag.indexOf("=") >= 0) {
+
+
+            const split = Utils.SplitFirst(tag, "=");
+            if (split[1] == "*") {
+                throw `Error while parsing tag '${tag}' in ${context}: detected a wildcard on a normal value. Use a regex pattern instead`
+            }
+            return new Tag(split[0], split[1])
+        }
+        throw `Error while parsing tag '${tag}' in ${context}: no key part and value part were found`
     }
 
     private static GetCount(key: string, value?: string) {

Models/ThemeConfig/Conversion/PrepareTheme.ts

@@ -417,11 +417,46 @@ class PreparePersonalTheme extends DesugaringStep<LayoutConfigJson> {
     
 }
 
+class WarnForUnsubstitutedLayersInTheme extends DesugaringStep<LayoutConfigJson>{
+    
+    constructor() {
+        super("Generates a warning if a theme uses an unsubstituted layer", ["layers"],"WarnForUnsubstitutedLayersInTheme");
+    }
+    
+    convert(json: LayoutConfigJson, context: string): { result: LayoutConfigJson; errors?: string[]; warnings?: string[]; information?: string[] } {
+        if(json.hideFromOverview === true){
+            return {result: json}
+        }
+        const warnings = []
+        for (const layer of json.layers) {
+            if(typeof layer === "string"){
+                continue
+            }
+            if(layer["builtin"] !== undefined){
+                continue
+            }
+            if(layer["source"]["geojson"] !== undefined){
+                // We turn a blind eye for import layers
+                continue
+            }
+            
+            const wrn = "The theme "+json.id+" has an inline layer: "+layer["id"]+". This is discouraged."
+            warnings.push(wrn)
+        }
+        return {
+            result: json,
+            warnings
+        };
+    }
+    
+}
+
 export class PrepareTheme extends Fuse<LayoutConfigJson> {
     constructor(state: DesugaringContext) {
         super(
             "Fully prepares and expands a theme",
             new PreparePersonalTheme(state),
+            // new WarnForUnsubstitutedLayersInTheme(),
             new OnEveryConcat("layers", new SubstituteLayer(state)),
             new SetDefault("socialImage", "assets/SocialImage.png", true),
             // We expand all tagrenderings first...

Models/ThemeConfig/LayerConfig.ts

@@ -24,6 +24,9 @@ import {Utils} from "../../Utils";
 import {TagsFilter} from "../../Logic/Tags/TagsFilter";
 import Table from "../../UI/Base/Table";
 import FilterConfigJson from "./Json/FilterConfigJson";
+import {And} from "../../Logic/Tags/And";
+import {Overpass} from "../../Logic/Osm/Overpass";
+import Constants from "../Constants";
 
 export default class LayerConfig extends WithContextLoader {
 
@@ -60,9 +63,9 @@ export default class LayerConfig extends WithContextLoader {
     public readonly filters: FilterConfig[];
     public readonly filterIsSameAs: string;
     public readonly forceLoad: boolean;
-    
+
-    public readonly syncSelection:  "no" | "local" | "theme-only" | "global"
+    public readonly syncSelection: "no" | "local" | "theme-only" | "global"
-    
+
     constructor(
         json: LayerConfigJson,
         context?: string,
@@ -109,8 +112,8 @@ export default class LayerConfig extends WithContextLoader {
         this.source = new SourceConfig(
             {
                 osmTags: osmTags,
-                            geojsonSource: json.source["geoJson"],
+                geojsonSource: json.source["geoJson"],
-               geojsonSourceLevel: json.source["geoJsonZoomLevel"],
+                geojsonSourceLevel: json.source["geoJsonZoomLevel"],
                 overpassScript: json.source["overpassScript"],
                 isOsmCache: json.source["isOsmCache"],
                 mercatorCrs: json.source["mercatorCrs"],
@@ -236,10 +239,9 @@ export default class LayerConfig extends WithContextLoader {
             const hasCenterRendering = this.mapRendering.some(r => r.location.has("centroid") || r.location.has("start") || r.location.has("end"))
 
             if (this.lineRendering.length === 0 && this.mapRendering.length === 0) {
-                console.log(json.mapRendering)
                 throw("The layer " + this.id + " does not have any maprenderings defined and will thus not show up on the map at all. If this is intentional, set maprenderings to 'null' instead of '[]'")
             } else if (!hasCenterRendering && this.lineRendering.length === 0 && !this.source.geojsonSource?.startsWith("https://api.openstreetmap.org/api/0.6/notes.json")) {
-                throw "The layer " + this.id + " might not render ways. This might result in dropped information (at "+context+")"
+                throw "The layer " + this.id + " might not render ways. This might result in dropped information (at " + context + ")"
             }
         }
 
@@ -251,10 +253,10 @@ export default class LayerConfig extends WithContextLoader {
 
         this.tagRenderings = (Utils.NoNull(json.tagRenderings) ?? []).map((tr, i) => new TagRenderingConfig(<TagRenderingConfigJson>tr, this.id + ".tagRenderings[" + i + "]"))
 
-        if(json.filter !== undefined && json.filter !== null && json.filter["sameAs"] !== undefined){
+        if (json.filter !== undefined && json.filter !== null && json.filter["sameAs"] !== undefined) {
             this.filterIsSameAs = json.filter["sameAs"]
             this.filters = []
-        }else{
+        } else {
             this.filters = (<FilterConfigJson[]>json.filter ?? []).map((option, i) => {
                 return new FilterConfig(option, `${context}.filter-[${i}]`)
             });
@@ -316,8 +318,8 @@ export default class LayerConfig extends WithContextLoader {
         }
         return mapRendering.GetBaseIcon(this.GetBaseTags())
     }
-    
+
-    public GetBaseTags(): any{
+    public GetBaseTags(): any {
         return TagUtils.changeAsProperties(this.source.osmTags.asChange({id: "node/-1"}))
     }
 
@@ -367,7 +369,7 @@ export default class LayerConfig extends WithContextLoader {
             extraProps.push(new Combine(["This layer will automatically load ", new Link(dep.neededLayer, "./" + dep.neededLayer + ".md"), " into the layout as it depends on it: ", dep.reason, "(" + dep.context + ")"]))
         }
 
-        for (const revDep of Utils.Dedup( layerIsNeededBy?.get(this.id) ?? [])) {
+        for (const revDep of Utils.Dedup(layerIsNeededBy?.get(this.id) ?? [])) {
             extraProps.push(new Combine(["This layer is needed as dependency for layer", new Link(revDep, "#" + revDep)]))
         }
 
@@ -402,7 +404,7 @@ export default class LayerConfig extends WithContextLoader {
             ]).SetClass("flex-col flex")
         }
 
-        const icon =  this.mapRendering
+        const icon = this.mapRendering
             .filter(mr => mr.location.has("point"))
             .map(mr => mr.icon?.render?.txt)
             .find(i => i !== undefined)
@@ -412,6 +414,15 @@ export default class LayerConfig extends WithContextLoader {
             iconImg = `<img src='https://mapcomplete.osm.be/${icon}' height="100px"> `
         }
 
+        let overpassLink: BaseUIElement = undefined;
+        if (Constants.priviliged_layers.indexOf(this.id) < 0) {
+            try {
+                overpassLink = new Link("Execute on overpass", Overpass.AsOverpassTurboLink(<TagsFilter> new And(neededTags).optimize()))
+            } catch (e) {
+                console.error("Could not generate overpasslink for " + this.id)
+            }
+        }
+
         return new Combine([
             new Combine([
                 new Title(this.id, 1),
@@ -427,7 +438,7 @@ export default class LayerConfig extends WithContextLoader {
             new Title("Basic tags for this layer", 2),
             "Elements must have the all of following tags to be shown on this layer:",
             new List(neededTags.map(t => t.asHumanString(true, false, {}))),
-
+            overpassLink,
             new Title("Supported attributes", 2),
             quickOverview,
             ...this.tagRenderings.map(tr => tr.GenerateDocumentation())