Fix duplicate buildings for grb layer; add default flag for filters, performance improvement

Logic/FeatureSource/FeaturePipeline.ts

@@ -204,7 +204,7 @@ export default class FeaturePipeline {
                     TiledFeatureSource.createHierarchy(src, {
                         layer: src.layer,
                         minZoomLevel: this.osmSourceZoomLevel,
-                        dontEnforceMinZoom: true,
+                        noDuplicates: true,
                         registerTile: (tile) => {
                             new RegisteringAllFromFeatureSourceActor(tile, state.allElements)
                             perLayerHierarchy.get(id).registerTile(tile)
@@ -276,7 +276,7 @@ export default class FeaturePipeline {
             (source) => TiledFeatureSource.createHierarchy(source, {
                 layer: source.layer,
                 minZoomLevel: source.layer.layerDef.minzoom,
-                dontEnforceMinZoom: true,
+                noDuplicates: true,
                 maxFeatureCount: state.layoutToUse.clustering.minNeededElements,
                 maxZoomLevel: state.layoutToUse.clustering.maxZoom,
                 registerTile: (tile) => {

Logic/FeatureSource/Sources/GeoJsonSource.ts

@@ -18,19 +18,13 @@ export default class GeoJsonSource implements FeatureSourceForLayer, Tiled {
     public readonly layer: FilteredLayer;
     public readonly tileIndex
     public readonly bbox;
-    private readonly seenids: Set<string> = new Set<string>()
+    private readonly seenids: Set<string>;
-    /**
+    private readonly idKey ?: string;
-     * Only used if the actual source is a tiled geojson.
-     * A big feature might be contained in multiple tiles.
-     * However, we only want to load them once. The blacklist thus contains all ids of all features previously seen
-     * @private
-     */
-    private readonly featureIdBlacklist?: UIEventSource<Set<string>>
 
     public constructor(flayer: FilteredLayer,
                        zxy?: [number, number, number] | BBox,
                        options?: {
-                           featureIdBlacklist?: UIEventSource<Set<string>>
+                           featureIdBlacklist?: Set<string>
                        }) {
 
         if (flayer.layerDef.source.geojsonZoomLevel !== undefined && zxy === undefined) {
@@ -38,7 +32,8 @@ export default class GeoJsonSource implements FeatureSourceForLayer, Tiled {
         }
 
         this.layer = flayer;
-        this.featureIdBlacklist = options?.featureIdBlacklist
+        this.idKey = flayer.layerDef.source.idKey
+        this.seenids = options?.featureIdBlacklist ?? new Set<string>()
         let url = flayer.layerDef.source.geojsonSource.replace("{layer}", flayer.layerDef.id);
         if (zxy !== undefined) {
             let tile_bbox: BBox;
@@ -106,6 +101,10 @@ export default class GeoJsonSource implements FeatureSourceForLayer, Tiled {
                         }
                     }
 
+                    if(self.idKey !== undefined){
+                        props.id = props[self.idKey]
+                    }
+                    
                     if (props.id === undefined) {
                         props.id = url + "/" + i;
                         feature.id = url + "/" + i;
@@ -117,10 +116,6 @@ export default class GeoJsonSource implements FeatureSourceForLayer, Tiled {
                     }
                     self.seenids.add(props.id)
 
-                    if (self.featureIdBlacklist?.data?.has(props.id)) {
-                        continue;
-                    }
-
                     let freshness: Date = time;
                     if (feature.properties["_last_edit:timestamp"] !== undefined) {
                         freshness = new Date(props["_last_edit:timestamp"])

Logic/FeatureSource/Sources/NewGeometryFromChangesFeatureSource.ts

@@ -20,7 +20,7 @@ export class NewGeometryFromChangesFeatureSource implements FeatureSource {
         const features = this.features.data;
         const self = this;
 
-        changes.pendingChanges.addCallbackAndRunD(changes => {
+        changes.pendingChanges.stabilized(100).addCallbackAndRunD(changes => {
             if (changes.length === 0) {
                 return;
             }

Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts

@@ -55,8 +55,7 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource {
             }
         }
 
-        const seenIds = new Set<string>();
+        const blackList = (new Set<string>())
-        const blackList = new UIEventSource(seenIds)
         super(
             layer,
             source.geojsonZoomLevel,
@@ -76,10 +75,7 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource {
                         featureIdBlacklist: blackList
                     }
                 )
-                src.features.addCallbackAndRunD(feats => {
+     
-                    feats.forEach(feat => seenIds.add(feat.feature.properties.id))
-                    blackList.ping();
-                })
                 registerLayer(src)
                 return src
             },

Logic/FeatureSource/TiledFeatureSource/TileHierarchyMerger.ts

@@ -21,7 +21,6 @@ export class TileHierarchyMerger implements TileHierarchy<FeatureSourceForLayer
      * Add another feature source for the given tile.
      * Entries for this tile will be merged
      * @param src
-     * @param index
      */
     public registerTile(src: FeatureSource & Tiled) {
 

Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource.ts

@@ -146,7 +146,10 @@ export default class TiledFeatureSource implements Tiled, IndexedFeatureSource,
         for (const feature of features) {
             const bbox = BBox.get(feature.feature)
 
-            if (this.options.dontEnforceMinZoom) {
+            // There are a few strategies to deal with features that cross tile boundaries
+            
+            if (this.options.noDuplicates) {
+                // Strategy 1: We put the feature into a somewhat matching tile
                 if (bbox.overlapsWith(this.upper_left.bbox)) {
                     ulf.push(feature)
                 } else if (bbox.overlapsWith(this.upper_right.bbox)) {
@@ -159,6 +162,7 @@ export default class TiledFeatureSource implements Tiled, IndexedFeatureSource,
                     overlapsboundary.push(feature)
                 }
             } else if (this.options.minZoomLevel === undefined) {
+                // Strategy 2: put it into a strictly matching tile (or in this tile, which is slightly too big)
                 if (bbox.isContainedIn(this.upper_left.bbox)) {
                     ulf.push(feature)
                 } else if (bbox.isContainedIn(this.upper_right.bbox)) {
@@ -171,7 +175,7 @@ export default class TiledFeatureSource implements Tiled, IndexedFeatureSource,
                     overlapsboundary.push(feature)
                 }
             } else {
-                // We duplicate a feature on a boundary into every tile as we need to get to the minZoomLevel
+                // Strategy 3: We duplicate a feature on a boundary into every tile as we need to get to the minZoomLevel
                 if (bbox.overlapsWith(this.upper_left.bbox)) {
                     ulf.push(feature)
                 }
@@ -201,10 +205,9 @@ export interface TiledFeatureSourceOptions {
     readonly minZoomLevel?: number,
     /**
      * IF minZoomLevel is set, and if a feature runs through a tile boundary, it would normally be duplicated.
-     * Setting 'dontEnforceMinZoomLevel' will still allow bigger zoom levels for those features.
+     * Setting 'dontEnforceMinZoomLevel' will assign to feature to some matching subtile.
-     * If 'pick_first' is set, the feature will not be duplicated but set to some tile
      */
-    readonly dontEnforceMinZoom?: boolean | "pick_first",
+    readonly noDuplicates?: boolean,
     readonly registerTile?: (tile: TiledFeatureSource & FeatureSourceForLayer & Tiled) => void,
     readonly layer?: FilteredLayer
 }

Models/ThemeConfig/FilterConfig.ts

@@ -18,6 +18,7 @@ export default class FilterConfig {
         originalTagsSpec: string | AndOrTagConfigJson
         fields: { name: string, type: string }[]
     }[];
+    public readonly defaultSelection : number
 
     constructor(json: FilterConfigJson, context: string) {
         if (json.options === undefined) {
@@ -35,6 +36,7 @@ export default class FilterConfig {
             throw `A filter was given where the options aren't a list at ${context}`
         }
         this.id = json.id;
+        let defaultSelection : number = undefined
         this.options = json.options.map((option, i) => {
             const ctx = `${context}.options[${i}]`;
             const question = Translations.T(
@@ -66,9 +68,18 @@ export default class FilterConfig {
                 }
             })
 
+            if(option.default){
+                if(defaultSelection === undefined){
+                    defaultSelection = i;
+                }else{
+                    throw `Invalid filter: multiple filters are set as default, namely ${i} and ${defaultSelection} at ${context}`
+                }
+            }
 
             return {question: question, osmTags: osmTags, fields, originalTagsSpec: option.osmTags};
         });
+        
+        this.defaultSelection = defaultSelection ?? 0
 
         if (this.options.some(o => o.fields.length > 0) && this.options.length > 1) {
             throw `Invalid filter at ${context}: a filter with textfields should only offer a single option.`
@@ -77,6 +88,8 @@ export default class FilterConfig {
         if (this.options.length > 1 && this.options[0].osmTags !== undefined) {
             throw "Error in " + context + "." + this.id + ": the first option of a multi-filter should always be the 'reset' option and not have any filters"
         }
+        
+        
     }
 
     public initState(): UIEventSource<FilterState> {
@@ -88,7 +101,14 @@ export default class FilterConfig {
             return "" + state.state
         }
 
-        const defaultValue = this.options.length > 1 ? "0" : ""
+        let defaultValue = ""
+        if(this.options.length > 1){
+            defaultValue = ""+this.defaultSelection 
+        }else{
+            if(this.defaultSelection > 0){
+                defaultValue = ""+this.defaultSelection
+            }
+        }
         const qp = QueryParameters.GetQueryParameter("filter-" + this.id, defaultValue, "State of filter " + this.id)
 
         if (this.options.length > 1) {

Models/ThemeConfig/Json/FilterConfigJson.ts

@@ -14,6 +14,7 @@ export default interface FilterConfigJson {
     options: {
         question: string | any;
         osmTags?: AndOrTagConfigJson | string,
+        default?: boolean,
         fields?: {
             name: string,
             type?: string | "string"

Models/ThemeConfig/Json/LayerConfigJson.ts

@@ -33,44 +33,65 @@ export interface LayerConfigJson {
 
 
     /**
-     * This determines where the data for the layer is fetched.
+     * This determines where the data for the layer is fetched: from OSM or from an external geojson dataset.
-     * There are some options:
      *
-     * # Query OSM directly
+     * If no 'geojson' is defined, data will be fetched from overpass and the OSM-API.
-     * source: {osmTags: "key=value"}
-     *  will fetch all objects with given tags from OSM.
-     *  Currently, this will create a query to overpass and fetch the data - in the future this might fetch from the OSM API
      *
-     * # Query OSM Via the overpass API with a custom script
+     * Every source _must_ define which tags _must_ be present in order to be picked up.
-     * source: {overpassScript: "<custom overpass tags>"} when you want to do special things. _This should be really rare_.
-     *      This means that the data will be pulled from overpass with this script, and will ignore the osmTags for the query
-     *      However, for the rest of the pipeline, the OsmTags will _still_ be used. This is important to enable layers etc...
      *
-     *
-     * # A single geojson-file
-     * source: {geoJson: "https://my.source.net/some-geo-data.geojson"}
-     *  fetches a geojson from a third party source
-     *
-     * # A tiled geojson source
-     * source: {geoJson: "https://my.source.net/some-tile-geojson-{layer}-{z}-{x}-{y}.geojson", geoJsonZoomLevel: 14}
-     *  to use a tiled geojson source. The web server must offer multiple geojsons. {z}, {x} and {y} are substituted by the location; {layer} is substituted with the id of the loaded layer
-     *
-     * Some API's use a BBOX instead of a tile, this can be used by specifying {y_min}, {y_max}, {x_min} and {x_max}
-     * Some API's use a mercator-projection (EPSG:900913) instead of WGS84. Set the flag `mercatorCrs: true`  in the source for this
-     *
-     * Note that both geojson-options might set a flag 'isOsmCache' indicating that the data originally comes from OSM too
-     *
-     *
-     * NOTE: the previous format was 'overpassTags: AndOrTagConfigJson | string', which is interpreted as a shorthand for source: {osmTags: "key=value"}
-     *  While still supported, this is considered deprecated
      */
-    source: ({ osmTags: AndOrTagConfigJson | string, overpassScript?: string } |
+    source: 
-        { osmTags: AndOrTagConfigJson | string, geoJson: string, geoJsonZoomLevel?: number, isOsmCache?: boolean, mercatorCrs?: boolean }) & ({
+        ({
-        /**
+            /**
-         * The maximum amount of seconds that a tile is allowed to linger in the cache
+             * Every source must set which tags have to be present in order to load the given layer.
-         */
+             */
-        maxCacheAge?: number
+            osmTags: AndOrTagConfigJson | string
-    })
+            /**
+             * The maximum amount of seconds that a tile is allowed to linger in the cache
+             */
+            maxCacheAge?: number
+        }) &
+        ({      /* # Query OSM Via the overpass API with a custom script
+            * source: {overpassScript: "<custom overpass tags>"} when you want to do special things. _This should be really rare_.
+            *      This means that the data will be pulled from overpass with this script, and will ignore the osmTags for the query
+            *      However, for the rest of the pipeline, the OsmTags will _still_ be used. This is important to enable layers etc...
+            */
+            overpassScript?: string
+        } |
+        {
+            /**
+             * The actual source of the data to load, if loaded via geojson.
+             *
+             * # A single geojson-file
+             * source: {geoJson: "https://my.source.net/some-geo-data.geojson"}
+             *  fetches a geojson from a third party source
+             *
+             * # A tiled geojson source
+             * source: {geoJson: "https://my.source.net/some-tile-geojson-{layer}-{z}-{x}-{y}.geojson", geoJsonZoomLevel: 14}
+             *  to use a tiled geojson source. The web server must offer multiple geojsons. {z}, {x} and {y} are substituted by the location; {layer} is substituted with the id of the loaded layer
+             *
+             * Some API's use a BBOX instead of a tile, this can be used by specifying {y_min}, {y_max}, {x_min} and {x_max}
+             */
+            geoJson: string,
+            /**
+             * To load a tiled geojson layer, set the zoomlevel of the tiles
+             */
+            geoJsonZoomLevel?: number,
+            /**
+             * Indicates that the upstream geojson data is OSM-derived.
+             * Useful for e.g. merging or for scripts generating this cache
+             */
+            isOsmCache?: boolean,
+            /**
+             * Some API's use a mercator-projection (EPSG:900913) instead of WGS84. Set the flag `mercatorCrs: true`  in the source for this
+             */
+            mercatorCrs?: boolean,
+            /**
+             * Some API's have an id-field, but give it a different name.
+             * Setting this key will rename this field into 'id'
+             */
+            idKey?: string
+        })
 
     /**
      *
@@ -113,7 +134,7 @@ export interface LayerConfigJson {
 
     /**
      * Advanced option - might be set by the theme compiler
-     * 
+     *
      * If true, this data will _always_ be loaded, even if the theme is disabled
      */
     forceLoad?: false | boolean
@@ -148,7 +169,7 @@ export interface LayerConfigJson {
      * If not specified, the OsmLink and wikipedia links will be used by default.
      * Use an empty array to hide them.
      * Note that "defaults" will insert all the default titleIcons (which are added automatically)
-     * 
+     *
      * Type: icon[]
      */
     titleIcons?: (string | TagRenderingConfigJson)[] | ["defaults"];
@@ -194,7 +215,7 @@ export interface LayerConfigJson {
 
         /**
          * Example images, which show real-life pictures of what such a feature might look like
-         * 
+         *
          * Type: image
          */
         exampleImages?: string[]
@@ -251,7 +272,7 @@ export interface LayerConfigJson {
     /**
      * All the extra questions for filtering
      */
-    filter?: (FilterConfigJson) [] | {sameAs: string},
+    filter?: (FilterConfigJson) [] | { sameAs: string },
 
     /**
      * This block defines under what circumstances the delete dialog is shown for objects of this layer.

Models/ThemeConfig/LayerConfig.ts

@@ -15,7 +15,6 @@ import LineRenderingConfig from "./LineRenderingConfig";
 import PointRenderingConfigJson from "./Json/PointRenderingConfigJson";
 import LineRenderingConfigJson from "./Json/LineRenderingConfigJson";
 import {TagRenderingConfigJson} from "./Json/TagRenderingConfigJson";
-import {UIEventSource} from "../../Logic/UIEventSource";
 import BaseUIElement from "../../UI/BaseUIElement";
 import Combine from "../../UI/Base/Combine";
 import Title from "../../UI/Base/Title";
@@ -108,11 +107,13 @@ export default class LayerConfig extends WithContextLoader {
         this.source = new SourceConfig(
             {
                 osmTags: osmTags,
-                geojsonSource: json.source["geoJson"],
+                            geojsonSource: json.source["geoJson"],
-                geojsonSourceLevel: json.source["geoJsonZoomLevel"],
+               geojsonSourceLevel: json.source["geoJsonZoomLevel"],
                 overpassScript: json.source["overpassScript"],
                 isOsmCache: json.source["isOsmCache"],
-                mercatorCrs: json.source["mercatorCrs"]
+                mercatorCrs: json.source["mercatorCrs"],
+                idKey: json.source["idKey"]
+
             },
             json.id
         );
@@ -236,7 +237,7 @@ export default class LayerConfig extends WithContextLoader {
                 console.log(json.mapRendering)
                 throw("The layer " + this.id + " does not have any maprenderings defined and will thus not show up on the map at all. If this is intentional, set maprenderings to 'null' instead of '[]'")
             } else if (!hasCenterRendering && this.lineRendering.length === 0 && !this.source.geojsonSource?.startsWith("https://api.openstreetmap.org/api/0.6/notes.json")) {
-                throw "The layer " + this.id + " might not render ways. This might result in dropped information"
+                throw "The layer " + this.id + " might not render ways. This might result in dropped information (at "+context+")"
             }
         }
 

Models/ThemeConfig/SourceConfig.ts

@@ -9,6 +9,7 @@ export default class SourceConfig {
     public geojsonZoomLevel?: number;
     public isOsmCacheLayer: boolean;
     public readonly mercatorCrs: boolean;
+    public readonly idKey : string
 
     constructor(params: {
         mercatorCrs?: boolean;
@@ -17,6 +18,7 @@ export default class SourceConfig {
         geojsonSource?: string,
         isOsmCache?: boolean,
         geojsonSourceLevel?: number,
+        idKey?: string
     }, context?: string) {
 
         let defined = 0;
@@ -47,5 +49,6 @@ export default class SourceConfig {
         this.geojsonZoomLevel = params.geojsonSourceLevel;
         this.isOsmCacheLayer = params.isOsmCache ?? false;
         this.mercatorCrs = params.mercatorCrs ?? false;
+        this.idKey= params.idKey 
     }
 }

UI/Popup/AutoApplyButton.ts

@@ -58,39 +58,9 @@ class ApplyButton extends UIElement {
         this.text = options.text
         this.icon = options.icon
         this.layer = this.state.filteredLayers.data.find(l => l.layerDef.id === this.target_layer_id)
-        this. tagRenderingConfig = this.layer.layerDef.tagRenderings.find(tr => tr.id === this.targetTagRendering)
+        this.tagRenderingConfig = this.layer.layerDef.tagRenderings.find(tr => tr.id === this.targetTagRendering)
 
     }
-    
-    private async Run() {
-            this.buttonState.setData("running")
-            try {
-                console.log("Applying auto-action on " + this.target_feature_ids.length + " features")
-
-                for (const targetFeatureId of this.target_feature_ids) {
-                    const featureTags = this.state.allElements.getEventSourceById(targetFeatureId)
-                    const rendering = this.tagRenderingConfig.GetRenderValue(featureTags.data).txt
-                    const specialRenderings = Utils.NoNull(SubstitutedTranslation.ExtractSpecialComponents(rendering)
-                        .map(x => x.special))
-                        .filter(v => v.func["supportsAutoAction"] === true)
-
-                    if(specialRenderings.length == 0){
-                        console.warn("AutoApply: feature "+targetFeatureId+" got a rendering without supported auto actions:", rendering)
-                    }
-                    
-                    for (const specialRendering of specialRenderings) {
-                        const action = <AutoAction>specialRendering.func
-                        await action.applyActionOn(this.state, featureTags, specialRendering.args)
-                    }
-                }
-                console.log("Flushing changes...")
-                await this.state.changes.flushChanges("Auto button")
-                this.buttonState.setData("done")
-            } catch (e) {
-                console.error("Error while running autoApply: ", e)
-             this.   buttonState.setData({error: e})
-            }
-    }
 
     protected InnerRender(): string | BaseUIElement {
         if (this.target_feature_ids.length === 0) {
@@ -105,7 +75,13 @@ class ApplyButton extends UIElement {
         const button = new SubtleButton(
             new Img(this.icon),
             this.text
-        ).onClick(() => self.Run());
+        ).onClick(() => {
+            this.buttonState.setData("running")
+            window.setTimeout(() => {
+
+            self.Run();
+            }, 50)
+        });
 
         const explanation = new Combine(["The following objects will be updated: ",
             ...this.target_feature_ids.map(id => new Combine([new Link(id, "https:/  /openstreetmap.org/" + id, true), ", "]))]).SetClass("subtle")
@@ -124,7 +100,7 @@ class ApplyButton extends UIElement {
             zoomToFeatures: true,
             features: new StaticFeatureSource(features, false),
             state: this.state,
-            layerToShow:this. layer.layerDef,
+            layerToShow: this.layer.layerDef,
         })
 
 
@@ -145,6 +121,37 @@ class ApplyButton extends UIElement {
         ))
     }
 
+    private async Run() {
+
+        
+        try {
+            console.log("Applying auto-action on " + this.target_feature_ids.length + " features")
+
+            for (const targetFeatureId of this.target_feature_ids) {
+                const featureTags = this.state.allElements.getEventSourceById(targetFeatureId)
+                const rendering = this.tagRenderingConfig.GetRenderValue(featureTags.data).txt
+                const specialRenderings = Utils.NoNull(SubstitutedTranslation.ExtractSpecialComponents(rendering)
+                    .map(x => x.special))
+                    .filter(v => v.func["supportsAutoAction"] === true)
+
+                if (specialRenderings.length == 0) {
+                    console.warn("AutoApply: feature " + targetFeatureId + " got a rendering without supported auto actions:", rendering)
+                }
+
+                for (const specialRendering of specialRenderings) {
+                    const action = <AutoAction>specialRendering.func
+                    await action.applyActionOn(this.state, featureTags, specialRendering.args)
+                }
+            }
+            console.log("Flushing changes...")
+            await this.state.changes.flushChanges("Auto button")
+            this.buttonState.setData("done")
+        } catch (e) {
+            console.error("Error while running autoApply: ", e)
+            this.buttonState.setData({error: e})
+        }
+    }
+
 }
 
 export default class AutoApplyButton implements SpecialVisualization {
@@ -215,15 +222,13 @@ export default class AutoApplyButton implements SpecialVisualization {
 
                 const loading = new Loading("Gathering which elements support auto-apply... ");
                 return new VariableUiElement(to_parse.map(ids => {
-                    if(ids === undefined){
+                    if (ids === undefined) {
                         return loading
                     }
 
                     return new ApplyButton(state, JSON.parse(ids), options);
                 }))
             })
-            
-
 
 
         } catch (e) {

assets/themes/grb_import/grb.json

@@ -369,7 +369,8 @@
                       ]
                     }
                   ]
-                }
+                },
+                "default": true
               }
             ]
           }
@@ -441,7 +442,8 @@
         "geoJson": "https://betadata.grbosm.site/grb?bbox={x_min},{y_min},{x_max},{y_max}",
         "geoJsonZoomLevel": 18,
         "mercatorCrs": true,
-        "maxCacheAge": 0
+        "maxCacheAge": 0,
+        "idKey": "osm_id"
       },
       "name": "GRB geometries",
       "title": "GRB outline",