Use IndexedDb to store cached geodata, fix #494. This should prevent crashes

Logic/Actors/OverpassFeatureSource.ts

@@ -10,6 +10,7 @@ import RelationsTracker from "../Osm/RelationsTracker";
 import {BBox} from "../BBox";
 import Loc from "../../Models/Loc";
 import LayerConfig from "../../Models/ThemeConfig/LayerConfig";
+import AllKnownLayers from "../../Customizations/AllKnownLayers";
 
 
 export default class OverpassFeatureSource implements FeatureSource {
@@ -121,6 +122,9 @@ export default class OverpassFeatureSource implements FeatureSource {
             if (typeof (layer) === "string") {
                 throw "A layer was not expanded!"
             }
+            if(AllKnownLayers.priviliged_layers.indexOf(layer.id) >= 0){
+                continue
+            }
             if (this.state.locationControl.data.zoom < layer.minzoom) {
                 continue;
             }

Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts

@@ -7,28 +7,92 @@ import FeatureSource, {Tiled} from "../FeatureSource";
 import {Tiles} from "../../../Models/TileRange";
 import {IdbLocalStorage} from "../../Web/IdbLocalStorage";
 import {UIEventSource} from "../../UIEventSource";
+import LayerConfig from "../../../Models/ThemeConfig/LayerConfig";
+import {BBox} from "../../BBox";
+import SimpleFeatureSource from "../Sources/SimpleFeatureSource";
+import FilteredLayer from "../../../Models/FilteredLayer";
 
 export default class SaveTileToLocalStorageActor {
     private readonly visitedTiles: UIEventSource<Map<number, Date>>
-    private readonly _layerId: string;
+    private readonly _layer: LayerConfig;
-    static storageKey: string = "";
+    private readonly _flayer : FilteredLayer
+    private readonly initializeTime = new Date()
 
-    constructor(layerId: string) {
+    constructor(layer: FilteredLayer) {
-        this._layerId = layerId;
+        this._flayer = layer
-        this.visitedTiles = IdbLocalStorage.Get("visited_tiles_" + layerId, 
+        this._layer = layer.layerDef
+        this.visitedTiles = IdbLocalStorage.Get("visited_tiles_" + this._layer.id, 
             {defaultValue: new Map<number, Date>(), })
+        this.visitedTiles.stabilized(100).addCallbackAndRunD(tiles => {
+            for (const key of Array.from(tiles.keys())) {
+                const tileFreshness = tiles.get(key)
+
+                const toOld = (this.initializeTime.getTime() -  tileFreshness.getTime()) > 1000 * this._layer.maxAgeOfCache
+                if(toOld){
+                    // Purge this tile
+                    this.SetIdb(key, undefined)
+                    console.debug("Purging tile",this._layer.id,key)         
+                    tiles.delete(key)
+                }
+            }
+            this.visitedTiles.ping()
+            return true;
+        })
+    }
+    
+    public LoadTilesFromDisk(currentBounds: UIEventSource<BBox>, 
+                       registerFreshness: (tileId: number, freshness: Date) => void,
+                       registerTile: ((src: FeatureSource & Tiled ) => void)){
+        const self = this;
+        this.visitedTiles.addCallbackD(tiles => {
+            if(tiles.size === 0){
+                // We don't do anything yet as probably not yet loaded from disk
+                // We'll unregister later on
+                return;
+            }
+            for (const key of Array.from(tiles.keys())) {
+                const tileFreshness = tiles.get(key)
+                if(tileFreshness > self.initializeTime){
+                    // This tile is loaded by another source
+                    continue
+                }
+                registerFreshness(key, tileFreshness)
+                
+                const tileBbox = BBox.fromTileIndex(key)
+                currentBounds.addCallbackAndRunD(bbox => {
+                    if(bbox.overlapsWith(tileBbox)){
+                        // The current tile should be loaded from disk
+                        this.GetIdb(key).then((features:{feature: any, freshness: Date}[] ) => {
+                            console.log("Loaded tile "+self._layer.id+"_"+key+" from disk")
+                            const src = new SimpleFeatureSource(self._flayer, key, new UIEventSource<{feature: any; freshness: Date}[]>(features))
+                            registerTile(src)
+                        })
+                        return true; // only load once: unregister
+                    }
+                })
+                
+            }
+            
+            return true; // Remove the callback
+            
+        })
     }
 
-    public loadAvailableTiles(){
+    private SetIdb(tileIndex, data){
-        this.visitedTiles.addCallbackAndRunD()
+        IdbLocalStorage.SetDirectly(this._layer.id+"_"+tileIndex, data)
     }
 
+    private GetIdb(tileIndex){
+      return IdbLocalStorage.GetDirectly(this._layer.id+"_"+tileIndex)
+    }
+    
     public addTile(tile: FeatureSource & Tiled){
+        const self = this
         tile.features.addCallbackAndRunD(features => {
             const now = new Date()
 
             if (features.length > 0) {
-                IdbLocalStorage.SetDirectly(this._layerId+"_"+tile.tileIndex, features)
+               self.SetIdb(tile.tileIndex, features)
             }
             // We _still_ write the time to know that this tile is empty!
             this.MarkVisited(tile.tileIndex, now)
@@ -46,6 +110,6 @@ export default class SaveTileToLocalStorageActor {
 
     public MarkVisited(tileId: number, freshness: Date) {
         this.visitedTiles.data.set(tileId, freshness)
-        this.visitedTiles.ping()
+        this.visitedTiles.ping()    
     }
 }

Logic/FeatureSource/FeaturePipeline.ts

@@ -11,7 +11,6 @@ import OverpassFeatureSource from "../Actors/OverpassFeatureSource";
 import GeoJsonSource from "./Sources/GeoJsonSource";
 import Loc from "../../Models/Loc";
 import RegisteringAllFromFeatureSourceActor from "./Actors/RegisteringAllFromFeatureSourceActor";
-import TiledFromLocalStorageSource from "./TiledFeatureSource/TiledFromLocalStorageSource";
 import SaveTileToLocalStorageActor from "./Actors/SaveTileToLocalStorageActor";
 import DynamicGeoJsonTileSource from "./TiledFeatureSource/DynamicGeoJsonTileSource";
 import {TileHierarchyMerger} from "./TiledFeatureSource/TileHierarchyMerger";
@@ -66,9 +65,6 @@ export default class FeaturePipeline {
 
         const self = this
         const expiryInSeconds = Math.min(...state.layoutToUse.layers.map(l => l.maxAgeOfCache))
-        for (const layer of state.layoutToUse.layers) {
-            TiledFromLocalStorageSource.cleanCacheForLayer(layer)
-        }
         this.oldestAllowedDate = new Date(new Date().getTime() - expiryInSeconds);
         this.osmSourceZoomLevel = state.osmApiTileSize.data;
         const useOsmApi = state.locationControl.map(l => l.zoom > (state.overpassMaxZoom.data ?? 12))
@@ -153,22 +149,22 @@ export default class FeaturePipeline {
                 continue
             }
             
-            this.localStorageSavers.set(filteredLayer.layerDef.id,  new SaveTileToLocalStorageActor(filteredLayer.layerDef.id))
+            const localTileSaver =  new SaveTileToLocalStorageActor(filteredLayer)
+            this.localStorageSavers.set(filteredLayer.layerDef.id, localTileSaver)
 
             if (source.geojsonSource === undefined) {
                 // This is an OSM layer
                 // We load the cached values and register them
                 // Getting data from upstream happens a bit lower
-                new TiledFromLocalStorageSource(filteredLayer,
+                localTileSaver.LoadTilesFromDisk(
-                    (src) => {
+                    state.currentBounds,
-                        new RegisteringAllFromFeatureSourceActor(src)
+                    (tileIndex, freshness) => self.freshnesses.get(id).addTileLoad(tileIndex, freshness),
-                        hierarchy.registerTile(src);
+                    (tile) => {
-                        src.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(src))
+                        new RegisteringAllFromFeatureSourceActor(tile)
-                    }, state)
+                        hierarchy.registerTile(tile);
-
+                        tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
-                TiledFromLocalStorageSource.GetFreshnesses(id).forEach((value, key) => {
+                    }
-                    self.freshnesses.get(id).addTileLoad(key, value)
+                )
-                })
 
                 continue
             }

Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts

@@ -1,116 +0,0 @@
-import FilteredLayer from "../../../Models/FilteredLayer";
-import {FeatureSourceForLayer, Tiled} from "../FeatureSource";
-import {UIEventSource} from "../../UIEventSource";
-import TileHierarchy from "./TileHierarchy";
-import SaveTileToLocalStorageActor from "../Actors/SaveTileToLocalStorageActor";
-import {BBox} from "../../BBox";
-import LayerConfig from "../../../Models/ThemeConfig/LayerConfig";
-
-export default class TiledFromLocalStorageSource implements TileHierarchy<FeatureSourceForLayer & Tiled> {
-    public readonly loadedTiles: Map<number, FeatureSourceForLayer & Tiled> = new Map<number, FeatureSourceForLayer & Tiled>();
-    private readonly layer: FilteredLayer;
-    private readonly handleFeatureSource: (src: FeatureSourceForLayer & Tiled, index: number) => void;
-    private readonly undefinedTiles: Set<number>;
-
-    constructor(layer: FilteredLayer,
-                handleFeatureSource: (src: FeatureSourceForLayer & Tiled, index: number) => void,
-                state: {
-                    currentBounds: UIEventSource<BBox>
-                }) {
-        this.layer = layer;
-        this.handleFeatureSource = handleFeatureSource;
-
-
-        this.undefinedTiles = new Set<number>()
-        const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-"
-        const knownTiles: number[] = Object.keys(localStorage)
-            .filter(key => {
-                return key.startsWith(prefix) && !key.endsWith("-time") && !key.endsWith("-format");
-            })
-            .map(key => {
-                return Number(key.substring(prefix.length));
-            })
-            .filter(i => !isNaN(i))
-
-        const self = this
-        state.currentBounds.map(bounds => {
-
-            if (bounds === undefined) {
-                return;
-            }
-            for (const knownTile of knownTiles) {
-
-                if (this.loadedTiles.has(knownTile)) {
-                    continue;
-                }
-                if (this.undefinedTiles.has(knownTile)) {
-                    continue;
-                }
-
-                if (!bounds.overlapsWith(BBox.fromTileIndex(knownTile))) {
-                    continue;
-                }
-                self.loadTile(knownTile)
-            }
-        })
-
-    }
-
-    public static GetFreshnesses(layerId: string): Map<number, Date> {
-        const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layerId + "-"
-        const freshnesses = new Map<number, Date>()
-        for (const key of Object.keys(localStorage)) {
-            if (!(key.startsWith(prefix) && key.endsWith("-time"))) {
-                continue
-            }
-            const index = Number(key.substring(prefix.length, key.length - "-time".length))
-            const time = Number(localStorage.getItem(key))
-            const freshness = new Date()
-            freshness.setTime(time)
-            freshnesses.set(index, freshness)
-        }
-        return freshnesses
-    }
-
-    static cleanCacheForLayer(layer: LayerConfig) {
-        const now = new Date()
-        const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.id + "-"
-        for (const key of Object.keys(localStorage)) {
-            if (!(key.startsWith(prefix) && key.endsWith("-time"))) {
-                continue
-            }
-            const index = Number(key.substring(prefix.length, key.length - "-time".length))
-            const time = Number(localStorage.getItem(key))
-            const timeDiff = (now.getTime() - time) / 1000
-
-            if (timeDiff >= layer.maxAgeOfCache) {
-                const k = prefix + index;
-                localStorage.removeItem(k)
-                localStorage.removeItem(k + "-format")
-                localStorage.removeItem(k + "-time")
-            }
-        }
-    }
-
-    private loadTile(neededIndex: number) {
-        try {
-            const key = SaveTileToLocalStorageActor.storageKey + "-" + this.layer.layerDef.id + "-" + neededIndex
-            const data = localStorage.getItem(key)
-            const features = JSON.parse(data)
-            const src = {
-                layer: this.layer,
-                features: new UIEventSource<{ feature: any; freshness: Date }[]>(features),
-                name: "FromLocalStorage(" + key + ")",
-                tileIndex: neededIndex,
-                bbox: BBox.fromTileIndex(neededIndex)
-            }
-            this.handleFeatureSource(src, neededIndex)
-            this.loadedTiles.set(neededIndex, src)
-        } catch (e) {
-            console.error("Could not load data tile from local storage due to", e)
-            this.undefinedTiles.add(neededIndex)
-        }
-    }
-
-
-}

Logic/State/MapState.ts

@@ -218,7 +218,6 @@ export default class MapState extends UserRelatedState {
                 let timeDiff = Number.MAX_VALUE // in seconds
                 const olderLocation = features.data[features.data.length - 2]
                 if (olderLocation !== undefined) {
-                    console.log("Previous location", previousLocation)
                     timeDiff = (new Date(previousLocation.freshness).getTime() - new Date(olderLocation.freshness).getTime()) / 1000
                 }
                 if (d < 20 && timeDiff < 60) {

Logic/UIEventSource.ts

@@ -223,6 +223,7 @@ export class UIEventSource<T> {
         for (const callback of this._callbacks) {
             if (callback(this.data) === true) {
                 // This callback wants to be deleted
+                // Note: it has to return precisely true in order to avoid accidental deletions
                 if (toDelete === undefined) {
                     toDelete = [callback]
                 } else {

Logic/Web/IdbLocalStorage.ts

@@ -8,12 +8,8 @@ export class IdbLocalStorage {
     
     public static Get<T>(key: string, options: { defaultValue?: T }): UIEventSource<T>{
         const src = new UIEventSource<T>(options.defaultValue, "idb-local-storage:"+key)
-        idb.get(key).then(v => {
+        idb.get(key).then(v => src.setData(v ?? options.defaultValue))
-            src.setData(v ?? options.defaultValue)
+        src.addCallback(v => idb.set(key, v))
-        })
-        src.stabilized(1000).addCallback(v => {
-            idb.set(key, v)
-        })
         return src;
         
     }
@@ -22,4 +18,7 @@ export class IdbLocalStorage {
         idb.set(key, value)
     }
 
+    static GetDirectly(key: string) {
+        return idb.get(key)
+    }
 }