WIP: use indexedDB as datastore for geotiles

Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts

@@ -3,56 +3,49 @@
  *
  * Technically, more an Actor then a featuresource, but it fits more neatly this ay
  */
-import {FeatureSourceForLayer} from "../FeatureSource";
+import FeatureSource, {Tiled} from "../FeatureSource";
 import {Tiles} from "../../../Models/TileRange";
+import {IdbLocalStorage} from "../../Web/IdbLocalStorage";
+import {UIEventSource} from "../../UIEventSource";
 
 export default class SaveTileToLocalStorageActor {
-    public static readonly storageKey: string = "cached-features";
+    private readonly visitedTiles: UIEventSource<Map<number, Date>>
-    public static readonly formatVersion: string = "2"
+    private readonly _layerId: string;
+    static storageKey: string = "";
 
-    constructor(source: FeatureSourceForLayer, tileIndex: number) {
+    constructor(layerId: string) {
+        this._layerId = layerId;
+        this.visitedTiles = IdbLocalStorage.Get("visited_tiles_" + layerId, 
+            {defaultValue: new Map<number, Date>(), })
+    }
 
-        source.features.addCallbackAndRunD(features => {
+    public loadAvailableTiles(){
-            const key = `${SaveTileToLocalStorageActor.storageKey}-${source.layer.layerDef.id}-${tileIndex}`
+        this.visitedTiles.addCallbackAndRunD()
+    }
+
+    public addTile(tile: FeatureSource & Tiled){
+        tile.features.addCallbackAndRunD(features => {
             const now = new Date()
 
-            try {
             if (features.length > 0) {
-                    localStorage.setItem(key, JSON.stringify(features));
+                IdbLocalStorage.SetDirectly(this._layerId+"_"+tile.tileIndex, features)
             }
             // We _still_ write the time to know that this tile is empty!
-                SaveTileToLocalStorageActor.MarkVisited(source.layer.layerDef.id, tileIndex, now)
+            this.MarkVisited(tile.tileIndex, now)
-            } catch (e) {
-                console.warn("Could not save the features to local storage:", e)
-            }
         })
     }
     
-
+    public poison(lon: number, lat: number) {
-    public static MarkVisited(layerId: string, tileId: number, freshness: Date) {
-        const key = `${SaveTileToLocalStorageActor.storageKey}-${layerId}-${tileId}`
-        try {
-            localStorage.setItem(key + "-time", JSON.stringify(freshness.getTime()))
-            localStorage.setItem(key + "-format", SaveTileToLocalStorageActor.formatVersion)
-        } catch (e) {
-            console.error("Could not mark tile ", key, "as visited")
-        }
-    }
-
-    public static poison(layers: string[], lon: number, lat: number) {
         for (let z = 0; z < 25; z++) {
-
             const {x, y} = Tiles.embedded_tile(lat, lon, z)
             const tileId = Tiles.tile_index(z, x, y)
-
+            this.visitedTiles.data.delete(tileId)
-            for (const layerId of layers) {
-
-                const key = `${SaveTileToLocalStorageActor.storageKey}-${layerId}-${tileId}`
-                localStorage.removeItem(key + "-time");
-                localStorage.removeItem(key + "-format")
-                localStorage.removeItem(key)
-            }
         }
 
     }
+
+    public MarkVisited(tileId: number, freshness: Date) {
+        this.visitedTiles.data.set(tileId, freshness)
+        this.visitedTiles.ping()
+    }
 }

Logic/FeatureSource/FeaturePipeline.ts

@@ -57,6 +57,8 @@ export default class FeaturePipeline {
     private readonly oldestAllowedDate: Date;
     private readonly osmSourceZoomLevel
     
+    private readonly localStorageSavers = new Map<string, SaveTileToLocalStorageActor>()
+
     constructor(
         handleFeatureSource: (source: FeatureSourceForLayer & Tiled) => void,
         state: MapState) {
@@ -77,7 +79,7 @@ export default class FeaturePipeline {
                 .map(ch => ch.changes)
                 .filter(coor => coor["lat"] !== undefined && coor["lon"] !== undefined)
                 .forEach(coor => {
-                    SaveTileToLocalStorageActor.poison(state.layoutToUse.layers.map(l => l.id), coor["lon"], coor["lat"])
+                    state.layoutToUse.layers.forEach(l => self.localStorageSavers.get(l.id).poison(coor["lon"], coor["lat"]))
                 })
         })
 
@@ -151,6 +153,8 @@ export default class FeaturePipeline {
                 continue
             }
             
+            this.localStorageSavers.set(filteredLayer.layerDef.id,  new SaveTileToLocalStorageActor(filteredLayer.layerDef.id))
+
             if (source.geojsonSource === undefined) {
                 // This is an OSM layer
                 // We load the cached values and register them
@@ -210,7 +214,7 @@ export default class FeaturePipeline {
             handleTile: tile => {
                 new RegisteringAllFromFeatureSourceActor(tile)
                 if (tile.layer.layerDef.maxAgeOfCache > 0) {
-                    new SaveTileToLocalStorageActor(tile, tile.tileIndex)
+                    self.localStorageSavers.get(tile.layer.layerDef.id).addTile(tile)
                 }
                 perLayerHierarchy.get(tile.layer.layerDef.id).registerTile(tile)
                 tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
@@ -219,10 +223,11 @@ export default class FeaturePipeline {
             state: state,
             markTileVisited: (tileId) =>
                 state.filteredLayers.data.forEach(flayer => {
-                    if (flayer.layerDef.maxAgeOfCache > 0) {
+                    const layer = flayer.layerDef
-                        SaveTileToLocalStorageActor.MarkVisited(flayer.layerDef.id, tileId, new Date())
+                    if (layer.maxAgeOfCache > 0) {
+                        self.localStorageSavers.get(layer.id).MarkVisited(tileId, new Date())
                     }
-                    self.freshnesses.get(flayer.layerDef.id).addTileLoad(tileId, new Date())
+                    self.freshnesses.get(layer.id).addTileLoad(tileId, new Date())
                 })
         })
 
@@ -252,10 +257,8 @@ export default class FeaturePipeline {
                 maxFeatureCount: state.layoutToUse.clustering.minNeededElements,
                 maxZoomLevel: state.layoutToUse.clustering.maxZoom,
                 registerTile: (tile) => {
-                    // We save the tile data for the given layer to local storage
+                    // We save the tile data for the given layer to local storage - data sourced from overpass
-                    if (source.layer.layerDef.source.geojsonSource === undefined || source.layer.layerDef.source.isOsmCacheLayer == true) {
+                    self.localStorageSavers.get(tile.layer.layerDef.id).addTile(tile)
-                        new SaveTileToLocalStorageActor(tile, tile.tileIndex)
-                    }
                     perLayerHierarchy.get(source.layer.layerDef.id).registerTile(new RememberingSource(tile))
                     tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
 
@@ -417,7 +420,7 @@ export default class FeaturePipeline {
                         const tileIndex = Tiles.tile_index(paddedToZoomLevel, x, y)
                         downloadedLayers.forEach(layer => {
                             self.freshnesses.get(layer.id).addTileLoad(tileIndex, date)
-                            SaveTileToLocalStorageActor.MarkVisited(layer.id, tileIndex, date)
+                            self.localStorageSavers.get(layer.id).MarkVisited(tileIndex, date)
                         })
                     })
 

Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts

@@ -3,7 +3,6 @@ import {FeatureSourceForLayer, Tiled} from "../FeatureSource";
 import {UIEventSource} from "../../UIEventSource";
 import TileHierarchy from "./TileHierarchy";
 import SaveTileToLocalStorageActor from "../Actors/SaveTileToLocalStorageActor";
-import {Tiles} from "../../../Models/TileRange";
 import {BBox} from "../../BBox";
 import LayerConfig from "../../../Models/ThemeConfig/LayerConfig";
 
@@ -33,21 +32,6 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
             })
             .filter(i => !isNaN(i))
 
-        console.debug("Layer", layer.layerDef.id, "has following tiles in available in localstorage", knownTiles.map(i => Tiles.tile_from_index(i).join("/")).join(", "))
-        for (const index of knownTiles) {
-
-            const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-" + index;
-            const version = localStorage.getItem(prefix + "-format")
-            if (version === undefined || version !== SaveTileToLocalStorageActor.formatVersion) {
-                // Invalid version! Remove this tile from local storage
-                localStorage.removeItem(prefix)
-                localStorage.removeItem(prefix + "-time")
-                localStorage.removeItem(prefix + "-format")
-                this.undefinedTiles.add(index)
-                console.log("Dropped old format tile", prefix)
-            }
-        }
-
         const self = this
         state.currentBounds.map(bounds => {
 
@@ -91,7 +75,6 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
     static cleanCacheForLayer(layer: LayerConfig) {
         const now = new Date()
         const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.id + "-"
-        console.log("Cleaning tiles of ", prefix, "with max age", layer.maxAgeOfCache)
         for (const key of Object.keys(localStorage)) {
             if (!(key.startsWith(prefix) && key.endsWith("-time"))) {
                 continue

Logic/State/MapState.ts

@@ -218,7 +218,8 @@ export default class MapState extends UserRelatedState {
                 let timeDiff = Number.MAX_VALUE // in seconds
                 const olderLocation = features.data[features.data.length - 2]
                 if (olderLocation !== undefined) {
-                    timeDiff = (previousLocation.freshness.getTime() - olderLocation.freshness.getTime()) / 1000
+                    console.log("Previous location", previousLocation)
+                    timeDiff = (new Date(previousLocation.freshness).getTime() - new Date(olderLocation.freshness).getTime()) / 1000
                 }
                 if (d < 20 && timeDiff < 60) {
                     // Do not append changes less then 20m - it's probably noise anyway

Logic/Web/IdbLocalStorage.ts

@@ -0,0 +1,25 @@
+import {UIEventSource} from "../UIEventSource";
+import * as idb from "idb-keyval"
+/**
+ * UIEventsource-wrapper around indexedDB key-value
+ */
+export class IdbLocalStorage {
+
+    
+    public static Get<T>(key: string, options: { defaultValue?: T }): UIEventSource<T>{
+        const src = new UIEventSource<T>(options.defaultValue, "idb-local-storage:"+key)
+        idb.get(key).then(v => {
+            src.setData(v ?? options.defaultValue)
+        })
+        src.stabilized(1000).addCallback(v => {
+            idb.set(key, v)
+        })
+        return src;
+        
+    }
+    
+    public static SetDirectly(key: string, value){
+        idb.set(key, value)
+    }
+
+}

package-lock.json

@@ -26,6 +26,7 @@
         "email-validator": "^2.0.4",
         "escape-html": "^1.0.3",
         "i18next-client": "^1.11.4",
+        "idb-keyval": "^6.0.3",
         "jquery": "^3.6.0",
         "jspdf": "^2.3.1",
         "latlon2country": "^1.1.3",
@@ -7435,6 +7436,14 @@
       "resolved": "https://registry.npmjs.org/icss-replace-symbols/-/icss-replace-symbols-1.1.0.tgz",
       "integrity": "sha1-Bupvg2ead0njhs/h/oEq5dsiPe0="
     },
+    "node_modules/idb-keyval": {
+      "version": "6.0.3",
+      "resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-6.0.3.tgz",
+      "integrity": "sha512-yh8V7CnE6EQMu9YDwQXhRxwZh4nv+8xm/HV4ZqK4IiYFJBWYGjJuykADJbSP+F/GDXUBwCSSNn/14IpGL81TuA==",
+      "dependencies": {
+        "safari-14-idb-fix": "^3.0.0"
+      }
+    },
     "node_modules/ieee754": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
@@ -14366,6 +14375,11 @@
       "integrity": "sha512-hcVC3wYEziELGGmEEXue7D75zbwIIVUMWAVbHItGPx0ziyXxrOMQx4rQEVEV45Ut/1IotuEvwqPopzIOkDMf0A==",
       "dev": true
     },
+    "node_modules/safari-14-idb-fix": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/safari-14-idb-fix/-/safari-14-idb-fix-3.0.0.tgz",
+      "integrity": "sha512-eBNFLob4PMq8JA1dGyFn6G97q3/WzNtFK4RnzT1fnLq+9RyrGknzYiM/9B12MnKAxuj1IXr7UKYtTNtjyKMBog=="
+    },
     "node_modules/safe-buffer": {
       "version": "5.2.1",
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@@ -24018,6 +24032,14 @@
       "resolved": "https://registry.npmjs.org/icss-replace-symbols/-/icss-replace-symbols-1.1.0.tgz",
       "integrity": "sha1-Bupvg2ead0njhs/h/oEq5dsiPe0="
     },
+    "idb-keyval": {
+      "version": "6.0.3",
+      "resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-6.0.3.tgz",
+      "integrity": "sha512-yh8V7CnE6EQMu9YDwQXhRxwZh4nv+8xm/HV4ZqK4IiYFJBWYGjJuykADJbSP+F/GDXUBwCSSNn/14IpGL81TuA==",
+      "requires": {
+        "safari-14-idb-fix": "^3.0.0"
+      }
+    },
     "ieee754": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
@@ -29713,6 +29735,11 @@
         }
       }
     },
+    "safari-14-idb-fix": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/safari-14-idb-fix/-/safari-14-idb-fix-3.0.0.tgz",
+      "integrity": "sha512-eBNFLob4PMq8JA1dGyFn6G97q3/WzNtFK4RnzT1fnLq+9RyrGknzYiM/9B12MnKAxuj1IXr7UKYtTNtjyKMBog=="
+    },
     "safe-buffer": {
       "version": "5.2.1",
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",

package.json

@@ -74,6 +74,7 @@
     "email-validator": "^2.0.4",
     "escape-html": "^1.0.3",
     "i18next-client": "^1.11.4",
+    "idb-keyval": "^6.0.3",
     "jquery": "^3.6.0",
     "jspdf": "^2.3.1",
     "latlon2country": "^1.1.3",

test.html

@@ -23,6 +23,8 @@
 <div id="extradiv">'extradiv' not attached</div>
 
 <script src="./test.ts"></script>
+<iframe src="https://staging.anyways.eu/mechelen-reroute/#map=13.70/4.47874/51.02723&route=bicycle.commute" width="100%" height="100%" style="min-width: 250px; min-height: 250px" title="Routeplanner"></iframe>
+
 
 
 </body>

test.ts

@@ -1 +0,0 @@
-console.log("Tests...")