Merge branch 'features/indexed-db' into develop

This commit is contained in:
pietervdvn 2021-11-16 02:57:34 +01:00
commit bf28292f50
11 changed files with 174 additions and 193 deletions

View file

@ -10,6 +10,7 @@ import RelationsTracker from "../Osm/RelationsTracker";
import {BBox} from "../BBox";
import Loc from "../../Models/Loc";
import LayerConfig from "../../Models/ThemeConfig/LayerConfig";
import AllKnownLayers from "../../Customizations/AllKnownLayers";
export default class OverpassFeatureSource implements FeatureSource {
@ -121,6 +122,9 @@ export default class OverpassFeatureSource implements FeatureSource {
if (typeof (layer) === "string") {
throw "A layer was not expanded!"
}
if(AllKnownLayers.priviliged_layers.indexOf(layer.id) >= 0){
continue
}
if (this.state.locationControl.data.zoom < layer.minzoom) {
continue;
}

View file

@ -3,56 +3,113 @@
*
* Technically, more an Actor then a featuresource, but it fits more neatly this ay
*/
import {FeatureSourceForLayer} from "../FeatureSource";
import FeatureSource, {Tiled} from "../FeatureSource";
import {Tiles} from "../../../Models/TileRange";
import {IdbLocalStorage} from "../../Web/IdbLocalStorage";
import {UIEventSource} from "../../UIEventSource";
import LayerConfig from "../../../Models/ThemeConfig/LayerConfig";
import {BBox} from "../../BBox";
import SimpleFeatureSource from "../Sources/SimpleFeatureSource";
import FilteredLayer from "../../../Models/FilteredLayer";
export default class SaveTileToLocalStorageActor {
public static readonly storageKey: string = "cached-features";
public static readonly formatVersion: string = "2"
private readonly visitedTiles: UIEventSource<Map<number, Date>>
private readonly _layer: LayerConfig;
private readonly _flayer : FilteredLayer
private readonly initializeTime = new Date()
constructor(source: FeatureSourceForLayer, tileIndex: number) {
constructor(layer: FilteredLayer) {
this._flayer = layer
this._layer = layer.layerDef
this.visitedTiles = IdbLocalStorage.Get("visited_tiles_" + this._layer.id,
{defaultValue: new Map<number, Date>(), })
this.visitedTiles.stabilized(100).addCallbackAndRunD(tiles => {
for (const key of Array.from(tiles.keys())) {
const tileFreshness = tiles.get(key)
source.features.addCallbackAndRunD(features => {
const key = `${SaveTileToLocalStorageActor.storageKey}-${source.layer.layerDef.id}-${tileIndex}`
const now = new Date()
try {
if (features.length > 0) {
localStorage.setItem(key, JSON.stringify(features));
const toOld = (this.initializeTime.getTime() - tileFreshness.getTime()) > 1000 * this._layer.maxAgeOfCache
if(toOld){
// Purge this tile
this.SetIdb(key, undefined)
console.debug("Purging tile",this._layer.id,key)
tiles.delete(key)
}
// We _still_ write the time to know that this tile is empty!
SaveTileToLocalStorageActor.MarkVisited(source.layer.layerDef.id, tileIndex, now)
} catch (e) {
console.warn("Could not save the features to local storage:", e)
}
this.visitedTiles.ping()
return true;
})
}
public LoadTilesFromDisk(currentBounds: UIEventSource<BBox>,
registerFreshness: (tileId: number, freshness: Date) => void,
registerTile: ((src: FeatureSource & Tiled ) => void)){
const self = this;
this.visitedTiles.addCallbackD(tiles => {
if(tiles.size === 0){
// We don't do anything yet as probably not yet loaded from disk
// We'll unregister later on
return;
}
for (const key of Array.from(tiles.keys())) {
const tileFreshness = tiles.get(key)
if(tileFreshness > self.initializeTime){
// This tile is loaded by another source
continue
}
registerFreshness(key, tileFreshness)
const tileBbox = BBox.fromTileIndex(key)
currentBounds.addCallbackAndRunD(bbox => {
if(bbox.overlapsWith(tileBbox)){
// The current tile should be loaded from disk
this.GetIdb(key).then((features:{feature: any, freshness: Date}[] ) => {
console.log("Loaded tile "+self._layer.id+"_"+key+" from disk")
const src = new SimpleFeatureSource(self._flayer, key, new UIEventSource<{feature: any; freshness: Date}[]>(features))
registerTile(src)
})
return true; // only load once: unregister
}
})
}
return true; // Remove the callback
})
}
public static MarkVisited(layerId: string, tileId: number, freshness: Date) {
const key = `${SaveTileToLocalStorageActor.storageKey}-${layerId}-${tileId}`
try {
localStorage.setItem(key + "-time", JSON.stringify(freshness.getTime()))
localStorage.setItem(key + "-format", SaveTileToLocalStorageActor.formatVersion)
} catch (e) {
console.error("Could not mark tile ", key, "as visited")
}
private SetIdb(tileIndex, data){
IdbLocalStorage.SetDirectly(this._layer.id+"_"+tileIndex, data)
}
public static poison(layers: string[], lon: number, lat: number) {
for (let z = 0; z < 25; z++) {
private GetIdb(tileIndex){
return IdbLocalStorage.GetDirectly(this._layer.id+"_"+tileIndex)
}
public addTile(tile: FeatureSource & Tiled){
const self = this
tile.features.addCallbackAndRunD(features => {
const now = new Date()
if (features.length > 0) {
self.SetIdb(tile.tileIndex, features)
}
// We _still_ write the time to know that this tile is empty!
this.MarkVisited(tile.tileIndex, now)
})
}
public poison(lon: number, lat: number) {
for (let z = 0; z < 25; z++) {
const {x, y} = Tiles.embedded_tile(lat, lon, z)
const tileId = Tiles.tile_index(z, x, y)
for (const layerId of layers) {
const key = `${SaveTileToLocalStorageActor.storageKey}-${layerId}-${tileId}`
localStorage.removeItem(key + "-time");
localStorage.removeItem(key + "-format")
localStorage.removeItem(key)
}
this.visitedTiles.data.delete(tileId)
}
}
public MarkVisited(tileId: number, freshness: Date) {
this.visitedTiles.data.set(tileId, freshness)
this.visitedTiles.ping()
}
}

View file

@ -11,7 +11,6 @@ import OverpassFeatureSource from "../Actors/OverpassFeatureSource";
import GeoJsonSource from "./Sources/GeoJsonSource";
import Loc from "../../Models/Loc";
import RegisteringAllFromFeatureSourceActor from "./Actors/RegisteringAllFromFeatureSourceActor";
import TiledFromLocalStorageSource from "./TiledFeatureSource/TiledFromLocalStorageSource";
import SaveTileToLocalStorageActor from "./Actors/SaveTileToLocalStorageActor";
import DynamicGeoJsonTileSource from "./TiledFeatureSource/DynamicGeoJsonTileSource";
import {TileHierarchyMerger} from "./TiledFeatureSource/TileHierarchyMerger";
@ -56,6 +55,8 @@ export default class FeaturePipeline {
private readonly oldestAllowedDate: Date;
private readonly osmSourceZoomLevel
private readonly localStorageSavers = new Map<string, SaveTileToLocalStorageActor>()
constructor(
handleFeatureSource: (source: FeatureSourceForLayer & Tiled) => void,
@ -64,9 +65,6 @@ export default class FeaturePipeline {
const self = this
const expiryInSeconds = Math.min(...state.layoutToUse.layers.map(l => l.maxAgeOfCache))
for (const layer of state.layoutToUse.layers) {
TiledFromLocalStorageSource.cleanCacheForLayer(layer)
}
this.oldestAllowedDate = new Date(new Date().getTime() - expiryInSeconds);
this.osmSourceZoomLevel = state.osmApiTileSize.data;
const useOsmApi = state.locationControl.map(l => l.zoom > (state.overpassMaxZoom.data ?? 12))
@ -77,7 +75,7 @@ export default class FeaturePipeline {
.map(ch => ch.changes)
.filter(coor => coor["lat"] !== undefined && coor["lon"] !== undefined)
.forEach(coor => {
SaveTileToLocalStorageActor.poison(state.layoutToUse.layers.map(l => l.id), coor["lon"], coor["lat"])
state.layoutToUse.layers.forEach(l => self.localStorageSavers.get(l.id).poison(coor["lon"], coor["lat"]))
})
})
@ -150,21 +148,23 @@ export default class FeaturePipeline {
handlePriviligedFeatureSource(state.homeLocation)
continue
}
const localTileSaver = new SaveTileToLocalStorageActor(filteredLayer)
this.localStorageSavers.set(filteredLayer.layerDef.id, localTileSaver)
if (source.geojsonSource === undefined) {
// This is an OSM layer
// We load the cached values and register them
// Getting data from upstream happens a bit lower
new TiledFromLocalStorageSource(filteredLayer,
(src) => {
new RegisteringAllFromFeatureSourceActor(src)
hierarchy.registerTile(src);
src.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(src))
}, state)
TiledFromLocalStorageSource.GetFreshnesses(id).forEach((value, key) => {
self.freshnesses.get(id).addTileLoad(key, value)
})
localTileSaver.LoadTilesFromDisk(
state.currentBounds,
(tileIndex, freshness) => self.freshnesses.get(id).addTileLoad(tileIndex, freshness),
(tile) => {
new RegisteringAllFromFeatureSourceActor(tile)
hierarchy.registerTile(tile);
tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
}
)
continue
}
@ -210,7 +210,7 @@ export default class FeaturePipeline {
handleTile: tile => {
new RegisteringAllFromFeatureSourceActor(tile)
if (tile.layer.layerDef.maxAgeOfCache > 0) {
new SaveTileToLocalStorageActor(tile, tile.tileIndex)
self.localStorageSavers.get(tile.layer.layerDef.id).addTile(tile)
}
perLayerHierarchy.get(tile.layer.layerDef.id).registerTile(tile)
tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
@ -219,10 +219,11 @@ export default class FeaturePipeline {
state: state,
markTileVisited: (tileId) =>
state.filteredLayers.data.forEach(flayer => {
if (flayer.layerDef.maxAgeOfCache > 0) {
SaveTileToLocalStorageActor.MarkVisited(flayer.layerDef.id, tileId, new Date())
const layer = flayer.layerDef
if (layer.maxAgeOfCache > 0) {
self.localStorageSavers.get(layer.id).MarkVisited(tileId, new Date())
}
self.freshnesses.get(flayer.layerDef.id).addTileLoad(tileId, new Date())
self.freshnesses.get(layer.id).addTileLoad(tileId, new Date())
})
})
@ -252,10 +253,8 @@ export default class FeaturePipeline {
maxFeatureCount: state.layoutToUse.clustering.minNeededElements,
maxZoomLevel: state.layoutToUse.clustering.maxZoom,
registerTile: (tile) => {
// We save the tile data for the given layer to local storage
if (source.layer.layerDef.source.geojsonSource === undefined || source.layer.layerDef.source.isOsmCacheLayer == true) {
new SaveTileToLocalStorageActor(tile, tile.tileIndex)
}
// We save the tile data for the given layer to local storage - data sourced from overpass
self.localStorageSavers.get(tile.layer.layerDef.id).addTile(tile)
perLayerHierarchy.get(source.layer.layerDef.id).registerTile(new RememberingSource(tile))
tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
@ -417,7 +416,7 @@ export default class FeaturePipeline {
const tileIndex = Tiles.tile_index(paddedToZoomLevel, x, y)
downloadedLayers.forEach(layer => {
self.freshnesses.get(layer.id).addTileLoad(tileIndex, date)
SaveTileToLocalStorageActor.MarkVisited(layer.id, tileIndex, date)
self.localStorageSavers.get(layer.id).MarkVisited(tileIndex, date)
})
})

View file

@ -1,133 +0,0 @@
import FilteredLayer from "../../../Models/FilteredLayer";
import {FeatureSourceForLayer, Tiled} from "../FeatureSource";
import {UIEventSource} from "../../UIEventSource";
import TileHierarchy from "./TileHierarchy";
import SaveTileToLocalStorageActor from "../Actors/SaveTileToLocalStorageActor";
import {Tiles} from "../../../Models/TileRange";
import {BBox} from "../../BBox";
import LayerConfig from "../../../Models/ThemeConfig/LayerConfig";
export default class TiledFromLocalStorageSource implements TileHierarchy<FeatureSourceForLayer & Tiled> {
public readonly loadedTiles: Map<number, FeatureSourceForLayer & Tiled> = new Map<number, FeatureSourceForLayer & Tiled>();
private readonly layer: FilteredLayer;
private readonly handleFeatureSource: (src: FeatureSourceForLayer & Tiled, index: number) => void;
private readonly undefinedTiles: Set<number>;
constructor(layer: FilteredLayer,
handleFeatureSource: (src: FeatureSourceForLayer & Tiled, index: number) => void,
state: {
currentBounds: UIEventSource<BBox>
}) {
this.layer = layer;
this.handleFeatureSource = handleFeatureSource;
this.undefinedTiles = new Set<number>()
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-"
const knownTiles: number[] = Object.keys(localStorage)
.filter(key => {
return key.startsWith(prefix) && !key.endsWith("-time") && !key.endsWith("-format");
})
.map(key => {
return Number(key.substring(prefix.length));
})
.filter(i => !isNaN(i))
console.debug("Layer", layer.layerDef.id, "has following tiles in available in localstorage", knownTiles.map(i => Tiles.tile_from_index(i).join("/")).join(", "))
for (const index of knownTiles) {
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-" + index;
const version = localStorage.getItem(prefix + "-format")
if (version === undefined || version !== SaveTileToLocalStorageActor.formatVersion) {
// Invalid version! Remove this tile from local storage
localStorage.removeItem(prefix)
localStorage.removeItem(prefix + "-time")
localStorage.removeItem(prefix + "-format")
this.undefinedTiles.add(index)
console.log("Dropped old format tile", prefix)
}
}
const self = this
state.currentBounds.map(bounds => {
if (bounds === undefined) {
return;
}
for (const knownTile of knownTiles) {
if (this.loadedTiles.has(knownTile)) {
continue;
}
if (this.undefinedTiles.has(knownTile)) {
continue;
}
if (!bounds.overlapsWith(BBox.fromTileIndex(knownTile))) {
continue;
}
self.loadTile(knownTile)
}
})
}
public static GetFreshnesses(layerId: string): Map<number, Date> {
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layerId + "-"
const freshnesses = new Map<number, Date>()
for (const key of Object.keys(localStorage)) {
if (!(key.startsWith(prefix) && key.endsWith("-time"))) {
continue
}
const index = Number(key.substring(prefix.length, key.length - "-time".length))
const time = Number(localStorage.getItem(key))
const freshness = new Date()
freshness.setTime(time)
freshnesses.set(index, freshness)
}
return freshnesses
}
static cleanCacheForLayer(layer: LayerConfig) {
const now = new Date()
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.id + "-"
console.log("Cleaning tiles of ", prefix, "with max age", layer.maxAgeOfCache)
for (const key of Object.keys(localStorage)) {
if (!(key.startsWith(prefix) && key.endsWith("-time"))) {
continue
}
const index = Number(key.substring(prefix.length, key.length - "-time".length))
const time = Number(localStorage.getItem(key))
const timeDiff = (now.getTime() - time) / 1000
if (timeDiff >= layer.maxAgeOfCache) {
const k = prefix + index;
localStorage.removeItem(k)
localStorage.removeItem(k + "-format")
localStorage.removeItem(k + "-time")
}
}
}
private loadTile(neededIndex: number) {
try {
const key = SaveTileToLocalStorageActor.storageKey + "-" + this.layer.layerDef.id + "-" + neededIndex
const data = localStorage.getItem(key)
const features = JSON.parse(data)
const src = {
layer: this.layer,
features: new UIEventSource<{ feature: any; freshness: Date }[]>(features),
name: "FromLocalStorage(" + key + ")",
tileIndex: neededIndex,
bbox: BBox.fromTileIndex(neededIndex)
}
this.handleFeatureSource(src, neededIndex)
this.loadedTiles.set(neededIndex, src)
} catch (e) {
console.error("Could not load data tile from local storage due to", e)
this.undefinedTiles.add(neededIndex)
}
}
}

View file

@ -218,7 +218,7 @@ export default class MapState extends UserRelatedState {
let timeDiff = Number.MAX_VALUE // in seconds
const olderLocation = features.data[features.data.length - 2]
if (olderLocation !== undefined) {
timeDiff = (previousLocation.freshness.getTime() - olderLocation.freshness.getTime()) / 1000
timeDiff = (new Date(previousLocation.freshness).getTime() - new Date(olderLocation.freshness).getTime()) / 1000
}
if (d < 20 && timeDiff < 60) {
// Do not append changes less then 20m - it's probably noise anyway

View file

@ -223,6 +223,7 @@ export class UIEventSource<T> {
for (const callback of this._callbacks) {
if (callback(this.data) === true) {
// This callback wants to be deleted
// Note: it has to return precisely true in order to avoid accidental deletions
if (toDelete === undefined) {
toDelete = [callback]
} else {

View file

@ -0,0 +1,24 @@
import {UIEventSource} from "../UIEventSource";
import * as idb from "idb-keyval"
/**
* UIEventsource-wrapper around indexedDB key-value
*/
export class IdbLocalStorage {
public static Get<T>(key: string, options: { defaultValue?: T }): UIEventSource<T>{
const src = new UIEventSource<T>(options.defaultValue, "idb-local-storage:"+key)
idb.get(key).then(v => src.setData(v ?? options.defaultValue))
src.addCallback(v => idb.set(key, v))
return src;
}
public static SetDirectly(key: string, value){
idb.set(key, value)
}
static GetDirectly(key: string) {
return idb.get(key)
}
}

27
package-lock.json generated
View file

@ -26,6 +26,7 @@
"email-validator": "^2.0.4",
"escape-html": "^1.0.3",
"i18next-client": "^1.11.4",
"idb-keyval": "^6.0.3",
"jquery": "^3.6.0",
"jspdf": "^2.3.1",
"latlon2country": "^1.1.3",
@ -7435,6 +7436,14 @@
"resolved": "https://registry.npmjs.org/icss-replace-symbols/-/icss-replace-symbols-1.1.0.tgz",
"integrity": "sha1-Bupvg2ead0njhs/h/oEq5dsiPe0="
},
"node_modules/idb-keyval": {
"version": "6.0.3",
"resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-6.0.3.tgz",
"integrity": "sha512-yh8V7CnE6EQMu9YDwQXhRxwZh4nv+8xm/HV4ZqK4IiYFJBWYGjJuykADJbSP+F/GDXUBwCSSNn/14IpGL81TuA==",
"dependencies": {
"safari-14-idb-fix": "^3.0.0"
}
},
"node_modules/ieee754": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
@ -14366,6 +14375,11 @@
"integrity": "sha512-hcVC3wYEziELGGmEEXue7D75zbwIIVUMWAVbHItGPx0ziyXxrOMQx4rQEVEV45Ut/1IotuEvwqPopzIOkDMf0A==",
"dev": true
},
"node_modules/safari-14-idb-fix": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/safari-14-idb-fix/-/safari-14-idb-fix-3.0.0.tgz",
"integrity": "sha512-eBNFLob4PMq8JA1dGyFn6G97q3/WzNtFK4RnzT1fnLq+9RyrGknzYiM/9B12MnKAxuj1IXr7UKYtTNtjyKMBog=="
},
"node_modules/safe-buffer": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
@ -24018,6 +24032,14 @@
"resolved": "https://registry.npmjs.org/icss-replace-symbols/-/icss-replace-symbols-1.1.0.tgz",
"integrity": "sha1-Bupvg2ead0njhs/h/oEq5dsiPe0="
},
"idb-keyval": {
"version": "6.0.3",
"resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-6.0.3.tgz",
"integrity": "sha512-yh8V7CnE6EQMu9YDwQXhRxwZh4nv+8xm/HV4ZqK4IiYFJBWYGjJuykADJbSP+F/GDXUBwCSSNn/14IpGL81TuA==",
"requires": {
"safari-14-idb-fix": "^3.0.0"
}
},
"ieee754": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
@ -29713,6 +29735,11 @@
}
}
},
"safari-14-idb-fix": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/safari-14-idb-fix/-/safari-14-idb-fix-3.0.0.tgz",
"integrity": "sha512-eBNFLob4PMq8JA1dGyFn6G97q3/WzNtFK4RnzT1fnLq+9RyrGknzYiM/9B12MnKAxuj1IXr7UKYtTNtjyKMBog=="
},
"safe-buffer": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",

View file

@ -74,6 +74,7 @@
"email-validator": "^2.0.4",
"escape-html": "^1.0.3",
"i18next-client": "^1.11.4",
"idb-keyval": "^6.0.3",
"jquery": "^3.6.0",
"jspdf": "^2.3.1",
"latlon2country": "^1.1.3",

View file

@ -23,6 +23,8 @@
<div id="extradiv">'extradiv' not attached</div>
<script src="./test.ts"></script>
<iframe src="https://staging.anyways.eu/mechelen-reroute/#map=13.70/4.47874/51.02723&route=bicycle.commute" width="100%" height="100%" style="min-width: 250px; min-height: 250px" title="Routeplanner"></iframe>
</body>

View file

@ -1 +0,0 @@
console.log("Tests...")