midgard/mapcomplete
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Fix bounds and loading with overpass

Browse source
This commit is contained in:
pietervdvn 2021-10-11 22:30:22 +02:00
parent 3157c35c62
commit 178cefceb4
7 changed files with 78 additions and 103 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
Logic/Actors/OverpassFeatureSource.ts
View file

@ -38,8 +38,7 @@ export default class OverpassFeatureSource implements FeatureSource {
readonly currentBounds: UIEventSource<BBox>
}
private readonly _isActive: UIEventSource<boolean>;
private readonly onBboxLoaded: (bbox: BBox, date: Date, layers: LayerConfig[]) => void;
private readonly padToTiles : number
private readonly onBboxLoaded: (bbox: BBox, date: Date, layers: LayerConfig[], zoomlevel: number) => void;
constructor(
state: {
readonly locationControl: UIEventSource<Loc>,
@ -50,20 +49,19 @@ export default class OverpassFeatureSource implements FeatureSource {
readonly currentBounds: UIEventSource<BBox>
},
options: {
padToTiles: number,
padToTiles: UIEventSource<number>,
isActive?: UIEventSource<boolean>,
relationTracker: RelationsTracker,
onBboxLoaded?: (bbox: BBox, date: Date, layers: LayerConfig[]) => void
onBboxLoaded?: (bbox: BBox, date: Date, layers: LayerConfig[], zoomlevel: number) => void
}) {
this.state = state
this._isActive = options.isActive;
this.padToTiles = options.padToTiles;
this.onBboxLoaded = options.onBboxLoaded
this.relationsTracker = options.relationTracker
const self = this;
state.currentBounds.addCallback(_ => {
self.update()
self.update(options.padToTiles.data)
})
}
@ -86,21 +84,21 @@ export default class OverpassFeatureSource implements FeatureSource {
return new Overpass(new Or(filters), extraScripts, interpreterUrl, this.state.overpassTimeout, this.relationsTracker);
}
private update() {
private update(paddedZoomLevel: number) {
if (!this._isActive.data) {
return;
}
const self = this;
this.updateAsync().then(bboxDate => {
this.updateAsync(paddedZoomLevel).then(bboxDate => {
if(bboxDate === undefined || self.onBboxLoaded === undefined){
return;
}
const [bbox, date, layers] = bboxDate
self.onBboxLoaded(bbox, date, layers)
self.onBboxLoaded(bbox, date, layers, paddedZoomLevel)
})
}
private async updateAsync(): Promise<[BBox, Date, LayerConfig[]]> {
private async updateAsync(padToZoomLevel: number): Promise<[BBox, Date, LayerConfig[]]> {
if (this.runningQuery.data) {
console.log("Still running a query, not updating");
return undefined;
@ -111,14 +109,11 @@ export default class OverpassFeatureSource implements FeatureSource {
return undefined;
}
const bounds = this.state.currentBounds.data?.pad(this.state.layoutToUse.widenFactor)?.expandToTileBounds(this.padToTiles);
const bounds = this.state.currentBounds.data?.pad(this.state.layoutToUse.widenFactor)?.expandToTileBounds(padToZoomLevel);
if (bounds === undefined) {
return undefined;
}
console.log("Current bounds are", this.state.currentBounds.data," padded with",this.state.layoutToUse.widenFactor+":",
this.state.currentBounds.data.pad(this.state.layoutToUse.widenFactor),
"Tileexpanded: ",this.state.currentBounds.data?.pad(this.state.layoutToUse.widenFactor)?.expandToTileBounds(this.padToTiles) )
const self = this;

4
Logic/FeatureSource/Actors/SaveTileToLocalStorageActor.ts
View file

@ -7,7 +7,7 @@ import {FeatureSourceForLayer} from "../FeatureSource";
export default class SaveTileToLocalStorageActor {
public static readonly storageKey: string = "cached-features";
public static readonly formatVersion: string = "1"
public static readonly formatVersion: string = "2"
constructor(source: FeatureSourceForLayer, tileIndex: number) {
source.features.addCallbackAndRunD(features => {
@ -31,6 +31,6 @@ export default class SaveTileToLocalStorageActor {
const key = `${SaveTileToLocalStorageActor.storageKey}-${layerId}-${tileId}`
localStorage.setItem(key + "-time", JSON.stringify(freshness.getTime()))
localStorage.setItem(key + "-format", SaveTileToLocalStorageActor.formatVersion)
console.log("Marked ", key, "as visited")
}
}

30
Logic/FeatureSource/FeaturePipeline.ts
View file

@ -155,7 +155,7 @@ export default class FeaturePipeline {
tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
}
})
}else{
} else {
new RegisteringAllFromFeatureSourceActor(src)
perLayerHierarchy.get(id).registerTile(src)
src.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(src))
@ -200,7 +200,7 @@ export default class FeaturePipeline {
new PerLayerFeatureSourceSplitter(state.filteredLayers,
(source) => TiledFeatureSource.createHierarchy(source, {
layer: source.layer,
minZoomLevel: this.osmSourceZoomLevel,
minZoomLevel: source.layer.layerDef.minzoom,
dontEnforceMinZoom: true,
maxFeatureCount: state.layoutToUse.clustering.minNeededElements,
maxZoomLevel: state.layoutToUse.clustering.maxZoom,
@ -233,7 +233,7 @@ export default class FeaturePipeline {
// Whenever fresh data comes in, we need to update the metatagging
self.newDataLoadedSignal.stabilized(1000).addCallback(src => {
self.newDataLoadedSignal.stabilized(1000).addCallback(_ => {
self.updateAllMetaTagging()
})
@ -270,15 +270,15 @@ export default class FeaturePipeline {
const self = this
return this.state.currentBounds.map(bbox => {
if (bbox === undefined) {
return
return undefined
}
if (!isSufficientlyZoomed.data) {
return;
return undefined;
}
const osmSourceZoomLevel = self.osmSourceZoomLevel
const range = bbox.containingTileRange(osmSourceZoomLevel)
const tileIndexes = []
if (range.total > 100) {
if (range.total >= 100) {
// Too much tiles!
return []
}
@ -288,7 +288,7 @@ export default class FeaturePipeline {
if (oldestDate !== undefined && oldestDate > this.oldestAllowedDate) {
console.debug("Skipping tile", osmSourceZoomLevel, x, y, "as a decently fresh one is available")
// The cached tiles contain decently fresh data
return;
return undefined;
}
tileIndexes.push(i)
})
@ -321,29 +321,30 @@ export default class FeaturePipeline {
}
const range = bbox.containingTileRange(zoom)
if (range.total > 100) {
if (range.total >= 5000) {
return false
}
const self = this;
const allFreshnesses = Tiles.MapRange(range, (x, y) => self.freshnessForVisibleLayers(zoom, x, y))
return allFreshnesses.some(freshness => freshness === undefined || freshness < this.oldestAllowedDate)
}, [state.locationControl])
const self = this;
const updater = new OverpassFeatureSource(state,
{
padToTiles: this.osmSourceZoomLevel,
padToTiles: state.locationControl.map(l => Math.min(15, l.zoom + 1)),
relationTracker: this.relationTracker,
isActive: useOsmApi.map(b => !b && overpassIsActive.data, [overpassIsActive]),
onBboxLoaded: ((bbox, date, downloadedLayers) => {
Tiles.MapRange(bbox.containingTileRange(self.osmSourceZoomLevel), (x, y) => {
onBboxLoaded: (bbox, date, downloadedLayers, paddedToZoomLevel) => {
Tiles.MapRange(bbox.containingTileRange(paddedToZoomLevel), (x, y) => {
const tileIndex = Tiles.tile_index(paddedToZoomLevel, x, y)
downloadedLayers.forEach(layer => {
SaveTileToLocalStorageActor.MarkVisited(layer.id, Tiles.tile_index(this.osmSourceZoomLevel, x, y), date)
self.freshnesses.get(layer.id).addTileLoad(tileIndex, date)
SaveTileToLocalStorageActor.MarkVisited(layer.id, tileIndex, date)
})
})
})
}
});
@ -379,7 +380,6 @@ export default class FeaturePipeline {
private updateAllMetaTagging() {
const self = this;
console.log("Reupdating all metatagging")
this.perLayerHierarchy.forEach(hierarchy => {
hierarchy.loadedTiles.forEach(src => {
self.applyMetaTags(src)

119
Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts
View file

@ -1,14 +1,16 @@
import FilteredLayer from "../../../Models/FilteredLayer";
import {FeatureSourceForLayer, Tiled} from "../FeatureSource";
import {UIEventSource} from "../../UIEventSource";
import Loc from "../../../Models/Loc";
import TileHierarchy from "./TileHierarchy";
import SaveTileToLocalStorageActor from "../Actors/SaveTileToLocalStorageActor";
import {Tiles} from "../../../Models/TileRange";
import {BBox} from "../../BBox";
export default class TiledFromLocalStorageSource implements TileHierarchy<FeatureSourceForLayer & Tiled> {
public loadedTiles: Map<number, FeatureSourceForLayer & Tiled> = new Map<number, FeatureSourceForLayer & Tiled>();
public readonly loadedTiles: Map<number, FeatureSourceForLayer & Tiled> = new Map<number, FeatureSourceForLayer & Tiled>();
private readonly layer: FilteredLayer;
private readonly handleFeatureSource: (src: FeatureSourceForLayer & Tiled, index: number) => void;
private readonly undefinedTiles: Set<number>;
public static GetFreshnesses(layerId: string): Map<number, Date> {
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layerId + "-"
@ -29,14 +31,15 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
constructor(layer: FilteredLayer,
handleFeatureSource: (src: FeatureSourceForLayer & Tiled, index: number) => void,
state: {
locationControl: UIEventSource<Loc>
leafletMap: any
currentBounds: UIEventSource<BBox>
}) {
this.layer = layer;
this.handleFeatureSource = handleFeatureSource;
const undefinedTiles = new Set<number>()
this.undefinedTiles = new Set<number>()
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-"
// @ts-ignore
const indexes: number[] = Object.keys(localStorage)
const knownTiles: number[] = Object.keys(localStorage)
.filter(key => {
return key.startsWith(prefix) && !key.endsWith("-time") && !key.endsWith("-format");
})
@ -45,8 +48,8 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
})
.filter(i => !isNaN(i))
console.debug("Layer", layer.layerDef.id, "has following tiles in available in localstorage", indexes.map(i => Tiles.tile_from_index(i).join("/")).join(", "))
for (const index of indexes) {
console.debug("Layer", layer.layerDef.id, "has following tiles in available in localstorage", knownTiles.map(i => Tiles.tile_from_index(i).join("/")).join(", "))
for (const index of knownTiles) {
const prefix = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-" + index;
const version = localStorage.getItem(prefix + "-format")
@ -55,78 +58,54 @@ export default class TiledFromLocalStorageSource implements TileHierarchy<Featur
localStorage.removeItem(prefix)
localStorage.removeItem(prefix+"-time")
localStorage.removeItem(prefix+"-format")
undefinedTiles.add(index)
this. undefinedTiles.add(index)
console.log("Dropped old format tile", prefix)
}
}
const zLevels = indexes.map(i => i % 100)
const indexesSet = new Set(indexes)
const maxZoom = Math.max(...zLevels)
const minZoom = Math.min(...zLevels)
const self = this;
const self = this
state.currentBounds.map(bounds => {
const neededTiles = state.locationControl.map(
location => {
if (!layer.isDisplayed.data) {
// No need to download! - the layer is disabled
return undefined;
}
if (location.zoom < layer.layerDef.minzoom) {
// No need to download! - the layer is disabled
return undefined;
}
// Yup, this is cheating to just get the bounds here
const bounds = state.leafletMap.data?.getBounds()
if (bounds === undefined) {
// We'll retry later
return undefined
}
const needed = []
for (let z = minZoom; z <= maxZoom; z++) {
const tileRange = Tiles.TileRangeBetween(z, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), bounds.getWest())
const neededZ = Tiles.MapRange(tileRange, (x, y) => Tiles.tile_index(z, x, y))
.filter(i => !self.loadedTiles.has(i) && !undefinedTiles.has(i) && indexesSet.has(i))
needed.push(...neededZ)
}
if (needed.length === 0) {
return undefined
}
return needed
if(bounds === undefined){
return;
}
, [layer.isDisplayed, state.leafletMap]).stabilized(50);
neededTiles.addCallbackAndRunD(neededIndexes => {
for (const neededIndex of neededIndexes) {
// We load the features from localStorage
try {
const key = SaveTileToLocalStorageActor.storageKey + "-" + layer.layerDef.id + "-" + neededIndex
const data = localStorage.getItem(key)
const features = JSON.parse(data)
const src = {
layer: layer,
features: new UIEventSource<{ feature: any; freshness: Date }[]>(features),
name: "FromLocalStorage(" + key + ")",
tileIndex: neededIndex,
bbox: BBox.fromTileIndex(neededIndex)
}
handleFeatureSource(src, neededIndex)
self.loadedTiles.set(neededIndex, src)
} catch (e) {
console.error("Could not load data tile from local storage due to", e)
undefinedTiles.add(neededIndex)
for (const knownTile of knownTiles) {
if(this.loadedTiles.has(knownTile)){
continue;
}
if(this.undefinedTiles.has(knownTile)){
continue;
}
if(!bounds.overlapsWith(BBox.fromTileIndex(knownTile))){
continue;
}
self.loadTile(knownTile)
}
})
}
private loadTile( neededIndex: number){
try {
const key = SaveTileToLocalStorageActor.storageKey + "-" + this.layer.layerDef.id + "-" + neededIndex
const data = localStorage.getItem(key)
const features = JSON.parse(data)
const src = {
layer: this.layer,
features: new UIEventSource<{ feature: any; freshness: Date }[]>(features),
name: "FromLocalStorage(" + key + ")",
tileIndex: neededIndex,
bbox: BBox.fromTileIndex(neededIndex)
}
this.handleFeatureSource(src, neededIndex)
this.loadedTiles.set(neededIndex, src)
} catch (e) {
console.error("Could not load data tile from local storage due to", e)
this.undefinedTiles.add(neededIndex)
}
}
}

2
Models/Constants.ts
View file

@ -2,7 +2,7 @@ import {Utils} from "../Utils";
export default class Constants {
public static vNumber = "0.10.4";
public static vNumber = "0.10.5";
public static ImgurApiKey = '7070e7167f0a25a'
public static readonly mapillary_client_token_v3 = 'TXhLaWthQ1d4RUg0czVxaTVoRjFJZzowNDczNjUzNmIyNTQyYzI2'
public static readonly mapillary_client_token_v4 = "MLY|4441509239301885|b40ad2d3ea105435bd40c7e76993ae85"

2
Models/TileRange.ts
View file

@ -15,7 +15,7 @@ export class Tiles {
public static MapRange<T>(tileRange: TileRange, f: (x: number, y: number) => T): T[] {
const result: T[] = []
const total = tileRange.total
if(total > 5000){
if(total > 100000){
throw "Tilerange too big"
}
for (let x = tileRange.xstart; x <= tileRange.xend; x++) {

1
assets/themes/toerisme_vlaanderen/toerisme_vlaanderen.json
View file

@ -37,6 +37,7 @@
{
"builtin": [
"bench",
"picnic_table",
"waste_basket"
],
"override": {