From 05298c2498bb908860f49f057b7466f4a8feb5ba Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Tue, 12 Nov 2024 13:17:36 +0100 Subject: [PATCH] Fix: actually download all OSM-data when downloading as geojson or CSV --- .../Sources/OverpassFeatureSource.ts | 9 +-- .../FeatureSource/Sources/ThemeSource.ts | 58 ++++++++++++++----- 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/src/Logic/FeatureSource/Sources/OverpassFeatureSource.ts b/src/Logic/FeatureSource/Sources/OverpassFeatureSource.ts index 7872d67bc..494d59e2d 100644 --- a/src/Logic/FeatureSource/Sources/OverpassFeatureSource.ts +++ b/src/Logic/FeatureSource/Sources/OverpassFeatureSource.ts @@ -49,14 +49,15 @@ export default class OverpassFeatureSource implements UpdatableFeatureSource { }, options?: { padToTiles?: Store - isActive?: Store + isActive?: Store, + ignoreZoom?: boolean } ) { this.state = state this._isActive = options?.isActive ?? new ImmutableStore(true) this.padToZoomLevel = options?.padToTiles const self = this - this._layersToDownload = state.zoom.map((zoom) => this.layersToDownload(zoom)) + this._layersToDownload = options?.ignoreZoom? new ImmutableStore(state.layers) : state.zoom.map((zoom) => this.layersToDownload(zoom)) state.bounds.mapD( (_) => { @@ -103,7 +104,7 @@ export default class OverpassFeatureSource implements UpdatableFeatureSource { * Download the relevant data from overpass. Attempt to use a different server if one fails; only downloads the relevant layers * @private */ - public async updateAsync(): Promise { + public async updateAsync(overrideBounds?: BBox): Promise { let data: any = undefined let lastUsed = 0 const start = new Date() @@ -122,7 +123,7 @@ export default class OverpassFeatureSource implements UpdatableFeatureSource { let bounds: BBox do { try { - bounds = this.state.bounds.data + bounds = overrideBounds ?? this.state.bounds.data ?.pad(this.state.widenFactor) ?.expandToTileBounds(this.padToZoomLevel?.data) if (!bounds) { diff --git a/src/Logic/FeatureSource/Sources/ThemeSource.ts b/src/Logic/FeatureSource/Sources/ThemeSource.ts index 80c85b44e..bc20862df 100644 --- a/src/Logic/FeatureSource/Sources/ThemeSource.ts +++ b/src/Logic/FeatureSource/Sources/ThemeSource.ts @@ -4,7 +4,7 @@ import { FeatureSource, UpdatableFeatureSource } from "../FeatureSource" import { Or } from "../../Tags/Or" import FeatureSwitchState from "../../State/FeatureSwitchState" import OverpassFeatureSource from "./OverpassFeatureSource" -import { Store, UIEventSource } from "../../UIEventSource" +import { ImmutableStore, Store, UIEventSource } from "../../UIEventSource" import OsmFeatureSource from "./OsmFeatureSource" import DynamicGeoJsonTileSource from "../TiledFeatureSource/DynamicGeoJsonTileSource" import { BBox } from "../../BBox" @@ -28,6 +28,13 @@ export default class ThemeSource extends FeatureSourceMerger { public static readonly fromCacheZoomLevel = 15 + /** + * This source is _only_ triggered when the data is downloaded for CSV export + * @private + */ + private readonly _downloadAll: OverpassFeatureSource + private readonly _mapBounds: Store + constructor( layers: LayerConfig[], featureSwitches: FeatureSwitchState, @@ -35,7 +42,7 @@ export default class ThemeSource extends FeatureSourceMerger { backend: string, isDisplayed: (id: string) => Store, mvtAvailableLayers: Set, - fullNodeDatabaseSource?: FullNodeDatabaseSource + fullNodeDatabaseSource?: FullNodeDatabaseSource, ) { const supportsForceDownload: UpdatableFeatureSource[] = [] @@ -56,7 +63,7 @@ export default class ThemeSource extends FeatureSourceMerger { { isActive: isDisplayed(layer.id), maxAge: layer.maxAgeOfCache, - } + }, ) fromCache.set(layer.id, src) } @@ -75,7 +82,7 @@ export default class ThemeSource extends FeatureSourceMerger { zoom, backend, featureSwitches, - fullNodeDatabaseSource + fullNodeDatabaseSource, ) nonMvtSources.push(osmApiSource) @@ -84,13 +91,14 @@ export default class ThemeSource extends FeatureSourceMerger { console.log( "Layers ", nonMvtLayers.map((l) => l.id), - " cannot be fetched from the cache server, defaulting to overpass/OSM-api" + " cannot be fetched from the cache server, defaulting to overpass/OSM-api", ) overpassSource = ThemeSource.setupOverpass(osmLayers, bounds, zoom, featureSwitches) nonMvtSources.push(overpassSource) supportsForceDownload.push(overpassSource) } + function setIsLoading() { const loading = overpassSource?.runningQuery?.data || osmApiSource?.isRunning?.data isLoading.setData(loading) @@ -100,21 +108,40 @@ export default class ThemeSource extends FeatureSourceMerger { osmApiSource?.isRunning?.addCallbackAndRun(() => setIsLoading()) const geojsonSources: UpdatableFeatureSource[] = geojsonlayers.map((l) => - ThemeSource.setupGeojsonSource(l, mapProperties, isDisplayed(l.id)) + ThemeSource.setupGeojsonSource(l, mapProperties, isDisplayed(l.id)), ) - super(...geojsonSources, ...Array.from(fromCache.values()), ...mvtSources, ...nonMvtSources) + const downloadAllBounds: UIEventSource = new UIEventSource(undefined) + const downloadAll= new OverpassFeatureSource({ + layers: layers.filter(l => l.isNormal()), + bounds: mapProperties.bounds, + zoom: mapProperties.zoom, + overpassUrl: featureSwitches.overpassUrl, + overpassTimeout: featureSwitches.overpassTimeout, + overpassMaxZoom: new ImmutableStore(99), + widenFactor: 0, + },{ + ignoreZoom: true + }) + + super(...geojsonSources, ...Array.from(fromCache.values()), ...mvtSources, ...nonMvtSources, downloadAll) this.isLoading = isLoading supportsForceDownload.push(...geojsonSources) supportsForceDownload.push(...mvtSources) // Non-mvt sources are handled by overpass + + + this._mapBounds = mapProperties.bounds + this._downloadAll = downloadAll + this.supportsForceDownload = supportsForceDownload + } private static setupMvtSource( layer: LayerConfig, mapProperties: { zoom: Store; bounds: Store }, - isActive?: Store + isActive?: Store, ): UpdatableFeatureSource { return new DynamicMvtileSource(layer, mapProperties, { isActive }) } @@ -122,12 +149,12 @@ export default class ThemeSource extends FeatureSourceMerger { private static setupGeojsonSource( layer: LayerConfig, mapProperties: { zoom: Store; bounds: Store }, - isActiveByFilter?: Store + isActiveByFilter?: Store, ): UpdatableFeatureSource { const source = layer.source const isActive = mapProperties.zoom.map( (z) => (isActiveByFilter?.data ?? true) && z >= layer.minzoom, - [isActiveByFilter] + [isActiveByFilter], ) if (source.geojsonZoomLevel === undefined) { // This is a 'load everything at once' geojson layer @@ -143,7 +170,7 @@ export default class ThemeSource extends FeatureSourceMerger { zoom: Store, backend: string, featureSwitches: FeatureSwitchState, - fullNodeDatabase: FullNodeDatabaseSource + fullNodeDatabase: FullNodeDatabaseSource, ): OsmFeatureSource | undefined { if (osmLayers.length == 0) { return undefined @@ -177,7 +204,7 @@ export default class ThemeSource extends FeatureSourceMerger { osmLayers: LayerConfig[], bounds: Store, zoom: Store, - featureSwitches: FeatureSwitchState + featureSwitches: FeatureSwitchState, ): OverpassFeatureSource | undefined { if (osmLayers.length == 0) { return undefined @@ -206,13 +233,14 @@ export default class ThemeSource extends FeatureSourceMerger { { padToTiles: zoom.map((zoom) => Math.min(15, zoom + 1)), isActive, - } + }, ) } public async downloadAll() { - console.log("Downloading all data") - await Promise.all(this.supportsForceDownload.map((i) => i.updateAsync())) + console.log("Downloading all data:") + await this._downloadAll.updateAsync(this._mapBounds.data) + // await Promise.all(this.supportsForceDownload.map((i) => i.updateAsync())) console.log("Done") } }