Fix: actually download all OSM-data when downloading as geojson or CSV

src/Logic/FeatureSource/Sources/OverpassFeatureSource.ts

@@ -49,14 +49,15 @@ export default class OverpassFeatureSource implements UpdatableFeatureSource {
         },
         options?: {
             padToTiles?: Store<number>
-            isActive?: Store<boolean>
+            isActive?: Store<boolean>,
+            ignoreZoom?: boolean
         }
     ) {
         this.state = state
         this._isActive = options?.isActive ?? new ImmutableStore(true)
         this.padToZoomLevel = options?.padToTiles
         const self = this
-        this._layersToDownload = state.zoom.map((zoom) => this.layersToDownload(zoom))
+        this._layersToDownload = options?.ignoreZoom? new ImmutableStore(state.layers) : state.zoom.map((zoom) => this.layersToDownload(zoom))
 
         state.bounds.mapD(
             (_) => {
@@ -103,7 +104,7 @@ export default class OverpassFeatureSource implements UpdatableFeatureSource {
      * Download the relevant data from overpass. Attempt to use a different server if one fails; only downloads the relevant layers
      * @private
      */
-    public async updateAsync(): Promise<void> {
+    public async updateAsync(overrideBounds?: BBox): Promise<void> {
         let data: any = undefined
         let lastUsed = 0
         const start = new Date()
@@ -122,7 +123,7 @@ export default class OverpassFeatureSource implements UpdatableFeatureSource {
         let bounds: BBox
         do {
             try {
-                bounds = this.state.bounds.data
+                bounds = overrideBounds ?? this.state.bounds.data
                     ?.pad(this.state.widenFactor)
                     ?.expandToTileBounds(this.padToZoomLevel?.data)
                 if (!bounds) {

src/Logic/FeatureSource/Sources/ThemeSource.ts

@@ -4,7 +4,7 @@ import { FeatureSource, UpdatableFeatureSource } from "../FeatureSource"
 import { Or } from "../../Tags/Or"
 import FeatureSwitchState from "../../State/FeatureSwitchState"
 import OverpassFeatureSource from "./OverpassFeatureSource"
-import { Store, UIEventSource } from "../../UIEventSource"
+import { ImmutableStore, Store, UIEventSource } from "../../UIEventSource"
 import OsmFeatureSource from "./OsmFeatureSource"
 import DynamicGeoJsonTileSource from "../TiledFeatureSource/DynamicGeoJsonTileSource"
 import { BBox } from "../../BBox"
@@ -28,6 +28,13 @@ export default class ThemeSource extends FeatureSourceMerger {
 
     public static readonly fromCacheZoomLevel = 15
 
+    /**
+     * This source is _only_ triggered when the data is downloaded for CSV export
+     * @private
+     */
+    private readonly _downloadAll: OverpassFeatureSource
+    private readonly _mapBounds: Store<BBox>
+
     constructor(
         layers: LayerConfig[],
         featureSwitches: FeatureSwitchState,
@@ -35,7 +42,7 @@ export default class ThemeSource extends FeatureSourceMerger {
         backend: string,
         isDisplayed: (id: string) => Store<boolean>,
         mvtAvailableLayers: Set<string>,
-        fullNodeDatabaseSource?: FullNodeDatabaseSource
+        fullNodeDatabaseSource?: FullNodeDatabaseSource,
     ) {
         const supportsForceDownload: UpdatableFeatureSource[] = []
 
@@ -56,7 +63,7 @@ export default class ThemeSource extends FeatureSourceMerger {
                     {
                         isActive: isDisplayed(layer.id),
                         maxAge: layer.maxAgeOfCache,
-                    }
+                    },
                 )
                 fromCache.set(layer.id, src)
             }
@@ -75,7 +82,7 @@ export default class ThemeSource extends FeatureSourceMerger {
             zoom,
             backend,
             featureSwitches,
-            fullNodeDatabaseSource
+            fullNodeDatabaseSource,
         )
         nonMvtSources.push(osmApiSource)
 
@@ -84,13 +91,14 @@ export default class ThemeSource extends FeatureSourceMerger {
             console.log(
                 "Layers ",
                 nonMvtLayers.map((l) => l.id),
-                " cannot be fetched from the cache server, defaulting to overpass/OSM-api"
+                " cannot be fetched from the cache server, defaulting to overpass/OSM-api",
             )
             overpassSource = ThemeSource.setupOverpass(osmLayers, bounds, zoom, featureSwitches)
             nonMvtSources.push(overpassSource)
             supportsForceDownload.push(overpassSource)
         }
 
+
         function setIsLoading() {
             const loading = overpassSource?.runningQuery?.data || osmApiSource?.isRunning?.data
             isLoading.setData(loading)
@@ -100,21 +108,40 @@ export default class ThemeSource extends FeatureSourceMerger {
         osmApiSource?.isRunning?.addCallbackAndRun(() => setIsLoading())
 
         const geojsonSources: UpdatableFeatureSource[] = geojsonlayers.map((l) =>
-            ThemeSource.setupGeojsonSource(l, mapProperties, isDisplayed(l.id))
+            ThemeSource.setupGeojsonSource(l, mapProperties, isDisplayed(l.id)),
         )
 
-        super(...geojsonSources, ...Array.from(fromCache.values()), ...mvtSources, ...nonMvtSources)
+        const downloadAllBounds: UIEventSource<BBox> = new UIEventSource<BBox>(undefined)
+       const downloadAll=  new OverpassFeatureSource({
+            layers: layers.filter(l => l.isNormal()),
+            bounds: mapProperties.bounds,
+            zoom: mapProperties.zoom,
+            overpassUrl: featureSwitches.overpassUrl,
+            overpassTimeout: featureSwitches.overpassTimeout,
+            overpassMaxZoom: new ImmutableStore(99),
+            widenFactor: 0,
+        },{
+          ignoreZoom: true
+      })
+
+        super(...geojsonSources, ...Array.from(fromCache.values()), ...mvtSources, ...nonMvtSources, downloadAll)
 
         this.isLoading = isLoading
         supportsForceDownload.push(...geojsonSources)
         supportsForceDownload.push(...mvtSources) // Non-mvt sources are handled by overpass
+
+
+        this._mapBounds = mapProperties.bounds
+        this._downloadAll = downloadAll
+
         this.supportsForceDownload = supportsForceDownload
+
     }
 
     private static setupMvtSource(
         layer: LayerConfig,
         mapProperties: { zoom: Store<number>; bounds: Store<BBox> },
-        isActive?: Store<boolean>
+        isActive?: Store<boolean>,
     ): UpdatableFeatureSource {
         return new DynamicMvtileSource(layer, mapProperties, { isActive })
     }
@@ -122,12 +149,12 @@ export default class ThemeSource extends FeatureSourceMerger {
     private static setupGeojsonSource(
         layer: LayerConfig,
         mapProperties: { zoom: Store<number>; bounds: Store<BBox> },
-        isActiveByFilter?: Store<boolean>
+        isActiveByFilter?: Store<boolean>,
     ): UpdatableFeatureSource {
         const source = layer.source
         const isActive = mapProperties.zoom.map(
             (z) => (isActiveByFilter?.data ?? true) && z >= layer.minzoom,
-            [isActiveByFilter]
+            [isActiveByFilter],
         )
         if (source.geojsonZoomLevel === undefined) {
             // This is a 'load everything at once' geojson layer
@@ -143,7 +170,7 @@ export default class ThemeSource extends FeatureSourceMerger {
         zoom: Store<number>,
         backend: string,
         featureSwitches: FeatureSwitchState,
-        fullNodeDatabase: FullNodeDatabaseSource
+        fullNodeDatabase: FullNodeDatabaseSource,
     ): OsmFeatureSource | undefined {
         if (osmLayers.length == 0) {
             return undefined
@@ -177,7 +204,7 @@ export default class ThemeSource extends FeatureSourceMerger {
         osmLayers: LayerConfig[],
         bounds: Store<BBox>,
         zoom: Store<number>,
-        featureSwitches: FeatureSwitchState
+        featureSwitches: FeatureSwitchState,
     ): OverpassFeatureSource | undefined {
         if (osmLayers.length == 0) {
             return undefined
@@ -206,13 +233,14 @@ export default class ThemeSource extends FeatureSourceMerger {
             {
                 padToTiles: zoom.map((zoom) => Math.min(15, zoom + 1)),
                 isActive,
-            }
+            },
         )
     }
 
     public async downloadAll() {
-        console.log("Downloading all data")
+        console.log("Downloading all data:")
-        await Promise.all(this.supportsForceDownload.map((i) => i.updateAsync()))
+        await this._downloadAll.updateAsync(this._mapBounds.data)
+       // await Promise.all(this.supportsForceDownload.map((i) => i.updateAsync()))
         console.log("Done")
     }
 }