midgard/mapcomplete
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Fix: actually download all OSM-data when downloading as geojson or CSV

Browse source
This commit is contained in:
Pieter Vander Vennet 2024-11-12 13:17:36 +01:00
parent 3d0a281ee1
commit 05298c2498
2 changed files with 48 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
src/Logic/FeatureSource/Sources/OverpassFeatureSource.ts
View file

@ -49,14 +49,15 @@ export default class OverpassFeatureSource implements UpdatableFeatureSource {
},
options?: {
padToTiles?: Store<number>
isActive?: Store<boolean>
isActive?: Store<boolean>,
ignoreZoom?: boolean
}
) {
this.state = state
this._isActive = options?.isActive ?? new ImmutableStore(true)
this.padToZoomLevel = options?.padToTiles
const self = this
this._layersToDownload = state.zoom.map((zoom) => this.layersToDownload(zoom))
this._layersToDownload = options?.ignoreZoom? new ImmutableStore(state.layers) : state.zoom.map((zoom) => this.layersToDownload(zoom))
state.bounds.mapD(
(_) => {
@ -103,7 +104,7 @@ export default class OverpassFeatureSource implements UpdatableFeatureSource {
* Download the relevant data from overpass. Attempt to use a different server if one fails; only downloads the relevant layers
* @private
*/
public async updateAsync(): Promise<void> {
public async updateAsync(overrideBounds?: BBox): Promise<void> {
let data: any = undefined
let lastUsed = 0
const start = new Date()
@ -122,7 +123,7 @@ export default class OverpassFeatureSource implements UpdatableFeatureSource {
let bounds: BBox
do {
try {
bounds = this.state.bounds.data
bounds = overrideBounds ?? this.state.bounds.data
?.pad(this.state.widenFactor)
?.expandToTileBounds(this.padToZoomLevel?.data)
if (!bounds) {

58
src/Logic/FeatureSource/Sources/ThemeSource.ts
View file

@ -4,7 +4,7 @@ import { FeatureSource, UpdatableFeatureSource } from "../FeatureSource"
import { Or } from "../../Tags/Or"
import FeatureSwitchState from "../../State/FeatureSwitchState"
import OverpassFeatureSource from "./OverpassFeatureSource"
import { Store, UIEventSource } from "../../UIEventSource"
import { ImmutableStore, Store, UIEventSource } from "../../UIEventSource"
import OsmFeatureSource from "./OsmFeatureSource"
import DynamicGeoJsonTileSource from "../TiledFeatureSource/DynamicGeoJsonTileSource"
import { BBox } from "../../BBox"
@ -28,6 +28,13 @@ export default class ThemeSource extends FeatureSourceMerger {
public static readonly fromCacheZoomLevel = 15
/**
* This source is _only_ triggered when the data is downloaded for CSV export
* @private
*/
private readonly _downloadAll: OverpassFeatureSource
private readonly _mapBounds: Store<BBox>
constructor(
layers: LayerConfig[],
featureSwitches: FeatureSwitchState,
@ -35,7 +42,7 @@ export default class ThemeSource extends FeatureSourceMerger {
backend: string,
isDisplayed: (id: string) => Store<boolean>,
mvtAvailableLayers: Set<string>,
fullNodeDatabaseSource?: FullNodeDatabaseSource
fullNodeDatabaseSource?: FullNodeDatabaseSource,
) {
const supportsForceDownload: UpdatableFeatureSource[] = []
@ -56,7 +63,7 @@ export default class ThemeSource extends FeatureSourceMerger {
{
isActive: isDisplayed(layer.id),
maxAge: layer.maxAgeOfCache,
}
},
)
fromCache.set(layer.id, src)
}
@ -75,7 +82,7 @@ export default class ThemeSource extends FeatureSourceMerger {
zoom,
backend,
featureSwitches,
fullNodeDatabaseSource
fullNodeDatabaseSource,
)
nonMvtSources.push(osmApiSource)
@ -84,13 +91,14 @@ export default class ThemeSource extends FeatureSourceMerger {
console.log(
"Layers ",
nonMvtLayers.map((l) => l.id),
" cannot be fetched from the cache server, defaulting to overpass/OSM-api"
" cannot be fetched from the cache server, defaulting to overpass/OSM-api",
)
overpassSource = ThemeSource.setupOverpass(osmLayers, bounds, zoom, featureSwitches)
nonMvtSources.push(overpassSource)
supportsForceDownload.push(overpassSource)
}
function setIsLoading() {
const loading = overpassSource?.runningQuery?.data || osmApiSource?.isRunning?.data
isLoading.setData(loading)
@ -100,21 +108,40 @@ export default class ThemeSource extends FeatureSourceMerger {
osmApiSource?.isRunning?.addCallbackAndRun(() => setIsLoading())
const geojsonSources: UpdatableFeatureSource[] = geojsonlayers.map((l) =>
ThemeSource.setupGeojsonSource(l, mapProperties, isDisplayed(l.id))
ThemeSource.setupGeojsonSource(l, mapProperties, isDisplayed(l.id)),
)
super(...geojsonSources, ...Array.from(fromCache.values()), ...mvtSources, ...nonMvtSources)
const downloadAllBounds: UIEventSource<BBox> = new UIEventSource<BBox>(undefined)
const downloadAll= new OverpassFeatureSource({
layers: layers.filter(l => l.isNormal()),
bounds: mapProperties.bounds,
zoom: mapProperties.zoom,
overpassUrl: featureSwitches.overpassUrl,
overpassTimeout: featureSwitches.overpassTimeout,
overpassMaxZoom: new ImmutableStore(99),
widenFactor: 0,
},{
ignoreZoom: true
})
super(...geojsonSources, ...Array.from(fromCache.values()), ...mvtSources, ...nonMvtSources, downloadAll)
this.isLoading = isLoading
supportsForceDownload.push(...geojsonSources)
supportsForceDownload.push(...mvtSources) // Non-mvt sources are handled by overpass
this._mapBounds = mapProperties.bounds
this._downloadAll = downloadAll
this.supportsForceDownload = supportsForceDownload
}
private static setupMvtSource(
layer: LayerConfig,
mapProperties: { zoom: Store<number>; bounds: Store<BBox> },
isActive?: Store<boolean>
isActive?: Store<boolean>,
): UpdatableFeatureSource {
return new DynamicMvtileSource(layer, mapProperties, { isActive })
}
@ -122,12 +149,12 @@ export default class ThemeSource extends FeatureSourceMerger {
private static setupGeojsonSource(
layer: LayerConfig,
mapProperties: { zoom: Store<number>; bounds: Store<BBox> },
isActiveByFilter?: Store<boolean>
isActiveByFilter?: Store<boolean>,
): UpdatableFeatureSource {
const source = layer.source
const isActive = mapProperties.zoom.map(
(z) => (isActiveByFilter?.data ?? true) && z >= layer.minzoom,
[isActiveByFilter]
[isActiveByFilter],
)
if (source.geojsonZoomLevel === undefined) {
// This is a 'load everything at once' geojson layer
@ -143,7 +170,7 @@ export default class ThemeSource extends FeatureSourceMerger {
zoom: Store<number>,
backend: string,
featureSwitches: FeatureSwitchState,
fullNodeDatabase: FullNodeDatabaseSource
fullNodeDatabase: FullNodeDatabaseSource,
): OsmFeatureSource | undefined {
if (osmLayers.length == 0) {
return undefined
@ -177,7 +204,7 @@ export default class ThemeSource extends FeatureSourceMerger {
osmLayers: LayerConfig[],
bounds: Store<BBox>,
zoom: Store<number>,
featureSwitches: FeatureSwitchState
featureSwitches: FeatureSwitchState,
): OverpassFeatureSource | undefined {
if (osmLayers.length == 0) {
return undefined
@ -206,13 +233,14 @@ export default class ThemeSource extends FeatureSourceMerger {
{
padToTiles: zoom.map((zoom) => Math.min(15, zoom + 1)),
isActive,
}
},
)
}
public async downloadAll() {
console.log("Downloading all data")
await Promise.all(this.supportsForceDownload.map((i) => i.updateAsync()))
console.log("Downloading all data:")
await this._downloadAll.updateAsync(this._mapBounds.data)
// await Promise.all(this.supportsForceDownload.map((i) => i.updateAsync()))
console.log("Done")
}
}