mapcomplete/Logic/FeatureSource/FeaturePipeline.ts
2023-01-22 01:09:16 +01:00

648 lines
27 KiB
TypeScript

import LayoutConfig from "../../Models/ThemeConfig/LayoutConfig"
import FilteringFeatureSource from "./Sources/FilteringFeatureSource"
import PerLayerFeatureSourceSplitter from "./PerLayerFeatureSourceSplitter"
import FeatureSource, { FeatureSourceForLayer, IndexedFeatureSource, Tiled } from "./FeatureSource"
import TiledFeatureSource from "./TiledFeatureSource/TiledFeatureSource"
import { Store, UIEventSource } from "../UIEventSource"
import { TileHierarchyTools } from "./TiledFeatureSource/TileHierarchy"
import RememberingSource from "./Sources/RememberingSource"
import OverpassFeatureSource from "../Actors/OverpassFeatureSource"
import GeoJsonSource from "./Sources/GeoJsonSource"
import Loc from "../../Models/Loc"
import RegisteringAllFromFeatureSourceActor from "./Actors/RegisteringAllFromFeatureSourceActor"
import SaveTileToLocalStorageActor from "./Actors/SaveTileToLocalStorageActor"
import DynamicGeoJsonTileSource from "./TiledFeatureSource/DynamicGeoJsonTileSource"
import { TileHierarchyMerger } from "./TiledFeatureSource/TileHierarchyMerger"
import RelationsTracker from "../Osm/RelationsTracker"
import { NewGeometryFromChangesFeatureSource } from "./Sources/NewGeometryFromChangesFeatureSource"
import ChangeGeometryApplicator from "./Sources/ChangeGeometryApplicator"
import { BBox } from "../BBox"
import OsmFeatureSource from "./TiledFeatureSource/OsmFeatureSource"
import { Tiles } from "../../Models/TileRange"
import TileFreshnessCalculator from "./TileFreshnessCalculator"
import FullNodeDatabaseSource from "./TiledFeatureSource/FullNodeDatabaseSource"
import MapState from "../State/MapState"
import { ElementStorage } from "../ElementStorage"
import { OsmFeature } from "../../Models/OsmFeature"
import LayerConfig from "../../Models/ThemeConfig/LayerConfig"
import { FilterState } from "../../Models/FilteredLayer"
import { GeoOperations } from "../GeoOperations"
import { Utils } from "../../Utils"
/**
* The features pipeline ties together a myriad of various datasources:
*
* - The Overpass-API
* - The OSM-API
* - Third-party geojson files, either sliced or directly.
*
* In order to truly understand this class, please have a look at the following diagram: https://cdn-images-1.medium.com/fit/c/800/618/1*qTK1iCtyJUr4zOyw4IFD7A.jpeg
*
*
*/
export default class FeaturePipeline {
public readonly sufficientlyZoomed: Store<boolean>
public readonly runningQuery: Store<boolean>
public readonly timeout: UIEventSource<number>
public readonly somethingLoaded: UIEventSource<boolean> = new UIEventSource<boolean>(false)
public readonly newDataLoadedSignal: UIEventSource<FeatureSource> =
new UIEventSource<FeatureSource>(undefined)
public readonly relationTracker: RelationsTracker
/**
* Keeps track of all raw OSM-nodes.
* Only initialized if 'type_node' is defined as layer
*/
public readonly fullNodeDatabase?: FullNodeDatabaseSource
private readonly overpassUpdater: OverpassFeatureSource
private state: MapState
private readonly perLayerHierarchy: Map<string, TileHierarchyMerger>
/**
* Keeps track of the age of the loaded data.
* Has one freshness-Calculator for every layer
* @private
*/
private readonly freshnesses = new Map<string, TileFreshnessCalculator>()
private readonly oldestAllowedDate: Date
private readonly osmSourceZoomLevel
private readonly localStorageSavers = new Map<string, SaveTileToLocalStorageActor>()
private readonly newGeometryHandler: NewGeometryFromChangesFeatureSource
constructor(
handleFeatureSource: (source: FeatureSourceForLayer & Tiled) => void,
state: MapState,
options?: {
/*Used for metatagging - will receive all the sources with changeGeometry applied but without filtering*/
handleRawFeatureSource: (source: FeatureSourceForLayer) => void
}
) {
this.state = state
const self = this
const expiryInSeconds = Math.min(
...(state.layoutToUse?.layers?.map((l) => l.maxAgeOfCache) ?? [])
)
this.oldestAllowedDate = new Date(new Date().getTime() - expiryInSeconds)
this.osmSourceZoomLevel = state.osmApiTileSize.data
const useOsmApi = state.locationControl.map(
(l) => l.zoom > (state.overpassMaxZoom.data ?? 12)
)
this.relationTracker = new RelationsTracker()
state.changes.allChanges.addCallbackAndRun((allChanges) => {
allChanges
.filter((ch) => ch.id < 0 && ch.changes !== undefined)
.map((ch) => ch.changes)
.filter((coor) => coor["lat"] !== undefined && coor["lon"] !== undefined)
.forEach((coor) => {
state.layoutToUse.layers.forEach((l) =>
self.localStorageSavers.get(l.id)?.poison(coor["lon"], coor["lat"])
)
})
})
this.sufficientlyZoomed = state.locationControl.map((location) => {
if (location?.zoom === undefined) {
return false
}
let minzoom = Math.min(
...state.filteredLayers.data.map((layer) => layer.layerDef.minzoom ?? 18)
)
return location.zoom >= minzoom
})
const neededTilesFromOsm = this.getNeededTilesFromOsm(this.sufficientlyZoomed)
const perLayerHierarchy = new Map<string, TileHierarchyMerger>()
this.perLayerHierarchy = perLayerHierarchy
// Given a tile, wraps it and passes it on to render (handled by 'handleFeatureSource'
function patchedHandleFeatureSource(
src: FeatureSourceForLayer & IndexedFeatureSource & Tiled
) {
// This will already contain the merged features for this tile. In other words, this will only be triggered once for every tile
const withChanges = new ChangeGeometryApplicator(src, state.changes)
const srcFiltered = new FilteringFeatureSource(state, src.tileIndex, withChanges)
handleFeatureSource(srcFiltered)
if (options?.handleRawFeatureSource) {
options.handleRawFeatureSource(withChanges)
}
self.somethingLoaded.setData(true)
// We do not mark as visited here, this is the responsability of the code near the actual loader (e.g. overpassLoader and OSMApiFeatureLoader)
}
function handlePriviligedFeatureSource(src: FeatureSourceForLayer & Tiled) {
// Passthrough to passed function, except that it registers as well
handleFeatureSource(src)
src.features.addCallbackAndRunD((fs) => {
fs.forEach((ff) => state.allElements.addOrGetElement(<any>ff.feature))
})
}
for (const filteredLayer of state.filteredLayers.data) {
const id = filteredLayer.layerDef.id
const source = filteredLayer.layerDef.source
const hierarchy = new TileHierarchyMerger(filteredLayer, (tile, _) =>
patchedHandleFeatureSource(tile)
)
perLayerHierarchy.set(id, hierarchy)
this.freshnesses.set(id, new TileFreshnessCalculator())
if (id === "type_node") {
this.fullNodeDatabase = new FullNodeDatabaseSource(filteredLayer, (tile) => {
new RegisteringAllFromFeatureSourceActor(tile, state.allElements)
perLayerHierarchy.get(tile.layer.layerDef.id).registerTile(tile)
tile.features.addCallbackAndRunD((_) => self.onNewDataLoaded(tile))
})
continue
}
if (id === "selected_element") {
handlePriviligedFeatureSource(state.selectedElementsLayer)
continue
}
if (id === "gps_location") {
handlePriviligedFeatureSource(state.currentUserLocation)
continue
}
if (id === "gps_location_history") {
handlePriviligedFeatureSource(state.historicalUserLocations)
continue
}
if (id === "gps_track") {
handlePriviligedFeatureSource(state.historicalUserLocationsTrack)
continue
}
if (id === "home_location") {
handlePriviligedFeatureSource(state.homeLocation)
continue
}
if (id === "current_view") {
handlePriviligedFeatureSource(state.currentView)
continue
}
const localTileSaver = new SaveTileToLocalStorageActor(filteredLayer)
this.localStorageSavers.set(filteredLayer.layerDef.id, localTileSaver)
if (source.geojsonSource === undefined) {
// This is an OSM layer
// We load the cached values and register them
// Getting data from upstream happens a bit lower
localTileSaver.LoadTilesFromDisk(
state.currentBounds,
state.locationControl,
(tileIndex, freshness) =>
self.freshnesses.get(id).addTileLoad(tileIndex, freshness),
(tile) => {
console.debug("Loaded tile ", id, tile.tileIndex, "from local cache")
new RegisteringAllFromFeatureSourceActor(tile, state.allElements)
hierarchy.registerTile(tile)
tile.features.addCallbackAndRunD((_) => self.onNewDataLoaded(tile))
}
)
continue
}
if (source.geojsonZoomLevel === undefined) {
// This is a 'load everything at once' geojson layer
const src = new GeoJsonSource(filteredLayer)
if (source.isOsmCacheLayer) {
// We split them up into tiles anyway as it is an OSM source
TiledFeatureSource.createHierarchy(src, {
layer: src.layer,
minZoomLevel: this.osmSourceZoomLevel,
noDuplicates: true,
registerTile: (tile) => {
new RegisteringAllFromFeatureSourceActor(tile, state.allElements)
perLayerHierarchy.get(id).registerTile(tile)
tile.features.addCallbackAndRunD((_) => self.onNewDataLoaded(tile))
},
})
} else {
new RegisteringAllFromFeatureSourceActor(src, state.allElements)
perLayerHierarchy.get(id).registerTile(src)
src.features.addCallbackAndRunD((_) => self.onNewDataLoaded(src))
}
} else {
new DynamicGeoJsonTileSource(
filteredLayer,
(tile) => {
new RegisteringAllFromFeatureSourceActor(tile, state.allElements)
perLayerHierarchy.get(id).registerTile(tile)
tile.features.addCallbackAndRunD((_) => self.onNewDataLoaded(tile))
},
state
)
}
}
const osmFeatureSource = new OsmFeatureSource({
isActive: useOsmApi,
neededTiles: neededTilesFromOsm,
handleTile: (tile) => {
new RegisteringAllFromFeatureSourceActor(tile, state.allElements)
if (tile.layer.layerDef.maxAgeOfCache > 0) {
const saver = self.localStorageSavers.get(tile.layer.layerDef.id)
if (saver === undefined) {
console.error(
"No localStorageSaver found for layer ",
tile.layer.layerDef.id
)
}
saver?.addTile(tile)
}
perLayerHierarchy.get(tile.layer.layerDef.id).registerTile(tile)
tile.features.addCallbackAndRunD((_) => self.onNewDataLoaded(tile))
},
state: state,
markTileVisited: (tileId) =>
state.filteredLayers.data.forEach((flayer) => {
const layer = flayer.layerDef
if (layer.maxAgeOfCache > 0) {
const saver = self.localStorageSavers.get(layer.id)
if (saver === undefined) {
console.error("No local storage saver found for ", layer.id)
} else {
saver.MarkVisited(tileId, new Date())
}
}
self.freshnesses.get(layer.id).addTileLoad(tileId, new Date())
}),
})
if (this.fullNodeDatabase !== undefined) {
osmFeatureSource.rawDataHandlers.push((osmJson, tileId) =>
this.fullNodeDatabase.handleOsmJson(osmJson, tileId)
)
}
const updater = this.initOverpassUpdater(state, useOsmApi)
this.overpassUpdater = updater
this.timeout = updater.timeout
// Actually load data from the overpass source
new PerLayerFeatureSourceSplitter(
state.filteredLayers,
(source) =>
TiledFeatureSource.createHierarchy(source, {
layer: source.layer,
minZoomLevel: source.layer.layerDef.minzoom,
noDuplicates: true,
maxFeatureCount: state.layoutToUse.clustering.minNeededElements,
maxZoomLevel: state.layoutToUse.clustering.maxZoom,
registerTile: (tile) => {
// We save the tile data for the given layer to local storage - data sourced from overpass
self.localStorageSavers.get(tile.layer.layerDef.id)?.addTile(tile)
perLayerHierarchy
.get(source.layer.layerDef.id)
.registerTile(new RememberingSource(tile))
tile.features.addCallbackAndRunD((f) => {
if (f.length === 0) {
return
}
self.onNewDataLoaded(tile)
})
},
}),
updater,
{
handleLeftovers: (leftOvers) => {
console.warn("Overpass returned a few non-matched features:", leftOvers)
},
}
)
// Also load points/lines that are newly added.
const newGeometry = new NewGeometryFromChangesFeatureSource(
state.changes,
state.allElements,
state.osmConnection._oauth_config.url
)
this.newGeometryHandler = newGeometry
newGeometry.features.addCallbackAndRun((geometries) => {
console.debug("New geometries are:", geometries)
})
new RegisteringAllFromFeatureSourceActor(newGeometry, state.allElements)
// A NewGeometryFromChangesFeatureSource does not split per layer, so we do this next
new PerLayerFeatureSourceSplitter(
state.filteredLayers,
(perLayer) => {
// We don't bother to split them over tiles as it'll contain little features by default, so we simply add them like this
perLayerHierarchy.get(perLayer.layer.layerDef.id).registerTile(perLayer)
// AT last, we always apply the metatags whenever possible
perLayer.features.addCallbackAndRunD((_) => {
self.onNewDataLoaded(perLayer)
})
},
newGeometry,
{
handleLeftovers: (leftOvers) => {
console.warn("Got some leftovers from the filteredLayers: ", leftOvers)
},
}
)
this.runningQuery = updater.runningQuery.map(
(overpass) => {
console.log(
"FeaturePipeline: runningQuery state changed: Overpass",
overpass ? "is querying," : "is idle,",
"osmFeatureSource is",
osmFeatureSource.isRunning
? "is running and needs " +
neededTilesFromOsm.data?.length +
" tiles (already got " +
osmFeatureSource.downloadedTiles.size +
" tiles )"
: "is idle"
)
return overpass || osmFeatureSource.isRunning.data
},
[osmFeatureSource.isRunning]
)
}
public GetAllFeaturesWithin(bbox: BBox): OsmFeature[][] {
const self = this
const tiles: OsmFeature[][] = []
Array.from(this.perLayerHierarchy.keys()).forEach((key) => {
const fetched: OsmFeature[][] = self.GetFeaturesWithin(key, bbox)
tiles.push(...fetched)
})
return tiles
}
public GetAllFeaturesAndMetaWithin(
bbox: BBox,
layerIdWhitelist?: Set<string>
): { features: OsmFeature[]; layer: string }[] {
const self = this
const tiles: { features: any[]; layer: string }[] = []
Array.from(this.perLayerHierarchy.keys()).forEach((key) => {
if (layerIdWhitelist !== undefined && !layerIdWhitelist.has(key)) {
return
}
return tiles.push({
layer: key,
features: [].concat(...self.GetFeaturesWithin(key, bbox)),
})
})
return tiles
}
/**
* Gets all the tiles which overlap with the given BBOX.
* This might imply that extra features might be shown
*/
public GetFeaturesWithin(layerId: string, bbox: BBox): OsmFeature[][] {
if (layerId === "*") {
return this.GetAllFeaturesWithin(bbox)
}
const requestedHierarchy = this.perLayerHierarchy.get(layerId)
if (requestedHierarchy === undefined) {
console.warn(
"Layer ",
layerId,
"is not defined. Try one of ",
Array.from(this.perLayerHierarchy.keys())
)
return undefined
}
return TileHierarchyTools.getTiles(requestedHierarchy, bbox)
.filter((featureSource) => featureSource.features?.data !== undefined)
.map((featureSource) => featureSource.features.data.map((fs) => fs.feature))
}
public GetTilesPerLayerWithin(
bbox: BBox,
handleTile: (tile: FeatureSourceForLayer & Tiled) => void
) {
Array.from(this.perLayerHierarchy.values()).forEach((hierarchy) => {
TileHierarchyTools.getTiles(hierarchy, bbox).forEach(handleTile)
})
}
private onNewDataLoaded(src: FeatureSource) {
this.newDataLoadedSignal.setData(src)
}
private freshnessForVisibleLayers(z: number, x: number, y: number): Date {
let oldestDate = undefined
for (const flayer of this.state.filteredLayers.data) {
if (!flayer.isDisplayed.data && !flayer.layerDef.forceLoad) {
continue
}
if (this.state.locationControl.data.zoom < flayer.layerDef.minzoom) {
continue
}
if (flayer.layerDef.maxAgeOfCache === 0) {
return undefined
}
const freshnessCalc = this.freshnesses.get(flayer.layerDef.id)
if (freshnessCalc === undefined) {
console.warn("No freshness tracker found for ", flayer.layerDef.id)
return undefined
}
const freshness = freshnessCalc.freshnessFor(z, x, y)
if (freshness === undefined) {
// SOmething is undefined --> we return undefined as we have to download
return undefined
}
if (oldestDate === undefined || oldestDate > freshness) {
oldestDate = freshness
}
}
return oldestDate
}
/*
* Gives an UIEventSource containing the tileIndexes of the tiles that should be loaded from OSM
* */
private getNeededTilesFromOsm(isSufficientlyZoomed: Store<boolean>): Store<number[]> {
const self = this
return this.state.currentBounds.map(
(bbox) => {
if (bbox === undefined) {
return []
}
if (!isSufficientlyZoomed.data) {
return []
}
const osmSourceZoomLevel = self.osmSourceZoomLevel
const range = bbox.containingTileRange(osmSourceZoomLevel)
const tileIndexes = []
if (range.total >= 100) {
// Too much tiles!
return undefined
}
Tiles.MapRange(range, (x, y) => {
const i = Tiles.tile_index(osmSourceZoomLevel, x, y)
const oldestDate = self.freshnessForVisibleLayers(osmSourceZoomLevel, x, y)
if (oldestDate !== undefined && oldestDate > this.oldestAllowedDate) {
console.debug(
"Skipping tile",
osmSourceZoomLevel,
x,
y,
"as a decently fresh one is available"
)
// The cached tiles contain decently fresh data
return undefined
}
tileIndexes.push(i)
})
return tileIndexes
},
[isSufficientlyZoomed]
)
}
private initOverpassUpdater(
state: {
allElements: ElementStorage
layoutToUse: LayoutConfig
currentBounds: Store<BBox>
locationControl: Store<Loc>
readonly overpassUrl: Store<string[]>
readonly overpassTimeout: Store<number>
readonly overpassMaxZoom: Store<number>
},
useOsmApi: Store<boolean>
): OverpassFeatureSource {
const minzoom = Math.min(...state.layoutToUse.layers.map((layer) => layer.minzoom))
const overpassIsActive = state.currentBounds.map(
(bbox) => {
if (bbox === undefined) {
console.debug("Disabling overpass source: no bbox")
return false
}
let zoom = state.locationControl.data.zoom
if (zoom < minzoom) {
// We are zoomed out over the zoomlevel of any layer
console.debug("Disabling overpass source: zoom < minzoom")
return false
}
const range = bbox.containingTileRange(zoom)
if (range.total >= 5000) {
// Let's assume we don't have so much data cached
return true
}
const self = this
const allFreshnesses = Tiles.MapRange(range, (x, y) =>
self.freshnessForVisibleLayers(zoom, x, y)
)
return allFreshnesses.some(
(freshness) => freshness === undefined || freshness < this.oldestAllowedDate
)
},
[state.locationControl]
)
const self = this
const updater = new OverpassFeatureSource(state, {
padToTiles: state.locationControl.map((l) => Math.min(15, l.zoom + 1)),
relationTracker: this.relationTracker,
isActive: useOsmApi.map((b) => !b && overpassIsActive.data, [overpassIsActive]),
freshnesses: this.freshnesses,
onBboxLoaded: (bbox, date, downloadedLayers, paddedToZoomLevel) => {
Tiles.MapRange(bbox.containingTileRange(paddedToZoomLevel), (x, y) => {
const tileIndex = Tiles.tile_index(paddedToZoomLevel, x, y)
downloadedLayers.forEach((layer) => {
self.freshnesses.get(layer.id).addTileLoad(tileIndex, date)
self.localStorageSavers.get(layer.id)?.MarkVisited(tileIndex, date)
})
})
},
})
// Register everything in the state' 'AllElements'
new RegisteringAllFromFeatureSourceActor(updater, state.allElements)
return updater
}
/**
* Builds upon 'GetAllFeaturesAndMetaWithin', but does stricter BBOX-checking and applies the filters
*/
public getAllVisibleElementsWithmeta(
bbox: BBox
): { center: [number, number]; element: OsmFeature; layer: LayerConfig }[] {
if (bbox === undefined) {
console.warn("No bbox")
return []
}
const layers = Utils.toIdRecord(this.state.layoutToUse.layers)
const elementsWithMeta: { features: OsmFeature[]; layer: string }[] =
this.GetAllFeaturesAndMetaWithin(bbox)
let elements: { center: [number, number]; element: OsmFeature; layer: LayerConfig }[] = []
let seenElements = new Set<string>()
for (const elementsWithMetaElement of elementsWithMeta) {
const layer = layers[elementsWithMetaElement.layer]
if (layer.title === undefined) {
continue
}
const filtered = this.state.filteredLayers.data.find((fl) => fl.layerDef == layer)
for (let i = 0; i < elementsWithMetaElement.features.length; i++) {
const element = elementsWithMetaElement.features[i]
if (!filtered.isDisplayed.data) {
continue
}
if (seenElements.has(element.properties.id)) {
continue
}
seenElements.add(element.properties.id)
if (!bbox.overlapsWith(BBox.get(element))) {
continue
}
if (layer?.isShown !== undefined && !layer.isShown.matchesProperties(element)) {
continue
}
const activeFilters: FilterState[] = Array.from(
filtered.appliedFilters.data.values()
)
if (
!activeFilters.every(
(filter) =>
filter?.currentFilter === undefined ||
filter?.currentFilter?.matchesProperties(element.properties)
)
) {
continue
}
const center = GeoOperations.centerpointCoordinates(element)
elements.push({
element,
center,
layer: layers[elementsWithMetaElement.layer],
})
}
}
return elements
}
/**
* Inject a new point
*/
InjectNewPoint(geojson) {
this.newGeometryHandler.features.data.push({
feature: geojson,
freshness: new Date(),
})
this.newGeometryHandler.features.ping()
}
}