midgard/mapcomplete
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch 'develop'

Browse source
This commit is contained in:
pietervdvn 2021-10-27 19:58:27 +02:00
commit f5f4bc7fde
17 changed files with 751 additions and 562 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
Logic/BBox.ts
View file

@ -1,5 +1,6 @@
import * as turf from "@turf/turf";
import {TileRange, Tiles} from "../Models/TileRange";
import {GeoOperations} from "./GeoOperations";
export class BBox {
@ -119,7 +120,7 @@ export class BBox {
pad(factor: number, maxIncrease = 2): BBox {
const latDiff = Math.min(maxIncrease / 2, Math.abs(this.maxLat - this.minLat) * factor)
const lonDiff =Math.min(maxIncrease / 2, Math.abs(this.maxLon - this.minLon) * factor)
const lonDiff = Math.min(maxIncrease / 2, Math.abs(this.maxLon - this.minLon) * factor)
return new BBox([[
this.minLon - lonDiff,
this.minLat - latDiff
@ -161,4 +162,16 @@ export class BBox {
const boundslr = Tiles.tile_bounds_lon_lat(lr.z, lr.x, lr.y)
return new BBox([].concat(boundsul, boundslr))
}
toMercator(): { minLat: number, maxLat: number, minLon: number, maxLon: number } {
const [minLon, minLat] = GeoOperations.ConvertWgs84To900913([this.minLon, this.minLat])
const [maxLon, maxLat] = GeoOperations.ConvertWgs84To900913([this.maxLon, this.maxLat])
return {
minLon, maxLon,
minLat, maxLat
}
}
}

18
Logic/DetermineLayout.ts
View file

@ -18,7 +18,6 @@ export default class DetermineLayout {
*/
public static async GetLayout(): Promise<[LayoutConfig, string]> {
const loadCustomThemeParam = QueryParameters.GetQueryParameter("userlayout", "false", "If not 'false', a custom (non-official) theme is loaded. This custom layout can be done in multiple ways: \n\n- The hash of the URL contains a base64-encoded .json-file containing the theme definition\n- The hash of the URL contains a lz-compressed .json-file, as generated by the custom theme generator\n- The parameter itself is an URL, in which case that URL will be downloaded. It should point to a .json of a theme")
const layoutFromBase64 = decodeURIComponent(loadCustomThemeParam.data);
@ -73,17 +72,13 @@ export default class DetermineLayout {
try {
const data = await Utils.downloadJson(link)
const parsed = await Utils.downloadJson(link)
console.log("Got ", parsed)
try {
let parsed = data;
if (typeof parsed == "string") {
parsed = JSON.parse(parsed);
}
// Overwrite the id to the url
parsed.id = link;
return new LayoutConfig(parsed, false).patchImages(link, data);
return new LayoutConfig(parsed, false).patchImages(link, JSON.stringify(parsed));
} catch (e) {
console.error(e)
DetermineLayout.ShowErrorOnCustomTheme(
`<a href="${link}">${link}</a> is invalid:`,
new FixedUiElement(e)
@ -92,6 +87,7 @@ export default class DetermineLayout {
}
} catch (e) {
console.erorr(e)
DetermineLayout.ShowErrorOnCustomTheme(
`<a href="${link}">${link}</a> is invalid - probably not found or invalid JSON:`,
new FixedUiElement(e)
@ -107,7 +103,7 @@ export default class DetermineLayout {
try {
// layoutFromBase64 contains the name of the theme. This is partly to do tracking with goat counter
const dedicatedHashFromLocalStorage = LocalStorageSource.Get(
"user-layout-" + userLayoutParam.data.replace(" ", "_")
"user-layout-" + userLayoutParam.data?.replace(" ", "_")
);
if (dedicatedHashFromLocalStorage.data?.length < 10) {
dedicatedHashFromLocalStorage.setData(undefined);
@ -134,6 +130,7 @@ export default class DetermineLayout {
try {
json = JSON.parse(Utils.UnMinify(LZString.decompressFromBase64(hash)))
} catch (e) {
console.error(e)
DetermineLayout.ShowErrorOnCustomTheme("Could not decode the hash", new FixedUiElement("Not a valid (LZ-compressed) JSON"))
return null;
}
@ -143,6 +140,7 @@ export default class DetermineLayout {
userLayoutParam.setData(layoutToUse.id);
return [layoutToUse, btoa(Utils.MinifyJSON(JSON.stringify(json)))];
} catch (e) {
console.error(e)
if (hash === undefined || hash.length < 10) {
DetermineLayout.ShowErrorOnCustomTheme("Could not load a theme from the hash", new FixedUiElement("Hash does not contain data"))
}

4
Logic/FeatureSource/FeaturePipeline.ts
View file

@ -205,7 +205,9 @@ export default class FeaturePipeline {
neededTiles: neededTilesFromOsm,
handleTile: tile => {
new RegisteringAllFromFeatureSourceActor(tile)
if (tile.layer.layerDef.maxAgeOfCache > 0) {
new SaveTileToLocalStorageActor(tile, tile.tileIndex)
}
perLayerHierarchy.get(tile.layer.layerDef.id).registerTile(tile)
tile.features.addCallbackAndRunD(_ => self.newDataLoadedSignal.setData(tile))
@ -213,7 +215,9 @@ export default class FeaturePipeline {
state: state,
markTileVisited: (tileId) =>
state.filteredLayers.data.forEach(flayer => {
if (flayer.layerDef.maxAgeOfCache > 0) {
SaveTileToLocalStorageActor.MarkVisited(flayer.layerDef.id, tileId, new Date())
}
self.freshnesses.get(flayer.layerDef.id).addTileLoad(tileId, new Date())
})
})

16
Logic/FeatureSource/Sources/GeoJsonSource.ts
View file

@ -7,6 +7,7 @@ import {Utils} from "../../../Utils";
import {FeatureSourceForLayer, Tiled} from "../FeatureSource";
import {Tiles} from "../../../Models/TileRange";
import {BBox} from "../../BBox";
import {GeoOperations} from "../../GeoOperations";
export default class GeoJsonSource implements FeatureSourceForLayer, Tiled {
@ -14,7 +15,6 @@ export default class GeoJsonSource implements FeatureSourceForLayer, Tiled {
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>;
public readonly name;
public readonly isOsmCache: boolean
private onFail: ((errorMsg: any, url: string) => void) = undefined;
private readonly seenids: Set<string> = new Set<string>()
public readonly layer: FilteredLayer;
@ -44,10 +44,20 @@ export default class GeoJsonSource implements FeatureSourceForLayer, Tiled {
let url = flayer.layerDef.source.geojsonSource.replace("{layer}", flayer.layerDef.id);
if (zxy !== undefined) {
const [z, x, y] = zxy;
let tile_bbox = BBox.fromTile(z, x, y)
let bounds : { minLat: number, maxLat: number, minLon: number, maxLon: number } = tile_bbox
if(this.layer.layerDef.source.mercatorCrs){
bounds = tile_bbox.toMercator()
}
url = url
.replace('{z}', "" + z)
.replace('{x}', "" + x)
.replace('{y}', "" + y)
.replace('{y_min}',""+bounds.minLat)
.replace('{y_max}',""+bounds.maxLat)
.replace('{x_min}',""+bounds.minLon)
.replace('{x_max}',""+bounds.maxLon)
this.tileIndex = Tiles.tile_index(z, x, y)
this.bbox = BBox.fromTile(z, x, y)
} else {
@ -72,6 +82,10 @@ export default class GeoJsonSource implements FeatureSourceForLayer, Tiled {
return;
}
if(self.layer.layerDef.source.mercatorCrs){
json = GeoOperations.GeoJsonToWGS84(json)
}
const time = new Date();
const newFeatures: { feature: any, freshness: Date } [] = []
let i = 0;

18
Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts
View file

@ -21,23 +21,27 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource {
throw "Invalid layer: geojsonSource expected"
}
const whitelistUrl = source.geojsonSource
.replace("{z}", ""+source.geojsonZoomLevel)
.replace("{x}_{y}.geojson", "overview.json")
.replace("{layer}",layer.layerDef.id)
let whitelist = undefined
if (source.geojsonSource.indexOf("{x}_{y}.geojson") > 0) {
const whitelistUrl = source.geojsonSource
.replace("{z}", "" + source.geojsonZoomLevel)
.replace("{x}_{y}.geojson", "overview.json")
.replace("{layer}", layer.layerDef.id)
Utils.downloadJson(whitelistUrl).then(
json => {
const data = new Map<number, Set<number>>();
for (const x in json) {
data.set(Number(x), new Set(json[x]))
}
console.log("The whitelist is", data, "based on ", json, "from", whitelistUrl)
whitelist = data
}
).catch(err => {
console.warn("No whitelist found for ", layer.layerDef.id, err)
})
}
const seenIds = new Set<string>();
const blackList = new UIEventSource(seenIds)
@ -45,9 +49,9 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource {
layer,
source.geojsonZoomLevel,
(zxy) => {
if(whitelist !== undefined){
if (whitelist !== undefined) {
const isWhiteListed = whitelist.get(zxy[1])?.has(zxy[2])
if(!isWhiteListed){
if (!isWhiteListed) {
console.log("Not downloading tile", ...zxy, "as it is not on the whitelist")
return undefined;
}

28
Logic/GeoOperations.ts
View file

@ -283,6 +283,34 @@ export class GeoOperations {
return headerValuesOrdered.map(v => JSON.stringify(v)).join(",") + "\n" + lines.join("\n")
}
private static readonly _earthRadius = 6378137;
private static readonly _originShift = 2 * Math.PI * GeoOperations._earthRadius / 2;
//Converts given lat/lon in WGS84 Datum to XY in Spherical Mercator EPSG:900913
public static ConvertWgs84To900913(lonLat: [number, number]): [number, number] {
const lon = lonLat[0];
const lat = lonLat[1];
const x = lon * GeoOperations._originShift / 180;
let y = Math.log(Math.tan((90 + lat) * Math.PI / 360)) / (Math.PI / 180);
y = y * GeoOperations._originShift / 180;
return [x, y];
}
//Converts XY point from (Spherical) Web Mercator EPSG:3785 (unofficially EPSG:900913) to lat/lon in WGS84 Datum
public static Convert900913ToWgs84(lonLat: [number, number]): [number, number] {
const lon = lonLat[0]
const lat = lonLat[1]
const x = 180 * lon / GeoOperations._originShift;
let y = 180 * lat / GeoOperations._originShift;
y = 180 / Math.PI * (2 * Math.atan(Math.exp(y * Math.PI / 180)) - Math.PI / 2);
return [x, y];
}
public static GeoJsonToWGS84(geojson){
return turf.toWgs84(geojson)
}
/**
* Calculates the intersection between two features.
* Returns the length if intersecting a linestring and a (multi)polygon (in meters), returns a surface area (in m²) if intersecting two (multi)polygons

4
Models/ThemeConfig/Json/LayerConfigJson.ts
View file

@ -53,6 +53,8 @@ export interface LayerConfigJson {
* source: {geoJson: "https://my.source.net/some-tile-geojson-{layer}-{z}-{x}-{y}.geojson", geoJsonZoomLevel: 14}
* to use a tiled geojson source. The web server must offer multiple geojsons. {z}, {x} and {y} are substituted by the location; {layer} is substituted with the id of the loaded layer
*
* Some API's use a BBOX instead of a tile, this can be used by specifying {y_min}, {y_max}, {x_min} and {x_max}
* Some API's use a mercator-projection (EPSG:900913) instead of WGS84. Set the flag `mercatorCrs: true` in the source for this
*
* Note that both geojson-options might set a flag 'isOsmCache' indicating that the data originally comes from OSM too
*
@ -61,7 +63,7 @@ export interface LayerConfigJson {
* While still supported, this is considered deprecated
*/
source: ({ osmTags: AndOrTagConfigJson | string, overpassScript?: string } |
{ osmTags: AndOrTagConfigJson | string, geoJson: string, geoJsonZoomLevel?: number, isOsmCache?: boolean }) & ({
{ osmTags: AndOrTagConfigJson | string, geoJson: string, geoJsonZoomLevel?: number, isOsmCache?: boolean, mercatorCrs?: boolean }) & ({
/**
* The maximum amount of seconds that a tile is allowed to linger in the cache
*/

1
Models/ThemeConfig/LayerConfig.ts
View file

@ -124,6 +124,7 @@ export default class LayerConfig {
geojsonSourceLevel: json.source["geoJsonZoomLevel"],
overpassScript: json.source["overpassScript"],
isOsmCache: json.source["isOsmCache"],
mercatorCrs: json.source["mercatorCrs"]
},
this.id
);

1
Models/ThemeConfig/LayoutConfig.ts
View file

@ -304,7 +304,6 @@ export default class LayoutConfig {
}
rewriting.forEach((value, key) => {
console.log("Rewriting", key, "==>", value)
originalJson = originalJson.replace(new RegExp(key, "g"), value)
})
return new LayoutConfig(JSON.parse(originalJson), false, "Layout rewriting")

10
Models/ThemeConfig/SourceConfig.ts
View file

@ -1,4 +1,5 @@
import {TagsFilter} from "../../Logic/Tags/TagsFilter";
import {RegexTag} from "../../Logic/Tags/RegexTag";
export default class SourceConfig {
@ -7,8 +8,10 @@ export default class SourceConfig {
public readonly geojsonSource?: string;
public readonly geojsonZoomLevel?: number;
public readonly isOsmCacheLayer: boolean;
public readonly mercatorCrs: boolean;
constructor(params: {
mercatorCrs?: boolean;
osmTags?: TagsFilter,
overpassScript?: string,
geojsonSource?: string,
@ -33,10 +36,15 @@ export default class SourceConfig {
console.error(params)
throw `Source said it is a OSM-cached layer, but didn't define the actual source of the cache (in context ${context})`
}
this.osmTags = params.osmTags;
if(params.geojsonSource !== undefined && params.geojsonSourceLevel !== undefined){
if(! ["x","y","x_min","x_max","y_min","Y_max"].some(toSearch => params.geojsonSource.indexOf(toSearch) > 0)){
throw `Source defines a geojson-zoomLevel, but does not specify {x} nor {y} (or equivalent), this is probably a bug (in context ${context})`
}}
this.osmTags = params.osmTags ?? new RegexTag("id",/.*/);
this.overpassScript = params.overpassScript;
this.geojsonSource = params.geojsonSource;
this.geojsonZoomLevel = params.geojsonSourceLevel;
this.isOsmCacheLayer = params.isOsmCache ?? false;
this.mercatorCrs = params.mercatorCrs ?? false;
}
}

6
UI/Popup/FeatureInfoBox.ts
View file

@ -83,7 +83,7 @@ export default class FeatureInfoBox extends ScrollableFullScreen {
layerConfig.allowMove
);
})
)
).SetClass("text-base")
);
}
@ -94,14 +94,14 @@ export default class FeatureInfoBox extends ScrollableFullScreen {
id,
layerConfig.deletion
))
))
).SetClass("text-base"))
}
if (layerConfig.allowSplit) {
editElements.push(
new VariableUiElement(tags.map(tags => tags.id).map(id =>
new SplitRoadWizard(id))
))
).SetClass("text-base"))
}

2
UI/SpecialVisualizations.ts
View file

@ -457,7 +457,7 @@ There are also some technicalities in your theme to keep in mind:
return new Combine([new FixedUiElement("The import button is disabled for unofficial themes to prevent accidents.").SetClass("alert"),
new FixedUiElement("To test, add 'test=true' to the URL. The changeset will be printed in the console. Please open a PR to officialize this theme to actually enable the import button.")])
}
const tgsSpec = args[0].split(",").map(spec => {
const tgsSpec = args[0].split(";").map(spec => {
const kv = spec.split("=").map(s => s.trim());
if (kv.length != 2) {
throw "Invalid key spec: multiple '=' found in " + spec

1
assets/layers/public_bookcase/public_bookcase.json
View file

@ -433,6 +433,7 @@
"id": "public_bookcase-website"
}
],
"allowMove": true,
"deletion": {
"softDeletionTags": {
"and": [

20
assets/themes/grb_import/README.md Normal file
View file

@ -0,0 +1,20 @@
GRB Import helper
===================
Preparing the CRAB dataset
--------------------------
````
# The original data is downloaded from https://download.vlaanderen.be/Producten/Detail?id=447&title=CRAB_Adressenlijst# (the GML-file here )
wget https://downloadagiv.blob.core.windows.net/crab-adressenlijst/GML/CRAB_Adressenlijst_GML.zip
# Extract the zip file
unzip CRAB_Adressenlijst_GML.zip
# convert the pesky GML file into geojson
ogr2ogr -progress -t_srs WGS84 -f \"GeoJson\" CRAB.geojson CrabAdr.gml
# When done, this big file is sliced into tiles with the slicer script
node --max_old_space_size=8000 $(which ts-node) ~/git/MapComplete/scripts/slice.ts CRAB.geojson 18 ~/git/pietervdvn.github.io/CRAB_2021_10_26
````

39
assets/themes/grb.json → assets/themes/grb_import/grb.json
View file

@ -22,7 +22,7 @@
"socialImage": "",
"layers": [
{
"id": "grb-fixmes",
"id": "osm-fixmes",
"name": {
"nl": "Fixmes op gebouwen"
},
@ -198,6 +198,43 @@
},
"wayHandling": 2,
"presets": []
},
{
"id": "crab-addresses 2021-10-26",
"source": {
"osmTags": "HUISNR~*",
"geoJson": "https://raw.githubusercontent.com/pietervdvn/pietervdvn.github.io/master/CRAB_2021_10_26/tile_{z}_{x}_{y}.geojson",
"#geoJson": "https://pietervdvn.github.io/CRAB_2021_10_26/tile_{z}_{x}_{y}.geojson",
"geoJsonZoomLevel": 18,
"maxCacheAge": 0
},
"minzoom": 19,
"name": "CRAB-addressen",
"title": "CRAB-adres",
"icon": "circle:#bb3322",
"iconSize": "15,15,center",
"tagRenderings": [
"all_tags",
{
"id": "import-button",
"render": "{import_button(addr:street=$STRAATNM; addr:housenumber=$HUISNR)}"
}
]
},
{
"id": "GRB",
"source": {
"geoJson": "https://betadata.grbosm.site/grb?bbox={x_min},{y_min},{x_max},{y_max}",
"geoJsonZoomLevel": 18,
"mercatorCrs": true,
"maxCacheAge": 0
},
"name": "GRB geometries",
"title": "GRB outline",
"minzoom": 19,
"tagRenderings": [
"all_tags"
]
}
],
"hideFromOverview": true,

2
index.ts
View file

@ -33,8 +33,6 @@ if (location.href.startsWith("http://buurtnatuur.be")) {
class Init {
public static Init(layoutToUse: LayoutConfig, encoded: string) {
if(layoutToUse === null){

114
scripts/slice.ts
View file

@ -4,11 +4,84 @@ import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSou
import * as readline from "readline";
import ScriptUtils from "./ScriptUtils";
async function readFeaturesFromLineDelimitedJsonFile(inputFile: string): Promise<any[]> {
const fileStream = fs.createReadStream(inputFile);
const rl = readline.createInterface({
input: fileStream,
crlfDelay: Infinity
});
// Note: we use the crlfDelay option to recognize all instances of CR LF
// ('\r\n') in input.txt as a single line break.
const allFeatures: any[] = []
// @ts-ignore
for await (const line of rl) {
try {
allFeatures.push(JSON.parse(line))
} catch (e) {
console.error("Could not parse", line)
break
}
if (allFeatures.length % 10000 === 0) {
ScriptUtils.erasableLog("Loaded ", allFeatures.length, "features up till now")
}
}
return allFeatures
}
async function readGeojsonLineByLine(inputFile: string): Promise<any[]> {
const fileStream = fs.createReadStream(inputFile);
const rl = readline.createInterface({
input: fileStream,
crlfDelay: Infinity
});
// Note: we use the crlfDelay option to recognize all instances of CR LF
// ('\r\n') in input.txt as a single line break.
const allFeatures: any[] = []
let featuresSeen = false
// @ts-ignore
for await (let line: string of rl) {
if (!featuresSeen && line.startsWith("\"features\":")) {
featuresSeen = true;
continue;
}
if (!featuresSeen) {
continue
}
if (line.endsWith(",")) {
line = line.substring(0, line.length - 1)
}
try {
allFeatures.push(JSON.parse(line))
} catch (e) {
console.error("Could not parse", line)
break
}
if (allFeatures.length % 10000 === 0) {
ScriptUtils.erasableLog("Loaded ", allFeatures.length, "features up till now")
}
}
return allFeatures
}
async function readFeaturesFromGeoJson(inputFile: string): Promise<any[]> {
try {
return JSON.parse(fs.readFileSync(inputFile, "UTF-8")).features
} catch (e) {
// We retry, but with a line-by-line approach
return await readGeojsonLineByLine(inputFile)
}
}
async function main(args: string[]) {
console.log("GeoJSON slicer")
if (args.length < 3) {
console.log("USAGE: <input-file.line-delimited-geojson> <target-zoom-level> <output-directory>")
console.log("USAGE: <input-file.geojson> <target-zoom-level> <output-directory>")
return
}
@ -23,37 +96,22 @@ async function main(args: string[]) {
console.log("Using directory ", outputDirectory)
const fileStream = fs.createReadStream(inputFile);
let allFeatures: any [];
if (inputFile.endsWith(".geojson")) {
allFeatures = await readFeaturesFromGeoJson(inputFile)
} else {
allFeatures = await readFeaturesFromLineDelimitedJsonFile(inputFile)
}
const rl = readline.createInterface({
input: fileStream,
crlfDelay: Infinity
});
// Note: we use the crlfDelay option to recognize all instances of CR LF
// ('\r\n') in input.txt as a single line break.
const allFeatures = []
// @ts-ignore
for await (const line of rl) {
// Each line in input.txt will be successively available here as `line`.
try{
allFeatures.push(JSON.parse(line))
}catch (e) {
console.error("Could not parse", line)
break
}
if(allFeatures.length % 10000 === 0){
ScriptUtils.erasableLog("Loaded ", allFeatures.length, "features up till now")
}
}
console.log("Loaded all", allFeatures.length, "points")
const keysToRemove = ["ID","STRAATNMID","NISCODE","GEMEENTE","POSTCODE","HERKOMST","APPTNR"]
const keysToRemove = ["ID", "STRAATNMID", "NISCODE", "GEMEENTE", "POSTCODE", "HERKOMST"]
for (const f of allFeatures) {
for (const keyToRm of keysToRemove) {
delete f.properties[keyToRm]
}
delete f.bbox
}
//const knownKeys = Utils.Dedup([].concat(...allFeatures.map(f => Object.keys(f.properties))))
@ -67,11 +125,15 @@ async function main(args: string[]) {
maxFeatureCount: Number.MAX_VALUE,
registerTile: tile => {
const path = `${outputDirectory}/tile_${tile.z}_${tile.x}_${tile.y}.geojson`
const features = tile.features.data.map(ff => ff.feature)
features.forEach(f => {
delete f.bbox
})
fs.writeFileSync(path, JSON.stringify({
"type": "FeatureCollection",
"features": tile.features.data.map(ff => ff.feature)
"features": features
}, null, " "))
console.log("Written ", path, "which has ", tile.features.data.length, "features")
ScriptUtils.erasableLog("Written ", path, "which has ", tile.features.data.length, "features")
}
}
)