From c98891d4bd67556b2e327fe239d09f92d4dcc56a Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Mon, 15 Jan 2024 15:23:40 +0100 Subject: [PATCH 01/61] Server: add first docs and experiments --- Docs/SettingUpPSQL.md | 11 ++++++++++ src/UI/Test.svelte | 47 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 Docs/SettingUpPSQL.md diff --git a/Docs/SettingUpPSQL.md b/Docs/SettingUpPSQL.md new file mode 100644 index 000000000..67c2ddeb2 --- /dev/null +++ b/Docs/SettingUpPSQL.md @@ -0,0 +1,11 @@ +sudo docker run --name some-postgis -e POSTGRES_PASSWORD=none -e POSTGRES_USER=user -d -p 5444:5432 -v /home/pietervdvn/data/pgsql/:/var/lib/postgresql/data postgis/postgis + +-> Via PGAdmin een database maken en: +1) Postgis activeren (rechtsklikken > Create > extension) +2) HStore activeren + +Installeer osm2pgsql (hint: compile from source is painless) + +pg_tileserv kan hier gedownload worden: https://github.com/CrunchyData/pg_tileserv + +DATABASE_URL=postgresql://user:none@localhost:5444/osm-poi ./pg_tileserv diff --git a/src/UI/Test.svelte b/src/UI/Test.svelte index c4d5d1823..91b4aa306 100644 --- a/src/UI/Test.svelte +++ b/src/UI/Test.svelte @@ -1,7 +1,50 @@ - +
+ +
From fb088059a5bddde88d0e7b6212720b4a0419e7fe Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Wed, 17 Jan 2024 02:30:13 +0100 Subject: [PATCH 02/61] Layerserver: improve docs, add stub of script that generates configuration file --- Docs/SettingUpPSQL.md | 43 +++++++- scripts/osm2pgsql/generateLayerFile.ts | 136 +++++++++++++++++++++++++ src/UI/Test.svelte | 20 ++++ 3 files changed, 194 insertions(+), 5 deletions(-) create mode 100644 scripts/osm2pgsql/generateLayerFile.ts diff --git a/Docs/SettingUpPSQL.md b/Docs/SettingUpPSQL.md index 67c2ddeb2..9c9e5e6f3 100644 --- a/Docs/SettingUpPSQL.md +++ b/Docs/SettingUpPSQL.md @@ -1,11 +1,44 @@ -sudo docker run --name some-postgis -e POSTGRES_PASSWORD=none -e POSTGRES_USER=user -d -p 5444:5432 -v /home/pietervdvn/data/pgsql/:/var/lib/postgresql/data postgis/postgis +# Setting up a synced OSM-server for quick layer access --> Via PGAdmin een database maken en: -1) Postgis activeren (rechtsklikken > Create > extension) -2) HStore activeren +## Setting up the SQL-server: -Installeer osm2pgsql (hint: compile from source is painless) +`sudo docker run --name some-postgis -e POSTGRES_PASSWORD=password -e POSTGRES_USER=user -d -p 5444:5432 -v /home/pietervdvn/data/pgsql/:/var/lib/postgresql/data postgis/postgis` + +Then, connect to this databank with PGAdmin, create a database within it. +Then activate following extensions for this database (right click > Create > Extension): + +- Postgis activeren (rechtsklikken > Create > extension) +- HStore activeren + +Install osm2pgsql (hint: compile from source is painless) pg_tileserv kan hier gedownload worden: https://github.com/CrunchyData/pg_tileserv DATABASE_URL=postgresql://user:none@localhost:5444/osm-poi ./pg_tileserv + +## Create export scripts for every layer + +Use scripts/osm2pgsl + +## Importing data + +To seed the database: + +```` +osm2pgsql -O flex -S drinking_water.lua -s --flat-nodes=import-help-file -d postgresql://user:none@localhost:5444/osm-poi andorra-latest.osm.pbf +```` + +## Deploying a tile server + +```` +export DATABASE_URL=postgresql://user:none@localhost:5444/osm-poi +./pg_tileserv +```` + +Tiles are available at: +```` +map.addSource("drinking_water", { +"type": "vector", +"tiles": ["http://127.0.0.2:7800/public.drinking_water/{z}/{x}/{y}.pbf"] // http://127.0.0.2:7800/public.drinking_water.json", +}) +```` diff --git a/scripts/osm2pgsql/generateLayerFile.ts b/scripts/osm2pgsql/generateLayerFile.ts new file mode 100644 index 000000000..670918a89 --- /dev/null +++ b/scripts/osm2pgsql/generateLayerFile.ts @@ -0,0 +1,136 @@ +import LayerConfig from "../../src/Models/ThemeConfig/LayerConfig" +import { TagsFilter } from "../../src/Logic/Tags/TagsFilter" +import { Tag } from "../../src/Logic/Tags/Tag" +import { And } from "../../src/Logic/Tags/And" +import Script from "../Script" +import { AllSharedLayers } from "../../src/Customizations/AllSharedLayers" +import fs from "fs" +import { Or } from "../../src/Logic/Tags/Or" +import { RegexTag } from "../../src/Logic/Tags/RegexTag" + +class LuaSnippets{ + /** + * The main piece of code that calls `process_poi` + */ + static tail = [ + "function osm2pgsql.process_node(object)", + " process_poi(object, object:as_point())", + "end", + "", + "function osm2pgsql.process_way(object)", + " if object.is_closed then", + " process_poi(object, object:as_polygon():centroid())", + " end", + "end", + ""].join("\n") + + public static combine(calls: string[]): string{ + return [ + `function process_poi(object, geom)`, + ...calls.map(c => " "+c+"(object, geom)"), + `end`, + ].join("\n") + } +} +class GenerateLayerLua { + private readonly _layer: LayerConfig + + constructor(layer: LayerConfig) { + this._layer = layer + } + public functionName(){ + const l = this._layer + return `process_poi_${l.id}` + } + + public generateFunction(): string { + const l = this._layer + return [ + `local pois_${l.id} = osm2pgsql.define_table({`, + ` name = '${l.id}',`, + " ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' },", + " columns = {", + " { column = 'tags', type = 'jsonb' },", + " { column = 'geom', type = 'point', not_null = true },", + " }" + + "})", + "", + "", + `function ${this.functionName()}(object, geom)`, + " local matches_filter = " + this.toLuaFilter(l.source.osmTags), + " if( not matches_filter) then", + " return", + " end", + " local a = {", + " geom = geom,", + " tags = object.tags", + " }", + " ", + ` pois_${l.id}:insert(a)`, + "end", + "" + ].join("\n") + } + + + private toLuaFilter(tag: TagsFilter, useParens: boolean = false): string { + if (tag instanceof Tag) { + return `object.tags["${tag.key}"] == "${tag.value}"` + } + if (tag instanceof And) { + const expr = tag.and.map(t => this.toLuaFilter(t, true)).join(" and ") + if (useParens) { + return "(" + expr + ")" + } + return expr + } + if (tag instanceof Or) { + const expr = tag.or.map(t => this.toLuaFilter(t, true)).join(" or ") + if (useParens) { + return "(" + expr + ")" + } + return expr + } + if (tag instanceof RegexTag) { + if(typeof tag.value === "string" && tag.invert){ + return `object.tags["${tag.key}"] ~= "${tag.value}"` + } + + let expr = `not string.find(object.tags["${tag.key}"], "${tag.value}")` + if (!tag.invert) { + expr = "not " + expr + } + if (useParens) { + expr = "(" + expr + ")" + } + return expr + } + let msg = "Could not handle" + tag.asHumanString(false, false, {}) + console.error(msg) + throw msg + } +} + +class GenerateLayerFile extends Script { + constructor() { + super("Generates a .lua-file to use with osm2pgsql") + } + + async main(args: string[]) { + let dw = AllSharedLayers.sharedLayers.get("drinking_water") + let t = AllSharedLayers.sharedLayers.get("toilet") + + const generators = [dw, t].map(l => new GenerateLayerLua(l)) + + const script = [ + ...generators.map(g => g.generateFunction()), + LuaSnippets.combine(generators.map(g => g.functionName())), + LuaSnippets.tail + ].join("\n\n\n") + const path = "build_db.lua" + fs.writeFileSync(path,script, "utf-8") + console.log("Written", path) + } +} + +new GenerateLayerFile().run() diff --git a/src/UI/Test.svelte b/src/UI/Test.svelte index 91b4aa306..5e13292b1 100644 --- a/src/UI/Test.svelte +++ b/src/UI/Test.svelte @@ -36,6 +36,26 @@ }, ) + map.addSource("toilet", { + "type": "vector", + "tiles": ["http://127.0.0.2:7800/public.toilet/{z}/{x}/{y}.pbf"] // http://127.0.0.2:7800/public.drinking_water.json", + }) + + map.addLayer( + { + "id": "toilet_layer", + "type": "circle", + "source": "toilet", + "source-layer": "public.toilet", + "paint": { + "circle-radius": 5, + "circle-color": "#0000ff", + "circle-stroke-width": 2, + "circle-stroke-color": "#000000", + }, + }, + ) + map.on('click', 'drinking_water_layer', (e) => { // Copy coordinates array. console.log(e) From 35228daa8fa2511ecb0407111431a4390006f782 Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Fri, 19 Jan 2024 02:00:05 +0100 Subject: [PATCH 03/61] Remove obsolete code --- src/Logic/FeatureSource/Sources/GeoJsonSource.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/Logic/FeatureSource/Sources/GeoJsonSource.ts b/src/Logic/FeatureSource/Sources/GeoJsonSource.ts index 998f88310..22bd6979a 100644 --- a/src/Logic/FeatureSource/Sources/GeoJsonSource.ts +++ b/src/Logic/FeatureSource/Sources/GeoJsonSource.ts @@ -103,7 +103,6 @@ export default class GeoJsonSource implements FeatureSource { const time = new Date() const newFeatures: Feature[] = [] let i = 0 - let skipped = 0 for (const feature of json.features) { if (feature.geometry.type === "Point") { // See https://github.com/maproulette/maproulette-backend/issues/242 @@ -131,16 +130,9 @@ export default class GeoJsonSource implements FeatureSource { i++ } if (self.seenids.has(props.id)) { - skipped++ continue } self.seenids.add(props.id) - - let freshness: Date = time - if (feature.properties["_last_edit:timestamp"] !== undefined) { - freshness = new Date(props["_last_edit:timestamp"]) - } - newFeatures.push(feature) } From ef2f1487c6dee52f52e81319b7f9f3bf9094bcce Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Mon, 22 Jan 2024 01:42:05 +0100 Subject: [PATCH 04/61] LayerServer: first version which can use a local MVT-server --- Docs/SettingUpPSQL.md | 4 +- assets/layers/shower/shower.json | 2 +- assets/layers/toilet/toilet.json | 2 +- assets/themes/toilets/toilets.json | 2 +- package-lock.json | 386 ++++++++++++++++++ package.json | 4 + scripts/osm2pgsql/generateLayerFile.ts | 16 +- scripts/osm2pgsql/tilecountServer.ts | 44 ++ .../FeatureSource/Sources/LayoutSource.ts | 22 +- src/Logic/FeatureSource/Sources/MvtSource.ts | 378 +++++++++++++++++ .../DynamicMvtTileSource.ts | 39 ++ .../TiledFeatureSource/DynamicTileSource.ts | 9 +- .../LocalStorageFeatureSource.ts | 4 +- src/Models/Constants.ts | 5 + .../ThemeConfig/Conversion/PrepareTheme.ts | 1 - src/UI/Map/ShowDataLayer.ts | 1 + src/UI/Test.svelte | 172 +++++--- 17 files changed, 1009 insertions(+), 82 deletions(-) create mode 100644 scripts/osm2pgsql/tilecountServer.ts create mode 100644 src/Logic/FeatureSource/Sources/MvtSource.ts create mode 100644 src/Logic/FeatureSource/TiledFeatureSource/DynamicMvtTileSource.ts diff --git a/Docs/SettingUpPSQL.md b/Docs/SettingUpPSQL.md index 9c9e5e6f3..c83ddb662 100644 --- a/Docs/SettingUpPSQL.md +++ b/Docs/SettingUpPSQL.md @@ -25,13 +25,13 @@ Use scripts/osm2pgsl To seed the database: ```` -osm2pgsql -O flex -S drinking_water.lua -s --flat-nodes=import-help-file -d postgresql://user:none@localhost:5444/osm-poi andorra-latest.osm.pbf +osm2pgsql -O flex -E 4326 -S build_db.lua -s --flat-nodes=import-help-file -d postgresql://user:password@localhost:5444/osm-poi andorra-latest.osm.pbf ```` ## Deploying a tile server ```` -export DATABASE_URL=postgresql://user:none@localhost:5444/osm-poi +export DATABASE_URL=postgresql://user:password@localhost:5444/osm-poi ./pg_tileserv ```` diff --git a/assets/layers/shower/shower.json b/assets/layers/shower/shower.json index 759004606..50fb25ed9 100644 --- a/assets/layers/shower/shower.json +++ b/assets/layers/shower/shower.json @@ -17,7 +17,7 @@ "source": { "osmTags": "amenity=shower" }, - "minzoom": 12, + "minzoom": 8, "title": { "render": { "en": "Shower", diff --git a/assets/layers/toilet/toilet.json b/assets/layers/toilet/toilet.json index 068ba43aa..2d58241b6 100644 --- a/assets/layers/toilet/toilet.json +++ b/assets/layers/toilet/toilet.json @@ -26,7 +26,7 @@ "source": { "osmTags": "amenity=toilets" }, - "minzoom": 12, + "minzoom": 9, "title": { "render": { "en": "Toilet", diff --git a/assets/themes/toilets/toilets.json b/assets/themes/toilets/toilets.json index 3879b0385..87afd0068 100644 --- a/assets/themes/toilets/toilets.json +++ b/assets/themes/toilets/toilets.json @@ -44,4 +44,4 @@ "shower" ], "widenFactor": 3 -} \ No newline at end of file +} diff --git a/package-lock.json b/package-lock.json index 92b0f86d2..2fa881903 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "@turf/length": "^6.5.0", "@turf/turf": "^6.5.0", "@types/dompurify": "^3.0.2", + "@types/pg": "^8.10.9", "@types/qrcode-generator": "^1.0.6", "@types/showdown": "^2.0.0", "chart.js": "^3.8.0", @@ -50,6 +51,8 @@ "osmtogeojson": "^3.0.0-beta.5", "panzoom": "^9.4.3", "papaparse": "^5.3.1", + "pbf": "^3.2.1", + "pg": "^8.11.3", "pic4carto": "^2.1.15", "prompt-sync": "^4.2.0", "qrcode-generator": "^1.4.4", @@ -4223,6 +4226,68 @@ "resolved": "https://registry.npmjs.org/@types/pbf/-/pbf-3.0.3.tgz", "integrity": "sha512-hw6bDMjvm+QTvEC+pRLpnTknQXoPu8Fnf+A+zX9HB7j/7RfYajFSbdukabo3adPwvvEHhIMafQl0R0Tpej7clQ==" }, + "node_modules/@types/pg": { + "version": "8.10.9", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.10.9.tgz", + "integrity": "sha512-UksbANNE/f8w0wOMxVKKIrLCbEMV+oM1uKejmwXr39olg4xqcfBDbXxObJAt6XxHbDa4XTKOlUEcEltXDX+XLQ==", + "dependencies": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^4.0.1" + } + }, + "node_modules/@types/pg/node_modules/pg-types": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-4.0.1.tgz", + "integrity": "sha512-hRCSDuLII9/LE3smys1hRHcu5QGcLs9ggT7I/TCs0IE+2Eesxi9+9RWAAwZ0yaGjxoWICF/YHLOEjydGujoJ+g==", + "dependencies": { + "pg-int8": "1.0.1", + "pg-numeric": "1.0.2", + "postgres-array": "~3.0.1", + "postgres-bytea": "~3.0.0", + "postgres-date": "~2.0.1", + "postgres-interval": "^3.0.0", + "postgres-range": "^1.1.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@types/pg/node_modules/postgres-array": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-3.0.2.tgz", + "integrity": "sha512-6faShkdFugNQCLwucjPcY5ARoW1SlbnrZjmGl0IrrqewpvxvhSLHimCVzqeuULCbG0fQv7Dtk1yDbG3xv7Veog==", + "engines": { + "node": ">=12" + } + }, + "node_modules/@types/pg/node_modules/postgres-bytea": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-3.0.0.tgz", + "integrity": "sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw==", + "dependencies": { + "obuf": "~1.1.2" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/@types/pg/node_modules/postgres-date": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-2.0.1.tgz", + "integrity": "sha512-YtMKdsDt5Ojv1wQRvUhnyDJNSr2dGIC96mQVKz7xufp07nfuFONzdaowrMHjlAzY6GDLd4f+LUHHAAM1h4MdUw==", + "engines": { + "node": ">=12" + } + }, + "node_modules/@types/pg/node_modules/postgres-interval": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-3.0.0.tgz", + "integrity": "sha512-BSNDnbyZCXSxgA+1f5UU2GmwhoI0aU5yMxRGO8CdFEcY2BQF9xm/7MqKnYoM1nJDk8nONNWDk9WeSmePFhQdlw==", + "engines": { + "node": ">=12" + } + }, "node_modules/@types/prompt-sync": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/@types/prompt-sync/-/prompt-sync-4.2.0.tgz", @@ -5277,6 +5342,14 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, + "node_modules/buffer-writer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==", + "engines": { + "node": ">=4" + } + }, "node_modules/bytewise": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/bytewise/-/bytewise-1.1.0.tgz", @@ -9606,6 +9679,11 @@ "node": ">= 0.4" } }, + "node_modules/obuf": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/obuf/-/obuf-1.1.2.tgz", + "integrity": "sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==" + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -9845,6 +9923,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/packet-reader": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" + }, "node_modules/panzoom": { "version": "9.4.3", "resolved": "https://registry.npmjs.org/panzoom/-/panzoom-9.4.3.tgz", @@ -9954,6 +10037,97 @@ "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==" }, + "node_modules/pg": { + "version": "8.11.3", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.3.tgz", + "integrity": "sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==", + "dependencies": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-connection-string": "^2.6.2", + "pg-pool": "^3.6.1", + "pg-protocol": "^1.6.0", + "pg-types": "^2.1.0", + "pgpass": "1.x" + }, + "engines": { + "node": ">= 8.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.1.1" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz", + "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.2.tgz", + "integrity": "sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-numeric": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pg-numeric/-/pg-numeric-1.0.2.tgz", + "integrity": "sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==", + "engines": { + "node": ">=4" + } + }, + "node_modules/pg-pool": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.1.tgz", + "integrity": "sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.0.tgz", + "integrity": "sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "dependencies": { + "split2": "^4.1.0" + } + }, "node_modules/pic4carto": { "version": "2.1.15", "resolved": "https://registry.npmjs.org/pic4carto/-/pic4carto-2.1.15.tgz", @@ -10138,6 +10312,46 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-range": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/postgres-range/-/postgres-range-1.1.3.tgz", + "integrity": "sha512-VdlZoocy5lCP0c/t66xAfclglEapXPCIVhqqJRncYpvbCgImF0w67aPKfbqUMr72tO2k5q0TdTZwCLjPTI6C9g==" + }, "node_modules/potpack": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/potpack/-/potpack-2.0.0.tgz", @@ -11461,6 +11675,14 @@ "node": ">=0.10.0" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/sshpk": { "version": "1.17.0", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.17.0.tgz", @@ -16835,6 +17057,55 @@ "resolved": "https://registry.npmjs.org/@types/pbf/-/pbf-3.0.3.tgz", "integrity": "sha512-hw6bDMjvm+QTvEC+pRLpnTknQXoPu8Fnf+A+zX9HB7j/7RfYajFSbdukabo3adPwvvEHhIMafQl0R0Tpej7clQ==" }, + "@types/pg": { + "version": "8.10.9", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.10.9.tgz", + "integrity": "sha512-UksbANNE/f8w0wOMxVKKIrLCbEMV+oM1uKejmwXr39olg4xqcfBDbXxObJAt6XxHbDa4XTKOlUEcEltXDX+XLQ==", + "requires": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^4.0.1" + }, + "dependencies": { + "pg-types": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-4.0.1.tgz", + "integrity": "sha512-hRCSDuLII9/LE3smys1hRHcu5QGcLs9ggT7I/TCs0IE+2Eesxi9+9RWAAwZ0yaGjxoWICF/YHLOEjydGujoJ+g==", + "requires": { + "pg-int8": "1.0.1", + "pg-numeric": "1.0.2", + "postgres-array": "~3.0.1", + "postgres-bytea": "~3.0.0", + "postgres-date": "~2.0.1", + "postgres-interval": "^3.0.0", + "postgres-range": "^1.1.1" + } + }, + "postgres-array": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-3.0.2.tgz", + "integrity": "sha512-6faShkdFugNQCLwucjPcY5ARoW1SlbnrZjmGl0IrrqewpvxvhSLHimCVzqeuULCbG0fQv7Dtk1yDbG3xv7Veog==" + }, + "postgres-bytea": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-3.0.0.tgz", + "integrity": "sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw==", + "requires": { + "obuf": "~1.1.2" + } + }, + "postgres-date": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-2.0.1.tgz", + "integrity": "sha512-YtMKdsDt5Ojv1wQRvUhnyDJNSr2dGIC96mQVKz7xufp07nfuFONzdaowrMHjlAzY6GDLd4f+LUHHAAM1h4MdUw==" + }, + "postgres-interval": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-3.0.0.tgz", + "integrity": "sha512-BSNDnbyZCXSxgA+1f5UU2GmwhoI0aU5yMxRGO8CdFEcY2BQF9xm/7MqKnYoM1nJDk8nONNWDk9WeSmePFhQdlw==" + } + } + }, "@types/prompt-sync": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/@types/prompt-sync/-/prompt-sync-4.2.0.tgz", @@ -17608,6 +17879,11 @@ "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" }, + "buffer-writer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==" + }, "bytewise": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/bytewise/-/bytewise-1.1.0.tgz", @@ -20874,6 +21150,11 @@ "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" }, + "obuf": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/obuf/-/obuf-1.1.2.tgz", + "integrity": "sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==" + }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -21049,6 +21330,11 @@ "p-limit": "^3.0.2" } }, + "packet-reader": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" + }, "panzoom": { "version": "9.4.3", "resolved": "https://registry.npmjs.org/panzoom/-/panzoom-9.4.3.tgz", @@ -21134,6 +21420,73 @@ "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", "integrity": "sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==" }, + "pg": { + "version": "8.11.3", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.3.tgz", + "integrity": "sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==", + "requires": { + "buffer-writer": "2.0.0", + "packet-reader": "1.0.0", + "pg-cloudflare": "^1.1.1", + "pg-connection-string": "^2.6.2", + "pg-pool": "^3.6.1", + "pg-protocol": "^1.6.0", + "pg-types": "^2.1.0", + "pgpass": "1.x" + } + }, + "pg-cloudflare": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz", + "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", + "optional": true + }, + "pg-connection-string": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.2.tgz", + "integrity": "sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==" + }, + "pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==" + }, + "pg-numeric": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pg-numeric/-/pg-numeric-1.0.2.tgz", + "integrity": "sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==" + }, + "pg-pool": { + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.1.tgz", + "integrity": "sha512-jizsIzhkIitxCGfPRzJn1ZdcosIt3pz9Sh3V01fm1vZnbnCMgmGl5wvGGdNN2EL9Rmb0EcFoCkixH4Pu+sP9Og==", + "requires": {} + }, + "pg-protocol": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.0.tgz", + "integrity": "sha512-M+PDm637OY5WM307051+bsDia5Xej6d9IR4GwJse1qA1DIhiKlksvrneZOYQq42OM+spubpcNYEo2FcKQrDk+Q==" + }, + "pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "requires": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + } + }, + "pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "requires": { + "split2": "^4.1.0" + } + }, "pic4carto": { "version": "2.1.15", "resolved": "https://registry.npmjs.org/pic4carto/-/pic4carto-2.1.15.tgz", @@ -21238,6 +21591,34 @@ "util-deprecate": "^1.0.2" } }, + "postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==" + }, + "postgres-bytea": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", + "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==" + }, + "postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==" + }, + "postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "requires": { + "xtend": "^4.0.0" + } + }, + "postgres-range": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/postgres-range/-/postgres-range-1.1.3.tgz", + "integrity": "sha512-VdlZoocy5lCP0c/t66xAfclglEapXPCIVhqqJRncYpvbCgImF0w67aPKfbqUMr72tO2k5q0TdTZwCLjPTI6C9g==" + }, "potpack": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/potpack/-/potpack-2.0.0.tgz", @@ -22195,6 +22576,11 @@ } } }, + "split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==" + }, "sshpk": { "version": "1.17.0", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.17.0.tgz", diff --git a/package.json b/package.json index 71f0b38be..5f961226b 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "oauth_secret": "NBWGhWDrD3QDB35xtVuxv4aExnmIt4FA_WgeLtwxasg", "url": "https://www.openstreetmap.org" }, + "mvt_layer_server": "http://127.0.0.1:7800/public.{layer}/{z}/{x}/{y}.pbf", "disabled:oauth_credentials": { "##": "DEV", "#": "This client-id is registered by 'MapComplete' on https://master.apis.dev.openstreetmap.org/", @@ -118,6 +119,7 @@ "@turf/length": "^6.5.0", "@turf/turf": "^6.5.0", "@types/dompurify": "^3.0.2", + "@types/pg": "^8.10.9", "@types/qrcode-generator": "^1.0.6", "@types/showdown": "^2.0.0", "chart.js": "^3.8.0", @@ -148,6 +150,8 @@ "osmtogeojson": "^3.0.0-beta.5", "panzoom": "^9.4.3", "papaparse": "^5.3.1", + "pbf": "^3.2.1", + "pg": "^8.11.3", "pic4carto": "^2.1.15", "prompt-sync": "^4.2.0", "qrcode-generator": "^1.4.4", diff --git a/scripts/osm2pgsql/generateLayerFile.ts b/scripts/osm2pgsql/generateLayerFile.ts index 670918a89..5867b8d9f 100644 --- a/scripts/osm2pgsql/generateLayerFile.ts +++ b/scripts/osm2pgsql/generateLayerFile.ts @@ -7,6 +7,7 @@ import { AllSharedLayers } from "../../src/Customizations/AllSharedLayers" import fs from "fs" import { Or } from "../../src/Logic/Tags/Or" import { RegexTag } from "../../src/Logic/Tags/RegexTag" +import { Utils } from "../../src/Utils" class LuaSnippets{ /** @@ -40,18 +41,24 @@ class GenerateLayerLua { } public functionName(){ const l = this._layer + if(!l.source?.osmTags){ + return undefined + } return `process_poi_${l.id}` } public generateFunction(): string { const l = this._layer + if(!l.source?.osmTags){ + return undefined + } return [ `local pois_${l.id} = osm2pgsql.define_table({`, ` name = '${l.id}',`, " ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' },", " columns = {", " { column = 'tags', type = 'jsonb' },", - " { column = 'geom', type = 'point', not_null = true },", + " { column = 'geom', type = 'point', projection = 4326, not_null = true },", " }" + "})", "", @@ -117,14 +124,13 @@ class GenerateLayerFile extends Script { } async main(args: string[]) { - let dw = AllSharedLayers.sharedLayers.get("drinking_water") - let t = AllSharedLayers.sharedLayers.get("toilet") + const layerNames = Array.from(AllSharedLayers.sharedLayers.values()) - const generators = [dw, t].map(l => new GenerateLayerLua(l)) + const generators = layerNames.map(l => new GenerateLayerLua(l)) const script = [ ...generators.map(g => g.generateFunction()), - LuaSnippets.combine(generators.map(g => g.functionName())), + LuaSnippets.combine(Utils.NoNull(generators.map(g => g.functionName()))), LuaSnippets.tail ].join("\n\n\n") const path = "build_db.lua" diff --git a/scripts/osm2pgsql/tilecountServer.ts b/scripts/osm2pgsql/tilecountServer.ts new file mode 100644 index 000000000..5c33c72b7 --- /dev/null +++ b/scripts/osm2pgsql/tilecountServer.ts @@ -0,0 +1,44 @@ +import { BBox } from "../../src/Logic/BBox" +import { Client } from "pg" + +/** + * Connects with a Postgis database, gives back how much items there are within the given BBOX + */ +export default class TilecountServer { + private readonly _client: Client + private isConnected = false + + constructor(connectionString: string) { + this._client = new Client(connectionString) + } + + async getCount(layer: string, bbox: BBox = undefined): Promise { + if (!this.isConnected) { + await this._client.connect() + this.isConnected = true + } + + let query = "SELECT COUNT(*) FROM " + layer + + if(bbox){ + query += ` WHERE ST_MakeEnvelope (${bbox.minLon}, ${bbox.minLat}, ${bbox.maxLon}, ${bbox.maxLat}, 4326) ~ geom` + } +console.log(query) + const result = await this._client.query(query) + return result.rows[0].count + } + + disconnect() { + this._client.end() + } +} + +const tcs = new TilecountServer("postgresql://user:none@localhost:5444/osm-poi") +console.log(">>>", await tcs.getCount("drinking_water", new BBox([ + [1.5052013991654007, + 42.57480750272123, + ], [ + 1.6663677350703097, + 42.499856652770745, + ]]))) +tcs.disconnect() diff --git a/src/Logic/FeatureSource/Sources/LayoutSource.ts b/src/Logic/FeatureSource/Sources/LayoutSource.ts index fa430f561..a46d81384 100644 --- a/src/Logic/FeatureSource/Sources/LayoutSource.ts +++ b/src/Logic/FeatureSource/Sources/LayoutSource.ts @@ -11,6 +11,9 @@ import DynamicGeoJsonTileSource from "../TiledFeatureSource/DynamicGeoJsonTileSo import { BBox } from "../../BBox" import LocalStorageFeatureSource from "../TiledFeatureSource/LocalStorageFeatureSource" import FullNodeDatabaseSource from "../TiledFeatureSource/FullNodeDatabaseSource" +import { Features } from "@rgossiaux/svelte-headlessui/types" +import DynamicMvtileSource from "../TiledFeatureSource/DynamicMvtTileSource" +import { layouts } from "chart.js" /** * This source will fetch the needed data from various sources for the given layout. @@ -44,14 +47,18 @@ export default class LayoutSource extends FeatureSourceMerger { maxAge: l.maxAgeOfCache, }) ) + console.log(mapProperties) + const mvtSources: FeatureSource[] = osmLayers.map(l => LayoutSource.setupMvtSource(l, mapProperties, isDisplayed(l.id))) + + /* const overpassSource = LayoutSource.setupOverpass( backend, osmLayers, bounds, zoom, featureSwitches - ) + )//*/ const osmApiSource = LayoutSource.setupOsmApiSource( osmLayers, @@ -61,22 +68,27 @@ export default class LayoutSource extends FeatureSourceMerger { featureSwitches, fullNodeDatabaseSource ) + const geojsonSources: FeatureSource[] = geojsonlayers.map((l) => LayoutSource.setupGeojsonSource(l, mapProperties, isDisplayed(l.id)) ) - super(overpassSource, osmApiSource, ...geojsonSources, ...fromCache) + + super(osmApiSource, ...geojsonSources, ...fromCache, ...mvtSources) const self = this function setIsLoading() { - const loading = overpassSource?.runningQuery?.data || osmApiSource?.isRunning?.data - self._isLoading.setData(loading) + // const loading = overpassSource?.runningQuery?.data || osmApiSource?.isRunning?.data + // self._isLoading.setData(loading) } - overpassSource?.runningQuery?.addCallbackAndRun((_) => setIsLoading()) + // overpassSource?.runningQuery?.addCallbackAndRun((_) => setIsLoading()) osmApiSource?.isRunning?.addCallbackAndRun((_) => setIsLoading()) } + private static setupMvtSource(layer: LayerConfig, mapProperties: { zoom: Store; bounds: Store }, isActive?: Store): FeatureSource{ + return new DynamicMvtileSource(layer, mapProperties, { isActive }) + } private static setupGeojsonSource( layer: LayerConfig, mapProperties: { zoom: Store; bounds: Store }, diff --git a/src/Logic/FeatureSource/Sources/MvtSource.ts b/src/Logic/FeatureSource/Sources/MvtSource.ts new file mode 100644 index 000000000..8c93eee9c --- /dev/null +++ b/src/Logic/FeatureSource/Sources/MvtSource.ts @@ -0,0 +1,378 @@ +import { Feature, Geometry } from "geojson" +import { Store, UIEventSource } from "../../UIEventSource" +import { FeatureSource } from "../FeatureSource" +import Pbf from "pbf" +import * as pbfCompile from "pbf/compile" +import * as PbfSchema from "protocol-buffers-schema" + +type Coords = [number, number][] + +class MvtFeatureBuilder { + private static readonly geom_types = ["Unknown", "Point", "LineString", "Polygon"] as const + private readonly _size: number + private readonly _x0: number + private readonly _y0: number + + constructor(extent: number, x: number, y: number, z: number) { + this._size = extent * Math.pow(2, z) + this._x0 = extent * x + this._y0 = extent * y + } + + public toGeoJson(geometry, typeIndex, properties): Feature { + let coords: [number, number] | Coords | Coords[] = this.encodeGeometry(geometry) + switch (typeIndex) { + case 1: + const points = [] + for (let i = 0; i < coords.length; i++) { + points[i] = coords[i][0] + } + coords = points + this.project(coords) + break + + case 2: + for (let i = 0; i < coords.length; i++) { + this.project(coords[i]) + } + break + + case 3: + let classified = this.classifyRings(coords) + for (let i = 0; i < coords.length; i++) { + for (let j = 0; j < coords[i].length; j++) { + this.project(classified[i][j]) + } + } + break + } + + let type: string = MvtFeatureBuilder.geom_types[typeIndex] + if (coords.length === 1) { + coords = coords[0] + } else { + type = "Multi" + type + } + + return { + type: "Feature", + geometry: { + type: type, + coordinates: coords, + }, + properties, + } + } + + private encodeGeometry(geometry: number[]) { + let cX = 0 + let cY = 0 + let coordss: Coords[] = [] + let currentRing: Coords = [] + for (let i = 0; i < geometry.length; i++) { + let commandInteger = geometry[i] + let commandId = commandInteger & 0x7 + let commandCount = commandInteger >> 3 + /* + Command Id Parameters Parameter Count + MoveTo 1 dX, dY 2 + LineTo 2 dX, dY 2 + ClosePath 7 No parameters 0 + */ + if (commandId === 1) { + // MoveTo means: we start a new ring + if (currentRing.length !== 0) { + coordss.push(currentRing) + currentRing = [] + } + } + if (commandId === 1 || commandId === 2){ + for (let j = 0; j < commandCount; j++) { + const dx = geometry[i + j * 2 + 1] + cX += ((dx >> 1) ^ (-(dx & 1))) + const dy = geometry[i + j * 2 + 2] + cY += ((dy >> 1) ^ (-(dy & 1))) + currentRing.push([cX, cY]) + } + i = commandCount * 2 + } + if(commandId === 7){ + currentRing.push([...currentRing[0]]) + } + + } + if (currentRing.length > 0) { + coordss.push(currentRing) + } + return coordss + } + + private signedArea(ring: Coords): number { + let sum = 0 + const len = ring.length + // J is basically (i - 1) % len + let j = len - 1 + let p1 + let p2 + for (let i = 0; i < len; i++) { + p1 = ring[i] + p2 = ring[j] + sum += (p2.x - p1.x) * (p1.y + p2.y) + j = i + } + return sum + } + + private classifyRings(rings: Coords[]): Coords[][] { + const len = rings.length + + if (len <= 1) return [rings] + + const polygons = [] + let polygon + // CounterClockWise + let ccw: boolean + + for (let i = 0; i < len; i++) { + const area = this.signedArea(rings[i]) + if (area === 0) continue + + if (ccw === undefined) { + ccw = area < 0 + } + if (ccw === (area < 0)) { + if (polygon) { + polygons.push(polygon) + } + polygon = [rings[i]] + + } else { + polygon.push(rings[i]) + } + } + if (polygon) { + polygons.push(polygon) + } + + return polygons + } + + /** + * Inline replacement of the location by projecting + * @param line + * @private + */ + private project(line: [number, number][]) { + const y0 = this._y0 + const x0 = this._x0 + const size = this._size + for (let i = 0; i < line.length; i++) { + let p = line[i] + let y2 = 180 - (p[1] + y0) * 360 / size + line[i] = [ + (p[0] + x0) * 360 / size - 180, + 360 / Math.PI * Math.atan(Math.exp(y2 * Math.PI / 180)) - 90, + ] + } + } +} + +export default class MvtSource implements FeatureSource { + + private static readonly schemaSpec = ` + package vector_tile; + +option optimize_for = LITE_RUNTIME; + +message Tile { + + // GeomType is described in section 4.3.4 of the specification + enum GeomType { + UNKNOWN = 0; + POINT = 1; + LINESTRING = 2; + POLYGON = 3; + } + + // Variant type encoding + // The use of values is described in section 4.1 of the specification + message Value { + // Exactly one of these values must be present in a valid message + optional string string_value = 1; + optional float float_value = 2; + optional double double_value = 3; + optional int64 int_value = 4; + optional uint64 uint_value = 5; + optional sint64 sint_value = 6; + optional bool bool_value = 7; + + extensions 8 to max; + } + + // Features are described in section 4.2 of the specification + message Feature { + optional uint64 id = 1 [ default = 0 ]; + + // Tags of this feature are encoded as repeated pairs of + // integers. + // A detailed description of tags is located in sections + // 4.2 and 4.4 of the specification + repeated uint32 tags = 2 [ packed = true ]; + + // The type of geometry stored in this feature. + optional GeomType type = 3 [ default = UNKNOWN ]; + + // Contains a stream of commands and parameters (vertices). + // A detailed description on geometry encoding is located in + // section 4.3 of the specification. + repeated uint32 geometry = 4 [ packed = true ]; + } + + // Layers are described in section 4.1 of the specification + message Layer { + // Any compliant implementation must first read the version + // number encoded in this message and choose the correct + // implementation for this version number before proceeding to + // decode other parts of this message. + required uint32 version = 15 [ default = 1 ]; + + required string name = 1; + + // The actual features in this tile. + repeated Feature features = 2; + + // Dictionary encoding for keys + repeated string keys = 3; + + // Dictionary encoding for values + repeated Value values = 4; + + // Although this is an "optional" field it is required by the specification. + // See https://github.com/mapbox/vector-tile-spec/issues/47 + optional uint32 extent = 5 [ default = 4096 ]; + + extensions 16 to max; + } + + repeated Layer layers = 3; + + extensions 16 to 8191; +} +` + private static readonly tile_schema = pbfCompile(PbfSchema.parse(MvtSource.schemaSpec)).Tile + + + private readonly _url: string + private readonly _layerName: string + private readonly _features: UIEventSource[]> = new UIEventSource[]>([]) + public readonly features: Store[]> = this._features + private readonly x: number + private readonly y: number + private readonly z: number + + constructor(url: string, x: number, y: number, z: number, layerName?: string) { + this._url = url + this._layerName = layerName + this.x = x + this.y = y + this.z = z + this.downloadSync() + } + + private getValue(v: { + // Exactly one of these values must be present in a valid message + string_value?: string, + float_value?: number, + double_value?: number, + int_value?: number, + uint_value?: number, + sint_value?: number, + bool_value?: boolean + }): string | number | undefined | boolean { + if (v.string_value !== "") { + return v.string_value + } + if (v.double_value !== 0) { + return v.double_value + } + if (v.float_value !== 0) { + return v.float_value + } + if (v.int_value !== 0) { + return v.int_value + } + if (v.uint_value !== 0) { + return v.uint_value + } + if (v.sint_value !== 0) { + return v.sint_value + } + if (v.bool_value !== false) { + return v.bool_value + } + return undefined + + } + + private downloadSync(){ + this.download().then(d => { + if(d.length === 0){ + return + } + return this._features.setData(d) + }).catch(e => {console.error(e)}) + } + private async download(): Promise { + const result = await fetch(this._url) + const buffer = await result.arrayBuffer() + const data = MvtSource.tile_schema.read(new Pbf(buffer)) + const layers = data.layers + let layer = data.layers[0] + if (layers.length > 1) { + if (!this._layerName) { + throw "Multiple layers in the downloaded tile, but no layername is given to choose from" + } + layer = layers.find(l => l.name === this._layerName) + } + if(!layer){ + return [] + } + const builder = new MvtFeatureBuilder(layer.extent, this.x, this.y, this.z) + const features: Feature[] = [] + + for (const feature of layer.features) { + const properties = this.inflateProperties(feature.tags, layer.keys, layer.values) + features.push(builder.toGeoJson(feature.geometry, feature.type, properties)) + } + + return features + } + + + private inflateProperties(tags: number[], keys: string[], values: { string_value: string }[]) { + const properties = {} + for (let i = 0; i < tags.length; i += 2) { + properties[keys[tags[i]]] = this.getValue(values[tags[i + 1]]) + } + let type: string + switch (properties["osm_type"]) { + case "N": + type = "node" + break + case "W": + type = "way" + break + case "R": + type = "relation" + break + } + properties["id"] = type + "/" + properties["osm_id"] + delete properties["osm_id"] + delete properties["osm_type"] + + return properties + } + +} diff --git a/src/Logic/FeatureSource/TiledFeatureSource/DynamicMvtTileSource.ts b/src/Logic/FeatureSource/TiledFeatureSource/DynamicMvtTileSource.ts new file mode 100644 index 000000000..ac8713332 --- /dev/null +++ b/src/Logic/FeatureSource/TiledFeatureSource/DynamicMvtTileSource.ts @@ -0,0 +1,39 @@ +import { Store } from "../../UIEventSource" +import DynamicTileSource from "./DynamicTileSource" +import { Utils } from "../../../Utils" +import { BBox } from "../../BBox" +import LayerConfig from "../../../Models/ThemeConfig/LayerConfig" +import MvtSource from "../Sources/MvtSource" +import { Tiles } from "../../../Models/TileRange" +import Constants from "../../../Models/Constants" + +export default class DynamicMvtileSource extends DynamicTileSource { + + constructor( + layer: LayerConfig, + mapProperties: { + zoom: Store + bounds: Store + }, + options?: { + isActive?: Store + }, + ) { + super( + mapProperties.zoom, + layer.minzoom, + (zxy) => { + const [z, x, y] = Tiles.tile_from_index(zxy) + const url = Utils.SubstituteKeys(Constants.VectorTileServer, + { + z, x, y, layer: layer.id, + }) + return new MvtSource(url, x, y, z) + }, + mapProperties, + { + isActive: options?.isActive, + }, + ) + } +} diff --git a/src/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts b/src/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts index e8e69ee2a..3bb5affd9 100644 --- a/src/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts +++ b/src/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts @@ -10,9 +10,9 @@ import FeatureSourceMerger from "../Sources/FeatureSourceMerger" */ export default class DynamicTileSource extends FeatureSourceMerger { constructor( - zoomlevel: number, + zoomlevel: Store, minzoom: number, - constructSource: (tileIndex) => FeatureSource, + constructSource: (tileIndex: number) => FeatureSource, mapProperties: { bounds: Store zoom: Store @@ -34,8 +34,9 @@ export default class DynamicTileSource extends FeatureSourceMerger { if (mapProperties.zoom.data < minzoom) { return undefined } + const z = Math.round(zoomlevel.data) const tileRange = Tiles.TileRangeBetween( - zoomlevel, + z, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), @@ -49,7 +50,7 @@ export default class DynamicTileSource extends FeatureSourceMerger { } const needed = Tiles.MapRange(tileRange, (x, y) => - Tiles.tile_index(zoomlevel, x, y) + Tiles.tile_index(z, x, y) ).filter((i) => !loadedTiles.has(i)) if (needed.length === 0) { return undefined diff --git a/src/Logic/FeatureSource/TiledFeatureSource/LocalStorageFeatureSource.ts b/src/Logic/FeatureSource/TiledFeatureSource/LocalStorageFeatureSource.ts index eac9d0859..11ce41080 100644 --- a/src/Logic/FeatureSource/TiledFeatureSource/LocalStorageFeatureSource.ts +++ b/src/Logic/FeatureSource/TiledFeatureSource/LocalStorageFeatureSource.ts @@ -1,5 +1,5 @@ import DynamicTileSource from "./DynamicTileSource" -import { Store } from "../../UIEventSource" +import { ImmutableStore, Store } from "../../UIEventSource" import { BBox } from "../../BBox" import TileLocalStorage from "../Actors/TileLocalStorage" import { Feature } from "geojson" @@ -27,7 +27,7 @@ export default class LocalStorageFeatureSource extends DynamicTileSource { options?.maxAge ?? 24 * 60 * 60 ) super( - zoomlevel, + new ImmutableStore(zoomlevel), layer.minzoom, (tileIndex) => new StaticFeatureSource( diff --git a/src/Models/Constants.ts b/src/Models/Constants.ts index 3551d6125..0d76b4fc5 100644 --- a/src/Models/Constants.ts +++ b/src/Models/Constants.ts @@ -154,6 +154,11 @@ export default class Constants { ] as const public static readonly defaultPinIcons: string[] = Constants._defaultPinIcons + /** + * The location that the MVT-layer is hosted. + * This is a MapLibre/MapBox vector tile server which hosts vector tiles for every (official) layer + */ + public static VectorTileServer: string | undefined = Constants.config.mvt_layer_server private static isRetina(): boolean { if (Utils.runningFromConsole) { diff --git a/src/Models/ThemeConfig/Conversion/PrepareTheme.ts b/src/Models/ThemeConfig/Conversion/PrepareTheme.ts index c5a613564..d4259c342 100644 --- a/src/Models/ThemeConfig/Conversion/PrepareTheme.ts +++ b/src/Models/ThemeConfig/Conversion/PrepareTheme.ts @@ -178,7 +178,6 @@ class AddDefaultLayers extends DesugaringStep { if (v === undefined) { const msg = `Default layer ${layerName} not found. ${state.sharedLayers.size} layers are available` if (layerName === "favourite") { - context.warn(msg) continue } context.err(msg) diff --git a/src/UI/Map/ShowDataLayer.ts b/src/UI/Map/ShowDataLayer.ts index 75c7213b0..64a3cfe94 100644 --- a/src/UI/Map/ShowDataLayer.ts +++ b/src/UI/Map/ShowDataLayer.ts @@ -303,6 +303,7 @@ class LineRenderingLayer { type: "FeatureCollection", features, }, + cluster: true, promoteId: "id", }) const linelayer = this._layername + "_line" diff --git a/src/UI/Test.svelte b/src/UI/Test.svelte index 5e13292b1..70f864f1b 100644 --- a/src/UI/Test.svelte +++ b/src/UI/Test.svelte @@ -1,70 +1,122 @@
- +
From 1d6c9ec1efc86d982071e6b6da932277e83594b4 Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Tue, 23 Jan 2024 22:03:02 +0100 Subject: [PATCH 05/61] LayerServer: make all layer ids unique --- assets/themes/bag/bag.json | 22 +++---- assets/themes/cyclenodes/cyclenodes.json | 5 +- assets/themes/cyclestreets/cyclestreets.json | 2 +- assets/themes/grb/grb.json | 14 ++--- assets/themes/healthcare/healthcare.json | 2 +- .../kerbs_and_crossings.json | 3 +- assets/themes/onwheels/onwheels.json | 1 + assets/themes/openlovemap/openlovemap.json | 2 +- assets/themes/transit/transit.json | 1 + assets/themes/uk_addresses/uk_addresses.json | 2 +- assets/themes/walkingnodes/walkingnodes.json | 5 +- ...eLayerFile.ts => generateBuildDbScript.ts} | 61 +++++++++++++------ 12 files changed, 74 insertions(+), 46 deletions(-) rename scripts/osm2pgsql/{generateLayerFile.ts => generateBuildDbScript.ts} (70%) diff --git a/assets/themes/bag/bag.json b/assets/themes/bag/bag.json index 1aa324bc3..05ddd4e4a 100644 --- a/assets/themes/bag/bag.json +++ b/assets/themes/bag/bag.json @@ -45,7 +45,7 @@ "hideFromOverview": true, "layers": [ { - "id": "osm:buildings", + "id": "osm_buildings", "name": "OSM Buildings", "title": "OSM Building", "description": "Layer showing buildings that are in OpenStreetMap", @@ -147,7 +147,7 @@ ] }, { - "id": "osm:adresses", + "id": "osm_adresses", "name": "OSM Adresses", "title": "OSM Adress", "description": "Layer showing adresses that are in OpenStreetMap", @@ -185,7 +185,7 @@ ] }, { - "id": "bag:pand", + "id": "bag_pand", "name": "BAG Buildings", "title": "BAG Building", "description": { @@ -207,7 +207,7 @@ }, "minzoom": 18, "calculatedTags": [ - "_overlaps_with_buildings=overlapWith(feat)('osm:buildings').filter(f => f.feat.properties.id.indexOf('-') < 0)", + "_overlaps_with_buildings=overlapWith(feat)('osm_buildings').filter(f => f.feat.properties.id.indexOf('-') < 0)", "_overlaps_with=feat(get)('_overlaps_with_buildings').find(f => f.overlap > 1 /* square meter */ )", "_overlaps_with_properties=feat(get)('_overlaps_with')?.feat?.properties", "_overlap_percentage=Math.round(100 * (feat(get)('_overlaps_with')?.overlap / feat(get)('_overlaps_with_properties')['_surface:strict']))", @@ -228,7 +228,7 @@ "render": { "special": { "type": "import_way_button", - "targetLayer": "osm:buildings", + "targetLayer": "osm_buildings", "tags": "building=$_bag_obj:building; ref:bag=$_bag_obj:ref:bag; source=BAG; source:date=$_bag_obj:source:date; start_date=$_bag_obj:start_date", "text": { "*": "Upload this building to OpenStreetMap" @@ -258,7 +258,7 @@ }, { "if": "_overlaps_with!=", - "then": "{conflate_button(osm:buildings, building=$_bag_obj:building; ref:bag=$_bag_obj:ref:bag; source=BAG; source:date=$_bag_obj:source:date; start_date=$_bag_obj:start_date, Replace the geometry in OpenStreetMap, , _osm_obj:id)}" + "then": "{conflate_button(osm_buildings, building=$_bag_obj:building; ref:bag=$_bag_obj:ref:bag; source=BAG; source:date=$_bag_obj:source:date; start_date=$_bag_obj:start_date, Replace the geometry in OpenStreetMap, , _osm_obj:id)}" }, { "if": { @@ -268,7 +268,7 @@ "_bag_obj:in_construction=true" ] }, - "then": "{import_way_button(osm:buildings, building=$_bag_obj:building; construction=$_bag_obj:construction; ref:bag=$_bag_obj:ref:bag; source=BAG; source:date=$_bag_obj:source:date; start_date=$_bag_obj:start_date, Upload this building to OpenStreetMap)}" + "then": "{import_way_button(osm_buildings, building=$_bag_obj:building; construction=$_bag_obj:construction; ref:bag=$_bag_obj:ref:bag; source=BAG; source:date=$_bag_obj:source:date; start_date=$_bag_obj:start_date, Upload this building to OpenStreetMap)}" } ] }, @@ -348,7 +348,7 @@ }, { "id": "Overlapping building", - "render": "
The overlapping osm:buildings is a {_osm_obj:building} and covers {_overlap_percentage}% of the BAG building.
The BAG-building covers {_reverse_overlap_percentage}% of the OSM building

BAG geometry:

{minimap(21, id):height:10rem;border-radius:1rem;overflow:hidden}

OSM geometry:

{minimap(21,_osm_obj:id):height:10rem;border-radius:1rem;overflow:hidden}
", + "render": "
The overlapping osm_buildings is a {_osm_obj:building} and covers {_overlap_percentage}% of the BAG building.
The BAG-building covers {_reverse_overlap_percentage}% of the OSM building

BAG geometry:

{minimap(21, id):height:10rem;border-radius:1rem;overflow:hidden}

OSM geometry:

{minimap(21,_osm_obj:id):height:10rem;border-radius:1rem;overflow:hidden}
", "condition": "_overlaps_with!=" }, { @@ -386,7 +386,7 @@ ] }, { - "id": "bag:verblijfsobject", + "id": "bag_verblijfsobject", "name": "BAG Addresses", "title": "BAG Address", "description": "Address information from the BAG register", @@ -398,7 +398,7 @@ }, "minzoom": 18, "calculatedTags": [ - "_closed_osm_addr:=closest(feat)('osm:adresses').properties", + "_closed_osm_addr:=closest(feat)('osm_adresses').properties", "_bag_obj:addr:housenumber=`${feat.properties.huisnummer}${feat.properties.huisletter}${(feat.properties.toevoeging != '') ? '-' : ''}${feat.properties.toevoeging}`", "_bag_obj:ref:bag=Number(feat.properties.identificatie)", "_bag_obj:source:date=new Date().toISOString().split('T')[0]", @@ -411,7 +411,7 @@ "tagRenderings": [ { "id": "Import button", - "render": "{import_button(osm:adresses, addr:city=$woonplaats; addr:housenumber=$_bag_obj:addr:housenumber; addr:postcode=$postcode; addr:street=$openbare_ruimte; ref:bag=$_bag_obj:ref:bag; source=BAG; source:date=$_bag_obj:source:date, Upload this adress to OpenStreetMap)}", + "render": "{import_button(osm_adresses, addr:city=$woonplaats; addr:housenumber=$_bag_obj:addr:housenumber; addr:postcode=$postcode; addr:street=$openbare_ruimte; ref:bag=$_bag_obj:ref:bag; source=BAG; source:date=$_bag_obj:source:date, Upload this adress to OpenStreetMap)}", "condition": "_imported_osm_object_found=false" }, { diff --git a/assets/themes/cyclenodes/cyclenodes.json b/assets/themes/cyclenodes/cyclenodes.json index 9e2d3ba35..8fdd4cab1 100644 --- a/assets/themes/cyclenodes/cyclenodes.json +++ b/assets/themes/cyclenodes/cyclenodes.json @@ -32,7 +32,7 @@ "hideFromOverview": true, "layers": [ { - "id": "node2node", + "id": "node2node_bicycle", "name": { "en": "Node to node links", "de": "Knotenpunktverbindungen", @@ -126,7 +126,7 @@ "pointRendering": null }, { - "id": "node", + "id": "node_bicycle", "name": { "en": "Nodes", "de": "Knotenpunkte", @@ -321,6 +321,7 @@ ], "override": { "minzoom": 16, + "id": "bicycle_guidepost", "source": { "osmTags": { "and": [ diff --git a/assets/themes/cyclestreets/cyclestreets.json b/assets/themes/cyclestreets/cyclestreets.json index 7b8ce2fc8..affd01ce7 100644 --- a/assets/themes/cyclestreets/cyclestreets.json +++ b/assets/themes/cyclestreets/cyclestreets.json @@ -290,7 +290,7 @@ ] }, { - "id": "all_streets", + "id": "not_cyclestreets", "name": { "nl": "Alle straten", "en": "All streets", diff --git a/assets/themes/grb/grb.json b/assets/themes/grb/grb.json index c4c955660..c2b64f77c 100644 --- a/assets/themes/grb/grb.json +++ b/assets/themes/grb/grb.json @@ -29,7 +29,7 @@ "hideFromOverview": true, "layers": [ { - "id": "osm-buildings", + "id": "osm_buildings_no_points", "name": "All OSM-buildings", "source": { "osmTags": { @@ -296,7 +296,7 @@ "name": "GRB geometries", "title": "GRB outline", "calculatedTags": [ - "_overlaps_with_buildings=overlapWith(feat)('osm-buildings').filter(f => f.feat.properties.id.indexOf('-') < 0) ?? []", + "_overlaps_with_buildings=overlapWith(feat)('osm_buildings_no_points').filter(f => f.feat.properties.id.indexOf('-') < 0) ?? []", "_overlaps_with=get(feat)('_overlaps_with_buildings').find(f => f.overlap > 1 /* square meter */ )", "_osm_obj:source:ref=get(feat)('_overlaps_with')?.feat?.properties['source:geometry:ref']", "_osm_obj:id=get(feat)('_overlaps_with')?.feat?.properties?.id", @@ -319,7 +319,7 @@ "tagRenderings": [ { "id": "Import-button", - "render": "{import_way_button(osm-buildings,building=$building;man_made=$man_made; source:geometry:date=$_grb_date; source:geometry:ref=$_grb_ref; addr:street=$addr:street; addr:housenumber=$addr:housenumber; building:min_level=$_building:min_level, Upload this building to OpenStreetMap,,_is_part_of_building=true,1,_moveable=true)}", + "render": "{import_way_button(osm_buildings_no_points,building=$building;man_made=$man_made; source:geometry:date=$_grb_date; source:geometry:ref=$_grb_ref; addr:street=$addr:street; addr:housenumber=$addr:housenumber; building:min_level=$_building:min_level, Upload this building to OpenStreetMap,,_is_part_of_building=true,1,_moveable=true)}", "mappings": [ { "#": "Failsafe", @@ -371,7 +371,7 @@ "addr:housenumber!:={_osm_obj:addr:housenumber}" ] }, - "then": "{conflate_button(osm-buildings,building=$_target_building_type; source:geometry:date=$_grb_date; source:geometry:ref=$_grb_ref; addr:street=$addr:street; addr:housenumber=$addr:housenumber, Replace the geometry in OpenStreetMap and add the address,,_osm_obj:id)}" + "then": "{conflate_button(osm_buildings_no_points,building=$_target_building_type; source:geometry:date=$_grb_date; source:geometry:ref=$_grb_ref; addr:street=$addr:street; addr:housenumber=$addr:housenumber, Replace the geometry in OpenStreetMap and add the address,,_osm_obj:id)}" }, { "if": { @@ -380,7 +380,7 @@ "_reverse_overlap_percentage>50" ] }, - "then": "{conflate_button(osm-buildings,building=$_target_building_type; source:geometry:date=$_grb_date; source:geometry:ref=$_grb_ref, Replace the geometry in OpenStreetMap,,_osm_obj:id)}" + "then": "{conflate_button(osm_buildings_no_points,building=$_target_building_type; source:geometry:date=$_grb_date; source:geometry:ref=$_grb_ref, Replace the geometry in OpenStreetMap,,_osm_obj:id)}" } ] }, @@ -612,7 +612,7 @@ "builtin": "crab_address", "override": { "calculatedTags+": [ - "_embedded_in=overlapWith(feat)('osm-buildings').filter(b => /* Do not match newly created objects */ b.feat.properties.id.indexOf('-') < 0)[0]?.feat?.properties ?? {}", + "_embedded_in=overlapWith(feat)('osm_buildings_no_points').filter(b => /* Do not match newly created objects */ b.feat.properties.id.indexOf('-') < 0)[0]?.feat?.properties ?? {}", "_embedding_nr=get(feat)('_embedded_in')['addr:housenumber']+(get(feat)('_embedded_in')['addr:unit'] ?? '')", "_embedding_street=get(feat)('_embedded_in')['addr:street']", "_embedding_id=get(feat)('_embedded_in').id", @@ -709,7 +709,7 @@ "text": { "nl": "Voeg dit adres als een nieuw adrespunt toe" }, - "snap_onto_layers": "osm-buildings" + "snap_onto_layers": "osm_buildings_no_points" } }, "mappings": [ diff --git a/assets/themes/healthcare/healthcare.json b/assets/themes/healthcare/healthcare.json index 51c577f0c..e06e7f305 100644 --- a/assets/themes/healthcare/healthcare.json +++ b/assets/themes/healthcare/healthcare.json @@ -39,7 +39,7 @@ { "builtin": "shops", "override": { - "id": "medical-shops", + "id": "medical_shops", "minzoom": 13, "=filter": [ "open_now", diff --git a/assets/themes/kerbs_and_crossings/kerbs_and_crossings.json b/assets/themes/kerbs_and_crossings/kerbs_and_crossings.json index 108cfd410..ce049f837 100644 --- a/assets/themes/kerbs_and_crossings/kerbs_and_crossings.json +++ b/assets/themes/kerbs_and_crossings/kerbs_and_crossings.json @@ -46,6 +46,7 @@ { "builtin": "crossings", "override": { + "id": "crossings_no_traffic_lights", "=presets": [ { "title": { @@ -87,4 +88,4 @@ }, "kerbs" ] -} \ No newline at end of file +} diff --git a/assets/themes/onwheels/onwheels.json b/assets/themes/onwheels/onwheels.json index 116f0a487..8577f595f 100644 --- a/assets/themes/onwheels/onwheels.json +++ b/assets/themes/onwheels/onwheels.json @@ -258,6 +258,7 @@ { "builtin": "parking_spaces", "override": { + "id": "parking_spaces_disabled", "source": { "osmTags": "parking_space=disabled" }, diff --git a/assets/themes/openlovemap/openlovemap.json b/assets/themes/openlovemap/openlovemap.json index 2fd24944b..1c9bf94c1 100644 --- a/assets/themes/openlovemap/openlovemap.json +++ b/assets/themes/openlovemap/openlovemap.json @@ -14,7 +14,7 @@ { "builtin": "shops", "override": { - "id": "erotic-shop", + "id": "erotic_shop", "source": { "osmTags": "shop=erotic" }, diff --git a/assets/themes/transit/transit.json b/assets/themes/transit/transit.json index 84c4a6abb..5676d6802 100644 --- a/assets/themes/transit/transit.json +++ b/assets/themes/transit/transit.json @@ -53,6 +53,7 @@ "builtin": "shelter", "override": { "minzoom": 18, + "id": "pt_shelter", "source": { "osmTags": { "and": [ diff --git a/assets/themes/uk_addresses/uk_addresses.json b/assets/themes/uk_addresses/uk_addresses.json index 0a05b9665..97bf12a81 100644 --- a/assets/themes/uk_addresses/uk_addresses.json +++ b/assets/themes/uk_addresses/uk_addresses.json @@ -185,7 +185,7 @@ ] }, { - "id": "address", + "id": "uk_address", "name": { "en": "Known addresses in OSM" }, diff --git a/assets/themes/walkingnodes/walkingnodes.json b/assets/themes/walkingnodes/walkingnodes.json index e20ffb727..78c89216c 100644 --- a/assets/themes/walkingnodes/walkingnodes.json +++ b/assets/themes/walkingnodes/walkingnodes.json @@ -21,7 +21,7 @@ "hideFromOverview": true, "layers": [ { - "id": "node2node", + "id": "node2node_hiking", "name": { "en": "Node to node links", "de": "Knotenpunktverbindungen", @@ -120,7 +120,7 @@ ] }, { - "id": "node", + "id": "node_hiking", "name": { "en": "Nodes", "de": "Knotenpunkte", @@ -279,6 +279,7 @@ ], "override": { "minzoom": 16, + "id": "guidepost_hiking", "source": { "osmTags": { "and": [ diff --git a/scripts/osm2pgsql/generateLayerFile.ts b/scripts/osm2pgsql/generateBuildDbScript.ts similarity index 70% rename from scripts/osm2pgsql/generateLayerFile.ts rename to scripts/osm2pgsql/generateBuildDbScript.ts index 5867b8d9f..a2c9363f2 100644 --- a/scripts/osm2pgsql/generateLayerFile.ts +++ b/scripts/osm2pgsql/generateBuildDbScript.ts @@ -9,7 +9,7 @@ import { Or } from "../../src/Logic/Tags/Or" import { RegexTag } from "../../src/Logic/Tags/RegexTag" import { Utils } from "../../src/Utils" -class LuaSnippets{ +class LuaSnippets { /** * The main piece of code that calls `process_poi` */ @@ -25,23 +25,25 @@ class LuaSnippets{ "end", ""].join("\n") - public static combine(calls: string[]): string{ + public static combine(calls: string[]): string { return [ `function process_poi(object, geom)`, - ...calls.map(c => " "+c+"(object, geom)"), + ...calls.map(c => " " + c + "(object, geom)"), `end`, ].join("\n") } } + class GenerateLayerLua { private readonly _layer: LayerConfig constructor(layer: LayerConfig) { this._layer = layer } - public functionName(){ + + public functionName() { const l = this._layer - if(!l.source?.osmTags){ + if (!l.source?.osmTags) { return undefined } return `process_poi_${l.id}` @@ -49,7 +51,7 @@ class GenerateLayerLua { public generateFunction(): string { const l = this._layer - if(!l.source?.osmTags){ + if (!l.source?.osmTags) { return undefined } return [ @@ -75,10 +77,38 @@ class GenerateLayerLua { " ", ` pois_${l.id}:insert(a)`, "end", - "" + "", ].join("\n") } + private regexTagToLua(tag: RegexTag) { + if (typeof tag.value === "string" && tag.invert) { + return `object.tags["${tag.key}"] ~= "${tag.value}"` + } + + if ("" + tag.value === "/.+/is" && !tag.invert) { + return `object.tags["${tag.key}"] ~= nil` + } + + if ("" + tag.value === "/.+/is" && tag.invert) { + return `object.tags["${tag.key}"] == nil` + } + + if (tag.matchesEmpty && !tag.invert) { + return `object.tags["${tag.key}"] == nil or object.tags["${tag.key}"] == ""` + } + + + if (tag.matchesEmpty && tag.invert) { + return `object.tags["${tag.key}"] ~= nil or object.tags["${tag.key}"] ~= ""` + } + + if (tag.invert) { + return `object.tags["${tag.key}"] == nil or not string.find(object.tags["${tag.key}"], "${tag.value}")` + } + + return `(object.tags["${tag.key}"] ~= nil and string.find(object.tags["${tag.key}"], "${tag.value}"))` + } private toLuaFilter(tag: TagsFilter, useParens: boolean = false): string { if (tag instanceof Tag) { @@ -99,14 +129,7 @@ class GenerateLayerLua { return expr } if (tag instanceof RegexTag) { - if(typeof tag.value === "string" && tag.invert){ - return `object.tags["${tag.key}"] ~= "${tag.value}"` - } - - let expr = `not string.find(object.tags["${tag.key}"], "${tag.value}")` - if (!tag.invert) { - expr = "not " + expr - } + let expr = this.regexTagToLua(tag) if (useParens) { expr = "(" + expr + ")" } @@ -124,17 +147,17 @@ class GenerateLayerFile extends Script { } async main(args: string[]) { - const layerNames = Array.from(AllSharedLayers.sharedLayers.values()) + const layers = Array.from(AllSharedLayers.sharedLayers.values()) - const generators = layerNames.map(l => new GenerateLayerLua(l)) + const generators = layers.filter(l => l.source.geojsonSource === undefined).map(l => new GenerateLayerLua(l)) const script = [ ...generators.map(g => g.generateFunction()), LuaSnippets.combine(Utils.NoNull(generators.map(g => g.functionName()))), - LuaSnippets.tail + LuaSnippets.tail, ].join("\n\n\n") const path = "build_db.lua" - fs.writeFileSync(path,script, "utf-8") + fs.writeFileSync(path, script, "utf-8") console.log("Written", path) } } From ee38cdb9d7e4fb050b6bcd57c8638b0205cf0b01 Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Tue, 23 Jan 2024 22:03:22 +0100 Subject: [PATCH 06/61] LayerServer: improve script, add unique and valid id check to layers --- Docs/SettingUpPSQL.md | 23 +- scripts/Script.ts | 9 +- scripts/generateLayerOverview.ts | 69 ++-- scripts/osm2pgsql/generateBuildDbScript.ts | 52 ++- .../ThemeConfig/Conversion/Validation.ts | 359 +++++++++++------- 5 files changed, 313 insertions(+), 199 deletions(-) diff --git a/Docs/SettingUpPSQL.md b/Docs/SettingUpPSQL.md index c83ddb662..dfa9989c6 100644 --- a/Docs/SettingUpPSQL.md +++ b/Docs/SettingUpPSQL.md @@ -10,26 +10,37 @@ Then activate following extensions for this database (right click > Create > Ext - Postgis activeren (rechtsklikken > Create > extension) - HStore activeren -Install osm2pgsql (hint: compile from source is painless) +Increase the max number of connections. osm2pgsql needs connection one per table (and a few more), and since we are making one table per layer in MapComplete, this amounts to a lot. -pg_tileserv kan hier gedownload worden: https://github.com/CrunchyData/pg_tileserv - -DATABASE_URL=postgresql://user:none@localhost:5444/osm-poi ./pg_tileserv +- Open PGAdmin, open the PGSQL-tool (CLI-button at the top) +- Run `max_connections = 2000;` and `show config_file;` to get the config file location (in docker). This is probably `/var/lib/postgresql/data/postgresql.conf` +- In a terminal, run `sudo docker exec -i bash` (run `sudo docker ps` to get the container id) +- `sed -i "s/max_connections = 100/max_connections = 5000/" /var/lib/postgresql/data/postgresql.conf` +- Validate with `cat /var/lib/postgresql/data/postgresql.conf | grep "max_connections"` +- `sudo docker restart ` ## Create export scripts for every layer -Use scripts/osm2pgsl +Use `vite-node ./scripts/osm2pgsql/generateBuildDbScript.ts` ## Importing data +Install osm2pgsql (hint: compile from source is painless) To seed the database: ```` -osm2pgsql -O flex -E 4326 -S build_db.lua -s --flat-nodes=import-help-file -d postgresql://user:password@localhost:5444/osm-poi andorra-latest.osm.pbf +osm2pgsql -O flex -E 4326 -S build_db.lua -s --flat-nodes=import-help-file -d postgresql://user:password@localhost:5444/osm-poi .osm.pbf ```` +Storing properties to table '"public"."osm2pgsql_properties" takes about 25 minutes with planet.osm + +Belgium (~555mb) takes 15m +World (80GB) should take 15m*160 = 2400m = 40hr + ## Deploying a tile server +pg_tileserv kan hier gedownload worden: https://github.com/CrunchyData/pg_tileserv + ```` export DATABASE_URL=postgresql://user:password@localhost:5444/osm-poi ./pg_tileserv diff --git a/scripts/Script.ts b/scripts/Script.ts index 48a727b22..25b0b14e7 100644 --- a/scripts/Script.ts +++ b/scripts/Script.ts @@ -13,8 +13,15 @@ export default abstract class Script { ScriptUtils.fixUtils() const args = [...process.argv] args.splice(0, 2) + const start = new Date() this.main(args) - .then((_) => console.log("All done")) + .then((_) =>{ + const end = new Date() + const millisNeeded = end.getTime() - start.getTime() + + const green = (s) => "\x1b[92m" + s + "\x1b[0m" + console.log(green("All done! (" + millisNeeded + " ms)")) + }) .catch((e) => console.log("ERROR:", e)) } diff --git a/scripts/generateLayerOverview.ts b/scripts/generateLayerOverview.ts index 39e8c6424..f60f96d10 100644 --- a/scripts/generateLayerOverview.ts +++ b/scripts/generateLayerOverview.ts @@ -9,7 +9,7 @@ import { DoesImageExist, PrevalidateTheme, ValidateLayer, - ValidateThemeAndLayers, + ValidateThemeAndLayers, ValidateThemeEnsemble, } from "../src/Models/ThemeConfig/Conversion/Validation" import { Translation } from "../src/UI/i18n/Translation" import { PrepareLayer } from "../src/Models/ThemeConfig/Conversion/PrepareLayer" @@ -25,6 +25,8 @@ import LayerConfig from "../src/Models/ThemeConfig/LayerConfig" import PointRenderingConfig from "../src/Models/ThemeConfig/PointRenderingConfig" import { ConversionContext } from "../src/Models/ThemeConfig/Conversion/ConversionContext" import { GenerateFavouritesLayer } from "./generateFavouritesLayer" +import LayoutConfig from "../src/Models/ThemeConfig/LayoutConfig" +import { TagsFilter } from "../src/Logic/Tags/TagsFilter" // This scripts scans 'src/assets/layers/*.json' for layer definition files and 'src/assets/themes/*.json' for theme definition files. // It spits out an overview of those to be used to load them @@ -123,6 +125,7 @@ class AddIconSummary extends DesugaringStep<{ raw: LayerConfigJson; parsed: Laye } } + class LayerOverviewUtils extends Script { public static readonly layerPath = "./src/assets/generated/layers/" public static readonly themePath = "./src/assets/generated/themes/" @@ -355,7 +358,6 @@ class LayerOverviewUtils extends Script { const layerWhitelist = new Set(args.find(a => a.startsWith("--layers=")) ?.substring("--layers=".length)?.split(",") ?? []) - const start = new Date() const forceReload = args.some((a) => a == "--force") const licensePaths = new Set() @@ -382,17 +384,21 @@ class LayerOverviewUtils extends Script { sharedLayers, recompiledThemes, forceReload, - themeWhitelist + themeWhitelist, ) - if (recompiledThemes.length > 0){ + new ValidateThemeEnsemble().convertStrict( + Array.from(sharedThemes.values()).map(th => new LayoutConfig(th, true))) + + + if (recompiledThemes.length > 0) { writeFileSync( "./src/assets/generated/known_layers.json", JSON.stringify({ layers: Array.from(sharedLayers.values()).filter((l) => l.id !== "favourite"), }), ) - } + } const mcChangesPath = "./assets/themes/mapcomplete-changes/mapcomplete-changes.json" if ( @@ -437,7 +443,7 @@ class LayerOverviewUtils extends Script { ) } - if(recompiledThemes.length > 0) { + if (recompiledThemes.length > 0) { writeFileSync( "./src/assets/generated/known_themes.json", JSON.stringify({ @@ -446,17 +452,10 @@ class LayerOverviewUtils extends Script { ) } - const end = new Date() - const millisNeeded = end.getTime() - start.getTime() if (AllSharedLayers.getSharedLayersConfigs().size == 0) { console.error( - "This was a bootstrapping-run. Run generate layeroverview again!(" + - millisNeeded + - " ms)", + "This was a bootstrapping-run. Run generate layeroverview again!" ) - } else { - const green = (s) => "\x1b[92m" + s + "\x1b[0m" - console.log(green("All done! (" + millisNeeded + " ms)")) } } @@ -482,7 +481,7 @@ class LayerOverviewUtils extends Script { private buildLayerIndex( doesImageExist: DoesImageExist, forceReload: boolean, - whitelist: Set + whitelist: Set, ): Map { // First, we expand and validate all builtin layers. These are written to src/assets/generated/layers // At the same time, an index of available layers is built. @@ -500,9 +499,9 @@ class LayerOverviewUtils extends Script { const recompiledLayers: string[] = [] let warningCount = 0 for (const sharedLayerPath of ScriptUtils.getLayerPaths()) { - if(whitelist.size > 0){ - const idByPath = sharedLayerPath.split("/").at(-1).split(".")[0] - if(Constants.priviliged_layers.indexOf( idByPath) < 0 && !whitelist.has(idByPath)){ + if (whitelist.size > 0) { + const idByPath = sharedLayerPath.split("/").at(-1).split(".")[0] + if (Constants.priviliged_layers.indexOf(idByPath) < 0 && !whitelist.has(idByPath)) { continue } } @@ -672,7 +671,7 @@ class LayerOverviewUtils extends Script { sharedLayers: Map, recompiledThemes: string[], forceReload: boolean, - whitelist: Set + whitelist: Set, ): Map { console.log(" ---------- VALIDATING BUILTIN THEMES ---------") const themeFiles = ScriptUtils.getThemeFiles() @@ -710,7 +709,7 @@ class LayerOverviewUtils extends Script { const themeInfo = themeFiles[i] const themePath = themeInfo.path let themeFile = themeInfo.parsed - if(whitelist.size > 0 && !whitelist.has(themeFile.id)){ + if (whitelist.size > 0 && !whitelist.has(themeFile.id)) { continue } @@ -795,21 +794,21 @@ class LayerOverviewUtils extends Script { } } - if(whitelist.size == 0){ - this.writeSmallOverview( - Array.from(fixed.values()).map((t) => { - return { - ...t, - hideFromOverview: t.hideFromOverview ?? false, - shortDescription: - t.shortDescription ?? - new Translation(t.description) - .FirstSentence() - .OnEveryLanguage((s) => parse_html(s).textContent).translations, - mustHaveLanguage: t.mustHaveLanguage?.length > 0, - } - }), - ) + if (whitelist.size == 0) { + this.writeSmallOverview( + Array.from(fixed.values()).map((t) => { + return { + ...t, + hideFromOverview: t.hideFromOverview ?? false, + shortDescription: + t.shortDescription ?? + new Translation(t.description) + .FirstSentence() + .OnEveryLanguage((s) => parse_html(s).textContent).translations, + mustHaveLanguage: t.mustHaveLanguage?.length > 0, + } + }), + ) } console.log( diff --git a/scripts/osm2pgsql/generateBuildDbScript.ts b/scripts/osm2pgsql/generateBuildDbScript.ts index a2c9363f2..0fce50b06 100644 --- a/scripts/osm2pgsql/generateBuildDbScript.ts +++ b/scripts/osm2pgsql/generateBuildDbScript.ts @@ -1,13 +1,13 @@ -import LayerConfig from "../../src/Models/ThemeConfig/LayerConfig" import { TagsFilter } from "../../src/Logic/Tags/TagsFilter" import { Tag } from "../../src/Logic/Tags/Tag" import { And } from "../../src/Logic/Tags/And" import Script from "../Script" -import { AllSharedLayers } from "../../src/Customizations/AllSharedLayers" import fs from "fs" import { Or } from "../../src/Logic/Tags/Or" import { RegexTag } from "../../src/Logic/Tags/RegexTag" import { Utils } from "../../src/Utils" +import { ValidateThemeEnsemble } from "../../src/Models/ThemeConfig/Conversion/Validation" +import { AllKnownLayouts } from "../../src/Customizations/AllKnownLayouts" class LuaSnippets { /** @@ -35,28 +35,31 @@ class LuaSnippets { } class GenerateLayerLua { - private readonly _layer: LayerConfig + private readonly _id: string + private readonly _tags: TagsFilter + private readonly _foundInThemes: string[] - constructor(layer: LayerConfig) { - this._layer = layer + constructor(id: string, tags: TagsFilter, foundInThemes: string[] = []) { + this._tags = tags + this._id = id + this._foundInThemes = foundInThemes } public functionName() { - const l = this._layer - if (!l.source?.osmTags) { + if (!this._tags) { return undefined } - return `process_poi_${l.id}` + return `process_poi_${this._id}` } public generateFunction(): string { - const l = this._layer - if (!l.source?.osmTags) { + if (!this._tags) { return undefined } return [ - `local pois_${l.id} = osm2pgsql.define_table({`, - ` name = '${l.id}',`, + `local pois_${this._id} = osm2pgsql.define_table({`, + this._foundInThemes ? "-- used in themes: " + this._foundInThemes.join(", ") : "", + ` name = '${this._id}',`, " ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' },", " columns = {", " { column = 'tags', type = 'jsonb' },", @@ -66,7 +69,7 @@ class GenerateLayerLua { "", "", `function ${this.functionName()}(object, geom)`, - " local matches_filter = " + this.toLuaFilter(l.source.osmTags), + " local matches_filter = " + this.toLuaFilter(this._tags), " if( not matches_filter) then", " return", " end", @@ -75,7 +78,7 @@ class GenerateLayerLua { " tags = object.tags", " }", " ", - ` pois_${l.id}:insert(a)`, + ` pois_${this._id}:insert(a)`, "end", "", ].join("\n") @@ -86,6 +89,8 @@ class GenerateLayerLua { return `object.tags["${tag.key}"] ~= "${tag.value}"` } + const v = ( tag.value).source.replace(/\\\//g, "/") + if ("" + tag.value === "/.+/is" && !tag.invert) { return `object.tags["${tag.key}"] ~= nil` } @@ -104,10 +109,10 @@ class GenerateLayerLua { } if (tag.invert) { - return `object.tags["${tag.key}"] == nil or not string.find(object.tags["${tag.key}"], "${tag.value}")` + return `object.tags["${tag.key}"] == nil or not string.find(object.tags["${tag.key}"], "${v}")` } - return `(object.tags["${tag.key}"] ~= nil and string.find(object.tags["${tag.key}"], "${tag.value}"))` + return `(object.tags["${tag.key}"] ~= nil and string.find(object.tags["${tag.key}"], "${v}"))` } private toLuaFilter(tag: TagsFilter, useParens: boolean = false): string { @@ -141,15 +146,21 @@ class GenerateLayerLua { } } -class GenerateLayerFile extends Script { +class GenerateBuildDbScript extends Script { constructor() { super("Generates a .lua-file to use with osm2pgsql") } async main(args: string[]) { - const layers = Array.from(AllSharedLayers.sharedLayers.values()) + const allNeededLayers = new ValidateThemeEnsemble().convertStrict( + AllKnownLayouts.allKnownLayouts.values(), + ) - const generators = layers.filter(l => l.source.geojsonSource === undefined).map(l => new GenerateLayerLua(l)) + const generators: GenerateLayerLua[] = [] + + allNeededLayers.forEach(({ tags, foundInTheme }, layerId) => { + generators.push(new GenerateLayerLua(layerId, tags, foundInTheme)) + }) const script = [ ...generators.map(g => g.generateFunction()), @@ -159,7 +170,8 @@ class GenerateLayerFile extends Script { const path = "build_db.lua" fs.writeFileSync(path, script, "utf-8") console.log("Written", path) + console.log(allNeededLayers.size+" layers will be created. Make sure to set 'max_connections' to at least "+(10 + allNeededLayers.size) ) } } -new GenerateLayerFile().run() +new GenerateBuildDbScript().run() diff --git a/src/Models/ThemeConfig/Conversion/Validation.ts b/src/Models/ThemeConfig/Conversion/Validation.ts index ed4c79098..6380df2e5 100644 --- a/src/Models/ThemeConfig/Conversion/Validation.ts +++ b/src/Models/ThemeConfig/Conversion/Validation.ts @@ -21,9 +21,7 @@ import PresetConfig from "../PresetConfig" import { TagsFilter } from "../../../Logic/Tags/TagsFilter" import { Translatable } from "../Json/Translatable" import { ConversionContext } from "./ConversionContext" -import * as eli from "../../../assets/editor-layer-index.json" import { AvailableRasterLayers } from "../../RasterLayers" -import Back from "../../../assets/svg/Back.svelte" import PointRenderingConfigJson from "../Json/PointRenderingConfigJson" class ValidateLanguageCompleteness extends DesugaringStep { @@ -33,7 +31,7 @@ class ValidateLanguageCompleteness extends DesugaringStep { super( "Checks that the given object is fully translated in the specified languages", [], - "ValidateLanguageCompleteness" + "ValidateLanguageCompleteness", ) this._languages = languages ?? ["en"] } @@ -47,18 +45,18 @@ class ValidateLanguageCompleteness extends DesugaringStep { .filter( (t) => t.tr.translations[neededLanguage] === undefined && - t.tr.translations["*"] === undefined + t.tr.translations["*"] === undefined, ) .forEach((missing) => { context .enter(missing.context.split(".")) .err( `The theme ${obj.id} should be translation-complete for ` + - neededLanguage + - ", but it lacks a translation for " + - missing.context + - ".\n\tThe known translation is " + - missing.tr.textFor("en") + neededLanguage + + ", but it lacks a translation for " + + missing.context + + ".\n\tThe known translation is " + + missing.tr.textFor("en"), ) }) } @@ -75,7 +73,7 @@ export class DoesImageExist extends DesugaringStep { constructor( knownImagePaths: Set, checkExistsSync: (path: string) => boolean = undefined, - ignore?: Set + ignore?: Set, ) { super("Checks if an image exists", [], "DoesImageExist") this._ignore = ignore @@ -111,15 +109,15 @@ export class DoesImageExist extends DesugaringStep { if (!this._knownImagePaths.has(image)) { if (this.doesPathExist === undefined) { context.err( - `Image with path ${image} not found or not attributed; it is used in ${context}` + `Image with path ${image} not found or not attributed; it is used in ${context}`, ) } else if (!this.doesPathExist(image)) { context.err( - `Image with path ${image} does not exist.\n Check for typo's and missing directories in the path.` + `Image with path ${image} does not exist.\n Check for typo's and missing directories in the path.`, ) } else { context.err( - `Image with path ${image} is not attributed (but it exists); execute 'npm run query:licenses' to add the license information and/or run 'npm run generate:licenses' to compile all the license info` + `Image with path ${image} is not attributed (but it exists); execute 'npm run query:licenses' to add the license information and/or run 'npm run generate:licenses' to compile all the license info`, ) } } @@ -143,7 +141,7 @@ export class ValidateTheme extends DesugaringStep { doesImageExist: DoesImageExist, path: string, isBuiltin: boolean, - sharedTagRenderings?: Set + sharedTagRenderings?: Set, ) { super("Doesn't change anything, but emits warnings and errors", [], "ValidateTheme") this._validateImage = doesImageExist @@ -162,15 +160,15 @@ export class ValidateTheme extends DesugaringStep { if (json["units"] !== undefined) { context.err( "The theme " + - json.id + - " has units defined - these should be defined on the layer instead. (Hint: use overrideAll: { '+units': ... }) " + json.id + + " has units defined - these should be defined on the layer instead. (Hint: use overrideAll: { '+units': ... }) ", ) } if (json["roamingRenderings"] !== undefined) { context.err( "Theme " + - json.id + - " contains an old 'roamingRenderings'. Use an 'overrideAll' instead" + json.id + + " contains an old 'roamingRenderings'. Use an 'overrideAll' instead", ) } } @@ -178,7 +176,7 @@ export class ValidateTheme extends DesugaringStep { if (!json.title) { context.enter("title").err(`The theme ${json.id} does not have a title defined.`) } - if(!json.icon){ + if (!json.icon) { context.enter("icon").err("A theme should have an icon") } if (this._isBuiltin && this._extractImages !== undefined) { @@ -188,10 +186,10 @@ export class ValidateTheme extends DesugaringStep { for (const remoteImage of remoteImages) { context.err( "Found a remote image: " + - remoteImage.path + - " in theme " + - json.id + - ", please download it." + remoteImage.path + + " in theme " + + json.id + + ", please download it.", ) } for (const image of images) { @@ -207,17 +205,17 @@ export class ValidateTheme extends DesugaringStep { const filename = this._path.substring( this._path.lastIndexOf("/") + 1, - this._path.length - 5 + this._path.length - 5, ) if (theme.id !== filename) { context.err( "Theme ids should be the same as the name.json, but we got id: " + - theme.id + - " and filename " + - filename + - " (" + - this._path + - ")" + theme.id + + " and filename " + + filename + + " (" + + this._path + + ")", ) } this._validateImage.convert(theme.icon, context.enter("icon")) @@ -225,13 +223,13 @@ export class ValidateTheme extends DesugaringStep { const dups = Utils.Duplicates(json.layers.map((layer) => layer["id"])) if (dups.length > 0) { context.err( - `The theme ${json.id} defines multiple layers with id ${dups.join(", ")}` + `The theme ${json.id} defines multiple layers with id ${dups.join(", ")}`, ) } if (json["mustHaveLanguage"] !== undefined) { new ValidateLanguageCompleteness(...json["mustHaveLanguage"]).convert( theme, - context + context, ) } if (!json.hideFromOverview && theme.id !== "personal" && this._isBuiltin) { @@ -239,7 +237,7 @@ export class ValidateTheme extends DesugaringStep { const targetLanguage = theme.title.SupportedLanguages()[0] if (targetLanguage !== "en") { context.err( - `TargetLanguage is not 'en' for public theme ${theme.id}, it is ${targetLanguage}. Move 'en' up in the title of the theme and set it as the first key` + `TargetLanguage is not 'en' for public theme ${theme.id}, it is ${targetLanguage}. Move 'en' up in the title of the theme and set it as the first key`, ) } @@ -282,6 +280,13 @@ export class ValidateTheme extends DesugaringStep { } } + for (let i = 0; i < theme.layers.length; i++) { + const layer = theme.layers[i] + if (!layer.id.match("[a-z][a-z0-9_]*")) { + context.enters("layers", i, "id").err("Invalid ID:" + layer.id + "should match [a-z][a-z0-9_]*") + } + } + return json } } @@ -291,7 +296,7 @@ export class ValidateThemeAndLayers extends Fuse { doesImageExist: DoesImageExist, path: string, isBuiltin: boolean, - sharedTagRenderings?: Set + sharedTagRenderings?: Set, ) { super( "Validates a theme and the contained layers", @@ -301,10 +306,10 @@ export class ValidateThemeAndLayers extends Fuse { new Each( new Bypass( (layer) => Constants.added_by_default.indexOf(layer.id) < 0, - new ValidateLayerConfig(undefined, isBuiltin, doesImageExist, false, true) - ) - ) - ) + new ValidateLayerConfig(undefined, isBuiltin, doesImageExist, false, true), + ), + ), + ), ) } } @@ -314,7 +319,7 @@ class OverrideShadowingCheck extends DesugaringStep { super( "Checks that an 'overrideAll' does not override a single override", [], - "OverrideShadowingCheck" + "OverrideShadowingCheck", ) } @@ -363,6 +368,22 @@ class MiscThemeChecks extends DesugaringStep { if (json.socialImage === "") { context.warn("Social image for theme " + json.id + " is the emtpy string") } + { + for (let i = 0; i < json.layers.length; i++) { + const l = json.layers[i] + if (l["override"]?.["source"] === undefined) { + continue + } + if (l["override"]?.["source"]?.["geoJson"]) { + continue // We don't care about external data as we won't cache it anyway + } + if (l["override"]["id"] !== undefined) { + continue + } + context.enters("layers", i).err("A layer which changes the source-tags must also change the ID") + } + } + return json } } @@ -372,7 +393,7 @@ export class PrevalidateTheme extends Fuse { super( "Various consistency checks on the raw JSON", new MiscThemeChecks(), - new OverrideShadowingCheck() + new OverrideShadowingCheck(), ) } } @@ -382,7 +403,7 @@ export class DetectConflictingAddExtraTags extends DesugaringStep ["_abc"] */ private static extractCalculatedTagNames( - layerConfig?: LayerConfigJson | { calculatedTags: string[] } + layerConfig?: LayerConfigJson | { calculatedTags: string[] }, ) { return ( layerConfig?.calculatedTags?.map((ct) => { @@ -617,16 +638,16 @@ export class DetectShadowedMappings extends DesugaringStep\` instead. The images found are ${images.join( - ", " - )}. (This check can be turned of by adding "#": "${ignoreToken}" in the mapping, but this is discouraged` + ", ", + )}. (This check can be turned of by adding "#": "${ignoreToken}" in the mapping, but this is discouraged`, ) } else { ctx.info( `Ignored image ${images.join( - ", " - )} in 'then'-clause of a mapping as this check has been disabled` + ", ", + )} in 'then'-clause of a mapping as this check has been disabled`, ) for (const image of images) { @@ -721,7 +742,7 @@ class ValidatePossibleLinks extends DesugaringStep does have `rel='noopener'` set", [], - "ValidatePossibleLinks" + "ValidatePossibleLinks", ) } @@ -751,21 +772,21 @@ class ValidatePossibleLinks extends DesugaringStep, - context: ConversionContext + context: ConversionContext, ): string | Record { if (typeof json === "string") { if (this.isTabnabbingProne(json)) { context.err( "The string " + - json + - " has a link targeting `_blank`, but it doesn't have `rel='noopener'` set. This gives rise to reverse tabnapping" + json + + " has a link targeting `_blank`, but it doesn't have `rel='noopener'` set. This gives rise to reverse tabnapping", ) } } else { for (const k in json) { if (this.isTabnabbingProne(json[k])) { context.err( - `The translation for ${k} '${json[k]}' has a link targeting \`_blank\`, but it doesn't have \`rel='noopener'\` set. This gives rise to reverse tabnapping` + `The translation for ${k} '${json[k]}' has a link targeting \`_blank\`, but it doesn't have \`rel='noopener'\` set. This gives rise to reverse tabnapping`, ) } } @@ -783,7 +804,7 @@ class CheckTranslation extends DesugaringStep { super( "Checks that a translation is valid and internally consistent", ["*"], - "CheckTranslation" + "CheckTranslation", ) this._allowUndefined = allowUndefined } @@ -829,17 +850,17 @@ class MiscTagRenderingChecks extends DesugaringStep { convert( json: TagRenderingConfigJson | QuestionableTagRenderingConfigJson, - context: ConversionContext + context: ConversionContext, ): TagRenderingConfigJson { if (json["special"] !== undefined) { context.err( - 'Detected `special` on the top level. Did you mean `{"render":{ "special": ... }}`' + "Detected `special` on the top level. Did you mean `{\"render\":{ \"special\": ... }}`", ) } if (Object.keys(json).length === 1 && typeof json["render"] === "string") { context.warn( - `use the content directly instead of {render: ${JSON.stringify(json["render"])}}` + `use the content directly instead of {render: ${JSON.stringify(json["render"])}}`, ) } @@ -851,7 +872,7 @@ class MiscTagRenderingChecks extends DesugaringStep { const mapping = json.mappings[i] CheckTranslation.noUndefined.convert( mapping.then, - context.enters("mappings", i, "then") + context.enters("mappings", i, "then"), ) if (!mapping.if) { context.enters("mappings", i).err("No `if` is defined") @@ -862,18 +883,18 @@ class MiscTagRenderingChecks extends DesugaringStep { context .enters("mappings", i, "then") .warn( - "A mapping should not start with 'yes' or 'no'. If the attribute is known, it will only show 'yes' or 'no' without the question, resulting in a weird phrasing in the information box" + "A mapping should not start with 'yes' or 'no'. If the attribute is known, it will only show 'yes' or 'no' without the question, resulting in a weird phrasing in the information box", ) } } } if (json["group"]) { - context.err('Groups are deprecated, use `"label": ["' + json["group"] + '"]` instead') + context.err("Groups are deprecated, use `\"label\": [\"" + json["group"] + "\"]` instead") } if (json["question"] && json.freeform?.key === undefined && json.mappings === undefined) { context.err( - "A question is defined, but no mappings nor freeform (key) are. Add at least one of them" + "A question is defined, but no mappings nor freeform (key) are. Add at least one of them", ) } if (json["question"] && !json.freeform && (json.mappings?.length ?? 0) == 1) { @@ -883,7 +904,7 @@ class MiscTagRenderingChecks extends DesugaringStep { context .enter("questionHint") .err( - "A questionHint is defined, but no question is given. As such, the questionHint will never be shown" + "A questionHint is defined, but no question is given. As such, the questionHint will never be shown", ) } @@ -893,10 +914,10 @@ class MiscTagRenderingChecks extends DesugaringStep { .enter("render") .err( "This tagRendering allows to set a value to key " + - json.freeform.key + - ", but does not define a `render`. Please, add a value here which contains `{" + - json.freeform.key + - "}`" + json.freeform.key + + ", but does not define a `render`. Please, add a value here which contains `{" + + json.freeform.key + + "}`", ) } else { const render = new Translation(json.render) @@ -927,7 +948,7 @@ class MiscTagRenderingChecks extends DesugaringStep { const keyFirstArg = ["canonical", "fediverse_link", "translated"] if ( keyFirstArg.some( - (funcName) => txt.indexOf(`{${funcName}(${json.freeform.key}`) >= 0 + (funcName) => txt.indexOf(`{${funcName}(${json.freeform.key}`) >= 0, ) ) { continue @@ -950,7 +971,7 @@ class MiscTagRenderingChecks extends DesugaringStep { context .enter("render") .err( - `The rendering for language ${ln} does not contain \`{${json.freeform.key}}\`. This is a bug, as this rendering should show exactly this freeform key!` + `The rendering for language ${ln} does not contain \`{${json.freeform.key}}\`. This is a bug, as this rendering should show exactly this freeform key!`, ) } } @@ -958,8 +979,8 @@ class MiscTagRenderingChecks extends DesugaringStep { if (json.render && json["question"] && json.freeform === undefined) { context.err( `Detected a tagrendering which takes input without freeform key in ${context}; the question is ${new Translation( - json["question"] - ).textFor("en")}` + json["question"], + ).textFor("en")}`, ) } @@ -970,9 +991,9 @@ class MiscTagRenderingChecks extends DesugaringStep { .enters("freeform", "type") .err( "Unknown type: " + - freeformType + - "; try one of " + - Validators.availableTypes.join(", ") + freeformType + + "; try one of " + + Validators.availableTypes.join(", "), ) } } @@ -1004,7 +1025,7 @@ export class ValidateTagRenderings extends Fuse { new On("question", new ValidatePossibleLinks()), new On("questionHint", new ValidatePossibleLinks()), new On("mappings", new Each(new On("then", new ValidatePossibleLinks()))), - new MiscTagRenderingChecks() + new MiscTagRenderingChecks(), ) } } @@ -1034,8 +1055,9 @@ export class PrevalidateLayer extends DesugaringStep { if (json.id?.toLowerCase() !== json.id) { context.enter("id").err(`The id of a layer should be lowercase: ${json.id}`) } - if (json.id?.match(/[a-z0-9-_]/) == null) { - context.enter("id").err(`The id of a layer should match [a-z0-9-_]*: ${json.id}`) + const layerRegex = /[a-zA-Z][a-zA-Z_0-9]+/ + if (json.id.match(layerRegex) === null) { + context.enter("id").err("Invalid ID. A layer ID should match " + layerRegex.source) } } @@ -1043,7 +1065,7 @@ export class PrevalidateLayer extends DesugaringStep { context .enter("source") .err( - "No source section is defined; please define one as data is not loaded otherwise" + "No source section is defined; please define one as data is not loaded otherwise", ) } else { if (json.source === "special" || json.source === "special:library") { @@ -1051,7 +1073,7 @@ export class PrevalidateLayer extends DesugaringStep { context .enters("source", "osmTags") .err( - "No osmTags defined in the source section - these should always be present, even for geojson layer" + "No osmTags defined in the source section - these should always be present, even for geojson layer", ) } else { const osmTags = TagUtils.Tag(json.source["osmTags"], context + "source.osmTags") @@ -1060,7 +1082,7 @@ export class PrevalidateLayer extends DesugaringStep { .enters("source", "osmTags") .err( "The source states tags which give a very wide selection: it only uses negative expressions, which will result in too much and unexpected data. Add at least one required tag. The tags are:\n\t" + - osmTags.asHumanString(false, false, {}) + osmTags.asHumanString(false, false, {}), ) } } @@ -1086,10 +1108,10 @@ export class PrevalidateLayer extends DesugaringStep { .enter("syncSelection") .err( "Invalid sync-selection: must be one of " + - LayerConfig.syncSelectionAllowed.map((v) => `'${v}'`).join(", ") + - " but got '" + - json.syncSelection + - "'" + LayerConfig.syncSelectionAllowed.map((v) => `'${v}'`).join(", ") + + " but got '" + + json.syncSelection + + "'", ) } if (json["pointRenderings"]?.length > 0) { @@ -1107,7 +1129,7 @@ export class PrevalidateLayer extends DesugaringStep { context.enter("pointRendering").err("There are no pointRenderings at all...") } - json.pointRendering?.forEach((pr,i) => this._validatePointRendering.convert(pr, context.enters("pointeRendering", i))) + json.pointRendering?.forEach((pr, i) => this._validatePointRendering.convert(pr, context.enters("pointeRendering", i))) if (json["mapRendering"]) { context.enter("mapRendering").err("This layer has a legacy 'mapRendering'") @@ -1123,8 +1145,8 @@ export class PrevalidateLayer extends DesugaringStep { if (!Constants.priviliged_layers.find((x) => x == json.id)) { context.err( "Layer " + - json.id + - " uses 'special' as source.osmTags. However, this layer is not a priviliged layer" + json.id + + " uses 'special' as source.osmTags. However, this layer is not a priviliged layer", ) } } @@ -1139,19 +1161,19 @@ export class PrevalidateLayer extends DesugaringStep { context .enter("title") .err( - "This layer does not have a title defined but it does have tagRenderings. Not having a title will disable the popups, resulting in an unclickable element. Please add a title. If not having a popup is intended and the tagrenderings need to be kept (e.g. in a library layer), set `title: null` to disable this error." + "This layer does not have a title defined but it does have tagRenderings. Not having a title will disable the popups, resulting in an unclickable element. Please add a title. If not having a popup is intended and the tagrenderings need to be kept (e.g. in a library layer), set `title: null` to disable this error.", ) } if (json.title === null) { context.info( - "Title is `null`. This results in an element that cannot be clicked - even though tagRenderings is set." + "Title is `null`. This results in an element that cannot be clicked - even though tagRenderings is set.", ) } { // Check for multiple, identical builtin questions - usability for studio users const duplicates = Utils.Duplicates( - json.tagRenderings.filter((tr) => typeof tr === "string") + json.tagRenderings.filter((tr) => typeof tr === "string"), ) for (let i = 0; i < json.tagRenderings.length; i++) { const tagRendering = json.tagRenderings[i] @@ -1181,7 +1203,7 @@ export class PrevalidateLayer extends DesugaringStep { { // duplicate ids in tagrenderings check const duplicates = Utils.NoNull( - Utils.Duplicates(Utils.NoNull((json.tagRenderings ?? []).map((tr) => tr["id"]))) + Utils.Duplicates(Utils.NoNull((json.tagRenderings ?? []).map((tr) => tr["id"]))), ) if (duplicates.length > 0) { // It is tempting to add an index to this warning; however, due to labels the indices here might be different from the index in the tagRendering list @@ -1219,8 +1241,8 @@ export class PrevalidateLayer extends DesugaringStep { if (json["overpassTags"] !== undefined) { context.err( "Layer " + - json.id + - 'still uses the old \'overpassTags\'-format. Please use "source": {"osmTags": }\' instead of "overpassTags": (note: this isn\'t your fault, the custom theme generator still spits out the old format)' + json.id + + "still uses the old 'overpassTags'-format. Please use \"source\": {\"osmTags\": }' instead of \"overpassTags\": (note: this isn't your fault, the custom theme generator still spits out the old format)", ) } const forbiddenTopLevel = [ @@ -1240,7 +1262,7 @@ export class PrevalidateLayer extends DesugaringStep { } if (json["hideUnderlayingFeaturesMinPercentage"] !== undefined) { context.err( - "Layer " + json.id + " contains an old 'hideUnderlayingFeaturesMinPercentage'" + "Layer " + json.id + " contains an old 'hideUnderlayingFeaturesMinPercentage'", ) } @@ -1257,9 +1279,9 @@ export class PrevalidateLayer extends DesugaringStep { if (this._path != undefined && this._path.indexOf(expected) < 0) { context.err( "Layer is in an incorrect place. The path is " + - this._path + - ", but expected " + - expected + this._path + + ", but expected " + + expected, ) } } @@ -1277,13 +1299,13 @@ export class PrevalidateLayer extends DesugaringStep { .enter(["tagRenderings", ...emptyIndexes]) .err( `Some tagrendering-ids are empty or have an emtpy string; this is not allowed (at ${emptyIndexes.join( - "," - )}])` + ",", + )}])`, ) } const duplicateIds = Utils.Duplicates( - (json.tagRenderings ?? [])?.map((f) => f["id"]).filter((id) => id !== "questions") + (json.tagRenderings ?? [])?.map((f) => f["id"]).filter((id) => id !== "questions"), ) if (duplicateIds.length > 0 && !Utils.runningFromConsole) { context @@ -1307,7 +1329,7 @@ export class PrevalidateLayer extends DesugaringStep { if (json.tagRenderings !== undefined) { new On( "tagRenderings", - new Each(new ValidateTagRenderings(json, this._doesImageExist)) + new Each(new ValidateTagRenderings(json, this._doesImageExist)), ).convert(json, context) } @@ -1334,7 +1356,7 @@ export class PrevalidateLayer extends DesugaringStep { context .enters("pointRendering", i, "marker", indexM, "icon", "condition") .err( - "Don't set a condition in a marker as this will result in an invisible but clickable element. Use extra filters in the source instead." + "Don't set a condition in a marker as this will result in an invisible but clickable element. Use extra filters in the source instead.", ) } } @@ -1372,9 +1394,9 @@ export class PrevalidateLayer extends DesugaringStep { .enters("presets", i, "tags") .err( "This preset does not match the required tags of this layer. This implies that a newly added point will not show up.\n A newly created point will have properties: " + - tags.asHumanString(false, false, {}) + - "\n The required tags are: " + - baseTags.asHumanString(false, false, {}) + tags.asHumanString(false, false, {}) + + "\n The required tags are: " + + baseTags.asHumanString(false, false, {}), ) } } @@ -1391,7 +1413,7 @@ export class ValidateLayerConfig extends DesugaringStep { isBuiltin: boolean, doesImageExist: DoesImageExist, studioValidations: boolean = false, - skipDefaultLayers: boolean = false + skipDefaultLayers: boolean = false, ) { super("Thin wrapper around 'ValidateLayer", [], "ValidateLayerConfig") this.validator = new ValidateLayer( @@ -1399,7 +1421,7 @@ export class ValidateLayerConfig extends DesugaringStep { isBuiltin, doesImageExist, studioValidations, - skipDefaultLayers + skipDefaultLayers, ) } @@ -1428,12 +1450,12 @@ class ValidatePointRendering extends DesugaringStep { } if (json.marker && !Array.isArray(json.marker)) { context.enter("marker").err( - "The marker in a pointRendering should be an array" + "The marker in a pointRendering should be an array", ) } if (json.location.length == 0) { - context.enter("location").err ( - "A pointRendering should have at least one 'location' to defined where it should be rendered. " + context.enter("location").err( + "A pointRendering should have at least one 'location' to defined where it should be rendered. ", ) } return json @@ -1441,41 +1463,44 @@ class ValidatePointRendering extends DesugaringStep { } } + export class ValidateLayer extends Conversion< LayerConfigJson, { parsed: LayerConfig; raw: LayerConfigJson } > { private readonly _skipDefaultLayers: boolean private readonly _prevalidation: PrevalidateLayer + constructor( path: string, isBuiltin: boolean, doesImageExist: DoesImageExist, studioValidations: boolean = false, - skipDefaultLayers: boolean = false + skipDefaultLayers: boolean = false, ) { super("Doesn't change anything, but emits warnings and errors", [], "ValidateLayer") this._prevalidation = new PrevalidateLayer( path, isBuiltin, doesImageExist, - studioValidations + studioValidations, ) this._skipDefaultLayers = skipDefaultLayers } convert( json: LayerConfigJson, - context: ConversionContext + context: ConversionContext, ): { parsed: LayerConfig; raw: LayerConfigJson } { context = context.inOperation(this.name) if (typeof json === "string") { context.err( - `Not a valid layer: the layerConfig is a string. 'npm run generate:layeroverview' might be needed` + `Not a valid layer: the layerConfig is a string. 'npm run generate:layeroverview' might be needed`, ) return undefined } + if (this._skipDefaultLayers && Constants.added_by_default.indexOf(json.id) >= 0) { return { parsed: undefined, raw: json } } @@ -1502,7 +1527,7 @@ export class ValidateLayer extends Conversion< context .enters("calculatedTags", i) .err( - `Invalid function definition: the custom javascript is invalid:${e}. The offending javascript code is:\n ${code}` + `Invalid function definition: the custom javascript is invalid:${e}. The offending javascript code is:\n ${code}`, ) } } @@ -1553,8 +1578,8 @@ export class ValidateFilter extends DesugaringStep { .enters("fields", i) .err( `Invalid filter: ${type} is not a valid textfield type.\n\tTry one of ${Array.from( - Validators.availableTypes - ).join(",")}` + Validators.availableTypes, + ).join(",")}`, ) } } @@ -1571,13 +1596,13 @@ export class DetectDuplicateFilters extends DesugaringStep<{ super( "Tries to detect layers where a shared filter can be used (or where similar filters occur)", [], - "DetectDuplicateFilters" + "DetectDuplicateFilters", ) } convert( json: { layers: LayerConfigJson[]; themes: LayoutConfigJson[] }, - context: ConversionContext + context: ConversionContext, ): { layers: LayerConfigJson[]; themes: LayoutConfigJson[] } { const { layers, themes } = json const perOsmTag = new Map< @@ -1641,7 +1666,7 @@ export class DetectDuplicateFilters extends DesugaringStep<{ filter: FilterConfigJson }[] >, - layout?: LayoutConfigJson | undefined + layout?: LayoutConfigJson | undefined, ): void { if (layer.filter === undefined || layer.filter === null) { return @@ -1681,7 +1706,7 @@ export class DetectDuplicatePresets extends DesugaringStep { super( "Detects mappings which have identical (english) names or identical mappings.", ["presets"], - "DetectDuplicatePresets" + "DetectDuplicatePresets", ) } @@ -1692,13 +1717,13 @@ export class DetectDuplicatePresets extends DesugaringStep { if (new Set(enNames).size != enNames.length) { const dups = Utils.Duplicates(enNames) const layersWithDup = json.layers.filter((l) => - l.presets.some((p) => dups.indexOf(p.title.textFor("en")) >= 0) + l.presets.some((p) => dups.indexOf(p.title.textFor("en")) >= 0), ) const layerIds = layersWithDup.map((l) => l.id) context.err( `This themes has multiple presets which are named:${dups}, namely layers ${layerIds.join( - ", " - )} this is confusing for contributors and is probably the result of reusing the same layer multiple times. Use \`{"override": {"=presets": []}}\` to remove some presets` + ", ", + )} this is confusing for contributors and is probably the result of reusing the same layer multiple times. Use \`{"override": {"=presets": []}}\` to remove some presets`, ) } @@ -1713,17 +1738,17 @@ export class DetectDuplicatePresets extends DesugaringStep { Utils.SameObject(presetATags, presetBTags) && Utils.sameList( presetA.preciseInput.snapToLayers, - presetB.preciseInput.snapToLayers + presetB.preciseInput.snapToLayers, ) ) { context.err( `This themes has multiple presets with the same tags: ${presetATags.asHumanString( false, false, - {} + {}, )}, namely the preset '${presets[i].title.textFor("en")}' and '${presets[ j - ].title.textFor("en")}'` + ].title.textFor("en")}'`, ) } } @@ -1732,3 +1757,63 @@ export class DetectDuplicatePresets extends DesugaringStep { return json } } + +export class ValidateThemeEnsemble extends Conversion> { + constructor() { + super("Validates that all themes together are logical, i.e. no duplicate ids exists within (overriden) themes", [], "ValidateThemeEnsemble") + } + + convert(json: LayoutConfig[], context: ConversionContext): Map { + + + const idToSource = new Map() + + for (const theme of json) { + for (const layer of theme.layers) { + if (typeof layer.source === "string") { + continue + } + if (Constants.priviliged_layers.indexOf(layer.id) >= 0) { + continue + } + if (!layer.source) { + console.log(theme, layer, layer.source) + context.enters(theme.id, "layers", "source", layer.id).err("No source defined") + continue + } + if (layer.source.geojsonSource) { + continue + } + const id = layer.id + const tags = layer.source.osmTags + if (!idToSource.has(id)) { + idToSource.set(id, { tags, foundInTheme: [theme.id] }) + continue + } + + const oldTags = idToSource.get(id).tags + const oldTheme = idToSource.get(id).foundInTheme + if (oldTags.shadows(tags) && tags.shadows(oldTags)) { + // All is good, all is well + oldTheme.push(theme.id) + continue + } + context.err(["The layer with id '" + id + "' is found in multiple themes with different tag definitions:", + "\t In theme " + oldTheme + ":\t" + oldTags.asHumanString(false, false, {}), + "\tIn theme " + theme.id + ":\t" + tags.asHumanString(false, false, {}), + + + ].join("\n")) + } + } + + + return idToSource + } +} From ee3e000cd166b931faaaf0faa0b0d23feba13b37 Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Fri, 26 Jan 2024 18:18:07 +0100 Subject: [PATCH 07/61] Add polygon merging --- Docs/SettingUpPSQL.md | 2 +- package.json | 2 +- scripts/osm2pgsql/generateBuildDbScript.ts | 274 +++++++++++------- src/Logic/FeatureSource/FeatureSource.ts | 6 + .../Sources/FeatureSourceMerger.ts | 32 +- src/Logic/FeatureSource/Sources/MvtSource.ts | 191 +++++++----- .../DynamicGeoJsonTileSource.ts | 4 +- .../DynamicMvtTileSource.ts | 61 +++- .../TiledFeatureSource/DynamicTileSource.ts | 99 ++++++- src/UI/Map/ShowDataLayer.ts | 1 - src/UI/Test.svelte | 93 +----- 11 files changed, 460 insertions(+), 305 deletions(-) diff --git a/Docs/SettingUpPSQL.md b/Docs/SettingUpPSQL.md index dfa9989c6..e3a4ae13b 100644 --- a/Docs/SettingUpPSQL.md +++ b/Docs/SettingUpPSQL.md @@ -29,7 +29,7 @@ Install osm2pgsql (hint: compile from source is painless) To seed the database: ```` -osm2pgsql -O flex -E 4326 -S build_db.lua -s --flat-nodes=import-help-file -d postgresql://user:password@localhost:5444/osm-poi .osm.pbf +osm2pgsql -O flex -S build_db.lua -s --flat-nodes=import-help-file -d postgresql://user:password@localhost:5444/osm-poi .osm.pbf ```` Storing properties to table '"public"."osm2pgsql_properties" takes about 25 minutes with planet.osm diff --git a/package.json b/package.json index 5f961226b..921eaee85 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "oauth_secret": "NBWGhWDrD3QDB35xtVuxv4aExnmIt4FA_WgeLtwxasg", "url": "https://www.openstreetmap.org" }, - "mvt_layer_server": "http://127.0.0.1:7800/public.{layer}/{z}/{x}/{y}.pbf", + "mvt_layer_server": "http://127.0.0.1:7800/public.{type}_{layer}/{z}/{x}/{y}.pbf", "disabled:oauth_credentials": { "##": "DEV", "#": "This client-id is registered by 'MapComplete' on https://master.apis.dev.openstreetmap.org/", diff --git a/scripts/osm2pgsql/generateBuildDbScript.ts b/scripts/osm2pgsql/generateBuildDbScript.ts index 0fce50b06..4559f4683 100644 --- a/scripts/osm2pgsql/generateBuildDbScript.ts +++ b/scripts/osm2pgsql/generateBuildDbScript.ts @@ -5,91 +5,57 @@ import Script from "../Script" import fs from "fs" import { Or } from "../../src/Logic/Tags/Or" import { RegexTag } from "../../src/Logic/Tags/RegexTag" -import { Utils } from "../../src/Utils" import { ValidateThemeEnsemble } from "../../src/Models/ThemeConfig/Conversion/Validation" import { AllKnownLayouts } from "../../src/Customizations/AllKnownLayouts" class LuaSnippets { - /** - * The main piece of code that calls `process_poi` - */ - static tail = [ - "function osm2pgsql.process_node(object)", - " process_poi(object, object:as_point())", + + public static helpers = [ + "function countTbl(tbl)\n" + + " local c = 0\n" + + " for n in pairs(tbl) do \n" + + " c = c + 1 \n" + + " end\n" + + " return c\n" + "end", - "", - "function osm2pgsql.process_way(object)", - " if object.is_closed then", - " process_poi(object, object:as_polygon():centroid())", - " end", - "end", - ""].join("\n") + ].join("\n") - public static combine(calls: string[]): string { - return [ - `function process_poi(object, geom)`, - ...calls.map(c => " " + c + "(object, geom)"), - `end`, - ].join("\n") - } -} - -class GenerateLayerLua { - private readonly _id: string - private readonly _tags: TagsFilter - private readonly _foundInThemes: string[] - - constructor(id: string, tags: TagsFilter, foundInThemes: string[] = []) { - this._tags = tags - this._id = id - this._foundInThemes = foundInThemes - } - - public functionName() { - if (!this._tags) { - return undefined + public static toLuaFilter(tag: TagsFilter, useParens: boolean = false): string { + if (tag instanceof Tag) { + return `object.tags["${tag.key}"] == "${tag.value}"` } - return `process_poi_${this._id}` - } - - public generateFunction(): string { - if (!this._tags) { - return undefined + if (tag instanceof And) { + const expr = tag.and.map(t => this.toLuaFilter(t, true)).join(" and ") + if (useParens) { + return "(" + expr + ")" + } + return expr } - return [ - `local pois_${this._id} = osm2pgsql.define_table({`, - this._foundInThemes ? "-- used in themes: " + this._foundInThemes.join(", ") : "", - ` name = '${this._id}',`, - " ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' },", - " columns = {", - " { column = 'tags', type = 'jsonb' },", - " { column = 'geom', type = 'point', projection = 4326, not_null = true },", - " }" + - "})", - "", - "", - `function ${this.functionName()}(object, geom)`, - " local matches_filter = " + this.toLuaFilter(this._tags), - " if( not matches_filter) then", - " return", - " end", - " local a = {", - " geom = geom,", - " tags = object.tags", - " }", - " ", - ` pois_${this._id}:insert(a)`, - "end", - "", - ].join("\n") + if (tag instanceof Or) { + const expr = tag.or.map(t => this.toLuaFilter(t, true)).join(" or ") + if (useParens) { + return "(" + expr + ")" + } + return expr + } + if (tag instanceof RegexTag) { + let expr = LuaSnippets.regexTagToLua(tag) + if (useParens) { + expr = "(" + expr + ")" + } + return expr + } + let msg = "Could not handle" + tag.asHumanString(false, false, {}) + console.error(msg) + throw msg } - private regexTagToLua(tag: RegexTag) { + private static regexTagToLua(tag: RegexTag) { if (typeof tag.value === "string" && tag.invert) { return `object.tags["${tag.key}"] ~= "${tag.value}"` } - const v = ( tag.value).source.replace(/\\\//g, "/") + const v = (tag.value).source.replace(/\\\//g, "/") if ("" + tag.value === "/.+/is" && !tag.invert) { return `object.tags["${tag.key}"] ~= nil` @@ -115,35 +81,58 @@ class GenerateLayerLua { return `(object.tags["${tag.key}"] ~= nil and string.find(object.tags["${tag.key}"], "${v}"))` } - private toLuaFilter(tag: TagsFilter, useParens: boolean = false): string { - if (tag instanceof Tag) { - return `object.tags["${tag.key}"] == "${tag.value}"` - } - if (tag instanceof And) { - const expr = tag.and.map(t => this.toLuaFilter(t, true)).join(" and ") - if (useParens) { - return "(" + expr + ")" - } - return expr - } - if (tag instanceof Or) { - const expr = tag.or.map(t => this.toLuaFilter(t, true)).join(" or ") - if (useParens) { - return "(" + expr + ")" - } - return expr - } - if (tag instanceof RegexTag) { - let expr = this.regexTagToLua(tag) - if (useParens) { - expr = "(" + expr + ")" - } - return expr - } - let msg = "Could not handle" + tag.asHumanString(false, false, {}) - console.error(msg) - throw msg +} + +class GenerateLayerLua { + private readonly _id: string + private readonly _tags: TagsFilter + private readonly _foundInThemes: string[] + + constructor(id: string, tags: TagsFilter, foundInThemes: string[] = []) { + this._tags = tags + this._id = id + this._foundInThemes = foundInThemes } + + public generateTables(): string { + if (!this._tags) { + return undefined + } + return [ + `db_tables.pois_${this._id} = osm2pgsql.define_table({`, + this._foundInThemes ? "-- used in themes: " + this._foundInThemes.join(", ") : "", + ` name = 'pois_${this._id}',`, + " ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' },", + " columns = {", + " { column = 'tags', type = 'jsonb' },", + " { column = 'geom', type = 'point', projection = 4326, not_null = true },", + " }", + "})", + "", + `db_tables.lines_${this._id} = osm2pgsql.define_table({`, + this._foundInThemes ? "-- used in themes: " + this._foundInThemes.join(", ") : "", + ` name = 'lines_${this._id}',`, + " ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' },", + " columns = {", + " { column = 'tags', type = 'jsonb' },", + " { column = 'geom', type = 'linestring', projection = 4326, not_null = true },", + " }", + "})", + + `db_tables.polygons_${this._id} = osm2pgsql.define_table({`, + this._foundInThemes ? "-- used in themes: " + this._foundInThemes.join(", ") : "", + ` name = 'polygons_${this._id}',`, + " ids = { type = 'any', type_column = 'osm_type', id_column = 'osm_id' },", + " columns = {", + " { column = 'tags', type = 'jsonb' },", + " { column = 'geom', type = 'polygon', projection = 4326, not_null = true },", + " }", + "})", + "", + ].join("\n") + } + + } class GenerateBuildDbScript extends Script { @@ -163,14 +152,93 @@ class GenerateBuildDbScript extends Script { }) const script = [ - ...generators.map(g => g.generateFunction()), - LuaSnippets.combine(Utils.NoNull(generators.map(g => g.functionName()))), - LuaSnippets.tail, + "local db_tables = {}", + LuaSnippets.helpers, + ...generators.map(g => g.generateTables()), + this.generateProcessPoi(allNeededLayers), + this.generateProcessWay(allNeededLayers), ].join("\n\n\n") const path = "build_db.lua" fs.writeFileSync(path, script, "utf-8") console.log("Written", path) - console.log(allNeededLayers.size+" layers will be created. Make sure to set 'max_connections' to at least "+(10 + allNeededLayers.size) ) + console.log(allNeededLayers.size + " layers will be created with 3 tables each. Make sure to set 'max_connections' to at least " + (10 + 3 * allNeededLayers.size)) + } + + private earlyAbort() { + return [" if countTbl(object.tags) == 0 then", + " return", + " end", + ""].join("\n") + } + + private generateProcessPoi(allNeededLayers: Map) { + const body: string[] = [] + allNeededLayers.forEach(({ tags }, layerId) => { + body.push( + this.insertInto(tags, layerId, "pois_").join("\n"), + ) + }) + + return [ + "function osm2pgsql.process_node(object)", + this.earlyAbort(), + " local geom = object:as_point()", + " local matches_filter = false", + body.join("\n"), + "end", + ].join("\n") + } + + /** + * If matches_filter + * @param tags + * @param layerId + * @param tableprefix + * @private + */ + private insertInto(tags: TagsFilter, layerId: string, tableprefix: "pois_" | "lines_" | "polygons_") { + const filter = LuaSnippets.toLuaFilter(tags) + return [ + " matches_filter = " + filter, + " if matches_filter then", + " db_tables." + tableprefix + layerId + ":insert({", + " geom = geom,", + " tags = object.tags", + " })", + " end", + ] + } + + private generateProcessWay(allNeededLayers: Map) { + const bodyLines: string[] = [] + allNeededLayers.forEach(({ tags }, layerId) => { + bodyLines.push(this.insertInto(tags, layerId, "lines_").join("\n")) + }) + + const bodyPolygons: string[] = [] + allNeededLayers.forEach(({ tags }, layerId) => { + bodyPolygons.push(this.insertInto(tags, layerId, "polygons_").join("\n")) + }) + + return [ + "function process_polygon(object, geom)", + " local matches_filter", + ...bodyPolygons, + "end", + "function process_linestring(object, geom)", + " local matches_filter", + ...bodyLines, + "end", + "", + "function osm2pgsql.process_way(object)", + this.earlyAbort(), + " if object.is_closed then", + " process_polygon(object, object:as_polygon())", + " else", + " process_linestring(object, object:as_linestring())", + " end", + "end", + ].join("\n") } } diff --git a/src/Logic/FeatureSource/FeatureSource.ts b/src/Logic/FeatureSource/FeatureSource.ts index 3b9798dbb..bd5ecbfc2 100644 --- a/src/Logic/FeatureSource/FeatureSource.ts +++ b/src/Logic/FeatureSource/FeatureSource.ts @@ -16,6 +16,12 @@ export interface FeatureSourceForLayer extends Feat readonly layer: FilteredLayer } +export interface FeatureSourceForTile extends FeatureSource { + readonly x: number + readonly y: number + readonly z: number + +} /** * A feature source which is aware of the indexes it contains */ diff --git a/src/Logic/FeatureSource/Sources/FeatureSourceMerger.ts b/src/Logic/FeatureSource/Sources/FeatureSourceMerger.ts index 951b688c5..7ea4d1635 100644 --- a/src/Logic/FeatureSource/Sources/FeatureSourceMerger.ts +++ b/src/Logic/FeatureSource/Sources/FeatureSourceMerger.ts @@ -2,43 +2,49 @@ import { Store, UIEventSource } from "../../UIEventSource" import { FeatureSource, IndexedFeatureSource } from "../FeatureSource" import { Feature } from "geojson" import { Utils } from "../../../Utils" +import DynamicTileSource from "../TiledFeatureSource/DynamicTileSource" /** - * + * The featureSourceMerger receives complete geometries from various sources. + * If multiple sources contain the same object (as determined by 'id'), only one copy of them is retained */ -export default class FeatureSourceMerger implements IndexedFeatureSource { +export default class FeatureSourceMerger implements IndexedFeatureSource { public features: UIEventSource = new UIEventSource([]) public readonly featuresById: Store> - private readonly _featuresById: UIEventSource> - private readonly _sources: FeatureSource[] = [] + protected readonly _featuresById: UIEventSource> + private readonly _sources: Src[] = [] /** * Merges features from different featureSources. * In case that multiple features have the same id, the latest `_version_number` will be used. Otherwise, we will take the last one */ - constructor(...sources: FeatureSource[]) { + constructor(...sources: Src[]) { this._featuresById = new UIEventSource>(new Map()) this.featuresById = this._featuresById const self = this sources = Utils.NoNull(sources) for (let source of sources) { source.features.addCallback(() => { - self.addData(sources.map((s) => s.features.data)) + self.addDataFromSources(sources) }) } - this.addData(sources.map((s) => s.features.data)) + this.addDataFromSources(sources) this._sources = sources } - public addSource(source: FeatureSource) { + public addSource(source: Src) { if (!source) { return } this._sources.push(source) source.features.addCallbackAndRun(() => { - this.addData(this._sources.map((s) => s.features.data)) + this.addDataFromSources(this._sources) }) } + protected addDataFromSources(sources: Src[]){ + this.addData(sources.map(s => s.features.data)) + } + protected addData(sources: Feature[][]) { sources = Utils.NoNull(sources) let somethingChanged = false @@ -56,7 +62,7 @@ export default class FeatureSourceMerger implements IndexedFeatureSource { const id = f.properties.id unseen.delete(id) if (!all.has(id)) { - // This is a new feature + // This is a new, previously unseen feature somethingChanged = true all.set(id, f) continue @@ -81,10 +87,8 @@ export default class FeatureSourceMerger implements IndexedFeatureSource { return } - const newList = [] - all.forEach((value) => { - newList.push(value) - }) + const newList = Array.from(all.values()) + this.features.setData(newList) this._featuresById.setData(all) } diff --git a/src/Logic/FeatureSource/Sources/MvtSource.ts b/src/Logic/FeatureSource/Sources/MvtSource.ts index 8c93eee9c..3aeb34400 100644 --- a/src/Logic/FeatureSource/Sources/MvtSource.ts +++ b/src/Logic/FeatureSource/Sources/MvtSource.ts @@ -1,6 +1,6 @@ import { Feature, Geometry } from "geojson" import { Store, UIEventSource } from "../../UIEventSource" -import { FeatureSource } from "../FeatureSource" +import { FeatureSourceForTile } from "../FeatureSource" import Pbf from "pbf" import * as pbfCompile from "pbf/compile" import * as PbfSchema from "protocol-buffers-schema" @@ -19,8 +19,67 @@ class MvtFeatureBuilder { this._y0 = extent * y } - public toGeoJson(geometry, typeIndex, properties): Feature { - let coords: [number, number] | Coords | Coords[] = this.encodeGeometry(geometry) + private static signedArea(ring: Coords): number { + let sum = 0 + const len = ring.length + // J is basically (i - 1) % len + let j = len - 1 + let p1 + let p2 + for (let i = 0; i < len; i++) { + p1 = ring[i] + p2 = ring[j] + sum += (p2.x - p1.x) * (p1.y + p2.y) + j = i + } + return sum + } + + /** + * + * const rings = [ [ [ 3.208361864089966, 51.186908820014736 ], [ 3.2084155082702637, 51.18689537073311 ], [ 3.208436965942383, 51.186888646090836 ], [ 3.2084155082702637, 51.18686174751187 ], [ 3.2084155082702637, 51.18685502286465 ], [ 3.2083725929260254, 51.18686847215807 ], [ 3.2083404064178467, 51.18687519680333 ], [ 3.208361864089966, 51.186908820014736 ] ] ] + * MvtFeatureBuilder.classifyRings(rings) // => [rings] + */ + private static classifyRings(rings: Coords[]): Coords[][] { + if (rings.length <= 0) { + throw "Now rings in polygon found" + } + if (rings.length == 1) { + return [rings] + } + + const polygons: Coords[][] = [] + let currentPolygon: Coords[] + + for (let i = 0; i < rings.length; i++) { + let ring = rings[i] + const area = this.signedArea(ring) + if (area === 0) { + // Weird, degenerate ring + continue + } + const ccw = area < 0 + + if (ccw === (area < 0)) { + if (currentPolygon) { + polygons.push(currentPolygon) + } + currentPolygon = [ring] + + } else { + currentPolygon.push(ring) + } + } + if (currentPolygon) { + polygons.push(currentPolygon) + } + + return polygons + } + + public toGeoJson(geometry: number[], typeIndex: 1 | 2 | 3, properties: any): Feature { + let coords: Coords[] = this.encodeGeometry(geometry) + let classified = undefined switch (typeIndex) { case 1: const points = [] @@ -38,9 +97,9 @@ class MvtFeatureBuilder { break case 3: - let classified = this.classifyRings(coords) - for (let i = 0; i < coords.length; i++) { - for (let j = 0; j < coords[i].length; j++) { + classified = MvtFeatureBuilder.classifyRings(coords) + for (let i = 0; i < classified.length; i++) { + for (let j = 0; j < classified[i].length; j++) { this.project(classified[i][j]) } } @@ -48,9 +107,11 @@ class MvtFeatureBuilder { } let type: string = MvtFeatureBuilder.geom_types[typeIndex] + let polygonCoords: Coords | Coords[] | Coords[][] if (coords.length === 1) { - coords = coords[0] + polygonCoords = (classified ?? coords)[0] } else { + polygonCoords = classified ?? coords type = "Multi" + type } @@ -58,13 +119,22 @@ class MvtFeatureBuilder { type: "Feature", geometry: { type: type, - coordinates: coords, + coordinates: polygonCoords, }, properties, } } - private encodeGeometry(geometry: number[]) { + /** + * + * const geometry = [9,233,8704,130,438,1455,270,653,248,423,368,493,362,381,330,267,408,301,406,221,402,157,1078,429,1002,449,1036,577,800,545,1586,1165,164,79,40] + * const builder = new MvtFeatureBuilder(4096, 66705, 43755, 17) + * const expected = [[3.2106759399175644,51.213658395282124],[3.2108227908611298,51.21396418776169],[3.2109133154153824,51.21410154168976],[3.210996463894844,51.214190590500664],[3.211119845509529,51.214294340548975],[3.211241215467453,51.2143745681588],[3.2113518565893173,51.21443085341426],[3.211488649249077,51.21449427925393],[3.2116247713565826,51.214540903490956],[3.211759552359581,51.21457408647774],[3.2121209800243378,51.214664394485254],[3.212456926703453,51.21475890267553],[3.2128042727708817,51.214880292910834],[3.213072493672371,51.214994962285544],[3.2136042416095734,51.21523984134939],[3.2136592268943787,51.21525664260963],[3.213672637939453,51.21525664260963]] + * builder.project(builder.encodeGeometry(geometry)[0]) // => expected + * @param geometry + * @private + */ + private encodeGeometry(geometry: number[]): Coords[] { let cX = 0 let cY = 0 let coordss: Coords[] = [] @@ -86,7 +156,7 @@ class MvtFeatureBuilder { currentRing = [] } } - if (commandId === 1 || commandId === 2){ + if (commandId === 1 || commandId === 2) { for (let j = 0; j < commandCount; j++) { const dx = geometry[i + j * 2 + 1] cX += ((dx >> 1) ^ (-(dx & 1))) @@ -94,10 +164,11 @@ class MvtFeatureBuilder { cY += ((dy >> 1) ^ (-(dy & 1))) currentRing.push([cX, cY]) } - i = commandCount * 2 + i += commandCount * 2 } - if(commandId === 7){ + if (commandId === 7) { currentRing.push([...currentRing[0]]) + i++ } } @@ -107,62 +178,12 @@ class MvtFeatureBuilder { return coordss } - private signedArea(ring: Coords): number { - let sum = 0 - const len = ring.length - // J is basically (i - 1) % len - let j = len - 1 - let p1 - let p2 - for (let i = 0; i < len; i++) { - p1 = ring[i] - p2 = ring[j] - sum += (p2.x - p1.x) * (p1.y + p2.y) - j = i - } - return sum - } - - private classifyRings(rings: Coords[]): Coords[][] { - const len = rings.length - - if (len <= 1) return [rings] - - const polygons = [] - let polygon - // CounterClockWise - let ccw: boolean - - for (let i = 0; i < len; i++) { - const area = this.signedArea(rings[i]) - if (area === 0) continue - - if (ccw === undefined) { - ccw = area < 0 - } - if (ccw === (area < 0)) { - if (polygon) { - polygons.push(polygon) - } - polygon = [rings[i]] - - } else { - polygon.push(rings[i]) - } - } - if (polygon) { - polygons.push(polygon) - } - - return polygons - } - /** * Inline replacement of the location by projecting - * @param line - * @private + * @param line the line which will be rewritten inline + * @return line */ - private project(line: [number, number][]) { + private project(line: Coords) { const y0 = this._y0 const x0 = this._x0 const size = this._size @@ -174,12 +195,13 @@ class MvtFeatureBuilder { 360 / Math.PI * Math.atan(Math.exp(y2 * Math.PI / 180)) - 90, ] } + return line } } -export default class MvtSource implements FeatureSource { +export default class MvtSource implements FeatureSourceForTile { - private static readonly schemaSpec = ` + private static readonly schemaSpec21 = ` package vector_tile; option optimize_for = LITE_RUNTIME; @@ -259,26 +281,30 @@ message Tile { extensions 16 to 8191; } ` - private static readonly tile_schema = pbfCompile(PbfSchema.parse(MvtSource.schemaSpec)).Tile - - + private static readonly tile_schema = (pbfCompile.default ?? pbfCompile)(PbfSchema.parse(MvtSource.schemaSpec21)).Tile + public readonly features: Store[]> private readonly _url: string private readonly _layerName: string private readonly _features: UIEventSource[]> = new UIEventSource[]>([]) - public readonly features: Store[]> = this._features - private readonly x: number - private readonly y: number - private readonly z: number + public readonly x: number + public readonly y: number + public readonly z: number - constructor(url: string, x: number, y: number, z: number, layerName?: string) { + constructor(url: string, x: number, y: number, z: number, layerName?: string, isActive?: Store) { this._url = url this._layerName = layerName this.x = x this.y = y this.z = z this.downloadSync() + this.features = this._features.map(fs => { + if (fs === undefined || isActive?.data === false) { + return [] + } + return fs + }, [isActive]) } private getValue(v: { @@ -316,16 +342,23 @@ message Tile { } - private downloadSync(){ + private downloadSync() { this.download().then(d => { - if(d.length === 0){ + if (d.length === 0) { return } return this._features.setData(d) - }).catch(e => {console.error(e)}) + }).catch(e => { + console.error(e) + }) } + private async download(): Promise { const result = await fetch(this._url) + if (result.status !== 200) { + console.error("Could not download tile " + this._url) + return [] + } const buffer = await result.arrayBuffer() const data = MvtSource.tile_schema.read(new Pbf(buffer)) const layers = data.layers @@ -336,7 +369,7 @@ message Tile { } layer = layers.find(l => l.name === this._layerName) } - if(!layer){ + if (!layer) { return [] } const builder = new MvtFeatureBuilder(layer.extent, this.x, this.y, this.z) diff --git a/src/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts b/src/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts index 88455e9bb..010af52ff 100644 --- a/src/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts +++ b/src/Logic/FeatureSource/TiledFeatureSource/DynamicGeoJsonTileSource.ts @@ -1,4 +1,4 @@ -import { Store } from "../../UIEventSource" +import { ImmutableStore, Store } from "../../UIEventSource" import DynamicTileSource from "./DynamicTileSource" import { Utils } from "../../../Utils" import GeoJsonSource from "../Sources/GeoJsonSource" @@ -65,7 +65,7 @@ export default class DynamicGeoJsonTileSource extends DynamicTileSource { const blackList = new Set() super( - source.geojsonZoomLevel, + new ImmutableStore(source.geojsonZoomLevel), layer.minzoom, (zxy) => { if (whitelist !== undefined) { diff --git a/src/Logic/FeatureSource/TiledFeatureSource/DynamicMvtTileSource.ts b/src/Logic/FeatureSource/TiledFeatureSource/DynamicMvtTileSource.ts index ac8713332..3b7476768 100644 --- a/src/Logic/FeatureSource/TiledFeatureSource/DynamicMvtTileSource.ts +++ b/src/Logic/FeatureSource/TiledFeatureSource/DynamicMvtTileSource.ts @@ -1,13 +1,45 @@ import { Store } from "../../UIEventSource" -import DynamicTileSource from "./DynamicTileSource" +import DynamicTileSource, { PolygonSourceMerger } from "./DynamicTileSource" import { Utils } from "../../../Utils" import { BBox } from "../../BBox" import LayerConfig from "../../../Models/ThemeConfig/LayerConfig" import MvtSource from "../Sources/MvtSource" import { Tiles } from "../../../Models/TileRange" import Constants from "../../../Models/Constants" +import FeatureSourceMerger from "../Sources/FeatureSourceMerger" -export default class DynamicMvtileSource extends DynamicTileSource { + +class PolygonMvtSource extends PolygonSourceMerger{ + constructor( layer: LayerConfig, + mapProperties: { + zoom: Store + bounds: Store + }, + options?: { + isActive?: Store + }) { + const roundedZoom = mapProperties.zoom.mapD(z => Math.min(Math.floor(z/2)*2, 14)) + super( + roundedZoom, + layer.minzoom, + (zxy) => { + const [z, x, y] = Tiles.tile_from_index(zxy) + const url = Utils.SubstituteKeys(Constants.VectorTileServer, + { + z, x, y, layer: layer.id, + type: "polygons", + }) + return new MvtSource(url, x, y, z) + }, + mapProperties, + { + isActive: options?.isActive, + }) + } +} + + +class PointMvtSource extends DynamicTileSource { constructor( layer: LayerConfig, @@ -19,14 +51,16 @@ export default class DynamicMvtileSource extends DynamicTileSource { isActive?: Store }, ) { + const roundedZoom = mapProperties.zoom.mapD(z => Math.min(Math.floor(z/2)*2, 14)) super( - mapProperties.zoom, + roundedZoom, layer.minzoom, (zxy) => { const [z, x, y] = Tiles.tile_from_index(zxy) const url = Utils.SubstituteKeys(Constants.VectorTileServer, { z, x, y, layer: layer.id, + type: "pois", }) return new MvtSource(url, x, y, z) }, @@ -37,3 +71,24 @@ export default class DynamicMvtileSource extends DynamicTileSource { ) } } + +export default class DynamicMvtileSource extends FeatureSourceMerger { + + constructor( + layer: LayerConfig, + mapProperties: { + zoom: Store + bounds: Store + }, + options?: { + isActive?: Store + }, + ) { + const roundedZoom = mapProperties.zoom.mapD(z => Math.floor(z)) + super( + new PointMvtSource(layer, mapProperties, options), + new PolygonMvtSource(layer, mapProperties, options) + + ) + } +} diff --git a/src/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts b/src/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts index 3bb5affd9..f905ae893 100644 --- a/src/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts +++ b/src/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts @@ -1,25 +1,37 @@ import { Store, Stores } from "../../UIEventSource" import { Tiles } from "../../../Models/TileRange" import { BBox } from "../../BBox" -import { FeatureSource } from "../FeatureSource" +import { FeatureSource, FeatureSourceForTile } from "../FeatureSource" import FeatureSourceMerger from "../Sources/FeatureSourceMerger" +import { Feature } from "geojson" +import { Utils } from "../../../Utils" +import { GeoOperations } from "../../GeoOperations" + /*** * A tiled source which dynamically loads the required tiles at a fixed zoom level. * A single featureSource will be initialized for every tile in view; which will later be merged into this featureSource */ -export default class DynamicTileSource extends FeatureSourceMerger { +export default class DynamicTileSource extends FeatureSourceMerger { + /** + * + * @param zoomlevel If {z} is specified in the source, the 'zoomlevel' will be used as zoomlevel to download from + * @param minzoom Only activate this feature source if zoomed in further then this + * @param constructSource + * @param mapProperties + * @param options + */ constructor( zoomlevel: Store, minzoom: number, - constructSource: (tileIndex: number) => FeatureSource, + constructSource: (tileIndex: number) => Src, mapProperties: { bounds: Store zoom: Store }, options?: { isActive?: Store - } + }, ) { super() const loadedTiles = new Set() @@ -34,32 +46,32 @@ export default class DynamicTileSource extends FeatureSourceMerger { if (mapProperties.zoom.data < minzoom) { return undefined } - const z = Math.round(zoomlevel.data) + const z = Math.floor(zoomlevel.data) const tileRange = Tiles.TileRangeBetween( z, bounds.getNorth(), bounds.getEast(), bounds.getSouth(), - bounds.getWest() + bounds.getWest(), ) if (tileRange.total > 500) { console.warn( - "Got a really big tilerange, bounds and location might be out of sync" + "Got a really big tilerange, bounds and location might be out of sync", ) return undefined } const needed = Tiles.MapRange(tileRange, (x, y) => - Tiles.tile_index(z, x, y) + Tiles.tile_index(z, x, y), ).filter((i) => !loadedTiles.has(i)) if (needed.length === 0) { return undefined } return needed }, - [options?.isActive, mapProperties.zoom] + [options?.isActive, mapProperties.zoom], ) - .stabilized(250) + .stabilized(250), ) neededTiles.addCallbackAndRunD((neededIndexes) => { @@ -70,3 +82,70 @@ export default class DynamicTileSource extends FeatureSourceMerger { }) } } + + +/** + * The PolygonSourceMerger receives various small pieces of bigger polygons and stitches them together. + * This is used to reconstruct polygons of vector tiles + */ +export class PolygonSourceMerger extends DynamicTileSource { + constructor( + zoomlevel: Store, + minzoom: number, + constructSource: (tileIndex: number) => FeatureSourceForTile, + mapProperties: { + bounds: Store + zoom: Store + }, + options?: { + isActive?: Store + }, + ) { + super(zoomlevel, minzoom, constructSource, mapProperties, options) + } + + protected addDataFromSources(sources: FeatureSourceForTile[]) { + sources = Utils.NoNull(sources) + const all: Map = new Map() + const zooms: Map = new Map() + + for (const source of sources) { + let z = source.z + for (const f of source.features.data) { + const id = f.properties.id + if(id.endsWith("146616907")){ + console.log("Horeca totaal") + } + if (!all.has(id)) { + // No other parts of this polygon have been seen before, simply add it + all.set(id, f) + zooms.set(id, z) + continue + } + + // A part of this object has been seen before, eventually from a different zoom level + const oldV = all.get(id) + const oldZ = zooms.get(id) + if (oldZ > z) { + // The store contains more detailed information, so we ignore this part which has a lower accuraccy + continue + } + if (oldZ < z) { + // The old value has worse accuracy then what we receive now, we throw it away + all.set(id, f) + zooms.set(id, z) + continue + } + const merged = GeoOperations.union(f, oldV) + merged.properties = oldV.properties + all.set(id, merged) + zooms.set(id, z) + } + } + + const newList = Array.from(all.values()) + this.features.setData(newList) + this._featuresById.setData(all) + } + +} diff --git a/src/UI/Map/ShowDataLayer.ts b/src/UI/Map/ShowDataLayer.ts index 64a3cfe94..75c7213b0 100644 --- a/src/UI/Map/ShowDataLayer.ts +++ b/src/UI/Map/ShowDataLayer.ts @@ -303,7 +303,6 @@ class LineRenderingLayer { type: "FeatureCollection", features, }, - cluster: true, promoteId: "id", }) const linelayer = this._layername + "_line" diff --git a/src/UI/Test.svelte b/src/UI/Test.svelte index 70f864f1b..4e278c4b1 100644 --- a/src/UI/Test.svelte +++ b/src/UI/Test.svelte @@ -4,20 +4,18 @@ import MaplibreMap from "./Map/MaplibreMap.svelte" import { Map as MlMap } from "maplibre-gl" import { MapLibreAdaptor } from "./Map/MapLibreAdaptor" - import Constants from "../Models/Constants" - import toilet from "../assets/generated/layers/toilet.json" + import shops from "../assets/generated/layers/shops.json" import LayerConfig from "../Models/ThemeConfig/LayerConfig" import DynamicMvtileSource from "../Logic/FeatureSource/TiledFeatureSource/DynamicMvtTileSource" import ShowDataLayer from "./Map/ShowDataLayer" - const tl = new LayerConfig(toilet) + const tl = new LayerConfig(shops) let map: UIEventSource = new UIEventSource(undefined) let adaptor = new MapLibreAdaptor(map) const src = new DynamicMvtileSource(tl, adaptor) - src.features.addCallbackAndRun(f => console.log(">>> Features are", f)) new ShowDataLayer(map, { layer: tl, features: src @@ -27,94 +25,7 @@ lat: 51.2095, lon: 3.2260, }) adaptor.zoom.setData(13) - const loadedIcons = new Set() - async function loadImage(map: MlMap, url: string, name: string): Promise { - return new Promise((resolve, reject) => { - if (loadedIcons.has(name)) { - return new Promise((resolve, reject) => resolve()) - } - loadedIcons.add(name) - if (Constants.defaultPinIcons.indexOf(url) >= 0) { - url = "./assets/svg/" + url + ".svg" - } - map.loadImage( - url, - (error, image) => { - if (error) { - reject(error) - } - map.addImage(name, image) - resolve() - }) - }) - } - - map.addCallbackAndRunD(map => { - map.on("load", async () => { - console.log("Onload") - await loadImage(map, "https://upload.wikimedia.org/wikipedia/commons/7/7c/201408_cat.png", "cat") - - /* - map.addSource("drinking_water", { - "type": "vector", - "tiles": ["http://127.0.0.2:7800/public.drinking_water/{z}/{x}/{y}.pbf"], // http://127.0.0.2:7800/public.drinking_water.json", - }) - - map.addLayer( - { - "id": "drinking_water_layer", - "type": "circle", - "source": "drinking_water", - "source-layer": "public.drinking_water", - "paint": { - "circle-radius": 5, - "circle-color": "#ff00ff", - "circle-stroke-width": 2, - "circle-stroke-color": "#000000", - }, - }, - )*/ - /* - map.addSource("toilet", { - "type": "vector", - "tiles": ["http://127.0.0.2:7800/public.toilet/{z}/{x}/{y}.pbf"], // http://127.0.0.2:7800/public.drinking_water.json", - }) - - map.addLayer( - { - "id": "toilet_layer", - "type": "circle", - "source": "toilet", - "source-layer": "public.toilet", - "paint": { - "circle-radius": 5, - "circle-color": "#0000ff", - "circle-stroke-width": 2, - "circle-stroke-color": "#000000", - }, - }, - ) - map.addLayer({ - "id": "points", - "type": "symbol", - "source": "toilet", - "source-layer": "public.toilet", - "layout": { - "icon-overlap": "always", - "icon-image": "cat", - "icon-size": 0.05, - }, - })*/ - - - map.on("click", "drinking_water_layer", (e) => { -// Copy coordinates array. - console.log(e) - console.warn(">>>", e.features[0]) - }) - }) - })
From 5b318236bfece8eb7a43d4e67ff676bcf3f54fc8 Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Fri, 2 Feb 2024 20:04:39 +0100 Subject: [PATCH 08/61] LayerServer: fix some bugs in build_db-script, decode MVT-tilesource for more features --- scripts/osm2pgsql/generateBuildDbScript.ts | 37 +++++- .../FeatureSource/Sources/LayoutSource.ts | 8 +- .../DynamicMvtTileSource.ts | 36 ++++- .../TiledFeatureSource/DynamicTileSource.ts | 70 +--------- .../TiledFeatureSource/LineSourceMerger.ts | 80 +++++++++++ .../TiledFeatureSource/PolygonSourceMerger.ts | 73 +++++++++++ src/Logic/GeoOperations.ts | 124 ++++++++++++++---- src/Logic/Osm/OsmObject.ts | 2 +- 8 files changed, 327 insertions(+), 103 deletions(-) create mode 100644 src/Logic/FeatureSource/TiledFeatureSource/LineSourceMerger.ts create mode 100644 src/Logic/FeatureSource/TiledFeatureSource/PolygonSourceMerger.ts diff --git a/scripts/osm2pgsql/generateBuildDbScript.ts b/scripts/osm2pgsql/generateBuildDbScript.ts index 4559f4683..5f6eb28b7 100644 --- a/scripts/osm2pgsql/generateBuildDbScript.ts +++ b/scripts/osm2pgsql/generateBuildDbScript.ts @@ -7,6 +7,7 @@ import { Or } from "../../src/Logic/Tags/Or" import { RegexTag } from "../../src/Logic/Tags/RegexTag" import { ValidateThemeEnsemble } from "../../src/Models/ThemeConfig/Conversion/Validation" import { AllKnownLayouts } from "../../src/Customizations/AllKnownLayouts" +import { OsmObject } from "../../src/Logic/Osm/OsmObject" class LuaSnippets { @@ -20,6 +21,33 @@ class LuaSnippets { "end", ].join("\n") + public static isPolygonFeature(): { blacklist: TagsFilter, whitelisted: TagsFilter } { + const dict = OsmObject.polygonFeatures + const or: TagsFilter[] = [] + const blacklisted : TagsFilter[] = [] + dict.forEach(({ values, blacklist }, k) => { + if(blacklist){ + if(values === undefined){ + blacklisted.push(new RegexTag(k, /.+/is)) + return + } + values.forEach(v => { + blacklisted.push(new RegexTag(k, v)) + }) + return + } + if (values === undefined || values === null) { + or.push(new RegexTag(k, /.+/is)) + return + } + values.forEach(v => { + or.push(new RegexTag(k, v)) + }) + }) + console.log("Polygon features are:", or.map(t => t.asHumanString(false, false, {}))) + return { blacklist: new Or(blacklisted), whitelisted: new Or(or) } + } + public static toLuaFilter(tag: TagsFilter, useParens: boolean = false): string { if (tag instanceof Tag) { return `object.tags["${tag.key}"] == "${tag.value}"` @@ -55,6 +83,10 @@ class LuaSnippets { return `object.tags["${tag.key}"] ~= "${tag.value}"` } + if (typeof tag.value === "string" && !tag.invert) { + return `object.tags["${tag.key}"] == "${tag.value}"` + } + const v = (tag.value).source.replace(/\\\//g, "/") if ("" + tag.value === "/.+/is" && !tag.invert) { @@ -220,6 +252,7 @@ class GenerateBuildDbScript extends Script { bodyPolygons.push(this.insertInto(tags, layerId, "polygons_").join("\n")) }) + const isPolygon = LuaSnippets.isPolygonFeature() return [ "function process_polygon(object, geom)", " local matches_filter", @@ -232,7 +265,9 @@ class GenerateBuildDbScript extends Script { "", "function osm2pgsql.process_way(object)", this.earlyAbort(), - " if object.is_closed then", + " local object_is_line = not object.is_closed or "+LuaSnippets.toLuaFilter(isPolygon.blacklist), + ` local object_is_area = object.is_closed and (object.tags["area"] == "yes" or (not object_is_line and ${LuaSnippets.toLuaFilter(isPolygon.whitelisted, true)}))`, + " if object_is_area then", " process_polygon(object, object:as_polygon())", " else", " process_linestring(object, object:as_linestring())", diff --git a/src/Logic/FeatureSource/Sources/LayoutSource.ts b/src/Logic/FeatureSource/Sources/LayoutSource.ts index a46d81384..9361217ce 100644 --- a/src/Logic/FeatureSource/Sources/LayoutSource.ts +++ b/src/Logic/FeatureSource/Sources/LayoutSource.ts @@ -59,7 +59,7 @@ export default class LayoutSource extends FeatureSourceMerger { zoom, featureSwitches )//*/ - +/* const osmApiSource = LayoutSource.setupOsmApiSource( osmLayers, bounds, @@ -67,14 +67,14 @@ export default class LayoutSource extends FeatureSourceMerger { backend, featureSwitches, fullNodeDatabaseSource - ) + )*/ const geojsonSources: FeatureSource[] = geojsonlayers.map((l) => LayoutSource.setupGeojsonSource(l, mapProperties, isDisplayed(l.id)) ) - super(osmApiSource, ...geojsonSources, ...fromCache, ...mvtSources) + super(...geojsonSources, ...fromCache, ...mvtSources) const self = this function setIsLoading() { @@ -83,7 +83,7 @@ export default class LayoutSource extends FeatureSourceMerger { } // overpassSource?.runningQuery?.addCallbackAndRun((_) => setIsLoading()) - osmApiSource?.isRunning?.addCallbackAndRun((_) => setIsLoading()) + // osmApiSource?.isRunning?.addCallbackAndRun((_) => setIsLoading()) } private static setupMvtSource(layer: LayerConfig, mapProperties: { zoom: Store; bounds: Store }, isActive?: Store): FeatureSource{ diff --git a/src/Logic/FeatureSource/TiledFeatureSource/DynamicMvtTileSource.ts b/src/Logic/FeatureSource/TiledFeatureSource/DynamicMvtTileSource.ts index 3b7476768..ed379bfb1 100644 --- a/src/Logic/FeatureSource/TiledFeatureSource/DynamicMvtTileSource.ts +++ b/src/Logic/FeatureSource/TiledFeatureSource/DynamicMvtTileSource.ts @@ -1,5 +1,5 @@ import { Store } from "../../UIEventSource" -import DynamicTileSource, { PolygonSourceMerger } from "./DynamicTileSource" +import DynamicTileSource from "./DynamicTileSource" import { Utils } from "../../../Utils" import { BBox } from "../../BBox" import LayerConfig from "../../../Models/ThemeConfig/LayerConfig" @@ -7,6 +7,8 @@ import MvtSource from "../Sources/MvtSource" import { Tiles } from "../../../Models/TileRange" import Constants from "../../../Models/Constants" import FeatureSourceMerger from "../Sources/FeatureSourceMerger" +import { LineSourceMerger } from "./LineSourceMerger" +import { PolygonSourceMerger } from "./PolygonSourceMerger" class PolygonMvtSource extends PolygonSourceMerger{ @@ -39,6 +41,36 @@ class PolygonMvtSource extends PolygonSourceMerger{ } +class LineMvtSource extends LineSourceMerger{ + constructor( layer: LayerConfig, + mapProperties: { + zoom: Store + bounds: Store + }, + options?: { + isActive?: Store + }) { + const roundedZoom = mapProperties.zoom.mapD(z => Math.min(Math.floor(z/2)*2, 14)) + super( + roundedZoom, + layer.minzoom, + (zxy) => { + const [z, x, y] = Tiles.tile_from_index(zxy) + const url = Utils.SubstituteKeys(Constants.VectorTileServer, + { + z, x, y, layer: layer.id, + type: "lines", + }) + return new MvtSource(url, x, y, z) + }, + mapProperties, + { + isActive: options?.isActive, + }) + } +} + + class PointMvtSource extends DynamicTileSource { constructor( @@ -84,9 +116,9 @@ export default class DynamicMvtileSource extends FeatureSourceMerger { isActive?: Store }, ) { - const roundedZoom = mapProperties.zoom.mapD(z => Math.floor(z)) super( new PointMvtSource(layer, mapProperties, options), + new LineMvtSource(layer, mapProperties, options), new PolygonMvtSource(layer, mapProperties, options) ) diff --git a/src/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts b/src/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts index f905ae893..efd109fd5 100644 --- a/src/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts +++ b/src/Logic/FeatureSource/TiledFeatureSource/DynamicTileSource.ts @@ -1,11 +1,8 @@ import { Store, Stores } from "../../UIEventSource" import { Tiles } from "../../../Models/TileRange" import { BBox } from "../../BBox" -import { FeatureSource, FeatureSourceForTile } from "../FeatureSource" +import { FeatureSource } from "../FeatureSource" import FeatureSourceMerger from "../Sources/FeatureSourceMerger" -import { Feature } from "geojson" -import { Utils } from "../../../Utils" -import { GeoOperations } from "../../GeoOperations" /*** @@ -84,68 +81,3 @@ export default class DynamicTileSource { - constructor( - zoomlevel: Store, - minzoom: number, - constructSource: (tileIndex: number) => FeatureSourceForTile, - mapProperties: { - bounds: Store - zoom: Store - }, - options?: { - isActive?: Store - }, - ) { - super(zoomlevel, minzoom, constructSource, mapProperties, options) - } - - protected addDataFromSources(sources: FeatureSourceForTile[]) { - sources = Utils.NoNull(sources) - const all: Map = new Map() - const zooms: Map = new Map() - - for (const source of sources) { - let z = source.z - for (const f of source.features.data) { - const id = f.properties.id - if(id.endsWith("146616907")){ - console.log("Horeca totaal") - } - if (!all.has(id)) { - // No other parts of this polygon have been seen before, simply add it - all.set(id, f) - zooms.set(id, z) - continue - } - - // A part of this object has been seen before, eventually from a different zoom level - const oldV = all.get(id) - const oldZ = zooms.get(id) - if (oldZ > z) { - // The store contains more detailed information, so we ignore this part which has a lower accuraccy - continue - } - if (oldZ < z) { - // The old value has worse accuracy then what we receive now, we throw it away - all.set(id, f) - zooms.set(id, z) - continue - } - const merged = GeoOperations.union(f, oldV) - merged.properties = oldV.properties - all.set(id, merged) - zooms.set(id, z) - } - } - - const newList = Array.from(all.values()) - this.features.setData(newList) - this._featuresById.setData(all) - } - -} diff --git a/src/Logic/FeatureSource/TiledFeatureSource/LineSourceMerger.ts b/src/Logic/FeatureSource/TiledFeatureSource/LineSourceMerger.ts new file mode 100644 index 000000000..8a1cb3909 --- /dev/null +++ b/src/Logic/FeatureSource/TiledFeatureSource/LineSourceMerger.ts @@ -0,0 +1,80 @@ +import { FeatureSourceForTile } from "../FeatureSource" +import { Store } from "../../UIEventSource" +import { BBox } from "../../BBox" +import { Utils } from "../../../Utils" +import { Feature, LineString, MultiLineString, Position } from "geojson" +import { Tiles } from "../../../Models/TileRange" +import { GeoOperations } from "../../GeoOperations" +import DynamicTileSource from "./DynamicTileSource" + +/** + * The PolygonSourceMerger receives various small pieces of bigger polygons and stitches them together. + * This is used to reconstruct polygons of vector tiles + */ +export class LineSourceMerger extends DynamicTileSource { + private readonly _zoomlevel: Store + + constructor( + zoomlevel: Store, + minzoom: number, + constructSource: (tileIndex: number) => FeatureSourceForTile, + mapProperties: { + bounds: Store + zoom: Store + }, + options?: { + isActive?: Store + }, + ) { + super(zoomlevel, minzoom, constructSource, mapProperties, options) + this._zoomlevel = zoomlevel + } + + protected addDataFromSources(sources: FeatureSourceForTile[]) { + sources = Utils.NoNull(sources) + const all: Map> = new Map() + const currentZoom = this._zoomlevel?.data ?? 0 + for (const source of sources) { + if(source.z != currentZoom){ + continue + } + const bboxCoors = Tiles.tile_bounds_lon_lat(source.z, source.x, source.y) + const bboxGeo = new BBox(bboxCoors).asGeoJson({}) + for (const f of source.features.data) { + const id = f.properties.id + const coordinates : Position[][] = [] + if(f.geometry.type === "LineString"){ + coordinates.push(f.geometry.coordinates) + }else if(f.geometry.type === "MultiLineString"){ + coordinates.push(...f.geometry.coordinates) + }else { + console.error("Invalid geometry type:", f.geometry.type) + continue + } + const oldV = all.get(id) + if(!oldV){ + + all.set(id, { + type: "Feature", + properties: f.properties, + geometry:{ + type:"MultiLineString", + coordinates + } + }) + continue + } + oldV.geometry.coordinates.push(...coordinates) + } + } + + const keys = Array.from(all.keys()) + for (const key of keys) { + all.set(key, GeoOperations.attemptLinearize(>all.get(key))) + } + const newList = Array.from(all.values()) + this.features.setData(newList) + this._featuresById.setData(all) + } + +} diff --git a/src/Logic/FeatureSource/TiledFeatureSource/PolygonSourceMerger.ts b/src/Logic/FeatureSource/TiledFeatureSource/PolygonSourceMerger.ts new file mode 100644 index 000000000..47327d872 --- /dev/null +++ b/src/Logic/FeatureSource/TiledFeatureSource/PolygonSourceMerger.ts @@ -0,0 +1,73 @@ +import { FeatureSourceForTile } from "../FeatureSource" +import { Store } from "../../UIEventSource" +import { BBox } from "../../BBox" +import { Utils } from "../../../Utils" +import { Feature } from "geojson" +import { GeoOperations } from "../../GeoOperations" +import DynamicTileSource from "./DynamicTileSource" + +/** + * The PolygonSourceMerger receives various small pieces of bigger polygons and stitches them together. + * This is used to reconstruct polygons of vector tiles + */ +export class PolygonSourceMerger extends DynamicTileSource { + constructor( + zoomlevel: Store, + minzoom: number, + constructSource: (tileIndex: number) => FeatureSourceForTile, + mapProperties: { + bounds: Store + zoom: Store + }, + options?: { + isActive?: Store + }, + ) { + super(zoomlevel, minzoom, constructSource, mapProperties, options) + } + + protected addDataFromSources(sources: FeatureSourceForTile[]) { + sources = Utils.NoNull(sources) + const all: Map = new Map() + const zooms: Map = new Map() + + for (const source of sources) { + let z = source.z + for (const f of source.features.data) { + const id = f.properties.id + if (id.endsWith("146616907")) { + console.log("Horeca totaal") + } + if (!all.has(id)) { + // No other parts of this polygon have been seen before, simply add it + all.set(id, f) + zooms.set(id, z) + continue + } + + // A part of this object has been seen before, eventually from a different zoom level + const oldV = all.get(id) + const oldZ = zooms.get(id) + if (oldZ > z) { + // The store contains more detailed information, so we ignore this part which has a lower accuraccy + continue + } + if (oldZ < z) { + // The old value has worse accuracy then what we receive now, we throw it away + all.set(id, f) + zooms.set(id, z) + continue + } + const merged = GeoOperations.union(f, oldV) + merged.properties = oldV.properties + all.set(id, merged) + zooms.set(id, z) + } + } + + const newList = Array.from(all.values()) + this.features.setData(newList) + this._featuresById.setData(all) + } + +} diff --git a/src/Logic/GeoOperations.ts b/src/Logic/GeoOperations.ts index c90869b83..7567a8bca 100644 --- a/src/Logic/GeoOperations.ts +++ b/src/Logic/GeoOperations.ts @@ -1,6 +1,6 @@ import { BBox } from "./BBox" import * as turf from "@turf/turf" -import { AllGeoJSON, booleanWithin, Coord } from "@turf/turf" +import { AllGeoJSON, booleanWithin, Coord, Lines } from "@turf/turf" import { Feature, FeatureCollection, @@ -156,7 +156,7 @@ export class GeoOperations { const intersection = GeoOperations.calculateIntersection( feature, otherFeature, - featureBBox + featureBBox, ) if (intersection === null) { continue @@ -195,7 +195,7 @@ export class GeoOperations { console.error( "Could not correctly calculate the overlap of ", feature, - ": unsupported type" + ": unsupported type", ) return result } @@ -224,7 +224,7 @@ export class GeoOperations { */ public static inside( pointCoordinate: [number, number] | Feature, - feature: Feature + feature: Feature, ): boolean { // ray-casting algorithm based on // http://www.ecse.rpi.edu/Homepages/wrf/Research/Short_Notes/pnpoly.html @@ -302,7 +302,7 @@ export class GeoOperations { */ public static nearestPoint( way: Feature, - point: [number, number] + point: [number, number], ): Feature< Point, { @@ -324,11 +324,11 @@ export class GeoOperations { public static forceLineString(way: Feature): Feature public static forceLineString( - way: Feature + way: Feature, ): Feature public static forceLineString( - way: Feature + way: Feature, ): Feature { if (way.geometry.type === "Polygon") { way = { ...way } @@ -448,7 +448,7 @@ export class GeoOperations { */ public static LineIntersections( feature: Feature, - otherFeature: Feature + otherFeature: Feature, ): [number, number][] { return turf .lineIntersect(feature, otherFeature) @@ -485,7 +485,7 @@ export class GeoOperations { locations: | Feature | Feature[], - title?: string + title?: string, ) { title = title?.trim() if (title === undefined || title === "") { @@ -506,7 +506,7 @@ export class GeoOperations { type: "Point", coordinates: p, }, - } + }, ) } for (const l of locationsWithMeta) { @@ -521,7 +521,7 @@ export class GeoOperations { trackPoints.push(trkpt) } const header = - '' + "" return ( header + "\n" + @@ -539,7 +539,7 @@ export class GeoOperations { */ public static toGpxPoints( locations: Feature[], - title?: string + title?: string, ) { title = title?.trim() if (title === undefined || title === "") { @@ -560,7 +560,7 @@ export class GeoOperations { trackPoints.push(trkpt) } const header = - '' + "" return ( header + "\n" + @@ -648,7 +648,7 @@ export class GeoOperations { }, }, distanceMeter, - { units: "meters" } + { units: "meters" }, ).geometry.coordinates } @@ -683,7 +683,7 @@ export class GeoOperations { */ static completelyWithin( feature: Feature, - possiblyEnclosingFeature: Feature + possiblyEnclosingFeature: Feature, ): boolean { return booleanWithin(feature, possiblyEnclosingFeature) } @@ -714,6 +714,23 @@ export class GeoOperations { } return kept } + + if (toSplit.geometry.type === "MultiLineString") { + const lines: Feature[][] = toSplit.geometry.coordinates.map(coordinates => + turf.lineSplit( {type: "LineString", coordinates}, boundary).features ) + const splitted: Feature[] = [].concat(...lines) + const kept: Feature[] = [] + for (const f of splitted) { + console.log("Checking", f) + if (!GeoOperations.inside(GeoOperations.centerpointCoordinates(f), boundary)) { + continue + } + f.properties = { ...toSplit.properties } + kept.push(f) + } + console.log(">>>", {lines, splitted, kept}) + return kept + } if (toSplit.geometry.type === "Polygon" || toSplit.geometry.type == "MultiPolygon") { const splitup = turf.intersect(>toSplit, boundary) splitup.properties = { ...toSplit.properties } @@ -739,7 +756,7 @@ export class GeoOperations { */ public static featureToCoordinateWithRenderingType( feature: Feature, - location: "point" | "centroid" | "start" | "end" | "projected_centerpoint" | string + location: "point" | "centroid" | "start" | "end" | "projected_centerpoint" | string, ): [number, number] | undefined { switch (location) { case "point": @@ -760,7 +777,7 @@ export class GeoOperations { const centerpoint = GeoOperations.centerpointCoordinates(feature) const projected = GeoOperations.nearestPoint( >feature, - centerpoint + centerpoint, ) return <[number, number]>projected.geometry.coordinates } @@ -937,7 +954,7 @@ export class GeoOperations { * GeoOperations.bearingToHuman(46) // => "NE" */ public static bearingToHuman( - bearing: number + bearing: number, ): "N" | "NE" | "E" | "SE" | "S" | "SW" | "W" | "NW" { while (bearing < 0) { bearing += 360 @@ -956,7 +973,7 @@ export class GeoOperations { * GeoOperations.bearingToHuman(46) // => "NE" */ public static bearingToHumanRelative( - bearing: number + bearing: number, ): | "straight" | "slight_right" @@ -975,18 +992,73 @@ export class GeoOperations { return GeoOperations.directionsRelative[segment] } + /** + * const coors = [[[3.217198532946432,51.218067],[3.216807134449482,51.21849812105347],[3.2164304037883706,51.2189272]],[[3.2176208,51.21760169669458],[3.217198560167068,51.218067]]] + * const f = {geometry: {coordinates: coors}} + * const merged = GeoOperations.attemptLinearize(f) + * merged.geometry.coordinates // => [[3.2176208,51.21760169669458],[3.217198532946432,51.218067], [3.216807134449482,51.21849812105347],[3.2164304037883706,51.2189272]] + */ + static attemptLinearize(multiLineStringFeature: Feature): Feature { + const coors = multiLineStringFeature.geometry.coordinates + if(coors.length === 0) { + console.error(multiLineStringFeature.geometry) + throw "Error: got degenerate multilinestring" + } + outer: for (let i = coors.length - 1; i >= 0; i--) { + // We try to match the first element of 'i' with another, earlier list `j` + // If a match is found with `j`, j is extended and `i` is scrapped + const iFirst = coors[i][0] + for (let j = 0; j < coors.length; j++) { + if (i == j) { + continue + } + + const jLast = coors[j].at(-1) + if (!(Math.abs(iFirst[0] - jLast[0]) < 0.000001 && Math.abs(iFirst[1] - jLast[1]) < 0.0000001)) { + continue + } + coors[j].splice(coors.length - 1, 1) + coors[j].push(...coors[i]) + coors.splice(i, 1) + continue outer + } + } + if(coors.length === 0) { + throw "No more coordinates found" + } + + if (coors.length === 1) { + return { + type: "Feature", + properties: multiLineStringFeature.properties, + geometry: { + type: "LineString", + coordinates: coors[0], + }, + } + } + return { + type: "Feature", + properties: multiLineStringFeature.properties, + geometry: { + type: "MultiLineString", + coordinates: coors, + }, + } + } + /** * Helper function which does the heavy lifting for 'inside' */ private static pointInPolygonCoordinates( x: number, y: number, - coordinates: [number, number][][] + coordinates: [number, number][][], ): boolean { const inside = GeoOperations.pointWithinRing( x, y, - /*This is the outer ring of the polygon */ coordinates[0] + /*This is the outer ring of the polygon */ coordinates[0], ) if (!inside) { return false @@ -995,7 +1067,7 @@ export class GeoOperations { const inHole = GeoOperations.pointWithinRing( x, y, - coordinates[i] /* These are inner rings, aka holes*/ + coordinates[i], /* These are inner rings, aka holes*/ ) if (inHole) { return false @@ -1033,7 +1105,7 @@ export class GeoOperations { feature, otherFeature, featureBBox: BBox, - otherFeatureBBox?: BBox + otherFeatureBBox?: BBox, ): number { if (feature.geometry.type === "LineString") { otherFeatureBBox = otherFeatureBBox ?? BBox.get(otherFeature) @@ -1082,7 +1154,7 @@ export class GeoOperations { let intersection = turf.lineSlice( turf.point(intersectionPointsArray[0]), turf.point(intersectionPointsArray[1]), - feature + feature, ) if (intersection == null) { @@ -1103,7 +1175,7 @@ export class GeoOperations { otherFeature, feature, otherFeatureBBox, - featureBBox + featureBBox, ) } @@ -1123,7 +1195,7 @@ export class GeoOperations { console.log("Applying fallback intersection...") const intersection = turf.intersect( turf.truncate(feature), - turf.truncate(otherFeature) + turf.truncate(otherFeature), ) if (intersection == null) { return null diff --git a/src/Logic/Osm/OsmObject.ts b/src/Logic/Osm/OsmObject.ts index f0e0215cd..d743b58c9 100644 --- a/src/Logic/Osm/OsmObject.ts +++ b/src/Logic/Osm/OsmObject.ts @@ -7,7 +7,7 @@ import { Feature, LineString, Polygon } from "geojson" export abstract class OsmObject { private static defaultBackend = "https://api.openstreetmap.org/" protected static backendURL = OsmObject.defaultBackend - private static polygonFeatures = OsmObject.constructPolygonFeatures() + public static polygonFeatures = OsmObject.constructPolygonFeatures() type: "node" | "way" | "relation" id: number /** From 74fb4bd5d1c9a8c33e68d47c48663e736ba79661 Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Thu, 15 Feb 2024 17:39:59 +0100 Subject: [PATCH 09/61] Add summary layer --- Docs/SettingUpPSQL.md | 11 + assets/layers/ice_cream/ice_cream.json | 1 + assets/layers/last_click/last_click.json | 2 +- assets/layers/summary/summary.json | 25 ++ assets/layers/toilet/toilet.json | 145 +++++++++-- assets/themes/shops/shops.json | 4 +- scripts/osm2pgsql/tilecountServer.ts | 232 ++++++++++++++++-- .../Sources/LastClickFeatureSource.ts | 20 +- .../TiledFeatureSource/SummaryTileSource.ts | 85 +++++++ src/Logic/UIEventSource.ts | 5 +- src/Models/Constants.ts | 4 +- src/Models/ThemeConfig/LayerConfig.ts | 5 +- src/Models/ThemeConfig/LayoutConfig.ts | 8 + .../ThemeConfig/PointRenderingConfig.ts | 3 +- src/Models/ThemeViewState.ts | 59 +++-- src/UI/DownloadFlow/DownloadButton.svelte | 1 - src/UI/Popup/AddNewPoint/AddNewPoint.svelte | 1 - src/UI/SpecialVisualization.ts | 1 - src/UI/ThemeViewGUI.svelte | 9 +- 19 files changed, 533 insertions(+), 88 deletions(-) create mode 100644 assets/layers/summary/summary.json create mode 100644 src/Logic/FeatureSource/TiledFeatureSource/SummaryTileSource.ts diff --git a/Docs/SettingUpPSQL.md b/Docs/SettingUpPSQL.md index e3a4ae13b..1b7ed9aa9 100644 --- a/Docs/SettingUpPSQL.md +++ b/Docs/SettingUpPSQL.md @@ -36,6 +36,17 @@ Storing properties to table '"public"."osm2pgsql_properties" takes about 25 minu Belgium (~555mb) takes 15m World (80GB) should take 15m*160 = 2400m = 40hr +73G Jan 23 00:22 planet-240115.osm.pbf: 2024-02-10 16:45:11 osm2pgsql took 871615s (242h 6m 55s; 10 days) overall on lain.local with RAID5 on 4 HDD disks, database is over 1Terrabyte (!) + +Server specs + +Lenovo thinkserver RD350, Intel Xeon E5-2600, 2Rx4 PC3 + 11 watt powered off, 73 watt idle, ~100 watt when importing + +HP ProLiant DL360 G7 (1U): 2Rx4 DDR3-memory (PC3) + Intel Xeon X56** + + ## Deploying a tile server diff --git a/assets/layers/ice_cream/ice_cream.json b/assets/layers/ice_cream/ice_cream.json index b6d6576a9..8b5ea0568 100644 --- a/assets/layers/ice_cream/ice_cream.json +++ b/assets/layers/ice_cream/ice_cream.json @@ -4,6 +4,7 @@ "en": "Ice cream parlors", "de": "Eisdielen" }, + "minzoom": 14, "description": { "en": "A place where ice cream is sold over the counter", "de": "Ein Ort, an dem Eiscreme an der Theke verkauft wird" diff --git a/assets/layers/last_click/last_click.json b/assets/layers/last_click/last_click.json index c99a4ce5b..335731bda 100644 --- a/assets/layers/last_click/last_click.json +++ b/assets/layers/last_click/last_click.json @@ -1,7 +1,7 @@ { "id": "last_click", "name": null, - "description": "This layer defines how to render the 'last click'-location. By default, it will show a marker with the possibility to add a new point (if there are some presets) and/or to add a new note (if the 'note' layer attribute is set). If none are possible, this layer won't show up", + "description": "This 'layer' is not really a layer, but contains part of the code how the popup to 'add a new marker' is displayed", "source": "special", "isShown": { "or": [ diff --git a/assets/layers/summary/summary.json b/assets/layers/summary/summary.json new file mode 100644 index 000000000..ecffca76a --- /dev/null +++ b/assets/layers/summary/summary.json @@ -0,0 +1,25 @@ +{ + "id": "summary", + "description": "Special layer which shows `count`", + "source": "special", + "name": "CLusters", + "title": { + "render": {"en": "Summary"} + }, + "tagRenderings": [ + "all_tags" + ], + "pointRendering": [ + { + "location": [ + "point", + "centroid" + ], + "iconSize": "25,25", + "label": { + "render": "{total}" + }, + "labelCssClasses": "bg-white w-6 h-6 text-lg rounded-full" + } + ] +} diff --git a/assets/layers/toilet/toilet.json b/assets/layers/toilet/toilet.json index 2d58241b6..192a60ab7 100644 --- a/assets/layers/toilet/toilet.json +++ b/assets/layers/toilet/toilet.json @@ -26,7 +26,7 @@ "source": { "osmTags": "amenity=toilets" }, - "minzoom": 9, + "minzoom": 10, "title": { "render": { "en": "Toilet", @@ -548,6 +548,115 @@ } ] }, + { + "condition": "toilets:position!=urinal", + "id": "gender_segregated", + "question": { + "en": "Are these toilets gender-segregated?", + "nl": "Zijn deze toiletten gescheiden op basis van geslacht?" + }, + "questionHint": { + "en": "Are there separate stalls or separate areas for men and women and are they signposted as such?", + "nl": "Is er een aparte ruimte voor mannen en vrouwen en zijn deze ruimtes ook expliciet aangegeven?" + }, + "mappings": [ + { + "if": "gender_segregated=yes", + "then": { + "en": "There is a separate, signposted area for men and women", + "nl": "Er zijn aparte ruimtes of toiletten voor mannen en vrouwen" + } + }, + { + "if": "gender_segregated=no", + "then": { + "en": "There is no separate, signposted area for men and women", + "nl": "Mannen en vrouwen gebruiken dezelfde ruimtes en toiletten" + } + } + ] + }, + { + "id": "menstrual_products", + "question": { + "en": "Are free, menstrual products distributed here?", + "nl": "Zijn er gratis menstruatieproducten beschikbaar?" + }, + "questionHint": { + "en": "This is only about menstrual products that are free of charge. If e.g. a vending machine is available which charges for menstrual products, ignore it for this question.", + "nl": "Dit gaat enkel over menstruatieproducten die gratis geschikbaar zijn. Indien er bv. een verkoopautomaat met menstruatieproducten is, negeer deze dan" + }, + "mappings": [ + { + "if": "toilets:menstrual_products=yes", + "then": { + "en": "Free menstrual products are available to all visitors of these toilets", + "nl": "Er zijn gratis menstruatieprocten beschikbaar voor alle bezoekers van deze toiletten" + } + }, + { + "if": "toilets:menstrual_products=limited", + "then": { + "en": "Free menstrual products are available to some visitors of these toilets", + "nl": "De gratis menstruatieproducten zijn enkel beschikbaar in een deel van de toiletten" + }, + "hideInAnswer": "gender_segregated=yes" + }, + { + "if": "toilets:menstrual_products=no", + "alsoShowIf": "toilets:menstrual_products=", + "then": { + "en": "No free menstrual products are available here", + "nl": "Er zijn geen gratis menstruatieproducten beschikbaar" + } + } + ] + }, + { + "id": "menstrual_products_location", + "question": { + "en": "Where are the free menstrual products located?", + "nl": "Waar bevinden de gratis menstruatieproducten zich?" + }, + "condition": { + "or": [ + "toilets:menstrual_products=limited", + "toilets:menstrual_products:location~*" + ] + }, + "render": { + "en": "The menstrual products are located in {toilets:menstrual_products:location}", + "nl": "De menstruatieproducten bevinden zich in {toilets:menstrual_products:location}" + }, + "freeform": { + "key": "toilets:menstrual_products:location", + "inline": true + }, + "mappings": [ + { + "then": { + "en": "The free, menstrual products are located in the toilet for women", + "nl": "De gratis menstruatieproducten bevinden zich in het vrouwentoilet" + }, + "if": "toilets:menstrual_products:location=female_toilet", + "alsoShowIf": "toilets:menstrual_products:location=" + }, + { + "then": { + "en": "The free, menstrual products are located in the toilet for men", + "nl": "De gratis menstruatieproducten bevinden zich in het mannentoilet" + }, + "if": "toilets:menstrual_products:location=male_toilet" + }, + { + "if": "toilets:menstrual_products:location=wheelchair_toilet", + "then": { + "en": "The free, menstrual products are located in the toilet for wheelchair users", + "nl": "De gratis menstruatieproducten bevinden zich in het rolstoeltoegankelijke toilet" + } + } + ] + }, { "id": "toilets-changing-table", "labels": [ @@ -576,7 +685,8 @@ "ca": "Hi ha un canviador per a nadons", "cs": "Přebalovací pult je k dispozici" }, - "if": "changing_table=yes" + "if": "changing_table=yes", + "icon": "./assets/layers/toilet/baby.svg" }, { "if": "changing_table=no", @@ -610,10 +720,10 @@ "cs": "Kde je umístěn přebalovací pult?" }, "render": { - "en": "The changing table is located at {changing_table:location}", - "de": "Die Wickeltabelle befindet sich in {changing_table:location}", + "en": "A changing table is located at {changing_table:location}", + "de": "Ein Wickeltisch befindet sich in {changing_table:location}", "fr": "Emplacement de la table à langer : {changing_table:location}", - "nl": "De luiertafel bevindt zich in {changing_table:location}", + "nl": "Er bevindt zich een luiertafel in {changing_table:location}", "it": "Il fasciatoio si trova presso {changing_table:location}", "es": "El cambiador está en {changing_table:location}", "da": "Puslebordet er placeret på {changing_table:location}", @@ -632,10 +742,10 @@ "mappings": [ { "then": { - "en": "The changing table is in the toilet for women. ", - "de": "Der Wickeltisch befindet sich in der Damentoilette. ", + "en": "A changing table is in the toilet for women", + "de": "Ein Wickeltisch ist in der Damentoilette vorhanden", "fr": "La table à langer est dans les toilettes pour femmes. ", - "nl": "De luiertafel bevindt zich in de vrouwentoiletten ", + "nl": "Er bevindt zich een luiertafel in de vrouwentoiletten ", "it": "Il fasciatoio è nei servizi igienici femminili. ", "da": "Puslebordet er på toilettet til kvinder. ", "ca": "El canviador està al lavabo per a dones. ", @@ -645,10 +755,10 @@ }, { "then": { - "en": "The changing table is in the toilet for men. ", - "de": "Der Wickeltisch befindet sich in der Herrentoilette. ", + "en": "A changing table is in the toilet for men", + "de": "Ein Wickeltisch ist in der Herrentoilette vorhanden", "fr": "La table à langer est dans les toilettes pour hommes. ", - "nl": "De luiertafel bevindt zich in de herentoiletten ", + "nl": "Er bevindt zich een luiertafel in de herentoiletten ", "it": "Il fasciatoio è nei servizi igienici maschili. ", "ca": "El canviador està al lavabo per a homes. ", "cs": "Přebalovací pult je na pánské toaletě. " @@ -658,10 +768,10 @@ { "if": "changing_table:location=wheelchair_toilet", "then": { - "en": "The changing table is in the toilet for wheelchair users. ", - "de": "Der Wickeltisch befindet sich in der Toilette für Rollstuhlfahrer. ", + "en": "A changing table is in the toilet for wheelchair users", + "de": "Ein Wickeltisch ist in der barrierefreien Toilette vorhanden", "fr": "La table à langer est dans les toilettes pour personnes à mobilité réduite. ", - "nl": "De luiertafel bevindt zich in de rolstoeltoegankelijke toilet ", + "nl": "Er bevindt zich een luiertafel in de rolstoeltoegankelijke toilet ", "it": "Il fasciatoio è nei servizi igienici per persone in sedia a rotelle. ", "da": "Puslebordet er på toilettet for kørestolsbrugere. ", "ca": "El canviador està al lavabo per a usuaris de cadira de rodes. ", @@ -671,10 +781,10 @@ { "if": "changing_table:location=dedicated_room", "then": { - "en": "The changing table is in a dedicated room. ", - "de": "Der Wickeltisch befindet sich in einem eigenen Raum. ", + "en": "A changing table is in a dedicated room", + "de": "Ein Wickeltisch befindet sich in einem eigenen Raum", "fr": "La table à langer est dans un espace dédié. ", - "nl": "De luiertafel bevindt zich in een daartoe voorziene kamer ", + "nl": "Er bevindt zich een luiertafel in een daartoe voorziene kamer ", "it": "Il fasciatoio è in una stanza dedicata. ", "es": "El cambiador está en una habitación dedicada ", "da": "Vuggestuen står i et særligt rum. ", @@ -683,6 +793,7 @@ } } ], + "multiAnswer": true, "id": "toilet-changing_table:location" }, { diff --git a/assets/themes/shops/shops.json b/assets/themes/shops/shops.json index 0024d9837..abe44dc0e 100644 --- a/assets/themes/shops/shops.json +++ b/assets/themes/shops/shops.json @@ -54,5 +54,7 @@ "pharmacy", "ice_cream" ], - "widenFactor": 3 + "overideAll": { + "minzoom": 16 + } } diff --git a/scripts/osm2pgsql/tilecountServer.ts b/scripts/osm2pgsql/tilecountServer.ts index 5c33c72b7..8980e9939 100644 --- a/scripts/osm2pgsql/tilecountServer.ts +++ b/scripts/osm2pgsql/tilecountServer.ts @@ -1,44 +1,240 @@ -import { BBox } from "../../src/Logic/BBox" import { Client } from "pg" +import http from "node:http" +import { Tiles } from "../../src/Models/TileRange" + +/** + * Just the OSM2PGSL default database + */ +interface PoiDatabaseMeta { + attributes + current_timestamp + db_format + flat_node_file + import_timestamp + output + prefix + replication_base_url + replication_sequence_number + replication_timestamp + style + updatable + version +} /** * Connects with a Postgis database, gives back how much items there are within the given BBOX */ -export default class TilecountServer { +class OsmPoiDatabase { + private static readonly prefixes: ReadonlyArray = ["pois", "lines", "polygons"] private readonly _client: Client private isConnected = false + private supportedLayers: string[] = undefined + private metaCache: PoiDatabaseMeta = undefined + private metaCacheDate: Date = undefined constructor(connectionString: string) { this._client = new Client(connectionString) } - async getCount(layer: string, bbox: BBox = undefined): Promise { + async getCount( + layer: string, + bbox: [[number, number], [number, number]] = undefined + ): Promise { if (!this.isConnected) { await this._client.connect() this.isConnected = true } - let query = "SELECT COUNT(*) FROM " + layer + let total = 0 - if(bbox){ - query += ` WHERE ST_MakeEnvelope (${bbox.minLon}, ${bbox.minLat}, ${bbox.maxLon}, ${bbox.maxLat}, 4326) ~ geom` + for (const prefix of OsmPoiDatabase.prefixes) { + let query = "SELECT COUNT(*) FROM " + prefix + "_" + layer + + if (bbox) { + query += ` WHERE ST_MakeEnvelope (${bbox[0][0]}, ${bbox[0][1]}, ${bbox[1][0]}, ${bbox[1][1]}, 4326) ~ geom` + } + console.log("Query:", query) + const result = await this._client.query(query) + total += Number(result.rows[0].count) } -console.log(query) - const result = await this._client.query(query) - return result.rows[0].count + return total } disconnect() { this._client.end() } + + async getLayers(): Promise { + if (this.supportedLayers !== undefined) { + return this.supportedLayers + } + const result = await this._client.query( + "SELECT table_name \n" + + "FROM information_schema.tables \n" + + "WHERE table_schema = 'public' AND table_name LIKE 'lines_%';" + ) + const layers = result.rows.map((r) => r.table_name.substring("lines_".length)) + this.supportedLayers = layers + return layers + } + + async getMeta(): Promise { + const now = new Date() + if (this.metaCache !== undefined) { + const diffSec = (this.metaCacheDate.getTime() - now.getTime()) / 1000 + if (diffSec < 120) { + return this.metaCache + } + } + const result = await this._client.query("SELECT * FROM public.osm2pgsql_properties") + const meta = {} + for (const { property, value } of result.rows) { + meta[property] = value + } + this.metaCacheDate = now + this.metaCache = meta + return this.metaCache + } } -const tcs = new TilecountServer("postgresql://user:none@localhost:5444/osm-poi") -console.log(">>>", await tcs.getCount("drinking_water", new BBox([ - [1.5052013991654007, - 42.57480750272123, - ], [ - 1.6663677350703097, - 42.499856652770745, - ]]))) -tcs.disconnect() +class Server { + constructor( + port: number, + handle: { + mustMatch: string | RegExp + mimetype: string + handle: (path: string) => Promise + }[] + ) { + handle.push({ + mustMatch: "", + mimetype: "text/html", + handle: async () => { + return `Supported endpoints are
    ${handle + .filter((h) => h.mustMatch !== "") + .map((h) => { + let l = h.mustMatch + if (typeof h.mustMatch === "string") { + l = `${l}` + } + return "
  • " + l + "
  • " + }) + .join("")}
` + }, + }) + http.createServer(async (req: http.IncomingMessage, res) => { + try { + console.log( + req.method + " " + req.url, + "from:", + req.headers.origin, + new Date().toISOString() + ) + + const url = new URL(`http://127.0.0.1/` + req.url) + let path = url.pathname + while (path.startsWith("/")) { + path = path.substring(1) + } + const handler = handle.find((h) => { + if (typeof h.mustMatch === "string") { + return h.mustMatch === path + } + if (path.match(h.mustMatch)) { + return true + } + }) + + if (handler === undefined || handler === null) { + res.writeHead(404, { "Content-Type": "text/html" }) + res.write("

Not found...

") + res.end() + return + } + + res.setHeader( + "Access-Control-Allow-Headers", + "Origin, X-Requested-With, Content-Type, Accept" + ) + res.setHeader("Access-Control-Allow-Origin", req.headers.origin ?? "*") + if (req.method === "OPTIONS") { + res.setHeader( + "Access-Control-Allow-Methods", + "POST, GET, OPTIONS, DELETE, UPDATE" + ) + res.writeHead(204, { "Content-Type": handler.mimetype }) + res.end() + return + } + if (req.method === "POST" || req.method === "UPDATE") { + return + } + + if (req.method === "DELETE") { + return + } + + try { + const result = await handler.handle(path) + res.writeHead(200, { "Content-Type": handler.mimetype }) + res.write(result) + res.end() + } catch (e) { + console.error("Could not handle request:", e) + res.writeHead(500) + res.write(e) + res.end() + } + } catch (e) { + console.error("FATAL:", e) + res.end() + } + }).listen(port) + console.log( + "Server is running on port " + port, + ". Supported endpoints are: " + handle.map((h) => h.mustMatch).join(", ") + ) + } +} + +const connectionString = "postgresql://user:password@localhost:5444/osm-poi" +const tcs = new OsmPoiDatabase(connectionString) +const server = new Server(2345, [ + { + mustMatch: "status.json", + mimetype: "application/json", + handle: async (path: string) => { + const layers = await tcs.getLayers() + const meta = await tcs.getMeta() + return JSON.stringify({ meta, layers }) + }, + }, + { + mustMatch: /[a-zA-Z0-9+]+\/[0-9]+\/[0-9]+\/[0-9]+\.json/, + mimetype: "application/json", // "application/vnd.geo+json", + async handle(path) { + console.log("Path is:", path, path.split(".")[0]) + const [layers, z, x, y] = path.split(".")[0].split("/") + + let sum = 0 + let properties: Record = {} + for (const layer of layers.split("+")) { + const count = await tcs.getCount( + layer, + Tiles.tile_bounds_lon_lat(Number(z), Number(x), Number(y)) + ) + properties[layer] = count + sum += count + } + + return JSON.stringify({ ...properties, total: sum }) + }, + }, +]) +console.log( + ">>>", + await tcs.getCount("drinking_water", [ + [3.194358020772171, 51.228073636083394], + [3.2839964396059145, 51.172701162680994], + ]) +) diff --git a/src/Logic/FeatureSource/Sources/LastClickFeatureSource.ts b/src/Logic/FeatureSource/Sources/LastClickFeatureSource.ts index c95857d70..3fe9b79e9 100644 --- a/src/Logic/FeatureSource/Sources/LastClickFeatureSource.ts +++ b/src/Logic/FeatureSource/Sources/LastClickFeatureSource.ts @@ -11,16 +11,15 @@ import { OsmTags } from "../../../Models/OsmFeature" * Highly specialized feature source. * Based on a lon/lat UIEVentSource, will generate the corresponding feature with the correct properties */ -export class LastClickFeatureSource implements WritableFeatureSource { - public readonly features: UIEventSource = new UIEventSource([]) - public readonly hasNoteLayer: boolean +export class LastClickFeatureSource { public readonly renderings: string[] - public readonly hasPresets: boolean private i: number = 0 + private readonly hasPresets: boolean + private readonly hasNoteLayer: boolean - constructor(location: Store<{ lon: number; lat: number }>, layout: LayoutConfig) { - this.hasNoteLayer = layout.layers.some((l) => l.id === "note") - this.hasPresets = layout.layers.some((l) => l.presets?.length > 0) + constructor(layout: LayoutConfig) { + this.hasNoteLayer = layout.hasNoteLayer() + this.hasPresets = layout.hasPresets() const allPresets: BaseUIElement[] = [] for (const layer of layout.layers) for (let i = 0; i < (layer.presets ?? []).length; i++) { @@ -43,16 +42,11 @@ export class LastClickFeatureSource implements WritableFeatureSource { Utils.runningFromConsole ? "" : uiElem.ConstructElement().innerHTML ) ) - - location.addCallbackAndRunD(({ lon, lat }) => { - this.features.setData([this.createFeature(lon, lat)]) - }) } public createFeature(lon: number, lat: number): Feature { const properties: OsmTags = { - lastclick: "yes", - id: "last_click_" + this.i, + id: "new_point_dialog", has_note_layer: this.hasNoteLayer ? "yes" : "no", has_presets: this.hasPresets ? "yes" : "no", renderings: this.renderings.join(""), diff --git a/src/Logic/FeatureSource/TiledFeatureSource/SummaryTileSource.ts b/src/Logic/FeatureSource/TiledFeatureSource/SummaryTileSource.ts new file mode 100644 index 000000000..2a7105fca --- /dev/null +++ b/src/Logic/FeatureSource/TiledFeatureSource/SummaryTileSource.ts @@ -0,0 +1,85 @@ +import DynamicTileSource from "./DynamicTileSource" +import { Store, UIEventSource } from "../../UIEventSource" +import { BBox } from "../../BBox" +import StaticFeatureSource from "../Sources/StaticFeatureSource" +import { Feature, Point } from "geojson" +import { Utils } from "../../../Utils" +import { Tiles } from "../../../Models/TileRange" + +/** + * Provides features summarizing the total amount of features at a given location + */ +export class SummaryTileSource extends DynamicTileSource { + private static readonly empty = [] + constructor( + cacheserver: string, + layers: string[], + zoomRounded: Store, + mapProperties: { + bounds: Store + zoom: Store + }, + options?: { + isActive?: Store + } + ) { + const layersSummed = layers.join("+") + super( + zoomRounded, + 0, // minzoom + (tileIndex) => { + const [z, x, y] = Tiles.tile_from_index(tileIndex) + const coordinates = Tiles.centerPointOf(z, x, y) + + const count = UIEventSource.FromPromiseWithErr( + Utils.downloadJson(`${cacheserver}/${layersSummed}/${z}/${x}/${y}.json`) + ) + const features: Store[]> = count.mapD((count) => { + if (count["error"] !== undefined) { + console.error( + "Could not download count for tile", + z, + x, + y, + "due to", + count["error"] + ) + return SummaryTileSource.empty + } + const counts = count["success"] + if (counts === undefined || counts["total"] === 0) { + return SummaryTileSource.empty + } + return [ + { + type: "Feature", + properties: { + id: "summary_" + tileIndex, + summary: "yes", + ...counts, + layers: layersSummed, + }, + geometry: { + type: "Point", + coordinates, + }, + }, + ] + }) + return new StaticFeatureSource( + features.map( + (f) => { + if (z !== zoomRounded.data) { + return SummaryTileSource.empty + } + return f + }, + [zoomRounded] + ) + ) + }, + mapProperties, + options + ) + } +} diff --git a/src/Logic/UIEventSource.ts b/src/Logic/UIEventSource.ts index ed48130ff..48bc9c6d9 100644 --- a/src/Logic/UIEventSource.ts +++ b/src/Logic/UIEventSource.ts @@ -638,8 +638,9 @@ export class UIEventSource extends Store implements Writable { promise: Promise ): UIEventSource<{ success: T } | { error: any } | undefined> { const src = new UIEventSource<{ success: T } | { error: any }>(undefined) - promise?.then((d) => src.setData({ success: d })) - promise?.catch((err) => src.setData({ error: err })) + promise + ?.then((d) => src.setData({ success: d })) + ?.catch((err) => src.setData({ error: err })) return src } diff --git a/src/Models/Constants.ts b/src/Models/Constants.ts index 0d76b4fc5..31181c5e6 100644 --- a/src/Models/Constants.ts +++ b/src/Models/Constants.ts @@ -24,6 +24,7 @@ export default class Constants { "range", "last_click", "favourite", + "summary", ] as const /** * Special layers which are not included in a theme by default @@ -36,7 +37,7 @@ export default class Constants { "import_candidate", "usersettings", "icons", - "filters" + "filters", ] as const /** * Layer IDs of layers which have special properties through built-in hooks @@ -151,7 +152,6 @@ export default class Constants { "mastodon", "party", "addSmall", - ] as const public static readonly defaultPinIcons: string[] = Constants._defaultPinIcons /** diff --git a/src/Models/ThemeConfig/LayerConfig.ts b/src/Models/ThemeConfig/LayerConfig.ts index 7b2feacd0..0f8b5facc 100644 --- a/src/Models/ThemeConfig/LayerConfig.ts +++ b/src/Models/ThemeConfig/LayerConfig.ts @@ -45,7 +45,6 @@ export default class LayerConfig extends WithContextLoader { public readonly isShown: TagsFilter public minzoom: number public minzoomVisible: number - public readonly maxzoom: number public readonly title?: TagRenderingConfig public readonly titleIcons: TagRenderingConfig[] public readonly mapRendering: PointRenderingConfig[] @@ -464,9 +463,7 @@ export default class LayerConfig extends WithContextLoader { return [ new Combine([ new Link( - Utils.runningFromConsole - ? "" - : Svg.statistics_svg().SetClass("w-4 h-4 mr-2"), + "", "https://taginfo.openstreetmap.org/keys/" + values.key + "#values", true ), diff --git a/src/Models/ThemeConfig/LayoutConfig.ts b/src/Models/ThemeConfig/LayoutConfig.ts index c9859b2c5..194089aeb 100644 --- a/src/Models/ThemeConfig/LayoutConfig.ts +++ b/src/Models/ThemeConfig/LayoutConfig.ts @@ -245,6 +245,14 @@ export default class LayoutConfig implements LayoutInformation { return this.layers.some((l) => l.isLeftRightSensitive()) } + public hasNoteLayer() { + return this.layers.some((l) => l.id === "note") + } + + public hasPresets() { + return this.layers.some((l) => l.presets?.length > 0) + } + public missingTranslations(extraInspection: any): { untranslated: Map total: number diff --git a/src/Models/ThemeConfig/PointRenderingConfig.ts b/src/Models/ThemeConfig/PointRenderingConfig.ts index a522c45a7..5f7974723 100644 --- a/src/Models/ThemeConfig/PointRenderingConfig.ts +++ b/src/Models/ThemeConfig/PointRenderingConfig.ts @@ -79,7 +79,6 @@ export default class PointRenderingConfig extends WithContextLoader { } }) - this.marker = (json.marker ?? []).map((m) => new IconConfig(m)) if (json.css !== undefined) { this.cssDef = this.tr("css", undefined) @@ -307,7 +306,7 @@ export default class PointRenderingConfig extends WithContextLoader { const label = self.label ?.GetRenderValue(tags) ?.Subs(tags) - ?.SetClass("block center absolute text-center marker-label") + ?.SetClass("flex items-center justify-center absolute marker-label") ?.SetClass(cssClassesLabel) if (cssLabel) { label.SetStyle(cssLabel) diff --git a/src/Models/ThemeViewState.ts b/src/Models/ThemeViewState.ts index 011cf335a..83602f151 100644 --- a/src/Models/ThemeViewState.ts +++ b/src/Models/ThemeViewState.ts @@ -62,7 +62,9 @@ import FavouritesFeatureSource from "../Logic/FeatureSource/Sources/FavouritesFe import { ProvidedImage } from "../Logic/ImageProviders/ImageProvider" import { GeolocationControlState } from "../UI/BigComponents/GeolocationControl" import Zoomcontrol from "../UI/Zoomcontrol" - +import { SummaryTileSource } from "../Logic/FeatureSource/TiledFeatureSource/SummaryTileSource" +import summaryLayer from "../assets/generated/layers/summary.json" +import { LayerConfigJson } from "./ThemeConfig/Json/LayerConfigJson" /** * * The themeviewState contains all the state needed for the themeViewGUI. @@ -140,7 +142,6 @@ export default class ThemeViewState implements SpecialVisualizationState { * Triggered by navigating the map with arrows or by pressing 'space' or 'enter' */ public readonly visualFeedback: UIEventSource = new UIEventSource(false) - private readonly newPointDialog: FilteredLayer constructor(layout: LayoutConfig) { Utils.initDomPurify() @@ -309,7 +310,6 @@ export default class ThemeViewState implements SpecialVisualizationState { fs.layer.layerDef.maxAgeOfCache ) }) - this.newPointDialog = this.layerState.filteredLayers.get("last_click") this.floors = this.featuresInView.features.stabilized(500).map((features) => { if (!features) { @@ -343,10 +343,7 @@ export default class ThemeViewState implements SpecialVisualizationState { return sorted }) - this.lastClickObject = new LastClickFeatureSource( - this.mapProperties.lastClickLocation, - this.layout - ) + this.lastClickObject = new LastClickFeatureSource(this.layout) this.osmObjectDownloader = new OsmObjectDownloader( this.osmConnection.Backend(), @@ -446,7 +443,6 @@ export default class ThemeViewState implements SpecialVisualizationState { const feature = this.lastClickObject.createFeature(lon, lat) this.featureProperties.trackFeature(feature) this.selectedElement.setData(feature) - this.selectedLayer.setData(this.newPointDialog.layerDef) } /** @@ -472,16 +468,6 @@ export default class ThemeViewState implements SpecialVisualizationState { this.userRelatedState.markLayoutAsVisited(this.layout) - this.selectedElement.addCallbackAndRunD((feature) => { - // As soon as we have a selected element, we clear the selected element - // This is to work around maplibre, which'll _first_ register the click on the map and only _then_ on the feature - // The only exception is if the last element is the 'add_new'-button, as we don't want it to disappear - if (feature.properties.id === "last_click") { - return - } - this.lastClickObject.features.setData([]) - }) - this.selectedElement.addCallback((selected) => { if (selected === undefined) { Zoomcontrol.resetzoom() @@ -656,6 +642,19 @@ export default class ThemeViewState implements SpecialVisualizationState { }) } + private setupSummaryLayer() { + const layers = this.layout.layers.filter( + (l) => + Constants.priviliged_layers.indexOf(l.id) < 0 && + l.source.geojsonSource === undefined + ) + return new SummaryTileSource( + "http://127.0.0.1:2345", + layers.map((l) => l.id), + this.mapProperties.zoom.map((z) => Math.max(Math.ceil(z) + 1, 0)), + this.mapProperties + ) + } /** * Add the special layers to the map */ @@ -683,6 +682,7 @@ export default class ThemeViewState implements SpecialVisualizationState { ), current_view: this.currentView, favourite: this.favourites, + summary: this.setupSummaryLayer(), } this.closestFeatures.registerSource(specialLayers.favourite, "favourite") @@ -720,15 +720,16 @@ export default class ThemeViewState implements SpecialVisualizationState { rangeIsDisplayed?.syncWith(this.featureSwitches.featureSwitchIsTesting, true) } - // enumarate all 'normal' layers and match them with the appropriate 'special' layer - if applicable + // enumerate all 'normal' layers and match them with the appropriate 'special' layer - if applicable this.layerState.filteredLayers.forEach((flayer) => { const id = flayer.layerDef.id const features: FeatureSource = specialLayers[id] if (features === undefined) { return } - if (id === "favourite") { - console.log("Matching special layer", id, flayer) + if (id === "summary") { + console.log("Skipping summary!") + return } this.featureProperties.trackFeatureSource(features) @@ -741,6 +742,20 @@ export default class ThemeViewState implements SpecialVisualizationState { selectedLayer: this.selectedLayer, }) }) + + const maxzoom = Math.min( + ...this.layout.layers + .filter((l) => Constants.priviliged_layers.indexOf(l.id) < 0) + .map((l) => l.minzoom) + ) + console.log("Maxzoom is", maxzoom) + new ShowDataLayer(this.map, { + features: specialLayers.summary, + layer: new LayerConfig(summaryLayer, "summaryLayer"), + doShowLayer: this.mapProperties.zoom.map((z) => z < maxzoom), + selectedLayer: this.selectedLayer, + selectedElement: this.selectedElement, + }) } /** @@ -761,8 +776,6 @@ export default class ThemeViewState implements SpecialVisualizationState { this.selectedElement.addCallback((selected) => { if (selected === undefined) { - // We did _unselect_ an item - we always remove the lastclick-object - this.lastClickObject.features.setData([]) this.selectedLayer.setData(undefined) this.focusOnMap() } diff --git a/src/UI/DownloadFlow/DownloadButton.svelte b/src/UI/DownloadFlow/DownloadButton.svelte index 229b74b9c..a8d0b1c72 100644 --- a/src/UI/DownloadFlow/DownloadButton.svelte +++ b/src/UI/DownloadFlow/DownloadButton.svelte @@ -35,7 +35,6 @@ async function clicked() { isExporting = true const gpsLayer = state.layerState.filteredLayers.get("gps_location") - state.lastClickObject.features.setData([]) state.userRelatedState.preferencesAsTags.data["__showTimeSensitiveIcons"] = "no" state.userRelatedState.preferencesAsTags.ping() const gpsIsDisplayed = gpsLayer.isDisplayed.data diff --git a/src/UI/Popup/AddNewPoint/AddNewPoint.svelte b/src/UI/Popup/AddNewPoint/AddNewPoint.svelte index d28c573fa..b1da84275 100644 --- a/src/UI/Popup/AddNewPoint/AddNewPoint.svelte +++ b/src/UI/Popup/AddNewPoint/AddNewPoint.svelte @@ -89,7 +89,6 @@ state.selectedElement.setData(undefined) // When aborted, we force the contributors to place the pin _again_ // This is because there might be a nearby object that was disabled; this forces them to re-evaluate the map - state.lastClickObject.features.setData([]) preciseInputIsTapped = false } diff --git a/src/UI/SpecialVisualization.ts b/src/UI/SpecialVisualization.ts index 2799a2e80..ab61da599 100644 --- a/src/UI/SpecialVisualization.ts +++ b/src/UI/SpecialVisualization.ts @@ -84,7 +84,6 @@ export interface SpecialVisualizationState { readonly preferencesAsTags: UIEventSource> readonly language: UIEventSource } - readonly lastClickObject: WritableFeatureSource readonly availableLayers: Store diff --git a/src/UI/ThemeViewGUI.svelte b/src/UI/ThemeViewGUI.svelte index 26e4f6e98..f6ee40867 100644 --- a/src/UI/ThemeViewGUI.svelte +++ b/src/UI/ThemeViewGUI.svelte @@ -96,6 +96,11 @@ if (element.properties.id.startsWith("current_view")) { return currentViewLayer } + console.log(">>> selected:", element) + if(element.properties.id === "new_point_dialog"){ + console.log(">>> searching last_click layer", layout) + return layout.layers.find(l => l.id === "last_click") + } if(element.properties.id === "location_track"){ return layout.layers.find(l => l.id === "gps_track") } @@ -259,7 +264,7 @@
- {#if state.lastClickObject.hasPresets || state.lastClickObject.hasNoteLayer} + {#if state.layout.hasPresets() || state.layout.hasNoteLayer()}