midgard/mapcomplete
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Refactoring of metatagging and extrafunctions to splice out the relation memberships, add calculatedTags and metatags into cache

Browse source
This commit is contained in:
pietervdvn 2021-04-22 13:30:00 +02:00
parent 362abbf079
commit 0dec1d0f75
10 changed files with 98 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
Logic/ExtraFunction.ts
View file

@ -2,6 +2,7 @@ import {GeoOperations} from "./GeoOperations";
import {UIElement} from "../UI/UIElement";
import Combine from "../UI/Base/Combine";
import State from "../State";
import {Relation} from "./Osm/ExtractRelations";
export class ExtraFunction {
@ -40,11 +41,11 @@ Some advanced functions are available on <b>feat</b> as well:
"overlapWith",
"Gives a list of features from the specified layer which this feature overlaps with, the amount of overlap in m². The returned value is <b>{ feat: GeoJSONFeature, overlap: number}</b>",
["...layerIds - one or more layer ids of the layer from which every feature is checked for overlap)"],
(featuresPerLayer, feat) => {
(params, feat) => {
return (...layerIds: string[]) => {
const result = []
for (const layerId of layerIds) {
const otherLayer = featuresPerLayer.get(layerId);
const otherLayer = params.featuresPerLayer.get(layerId);
if (otherLayer === undefined) {
continue;
}
@ -80,10 +81,10 @@ Some advanced functions are available on <b>feat</b> as well:
"closest",
"Given either a list of geojson features or a single layer name, gives the single object which is nearest to the feature. In the case of ways/polygons, only the centerpoint is considered.",
["list of features"],
(featuresPerLayer, feature) => {
(params, feature) => {
return (features) => {
if (typeof features === "string") {
features = featuresPerLayer.get(features)
features = params.featuresPerLayer.get(features)
}
let closestFeature = undefined;
let closestDistance = undefined;
@ -118,11 +119,8 @@ Some advanced functions are available on <b>feat</b> as well:
"memberships",
"Gives a list of {role: string, relation: Relation}-objects, containing all the relations that this feature is part of. \n\nFor example: `_part_of_walking_routes=feat.memberships().map(r => r.relation.tags.name).join(';')`",
[],
(featuresPerLayer, feature) => {
return () => {
return State.state.knownRelations.data?.get(feature.id) ?? [];
}
(params, feature) => {
return () => params.relations ?? [];
}
)
@ -130,9 +128,9 @@ Some advanced functions are available on <b>feat</b> as well:
private readonly _name: string;
private readonly _args: string[];
private readonly _doc: string;
private readonly _f: (featuresPerLayer: Map<string, any[]>, feat: any) => any;
private readonly _f: (params: {featuresPerLayer: Map<string, any[]>, relations: {role: string, relation: Relation}[]}, feat: any) => any;
constructor(name: string, doc: string, args: string[], f: ((featuresPerLayer: Map<string, any[]>, feat: any) => any)) {
constructor(name: string, doc: string, args: string[], f: ((params: {featuresPerLayer: Map<string, any[]>, relations: {role: string, relation: Relation}[]}, feat: any) => any)) {
this._name = name;
this._doc = doc;
this._args = args;
@ -140,9 +138,9 @@ Some advanced functions are available on <b>feat</b> as well:
}
public static FullPatchFeature(featuresPerLayer: Map<string, any[]>, feature) {
public static FullPatchFeature(featuresPerLayer: Map<string, any[]>,relations: {role: string, relation: Relation}[], feature) {
for (const func of ExtraFunction.allFuncs) {
func.PatchFeature(featuresPerLayer, feature);
func.PatchFeature(featuresPerLayer, relations, feature);
}
}
@ -168,7 +166,8 @@ Some advanced functions are available on <b>feat</b> as well:
]);
}
public PatchFeature(featuresPerLayer: Map<string, any[]>, feature: any) {
feature[this._name] = this._f(featuresPerLayer, feature);
public PatchFeature(featuresPerLayer: Map<string, any[]>, relations: {role: string, relation: Relation}[], feature: any) {
feature[this._name] = this._f({featuresPerLayer: featuresPerLayer, relations: relations}, feature);
}
}

4
Logic/FeatureSource/FeatureDuplicatorPerLayer.ts
View file

@ -11,6 +11,10 @@ import LayerConfig from "../../Customizations/JSON/LayerConfig";
export default class FeatureDuplicatorPerLayer implements FeatureSource {
public readonly features: UIEventSource<{ feature: any; freshness: Date }[]>;
public static GetMatchingLayerId(){
}
constructor(layers: UIEventSource<{ layerDef: LayerConfig }[]>, upstream: FeatureSource) {
this.features = upstream.features.map(features => {

2
Logic/FeatureSource/FeaturePipeline.ts
View file

@ -35,7 +35,7 @@ export default class FeaturePipeline implements FeatureSource {
const geojsonSources: FeatureSource [] = GeoJsonSource
.ConstructMultiSource(flayers.data, locationControl)
.map(geojsonSource => new RegisteringFeatureSource(new FeatureDuplicatorPerLayer(flayers, geojsonSource)));
.map(geojsonSource => new RegisteringFeatureSource(new FeatureDuplicatorPerLayer(flayers, geojsonSource)));
const amendedLocalStorageSource =
new RememberingSource(new RegisteringFeatureSource(new FeatureDuplicatorPerLayer(flayers, new LocalStorageSource(layout))

3
Logic/FeatureSource/MetaTaggingFeatureSource.ts
View file

@ -3,6 +3,7 @@ import {UIEventSource} from "../UIEventSource";
import State from "../../State";
import Hash from "../Web/Hash";
import MetaTagging from "../MetaTagging";
import ExtractRelations from "../Osm/ExtractRelations";
export default class MetaTaggingFeatureSource implements FeatureSource {
features: UIEventSource<{ feature: any; freshness: Date }[]> = new UIEventSource<{feature: any; freshness: Date}[]>(undefined);
@ -21,7 +22,7 @@ export default class MetaTaggingFeatureSource implements FeatureSource {
}
})
MetaTagging.addMetatags(featuresFreshness, State.state.layoutToUse.data.layers);
MetaTagging.addMetatags(featuresFreshness, State.state.knownRelations.data, State.state.layoutToUse.data.layers);
self.features.setData(featuresFreshness);
});
}

29
Logic/MetaTagging.ts
View file

@ -1,6 +1,14 @@
import LayerConfig from "../Customizations/JSON/LayerConfig";
import SimpleMetaTagger from "./SimpleMetaTagger";
import {ExtraFunction} from "./ExtraFunction";
import State from "../State";
import {Relation} from "./Osm/ExtractRelations";
interface Params {
featuresPerLayer: Map<string, any[]>,
memberships: Map<string, { role: string, relation: Relation }[]>
}
/**
* Metatagging adds various tags to the elements, e.g. lat, lon, surface area, ...
@ -14,7 +22,8 @@ export default class MetaTagging {
* An actor which adds metatags on every feature in the given object
* The features are a list of geojson-features, with a "properties"-field and geometry
*/
static addMetatags(features: { feature: any; freshness: Date }[], layers: LayerConfig[]) {
static addMetatags(features: { feature: any; freshness: Date }[],
relations: Map<string, { role: string, relation: Relation }[]>, layers: LayerConfig[]) {
for (const metatag of SimpleMetaTagger.metatags) {
try {
@ -26,7 +35,7 @@ export default class MetaTagging {
}
// The functions - per layer - which add the new keys
const layerFuncs = new Map<string, ((featursPerLayer: Map<string, any[]>, feature: any) => void)>();
const layerFuncs = new Map<string, ((params: Params, feature: any) => void)>();
for (const layer of layers) {
layerFuncs.set(layer.id, this.createRetaggingFunc(layer));
}
@ -48,27 +57,26 @@ export default class MetaTagging {
if (f === undefined) {
continue;
}
f(featuresPerLayer, feature.feature)
f({featuresPerLayer: featuresPerLayer, memberships: relations}, feature.feature)
}
}
private static createRetaggingFunc(layer: LayerConfig): ((featuresPerLayer: Map<string, any[]>, feature: any) => void) {
private static createRetaggingFunc(layer: LayerConfig):
((params: Params, feature: any) => void) {
const calculatedTags: [string, string][] = layer.calculatedTags;
if (calculatedTags === undefined) {
return undefined;
}
const functions: ((featuresPerLayer: Map<string, any[]>, feature: any) => void)[] = [];
const functions: ((params: Params, feature: any) => void)[] = [];
for (const entry of calculatedTags) {
const key = entry[0]
const code = entry[1];
if (code === undefined) {
continue;
}
const func = new Function("feat", "return " + code + ";");
const f = (featuresPerLayer, feature: any) => {
@ -76,16 +84,17 @@ export default class MetaTagging {
}
functions.push(f)
}
return (featuresPerLayer: Map<string, any[]>, feature) => {
return (params: Params, feature) => {
const tags = feature.properties
if (tags === undefined) {
return;
}
ExtraFunction.FullPatchFeature(featuresPerLayer, feature);
const relations = params.memberships.get(feature.properties.id)
ExtraFunction.FullPatchFeature(params.featuresPerLayer, relations, feature);
try {
for (const f of functions) {
f(featuresPerLayer, feature);
f(params, feature);
}
} catch (e) {
console.error("While calculating a tag value: ", e)

3
Logic/Osm/ExtractRelations.ts
View file

@ -17,7 +17,6 @@ export default class ExtractRelations {
public static RegisterRelations(overpassJson: any) : void{
const memberships = ExtractRelations.BuildMembershipTable(ExtractRelations.GetRelationElements(overpassJson))
console.log("Assigned memberships: ", memberships)
State.state.knownRelations.setData(memberships)
}
@ -40,7 +39,7 @@ export default class ExtractRelations {
* @param relations
* @constructor
*/
private static BuildMembershipTable(relations: Relation[]): Map<string, { role: string, relation: Relation, }[]> {
public static BuildMembershipTable(relations: Relation[]): Map<string, { role: string, relation: Relation }[]> {
const memberships = new Map<string, { role: string, relation: Relation }[]>()
for (const relation of relations) {

10
Logic/SimpleMetaTagger.ts
View file

@ -9,6 +9,7 @@ import {UIElement} from "../UI/UIElement";
import Combine from "../UI/Base/Combine";
import UpdateTagsFromOsmAPI from "./Actors/UpdateTagsFromOsmAPI";
export default class SimpleMetaTagger {
public readonly keys: string[];
public readonly doc: string;
@ -89,7 +90,12 @@ export default class SimpleMetaTagger {
["_isOpen", "_isOpen:description"],
"If 'opening_hours' is present, it will add the current state of the feature (being 'yes' or 'no')",
(feature => {
if(Utils.runningFromConsole){
// We are running from console, thus probably creating a cache
// isOpen is irrelevant
return
}
const tagsSource = State.state.allElements.getEventSourceById(feature.properties.id);
tagsSource.addCallbackAndRun(tags => {
if (tags.opening_hours === undefined || tags._country === undefined) {
@ -317,7 +323,7 @@ export default class SimpleMetaTagger {
];
static GetCountryCodeFor(lon: number, lat: number, callback: (country: string) => void) {
SimpleMetaTagger.coder.GetCountryCodeFor(lon, lat, callback)
SimpleMetaTagger.coder?.GetCountryCodeFor(lon, lat, callback)
}
static HelpText(): UIElement {

18
assets/layers/slow_roads/slow_roads.json
View file

@ -159,32 +159,34 @@
"width": {
"render": "4"
},
"dashArray": "2 10 6 10",
"color": {
"render": "#bb2",
"dashArray": {
"render": "",
"mappings": [
{
"if": "highway=cycleway",
"then": "#00c"
"then": ""
},
{
"if": "highway=path",
"then": "#bb2"
"then": "10 3"
},
{
"if": "highway=footway",
"then": "#c30"
"then": "10 10"
},
{
"if": "highway=pedestrian",
"then": "#3c3"
"then": "10 10"
},
{
"if": "highway=living_street",
"then": "#ccc"
"then": "10 5 3 5"
}
]
},
"color": {
"render": "#eaba2a"
},
"presets": [
]
}

2
package.json
View file

@ -15,7 +15,7 @@
"generate:translations": "ts-node scripts/generateTranslations.ts",
"generate:layouts": "ts-node scripts/generateLayouts.ts",
"generate:docs": "ts-node scripts/generateDocs.ts && ts-node scripts/generateTaginfoProjectFiles.ts",
"generate:cache:speelplekken": "ts-node scripts/generateCache.ts speelplekken 14 ./cache/speelplekken 51.2003 4.3925 51.1058 4.5087",
"generate:cache:speelplekken": "ts-node scripts/generateCache.ts speelplekken 14 ../pietervdvn.github.io/speelplekken_cache/ 51.20 4.37 51.11 4.51",
"generate:layeroverview": "ts-node scripts/generateLayerOverview.ts --no-fail",
"generate:licenses": "ts-node scripts/generateLicenseInfo.ts --no-fail",
"validate:layeroverview": "ts-node scripts/generateLayerOverview.ts --report",

52
scripts/generateCache.ts
View file

@ -14,6 +14,8 @@ import ScriptUtils from "./ScriptUtils";
import ExtractRelations from "../Logic/Osm/ExtractRelations";
import * as OsmToGeoJson from "osmtogeojson";
import {Script} from "vm";
import MetaTagging from "../Logic/MetaTagging";
import State from "../State";
function createOverpassObject(theme: LayoutConfig) {
let filters: TagsFilter[] = [];
@ -26,8 +28,8 @@ function createOverpassObject(theme: LayoutConfig) {
continue;
}
if (layer.source.geojsonSource !== undefined) {
// Not our responsibility to download this layer!
continue;
// We download these anyway - we are building the cache after all!
//continue;
}
@ -68,17 +70,17 @@ function metaJsonName(targetDir: string, x: number, y: number, z: number): strin
return targetDir + "_" + z + "_" + x + "_" + y + ".meta.json"
}
async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass) {
async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass) {
let downloaded = 0
for (let x = r.xstart; x <= r.xend; x++) {
for (let y = r.ystart; y <= r.yend; y++) {
console.log("x:", (x - r.xstart), "/", (r.xend - r.xstart), "; y:", (y - r.ystart), "/", (r.yend - r.ystart), "; total: ", downloaded, "/", r.total)
downloaded++;
const filename = rawJsonName(targetdir, x, y, r.zoomlevel)
if (existsSync(filename)) {
console.log("Already exists: ", filename)
continue;
}
console.log("x:", (x - r.xstart), "/", (r.xend - r.xstart), "; y:", (y - r.ystart), "/", (r.yend - r.ystart), "; total: ", downloaded, "/", r.total)
const boundsArr = Utils.tile_bounds(r.zoomlevel, x, y)
const bounds = {
@ -87,28 +89,35 @@ async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass)
east: Math.max(boundsArr[0][1], boundsArr[1][1]),
west: Math.min(boundsArr[0][1], boundsArr[1][1])
}
console.log("Downloading tile", r.zoomlevel, x, y, "with bounds", bounds)
const url = overpass.buildQuery("[bbox:" + bounds.south + "," + bounds.west + "," + bounds.north + "," + bounds.east + "]")
let gotResponse = false
ScriptUtils.DownloadJSON(url,
chunks => {
gotResponse = true;
saveResponse(chunks, filename)
})
await ScriptUtils.sleep(10000)
console.debug("Waking up")
while (!gotResponse) {
await ScriptUtils.sleep(10000)
console.debug("Waking up")
if (!gotResponse) {
console.log("Didn't get an answer yet - waiting more")
}
}
}
}
}
async function postProcess(targetdir: string, r: TileRange) {
async function postProcess(targetdir: string, r: TileRange, theme: LayoutConfig) {
let processed = 0;
for (let x = r.xstart; x <= r.xend; x++) {
for (let y = r.ystart; y <= r.yend; y++) {
processed++;
const filename = rawJsonName(targetdir, x, y, r.zoomlevel)
console.log(" Post processing", processed, "/",r. total, filename)
console.log(" Post processing", processed, "/", r.total, filename)
if (!existsSync(filename)) {
throw "Not found - and not downloaded. Run this script again!: " + filename
}
@ -118,11 +127,26 @@ async function postProcess(targetdir: string, r: TileRange) {
// Create and save the geojson file - which is the main chunk of the data
const geojson = OsmToGeoJson.default(rawOsm);
const osmTime = new Date(rawOsm.osm3s.timestamp_osm_base);
for (const feature of geojson.features) {
for (const layer of theme.layers) {
if (layer.source.osmTags.matchesProperties(feature.properties)) {
feature["_matching_layer_id"] = layer.id;
break;
}
}
}
const featuresFreshness = geojson.features.map(feature => ({
freshness: osmTime,
feature: feature
}));
// Extract the relationship information
const relations = ExtractRelations.BuildMembershipTable(ExtractRelations.GetRelationElements(rawOsm))
MetaTagging.addMetatags(featuresFreshness, relations, theme.layers);
writeFileSync(geoJsonName(targetdir, x, y, r.zoomlevel), JSON.stringify(geojson))
// Extract the relationship information
const relations = ExtractRelations.GetRelationElements(rawOsm)
const osmTime = new Date(rawOsm.osm3s.timestamp_osm_base);
const meta = {
freshness: osmTime,
@ -145,7 +169,7 @@ async function main(args: string[]) {
}
const themeName = args[0]
const zoomlevel = Number(args[1])
const targetdir = args[2]
const targetdir = args[2] + "/" + themeName
const lat0 = Number(args[3])
const lon0 = Number(args[4])
const lat1 = Number(args[5])
@ -167,7 +191,7 @@ async function main(args: string[]) {
await downloadRaw(targetdir, tileRange, overpass)
await postProcess(targetdir, tileRange)
await postProcess(targetdir, tileRange, theme)
}