From 352414b29d6bbf68d1e40ed21cbaa817f38a6dd4 Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Mon, 26 Feb 2024 02:24:46 +0100 Subject: [PATCH] Add module to fetch data (via a proxy) from the website with jsonld --- assets/themes/velopark/velopark.json | 13 +- package.json | 4 +- scripts/ScriptUtils.ts | 11 +- scripts/importscripts/compareWebsiteData.ts | 80 ++++++ scripts/scrapeOsm.ts | 0 scripts/serverLdScrape.ts | 1 + scripts/velopark/veloParkToGeojson.ts | 25 +- src/Logic/UIEventSource.ts | 7 +- src/Logic/Web/LinkedDataLoader.ts | 67 ++++- src/UI/Comparison/ComparisonAction.svelte | 71 ++--- src/UI/Comparison/ComparisonTable.svelte | 12 +- src/UI/Comparison/ComparisonTool.svelte | 81 ++---- src/UI/Image/AttributedImage.svelte | 2 +- .../InputElement/Validators/UrlValidator.ts | 5 + src/UI/LinkedDataDisplay.svelte | 90 ------ src/UI/Map/ShowDataLayer.ts | 2 +- src/UI/SpecialVisualizations.ts | 268 +++++++++--------- 17 files changed, 388 insertions(+), 351 deletions(-) create mode 100644 scripts/importscripts/compareWebsiteData.ts create mode 100644 scripts/scrapeOsm.ts delete mode 100644 src/UI/LinkedDataDisplay.svelte diff --git a/assets/themes/velopark/velopark.json b/assets/themes/velopark/velopark.json index 2d43fdcbb..96b99a73f 100644 --- a/assets/themes/velopark/velopark.json +++ b/assets/themes/velopark/velopark.json @@ -90,11 +90,8 @@ "id": "show-data-velopark", "render": { "special": { - "type": "compare_data", - "url": "ref:velopark", - "host": "https://data.velopark.be", - "postprocessing": "velopark", - "readonly": "yes" + "type": "linked_data_from_website", + "key": "ref:velopark" } } }, @@ -338,10 +335,8 @@ }, "render": { "special": { - "type": "compare_data", - "url": "ref:velopark", - "host": "https://data.velopark.be", - "postprocessing": "velopark" + "type": "linked_data_from_website", + "key": "ref:velopark" } } } diff --git a/package.json b/package.json index b0c8bb90e..6a6d3ba8e 100644 --- a/package.json +++ b/package.json @@ -94,7 +94,9 @@ "weblate-fix-heavy": "git fetch weblate-hosted-layers; git fetch weblate-hosted-core; git merge weblate-hosted-layers/master weblate-hosted-core/master ", "housekeeping": "git pull && npx update-browserslist-db@latest && npm run weblate-fix-heavy && npm run generate && npm run generate:docs && npm run generate:contributor-list && vite-node scripts/fetchLanguages.ts && npm run format && git add assets/ langs/ Docs/ **/*.ts Docs/* src/* && git commit -m 'chore: automated housekeeping...'", "reuse-compliance": "reuse lint", - "backup:images": "vite-node scripts/generateImageAnalysis.ts -- ~/data/imgur-image-backup/" + "backup:images": "vite-node scripts/generateImageAnalysis.ts -- ~/data/imgur-image-backup/", + "dloadVelopark": "vite-node scripts/velopark/veloParkToGeojson.ts ", + "scrapeWebsites": "vite-node scripts/importscripts/compareWebsiteData.ts -- ~/Downloads/ShopsWithWebsiteNodes.csv ~/data/scraped_websites/\n" }, "keywords": [ "OpenStreetMap", diff --git a/scripts/ScriptUtils.ts b/scripts/ScriptUtils.ts index 0f969da76..cff0f8c86 100644 --- a/scripts/ScriptUtils.ts +++ b/scripts/ScriptUtils.ts @@ -148,7 +148,16 @@ export default class ScriptUtils { const data = await ScriptUtils.Download(url, headers) return JSON.parse(data["content"]) } - + public static async DownloadFetch( + url: string, + headers?: any + ): Promise<{ content: string } | { redirect: string }> { + console.log("Fetching", url) + const req = await fetch(url, {headers}) + const data= await req.text() + console.log("Fetched", url,data) + return {content: data} + } public static Download( url: string, headers?: any diff --git a/scripts/importscripts/compareWebsiteData.ts b/scripts/importscripts/compareWebsiteData.ts new file mode 100644 index 000000000..87233d34d --- /dev/null +++ b/scripts/importscripts/compareWebsiteData.ts @@ -0,0 +1,80 @@ +import fs from "fs" +// import readline from "readline" +import Script from "../Script" +import LinkedDataLoader from "../../src/Logic/Web/LinkedDataLoader" +import UrlValidator from "../../src/UI/InputElement/Validators/UrlValidator" +// vite-node scripts/importscripts/compareWebsiteData.ts -- ~/Downloads/ShopsWithWebsiteNodes.csv ~/data/scraped_websites/ +/* +class CompareWebsiteData extends Script { + constructor() { + super("Given a csv file with 'id', 'tags' and 'website', attempts to fetch jsonld and compares the attributes. Usage: csv-file datadir") + } + + private readonly urlFormatter = new UrlValidator() + async getWithCache(cachedir : string, url: string): Promise{ + const filename= cachedir+"/"+encodeURIComponent(url) + if(fs.existsSync(filename)){ + return JSON.parse(fs.readFileSync(filename, "utf-8")) + } + const jsonLd = await LinkedDataLoader.fetchJsonLdWithProxy(url) + console.log("Got:", jsonLd) + fs.writeFileSync(filename, JSON.stringify(jsonLd)) + return jsonLd + } + async handleEntry(line: string, cachedir: string, targetfile: string) : Promise{ + const id = JSON.parse(line.split(",")[0]) + let tags = line.substring(line.indexOf("{") - 1) + tags = tags.substring(1, tags.length - 1) + tags = tags.replace(/""/g, "\"") + const data = JSON.parse(tags) + + const website = data.website //this.urlFormatter.reformat(data.website) + if(!website.startsWith("https://stores.delhaize.be")){ + return false + } + console.log(website) + const jsonld = await this.getWithCache(cachedir, website) + console.log(jsonld) + if(Object.keys(jsonld).length === 0){ + return false + } + const diff = LinkedDataLoader.removeDuplicateData(jsonld, data) + fs.appendFileSync(targetfile, id +", "+ JSON.stringify(diff)+"\n") + return true + } + + async main(args: string[]): Promise { + if (args.length < 2) { + throw "Not enough arguments" + } + + + const readInterface = readline.createInterface({ + input: fs.createReadStream(args[0]), + }) + + let handled = 0 + let diffed = 0 + const targetfile = "diff.csv" + fs.writeFileSync(targetfile, "id, diff-json\n") + for await (const line of readInterface) { + try { + if(line.startsWith("\"id\"")){ + continue + } + const madeComparison = await this.handleEntry(line, args[1], targetfile) + handled ++ + diffed = diffed + (madeComparison ? 1 : 0) + if(handled % 1000 == 0){ + // console.log("Handled ",handled," got ",diffed,"diff results") + } + } catch (e) { + // console.error(e) + } + } + + } +} + +new CompareWebsiteData().run() +*/ diff --git a/scripts/scrapeOsm.ts b/scripts/scrapeOsm.ts new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/serverLdScrape.ts b/scripts/serverLdScrape.ts index d1770a2b1..5a08f27a6 100644 --- a/scripts/serverLdScrape.ts +++ b/scripts/serverLdScrape.ts @@ -15,6 +15,7 @@ class ServerLdScrape extends Script { mimetype: "application/ld+json", async handle(content, searchParams: URLSearchParams) { const url = searchParams.get("url") + console.log("Fetching", url) if (cache[url]) { return JSON.stringify(cache[url]) } diff --git a/scripts/velopark/veloParkToGeojson.ts b/scripts/velopark/veloParkToGeojson.ts index 6e7a5582c..1afa3996d 100644 --- a/scripts/velopark/veloParkToGeojson.ts +++ b/scripts/velopark/veloParkToGeojson.ts @@ -1,39 +1,42 @@ import Script from "../Script" -import { Utils } from "../../src/Utils" -import VeloparkLoader, { VeloparkData } from "../../src/Logic/Web/VeloparkLoader" import fs from "fs" import { Overpass } from "../../src/Logic/Osm/Overpass" import { RegexTag } from "../../src/Logic/Tags/RegexTag" import Constants from "../../src/Models/Constants" import { ImmutableStore } from "../../src/Logic/UIEventSource" import { BBox } from "../../src/Logic/BBox" +import LinkedDataLoader from "../../src/Logic/Web/LinkedDataLoader" class VeloParkToGeojson extends Script { constructor() { super( - "Downloads the latest Velopark data and converts it to a geojson, which will be saved at the current directory" + "Downloads the latest Velopark data and converts it to a geojson, which will be saved at the current directory", ) } exportTo(filename: string, features) { - fs.writeFileSync( - filename + "_" + new Date().toISOString() + ".geojson", + features = features.slice(0,25) // TODO REMOVE + const file = filename + "_" + /*new Date().toISOString() + */".geojson" + fs.writeFileSync(file, JSON.stringify( { type: "FeatureCollection", + "#":"Only 25 features are shown!", // TODO REMOVE features, }, null, - " " - ) + " ", + ), ) + console.log("Written",file) } async main(args: string[]): Promise { console.log("Downloading velopark data") // Download data for NIS-code 1000. 1000 means: all of belgium const url = "https://www.velopark.be/api/parkings/1000" - const data = await Utils.downloadJson(url) + const allVelopark = await LinkedDataLoader.fetchJsonLd(url, { country: "be" }) + this.exportTo("velopark_all", allVelopark) const bboxBelgium = new BBox([ [2.51357303225, 49.5294835476], @@ -44,15 +47,13 @@ class VeloParkToGeojson extends Script { [], Constants.defaultOverpassUrls[0], new ImmutableStore(60 * 5), - false + false, ) const alreadyLinkedFeatures = await alreadyLinkedQuery.queryGeoJson(bboxBelgium) const seenIds = new Set( - alreadyLinkedFeatures[0].features.map((f) => f.properties["ref:velopark"]) + alreadyLinkedFeatures[0].features.map((f) => f.properties["ref:velopark"]), ) console.log("OpenStreetMap contains", seenIds.size, "bicycle parkings with a velopark ref") - const allVelopark = data.map((f) => VeloparkLoader.convert(f)) - this.exportTo("velopark_all", allVelopark) const features = allVelopark.filter((f) => !seenIds.has(f.properties["ref:velopark"])) diff --git a/src/Logic/UIEventSource.ts b/src/Logic/UIEventSource.ts index 70aa5da87..af5a033fa 100644 --- a/src/Logic/UIEventSource.ts +++ b/src/Logic/UIEventSource.ts @@ -237,8 +237,11 @@ export abstract class Store implements Readable { public bindD(f: (t: Exclude) => Store): Store { return this.bind((t) => { - if (t === undefined || t === null) { - return t + if(t=== null){ + return null + } + if (t === undefined ) { + return undefined } return f(>t) }) diff --git a/src/Logic/Web/LinkedDataLoader.ts b/src/Logic/Web/LinkedDataLoader.ts index 8b738983f..5c317bccd 100644 --- a/src/Logic/Web/LinkedDataLoader.ts +++ b/src/Logic/Web/LinkedDataLoader.ts @@ -6,7 +6,11 @@ import PhoneValidator from "../../UI/InputElement/Validators/PhoneValidator" import EmailValidator from "../../UI/InputElement/Validators/EmailValidator" import { Validator } from "../../UI/InputElement/Validator" import UrlValidator from "../../UI/InputElement/Validators/UrlValidator" +import Constants from "../../Models/Constants" +interface JsonLdLoaderOptions { + country?: string +} export default class LinkedDataLoader { private static readonly COMPACTING_CONTEXT = { name: "http://schema.org/name", @@ -43,6 +47,10 @@ export default class LinkedDataLoader { "http://schema.org/contactPoint", ] + private static ignoreTypes = [ + "Breadcrumblist" + ] + static async geoToGeometry(geo): Promise { const context = { lat: { @@ -102,15 +110,30 @@ export default class LinkedDataLoader { return OH.ToString(OH.MergeTimes(allRules)) } - static async fetchJsonLd(url: string, country?: string): Promise> { - const proxy = "http://127.0.0.1:2346/extractgraph" // "https://cache.mapcomplete.org/extractgraph" - const data = await Utils.downloadJson(`${proxy}?url=${url}`) - const compacted = await jsonld.compact(data, LinkedDataLoader.COMPACTING_CONTEXT) + static async fetchJsonLdWithProxy(url: string, options?: JsonLdLoaderOptions): Promise { + const urlWithProxy = Constants.linkedDataProxy.replace("{url}", encodeURIComponent(url)) + return await this.fetchJsonLd(urlWithProxy, options) + } + + /** + * + * + * { + * "content": "{\"@context\":\"http://schema.org\",\"@type\":\"LocalBusiness\",\"@id\":\"http://stores.delhaize.be/nl/ad-delhaize-munsterbilzen\",\"name\":\"AD Delhaize Munsterbilzen\",\"url\":\"http://stores.delhaize.be/nl/ad-delhaize-munsterbilzen\",\"logo\":\"https://stores.delhaize.be/build/images/web/shop/delhaize-be/favicon.ico\",\"image\":\"http://stores.delhaize.be/image/mobilosoft-testing?apiPath=rehab/delhaize-be/images/location/ad%20delhaize%20image%20ge%CC%81ne%CC%81rale%20%281%29%201652787176865&imageSize=h_500\",\"email\":\"\",\"telephone\":\"+3289413520\",\"address\":{\"@type\":\"PostalAddress\",\"streetAddress\":\"Waterstraat, 18\",\"addressLocality\":\"Bilzen\",\"postalCode\":\"3740\",\"addressCountry\":\"BE\"},\"geo\":{\"@type\":\"GeoCoordinates\",\"latitude\":50.8906898,\"longitude\":5.5260586},\"openingHoursSpecification\":[{\"@type\":\"OpeningHoursSpecification\",\"dayOfWeek\":\"Tuesday\",\"opens\":\"08:00\",\"closes\":\"18:30\"},{\"@type\":\"OpeningHoursSpecification\",\"dayOfWeek\":\"Wednesday\",\"opens\":\"08:00\",\"closes\":\"18:30\"},{\"@type\":\"OpeningHoursSpecification\",\"dayOfWeek\":\"Thursday\",\"opens\":\"08:00\",\"closes\":\"18:30\"},{\"@type\":\"OpeningHoursSpecification\",\"dayOfWeek\":\"Friday\",\"opens\":\"08:00\",\"closes\":\"18:30\"},{\"@type\":\"OpeningHoursSpecification\",\"dayOfWeek\":\"Saturday\",\"opens\":\"08:00\",\"closes\":\"18:30\"},{\"@type\":\"OpeningHoursSpecification\",\"dayOfWeek\":\"Sunday\",\"opens\":\"08:00\",\"closes\":\"12:00\"},{\"@type\":\"OpeningHoursSpecification\",\"dayOfWeek\":\"Monday\",\"opens\":\"12:00\",\"closes\":\"18:30\"}],\"@base\":\"https://stores.delhaize.be/nl/ad-delhaize-munsterbilzen\"}" + * } + */ + private static async compact(data: any, options?: JsonLdLoaderOptions): Promise{ + console.log("Compacting",data) + if(Array.isArray(data)) { + return await Promise.all(data.map(d => LinkedDataLoader.compact(d))) + } + const country = options?.country + const compacted = await jsonld.compact(data, LinkedDataLoader.COMPACTING_CONTEXT) compacted["opening_hours"] = await LinkedDataLoader.ohToOsmFormat( compacted["opening_hours"] ) if (compacted["openingHours"]) { - const ohspec: string[] = compacted["openingHours"] + const ohspec: string[] = compacted["openingHours"] compacted["opening_hours"] = OH.simplify( ohspec.map((r) => LinkedDataLoader.ohStringToOsmFormat(r)).join("; ") ) @@ -138,5 +161,39 @@ export default class LinkedDataLoader { } } return compacted + + } + static async fetchJsonLd(url: string, options?: JsonLdLoaderOptions): Promise { + const data = await Utils.downloadJson(url) + return await LinkedDataLoader.compact(data, options) + } + + /** + * Only returns different items + * @param externalData + * @param currentData + */ + static removeDuplicateData(externalData: Record, currentData: Record) : Record{ + const d = { ...externalData } + delete d["@context"] + for (const k in d) { + const v = currentData[k] + if (!v) { + continue + } + if (k === "opening_hours") { + const oh = [].concat(...v.split(";").map(r => OH.ParseRule(r) ?? [])) + const merged = OH.ToString(OH.MergeTimes(oh ?? [])) + if (merged === d[k]) { + delete d[k] + continue + } + } + if (v === d[k]) { + delete d[k] + } + delete d.geo + } + return d } } diff --git a/src/UI/Comparison/ComparisonAction.svelte b/src/UI/Comparison/ComparisonAction.svelte index 233e31c70..c44e0dd7c 100644 --- a/src/UI/Comparison/ComparisonAction.svelte +++ b/src/UI/Comparison/ComparisonAction.svelte @@ -1,48 +1,51 @@ {key} + {#if $tags[key]} + {$tags[key]} + {/if} {#if externalProperties[key].startsWith("http")} diff --git a/src/UI/Comparison/ComparisonTable.svelte b/src/UI/Comparison/ComparisonTable.svelte index f71c830e3..c806e2953 100644 --- a/src/UI/Comparison/ComparisonTable.svelte +++ b/src/UI/Comparison/ComparisonTable.svelte @@ -26,23 +26,21 @@ let externalKeys: string[] = Object.keys(externalProperties).sort() const imageKeyRegex = /image|image:[0-9]+/ - console.log("Calculating knwon images") let knownImages = new Set( Object.keys(osmProperties) .filter((k) => k.match(imageKeyRegex)) .map((k) => osmProperties[k]) ) - console.log("Known images are:", knownImages) let unknownImages = externalKeys .filter((k) => k.match(imageKeyRegex)) .map((k) => externalProperties[k]) .filter((i) => !knownImages.has(i)) let propertyKeysExternal = externalKeys.filter((k) => k.match(imageKeyRegex) === null) - let missing = propertyKeysExternal.filter((k) => osmProperties[k] === undefined) + let missing = propertyKeysExternal.filter((k) => osmProperties[k] === undefined && typeof externalProperties[k] === "string") let same = propertyKeysExternal.filter((key) => osmProperties[key] === externalProperties[key]) let different = propertyKeysExternal.filter( - (key) => osmProperties[key] !== undefined && osmProperties[key] !== externalProperties[key] + (key) => osmProperties[key] !== undefined && osmProperties[key] !== externalProperties[key] && typeof externalProperties[key] === "string" ) let currentStep: "init" | "applying_all" | "all_applied" = "init" @@ -68,11 +66,7 @@ External {#each different as key} - - {key} - {osmProperties[key]} - {externalProperties[key]} - + {/each} {/if} diff --git a/src/UI/Comparison/ComparisonTool.svelte b/src/UI/Comparison/ComparisonTool.svelte index 29c1d53aa..7f96ced49 100644 --- a/src/UI/Comparison/ComparisonTool.svelte +++ b/src/UI/Comparison/ComparisonTool.svelte @@ -1,65 +1,34 @@ - -{#if error !== undefined} +{#if $externalData === null} + +{:else if $externalData === undefined} + {$externalData} +{:else if $externalData["error"] !== undefined}
- Something went wrong: {error} + Something went wrong: {$externalData["error"]}
-{:else if data === undefined} - - Loading {$tags[url]} - -{:else if data.properties !== undefined} +{:else if $externalData["success"] !== undefined} let fallbackImage: string = undefined if (image.provider === Mapillary.singleton) { fallbackImage = "./assets/svg/blocked.svg" diff --git a/src/UI/InputElement/Validators/UrlValidator.ts b/src/UI/InputElement/Validators/UrlValidator.ts index a17754ab6..aa70abc15 100644 --- a/src/UI/InputElement/Validators/UrlValidator.ts +++ b/src/UI/InputElement/Validators/UrlValidator.ts @@ -41,6 +41,11 @@ export default class UrlValidator extends Validator { "AdGroup", "TargetId", "msclkid", + "pk_source", + "pk_medium", + "pk_campaign", + "pk_content", + "pk_kwd" ] for (const dontLike of blacklistedTrackingParams) { url.searchParams.delete(dontLike.toLowerCase()) diff --git a/src/UI/LinkedDataDisplay.svelte b/src/UI/LinkedDataDisplay.svelte deleted file mode 100644 index d7b106dbe..000000000 --- a/src/UI/LinkedDataDisplay.svelte +++ /dev/null @@ -1,90 +0,0 @@ - -{#if $error} -
- {$error} -
-{:else if $url} -
- {#if $dataCleaned !== undefined && Object.keys($dataCleaned).length === 0} - No new data from website - {:else if !$data} - - {:else} - {$distanceToFeature} -
    - {#each Object.keys($dataCleaned) as k} -
  • - {k}: {JSON.stringify($dataCleaned[k])} {$tagsSource[k]} {($dataCleaned[k]) === $tagsSource[k]} -
    • - {/each} -
- {/if} -
-{/if} diff --git a/src/UI/Map/ShowDataLayer.ts b/src/UI/Map/ShowDataLayer.ts index 295d1fac1..2b6fac354 100644 --- a/src/UI/Map/ShowDataLayer.ts +++ b/src/UI/Map/ShowDataLayer.ts @@ -459,7 +459,7 @@ class LineRenderingLayer { } else { const tags = this._fetchStore(id) this._listenerInstalledOn.add(id) - tags.addCallbackAndRunD((properties) => { + tags?.addCallbackAndRunD((properties) => { // Make sure to use 'getSource' here, the layer names are different! try { if (map.getSource(this._layername) === undefined) { diff --git a/src/UI/SpecialVisualizations.ts b/src/UI/SpecialVisualizations.ts index 808f046fe..6c5afafd7 100644 --- a/src/UI/SpecialVisualizations.ts +++ b/src/UI/SpecialVisualizations.ts @@ -3,11 +3,7 @@ import { FixedUiElement } from "./Base/FixedUiElement" import BaseUIElement from "./BaseUIElement" import Title from "./Base/Title" import Table from "./Base/Table" -import { - RenderingSpecification, - SpecialVisualization, - SpecialVisualizationState, -} from "./SpecialVisualization" +import { RenderingSpecification, SpecialVisualization, SpecialVisualizationState } from "./SpecialVisualization" import { HistogramViz } from "./Popup/HistogramViz" import { MinimapViz } from "./Popup/MinimapViz" import { ShareLinkViz } from "./Popup/ShareLinkViz" @@ -93,7 +89,7 @@ import SpecialVisualisationUtils from "./SpecialVisualisationUtils" import LoginButton from "./Base/LoginButton.svelte" import Toggle from "./Input/Toggle" import ImportReviewIdentity from "./Reviews/ImportReviewIdentity.svelte" -import LinkedDataDisplay from "./LinkedDataDisplay.svelte" +import LinkedDataLoader from "../Logic/Web/LinkedDataLoader" class NearbyImageVis implements SpecialVisualization { // Class must be in SpecialVisualisations due to weird cyclical import that breaks the tests @@ -120,7 +116,7 @@ class NearbyImageVis implements SpecialVisualization { tags: UIEventSource>, args: string[], feature: Feature, - layer: LayerConfig + layer: LayerConfig, ): BaseUIElement { const isOpen = args[0] === "open" const readonly = args[1] === "readonly" @@ -187,7 +183,7 @@ class StealViz implements SpecialVisualization { selectedElement: otherFeature, state, layer, - }) + }), ) } if (elements.length === 1) { @@ -195,8 +191,8 @@ class StealViz implements SpecialVisualization { } return new Combine(elements).SetClass("flex flex-col") }, - [state.indexedFeatures.featuresById] - ) + [state.indexedFeatures.featuresById], + ), ) } @@ -235,7 +231,7 @@ export class QuestionViz implements SpecialVisualization { tags: UIEventSource>, args: string[], feature: Feature, - layer: LayerConfig + layer: LayerConfig, ): BaseUIElement { const labels = args[0] ?.split(";") @@ -271,38 +267,38 @@ export default class SpecialVisualizations { viz.docs, viz.args.length > 0 ? new Table( - ["name", "default", "description"], - viz.args.map((arg) => { - let defaultArg = arg.defaultValue ?? "_undefined_" - if (defaultArg == "") { - defaultArg = "_empty string_" - } - return [arg.name, defaultArg, arg.doc] - }) - ) + ["name", "default", "description"], + viz.args.map((arg) => { + let defaultArg = arg.defaultValue ?? "_undefined_" + if (defaultArg == "") { + defaultArg = "_empty string_" + } + return [arg.name, defaultArg, arg.doc] + }), + ) : undefined, new Title("Example usage of " + viz.funcName, 4), new FixedUiElement( viz.example ?? - "`{" + - viz.funcName + - "(" + - viz.args.map((arg) => arg.defaultValue).join(",") + - ")}`" + "`{" + + viz.funcName + + "(" + + viz.args.map((arg) => arg.defaultValue).join(",") + + ")}`", ).SetClass("literal-code"), ]) } public static constructSpecification( template: string, - extraMappings: SpecialVisualization[] = [] + extraMappings: SpecialVisualization[] = [], ): RenderingSpecification[] { return SpecialVisualisationUtils.constructSpecification(template, extraMappings) } public static HelpMessage() { const helpTexts = SpecialVisualizations.specialVisualizations.map((viz) => - SpecialVisualizations.DocumentationFor(viz) + SpecialVisualizations.DocumentationFor(viz), ) return new Combine([ @@ -336,10 +332,10 @@ export default class SpecialVisualizations { }, }, null, - " " - ) + " ", + ), ).SetClass("code"), - 'In other words: use `{ "before": ..., "after": ..., "special": {"type": ..., "argname": ...argvalue...}`. The args are in the `special` block; an argvalue can be a string, a translation or another value. (Refer to class `RewriteSpecial` in case of problems)', + "In other words: use `{ \"before\": ..., \"after\": ..., \"special\": {\"type\": ..., \"argname\": ...argvalue...}`. The args are in the `special` block; an argvalue can be a string, a translation or another value. (Refer to class `RewriteSpecial` in case of problems)", ]).SetClass("flex flex-col"), ...helpTexts, ]).SetClass("flex flex-col") @@ -348,20 +344,20 @@ export default class SpecialVisualizations { // noinspection JSUnusedGlobalSymbols public static renderExampleOfSpecial( state: SpecialVisualizationState, - s: SpecialVisualization + s: SpecialVisualization, ): BaseUIElement { const examples = s.structuredExamples === undefined ? [] : s.structuredExamples().map((e) => { - return s.constr( - state, - new UIEventSource>(e.feature.properties), - e.args, - e.feature, - undefined - ) - }) + return s.constr( + state, + new UIEventSource>(e.feature.properties), + e.args, + e.feature, + undefined, + ) + }) return new Combine([new Title(s.funcName), s.docs, ...examples]) } @@ -401,7 +397,7 @@ export default class SpecialVisualizations { assignTo: state.userRelatedState.language, availableLanguages: state.layout.language, preferredLanguages: state.osmConnection.userDetails.map( - (ud) => ud.languages + (ud) => ud.languages, ), }) }, @@ -426,7 +422,7 @@ export default class SpecialVisualizations { constr( state: SpecialVisualizationState, - tagSource: UIEventSource> + tagSource: UIEventSource>, ): BaseUIElement { return new VariableUiElement( tagSource @@ -436,7 +432,7 @@ export default class SpecialVisualizations { return new SplitRoadWizard(id, state) } return undefined - }) + }), ) }, }, @@ -450,7 +446,7 @@ export default class SpecialVisualizations { tagSource: UIEventSource>, argument: string[], feature: Feature, - layer: LayerConfig + layer: LayerConfig, ): BaseUIElement { if (feature.geometry.type !== "Point") { return undefined @@ -473,7 +469,7 @@ export default class SpecialVisualizations { tagSource: UIEventSource>, argument: string[], feature: Feature, - layer: LayerConfig + layer: LayerConfig, ): BaseUIElement { if (!layer.deletion) { return undefined @@ -501,7 +497,7 @@ export default class SpecialVisualizations { state: SpecialVisualizationState, tagSource: UIEventSource>, argument: string[], - feature: Feature + feature: Feature, ): BaseUIElement { const [lon, lat] = GeoOperations.centerpointCoordinates(feature) return new SvelteUIElement(CreateNewNote, { @@ -565,7 +561,7 @@ export default class SpecialVisualizations { .map((tags) => tags[args[0]]) .map((wikidata) => { wikidata = Utils.NoEmpty( - wikidata?.split(";")?.map((wd) => wd.trim()) ?? [] + wikidata?.split(";")?.map((wd) => wd.trim()) ?? [], )[0] const entry = Wikidata.LoadWikidataEntry(wikidata) return new VariableUiElement( @@ -575,9 +571,9 @@ export default class SpecialVisualizations { } const response = e["success"] return Translation.fromMap(response.labels) - }) + }), ) - }) + }), ), }, new MapillaryLinkVis(), @@ -591,7 +587,7 @@ export default class SpecialVisualizations { tags: UIEventSource>, _, __, - layer: LayerConfig + layer: LayerConfig, ) => new SvelteUIElement(AllTagsPanel, { tags, layer }), }, { @@ -613,7 +609,7 @@ export default class SpecialVisualizations { return new ImageCarousel( AllImageProviders.LoadImagesFor(tags, imagePrefixes), tags, - state + state, ) }, }, @@ -669,7 +665,7 @@ export default class SpecialVisualizations { { nameKey: nameKey, fallbackName, - } + }, ) return new SvelteUIElement(StarsBarIcon, { score: reviews.average, @@ -702,7 +698,7 @@ export default class SpecialVisualizations { { nameKey: nameKey, fallbackName, - } + }, ) return new SvelteUIElement(ReviewForm, { reviews, state, tags, feature, layer }) }, @@ -734,7 +730,7 @@ export default class SpecialVisualizations { { nameKey: nameKey, fallbackName, - } + }, ) return new SvelteUIElement(AllReviews, { reviews, state, tags, feature, layer }) }, @@ -754,7 +750,7 @@ export default class SpecialVisualizations { tagSource: UIEventSource>, argument: string[], feature: Feature, - layer: LayerConfig + layer: LayerConfig, ): BaseUIElement { const [text] = argument return new SvelteUIElement(ImportReviewIdentity, { state, text }) @@ -813,7 +809,7 @@ export default class SpecialVisualizations { tags: UIEventSource>, args: string[], feature: Feature, - layer: LayerConfig + layer: LayerConfig, ): SvelteUIElement { const keyToUse = args[0] const prefix = args[1] @@ -850,17 +846,17 @@ export default class SpecialVisualizations { return undefined } const allUnits: Unit[] = [].concat( - ...(state?.layout?.layers?.map((lyr) => lyr.units) ?? []) + ...(state?.layout?.layers?.map((lyr) => lyr.units) ?? []), ) const unit = allUnits.filter((unit) => - unit.isApplicableToKey(key) + unit.isApplicableToKey(key), )[0] if (unit === undefined) { return value } const getCountry = () => tagSource.data._country return unit.asHumanLongValue(value, getCountry) - }) + }), ) }, }, @@ -877,7 +873,7 @@ export default class SpecialVisualizations { new Combine([ t.downloadFeatureAsGeojson.SetClass("font-bold text-lg"), t.downloadGeoJsonHelper.SetClass("subtle"), - ]).SetClass("flex flex-col") + ]).SetClass("flex flex-col"), ) .onClick(() => { console.log("Exporting as Geojson") @@ -890,7 +886,7 @@ export default class SpecialVisualizations { title + "_mapcomplete_export.geojson", { mimetype: "application/vnd.geo+json", - } + }, ) }) .SetClass("w-full") @@ -926,7 +922,7 @@ export default class SpecialVisualizations { constr: (state) => { return new SubtleButton( Svg.delete_icon_svg().SetStyle("height: 1.5rem"), - Translations.t.general.removeLocationHistory + Translations.t.general.removeLocationHistory, ).onClick(() => { state.historicalUserLocations.features.setData([]) state.selectedElement.setData(undefined) @@ -964,10 +960,10 @@ export default class SpecialVisualizations { .filter((c) => c.text !== "") .map( (c, i) => - new NoteCommentElement(c, state, i, comments.length) - ) + new NoteCommentElement(c, state, i, comments.length), + ), ).SetClass("flex flex-col") - }) + }), ), }, { @@ -1001,7 +997,7 @@ export default class SpecialVisualizations { tagsSource: UIEventSource>, _: string[], feature: Feature, - layer: LayerConfig + layer: LayerConfig, ) => new VariableUiElement( tagsSource.map((tags) => { @@ -1019,7 +1015,7 @@ export default class SpecialVisualizations { feature, layer, }).SetClass("px-1") - }) + }), ), }, { @@ -1035,8 +1031,8 @@ export default class SpecialVisualizations { let challenge = Stores.FromPromise( Utils.downloadJsonCached( `${Maproulette.defaultEndpoint}/challenge/${parentId}`, - 24 * 60 * 60 * 1000 - ) + 24 * 60 * 60 * 1000, + ), ) return new VariableUiElement( @@ -1061,7 +1057,7 @@ export default class SpecialVisualizations { } else { return [title, new List(listItems)] } - }) + }), ) }, docs: "Fetches the metadata of MapRoulette campaign that this task is part of and shows those details (namely `title`, `description` and `instruction`).\n\nThis reads the property `mr_challengeId` to detect the parent campaign.", @@ -1075,15 +1071,15 @@ export default class SpecialVisualizations { "\n" + "```json\n" + "{\n" + - ' "id": "mark_duplicate",\n' + - ' "render": {\n' + - ' "special": {\n' + - ' "type": "maproulette_set_status",\n' + - ' "message": {\n' + - ' "en": "Mark as not found or false positive"\n' + + " \"id\": \"mark_duplicate\",\n" + + " \"render\": {\n" + + " \"special\": {\n" + + " \"type\": \"maproulette_set_status\",\n" + + " \"message\": {\n" + + " \"en\": \"Mark as not found or false positive\"\n" + " },\n" + - ' "status": "2",\n' + - ' "image": "close"\n' + + " \"status\": \"2\",\n" + + " \"image\": \"close\"\n" + " }\n" + " }\n" + "}\n" + @@ -1163,8 +1159,8 @@ export default class SpecialVisualizations { const fsBboxed = new BBoxFeatureSourceForLayer(fs, bbox) return new StatisticsPanel(fsBboxed) }, - [state.mapProperties.bounds] - ) + [state.mapProperties.bounds], + ), ) }, }, @@ -1230,7 +1226,7 @@ export default class SpecialVisualizations { constr( state: SpecialVisualizationState, tagSource: UIEventSource>, - args: string[] + args: string[], ): BaseUIElement { let [text, href, classnames, download, ariaLabel] = args if (download === "") { @@ -1244,14 +1240,14 @@ export default class SpecialVisualizations { text: Utils.SubstituteKeys(text, tags), href: Utils.SubstituteKeys(href, tags).replaceAll( / /g, - "%20" + "%20", ) /* Chromium based browsers eat the spaces */, classnames, download: Utils.SubstituteKeys(download, tags), ariaLabel: Utils.SubstituteKeys(ariaLabel, tags), newTab, - }) - ) + }), + ), ) }, }, @@ -1273,7 +1269,7 @@ export default class SpecialVisualizations { }, }, null, - " " + " ", ) + "\n```", args: [ @@ -1297,7 +1293,7 @@ export default class SpecialVisualizations { featureTags: UIEventSource>, args: string[], feature: Feature, - layer: LayerConfig + layer: LayerConfig, ) { const [key, tr, classesRaw] = args let classes = classesRaw ?? "" @@ -1322,7 +1318,7 @@ export default class SpecialVisualizations { elements.push(subsTr) } return elements - }) + }), ) }, }, @@ -1342,7 +1338,7 @@ export default class SpecialVisualizations { tagSource: UIEventSource>, argument: string[], feature: Feature, - layer: LayerConfig + layer: LayerConfig, ): BaseUIElement { return new VariableUiElement( tagSource.map((tags) => { @@ -1354,7 +1350,7 @@ export default class SpecialVisualizations { console.error("Cannot create a translation for", v, "due to", e) return JSON.stringify(v) } - }) + }), ) }, }, @@ -1374,7 +1370,7 @@ export default class SpecialVisualizations { tagSource: UIEventSource>, argument: string[], feature: Feature, - layer: LayerConfig + layer: LayerConfig, ): BaseUIElement { const key = argument[0] const validator = new FediverseValidator() @@ -1384,7 +1380,7 @@ export default class SpecialVisualizations { .map((fediAccount) => { fediAccount = validator.reformat(fediAccount) const [_, username, host] = fediAccount.match( - FediverseValidator.usernameAtServer + FediverseValidator.usernameAtServer, ) const normalLink = new SvelteUIElement(Link, { @@ -1396,10 +1392,10 @@ export default class SpecialVisualizations { const loggedInContributorMastodon = state.userRelatedState?.preferencesAsTags?.data?.[ "_mastodon_link" - ] + ] console.log( "LoggedinContributorMastodon", - loggedInContributorMastodon + loggedInContributorMastodon, ) if (!loggedInContributorMastodon) { return normalLink @@ -1415,7 +1411,7 @@ export default class SpecialVisualizations { newTab: true, }).SetClass("button"), ]) - }) + }), ) }, }, @@ -1435,7 +1431,7 @@ export default class SpecialVisualizations { tagSource: UIEventSource>, args: string[], feature: Feature, - layer: LayerConfig + layer: LayerConfig, ): BaseUIElement { return new FixedUiElement("{" + args[0] + "}") }, @@ -1456,7 +1452,7 @@ export default class SpecialVisualizations { tagSource: UIEventSource>, argument: string[], feature: Feature, - layer: LayerConfig + layer: LayerConfig, ): BaseUIElement { const key = argument[0] ?? "value" return new VariableUiElement( @@ -1474,12 +1470,12 @@ export default class SpecialVisualizations { } catch (e) { return new FixedUiElement( "Could not parse this tag: " + - JSON.stringify(value) + - " due to " + - e + JSON.stringify(value) + + " due to " + + e, ).SetClass("alert") } - }) + }), ) }, }, @@ -1500,7 +1496,7 @@ export default class SpecialVisualizations { tagSource: UIEventSource>, argument: string[], feature: Feature, - layer: LayerConfig + layer: LayerConfig, ): BaseUIElement { const giggityUrl = argument[0] return new SvelteUIElement(Giggity, { tags: tagSource, state, giggityUrl }) @@ -1516,12 +1512,12 @@ export default class SpecialVisualizations { _: UIEventSource>, argument: string[], feature: Feature, - layer: LayerConfig + layer: LayerConfig, ): BaseUIElement { const tags = (( state )).geolocation.currentUserLocation.features.map( - (features) => features[0]?.properties + (features) => features[0]?.properties, ) return new Combine([ new SvelteUIElement(OrientationDebugPanel, {}), @@ -1543,7 +1539,7 @@ export default class SpecialVisualizations { tagSource: UIEventSource>, argument: string[], feature: Feature, - layer: LayerConfig + layer: LayerConfig, ): BaseUIElement { return new SvelteUIElement(MarkAsFavourite, { tags: tagSource, @@ -1563,7 +1559,7 @@ export default class SpecialVisualizations { tagSource: UIEventSource>, argument: string[], feature: Feature, - layer: LayerConfig + layer: LayerConfig, ): BaseUIElement { return new SvelteUIElement(MarkAsFavouriteMini, { tags: tagSource, @@ -1583,7 +1579,7 @@ export default class SpecialVisualizations { tagSource: UIEventSource>, argument: string[], feature: Feature, - layer: LayerConfig + layer: LayerConfig, ): BaseUIElement { return new SvelteUIElement(DirectionIndicator, { state, feature }) }, @@ -1598,7 +1594,7 @@ export default class SpecialVisualizations { tagSource: UIEventSource>, argument: string[], feature: Feature, - layer: LayerConfig + layer: LayerConfig, ): BaseUIElement { return new VariableUiElement( tagSource @@ -1620,9 +1616,9 @@ export default class SpecialVisualizations { `${window.location.protocol}//${window.location.host}${window.location.pathname}?${layout}lat=${lat}&lon=${lon}&z=15` + `#${id}` return new Img(new Qr(url).toImageElement(75)).SetStyle( - "width: 75px" + "width: 75px", ) - }) + }), ) }, }, @@ -1642,7 +1638,7 @@ export default class SpecialVisualizations { tagSource: UIEventSource>, args: string[], feature: Feature, - layer: LayerConfig + layer: LayerConfig, ): BaseUIElement { const key = args[0] === "" ? "_direction:centerpoint" : args[0] return new VariableUiElement( @@ -1653,11 +1649,11 @@ export default class SpecialVisualizations { }) .mapD((value) => { const dir = GeoOperations.bearingToHuman( - GeoOperations.parseBearing(value) + GeoOperations.parseBearing(value), ) console.log("Human dir", dir) return Translations.t.general.visualFeedback.directionsAbsolute[dir] - }) + }), ) }, }, @@ -1675,11 +1671,6 @@ export default class SpecialVisualizations { required: true, doc: "The domain name(s) where data might be fetched from - this is needed to set the CSP. A domain must include 'https', e.g. 'https://example.com'. For multiple domains, separate them with ';'. If you don't know the possible domains, use '*'. ", }, - { - name: "postprocessing", - required: false, - doc: "Apply some postprocessing. Currently, only 'velopark' is allowed as value", - }, { name: "readonly", required: false, @@ -1692,19 +1683,19 @@ export default class SpecialVisualizations { tagSource: UIEventSource>, args: string[], feature: Feature, - layer: LayerConfig + layer: LayerConfig, ): BaseUIElement { const url = args[0] - const postprocessVelopark = args[2] === "velopark" const readonly = args[3] === "yes" + const externalData = Stores.FromPromiseWithErr(Utils.downloadJson(url)) return new SvelteUIElement(ComparisonTool, { url, - postprocessVelopark, state, tags: tagSource, layer, feature, readonly, + externalData, }) }, }, @@ -1718,12 +1709,12 @@ export default class SpecialVisualizations { tagSource: UIEventSource>, args: string[], feature: Feature, - layer: LayerConfig + layer: LayerConfig, ): BaseUIElement { return new Toggle( undefined, new SvelteUIElement(LoginButton), - state.osmConnection.isLoggedIn + state.osmConnection.isLoggedIn, ) }, }, @@ -1740,19 +1731,36 @@ export default class SpecialVisualizations { needsUrls: [Constants.linkedDataProxy], constr( state: SpecialVisualizationState, - tagsSource: UIEventSource>, + tags: UIEventSource>, argument: string[], feature: Feature, - layer: LayerConfig + layer: LayerConfig, ): BaseUIElement { const key = argument[0] ?? "website" - return new SvelteUIElement(LinkedDataDisplay, { - feature, - state, - tagsSource, - key, - layer, + let url = tags.mapD(tags => { + if (!tags._country || !tags[key] || tags[key] === "undefined") { + return null + } + return ({ url: tags[key], country: tags._country }) }) + const externalData: Store<{ success: { content: any } } | { + error: string + } | undefined | null> = url.bindD(({ + url, + country, + }) => Stores.FromPromiseWithErr(LinkedDataLoader.fetchJsonLdWithProxy(url, { country }))) + + + + return new Toggle( + new SvelteUIElement(ComparisonTool, { + feature, + state, + tags, + layer, + externalData, + }), undefined, url.map(url => !!url), + ) }, }, ] @@ -1766,7 +1774,7 @@ export default class SpecialVisualizations { throw ( "Invalid special visualisation found: funcName is undefined for " + invalid.map((sp) => sp.i).join(", ") + - '. Did you perhaps type \n funcName: "funcname" // type declaration uses COLON\ninstead of:\n funcName = "funcName" // value definition uses EQUAL' + ". Did you perhaps type \n funcName: \"funcname\" // type declaration uses COLON\ninstead of:\n funcName = \"funcName\" // value definition uses EQUAL" ) }