Add module to fetch data (via a proxy) from the website with jsonld

This commit is contained in:
Pieter Vander Vennet 2024-02-26 02:24:46 +01:00
parent 1b06eee15b
commit 352414b29d
17 changed files with 388 additions and 351 deletions

View file

@ -90,11 +90,8 @@
"id": "show-data-velopark",
"render": {
"special": {
"type": "compare_data",
"url": "ref:velopark",
"host": "https://data.velopark.be",
"postprocessing": "velopark",
"readonly": "yes"
"type": "linked_data_from_website",
"key": "ref:velopark"
}
}
},
@ -338,10 +335,8 @@
},
"render": {
"special": {
"type": "compare_data",
"url": "ref:velopark",
"host": "https://data.velopark.be",
"postprocessing": "velopark"
"type": "linked_data_from_website",
"key": "ref:velopark"
}
}
}

View file

@ -94,7 +94,9 @@
"weblate-fix-heavy": "git fetch weblate-hosted-layers; git fetch weblate-hosted-core; git merge weblate-hosted-layers/master weblate-hosted-core/master ",
"housekeeping": "git pull && npx update-browserslist-db@latest && npm run weblate-fix-heavy && npm run generate && npm run generate:docs && npm run generate:contributor-list && vite-node scripts/fetchLanguages.ts && npm run format && git add assets/ langs/ Docs/ **/*.ts Docs/* src/* && git commit -m 'chore: automated housekeeping...'",
"reuse-compliance": "reuse lint",
"backup:images": "vite-node scripts/generateImageAnalysis.ts -- ~/data/imgur-image-backup/"
"backup:images": "vite-node scripts/generateImageAnalysis.ts -- ~/data/imgur-image-backup/",
"dloadVelopark": "vite-node scripts/velopark/veloParkToGeojson.ts ",
"scrapeWebsites": "vite-node scripts/importscripts/compareWebsiteData.ts -- ~/Downloads/ShopsWithWebsiteNodes.csv ~/data/scraped_websites/\n"
},
"keywords": [
"OpenStreetMap",

View file

@ -148,7 +148,16 @@ export default class ScriptUtils {
const data = await ScriptUtils.Download(url, headers)
return JSON.parse(data["content"])
}
public static async DownloadFetch(
url: string,
headers?: any
): Promise<{ content: string } | { redirect: string }> {
console.log("Fetching", url)
const req = await fetch(url, {headers})
const data= await req.text()
console.log("Fetched", url,data)
return {content: data}
}
public static Download(
url: string,
headers?: any

View file

@ -0,0 +1,80 @@
import fs from "fs"
// import readline from "readline"
import Script from "../Script"
import LinkedDataLoader from "../../src/Logic/Web/LinkedDataLoader"
import UrlValidator from "../../src/UI/InputElement/Validators/UrlValidator"
// vite-node scripts/importscripts/compareWebsiteData.ts -- ~/Downloads/ShopsWithWebsiteNodes.csv ~/data/scraped_websites/
/*
class CompareWebsiteData extends Script {
constructor() {
super("Given a csv file with 'id', 'tags' and 'website', attempts to fetch jsonld and compares the attributes. Usage: csv-file datadir")
}
private readonly urlFormatter = new UrlValidator()
async getWithCache(cachedir : string, url: string): Promise<any>{
const filename= cachedir+"/"+encodeURIComponent(url)
if(fs.existsSync(filename)){
return JSON.parse(fs.readFileSync(filename, "utf-8"))
}
const jsonLd = await LinkedDataLoader.fetchJsonLdWithProxy(url)
console.log("Got:", jsonLd)
fs.writeFileSync(filename, JSON.stringify(jsonLd))
return jsonLd
}
async handleEntry(line: string, cachedir: string, targetfile: string) : Promise<boolean>{
const id = JSON.parse(line.split(",")[0])
let tags = line.substring(line.indexOf("{") - 1)
tags = tags.substring(1, tags.length - 1)
tags = tags.replace(/""/g, "\"")
const data = JSON.parse(tags)
const website = data.website //this.urlFormatter.reformat(data.website)
if(!website.startsWith("https://stores.delhaize.be")){
return false
}
console.log(website)
const jsonld = await this.getWithCache(cachedir, website)
console.log(jsonld)
if(Object.keys(jsonld).length === 0){
return false
}
const diff = LinkedDataLoader.removeDuplicateData(jsonld, data)
fs.appendFileSync(targetfile, id +", "+ JSON.stringify(diff)+"\n")
return true
}
async main(args: string[]): Promise<void> {
if (args.length < 2) {
throw "Not enough arguments"
}
const readInterface = readline.createInterface({
input: fs.createReadStream(args[0]),
})
let handled = 0
let diffed = 0
const targetfile = "diff.csv"
fs.writeFileSync(targetfile, "id, diff-json\n")
for await (const line of readInterface) {
try {
if(line.startsWith("\"id\"")){
continue
}
const madeComparison = await this.handleEntry(line, args[1], targetfile)
handled ++
diffed = diffed + (madeComparison ? 1 : 0)
if(handled % 1000 == 0){
// console.log("Handled ",handled," got ",diffed,"diff results")
}
} catch (e) {
// console.error(e)
}
}
}
}
new CompareWebsiteData().run()
*/

0
scripts/scrapeOsm.ts Normal file
View file

View file

@ -15,6 +15,7 @@ class ServerLdScrape extends Script {
mimetype: "application/ld+json",
async handle(content, searchParams: URLSearchParams) {
const url = searchParams.get("url")
console.log("Fetching", url)
if (cache[url]) {
return JSON.stringify(cache[url])
}

View file

@ -1,39 +1,42 @@
import Script from "../Script"
import { Utils } from "../../src/Utils"
import VeloparkLoader, { VeloparkData } from "../../src/Logic/Web/VeloparkLoader"
import fs from "fs"
import { Overpass } from "../../src/Logic/Osm/Overpass"
import { RegexTag } from "../../src/Logic/Tags/RegexTag"
import Constants from "../../src/Models/Constants"
import { ImmutableStore } from "../../src/Logic/UIEventSource"
import { BBox } from "../../src/Logic/BBox"
import LinkedDataLoader from "../../src/Logic/Web/LinkedDataLoader"
class VeloParkToGeojson extends Script {
constructor() {
super(
"Downloads the latest Velopark data and converts it to a geojson, which will be saved at the current directory"
"Downloads the latest Velopark data and converts it to a geojson, which will be saved at the current directory",
)
}
exportTo(filename: string, features) {
fs.writeFileSync(
filename + "_" + new Date().toISOString() + ".geojson",
features = features.slice(0,25) // TODO REMOVE
const file = filename + "_" + /*new Date().toISOString() + */".geojson"
fs.writeFileSync(file,
JSON.stringify(
{
type: "FeatureCollection",
"#":"Only 25 features are shown!", // TODO REMOVE
features,
},
null,
" "
)
" ",
),
)
console.log("Written",file)
}
async main(args: string[]): Promise<void> {
console.log("Downloading velopark data")
// Download data for NIS-code 1000. 1000 means: all of belgium
const url = "https://www.velopark.be/api/parkings/1000"
const data = <VeloparkData[]>await Utils.downloadJson(url)
const allVelopark = await LinkedDataLoader.fetchJsonLd(url, { country: "be" })
this.exportTo("velopark_all", allVelopark)
const bboxBelgium = new BBox([
[2.51357303225, 49.5294835476],
@ -44,15 +47,13 @@ class VeloParkToGeojson extends Script {
[],
Constants.defaultOverpassUrls[0],
new ImmutableStore(60 * 5),
false
false,
)
const alreadyLinkedFeatures = await alreadyLinkedQuery.queryGeoJson(bboxBelgium)
const seenIds = new Set<string>(
alreadyLinkedFeatures[0].features.map((f) => f.properties["ref:velopark"])
alreadyLinkedFeatures[0].features.map((f) => f.properties["ref:velopark"]),
)
console.log("OpenStreetMap contains", seenIds.size, "bicycle parkings with a velopark ref")
const allVelopark = data.map((f) => VeloparkLoader.convert(f))
this.exportTo("velopark_all", allVelopark)
const features = allVelopark.filter((f) => !seenIds.has(f.properties["ref:velopark"]))

View file

@ -237,8 +237,11 @@ export abstract class Store<T> implements Readable<T> {
public bindD<X>(f: (t: Exclude<T, undefined | null>) => Store<X>): Store<X> {
return this.bind((t) => {
if (t === undefined || t === null) {
return <undefined | null>t
if(t=== null){
return null
}
if (t === undefined ) {
return undefined
}
return f(<Exclude<T, undefined | null>>t)
})

View file

@ -6,7 +6,11 @@ import PhoneValidator from "../../UI/InputElement/Validators/PhoneValidator"
import EmailValidator from "../../UI/InputElement/Validators/EmailValidator"
import { Validator } from "../../UI/InputElement/Validator"
import UrlValidator from "../../UI/InputElement/Validators/UrlValidator"
import Constants from "../../Models/Constants"
interface JsonLdLoaderOptions {
country?: string
}
export default class LinkedDataLoader {
private static readonly COMPACTING_CONTEXT = {
name: "http://schema.org/name",
@ -43,6 +47,10 @@ export default class LinkedDataLoader {
"http://schema.org/contactPoint",
]
private static ignoreTypes = [
"Breadcrumblist"
]
static async geoToGeometry(geo): Promise<Geometry> {
const context = {
lat: {
@ -102,15 +110,30 @@ export default class LinkedDataLoader {
return OH.ToString(OH.MergeTimes(allRules))
}
static async fetchJsonLd(url: string, country?: string): Promise<Record<string, any>> {
const proxy = "http://127.0.0.1:2346/extractgraph" // "https://cache.mapcomplete.org/extractgraph"
const data = await Utils.downloadJson(`${proxy}?url=${url}`)
const compacted = await jsonld.compact(data, LinkedDataLoader.COMPACTING_CONTEXT)
static async fetchJsonLdWithProxy(url: string, options?: JsonLdLoaderOptions): Promise<any> {
const urlWithProxy = Constants.linkedDataProxy.replace("{url}", encodeURIComponent(url))
return await this.fetchJsonLd(urlWithProxy, options)
}
/**
*
*
* {
* "content": "{\"@context\":\"http://schema.org\",\"@type\":\"LocalBusiness\",\"@id\":\"http://stores.delhaize.be/nl/ad-delhaize-munsterbilzen\",\"name\":\"AD Delhaize Munsterbilzen\",\"url\":\"http://stores.delhaize.be/nl/ad-delhaize-munsterbilzen\",\"logo\":\"https://stores.delhaize.be/build/images/web/shop/delhaize-be/favicon.ico\",\"image\":\"http://stores.delhaize.be/image/mobilosoft-testing?apiPath=rehab/delhaize-be/images/location/ad%20delhaize%20image%20ge%CC%81ne%CC%81rale%20%281%29%201652787176865&imageSize=h_500\",\"email\":\"\",\"telephone\":\"+3289413520\",\"address\":{\"@type\":\"PostalAddress\",\"streetAddress\":\"Waterstraat, 18\",\"addressLocality\":\"Bilzen\",\"postalCode\":\"3740\",\"addressCountry\":\"BE\"},\"geo\":{\"@type\":\"GeoCoordinates\",\"latitude\":50.8906898,\"longitude\":5.5260586},\"openingHoursSpecification\":[{\"@type\":\"OpeningHoursSpecification\",\"dayOfWeek\":\"Tuesday\",\"opens\":\"08:00\",\"closes\":\"18:30\"},{\"@type\":\"OpeningHoursSpecification\",\"dayOfWeek\":\"Wednesday\",\"opens\":\"08:00\",\"closes\":\"18:30\"},{\"@type\":\"OpeningHoursSpecification\",\"dayOfWeek\":\"Thursday\",\"opens\":\"08:00\",\"closes\":\"18:30\"},{\"@type\":\"OpeningHoursSpecification\",\"dayOfWeek\":\"Friday\",\"opens\":\"08:00\",\"closes\":\"18:30\"},{\"@type\":\"OpeningHoursSpecification\",\"dayOfWeek\":\"Saturday\",\"opens\":\"08:00\",\"closes\":\"18:30\"},{\"@type\":\"OpeningHoursSpecification\",\"dayOfWeek\":\"Sunday\",\"opens\":\"08:00\",\"closes\":\"12:00\"},{\"@type\":\"OpeningHoursSpecification\",\"dayOfWeek\":\"Monday\",\"opens\":\"12:00\",\"closes\":\"18:30\"}],\"@base\":\"https://stores.delhaize.be/nl/ad-delhaize-munsterbilzen\"}"
* }
*/
private static async compact(data: any, options?: JsonLdLoaderOptions): Promise<any>{
console.log("Compacting",data)
if(Array.isArray(data)) {
return await Promise.all(data.map(d => LinkedDataLoader.compact(d)))
}
const country = options?.country
const compacted = await jsonld.compact(data, <any> LinkedDataLoader.COMPACTING_CONTEXT)
compacted["opening_hours"] = await LinkedDataLoader.ohToOsmFormat(
compacted["opening_hours"]
)
if (compacted["openingHours"]) {
const ohspec: string[] = compacted["openingHours"]
const ohspec: string[] = <any> compacted["openingHours"]
compacted["opening_hours"] = OH.simplify(
ohspec.map((r) => LinkedDataLoader.ohStringToOsmFormat(r)).join("; ")
)
@ -138,5 +161,39 @@ export default class LinkedDataLoader {
}
}
return <any>compacted
}
static async fetchJsonLd(url: string, options?: JsonLdLoaderOptions): Promise<any> {
const data = await Utils.downloadJson(url)
return await LinkedDataLoader.compact(data, options)
}
/**
* Only returns different items
* @param externalData
* @param currentData
*/
static removeDuplicateData(externalData: Record<string, string>, currentData: Record<string, string>) : Record<string, string>{
const d = { ...externalData }
delete d["@context"]
for (const k in d) {
const v = currentData[k]
if (!v) {
continue
}
if (k === "opening_hours") {
const oh = [].concat(...v.split(";").map(r => OH.ParseRule(r) ?? []))
const merged = OH.ToString(OH.MergeTimes(oh ?? []))
if (merged === d[k]) {
delete d[k]
continue
}
}
if (v === d[k]) {
delete d[k]
}
delete d.geo
}
return d
}
}

View file

@ -1,48 +1,51 @@
<script lang="ts">
import { UIEventSource } from "../../Logic/UIEventSource"
import type { OsmTags } from "../../Models/OsmFeature"
import type { SpecialVisualizationState } from "../SpecialVisualization"
import type { Feature } from "geojson"
import LayerConfig from "../../Models/ThemeConfig/LayerConfig"
import ChangeTagAction from "../../Logic/Osm/Actions/ChangeTagAction"
import { Tag } from "../../Logic/Tags/Tag"
import Loading from "../Base/Loading.svelte"
import { UIEventSource } from "../../Logic/UIEventSource"
import type { OsmTags } from "../../Models/OsmFeature"
import type { SpecialVisualizationState } from "../SpecialVisualization"
import type { Feature } from "geojson"
import LayerConfig from "../../Models/ThemeConfig/LayerConfig"
import ChangeTagAction from "../../Logic/Osm/Actions/ChangeTagAction"
import { Tag } from "../../Logic/Tags/Tag"
import Loading from "../Base/Loading.svelte"
export let key: string
export let externalProperties: Record<string, string>
export let key: string
export let externalProperties: Record<string, string>
export let tags: UIEventSource<OsmTags>
export let state: SpecialVisualizationState
export let feature: Feature
export let layer: LayerConfig
export let tags: UIEventSource<OsmTags>
export let state: SpecialVisualizationState
export let feature: Feature
export let layer: LayerConfig
export let readonly = false
export let readonly = false
let currentStep: "init" | "applying" | "done" = "init"
let currentStep: "init" | "applying" | "done" = "init"
/**
* Copy the given key into OSM
* @param key
*/
async function apply(key: string) {
currentStep = "applying"
const change = new ChangeTagAction(
tags.data.id,
new Tag(key, externalProperties[key]),
tags.data,
{
theme: state.layout.id,
changeType: "import",
}
)
await state.changes.applyChanges(await change.CreateChangeDescriptions())
currentStep = "done"
}
/**
* Copy the given key into OSM
* @param key
*/
async function apply(key: string) {
currentStep = "applying"
const change = new ChangeTagAction(
tags.data.id,
new Tag(key, externalProperties[key]),
tags.data,
{
theme: state.layout.id,
changeType: "import",
},
)
await state.changes.applyChanges(await change.CreateChangeDescriptions())
currentStep = "done"
}
</script>
<tr>
<td><b>{key}</b></td>
{#if $tags[key]}
{$tags[key]}
{/if}
<td>
{#if externalProperties[key].startsWith("http")}
<a href={externalProperties[key]} target="_blank">

View file

@ -26,23 +26,21 @@
let externalKeys: string[] = Object.keys(externalProperties).sort()
const imageKeyRegex = /image|image:[0-9]+/
console.log("Calculating knwon images")
let knownImages = new Set(
Object.keys(osmProperties)
.filter((k) => k.match(imageKeyRegex))
.map((k) => osmProperties[k])
)
console.log("Known images are:", knownImages)
let unknownImages = externalKeys
.filter((k) => k.match(imageKeyRegex))
.map((k) => externalProperties[k])
.filter((i) => !knownImages.has(i))
let propertyKeysExternal = externalKeys.filter((k) => k.match(imageKeyRegex) === null)
let missing = propertyKeysExternal.filter((k) => osmProperties[k] === undefined)
let missing = propertyKeysExternal.filter((k) => osmProperties[k] === undefined && typeof externalProperties[k] === "string")
let same = propertyKeysExternal.filter((key) => osmProperties[key] === externalProperties[key])
let different = propertyKeysExternal.filter(
(key) => osmProperties[key] !== undefined && osmProperties[key] !== externalProperties[key]
(key) => osmProperties[key] !== undefined && osmProperties[key] !== externalProperties[key] && typeof externalProperties[key] === "string"
)
let currentStep: "init" | "applying_all" | "all_applied" = "init"
@ -68,11 +66,7 @@
<th>External</th>
</tr>
{#each different as key}
<tr>
<td>{key}</td>
<td>{osmProperties[key]}</td>
<td>{externalProperties[key]}</td>
</tr>
<ComparisonAction {key} {state} {tags} {externalProperties} {layer} {feature} {readonly} />
{/each}
</table>
{/if}

View file

@ -1,65 +1,34 @@
<script lang="ts">
/**
* The comparison tool loads json-data from a speficied URL, eventually post-processes it
* and compares it with the current object
*/
import { onMount } from "svelte"
import { Utils } from "../../Utils"
import VeloparkLoader from "../../Logic/Web/VeloparkLoader"
import Loading from "../Base/Loading.svelte"
import type { SpecialVisualizationState } from "../SpecialVisualization"
import { UIEventSource } from "../../Logic/UIEventSource"
import ComparisonTable from "./ComparisonTable.svelte"
import LayerConfig from "../../Models/ThemeConfig/LayerConfig"
import type { Feature } from "geojson"
import type { OsmTags } from "../../Models/OsmFeature"
/**
* The comparison tool loads json-data from a speficied URL, eventually post-processes it
* and compares it with the current object
*/
import Loading from "../Base/Loading.svelte"
import type { SpecialVisualizationState } from "../SpecialVisualization"
import { Store, UIEventSource } from "../../Logic/UIEventSource"
import ComparisonTable from "./ComparisonTable.svelte"
import LayerConfig from "../../Models/ThemeConfig/LayerConfig"
import type { Feature } from "geojson"
import type { OsmTags } from "../../Models/OsmFeature"
export let url: string
export let postprocessVelopark: boolean
export let state: SpecialVisualizationState
export let tags: UIEventSource<OsmTags>
export let layer: LayerConfig
export let feature: Feature
export let readonly = false
let data: any = undefined
let error: any = undefined
onMount(async () => {
const _url = tags.data[url]
if (!_url) {
error = "No URL found in attribute" + url
}
try {
console.log("Attempting to download", _url)
const downloaded = await Utils.downloadJsonAdvanced(_url)
if (downloaded["error"]) {
console.error(downloaded)
error = downloaded["error"]
return
}
if (postprocessVelopark) {
data = VeloparkLoader.convert(downloaded["content"])
return
}
data = downloaded["content"]
} catch (e) {
console.error(e)
error = "" + e
}
})
export let externalData: Store<{ success: {content: Record<string, string> } } | { error: string } | undefined | null /* null if no URL is found, undefined if loading*/>
export let state: SpecialVisualizationState
export let tags: UIEventSource<OsmTags>
export let layer: LayerConfig
export let feature: Feature
export let readonly = false
</script>
{#if error !== undefined}
{#if $externalData === null}
<!-- empty block -->
{:else if $externalData === undefined}
<Loading>{$externalData}</Loading>
{:else if $externalData["error"] !== undefined}
<div class="alert">
Something went wrong: {error}
Something went wrong: {$externalData["error"]}
</div>
{:else if data === undefined}
<Loading>
Loading {$tags[url]}
</Loading>
{:else if data.properties !== undefined}
{:else if $externalData["success"] !== undefined}
<ComparisonTable
externalProperties={data.properties}
externalProperties={$externalData["success"]}
osmProperties={$tags}
{state}
{feature}

View file

@ -7,7 +7,7 @@
import { Mapillary } from "../../Logic/ImageProviders/Mapillary"
import { UIEventSource } from "../../Logic/UIEventSource"
export let image: ProvidedImage
export let image: Partial<ProvidedImage>
let fallbackImage: string = undefined
if (image.provider === Mapillary.singleton) {
fallbackImage = "./assets/svg/blocked.svg"

View file

@ -41,6 +41,11 @@ export default class UrlValidator extends Validator {
"AdGroup",
"TargetId",
"msclkid",
"pk_source",
"pk_medium",
"pk_campaign",
"pk_content",
"pk_kwd"
]
for (const dontLike of blacklistedTrackingParams) {
url.searchParams.delete(dontLike.toLowerCase())

View file

@ -1,90 +0,0 @@
<script lang="ts">
/**
* Shows attributes that are loaded via linked data and which are suitable for import
*/
import type { SpecialVisualizationState } from "./SpecialVisualization"
import type { Store } from "../Logic/UIEventSource"
import { Stores, UIEventSource } from "../Logic/UIEventSource"
import type { Feature, Geometry } from "geojson"
import LayerConfig from "../Models/ThemeConfig/LayerConfig"
import LinkedDataLoader from "../Logic/Web/LinkedDataLoader"
import Loading from "./Base/Loading.svelte"
import { GeoOperations } from "../Logic/GeoOperations"
import { OH } from "./OpeningHours/OpeningHours"
export let state: SpecialVisualizationState
export let tagsSource: UIEventSource<Record<string, string>>
export let argument: string[]
export let feature: Feature
export let layer: LayerConfig
export let key: string
let url = tagsSource.mapD(tags => {
if (!tags._country || !tags[key] || tags[key] === "undefined") {
return undefined
}
return ({ url: tags[key], country: tags._country })
})
let dataWithErr = url.bindD(({ url, country }) => {
return Stores.FromPromiseWithErr(LinkedDataLoader.fetchJsonLd(url, country))
})
let error = dataWithErr.mapD(d => d["error"])
let data = dataWithErr.mapD(d => d["success"])
let distanceToFeature: Store<string> = data.mapD(d => d.geo).mapD(geo => {
const dist = Math.round(GeoOperations.distanceBetween(
GeoOperations.centerpointCoordinates(<Geometry>geo), GeoOperations.centerpointCoordinates(feature)))
return dist + "m"
})
let dataCleaned = data.mapD(d => {
const featureTags = tagsSource.data
console.log("Downloaded data is", d)
d = { ...d }
delete d["@context"]
for (const k in d) {
const v = featureTags[k]
if (!v) {
continue
}
if (k === "opening_hours") {
const oh = [].concat(...v.split(";").map(r => OH.ParseRule(r) ?? []))
const merged = OH.ToString(OH.MergeTimes(oh ?? []))
if (merged === d[k]) {
delete d[k]
continue
}
}
if (featureTags[k] === d[k]) {
delete d[k]
}
delete d.geo
}
return d
}, [tagsSource])
</script>
{#if $error}
<div class="alert">
{$error}
</div>
{:else if $url}
<div class="flex flex-col border border-dashed border-gray-500 p-1">
{#if $dataCleaned !== undefined && Object.keys($dataCleaned).length === 0}
No new data from website
{:else if !$data}
<Loading />
{:else}
{$distanceToFeature}
<ul>
{#each Object.keys($dataCleaned) as k}
<li>
<b>{k}</b>: {JSON.stringify($dataCleaned[k])} {$tagsSource[k]} {($dataCleaned[k]) === $tagsSource[k]}
</li>
{/each}
</ul>
{/if}
</div>
{/if}

View file

@ -459,7 +459,7 @@ class LineRenderingLayer {
} else {
const tags = this._fetchStore(id)
this._listenerInstalledOn.add(id)
tags.addCallbackAndRunD((properties) => {
tags?.addCallbackAndRunD((properties) => {
// Make sure to use 'getSource' here, the layer names are different!
try {
if (map.getSource(this._layername) === undefined) {

View file

@ -3,11 +3,7 @@ import { FixedUiElement } from "./Base/FixedUiElement"
import BaseUIElement from "./BaseUIElement"
import Title from "./Base/Title"
import Table from "./Base/Table"
import {
RenderingSpecification,
SpecialVisualization,
SpecialVisualizationState,
} from "./SpecialVisualization"
import { RenderingSpecification, SpecialVisualization, SpecialVisualizationState } from "./SpecialVisualization"
import { HistogramViz } from "./Popup/HistogramViz"
import { MinimapViz } from "./Popup/MinimapViz"
import { ShareLinkViz } from "./Popup/ShareLinkViz"
@ -93,7 +89,7 @@ import SpecialVisualisationUtils from "./SpecialVisualisationUtils"
import LoginButton from "./Base/LoginButton.svelte"
import Toggle from "./Input/Toggle"
import ImportReviewIdentity from "./Reviews/ImportReviewIdentity.svelte"
import LinkedDataDisplay from "./LinkedDataDisplay.svelte"
import LinkedDataLoader from "../Logic/Web/LinkedDataLoader"
class NearbyImageVis implements SpecialVisualization {
// Class must be in SpecialVisualisations due to weird cyclical import that breaks the tests
@ -120,7 +116,7 @@ class NearbyImageVis implements SpecialVisualization {
tags: UIEventSource<Record<string, string>>,
args: string[],
feature: Feature,
layer: LayerConfig
layer: LayerConfig,
): BaseUIElement {
const isOpen = args[0] === "open"
const readonly = args[1] === "readonly"
@ -187,7 +183,7 @@ class StealViz implements SpecialVisualization {
selectedElement: otherFeature,
state,
layer,
})
}),
)
}
if (elements.length === 1) {
@ -195,8 +191,8 @@ class StealViz implements SpecialVisualization {
}
return new Combine(elements).SetClass("flex flex-col")
},
[state.indexedFeatures.featuresById]
)
[state.indexedFeatures.featuresById],
),
)
}
@ -235,7 +231,7 @@ export class QuestionViz implements SpecialVisualization {
tags: UIEventSource<Record<string, string>>,
args: string[],
feature: Feature,
layer: LayerConfig
layer: LayerConfig,
): BaseUIElement {
const labels = args[0]
?.split(";")
@ -271,38 +267,38 @@ export default class SpecialVisualizations {
viz.docs,
viz.args.length > 0
? new Table(
["name", "default", "description"],
viz.args.map((arg) => {
let defaultArg = arg.defaultValue ?? "_undefined_"
if (defaultArg == "") {
defaultArg = "_empty string_"
}
return [arg.name, defaultArg, arg.doc]
})
)
["name", "default", "description"],
viz.args.map((arg) => {
let defaultArg = arg.defaultValue ?? "_undefined_"
if (defaultArg == "") {
defaultArg = "_empty string_"
}
return [arg.name, defaultArg, arg.doc]
}),
)
: undefined,
new Title("Example usage of " + viz.funcName, 4),
new FixedUiElement(
viz.example ??
"`{" +
viz.funcName +
"(" +
viz.args.map((arg) => arg.defaultValue).join(",") +
")}`"
"`{" +
viz.funcName +
"(" +
viz.args.map((arg) => arg.defaultValue).join(",") +
")}`",
).SetClass("literal-code"),
])
}
public static constructSpecification(
template: string,
extraMappings: SpecialVisualization[] = []
extraMappings: SpecialVisualization[] = [],
): RenderingSpecification[] {
return SpecialVisualisationUtils.constructSpecification(template, extraMappings)
}
public static HelpMessage() {
const helpTexts = SpecialVisualizations.specialVisualizations.map((viz) =>
SpecialVisualizations.DocumentationFor(viz)
SpecialVisualizations.DocumentationFor(viz),
)
return new Combine([
@ -336,10 +332,10 @@ export default class SpecialVisualizations {
},
},
null,
" "
)
" ",
),
).SetClass("code"),
'In other words: use `{ "before": ..., "after": ..., "special": {"type": ..., "argname": ...argvalue...}`. The args are in the `special` block; an argvalue can be a string, a translation or another value. (Refer to class `RewriteSpecial` in case of problems)',
"In other words: use `{ \"before\": ..., \"after\": ..., \"special\": {\"type\": ..., \"argname\": ...argvalue...}`. The args are in the `special` block; an argvalue can be a string, a translation or another value. (Refer to class `RewriteSpecial` in case of problems)",
]).SetClass("flex flex-col"),
...helpTexts,
]).SetClass("flex flex-col")
@ -348,20 +344,20 @@ export default class SpecialVisualizations {
// noinspection JSUnusedGlobalSymbols
public static renderExampleOfSpecial(
state: SpecialVisualizationState,
s: SpecialVisualization
s: SpecialVisualization,
): BaseUIElement {
const examples =
s.structuredExamples === undefined
? []
: s.structuredExamples().map((e) => {
return s.constr(
state,
new UIEventSource<Record<string, string>>(e.feature.properties),
e.args,
e.feature,
undefined
)
})
return s.constr(
state,
new UIEventSource<Record<string, string>>(e.feature.properties),
e.args,
e.feature,
undefined,
)
})
return new Combine([new Title(s.funcName), s.docs, ...examples])
}
@ -401,7 +397,7 @@ export default class SpecialVisualizations {
assignTo: state.userRelatedState.language,
availableLanguages: state.layout.language,
preferredLanguages: state.osmConnection.userDetails.map(
(ud) => ud.languages
(ud) => ud.languages,
),
})
},
@ -426,7 +422,7 @@ export default class SpecialVisualizations {
constr(
state: SpecialVisualizationState,
tagSource: UIEventSource<Record<string, string>>
tagSource: UIEventSource<Record<string, string>>,
): BaseUIElement {
return new VariableUiElement(
tagSource
@ -436,7 +432,7 @@ export default class SpecialVisualizations {
return new SplitRoadWizard(<WayId>id, state)
}
return undefined
})
}),
)
},
},
@ -450,7 +446,7 @@ export default class SpecialVisualizations {
tagSource: UIEventSource<Record<string, string>>,
argument: string[],
feature: Feature,
layer: LayerConfig
layer: LayerConfig,
): BaseUIElement {
if (feature.geometry.type !== "Point") {
return undefined
@ -473,7 +469,7 @@ export default class SpecialVisualizations {
tagSource: UIEventSource<Record<string, string>>,
argument: string[],
feature: Feature,
layer: LayerConfig
layer: LayerConfig,
): BaseUIElement {
if (!layer.deletion) {
return undefined
@ -501,7 +497,7 @@ export default class SpecialVisualizations {
state: SpecialVisualizationState,
tagSource: UIEventSource<Record<string, string>>,
argument: string[],
feature: Feature
feature: Feature,
): BaseUIElement {
const [lon, lat] = GeoOperations.centerpointCoordinates(feature)
return new SvelteUIElement(CreateNewNote, {
@ -565,7 +561,7 @@ export default class SpecialVisualizations {
.map((tags) => tags[args[0]])
.map((wikidata) => {
wikidata = Utils.NoEmpty(
wikidata?.split(";")?.map((wd) => wd.trim()) ?? []
wikidata?.split(";")?.map((wd) => wd.trim()) ?? [],
)[0]
const entry = Wikidata.LoadWikidataEntry(wikidata)
return new VariableUiElement(
@ -575,9 +571,9 @@ export default class SpecialVisualizations {
}
const response = <WikidataResponse>e["success"]
return Translation.fromMap(response.labels)
})
}),
)
})
}),
),
},
new MapillaryLinkVis(),
@ -591,7 +587,7 @@ export default class SpecialVisualizations {
tags: UIEventSource<Record<string, string>>,
_,
__,
layer: LayerConfig
layer: LayerConfig,
) => new SvelteUIElement(AllTagsPanel, { tags, layer }),
},
{
@ -613,7 +609,7 @@ export default class SpecialVisualizations {
return new ImageCarousel(
AllImageProviders.LoadImagesFor(tags, imagePrefixes),
tags,
state
state,
)
},
},
@ -669,7 +665,7 @@ export default class SpecialVisualizations {
{
nameKey: nameKey,
fallbackName,
}
},
)
return new SvelteUIElement(StarsBarIcon, {
score: reviews.average,
@ -702,7 +698,7 @@ export default class SpecialVisualizations {
{
nameKey: nameKey,
fallbackName,
}
},
)
return new SvelteUIElement(ReviewForm, { reviews, state, tags, feature, layer })
},
@ -734,7 +730,7 @@ export default class SpecialVisualizations {
{
nameKey: nameKey,
fallbackName,
}
},
)
return new SvelteUIElement(AllReviews, { reviews, state, tags, feature, layer })
},
@ -754,7 +750,7 @@ export default class SpecialVisualizations {
tagSource: UIEventSource<Record<string, string>>,
argument: string[],
feature: Feature,
layer: LayerConfig
layer: LayerConfig,
): BaseUIElement {
const [text] = argument
return new SvelteUIElement(ImportReviewIdentity, { state, text })
@ -813,7 +809,7 @@ export default class SpecialVisualizations {
tags: UIEventSource<Record<string, string>>,
args: string[],
feature: Feature,
layer: LayerConfig
layer: LayerConfig,
): SvelteUIElement {
const keyToUse = args[0]
const prefix = args[1]
@ -850,17 +846,17 @@ export default class SpecialVisualizations {
return undefined
}
const allUnits: Unit[] = [].concat(
...(state?.layout?.layers?.map((lyr) => lyr.units) ?? [])
...(state?.layout?.layers?.map((lyr) => lyr.units) ?? []),
)
const unit = allUnits.filter((unit) =>
unit.isApplicableToKey(key)
unit.isApplicableToKey(key),
)[0]
if (unit === undefined) {
return value
}
const getCountry = () => tagSource.data._country
return unit.asHumanLongValue(value, getCountry)
})
}),
)
},
},
@ -877,7 +873,7 @@ export default class SpecialVisualizations {
new Combine([
t.downloadFeatureAsGeojson.SetClass("font-bold text-lg"),
t.downloadGeoJsonHelper.SetClass("subtle"),
]).SetClass("flex flex-col")
]).SetClass("flex flex-col"),
)
.onClick(() => {
console.log("Exporting as Geojson")
@ -890,7 +886,7 @@ export default class SpecialVisualizations {
title + "_mapcomplete_export.geojson",
{
mimetype: "application/vnd.geo+json",
}
},
)
})
.SetClass("w-full")
@ -926,7 +922,7 @@ export default class SpecialVisualizations {
constr: (state) => {
return new SubtleButton(
Svg.delete_icon_svg().SetStyle("height: 1.5rem"),
Translations.t.general.removeLocationHistory
Translations.t.general.removeLocationHistory,
).onClick(() => {
state.historicalUserLocations.features.setData([])
state.selectedElement.setData(undefined)
@ -964,10 +960,10 @@ export default class SpecialVisualizations {
.filter((c) => c.text !== "")
.map(
(c, i) =>
new NoteCommentElement(c, state, i, comments.length)
)
new NoteCommentElement(c, state, i, comments.length),
),
).SetClass("flex flex-col")
})
}),
),
},
{
@ -1001,7 +997,7 @@ export default class SpecialVisualizations {
tagsSource: UIEventSource<Record<string, string>>,
_: string[],
feature: Feature,
layer: LayerConfig
layer: LayerConfig,
) =>
new VariableUiElement(
tagsSource.map((tags) => {
@ -1019,7 +1015,7 @@ export default class SpecialVisualizations {
feature,
layer,
}).SetClass("px-1")
})
}),
),
},
{
@ -1035,8 +1031,8 @@ export default class SpecialVisualizations {
let challenge = Stores.FromPromise(
Utils.downloadJsonCached(
`${Maproulette.defaultEndpoint}/challenge/${parentId}`,
24 * 60 * 60 * 1000
)
24 * 60 * 60 * 1000,
),
)
return new VariableUiElement(
@ -1061,7 +1057,7 @@ export default class SpecialVisualizations {
} else {
return [title, new List(listItems)]
}
})
}),
)
},
docs: "Fetches the metadata of MapRoulette campaign that this task is part of and shows those details (namely `title`, `description` and `instruction`).\n\nThis reads the property `mr_challengeId` to detect the parent campaign.",
@ -1075,15 +1071,15 @@ export default class SpecialVisualizations {
"\n" +
"```json\n" +
"{\n" +
' "id": "mark_duplicate",\n' +
' "render": {\n' +
' "special": {\n' +
' "type": "maproulette_set_status",\n' +
' "message": {\n' +
' "en": "Mark as not found or false positive"\n' +
" \"id\": \"mark_duplicate\",\n" +
" \"render\": {\n" +
" \"special\": {\n" +
" \"type\": \"maproulette_set_status\",\n" +
" \"message\": {\n" +
" \"en\": \"Mark as not found or false positive\"\n" +
" },\n" +
' "status": "2",\n' +
' "image": "close"\n' +
" \"status\": \"2\",\n" +
" \"image\": \"close\"\n" +
" }\n" +
" }\n" +
"}\n" +
@ -1163,8 +1159,8 @@ export default class SpecialVisualizations {
const fsBboxed = new BBoxFeatureSourceForLayer(fs, bbox)
return new StatisticsPanel(fsBboxed)
},
[state.mapProperties.bounds]
)
[state.mapProperties.bounds],
),
)
},
},
@ -1230,7 +1226,7 @@ export default class SpecialVisualizations {
constr(
state: SpecialVisualizationState,
tagSource: UIEventSource<Record<string, string>>,
args: string[]
args: string[],
): BaseUIElement {
let [text, href, classnames, download, ariaLabel] = args
if (download === "") {
@ -1244,14 +1240,14 @@ export default class SpecialVisualizations {
text: Utils.SubstituteKeys(text, tags),
href: Utils.SubstituteKeys(href, tags).replaceAll(
/ /g,
"%20"
"%20",
) /* Chromium based browsers eat the spaces */,
classnames,
download: Utils.SubstituteKeys(download, tags),
ariaLabel: Utils.SubstituteKeys(ariaLabel, tags),
newTab,
})
)
}),
),
)
},
},
@ -1273,7 +1269,7 @@ export default class SpecialVisualizations {
},
},
null,
" "
" ",
) +
"\n```",
args: [
@ -1297,7 +1293,7 @@ export default class SpecialVisualizations {
featureTags: UIEventSource<Record<string, string>>,
args: string[],
feature: Feature,
layer: LayerConfig
layer: LayerConfig,
) {
const [key, tr, classesRaw] = args
let classes = classesRaw ?? ""
@ -1322,7 +1318,7 @@ export default class SpecialVisualizations {
elements.push(subsTr)
}
return elements
})
}),
)
},
},
@ -1342,7 +1338,7 @@ export default class SpecialVisualizations {
tagSource: UIEventSource<Record<string, string>>,
argument: string[],
feature: Feature,
layer: LayerConfig
layer: LayerConfig,
): BaseUIElement {
return new VariableUiElement(
tagSource.map((tags) => {
@ -1354,7 +1350,7 @@ export default class SpecialVisualizations {
console.error("Cannot create a translation for", v, "due to", e)
return JSON.stringify(v)
}
})
}),
)
},
},
@ -1374,7 +1370,7 @@ export default class SpecialVisualizations {
tagSource: UIEventSource<Record<string, string>>,
argument: string[],
feature: Feature,
layer: LayerConfig
layer: LayerConfig,
): BaseUIElement {
const key = argument[0]
const validator = new FediverseValidator()
@ -1384,7 +1380,7 @@ export default class SpecialVisualizations {
.map((fediAccount) => {
fediAccount = validator.reformat(fediAccount)
const [_, username, host] = fediAccount.match(
FediverseValidator.usernameAtServer
FediverseValidator.usernameAtServer,
)
const normalLink = new SvelteUIElement(Link, {
@ -1396,10 +1392,10 @@ export default class SpecialVisualizations {
const loggedInContributorMastodon =
state.userRelatedState?.preferencesAsTags?.data?.[
"_mastodon_link"
]
]
console.log(
"LoggedinContributorMastodon",
loggedInContributorMastodon
loggedInContributorMastodon,
)
if (!loggedInContributorMastodon) {
return normalLink
@ -1415,7 +1411,7 @@ export default class SpecialVisualizations {
newTab: true,
}).SetClass("button"),
])
})
}),
)
},
},
@ -1435,7 +1431,7 @@ export default class SpecialVisualizations {
tagSource: UIEventSource<Record<string, string>>,
args: string[],
feature: Feature,
layer: LayerConfig
layer: LayerConfig,
): BaseUIElement {
return new FixedUiElement("{" + args[0] + "}")
},
@ -1456,7 +1452,7 @@ export default class SpecialVisualizations {
tagSource: UIEventSource<Record<string, string>>,
argument: string[],
feature: Feature,
layer: LayerConfig
layer: LayerConfig,
): BaseUIElement {
const key = argument[0] ?? "value"
return new VariableUiElement(
@ -1474,12 +1470,12 @@ export default class SpecialVisualizations {
} catch (e) {
return new FixedUiElement(
"Could not parse this tag: " +
JSON.stringify(value) +
" due to " +
e
JSON.stringify(value) +
" due to " +
e,
).SetClass("alert")
}
})
}),
)
},
},
@ -1500,7 +1496,7 @@ export default class SpecialVisualizations {
tagSource: UIEventSource<Record<string, string>>,
argument: string[],
feature: Feature,
layer: LayerConfig
layer: LayerConfig,
): BaseUIElement {
const giggityUrl = argument[0]
return new SvelteUIElement(Giggity, { tags: tagSource, state, giggityUrl })
@ -1516,12 +1512,12 @@ export default class SpecialVisualizations {
_: UIEventSource<Record<string, string>>,
argument: string[],
feature: Feature,
layer: LayerConfig
layer: LayerConfig,
): BaseUIElement {
const tags = (<ThemeViewState>(
state
)).geolocation.currentUserLocation.features.map(
(features) => features[0]?.properties
(features) => features[0]?.properties,
)
return new Combine([
new SvelteUIElement(OrientationDebugPanel, {}),
@ -1543,7 +1539,7 @@ export default class SpecialVisualizations {
tagSource: UIEventSource<Record<string, string>>,
argument: string[],
feature: Feature,
layer: LayerConfig
layer: LayerConfig,
): BaseUIElement {
return new SvelteUIElement(MarkAsFavourite, {
tags: tagSource,
@ -1563,7 +1559,7 @@ export default class SpecialVisualizations {
tagSource: UIEventSource<Record<string, string>>,
argument: string[],
feature: Feature,
layer: LayerConfig
layer: LayerConfig,
): BaseUIElement {
return new SvelteUIElement(MarkAsFavouriteMini, {
tags: tagSource,
@ -1583,7 +1579,7 @@ export default class SpecialVisualizations {
tagSource: UIEventSource<Record<string, string>>,
argument: string[],
feature: Feature,
layer: LayerConfig
layer: LayerConfig,
): BaseUIElement {
return new SvelteUIElement(DirectionIndicator, { state, feature })
},
@ -1598,7 +1594,7 @@ export default class SpecialVisualizations {
tagSource: UIEventSource<Record<string, string>>,
argument: string[],
feature: Feature,
layer: LayerConfig
layer: LayerConfig,
): BaseUIElement {
return new VariableUiElement(
tagSource
@ -1620,9 +1616,9 @@ export default class SpecialVisualizations {
`${window.location.protocol}//${window.location.host}${window.location.pathname}?${layout}lat=${lat}&lon=${lon}&z=15` +
`#${id}`
return new Img(new Qr(url).toImageElement(75)).SetStyle(
"width: 75px"
"width: 75px",
)
})
}),
)
},
},
@ -1642,7 +1638,7 @@ export default class SpecialVisualizations {
tagSource: UIEventSource<Record<string, string>>,
args: string[],
feature: Feature,
layer: LayerConfig
layer: LayerConfig,
): BaseUIElement {
const key = args[0] === "" ? "_direction:centerpoint" : args[0]
return new VariableUiElement(
@ -1653,11 +1649,11 @@ export default class SpecialVisualizations {
})
.mapD((value) => {
const dir = GeoOperations.bearingToHuman(
GeoOperations.parseBearing(value)
GeoOperations.parseBearing(value),
)
console.log("Human dir", dir)
return Translations.t.general.visualFeedback.directionsAbsolute[dir]
})
}),
)
},
},
@ -1675,11 +1671,6 @@ export default class SpecialVisualizations {
required: true,
doc: "The domain name(s) where data might be fetched from - this is needed to set the CSP. A domain must include 'https', e.g. 'https://example.com'. For multiple domains, separate them with ';'. If you don't know the possible domains, use '*'. ",
},
{
name: "postprocessing",
required: false,
doc: "Apply some postprocessing. Currently, only 'velopark' is allowed as value",
},
{
name: "readonly",
required: false,
@ -1692,19 +1683,19 @@ export default class SpecialVisualizations {
tagSource: UIEventSource<Record<string, string>>,
args: string[],
feature: Feature,
layer: LayerConfig
layer: LayerConfig,
): BaseUIElement {
const url = args[0]
const postprocessVelopark = args[2] === "velopark"
const readonly = args[3] === "yes"
const externalData = Stores.FromPromiseWithErr(Utils.downloadJson(url))
return new SvelteUIElement(ComparisonTool, {
url,
postprocessVelopark,
state,
tags: tagSource,
layer,
feature,
readonly,
externalData,
})
},
},
@ -1718,12 +1709,12 @@ export default class SpecialVisualizations {
tagSource: UIEventSource<Record<string, string>>,
args: string[],
feature: Feature,
layer: LayerConfig
layer: LayerConfig,
): BaseUIElement {
return new Toggle(
undefined,
new SvelteUIElement(LoginButton),
state.osmConnection.isLoggedIn
state.osmConnection.isLoggedIn,
)
},
},
@ -1740,19 +1731,36 @@ export default class SpecialVisualizations {
needsUrls: [Constants.linkedDataProxy],
constr(
state: SpecialVisualizationState,
tagsSource: UIEventSource<Record<string, string>>,
tags: UIEventSource<Record<string, string>>,
argument: string[],
feature: Feature,
layer: LayerConfig
layer: LayerConfig,
): BaseUIElement {
const key = argument[0] ?? "website"
return new SvelteUIElement(LinkedDataDisplay, {
feature,
state,
tagsSource,
key,
layer,
let url = tags.mapD(tags => {
if (!tags._country || !tags[key] || tags[key] === "undefined") {
return null
}
return ({ url: tags[key], country: tags._country })
})
const externalData: Store<{ success: { content: any } } | {
error: string
} | undefined | null> = url.bindD(({
url,
country,
}) => Stores.FromPromiseWithErr(LinkedDataLoader.fetchJsonLdWithProxy(url, { country })))
return new Toggle(
new SvelteUIElement(ComparisonTool, {
feature,
state,
tags,
layer,
externalData,
}), undefined, url.map(url => !!url),
)
},
},
]
@ -1766,7 +1774,7 @@ export default class SpecialVisualizations {
throw (
"Invalid special visualisation found: funcName is undefined for " +
invalid.map((sp) => sp.i).join(", ") +
'. Did you perhaps type \n funcName: "funcname" // type declaration uses COLON\ninstead of:\n funcName = "funcName" // value definition uses EQUAL'
". Did you perhaps type \n funcName: \"funcname\" // type declaration uses COLON\ninstead of:\n funcName = \"funcName\" // value definition uses EQUAL"
)
}