From 734be4a702f1ef0394bc72e4f16724b2237a44cd Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Wed, 19 Jun 2024 03:22:57 +0200 Subject: [PATCH] More tweaks to the linked data loader --- scripts/downloadLinkedDataList.ts | 4 +- scripts/importscripts/compareWebsiteData.ts | 2 +- src/Logic/Web/LinkedDataLoader.ts | 75 ++++++++++++--------- src/UI/Comparison/ComparisonAction.svelte | 36 +++++----- src/UI/SpecialVisualizations.ts | 14 +++- 5 files changed, 76 insertions(+), 55 deletions(-) diff --git a/scripts/downloadLinkedDataList.ts b/scripts/downloadLinkedDataList.ts index 1f45e3348..caacf509b 100644 --- a/scripts/downloadLinkedDataList.ts +++ b/scripts/downloadLinkedDataList.ts @@ -2,14 +2,14 @@ import Script from "./Script" import LinkedDataLoader from "../src/Logic/Web/LinkedDataLoader" import { writeFileSync } from "fs" -export default class DownloadLinkedDataList extends Script { +class DownloadLinkedDataList extends Script { constructor() { super("Downloads the localBusinesses from the given location. Usage: url [--no-proxy]") } async main([url, noProxy]: string[]): Promise { const useProxy = noProxy !== "--no-proxy" - const data = await LinkedDataLoader.fetchJsonLd(url, {}, useProxy) + const data = await LinkedDataLoader.fetchJsonLd(url, {}, useProxy ? "proxy" : "fetch-lod") const path = "linked_data_" + url.replace(/[^a-zA-Z0-9_]/g, "_") + ".jsonld" writeFileSync(path, JSON.stringify(data), "utf8") console.log("Written", path) diff --git a/scripts/importscripts/compareWebsiteData.ts b/scripts/importscripts/compareWebsiteData.ts index 22ff671e4..be041f506 100644 --- a/scripts/importscripts/compareWebsiteData.ts +++ b/scripts/importscripts/compareWebsiteData.ts @@ -17,7 +17,7 @@ class CompareWebsiteData extends Script { if (fs.existsSync(filename)) { return JSON.parse(fs.readFileSync(filename, "utf-8")) } - const jsonLd = await LinkedDataLoader.fetchJsonLd(url, undefined, true) + const jsonLd = await LinkedDataLoader.fetchJsonLd(url, undefined, "proxy") console.log("Got:", jsonLd) fs.writeFileSync(filename, JSON.stringify(jsonLd)) return jsonLd diff --git a/src/Logic/Web/LinkedDataLoader.ts b/src/Logic/Web/LinkedDataLoader.ts index d07397ad3..80263d4b6 100644 --- a/src/Logic/Web/LinkedDataLoader.ts +++ b/src/Logic/Web/LinkedDataLoader.ts @@ -27,23 +27,23 @@ export default class LinkedDataLoader { opening_hours: { "@id": "http://schema.org/openingHoursSpecification" }, openingHours: { "@id": "http://schema.org/openingHours", "@container": "@set" }, geo: { "@id": "http://schema.org/geo" }, - alt_name: { "@id": "http://schema.org/alternateName" }, + alt_name: { "@id": "http://schema.org/alternateName" } } private static COMPACTING_CONTEXT_OH = { dayOfWeek: { "@id": "http://schema.org/dayOfWeek", "@container": "@set" }, closes: { "@id": "http://schema.org/closes", - "@type": "http://www.w3.org/2001/XMLSchema#time", + "@type": "http://www.w3.org/2001/XMLSchema#time" }, opens: { "@id": "http://schema.org/opens", - "@type": "http://www.w3.org/2001/XMLSchema#time", - }, + "@type": "http://www.w3.org/2001/XMLSchema#time" + } } private static formatters: Record<"phone" | "email" | "website", Validator> = { phone: new PhoneValidator(), email: new EmailValidator(), - website: new UrlValidator(undefined, undefined, true), + website: new UrlValidator(undefined, undefined, true) } private static ignoreKeys = [ "http://schema.org/logo", @@ -56,7 +56,7 @@ export default class LinkedDataLoader { "http://schema.org/description", "http://schema.org/hasMap", "http://schema.org/priceRange", - "http://schema.org/contactPoint", + "http://schema.org/contactPoint" ] private static shapeToPolygon(str: string): Polygon { @@ -69,8 +69,8 @@ export default class LinkedDataLoader { .trim() .split(" ") .map((n) => Number(n)) - ), - ], + ) + ] } } @@ -92,18 +92,18 @@ export default class LinkedDataLoader { const context = { lat: { "@id": "http://schema.org/latitude", - "@type": "http://www.w3.org/2001/XMLSchema#double", + "@type": "http://www.w3.org/2001/XMLSchema#double" }, lon: { "@id": "http://schema.org/longitude", - "@type": "http://www.w3.org/2001/XMLSchema#double", - }, + "@type": "http://www.w3.org/2001/XMLSchema#double" + } } const flattened = await jsonld.compact(geo, context) return { type: "Point", - coordinates: [Number(flattened.lon), Number(flattened.lat)], + coordinates: [Number(flattened.lon), Number(flattened.lat)] } } @@ -236,13 +236,28 @@ export default class LinkedDataLoader { static async fetchJsonLd( url: string, options?: JsonLdLoaderOptions, - useProxy: boolean = false + mode: "fetch-lod" | "fetch-raw" | "proxy" ): Promise { - if (useProxy) { + if (mode === "proxy") { url = Constants.linkedDataProxy.replace("{url}", encodeURIComponent(url)) } - const data = await Utils.downloadJson(url) - return await LinkedDataLoader.compact(data, options) + if (mode !== "fetch-raw") { + const data = await Utils.downloadJson(url) + return await LinkedDataLoader.compact(data, options) + } + + + let htmlContent = await Utils.download(url) + const div = document.createElement("div") + div.innerHTML = htmlContent + const script = Array.from(div.getElementsByTagName("script")) + .find(script => script.type === "application/ld+json") + + + const snippet = JSON.parse(script.textContent) + snippet["@base"] = url + return await LinkedDataLoader.compact(snippet, options) + } /** @@ -293,7 +308,7 @@ export default class LinkedDataLoader { if (properties["latitude"] && properties["longitude"]) { geometry = { type: "Point", - coordinates: [Number(properties["longitude"]), Number(properties["latitude"])], + coordinates: [Number(properties["longitude"]), Number(properties["latitude"])] } delete properties["latitude"] delete properties["longitude"] @@ -305,7 +320,7 @@ export default class LinkedDataLoader { const geo: GeoJSON = { type: "Feature", properties, - geometry, + geometry } delete linkedData.geo delete properties.shape @@ -423,7 +438,7 @@ export default class LinkedDataLoader { "brede publiek", "iedereen", "bezoekers", - "iedereen - vooral bezoekers gemeentehuis of bibliotheek.", + "iedereen - vooral bezoekers gemeentehuis of bibliotheek." ].indexOf(audience.toLowerCase()) >= 0 ) { return "yes" @@ -506,7 +521,7 @@ export default class LinkedDataLoader { mv: "http://schema.mobivoc.org/", gr: "http://purl.org/goodrelations/v1#", vp: "https://data.velopark.be/openvelopark/vocabulary#", - vpt: "https://data.velopark.be/openvelopark/terms#", + vpt: "https://data.velopark.be/openvelopark/terms#" }, [url], undefined, @@ -527,7 +542,7 @@ export default class LinkedDataLoader { mv: "http://schema.mobivoc.org/", gr: "http://purl.org/goodrelations/v1#", vp: "https://data.velopark.be/openvelopark/vocabulary#", - vpt: "https://data.velopark.be/openvelopark/terms#", + vpt: "https://data.velopark.be/openvelopark/terms#" }, [url], "g", @@ -670,20 +685,20 @@ export default class LinkedDataLoader { const withProxyUrl = Constants.linkedDataProxy.replace("{url}", encodeURIComponent(url)) const optionalPaths: Record> = { "schema:interactionService": { - "schema:url": "website", + "schema:url": "website" }, "mv:operatedBy": { - "gr:legalName": "operator", + "gr:legalName": "operator" }, "schema:contactPoint": { "schema:email": "email", - "schema:telephone": "phone", + "schema:telephone": "phone" }, - "schema:dateModified": "_last_edit_timestamp", + "schema:dateModified": "_last_edit_timestamp" } if (includeExtras) { optionalPaths["schema:address"] = { - "schema:streetAddress": "addr", + "schema:streetAddress": "addr" } optionalPaths["schema:name"] = "name" optionalPaths["schema:description"] = "description" @@ -701,19 +716,19 @@ export default class LinkedDataLoader { "schema:geo": { "schema:latitude": "latitude", "schema:longitude": "longitude", - "schema:polygon": "shape", + "schema:polygon": "shape" }, "schema:priceSpecification": { "mv:freeOfCharge": "fee", - "schema:price": "charge", - }, + "schema:price": "charge" + } } const extra = [ "schema:priceSpecification [ mv:dueForTime [ mv:timeStartValue ?chargeStart; mv:timeEndValue ?chargeEnd; mv:timeUnit ?timeUnit ] ]", "vp:allows [vp:bicycleType ; vp:bicyclesAmount ?capacityCargobike; vp:bicycleType ?cargoBikeType]", "vp:allows [vp:bicycleType ; vp:bicyclesAmount ?capacityElectric; vp:bicycleType ?electricBikeType]", - "vp:allows [vp:bicycleType ; vp:bicyclesAmount ?capacityTandem; vp:bicycleType ?tandemBikeType]", + "vp:allows [vp:bicycleType ; vp:bicyclesAmount ?capacityTandem; vp:bicycleType ?tandemBikeType]" ] const unpatched = await this.fetchEntry( diff --git a/src/UI/Comparison/ComparisonAction.svelte b/src/UI/Comparison/ComparisonAction.svelte index d5f5208a8..dc3a124d5 100644 --- a/src/UI/Comparison/ComparisonAction.svelte +++ b/src/UI/Comparison/ComparisonAction.svelte @@ -66,24 +66,22 @@
-
-
-
- {#if renderingExternal} - - {:else} -
- {key} - {externalProperties[key]} -
- {/if} -
+
+
+ {#if renderingExternal} + + {:else} +
+ {key} + {externalProperties[key]} +
+ {/if} {#if !readonly && ($isTesting || $isDebug || $showTags === "yes" || $showTags === "always" || $showTags === "full")}
@@ -103,7 +101,7 @@ {#if !readonly} {#if currentStep === "init"}