Further stabilization of wikipedia box
This commit is contained in:
parent
393d5d8932
commit
a89d303ecd
15 changed files with 169 additions and 109 deletions
|
@ -150,7 +150,6 @@ export class InitUiElements {
|
||||||
if (userDetails === undefined) {
|
if (userDetails === undefined) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
console.log("Adding home location of ", userDetails)
|
|
||||||
const home = userDetails.home;
|
const home = userDetails.home;
|
||||||
if (home === undefined) {
|
if (home === undefined) {
|
||||||
return userDetails.loggedIn; // If logged in, the home is not set and we unregister. If not logged in, we stay registered if a login still comes
|
return userDetails.loggedIn; // If logged in, the home is not set and we unregister. If not logged in, we stay registered if a login still comes
|
||||||
|
|
|
@ -23,7 +23,10 @@ export class WikidataImageProvider extends ImageProvider {
|
||||||
}
|
}
|
||||||
|
|
||||||
public async ExtractUrls(key: string, value: string): Promise<Promise<ProvidedImage>[]> {
|
public async ExtractUrls(key: string, value: string): Promise<Promise<ProvidedImage>[]> {
|
||||||
const entity = await Wikidata.LoadWikidataEntry(value)
|
const entity = await Wikidata.LoadWikidataEntryAsync(value)
|
||||||
|
if(entity === undefined){
|
||||||
|
return []
|
||||||
|
}
|
||||||
|
|
||||||
const allImages : Promise<ProvidedImage>[] = []
|
const allImages : Promise<ProvidedImage>[] = []
|
||||||
// P18 is the claim 'depicted in this image'
|
// P18 is the claim 'depicted in this image'
|
||||||
|
@ -32,7 +35,7 @@ export class WikidataImageProvider extends ImageProvider {
|
||||||
allImages.push(...promises)
|
allImages.push(...promises)
|
||||||
}
|
}
|
||||||
|
|
||||||
const commons =entity.wikisites.get("commons")
|
const commons = entity.commons
|
||||||
if (commons !== undefined) {
|
if (commons !== undefined) {
|
||||||
const promises = await WikimediaImageProvider.singleton.ExtractUrls(undefined , commons)
|
const promises = await WikimediaImageProvider.singleton.ExtractUrls(undefined , commons)
|
||||||
allImages.push(...promises)
|
allImages.push(...promises)
|
||||||
|
|
|
@ -18,7 +18,8 @@ export class Overpass {
|
||||||
private _relationTracker: RelationsTracker;
|
private _relationTracker: RelationsTracker;
|
||||||
|
|
||||||
|
|
||||||
constructor(filter: TagsFilter, extraScripts: string[],
|
constructor(filter: TagsFilter,
|
||||||
|
extraScripts: string[],
|
||||||
interpreterUrl: string,
|
interpreterUrl: string,
|
||||||
timeout: UIEventSource<number>,
|
timeout: UIEventSource<number>,
|
||||||
relationTracker: RelationsTracker,
|
relationTracker: RelationsTracker,
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
import {Utils} from "../Utils";
|
import {Utils} from "../Utils";
|
||||||
|
import * as Events from "events";
|
||||||
|
|
||||||
export class UIEventSource<T> {
|
export class UIEventSource<T> {
|
||||||
|
|
||||||
|
@ -32,14 +33,14 @@ export class UIEventSource<T> {
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
public static flatten<X>(source: UIEventSource<UIEventSource<X>>, possibleSources: UIEventSource<any>[]): UIEventSource<X> {
|
public static flatten<X>(source: UIEventSource<UIEventSource<X>>, possibleSources?: UIEventSource<any>[]): UIEventSource<X> {
|
||||||
const sink = new UIEventSource<X>(source.data?.data);
|
const sink = new UIEventSource<X>(source.data?.data);
|
||||||
|
|
||||||
source.addCallback((latestData) => {
|
source.addCallback((latestData) => {
|
||||||
sink.setData(latestData?.data);
|
sink.setData(latestData?.data);
|
||||||
});
|
});
|
||||||
|
|
||||||
for (const possibleSource of possibleSources) {
|
for (const possibleSource of possibleSources ?? []) {
|
||||||
possibleSource?.addCallback(() => {
|
possibleSource?.addCallback(() => {
|
||||||
sink.setData(source.data?.data);
|
sink.setData(source.data?.data);
|
||||||
})
|
})
|
||||||
|
@ -186,6 +187,25 @@ export class UIEventSource<T> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Monadic bind function
|
||||||
|
*/
|
||||||
|
public bind<X>(f: ((t: T) => UIEventSource<X>)): UIEventSource<X>{
|
||||||
|
const sink = new UIEventSource<X>( undefined )
|
||||||
|
const seenEventSources = new Set<UIEventSource<X>>();
|
||||||
|
this.addCallbackAndRun(data => {
|
||||||
|
const eventSource = f(data)
|
||||||
|
if(eventSource === undefined){
|
||||||
|
sink.setData(undefined)
|
||||||
|
}else if(!seenEventSources.has(eventSource)){
|
||||||
|
eventSource.addCallbackAndRun(mappedData => sink.setData(mappedData))
|
||||||
|
seenEventSources.add(eventSource)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
return sink;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Monoidal map:
|
* Monoidal map:
|
||||||
* Given a function 'f', will construct a new UIEventSource where the contents will always be "f(this.data)'
|
* Given a function 'f', will construct a new UIEventSource where the contents will always be "f(this.data)'
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
import {Utils} from "../../Utils";
|
import {Utils} from "../../Utils";
|
||||||
|
import {UIEventSource} from "../UIEventSource";
|
||||||
|
|
||||||
|
|
||||||
export interface WikidataResponse {
|
export interface WikidataResponse {
|
||||||
|
@ -62,15 +63,23 @@ export default class Wikidata {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
private static readonly _cache = new Map<number, UIEventSource<{success: WikidataResponse} | {error: any}>>()
|
||||||
* Loads a wikidata page
|
public static LoadWikidataEntry(value: string | number): UIEventSource<{success: WikidataResponse} | {error: any}> {
|
||||||
* @returns the entity of the given value
|
const key = this.ExtractKey(value)
|
||||||
*/
|
const cached = Wikidata._cache.get(key)
|
||||||
public static async LoadWikidataEntry(value: string | number): Promise<WikidataResponse> {
|
if(cached !== undefined){
|
||||||
const wikidataUrl = "https://www.wikidata.org/wiki/"
|
return cached
|
||||||
if (typeof value === "number") {
|
|
||||||
value = "Q" + value
|
|
||||||
}
|
}
|
||||||
|
const src = UIEventSource.FromPromiseWithErr(Wikidata.LoadWikidataEntryAsync(key))
|
||||||
|
Wikidata._cache.set(key, src)
|
||||||
|
return src;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static ExtractKey(value: string | number) : number{
|
||||||
|
if (typeof value === "number") {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
const wikidataUrl = "https://www.wikidata.org/wiki/"
|
||||||
if (value.startsWith(wikidataUrl)) {
|
if (value.startsWith(wikidataUrl)) {
|
||||||
value = value.substring(wikidataUrl.length)
|
value = value.substring(wikidataUrl.length)
|
||||||
}
|
}
|
||||||
|
@ -78,12 +87,30 @@ export default class Wikidata {
|
||||||
// Probably some random link in the image field - we skip it
|
// Probably some random link in the image field - we skip it
|
||||||
return undefined
|
return undefined
|
||||||
}
|
}
|
||||||
if (!value.startsWith("Q")) {
|
if (value.startsWith("Q")) {
|
||||||
value = "Q" + value
|
value = value.substring(1)
|
||||||
}
|
}
|
||||||
const url = "https://www.wikidata.org/wiki/Special:EntityData/" + value + ".json";
|
const n = Number(value)
|
||||||
|
if(isNaN(n)){
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
return n;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Loads a wikidata page
|
||||||
|
* @returns the entity of the given value
|
||||||
|
*/
|
||||||
|
public static async LoadWikidataEntryAsync(value: string | number): Promise<WikidataResponse> {
|
||||||
|
const id = Wikidata.ExtractKey(value)
|
||||||
|
if(id === undefined){
|
||||||
|
console.warn("Could not extract a wikidata entry from", value)
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
console.log("Requesting wikidata with id", id)
|
||||||
|
const url = "https://www.wikidata.org/wiki/Special:EntityData/Q" + id + ".json";
|
||||||
const response = await Utils.downloadJson(url)
|
const response = await Utils.downloadJson(url)
|
||||||
return Wikidata.ParseResponse(response.entities[value]);
|
return Wikidata.ParseResponse(response.entities["Q" + id])
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
|
@ -61,7 +61,6 @@ export default class Wikipedia {
|
||||||
|
|
||||||
const links = Array.from(content.getElementsByTagName("a"))
|
const links = Array.from(content.getElementsByTagName("a"))
|
||||||
|
|
||||||
console.log("Links are", links)
|
|
||||||
// Rewrite relative links to absolute links + open them in a new tab
|
// Rewrite relative links to absolute links + open them in a new tab
|
||||||
links.filter(link => link.getAttribute("href")?.startsWith("/") ?? false).
|
links.filter(link => link.getAttribute("href")?.startsWith("/") ?? false).
|
||||||
forEach(link => {
|
forEach(link => {
|
||||||
|
|
|
@ -39,13 +39,13 @@ export default class AllDownloads extends ScrollableFullScreen {
|
||||||
|
|
||||||
const loading = Svg.loading_svg().SetClass("animate-rotate");
|
const loading = Svg.loading_svg().SetClass("animate-rotate");
|
||||||
|
|
||||||
|
const dloadTrans = Translations.t.general.download
|
||||||
const icon = new Toggle(loading, Svg.floppy_ui(), isExporting);
|
const icon = new Toggle(loading, Svg.floppy_ui(), isExporting);
|
||||||
const text = new Toggle(
|
const text = new Toggle(
|
||||||
new FixedUiElement("Exporting..."),
|
dloadTrans.exporting.Clone(),
|
||||||
|
|
||||||
new Combine([
|
new Combine([
|
||||||
Translations.t.general.download.downloadAsPdf.Clone().SetClass("font-bold"),
|
dloadTrans.downloadAsPdf.Clone().SetClass("font-bold"),
|
||||||
Translations.t.general.download.downloadAsPdfHelper.Clone()]
|
dloadTrans.downloadAsPdfHelper.Clone()]
|
||||||
).SetClass("flex flex-col")
|
).SetClass("flex flex-col")
|
||||||
.onClick(() => {
|
.onClick(() => {
|
||||||
generatePdf()
|
generatePdf()
|
||||||
|
|
|
@ -121,7 +121,6 @@ export default class FilterView extends VariableUiElement {
|
||||||
listFilterElements.map((input) => input[1].data)
|
listFilterElements.map((input) => input[1].data)
|
||||||
);
|
);
|
||||||
|
|
||||||
console.log(listTagsFilters, oldValue)
|
|
||||||
flayer.appliedFilters.setData(listTagsFilters);
|
flayer.appliedFilters.setData(listTagsFilters);
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
|
|
|
@ -32,7 +32,7 @@ export default class ExportPDF {
|
||||||
private readonly mapH = 210;
|
private readonly mapH = 210;
|
||||||
private readonly scaling = 2
|
private readonly scaling = 2
|
||||||
private readonly freeDivId: string;
|
private readonly freeDivId: string;
|
||||||
private readonly _layout: UIEventSource<LayoutConfig>;
|
private readonly _layout: LayoutConfig;
|
||||||
private _screenhotTaken = false;
|
private _screenhotTaken = false;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
|
@ -41,7 +41,7 @@ export default class ExportPDF {
|
||||||
location: UIEventSource<Loc>,
|
location: UIEventSource<Loc>,
|
||||||
background?: UIEventSource<BaseLayer>
|
background?: UIEventSource<BaseLayer>
|
||||||
features: FeaturePipeline,
|
features: FeaturePipeline,
|
||||||
layout: UIEventSource<LayoutConfig>
|
layout: LayoutConfig
|
||||||
}
|
}
|
||||||
) {
|
) {
|
||||||
|
|
||||||
|
@ -87,7 +87,6 @@ export default class ExportPDF {
|
||||||
minimap.leafletMap .addCallbackAndRunD(leaflet => {
|
minimap.leafletMap .addCallbackAndRunD(leaflet => {
|
||||||
const bounds = BBox.fromLeafletBounds(leaflet.getBounds().pad(0.2))
|
const bounds = BBox.fromLeafletBounds(leaflet.getBounds().pad(0.2))
|
||||||
options.features.GetTilesPerLayerWithin(bounds, tile => {
|
options.features.GetTilesPerLayerWithin(bounds, tile => {
|
||||||
console.log("REndering", tile.name)
|
|
||||||
new ShowDataLayer(
|
new ShowDataLayer(
|
||||||
{
|
{
|
||||||
features: tile,
|
features: tile,
|
||||||
|
@ -108,13 +107,13 @@ export default class ExportPDF {
|
||||||
}
|
}
|
||||||
|
|
||||||
private async CreatePdf(leaflet: L.Map) {
|
private async CreatePdf(leaflet: L.Map) {
|
||||||
|
console.log("PDF creation started")
|
||||||
const t = Translations.t.general.pdf;
|
const t = Translations.t.general.pdf;
|
||||||
const layout = this._layout.data
|
const layout = this._layout
|
||||||
const screenshotter = new SimpleMapScreenshoter();
|
const screenshotter = new SimpleMapScreenshoter();
|
||||||
//minimap op index.html -> hidden daar alles op doen en dan weg
|
//minimap op index.html -> hidden daar alles op doen en dan weg
|
||||||
//minimap - leaflet map ophalen - boundaries ophalen - State.state.featurePipeline
|
//minimap - leaflet map ophalen - boundaries ophalen - State.state.featurePipeline
|
||||||
screenshotter.addTo(leaflet);
|
screenshotter.addTo(leaflet);
|
||||||
console.log("Taking screenshot")
|
|
||||||
|
|
||||||
|
|
||||||
let doc = new jsPDF('landscape');
|
let doc = new jsPDF('landscape');
|
||||||
|
@ -164,7 +163,6 @@ export default class ExportPDF {
|
||||||
const imgSource = layout.icon
|
const imgSource = layout.icon
|
||||||
const imgType = imgSource.substr(imgSource.lastIndexOf(".") + 1);
|
const imgType = imgSource.substr(imgSource.lastIndexOf(".") + 1);
|
||||||
img.src = imgSource
|
img.src = imgSource
|
||||||
console.log(imgType)
|
|
||||||
if (imgType.toLowerCase() === "svg") {
|
if (imgType.toLowerCase() === "svg") {
|
||||||
new FixedUiElement("").AttachTo(this.freeDivId)
|
new FixedUiElement("").AttachTo(this.freeDivId)
|
||||||
|
|
||||||
|
|
|
@ -8,78 +8,69 @@ import BaseUIElement from "./BaseUIElement";
|
||||||
import Title from "./Base/Title";
|
import Title from "./Base/Title";
|
||||||
import Translations from "./i18n/Translations";
|
import Translations from "./i18n/Translations";
|
||||||
import Svg from "../Svg";
|
import Svg from "../Svg";
|
||||||
import Wikidata from "../Logic/Web/Wikidata";
|
import Wikidata, {WikidataResponse} from "../Logic/Web/Wikidata";
|
||||||
import Locale from "./i18n/Locale";
|
import Locale from "./i18n/Locale";
|
||||||
|
import Toggle from "./Input/Toggle";
|
||||||
|
|
||||||
export default class WikipediaBox extends Combine {
|
export default class WikipediaBox extends Toggle {
|
||||||
|
|
||||||
private static async ExtractWikiPages(wikidata): Promise<Map<string, string>> {
|
|
||||||
return (await Wikidata.LoadWikidataEntry(wikidata)).wikisites
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private static _cache = new Map()
|
|
||||||
|
|
||||||
constructor(wikidataId: string | UIEventSource<string>) {
|
constructor(wikidataId: string | UIEventSource<string>) {
|
||||||
const wp = Translations.t.general.wikipedia;
|
const wp = Translations.t.general.wikipedia;
|
||||||
if(typeof wikidataId === "string"){
|
if (typeof wikidataId === "string") {
|
||||||
wikidataId = new UIEventSource(wikidataId)
|
wikidataId = new UIEventSource(wikidataId)
|
||||||
}
|
}
|
||||||
|
|
||||||
const knownPages = new UIEventSource<{success:Map<string, string>}|{error:any}>(undefined)
|
|
||||||
|
|
||||||
wikidataId.addCallbackAndRunD(wikidataId => {
|
const wikibox = wikidataId
|
||||||
WikipediaBox.ExtractWikiPages(wikidataId).then(pages => {
|
.bind(id => {
|
||||||
knownPages.setData({success:pages})
|
console.log("Wikidata is", id)
|
||||||
}).catch(err=> {
|
if(id === undefined){
|
||||||
knownPages.setData({error: err})
|
return undefined
|
||||||
|
}
|
||||||
|
console.log("Initing load WIkidataentry with id", id)
|
||||||
|
return Wikidata.LoadWikidataEntry(id);
|
||||||
})
|
})
|
||||||
})
|
.map(maybewikidata => {
|
||||||
|
if (maybewikidata === undefined) {
|
||||||
const cachedPages = new Map<string, BaseUIElement>()
|
|
||||||
|
|
||||||
const contents = new VariableUiElement(
|
|
||||||
knownPages.map(pages => {
|
|
||||||
|
|
||||||
if (pages === undefined) {
|
|
||||||
return new Loading(wp.loading.Clone())
|
return new Loading(wp.loading.Clone())
|
||||||
}
|
}
|
||||||
if (pages["error"] !== undefined) {
|
if (maybewikidata["error"] !== undefined) {
|
||||||
return wp.failed.Clone().SetClass("alert p-4")
|
return wp.failed.Clone().SetClass("alert p-4")
|
||||||
}
|
}
|
||||||
const dict: Map<string, string> = pages["success"]
|
const wikidata = <WikidataResponse>maybewikidata["success"]
|
||||||
|
console.log("Got wikidata response", wikidata)
|
||||||
|
if (wikidata.wikisites.size === 0) {
|
||||||
|
return wp.noWikipediaPage.Clone()
|
||||||
|
}
|
||||||
|
|
||||||
const preferredLanguage = [Locale.language.data, "en", Array.from(dict.keys())[0]]
|
const preferredLanguage = [Locale.language.data, "en", Array.from(wikidata.wikisites.keys())[0]]
|
||||||
let language
|
let language
|
||||||
let pagetitle;
|
let pagetitle;
|
||||||
let i = 0
|
let i = 0
|
||||||
do {
|
do {
|
||||||
language = preferredLanguage[i]
|
language = preferredLanguage[i]
|
||||||
pagetitle = dict.get(language)
|
pagetitle = wikidata.wikisites.get(language)
|
||||||
i++;
|
i++;
|
||||||
if(i >= preferredLanguage.length){
|
|
||||||
return wp.noWikipediaPage.Clone()
|
|
||||||
}
|
|
||||||
} while (pagetitle === undefined)
|
} while (pagetitle === undefined)
|
||||||
|
return WikipediaBox.createContents(pagetitle, language)
|
||||||
if(cachedPages.has(language)){
|
|
||||||
return cachedPages.get(language)
|
|
||||||
}
|
|
||||||
|
|
||||||
const page = WikipediaBox.createContents(pagetitle, language);
|
|
||||||
cachedPages.set(language, page)
|
|
||||||
return page
|
|
||||||
}, [Locale.language])
|
}, [Locale.language])
|
||||||
|
|
||||||
|
|
||||||
|
const contents = new VariableUiElement(
|
||||||
|
wikibox
|
||||||
).SetClass("overflow-auto normal-background rounded-lg")
|
).SetClass("overflow-auto normal-background rounded-lg")
|
||||||
|
|
||||||
|
|
||||||
super([
|
const mainContent = new Combine([
|
||||||
new Combine([Svg.wikipedia_ui().SetStyle("width: 1.5rem").SetClass("mr-3"),
|
new Combine([Svg.wikipedia_ui().SetStyle("width: 1.5rem").SetClass("mr-3"),
|
||||||
new Title(Translations.t.general.wikipedia.wikipediaboxTitle.Clone(), 2)]).SetClass("flex"),
|
new Title(Translations.t.general.wikipedia.wikipediaboxTitle.Clone(), 2)]).SetClass("flex"),
|
||||||
contents])
|
contents]).SetClass("block rounded-xl subtle-background m-1 p-2 flex flex-col")
|
||||||
|
super(
|
||||||
this
|
mainContent,
|
||||||
.SetClass("block rounded-xl subtle-background m-1 p-2 flex flex-col")
|
undefined,
|
||||||
|
wikidataId.map(id => id !== undefined)
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -103,6 +94,7 @@ export default class WikipediaBox extends Combine {
|
||||||
return new FixedUiElement(htmlContent["success"]).SetClass("wikipedia-article")
|
return new FixedUiElement(htmlContent["success"]).SetClass("wikipedia-article")
|
||||||
}
|
}
|
||||||
if (htmlContent["error"]) {
|
if (htmlContent["error"]) {
|
||||||
|
console.warn("Loading wikipage failed due to", htmlContent["error"])
|
||||||
return wp.failed.Clone().SetClass("alert p-4")
|
return wp.failed.Clone().SetClass("alert p-4")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -174,6 +174,7 @@
|
||||||
"downloadAsPdf": "Download a PDF of the current map",
|
"downloadAsPdf": "Download a PDF of the current map",
|
||||||
"downloadAsPdfHelper": "Ideal to print the current map",
|
"downloadAsPdfHelper": "Ideal to print the current map",
|
||||||
"downloadGeojson": "Download visible data as geojson",
|
"downloadGeojson": "Download visible data as geojson",
|
||||||
|
"exporting": "Exporting...",
|
||||||
"downloadGeoJsonHelper": "Compatible with QGIS, ArcGIS, ESRI, ...",
|
"downloadGeoJsonHelper": "Compatible with QGIS, ArcGIS, ESRI, ...",
|
||||||
"downloadCSV": "Download visible data as CSV",
|
"downloadCSV": "Download visible data as CSV",
|
||||||
"downloadCSVHelper": "Compatible with LibreOffice Calc, Excel, …",
|
"downloadCSVHelper": "Compatible with LibreOffice Calc, Excel, …",
|
||||||
|
|
|
@ -11,7 +11,7 @@
|
||||||
"start": "npm run start:prepare && npm-run-all --parallel start:parallel:*",
|
"start": "npm run start:prepare && npm-run-all --parallel start:parallel:*",
|
||||||
"strt": "npm run start:prepare && npm run start:parallel:parcel",
|
"strt": "npm run start:prepare && npm run start:parallel:parcel",
|
||||||
"start:prepare": "ts-node scripts/generateLayerOverview.ts --no-fail && npm run increase-memory",
|
"start:prepare": "ts-node scripts/generateLayerOverview.ts --no-fail && npm run increase-memory",
|
||||||
"start:parallel:parcel": "parcel *.html UI/** Logic/** assets/*.json assets/svg/* assets/generated/* assets/layers/*/*.svg assets/tagRenderings/*.json assets/themes/*/*.svg assets/themes/*/*.png vendor/* vendor/*/*",
|
"start:parallel:parcel": "parcel *.html UI/** Logic/** assets/*.json assets/svg/* assets/generated/* assets/layers/*/*.svg assets/layers/*/*.jpg assets/layers/*/*.png assets/tagRenderings/*.json assets/themes/*/*.svg assets/themes/*/*.jpg assets/themes/*/*.png vendor/* vendor/*/*",
|
||||||
"start:parallel:tailwindcli": "tailwindcss -i index.css -o css/index-tailwind-output.css --watch",
|
"start:parallel:tailwindcli": "tailwindcss -i index.css -o css/index-tailwind-output.css --watch",
|
||||||
"test": "ts-node test/TestAll.ts",
|
"test": "ts-node test/TestAll.ts",
|
||||||
"init": "npm ci && npm run generate && npm run generate:editor-layer-index && npm run generate:layouts && npm run clean",
|
"init": "npm ci && npm run generate && npm run generate:editor-layer-index && npm run generate:layouts && npm run clean",
|
||||||
|
|
|
@ -2,9 +2,6 @@
|
||||||
* Generates a collection of geojson files based on an overpass query for a given theme
|
* Generates a collection of geojson files based on an overpass query for a given theme
|
||||||
*/
|
*/
|
||||||
import {Utils} from "../Utils";
|
import {Utils} from "../Utils";
|
||||||
|
|
||||||
Utils.runningFromConsole = true
|
|
||||||
|
|
||||||
import {Overpass} from "../Logic/Osm/Overpass";
|
import {Overpass} from "../Logic/Osm/Overpass";
|
||||||
import {existsSync, readFileSync, writeFileSync} from "fs";
|
import {existsSync, readFileSync, writeFileSync} from "fs";
|
||||||
import {TagsFilter} from "../Logic/Tags/TagsFilter";
|
import {TagsFilter} from "../Logic/Tags/TagsFilter";
|
||||||
|
@ -22,12 +19,13 @@ import FilteredLayer from "../Models/FilteredLayer";
|
||||||
import FeatureSource, {FeatureSourceForLayer} from "../Logic/FeatureSource/FeatureSource";
|
import FeatureSource, {FeatureSourceForLayer} from "../Logic/FeatureSource/FeatureSource";
|
||||||
import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSource";
|
import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSource";
|
||||||
import TiledFeatureSource from "../Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource";
|
import TiledFeatureSource from "../Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource";
|
||||||
|
import Constants from "../Models/Constants";
|
||||||
|
|
||||||
|
|
||||||
ScriptUtils.fixUtils()
|
ScriptUtils.fixUtils()
|
||||||
|
|
||||||
|
|
||||||
function createOverpassObject(theme: LayoutConfig, relationTracker: RelationsTracker) {
|
function createOverpassObject(theme: LayoutConfig, relationTracker: RelationsTracker, backend: string) {
|
||||||
let filters: TagsFilter[] = [];
|
let filters: TagsFilter[] = [];
|
||||||
let extraScripts: string[] = [];
|
let extraScripts: string[] = [];
|
||||||
for (const layer of theme.layers) {
|
for (const layer of theme.layers) {
|
||||||
|
@ -58,7 +56,7 @@ function createOverpassObject(theme: LayoutConfig, relationTracker: RelationsTra
|
||||||
if (filters.length + extraScripts.length === 0) {
|
if (filters.length + extraScripts.length === 0) {
|
||||||
throw "Nothing to download! The theme doesn't declare anything to download"
|
throw "Nothing to download! The theme doesn't declare anything to download"
|
||||||
}
|
}
|
||||||
return new Overpass(new Or(filters), extraScripts, new UIEventSource<string>("https://overpass.kumi.systems/api/interpreter"), //https://overpass-api.de/api/interpreter"),
|
return new Overpass(new Or(filters), extraScripts, backend,
|
||||||
new UIEventSource<number>(60), relationTracker);
|
new UIEventSource<number>(60), relationTracker);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -71,7 +69,7 @@ function geoJsonName(targetDir: string, x: number, y: number, z: number): string
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Downloads the given feature and saves them to disk
|
/// Downloads the given feature and saves them to disk
|
||||||
async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass)/* : {failed: number, skipped :number} */ {
|
async function downloadRaw(targetdir: string, r: TileRange, theme: LayoutConfig, relationTracker: RelationsTracker)/* : {failed: number, skipped :number} */ {
|
||||||
let downloaded = 0
|
let downloaded = 0
|
||||||
let failed = 0
|
let failed = 0
|
||||||
let skipped = 0
|
let skipped = 0
|
||||||
|
@ -93,35 +91,28 @@ async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass)/
|
||||||
east: Math.max(boundsArr[0][1], boundsArr[1][1]),
|
east: Math.max(boundsArr[0][1], boundsArr[1][1]),
|
||||||
west: Math.min(boundsArr[0][1], boundsArr[1][1])
|
west: Math.min(boundsArr[0][1], boundsArr[1][1])
|
||||||
}
|
}
|
||||||
|
const overpass = createOverpassObject(theme, relationTracker, Constants.defaultOverpassUrls[(downloaded + failed) % Constants.defaultOverpassUrls.length])
|
||||||
const url = overpass.buildQuery("[bbox:" + bounds.south + "," + bounds.west + "," + bounds.north + "," + bounds.east + "]")
|
const url = overpass.buildQuery("[bbox:" + bounds.south + "," + bounds.west + "," + bounds.north + "," + bounds.east + "]")
|
||||||
|
|
||||||
await ScriptUtils.DownloadJSON(url)
|
try {
|
||||||
.then(json => {
|
|
||||||
if (json.elements.length === 0) {
|
|
||||||
console.log("Got an empty response!")
|
|
||||||
if ((<string>json.remark ?? "").startsWith("runtime error")) {
|
|
||||||
console.error("Got a runtime error: ", json.remark)
|
|
||||||
failed++;
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
const json = await ScriptUtils.DownloadJSON(url)
|
||||||
|
if (json.elements.length === 0) {
|
||||||
|
console.log("Got an empty response!")
|
||||||
console.log("Got the response - writing to ", filename)
|
if ((<string>json.remark ?? "").startsWith("runtime error")) {
|
||||||
writeFileSync(filename, JSON.stringify(json, null, " "));
|
console.error("Got a runtime error: ", json.remark)
|
||||||
|
failed++;
|
||||||
}
|
}
|
||||||
)
|
|
||||||
.catch(err => {
|
|
||||||
console.log(url)
|
|
||||||
console.log("Could not download - probably hit the rate limit; waiting a bit. (" + err + ")")
|
|
||||||
failed++;
|
|
||||||
return ScriptUtils.sleep(60000).then(() => console.log("Waiting is done"))
|
|
||||||
})
|
|
||||||
|
|
||||||
if (x < r.xend || y < r.yend) {
|
} else {
|
||||||
console.debug("Cooling down 10s")
|
console.log("Got the response - writing to ", filename)
|
||||||
await ScriptUtils.sleep(10000)
|
writeFileSync(filename, JSON.stringify(json, null, " "));
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.log(url)
|
||||||
|
console.log("Could not download - probably hit the rate limit; waiting a bit. (" + err + ")")
|
||||||
|
failed++;
|
||||||
|
await ScriptUtils.sleep(1000)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -291,11 +282,10 @@ async function main(args: string[]) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
const relationTracker = new RelationsTracker()
|
const relationTracker = new RelationsTracker()
|
||||||
const overpass = createOverpassObject(theme, relationTracker)
|
|
||||||
|
|
||||||
let failed = 0;
|
let failed = 0;
|
||||||
do {
|
do {
|
||||||
const cachingResult = await downloadRaw(targetdir, tileRange, overpass)
|
const cachingResult = await downloadRaw(targetdir, tileRange, theme, relationTracker)
|
||||||
failed = cachingResult.failed
|
failed = cachingResult.failed
|
||||||
if (failed > 0) {
|
if (failed > 0) {
|
||||||
await ScriptUtils.sleep(30000)
|
await ScriptUtils.sleep(30000)
|
||||||
|
|
|
@ -10,6 +10,7 @@ import RelationSplitHandlerSpec from "./RelationSplitHandler.spec";
|
||||||
import SplitActionSpec from "./SplitAction.spec";
|
import SplitActionSpec from "./SplitAction.spec";
|
||||||
import {Utils} from "../Utils";
|
import {Utils} from "../Utils";
|
||||||
import TileFreshnessCalculatorSpec from "./TileFreshnessCalculator.spec";
|
import TileFreshnessCalculatorSpec from "./TileFreshnessCalculator.spec";
|
||||||
|
import WikidataSpecTest from "./Wikidata.spec.test";
|
||||||
|
|
||||||
|
|
||||||
ScriptUtils.fixUtils()
|
ScriptUtils.fixUtils()
|
||||||
|
@ -23,7 +24,8 @@ const allTests = [
|
||||||
new UnitsSpec(),
|
new UnitsSpec(),
|
||||||
new RelationSplitHandlerSpec(),
|
new RelationSplitHandlerSpec(),
|
||||||
new SplitActionSpec(),
|
new SplitActionSpec(),
|
||||||
new TileFreshnessCalculatorSpec()
|
new TileFreshnessCalculatorSpec(),
|
||||||
|
new WikidataSpecTest()
|
||||||
]
|
]
|
||||||
|
|
||||||
Utils.externalDownloadFunction = async (url) => {
|
Utils.externalDownloadFunction = async (url) => {
|
||||||
|
|
29
test/Wikidata.spec.test.ts
Normal file
29
test/Wikidata.spec.test.ts
Normal file
|
@ -0,0 +1,29 @@
|
||||||
|
import Wikidata from "../Logic/Web/Wikidata";
|
||||||
|
import * as assert from "assert";
|
||||||
|
import {equal} from "assert";
|
||||||
|
import T from "./TestHelper";
|
||||||
|
import {Utils} from "../Utils";
|
||||||
|
|
||||||
|
export default class WikidataSpecTest extends T {
|
||||||
|
constructor() {
|
||||||
|
super("Wikidata",
|
||||||
|
[
|
||||||
|
["download wikidata",
|
||||||
|
async () => {
|
||||||
|
|
||||||
|
Utils.injectJsonDownloadForTests(
|
||||||
|
"https://www.wikidata.org/wiki/Special:EntityData/Q14517013.json" ,
|
||||||
|
{"entities":{"Q14517013":{"pageid":16187848,"ns":0,"title":"Q14517013","lastrevid":1408823680,"modified":"2021-04-26T07:35:01Z","type":"item","id":"Q14517013","labels":{"nl":{"language":"nl","value":"Vredesmolen"},"en":{"language":"en","value":"Peace Mill"}},"descriptions":{"nl":{"language":"nl","value":"molen in West-Vlaanderen"}},"aliases":{},"claims":{"P625":[{"mainsnak":{"snaktype":"value","property":"P625","hash":"d86538f14e8cca00bbf30fb029829aacbc6903a0","datavalue":{"value":{"latitude":50.99444,"longitude":2.92528,"altitude":null,"precision":0.0001,"globe":"http://www.wikidata.org/entity/Q2"},"type":"globecoordinate"},"datatype":"globe-coordinate"},"type":"statement","id":"Q14517013$DBFBFD69-F54D-4C92-A7F4-A44F876E5776","rank":"normal","references":[{"hash":"732ec1c90a6f0694c7db9a71bf09fe7f2b674172","snaks":{"P143":[{"snaktype":"value","property":"P143","hash":"9123b0de1cc9c3954366ba797d598e4e1ea4146f","datavalue":{"value":{"entity-type":"item","numeric-id":10000,"id":"Q10000"},"type":"wikibase-entityid"},"datatype":"wikibase-item"}]},"snaks-order":["P143"]}]}],"P17":[{"mainsnak":{"snaktype":"value","property":"P17","hash":"c2859f311753176d6bdfa7da54ceeeac7acb52c8","datavalue":{"value":{"entity-type":"item","numeric-id":31,"id":"Q31"},"type":"wikibase-entityid"},"datatype":"wikibase-item"},"type":"statement","id":"Q14517013$C12E4DA5-44E1-41ED-BF3D-C84381246429","rank":"normal"}],"P18":[{"mainsnak":{"snaktype":"value","property":"P18","hash":"af765166ecaa7d01ea800812b5b356886b8849a0","datavalue":{"value":"Klerken Vredesmolen R01.jpg","type":"string"},"datatype":"commonsMedia"},"type":"statement","id":"Q14517013$5291801E-11BE-4CE7-8F42-D0D6A120F390","rank":"normal"}],"P2867":[{"mainsnak":{"snaktype":"value","property":"P2867","hash":"b1c627972ba2cc71e3567d2fb56cb5f90dd64007","datavalue":{"value":"893","type":"string"},"datatype":"external-id"},"type":"statement","id":"Q14517013$2aff9dcd-4d24-cd92-b5af-f6268425695f","rank":"normal"}],"P31":[{"mainsnak":{"snaktype":"value","property":"P31","hash":"9b48263bb51c506553aac2281ae331353b5c9002","datavalue":{"value":{"entity-type":"item","numeric-id":38720,"id":"Q38720"},"type":"wikibase-entityid"},"datatype":"wikibase-item"},"type":"statement","id":"Q14517013$46dd9d89-4999-eee6-20a4-c4f6650b1d9c","rank":"normal"},{"mainsnak":{"snaktype":"value","property":"P31","hash":"a1d6f3409c57de0361c68263c9397a99dabe19ea","datavalue":{"value":{"entity-type":"item","numeric-id":3851468,"id":"Q3851468"},"type":"wikibase-entityid"},"datatype":"wikibase-item"},"type":"statement","id":"Q14517013$C83A8B1F-7798-493A-86C9-EC0EFEE356B3","rank":"normal"},{"mainsnak":{"snaktype":"value","property":"P31","hash":"ee5ba9185bdf9f0eb80b52e1cdc70c5883fac95a","datavalue":{"value":{"entity-type":"item","numeric-id":623605,"id":"Q623605"},"type":"wikibase-entityid"},"datatype":"wikibase-item"},"type":"statement","id":"Q14517013$CF74DC2E-6814-4755-9BAD-6EE9FEF637DD","rank":"normal"}],"P2671":[{"mainsnak":{"snaktype":"value","property":"P2671","hash":"83fb38a3c6407f7d0d7bb051d1c31cea8ae26975","datavalue":{"value":"/g/121cb15z","type":"string"},"datatype":"external-id"},"type":"statement","id":"Q14517013$E6FFEF32-0131-42FD-9C66-1A406B68059A","rank":"normal"}]},"sitelinks":{"commonswiki":{"site":"commonswiki","title":"Category:Vredesmolen, Klerken","badges":[],"url":"https://commons.wikimedia.org/wiki/Category:Vredesmolen,_Klerken"},"nlwiki":{"site":"nlwiki","title":"Vredesmolen","badges":[],"url":"https://nl.wikipedia.org/wiki/Vredesmolen"}}}}}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
const wdata = await Wikidata.LoadWikidataEntryAsync(14517013)
|
||||||
|
T.isTrue(wdata.wikisites.has("nl"), "dutch for wikisite not found")
|
||||||
|
equal("Vredesmolen", wdata.wikisites.get("nl"))
|
||||||
|
}
|
||||||
|
|
||||||
|
]
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
Loading…
Reference in a new issue