Further stabilization of wikipedia box

This commit is contained in:
pietervdvn 2021-10-03 01:38:57 +02:00
parent 393d5d8932
commit a89d303ecd
15 changed files with 169 additions and 109 deletions

View file

@ -150,7 +150,6 @@ export class InitUiElements {
if (userDetails === undefined) {
return false;
}
console.log("Adding home location of ", userDetails)
const home = userDetails.home;
if (home === undefined) {
return userDetails.loggedIn; // If logged in, the home is not set and we unregister. If not logged in, we stay registered if a login still comes

View file

@ -23,7 +23,10 @@ export class WikidataImageProvider extends ImageProvider {
}
public async ExtractUrls(key: string, value: string): Promise<Promise<ProvidedImage>[]> {
const entity = await Wikidata.LoadWikidataEntry(value)
const entity = await Wikidata.LoadWikidataEntryAsync(value)
if(entity === undefined){
return []
}
const allImages : Promise<ProvidedImage>[] = []
// P18 is the claim 'depicted in this image'
@ -32,7 +35,7 @@ export class WikidataImageProvider extends ImageProvider {
allImages.push(...promises)
}
const commons =entity.wikisites.get("commons")
const commons = entity.commons
if (commons !== undefined) {
const promises = await WikimediaImageProvider.singleton.ExtractUrls(undefined , commons)
allImages.push(...promises)

View file

@ -18,7 +18,8 @@ export class Overpass {
private _relationTracker: RelationsTracker;
constructor(filter: TagsFilter, extraScripts: string[],
constructor(filter: TagsFilter,
extraScripts: string[],
interpreterUrl: string,
timeout: UIEventSource<number>,
relationTracker: RelationsTracker,

View file

@ -1,4 +1,5 @@
import {Utils} from "../Utils";
import * as Events from "events";
export class UIEventSource<T> {
@ -32,14 +33,14 @@ export class UIEventSource<T> {
return [];
}
public static flatten<X>(source: UIEventSource<UIEventSource<X>>, possibleSources: UIEventSource<any>[]): UIEventSource<X> {
public static flatten<X>(source: UIEventSource<UIEventSource<X>>, possibleSources?: UIEventSource<any>[]): UIEventSource<X> {
const sink = new UIEventSource<X>(source.data?.data);
source.addCallback((latestData) => {
sink.setData(latestData?.data);
});
for (const possibleSource of possibleSources) {
for (const possibleSource of possibleSources ?? []) {
possibleSource?.addCallback(() => {
sink.setData(source.data?.data);
})
@ -186,6 +187,25 @@ export class UIEventSource<T> {
}
}
/**
* Monadic bind function
*/
public bind<X>(f: ((t: T) => UIEventSource<X>)): UIEventSource<X>{
const sink = new UIEventSource<X>( undefined )
const seenEventSources = new Set<UIEventSource<X>>();
this.addCallbackAndRun(data => {
const eventSource = f(data)
if(eventSource === undefined){
sink.setData(undefined)
}else if(!seenEventSources.has(eventSource)){
eventSource.addCallbackAndRun(mappedData => sink.setData(mappedData))
seenEventSources.add(eventSource)
}
})
return sink;
}
/**
* Monoidal map:
* Given a function 'f', will construct a new UIEventSource where the contents will always be "f(this.data)'

View file

@ -1,4 +1,5 @@
import {Utils} from "../../Utils";
import {UIEventSource} from "../UIEventSource";
export interface WikidataResponse {
@ -62,15 +63,23 @@ export default class Wikidata {
}
}
/**
* Loads a wikidata page
* @returns the entity of the given value
*/
public static async LoadWikidataEntry(value: string | number): Promise<WikidataResponse> {
const wikidataUrl = "https://www.wikidata.org/wiki/"
if (typeof value === "number") {
value = "Q" + value
private static readonly _cache = new Map<number, UIEventSource<{success: WikidataResponse} | {error: any}>>()
public static LoadWikidataEntry(value: string | number): UIEventSource<{success: WikidataResponse} | {error: any}> {
const key = this.ExtractKey(value)
const cached = Wikidata._cache.get(key)
if(cached !== undefined){
return cached
}
const src = UIEventSource.FromPromiseWithErr(Wikidata.LoadWikidataEntryAsync(key))
Wikidata._cache.set(key, src)
return src;
}
private static ExtractKey(value: string | number) : number{
if (typeof value === "number") {
return value
}
const wikidataUrl = "https://www.wikidata.org/wiki/"
if (value.startsWith(wikidataUrl)) {
value = value.substring(wikidataUrl.length)
}
@ -78,12 +87,30 @@ export default class Wikidata {
// Probably some random link in the image field - we skip it
return undefined
}
if (!value.startsWith("Q")) {
value = "Q" + value
if (value.startsWith("Q")) {
value = value.substring(1)
}
const url = "https://www.wikidata.org/wiki/Special:EntityData/" + value + ".json";
const n = Number(value)
if(isNaN(n)){
return undefined
}
return n;
}
/**
* Loads a wikidata page
* @returns the entity of the given value
*/
public static async LoadWikidataEntryAsync(value: string | number): Promise<WikidataResponse> {
const id = Wikidata.ExtractKey(value)
if(id === undefined){
console.warn("Could not extract a wikidata entry from", value)
return undefined;
}
console.log("Requesting wikidata with id", id)
const url = "https://www.wikidata.org/wiki/Special:EntityData/Q" + id + ".json";
const response = await Utils.downloadJson(url)
return Wikidata.ParseResponse(response.entities[value]);
return Wikidata.ParseResponse(response.entities["Q" + id])
}
}

View file

@ -61,7 +61,6 @@ export default class Wikipedia {
const links = Array.from(content.getElementsByTagName("a"))
console.log("Links are", links)
// Rewrite relative links to absolute links + open them in a new tab
links.filter(link => link.getAttribute("href")?.startsWith("/") ?? false).
forEach(link => {

View file

@ -39,13 +39,13 @@ export default class AllDownloads extends ScrollableFullScreen {
const loading = Svg.loading_svg().SetClass("animate-rotate");
const dloadTrans = Translations.t.general.download
const icon = new Toggle(loading, Svg.floppy_ui(), isExporting);
const text = new Toggle(
new FixedUiElement("Exporting..."),
dloadTrans.exporting.Clone(),
new Combine([
Translations.t.general.download.downloadAsPdf.Clone().SetClass("font-bold"),
Translations.t.general.download.downloadAsPdfHelper.Clone()]
dloadTrans.downloadAsPdf.Clone().SetClass("font-bold"),
dloadTrans.downloadAsPdfHelper.Clone()]
).SetClass("flex flex-col")
.onClick(() => {
generatePdf()

View file

@ -121,7 +121,6 @@ export default class FilterView extends VariableUiElement {
listFilterElements.map((input) => input[1].data)
);
console.log(listTagsFilters, oldValue)
flayer.appliedFilters.setData(listTagsFilters);
})
);

View file

@ -32,7 +32,7 @@ export default class ExportPDF {
private readonly mapH = 210;
private readonly scaling = 2
private readonly freeDivId: string;
private readonly _layout: UIEventSource<LayoutConfig>;
private readonly _layout: LayoutConfig;
private _screenhotTaken = false;
constructor(
@ -41,7 +41,7 @@ export default class ExportPDF {
location: UIEventSource<Loc>,
background?: UIEventSource<BaseLayer>
features: FeaturePipeline,
layout: UIEventSource<LayoutConfig>
layout: LayoutConfig
}
) {
@ -87,7 +87,6 @@ export default class ExportPDF {
minimap.leafletMap .addCallbackAndRunD(leaflet => {
const bounds = BBox.fromLeafletBounds(leaflet.getBounds().pad(0.2))
options.features.GetTilesPerLayerWithin(bounds, tile => {
console.log("REndering", tile.name)
new ShowDataLayer(
{
features: tile,
@ -108,13 +107,13 @@ export default class ExportPDF {
}
private async CreatePdf(leaflet: L.Map) {
console.log("PDF creation started")
const t = Translations.t.general.pdf;
const layout = this._layout.data
const layout = this._layout
const screenshotter = new SimpleMapScreenshoter();
//minimap op index.html -> hidden daar alles op doen en dan weg
//minimap - leaflet map ophalen - boundaries ophalen - State.state.featurePipeline
screenshotter.addTo(leaflet);
console.log("Taking screenshot")
let doc = new jsPDF('landscape');
@ -164,7 +163,6 @@ export default class ExportPDF {
const imgSource = layout.icon
const imgType = imgSource.substr(imgSource.lastIndexOf(".") + 1);
img.src = imgSource
console.log(imgType)
if (imgType.toLowerCase() === "svg") {
new FixedUiElement("").AttachTo(this.freeDivId)

View file

@ -8,78 +8,69 @@ import BaseUIElement from "./BaseUIElement";
import Title from "./Base/Title";
import Translations from "./i18n/Translations";
import Svg from "../Svg";
import Wikidata from "../Logic/Web/Wikidata";
import Wikidata, {WikidataResponse} from "../Logic/Web/Wikidata";
import Locale from "./i18n/Locale";
import Toggle from "./Input/Toggle";
export default class WikipediaBox extends Combine {
export default class WikipediaBox extends Toggle {
private static async ExtractWikiPages(wikidata): Promise<Map<string, string>> {
return (await Wikidata.LoadWikidataEntry(wikidata)).wikisites
}
private static _cache = new Map()
constructor(wikidataId: string | UIEventSource<string>) {
const wp = Translations.t.general.wikipedia;
if(typeof wikidataId === "string"){
if (typeof wikidataId === "string") {
wikidataId = new UIEventSource(wikidataId)
}
const knownPages = new UIEventSource<{success:Map<string, string>}|{error:any}>(undefined)
wikidataId.addCallbackAndRunD(wikidataId => {
WikipediaBox.ExtractWikiPages(wikidataId).then(pages => {
knownPages.setData({success:pages})
}).catch(err=> {
knownPages.setData({error: err})
})
})
const cachedPages = new Map<string, BaseUIElement>()
const contents = new VariableUiElement(
knownPages.map(pages => {
if (pages === undefined) {
const wikibox = wikidataId
.bind(id => {
console.log("Wikidata is", id)
if(id === undefined){
return undefined
}
console.log("Initing load WIkidataentry with id", id)
return Wikidata.LoadWikidataEntry(id);
})
.map(maybewikidata => {
if (maybewikidata === undefined) {
return new Loading(wp.loading.Clone())
}
if (pages["error"] !== undefined) {
if (maybewikidata["error"] !== undefined) {
return wp.failed.Clone().SetClass("alert p-4")
}
const dict: Map<string, string> = pages["success"]
const wikidata = <WikidataResponse>maybewikidata["success"]
console.log("Got wikidata response", wikidata)
if (wikidata.wikisites.size === 0) {
return wp.noWikipediaPage.Clone()
}
const preferredLanguage = [Locale.language.data, "en", Array.from(dict.keys())[0]]
const preferredLanguage = [Locale.language.data, "en", Array.from(wikidata.wikisites.keys())[0]]
let language
let pagetitle;
let i = 0
do {
language = preferredLanguage[i]
pagetitle = dict.get(language)
pagetitle = wikidata.wikisites.get(language)
i++;
if(i >= preferredLanguage.length){
return wp.noWikipediaPage.Clone()
}
} while (pagetitle === undefined)
if(cachedPages.has(language)){
return cachedPages.get(language)
}
const page = WikipediaBox.createContents(pagetitle, language);
cachedPages.set(language, page)
return page
return WikipediaBox.createContents(pagetitle, language)
}, [Locale.language])
const contents = new VariableUiElement(
wikibox
).SetClass("overflow-auto normal-background rounded-lg")
super([
const mainContent = new Combine([
new Combine([Svg.wikipedia_ui().SetStyle("width: 1.5rem").SetClass("mr-3"),
new Title(Translations.t.general.wikipedia.wikipediaboxTitle.Clone(), 2)]).SetClass("flex"),
contents])
this
.SetClass("block rounded-xl subtle-background m-1 p-2 flex flex-col")
contents]).SetClass("block rounded-xl subtle-background m-1 p-2 flex flex-col")
super(
mainContent,
undefined,
wikidataId.map(id => id !== undefined)
)
}
/**
@ -103,6 +94,7 @@ export default class WikipediaBox extends Combine {
return new FixedUiElement(htmlContent["success"]).SetClass("wikipedia-article")
}
if (htmlContent["error"]) {
console.warn("Loading wikipage failed due to", htmlContent["error"])
return wp.failed.Clone().SetClass("alert p-4")
}

View file

@ -174,6 +174,7 @@
"downloadAsPdf": "Download a PDF of the current map",
"downloadAsPdfHelper": "Ideal to print the current map",
"downloadGeojson": "Download visible data as geojson",
"exporting": "Exporting...",
"downloadGeoJsonHelper": "Compatible with QGIS, ArcGIS, ESRI, ...",
"downloadCSV": "Download visible data as CSV",
"downloadCSVHelper": "Compatible with LibreOffice Calc, Excel, …",

View file

@ -11,7 +11,7 @@
"start": "npm run start:prepare && npm-run-all --parallel start:parallel:*",
"strt": "npm run start:prepare && npm run start:parallel:parcel",
"start:prepare": "ts-node scripts/generateLayerOverview.ts --no-fail && npm run increase-memory",
"start:parallel:parcel": "parcel *.html UI/** Logic/** assets/*.json assets/svg/* assets/generated/* assets/layers/*/*.svg assets/tagRenderings/*.json assets/themes/*/*.svg assets/themes/*/*.png vendor/* vendor/*/*",
"start:parallel:parcel": "parcel *.html UI/** Logic/** assets/*.json assets/svg/* assets/generated/* assets/layers/*/*.svg assets/layers/*/*.jpg assets/layers/*/*.png assets/tagRenderings/*.json assets/themes/*/*.svg assets/themes/*/*.jpg assets/themes/*/*.png vendor/* vendor/*/*",
"start:parallel:tailwindcli": "tailwindcss -i index.css -o css/index-tailwind-output.css --watch",
"test": "ts-node test/TestAll.ts",
"init": "npm ci && npm run generate && npm run generate:editor-layer-index && npm run generate:layouts && npm run clean",

View file

@ -2,9 +2,6 @@
* Generates a collection of geojson files based on an overpass query for a given theme
*/
import {Utils} from "../Utils";
Utils.runningFromConsole = true
import {Overpass} from "../Logic/Osm/Overpass";
import {existsSync, readFileSync, writeFileSync} from "fs";
import {TagsFilter} from "../Logic/Tags/TagsFilter";
@ -22,12 +19,13 @@ import FilteredLayer from "../Models/FilteredLayer";
import FeatureSource, {FeatureSourceForLayer} from "../Logic/FeatureSource/FeatureSource";
import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSource";
import TiledFeatureSource from "../Logic/FeatureSource/TiledFeatureSource/TiledFeatureSource";
import Constants from "../Models/Constants";
ScriptUtils.fixUtils()
function createOverpassObject(theme: LayoutConfig, relationTracker: RelationsTracker) {
function createOverpassObject(theme: LayoutConfig, relationTracker: RelationsTracker, backend: string) {
let filters: TagsFilter[] = [];
let extraScripts: string[] = [];
for (const layer of theme.layers) {
@ -58,7 +56,7 @@ function createOverpassObject(theme: LayoutConfig, relationTracker: RelationsTra
if (filters.length + extraScripts.length === 0) {
throw "Nothing to download! The theme doesn't declare anything to download"
}
return new Overpass(new Or(filters), extraScripts, new UIEventSource<string>("https://overpass.kumi.systems/api/interpreter"), //https://overpass-api.de/api/interpreter"),
return new Overpass(new Or(filters), extraScripts, backend,
new UIEventSource<number>(60), relationTracker);
}
@ -71,7 +69,7 @@ function geoJsonName(targetDir: string, x: number, y: number, z: number): string
}
/// Downloads the given feature and saves them to disk
async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass)/* : {failed: number, skipped :number} */ {
async function downloadRaw(targetdir: string, r: TileRange, theme: LayoutConfig, relationTracker: RelationsTracker)/* : {failed: number, skipped :number} */ {
let downloaded = 0
let failed = 0
let skipped = 0
@ -93,35 +91,28 @@ async function downloadRaw(targetdir: string, r: TileRange, overpass: Overpass)/
east: Math.max(boundsArr[0][1], boundsArr[1][1]),
west: Math.min(boundsArr[0][1], boundsArr[1][1])
}
const overpass = createOverpassObject(theme, relationTracker, Constants.defaultOverpassUrls[(downloaded + failed) % Constants.defaultOverpassUrls.length])
const url = overpass.buildQuery("[bbox:" + bounds.south + "," + bounds.west + "," + bounds.north + "," + bounds.east + "]")
await ScriptUtils.DownloadJSON(url)
.then(json => {
if (json.elements.length === 0) {
console.log("Got an empty response!")
if ((<string>json.remark ?? "").startsWith("runtime error")) {
console.error("Got a runtime error: ", json.remark)
failed++;
return
}
try {
}
console.log("Got the response - writing to ", filename)
writeFileSync(filename, JSON.stringify(json, null, " "));
const json = await ScriptUtils.DownloadJSON(url)
if (json.elements.length === 0) {
console.log("Got an empty response!")
if ((<string>json.remark ?? "").startsWith("runtime error")) {
console.error("Got a runtime error: ", json.remark)
failed++;
}
)
.catch(err => {
console.log(url)
console.log("Could not download - probably hit the rate limit; waiting a bit. (" + err + ")")
failed++;
return ScriptUtils.sleep(60000).then(() => console.log("Waiting is done"))
})
if (x < r.xend || y < r.yend) {
console.debug("Cooling down 10s")
await ScriptUtils.sleep(10000)
} else {
console.log("Got the response - writing to ", filename)
writeFileSync(filename, JSON.stringify(json, null, " "));
}
} catch (err) {
console.log(url)
console.log("Could not download - probably hit the rate limit; waiting a bit. (" + err + ")")
failed++;
await ScriptUtils.sleep(1000)
}
}
}
@ -291,11 +282,10 @@ async function main(args: string[]) {
return
}
const relationTracker = new RelationsTracker()
const overpass = createOverpassObject(theme, relationTracker)
let failed = 0;
do {
const cachingResult = await downloadRaw(targetdir, tileRange, overpass)
const cachingResult = await downloadRaw(targetdir, tileRange, theme, relationTracker)
failed = cachingResult.failed
if (failed > 0) {
await ScriptUtils.sleep(30000)

View file

@ -10,6 +10,7 @@ import RelationSplitHandlerSpec from "./RelationSplitHandler.spec";
import SplitActionSpec from "./SplitAction.spec";
import {Utils} from "../Utils";
import TileFreshnessCalculatorSpec from "./TileFreshnessCalculator.spec";
import WikidataSpecTest from "./Wikidata.spec.test";
ScriptUtils.fixUtils()
@ -23,7 +24,8 @@ const allTests = [
new UnitsSpec(),
new RelationSplitHandlerSpec(),
new SplitActionSpec(),
new TileFreshnessCalculatorSpec()
new TileFreshnessCalculatorSpec(),
new WikidataSpecTest()
]
Utils.externalDownloadFunction = async (url) => {

View file

@ -0,0 +1,29 @@
import Wikidata from "../Logic/Web/Wikidata";
import * as assert from "assert";
import {equal} from "assert";
import T from "./TestHelper";
import {Utils} from "../Utils";
export default class WikidataSpecTest extends T {
constructor() {
super("Wikidata",
[
["download wikidata",
async () => {
Utils.injectJsonDownloadForTests(
"https://www.wikidata.org/wiki/Special:EntityData/Q14517013.json" ,
{"entities":{"Q14517013":{"pageid":16187848,"ns":0,"title":"Q14517013","lastrevid":1408823680,"modified":"2021-04-26T07:35:01Z","type":"item","id":"Q14517013","labels":{"nl":{"language":"nl","value":"Vredesmolen"},"en":{"language":"en","value":"Peace Mill"}},"descriptions":{"nl":{"language":"nl","value":"molen in West-Vlaanderen"}},"aliases":{},"claims":{"P625":[{"mainsnak":{"snaktype":"value","property":"P625","hash":"d86538f14e8cca00bbf30fb029829aacbc6903a0","datavalue":{"value":{"latitude":50.99444,"longitude":2.92528,"altitude":null,"precision":0.0001,"globe":"http://www.wikidata.org/entity/Q2"},"type":"globecoordinate"},"datatype":"globe-coordinate"},"type":"statement","id":"Q14517013$DBFBFD69-F54D-4C92-A7F4-A44F876E5776","rank":"normal","references":[{"hash":"732ec1c90a6f0694c7db9a71bf09fe7f2b674172","snaks":{"P143":[{"snaktype":"value","property":"P143","hash":"9123b0de1cc9c3954366ba797d598e4e1ea4146f","datavalue":{"value":{"entity-type":"item","numeric-id":10000,"id":"Q10000"},"type":"wikibase-entityid"},"datatype":"wikibase-item"}]},"snaks-order":["P143"]}]}],"P17":[{"mainsnak":{"snaktype":"value","property":"P17","hash":"c2859f311753176d6bdfa7da54ceeeac7acb52c8","datavalue":{"value":{"entity-type":"item","numeric-id":31,"id":"Q31"},"type":"wikibase-entityid"},"datatype":"wikibase-item"},"type":"statement","id":"Q14517013$C12E4DA5-44E1-41ED-BF3D-C84381246429","rank":"normal"}],"P18":[{"mainsnak":{"snaktype":"value","property":"P18","hash":"af765166ecaa7d01ea800812b5b356886b8849a0","datavalue":{"value":"Klerken Vredesmolen R01.jpg","type":"string"},"datatype":"commonsMedia"},"type":"statement","id":"Q14517013$5291801E-11BE-4CE7-8F42-D0D6A120F390","rank":"normal"}],"P2867":[{"mainsnak":{"snaktype":"value","property":"P2867","hash":"b1c627972ba2cc71e3567d2fb56cb5f90dd64007","datavalue":{"value":"893","type":"string"},"datatype":"external-id"},"type":"statement","id":"Q14517013$2aff9dcd-4d24-cd92-b5af-f6268425695f","rank":"normal"}],"P31":[{"mainsnak":{"snaktype":"value","property":"P31","hash":"9b48263bb51c506553aac2281ae331353b5c9002","datavalue":{"value":{"entity-type":"item","numeric-id":38720,"id":"Q38720"},"type":"wikibase-entityid"},"datatype":"wikibase-item"},"type":"statement","id":"Q14517013$46dd9d89-4999-eee6-20a4-c4f6650b1d9c","rank":"normal"},{"mainsnak":{"snaktype":"value","property":"P31","hash":"a1d6f3409c57de0361c68263c9397a99dabe19ea","datavalue":{"value":{"entity-type":"item","numeric-id":3851468,"id":"Q3851468"},"type":"wikibase-entityid"},"datatype":"wikibase-item"},"type":"statement","id":"Q14517013$C83A8B1F-7798-493A-86C9-EC0EFEE356B3","rank":"normal"},{"mainsnak":{"snaktype":"value","property":"P31","hash":"ee5ba9185bdf9f0eb80b52e1cdc70c5883fac95a","datavalue":{"value":{"entity-type":"item","numeric-id":623605,"id":"Q623605"},"type":"wikibase-entityid"},"datatype":"wikibase-item"},"type":"statement","id":"Q14517013$CF74DC2E-6814-4755-9BAD-6EE9FEF637DD","rank":"normal"}],"P2671":[{"mainsnak":{"snaktype":"value","property":"P2671","hash":"83fb38a3c6407f7d0d7bb051d1c31cea8ae26975","datavalue":{"value":"/g/121cb15z","type":"string"},"datatype":"external-id"},"type":"statement","id":"Q14517013$E6FFEF32-0131-42FD-9C66-1A406B68059A","rank":"normal"}]},"sitelinks":{"commonswiki":{"site":"commonswiki","title":"Category:Vredesmolen, Klerken","badges":[],"url":"https://commons.wikimedia.org/wiki/Category:Vredesmolen,_Klerken"},"nlwiki":{"site":"nlwiki","title":"Vredesmolen","badges":[],"url":"https://nl.wikipedia.org/wiki/Vredesmolen"}}}}}
)
const wdata = await Wikidata.LoadWikidataEntryAsync(14517013)
T.isTrue(wdata.wikisites.has("nl"), "dutch for wikisite not found")
equal("Vredesmolen", wdata.wikisites.get("nl"))
}
]
]);
}
}