midgard/mapcomplete
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Refactoring of image detection, fix loading wikimedia images

Browse source
This commit is contained in:
pietervdvn 2021-09-29 23:56:59 +02:00
parent 4da6070b28
commit a6e8714ae0
21 changed files with 468 additions and 528 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

173
Logic/Actors/ImageSearcher.ts
View file

@ -1,173 +0,0 @@
import {ImagesInCategory, Wikidata, Wikimedia} from "../ImageProviders/Wikimedia";
import {UIEventSource} from "../UIEventSource";
/**
* There are multiple way to fetch images for an object
* 1) There is an image tag
* 2) There is an image tag, the image tag contains multiple ';'-separated URLS
* 3) there are multiple image tags, e.g. 'image', 'image:0', 'image:1', and 'image_0', 'image_1' - however, these are pretty rare so we are gonna ignore them
* 4) There is a wikimedia_commons-tag, which either has a 'File': or a 'category:' containing images
* 5) There is a wikidata-tag, and the wikidata item either has an 'image' attribute or has 'a link to a wikimedia commons category'
* 6) There is a wikipedia article, from which we can deduct the wikidata item
*
* For some images, author and license should be shown
*/
/**
* Class which search for all the possible locations for images and which builds a list of UI-elements for it.
* Note that this list is embedded into an UIEVentSource, ready to put it into a carousel.
*
*/
export class ImageSearcher extends UIEventSource<{ key: string, url: string }[]> {
private static _cache = new Map<string, ImageSearcher>();
private readonly _wdItem = new UIEventSource<string>("");
private readonly _commons = new UIEventSource<string>("");
private constructor(tags: UIEventSource<any>, imagePrefix = "image", loadSpecial = true) {
super([])
const self = this;
function AddImages(images: { key: string, url: string }[]) {
const oldUrls = self.data.map(kurl => kurl.url);
let somethingChanged = false;
for (const image of images) {
const url = image.url;
if (url === undefined || url === null || url === "") {
continue;
}
if (oldUrls.indexOf(url) >= 0) {
// Already exists
continue;
}
self.data.push(image);
somethingChanged = true;
}
if (somethingChanged) {
self.ping();
}
}
function addImage(image: string) {
AddImages([{url: image, key: undefined}]);
}
// By wrapping this in a UIEventSource, we prevent multiple queries of loadWikiData
this._wdItem.addCallback(wdItemContents => {
ImageSearcher.loadWikidata(wdItemContents, addImage);
});
this._commons.addCallback(commonsData => {
ImageSearcher.LoadCommons(commonsData, addImage)
});
tags.addCallbackAndRun(tags => {
AddImages(ImageSearcher.LoadImages(tags, imagePrefix));
});
if (loadSpecial) {
tags.addCallbackAndRunD(tags => {
const wdItem = tags.wikidata;
if (wdItem !== undefined) {
self._wdItem.setData(wdItem);
}
const commons = tags.wikimedia_commons;
if (commons !== undefined) {
self._commons.setData(commons);
}
if (tags.mapillary) {
let mapillary = tags.mapillary;
const prefix = "https://www.mapillary.com/map/im/";
let regex = /https?:\/\/www.mapillary.com\/app\/.*pKey=([^&]*).*/
let match = mapillary.match(regex);
if (match) {
mapillary = match[1];
}
if (mapillary.indexOf(prefix) < 0) {
mapillary = prefix + mapillary;
}
AddImages([{url: mapillary, key: undefined}]);
}
})
}
}
public static construct(tags: UIEventSource<any>, imagePrefix = "image", loadSpecial = true): ImageSearcher {
const key = tags.data["id"] + " " + imagePrefix + loadSpecial;
if (tags.data["id"] !== undefined && ImageSearcher._cache.has(key)) {
return ImageSearcher._cache.get(key)
}
const searcher = new ImageSearcher(tags, imagePrefix, loadSpecial);
ImageSearcher._cache.set(key, searcher)
return searcher;
}
private static loadWikidata(wikidataItem, addImage: ((url: string) => void)): void {
// Load the wikidata item, then detect usage on 'commons'
let allWikidataId = wikidataItem.split(";");
for (let wikidataId of allWikidataId) {
// @ts-ignore
if (wikidataId.startsWith("Q")) {
wikidataId = wikidataId.substr(1);
}
Wikimedia.GetWikiData(parseInt(wikidataId), (wd: Wikidata) => {
addImage(wd.image);
Wikimedia.GetCategoryFiles(wd.commonsWiki, (images: ImagesInCategory) => {
for (const image of images.images) {
if (image.startsWith("File:")) {
addImage(image);
}
}
})
})
}
}
private static LoadCommons(commonsData: string, addImage: ((url: string) => void)): void {
const allCommons: string[] = commonsData.split(";");
for (const commons of allCommons) {
if (commons.startsWith("Category:")) {
Wikimedia.GetCategoryFiles(commons, (images: ImagesInCategory) => {
for (const image of images.images) {
if (image.startsWith("File:")) {
addImage(image)
}
}
})
} else {
if (commons.startsWith("File:")) {
addImage(commons)
}
}
}
}
private static LoadImages(tags: any, imagePrefix: string): { key: string, url: string }[] {
const imageTag = tags[imagePrefix];
const images: { key: string, url: string }[] = [];
if (imageTag !== undefined) {
const bareImages = imageTag.split(";");
for (const bareImage of bareImages) {
images.push({key: imagePrefix, url: bareImage})
}
}
for (const key in tags) {
if (key.startsWith(imagePrefix + ":")) {
const url = tags[key]
images.push({key: key, url: url})
}
}
return images;
}
}

3
Logic/FeatureSource/TiledFeatureSource/TiledFromLocalStorageSource.ts
View file

@ -24,11 +24,12 @@ public tileFreshness : Map<number, Date> = new Map<number, Date>()
// @ts-ignore
const indexes: number[] = Object.keys(localStorage)
.filter(key => {
return key.startsWith(prefix) && !key.endsWith("-time");
return key.startsWith(prefix) && !key.endsWith("-time") && !key.endsWith("-format");
})
.map(key => {
return Number(key.substring(prefix.length));
})
.filter(i => !isNaN(i))
console.debug("Layer", layer.layerDef.id, "has following tiles in available in localstorage", indexes.map(i => Tiles.tile_from_index(i).join("/")).join(", "))
for (const index of indexes) {

54
Logic/ImageProviders/AllImageProviders.ts
View file

@ -1,9 +1,59 @@
import {Mapillary} from "./Mapillary";
import {Wikimedia} from "./Wikimedia";
import {WikimediaImageProvider} from "./WikimediaImageProvider";
import {Imgur} from "./Imgur";
import GenericImageProvider from "./GenericImageProvider";
import {UIEventSource} from "../UIEventSource";
import ImageProvider, {ProvidedImage} from "./ImageProvider";
import {WikidataImageProvider} from "./WikidataImageProvider";
import {Utils} from "../../Utils";
/**
* A generic 'from the interwebz' image picker, without attribution
*/
export default class AllImageProviders {
public static ImageAttributionSource = [Imgur.singleton, Mapillary.singleton, Wikimedia.singleton]
public static ImageAttributionSource: ImageProvider[] = [
Imgur.singleton,
Mapillary.singleton,
WikidataImageProvider.singleton,
WikimediaImageProvider.singleton,
new GenericImageProvider(Imgur.defaultValuePrefix)]
private static _cache: Map<string, UIEventSource<ProvidedImage[]>> = new Map<string, UIEventSource<ProvidedImage[]>>()
public static LoadImagesFor(tags: UIEventSource<any>, imagePrefix: string, loadSpecialSource: boolean): UIEventSource<ProvidedImage[]> {
const id = tags.data.id
if (id === undefined) {
return undefined;
}
const cached = this._cache.get(tags.data.id)
if (cached !== undefined) {
return cached
}
const source = new UIEventSource([])
this._cache.set(id, source)
const allSources = []
for (const imageProvider of AllImageProviders.ImageAttributionSource) {
const singleSource = imageProvider.GetRelevantUrls(tags)
allSources.push(singleSource)
singleSource.addCallbackAndRunD(_ => {
const all : ProvidedImage[] = [].concat(...allSources.map(source => source.data))
const uniq = []
const seen = new Set<string>()
for (const img of all) {
if(seen.has(img.url)){
continue
}
seen.add(img.url)
uniq.push(img)
}
source.setData(uniq)
})
}
return source;
}
}

36
Logic/ImageProviders/GenericImageProvider.ts Normal file
View file

@ -0,0 +1,36 @@
import ImageProvider, {ProvidedImage} from "./ImageProvider";
export default class GenericImageProvider extends ImageProvider {
public defaultKeyPrefixes: string[] = ["image"];
private readonly _valuePrefixBlacklist: string[];
public constructor(valuePrefixBlacklist: string[]) {
super();
this._valuePrefixBlacklist = valuePrefixBlacklist;
}
protected DownloadAttribution(url: string) {
return undefined
}
async ExtractUrls(key: string, value: string): Promise<Promise<ProvidedImage>[]> {
if (this._valuePrefixBlacklist.some(prefix => value.startsWith(prefix))) {
return []
}
return [Promise.resolve({
key: key,
url: value,
provider: this
})]
}
SourceIcon(backlinkSource?: string) {
return undefined;
}
}

33
Logic/ImageProviders/ImageAttributionSource.ts
View file

@ -1,33 +0,0 @@
import {UIEventSource} from "../UIEventSource";
import {LicenseInfo} from "./Wikimedia";
import BaseUIElement from "../../UI/BaseUIElement";
export default abstract class ImageAttributionSource {
private _cache = new Map<string, UIEventSource<LicenseInfo>>()
GetAttributionFor(url: string): UIEventSource<LicenseInfo> {
const cached = this._cache.get(url);
if (cached !== undefined) {
return cached;
}
const src = new UIEventSource(undefined)
this._cache.set(url, src)
this.DownloadAttribution(url).then(license =>
src.setData(license))
.catch(e => console.error("Could not download license information for ", url, " due to", e))
return src;
}
public abstract SourceIcon(backlinkSource?: string): BaseUIElement;
/*Converts a value to a URL. Can return null if not applicable*/
public PrepareUrl(value: string): string | UIEventSource<string> {
return value;
}
protected abstract DownloadAttribution(url: string): Promise<LicenseInfo>;
}

64
Logic/ImageProviders/ImageProvider.ts Normal file
View file

@ -0,0 +1,64 @@
import {UIEventSource} from "../UIEventSource";
import BaseUIElement from "../../UI/BaseUIElement";
import {LicenseInfo} from "./LicenseInfo";
export interface ProvidedImage {
url: string, key: string, provider: ImageProvider
}
export default abstract class ImageProvider {
protected abstract readonly defaultKeyPrefixes : string[]
private _cache = new Map<string, UIEventSource<LicenseInfo>>()
GetAttributionFor(url: string): UIEventSource<LicenseInfo> {
const cached = this._cache.get(url);
if (cached !== undefined) {
return cached;
}
const src =UIEventSource.FromPromise(this.DownloadAttribution(url))
this._cache.set(url, src)
return src;
}
public abstract SourceIcon(backlinkSource?: string): BaseUIElement;
protected abstract DownloadAttribution(url: string): Promise<LicenseInfo>;
/**
* Given a properies object, maps it onto _all_ the available pictures for this imageProvider
*/
public GetRelevantUrls(allTags: UIEventSource<any>, options?: {
prefixes?: string[]
}):UIEventSource<ProvidedImage[]> {
const prefixes = options?.prefixes ?? this.defaultKeyPrefixes
const relevantUrls = new UIEventSource<{ url: string; key: string; provider: ImageProvider }[]>([])
const seenValues = new Set<string>()
allTags.addCallbackAndRunD(tags => {
for (const key in tags) {
if(!prefixes.some(prefix => key.startsWith(prefix))){
continue
}
const value = tags[key]
if(seenValues.has(value)){
continue
}
seenValues.add(value)
this.ExtractUrls(key, value).then(promises => {
for (const promise of promises) {
promise.then(providedImage => {
relevantUrls.data.push(providedImage)
relevantUrls.ping()
})
}
})
}
})
return relevantUrls
}
public abstract ExtractUrls(key: string, value: string) : Promise<Promise<ProvidedImage>[]>;
}

23
Logic/ImageProviders/Imgur.ts
View file

@ -1,12 +1,14 @@
// @ts-ignore
import $ from "jquery"
import {LicenseInfo} from "./Wikimedia";
import ImageAttributionSource from "./ImageAttributionSource";
import ImageProvider, {ProvidedImage} from "./ImageProvider";
import BaseUIElement from "../../UI/BaseUIElement";
import {Utils} from "../../Utils";
import Constants from "../../Models/Constants";
import {LicenseInfo} from "./LicenseInfo";
export class Imgur extends ImageAttributionSource {
export class Imgur extends ImageProvider {
public static readonly defaultValuePrefix = ["https://i.imgur.com"]
public readonly defaultKeyPrefixes: string[] = ["image"];
public static readonly singleton = new Imgur();
@ -87,7 +89,7 @@ export class Imgur extends ImageAttributionSource {
return undefined;
}
protected async DownloadAttribution(url: string): Promise<LicenseInfo> {
protected DownloadAttribution: (url: string) => Promise<LicenseInfo> = async (url: string) => {
const hash = url.substr("https://i.imgur.com/".length).split(".jpg")[0];
const apiUrl = 'https://api.imgur.com/3/image/' + hash;
@ -110,5 +112,16 @@ export class Imgur extends ImageAttributionSource {
return licenseInfo
}
public async ExtractUrls(key: string, value: string): Promise<Promise<ProvidedImage>[]> {
if (Imgur.defaultValuePrefix.some(prefix => value.startsWith(prefix))) {
return [Promise.resolve({
url: value,
key: key,
provider: this
})]
}
return []
}
}

10
Logic/ImageProviders/LicenseInfo.ts Normal file
View file

@ -0,0 +1,10 @@
export class LicenseInfo {
artist: string = "";
license: string = "";
licenseShortName: string = "";
usageTerms: string = "";
attributionRequired: boolean = false;
copyrighted: boolean = false;
credit: string = "";
description: string = "";
}

47
Logic/ImageProviders/Mapillary.ts
View file

@ -1,19 +1,19 @@
import {LicenseInfo} from "./Wikimedia";
import ImageAttributionSource from "./ImageAttributionSource";
import ImageProvider, {ProvidedImage} from "./ImageProvider";
import BaseUIElement from "../../UI/BaseUIElement";
import {UIEventSource} from "../UIEventSource";
import Svg from "../../Svg";
import {Utils} from "../../Utils";
import {LicenseInfo} from "./LicenseInfo";
import Constants from "../../Models/Constants";
export class Mapillary extends ImageAttributionSource {
export class Mapillary extends ImageProvider {
defaultKeyPrefixes = ["mapillary"]
public static readonly singleton = new Mapillary();
private static readonly v4_cached_urls = new Map<string, UIEventSource<string>>();
private static readonly client_token_v3 = 'TXhLaWthQ1d4RUg0czVxaTVoRjFJZzowNDczNjUzNmIyNTQyYzI2'
private static readonly client_token_v4 = "MLY|4441509239301885|b40ad2d3ea105435bd40c7e76993ae85"
private constructor() {
super();
}
@ -56,26 +56,31 @@ export class Mapillary extends ImageAttributionSource {
return Svg.mapillary_svg();
}
PrepareUrl(value: string): string | UIEventSource<string> {
const keyV = Mapillary.ExtractKeyFromURL(value)
if (!keyV.isApiv4) {
return `https://images.mapillary.com/${keyV.key}/thumb-640.jpg?client_id=${Mapillary.client_token_v3}`
} else {
const key = keyV.key;
if (Mapillary.v4_cached_urls.has(key)) {
return Mapillary.v4_cached_urls.get(key)
async ExtractUrls(key: string, value: string): Promise<Promise<ProvidedImage>[]> {
return [this.PrepareUrlAsync(key, value)]
}
const metadataUrl = 'https://graph.mapillary.com/' + key + '?fields=thumb_1024_url&&access_token=' + Mapillary.client_token_v4;
private async PrepareUrlAsync(key: string, value: string): Promise<ProvidedImage> {
const keyV = Mapillary.ExtractKeyFromURL(value)
if (!keyV.isApiv4) {
const url = `https://images.mapillary.com/${keyV.key}/thumb-640.jpg?client_id=${Constants.mapillary_client_token_v3}`
return {
url: url,
provider: this,
key: key
}
} else {
const key = keyV.key;
const metadataUrl = 'https://graph.mapillary.com/' + key + '?fields=thumb_1024_url&&access_token=' + Constants.mapillary_client_token_v4;
const source = new UIEventSource<string>(undefined)
Mapillary.v4_cached_urls.set(key, source)
Utils.downloadJson(metadataUrl).then(
json => {
console.warn("Got response on mapillary image", json, json["thumb_1024_url"])
return source.setData(json["thumb_1024_url"]);
const response = await Utils.downloadJson(metadataUrl)
const url = <string> response["thumb_1024_url"];
return {
url: url,
provider: this,
key: key
}
)
return source
}
}

51
Logic/ImageProviders/WikidataImageProvider.ts Normal file
View file

@ -0,0 +1,51 @@
import {Utils} from "../../Utils";
import ImageProvider, {ProvidedImage} from "./ImageProvider";
import BaseUIElement from "../../UI/BaseUIElement";
import Svg from "../../Svg";
import {WikimediaImageProvider} from "./WikimediaImageProvider";
export class WikidataImageProvider extends ImageProvider {
public SourceIcon(backlinkSource?: string): BaseUIElement {
throw Svg.wikidata_svg();
}
public static readonly singleton = new WikidataImageProvider()
public readonly defaultKeyPrefixes = ["wikidata"]
private constructor() {
super()
}
protected DownloadAttribution(url: string): Promise<any> {
throw new Error("Method not implemented; shouldn't be needed!");
}
public async ExtractUrls(key: string, value: string): Promise<Promise<ProvidedImage>[]> {
const wikidataUrl = "https://www.wikidata.org/wiki/"
if (value.startsWith(wikidataUrl)) {
value = value.substring(wikidataUrl.length)
}
if (!value.startsWith("Q")) {
value = "Q" + value
}
const url = "https://www.wikidata.org/wiki/Special:EntityData/" + value + ".json";
const response = await Utils.downloadJson(url)
const entity = response.entities[value];
const commons = entity.sitelinks.commonswiki;
// P18 is the claim 'depicted in this image'
const image = entity.claims.P18?.[0]?.mainsnak?.datavalue?.value;
const allImages = []
if (image !== undefined) {
// We found a 'File://'
const promises = await WikimediaImageProvider.singleton.ExtractUrls(key, image)
allImages.push(...promises)
}
if (commons !== undefined) {
const promises = await WikimediaImageProvider.singleton.ExtractUrls(commons, image)
allImages.push(...promises)
}
return allImages
}
}

185
Logic/ImageProviders/Wikimedia.ts
View file

@ -1,185 +0,0 @@
import ImageAttributionSource from "./ImageAttributionSource";
import BaseUIElement from "../../UI/BaseUIElement";
import Svg from "../../Svg";
import Link from "../../UI/Base/Link";
import {Utils} from "../../Utils";
/**
* This module provides endpoints for wikipedia/wikimedia and others
*/
export class Wikimedia extends ImageAttributionSource {
public static readonly singleton = new Wikimedia();
private constructor() {
super();
}
static ImageNameToUrl(filename: string, width: number = 500, height: number = 200): string {
filename = encodeURIComponent(filename);
return "https://commons.wikimedia.org/wiki/Special:FilePath/" + filename + "?width=" + width + "&height=" + height;
}
static GetCategoryFiles(categoryName: string, handleCategory: ((ImagesInCategory: ImagesInCategory) => void),
alreadyLoaded = 0,
continueParameter: { k: string, param: string } = undefined) {
if (categoryName === undefined || categoryName === null || categoryName === "") {
return;
}
// @ts-ignore
if (!categoryName.startsWith("Category:")) {
categoryName = "Category:" + categoryName;
}
let url = "https://commons.wikimedia.org/w/api.php?" +
"action=query&list=categorymembers&format=json&" +
"&origin=*" +
"&cmtitle=" + encodeURIComponent(categoryName);
if (continueParameter !== undefined) {
url = url + "&" + continueParameter.k + "=" + continueParameter.param;
}
const self = this;
console.log("Loading a wikimedia category: ", url)
Utils.downloadJson(url).then((response) => {
let imageOverview = new ImagesInCategory();
let members = response.query?.categorymembers;
if (members === undefined) {
members = [];
}
for (const member of members) {
imageOverview.images.push(member.title);
}
console.log("Got images! ", imageOverview)
if (response.continue === undefined) {
handleCategory(imageOverview);
return;
}
if (alreadyLoaded > 10) {
console.log(`Recursive wikimedia category load stopped for ${categoryName} - got already enough images now (${alreadyLoaded})`)
handleCategory(imageOverview)
return;
}
self.GetCategoryFiles(categoryName,
(recursiveImages) => {
recursiveImages.images.push(...imageOverview.images);
handleCategory(recursiveImages);
},
alreadyLoaded + 10,
{k: "cmcontinue", param: response.continue.cmcontinue})
});
}
static GetWikiData(id: number, handleWikidata: ((Wikidata) => void)) {
const url = "https://www.wikidata.org/wiki/Special:EntityData/Q" + id + ".json";
Utils.downloadJson(url).then(response => {
const entity = response.entities["Q" + id];
const commons = entity.sitelinks.commonswiki;
const wd = new Wikidata();
wd.commonsWiki = commons?.title;
// P18 is the claim 'depicted in this image'
const image = entity.claims.P18?.[0]?.mainsnak?.datavalue?.value;
if (image) {
wd.image = "File:" + image;
}
handleWikidata(wd);
});
}
private static ExtractFileName(url: string) {
if (!url.startsWith("http")) {
return url;
}
const path = new URL(url).pathname
return path.substring(path.lastIndexOf("/") + 1);
}
SourceIcon(backlink: string): BaseUIElement {
const img = Svg.wikimedia_commons_white_svg()
.SetStyle("width:2em;height: 2em");
if (backlink === undefined) {
return img
}
return new Link(Svg.wikimedia_commons_white_img,
`https://commons.wikimedia.org/wiki/${backlink}`, true)
}
PrepareUrl(value: string): string {
if (value.toLowerCase().startsWith("https://commons.wikimedia.org/wiki/")) {
return value;
}
return Wikimedia.ImageNameToUrl(value, 500, 400)
.replace(/'/g, '%27');
}
protected async DownloadAttribution(filename: string): Promise<LicenseInfo> {
filename = Wikimedia.ExtractFileName(filename)
if (filename === "") {
return undefined;
}
const url = "https://en.wikipedia.org/w/" +
"api.php?action=query&prop=imageinfo&iiprop=extmetadata&" +
"titles=" + filename +
"&format=json&origin=*";
const data = await Utils.downloadJson(url)
const licenseInfo = new LicenseInfo();
const license = (data.query.pages[-1].imageinfo ?? [])[0]?.extmetadata;
if (license === undefined) {
console.error("This file has no usable metedata or license attached... Please fix the license info file yourself!")
return undefined;
}
licenseInfo.artist = license.Artist?.value;
licenseInfo.license = license.License?.value;
licenseInfo.copyrighted = license.Copyrighted?.value;
licenseInfo.attributionRequired = license.AttributionRequired?.value;
licenseInfo.usageTerms = license.UsageTerms?.value;
licenseInfo.licenseShortName = license.LicenseShortName?.value;
licenseInfo.credit = license.Credit?.value;
licenseInfo.description = license.ImageDescription?.value;
return licenseInfo;
}
}
export class Wikidata {
commonsWiki: string;
image: string;
}
export class ImagesInCategory {
// Filenames of relevant images
images: string[] = [];
}
export class LicenseInfo {
artist: string = "";
license: string = "";
licenseShortName: string = "";
usageTerms: string = "";
attributionRequired: boolean = false;
copyrighted: boolean = false;
credit: string = "";
description: string = "";
}

163
Logic/ImageProviders/WikimediaImageProvider.ts Normal file
View file

@ -0,0 +1,163 @@
import ImageProvider, {ProvidedImage} from "./ImageProvider";
import BaseUIElement from "../../UI/BaseUIElement";
import Svg from "../../Svg";
import Link from "../../UI/Base/Link";
import {Utils} from "../../Utils";
import {LicenseInfo} from "./LicenseInfo";
/**
* This module provides endpoints for wikimedia and others
*/
export class WikimediaImageProvider extends ImageProvider {
public readonly defaultKeyPrefixes = ["wikimedia_commons"]
public static readonly singleton = new WikimediaImageProvider();
private constructor() {
super();
}
/**
* Recursively walks a wikimedia commons category in order to search for (image) files
* Returns (a promise of) a list of URLS
* @param categoryName The name of the wikimedia category
* @param maxLoad: the maximum amount of images to return
* @param continueParameter: if the page indicates that more pages should be loaded, this uses a token to continue. Provided by wikimedia
*/
private static async GetImagesInCategory(categoryName: string,
maxLoad = 10,
continueParameter: string = undefined): Promise<string[]> {
if (categoryName === undefined || categoryName === null || categoryName === "") {
return [];
}
if (!categoryName.startsWith("Category:")) {
categoryName = "Category:" + categoryName;
}
let url = "https://commons.wikimedia.org/w/api.php?" +
"action=query&list=categorymembers&format=json&" +
"&origin=*" +
"&cmtitle=" + encodeURIComponent(categoryName);
if (continueParameter !== undefined) {
url = `${url}&cmcontinue=${continueParameter}`;
}
console.log("Loading a wikimedia category: ", url)
const response = await Utils.downloadJson(url)
const members = response.query?.categorymembers ?? [];
const imageOverview: string[] = members.map(member => member.title);
if (response.continue === undefined) {
// We are done crawling through the category - no continuation in sight
return imageOverview;
}
if (maxLoad - imageOverview.length <= 0) {
console.log(`Recursive wikimedia category load stopped for ${categoryName}`)
return imageOverview;
}
// We do have a continue token - let's load the next page
const recursive = await this.GetImagesInCategory(categoryName, maxLoad - imageOverview.length, response.continue.cmcontinue)
imageOverview.push(...recursive)
return imageOverview
}
private static ExtractFileName(url: string) {
if (!url.startsWith("http")) {
return url;
}
const path = new URL(url).pathname
return path.substring(path.lastIndexOf("/") + 1);
}
SourceIcon(backlink: string): BaseUIElement {
const img = Svg.wikimedia_commons_white_svg()
.SetStyle("width:2em;height: 2em");
if (backlink === undefined) {
return img
}
return new Link(Svg.wikimedia_commons_white_img,
`https://commons.wikimedia.org/wiki/${backlink}`, true)
}
private PrepareUrl(value: string): string {
if (value.toLowerCase().startsWith("https://commons.wikimedia.org/wiki/")) {
return value;
}
return (`https://commons.wikimedia.org/wiki/Special:FilePath/${encodeURIComponent(value)}?width=500&height=400`)
}
protected async DownloadAttribution(filename: string): Promise<LicenseInfo> {
filename = WikimediaImageProvider.ExtractFileName(filename)
if (filename === "") {
return undefined;
}
const url = "https://en.wikipedia.org/w/" +
"api.php?action=query&prop=imageinfo&iiprop=extmetadata&" +
"titles=" + filename +
"&format=json&origin=*";
const data = await Utils.downloadJson(url)
const licenseInfo = new LicenseInfo();
const license = (data.query.pages[-1].imageinfo ?? [])[0]?.extmetadata;
if (license === undefined) {
console.error("This file has no usable metedata or license attached... Please fix the license info file yourself!")
return undefined;
}
licenseInfo.artist = license.Artist?.value;
licenseInfo.license = license.License?.value;
licenseInfo.copyrighted = license.Copyrighted?.value;
licenseInfo.attributionRequired = license.AttributionRequired?.value;
licenseInfo.usageTerms = license.UsageTerms?.value;
licenseInfo.licenseShortName = license.LicenseShortName?.value;
licenseInfo.credit = license.Credit?.value;
licenseInfo.description = license.ImageDescription?.value;
return licenseInfo;
}
private async UrlForImage(image: string): Promise<ProvidedImage>{
if(!image.startsWith("File:")){
image = "File:"+image
}
return {url: this.PrepareUrl(image), key: undefined, provider: this}
}
public async ExtractUrls(key: string, value: string): Promise<Promise<ProvidedImage>[]> {
const commonsPrefix = "https://commons.wikimedia.org/wiki/"
if(value.startsWith(commonsPrefix)){
value = value.substring(commonsPrefix.length)
} else if(value.startsWith("https://upload.wikimedia.org")){
const result : ProvidedImage = {
key: undefined,
url: value,
provider: this
}
return [Promise.resolve(result)]
}
if(value.startsWith("Category:")){
const urls = await WikimediaImageProvider.GetImagesInCategory(value)
return urls.map(image => this.UrlForImage(image))
}
if(value.startsWith("File:")){
return [this.UrlForImage(value)]
}
if(value.startsWith("http")){
// PRobably an error
return []
}
// We do a last effort and assume this is a file
return [this.UrlForImage("File:"+value)]
}
}

2
Logic/UIEventSource.ts
View file

@ -64,7 +64,7 @@ export class UIEventSource<T> {
public static FromPromise<T>(promise : Promise<T>): UIEventSource<T>{
const src = new UIEventSource<T>(undefined)
promise.then(d => src.setData(d))
promise?.then(d => src.setData(d))
return src
}

7
Models/Constants.ts
View file

@ -2,8 +2,11 @@ import {Utils} from "../Utils";
export default class Constants {
public static vNumber = "0.10.0-rc0";
public static vNumber = "0.10.0-rc1";
public static ImgurApiKey = '7070e7167f0a25a'
public static readonly mapillary_client_token_v3 = 'TXhLaWthQ1d4RUg0czVxaTVoRjFJZzowNDczNjUzNmIyNTQyYzI2'
public static readonly mapillary_client_token_v4 = "MLY|4441509239301885|b40ad2d3ea105435bd40c7e76993ae85"
public static defaultOverpassUrls = [
// The official instance, 10000 queries per day per project allowed
"https://overpass-api.de/api/interpreter",
@ -15,6 +18,8 @@ export default class Constants {
"https://overpass.openstreetmap.fr/api/interpreter"
]
// The user journey states thresholds when a new feature gets unlocked
public static userJourney = {
moreScreenUnlock: 1,

23
UI/Image/AttributedImage.ts
View file

@ -1,30 +1,19 @@
import Combine from "../Base/Combine";
import Attribution from "./Attribution";
import Img from "../Base/Img";
import ImageAttributionSource from "../../Logic/ImageProviders/ImageAttributionSource";
import {ProvidedImage} from "../../Logic/ImageProviders/ImageProvider";
import BaseUIElement from "../BaseUIElement";
import {VariableUiElement} from "../Base/VariableUIElement";
import Loading from "../Base/Loading";
export class AttributedImage extends Combine {
constructor(urlSource: string, imgSource: ImageAttributionSource) {
const preparedUrl = imgSource.PrepareUrl(urlSource)
constructor(imageInfo: ProvidedImage) {
let img: BaseUIElement;
let attr: BaseUIElement
if (typeof preparedUrl === "string") {
img = new Img(urlSource);
attr = new Attribution(imgSource.GetAttributionFor(urlSource), imgSource.SourceIcon())
} else {
img = new VariableUiElement(preparedUrl.map(url => {
if(url === undefined){
return new Loading()
}
return new Img(url, false, {fallbackImage: './assets/svg/blocked.svg'});
}))
attr = new VariableUiElement(preparedUrl.map(_ => new Attribution(imgSource.GetAttributionFor(urlSource), imgSource.SourceIcon())))
}
img = new Img(imageInfo.url);
attr = new Attribution(imageInfo.provider.GetAttributionFor(imageInfo.url),
imageInfo.provider.SourceIcon(),
)
super([img, attr]);

2
UI/Image/Attribution.ts
View file

@ -3,7 +3,7 @@ import Translations from "../i18n/Translations";
import BaseUIElement from "../BaseUIElement";
import {VariableUiElement} from "../Base/VariableUIElement";
import {UIEventSource} from "../../Logic/UIEventSource";
import {LicenseInfo} from "../../Logic/ImageProviders/Wikimedia";
import {LicenseInfo} from "../../Logic/ImageProviders/LicenseInfo";
export default class Attribution extends VariableUiElement {

49
UI/Image/ImageCarousel.ts
View file

@ -4,19 +4,20 @@ import Combine from "../Base/Combine";
import DeleteImage from "./DeleteImage";
import {AttributedImage} from "./AttributedImage";
import BaseUIElement from "../BaseUIElement";
import Img from "../Base/Img";
import Toggle from "../Input/Toggle";
import {Wikimedia} from "../../Logic/ImageProviders/Wikimedia";
import {Imgur} from "../../Logic/ImageProviders/Imgur";
import {Mapillary} from "../../Logic/ImageProviders/Mapillary";
import ImageProvider from "../../Logic/ImageProviders/ImageProvider";
export class ImageCarousel extends Toggle {
constructor(images: UIEventSource<{ key: string, url: string }[]>, tags: UIEventSource<any>) {
const uiElements = images.map((imageURLS: { key: string, url: string }[]) => {
constructor(images: UIEventSource<{ key: string, url: string, provider: ImageProvider }[]>, tags: UIEventSource<any>) {
const uiElements = images.map((imageURLS: { key: string, url: string, provider: ImageProvider }[]) => {
const uiElements: BaseUIElement[] = [];
for (const url of imageURLS) {
let image = ImageCarousel.CreateImageElement(url.url)
try {
let image = new AttributedImage(url)
if (url.key !== undefined) {
image = new Combine([
image,
@ -27,6 +28,11 @@ export class ImageCarousel extends Toggle {
.SetClass("w-full block")
.SetStyle("min-width: 50px; background: grey;")
uiElements.push(image);
} catch (e) {
console.error("Could not generate image element for", url.url, "due to", e)
}
}
return uiElements;
});
@ -38,33 +44,4 @@ export class ImageCarousel extends Toggle {
)
this.SetClass("block w-full");
}
/***
* Creates either a 'simpleimage' or a 'wikimediaimage' based on the string
* @param url
* @constructor
*/
private static CreateImageElement(url: string): BaseUIElement {
// @ts-ignore
let attrSource: ImageAttributionSource = undefined;
if (url.startsWith("File:")) {
attrSource = Wikimedia.singleton
} else if (url.toLowerCase().startsWith("https://commons.wikimedia.org/wiki/")) {
attrSource = Wikimedia.singleton;
} else if (url.toLowerCase().startsWith("https://i.imgur.com/")) {
attrSource = Imgur.singleton
} else if (url.toLowerCase().startsWith("https://www.mapillary.com/map/im/")) {
attrSource = Mapillary.singleton
} else {
return new Img(url);
}
try {
return new AttributedImage(url, attrSource)
} catch (e) {
console.error("Could not create an image: ", e)
return undefined;
}
}
}

7
UI/SpecialVisualizations.ts
View file

@ -13,20 +13,19 @@ import Translations from "./i18n/Translations";
import ReviewForm from "./Reviews/ReviewForm";
import OpeningHoursVisualization from "./OpeningHours/OpeningHoursVisualization";
import State from "../State";
import {ImageSearcher} from "../Logic/Actors/ImageSearcher";
import BaseUIElement from "./BaseUIElement";
import Title from "./Base/Title";
import Table from "./Base/Table";
import Histogram from "./BigComponents/Histogram";
import Loc from "../Models/Loc";
import {Utils} from "../Utils";
import BaseLayer from "../Models/BaseLayer";
import LayerConfig from "../Models/ThemeConfig/LayerConfig";
import ImportButton from "./BigComponents/ImportButton";
import {Tag} from "../Logic/Tags/Tag";
import StaticFeatureSource from "../Logic/FeatureSource/Sources/StaticFeatureSource";
import ShowDataMultiLayer from "./ShowDataLayer/ShowDataMultiLayer";
import Minimap from "./Base/Minimap";
import AllImageProviders from "../Logic/ImageProviders/AllImageProviders";
export interface SpecialVisualization {
funcName: string,
@ -76,9 +75,7 @@ export default class SpecialVisualizations {
constr: (state: State, tags, args) => {
const imagePrefix = args[0];
const loadSpecial = args[1].toLowerCase() === "true";
const searcher: UIEventSource<{ key: string, url: string }[]> = ImageSearcher.construct(tags, imagePrefix, loadSpecial);
return new ImageCarousel(searcher, tags);
return new ImageCarousel(AllImageProviders.LoadImagesFor(tags, imagePrefix, loadSpecial), tags);
}
},
{

2
assets/themes/natuurpunt/natuurpunt.json
View file

@ -75,7 +75,7 @@
"icon": {
"render": "circle:#FE6F32;./assets/themes/natuurpunt/nature_reserve.svg"
},
"presets": null
"presets": []
}
},
{

28
test/ImageSearcher.spec.ts
View file

@ -1,28 +0,0 @@
import {Utils} from "../Utils";
import {equal} from "assert";
import T from "./TestHelper";
import {UIEventSource} from "../Logic/UIEventSource";
import {ImageSearcher} from "../Logic/Actors/ImageSearcher";
Utils.runningFromConsole = true;
export default class ImageSearcherSpec extends T {
constructor() {
super("imagesearcher", [
[
"Should find images",
() => {
const tags = new UIEventSource({
"mapillary": "https://www.mapillary.com/app/?pKey=bYH6FFl8LXAPapz4PNSh3Q"
});
const searcher = ImageSearcher.construct(tags)
const result = searcher.data[0];
equal(result.url, "https://www.mapillary.com/map/im/bYH6FFl8LXAPapz4PNSh3Q");
}
],
]);
}
}

2
test/TestAll.ts
View file

@ -1,7 +1,6 @@
import TagSpec from "./Tag.spec";
import ImageAttributionSpec from "./ImageAttribution.spec";
import GeoOperationsSpec from "./GeoOperations.spec";
import ImageSearcherSpec from "./ImageSearcher.spec";
import ThemeSpec from "./Theme.spec";
import UtilsSpec from "./Utils.spec";
import OsmObjectSpec from "./OsmObject.spec";
@ -18,7 +17,6 @@ const allTests = [
new TagSpec(),
new ImageAttributionSpec(),
new GeoOperationsSpec(),
new ImageSearcherSpec(),
new ThemeSpec(),
new UtilsSpec(),
new UnitsSpec(),