From aaaf876257df921955f462f076505c8ae7202f3f Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Tue, 22 Jun 2021 14:21:32 +0200 Subject: [PATCH] Move imageAttributionSources around, improve fixTheme script --- Logic/Actors/ImageSearcher.ts | 2 +- Logic/ImageProviders/AllImageProviders.ts | 9 ++++ .../ImageAttributionSource.ts | 2 +- Logic/{Web => ImageProviders}/Imgur.ts | 0 .../{Web => ImageProviders}/ImgurUploader.ts | 0 Logic/{Web => ImageProviders}/Mapillary.ts | 0 Logic/{Web => ImageProviders}/Wikimedia.ts | 36 ++++++++------ UI/Image/AttributedImage.ts | 2 +- UI/Image/Attribution.ts | 2 +- UI/Image/ImageCarousel.ts | 7 ++- UI/Image/ImageUploadFlow.ts | 2 +- Utils.ts | 29 ++++++++++- scripts/ScriptUtils.ts | 27 ++++++++++- scripts/fixTheme.ts | 48 ++++++++++++------- test.ts | 2 +- 15 files changed, 125 insertions(+), 43 deletions(-) create mode 100644 Logic/ImageProviders/AllImageProviders.ts rename Logic/{Web => ImageProviders}/ImageAttributionSource.ts (92%) rename Logic/{Web => ImageProviders}/Imgur.ts (100%) rename Logic/{Web => ImageProviders}/ImgurUploader.ts (100%) rename Logic/{Web => ImageProviders}/Mapillary.ts (100%) rename Logic/{Web => ImageProviders}/Wikimedia.ts (82%) diff --git a/Logic/Actors/ImageSearcher.ts b/Logic/Actors/ImageSearcher.ts index 3ea510152..165442d6f 100644 --- a/Logic/Actors/ImageSearcher.ts +++ b/Logic/Actors/ImageSearcher.ts @@ -1,4 +1,4 @@ -import {ImagesInCategory, Wikidata, Wikimedia} from "../Web/Wikimedia"; +import {ImagesInCategory, Wikidata, Wikimedia} from "../ImageProviders/Wikimedia"; import {UIEventSource} from "../UIEventSource"; /** diff --git a/Logic/ImageProviders/AllImageProviders.ts b/Logic/ImageProviders/AllImageProviders.ts new file mode 100644 index 000000000..89ae0a142 --- /dev/null +++ b/Logic/ImageProviders/AllImageProviders.ts @@ -0,0 +1,9 @@ +import {Mapillary} from "./Mapillary"; +import {Wikimedia} from "./Wikimedia"; +import {Imgur} from "./Imgur"; + +export default class AllImageProviders{ + + public static ImageAttributionSource = [Imgur.singleton, Mapillary.singleton, Wikimedia.singleton] + +} \ No newline at end of file diff --git a/Logic/Web/ImageAttributionSource.ts b/Logic/ImageProviders/ImageAttributionSource.ts similarity index 92% rename from Logic/Web/ImageAttributionSource.ts rename to Logic/ImageProviders/ImageAttributionSource.ts index 689a32c46..451909cfb 100644 --- a/Logic/Web/ImageAttributionSource.ts +++ b/Logic/ImageProviders/ImageAttributionSource.ts @@ -5,7 +5,6 @@ import BaseUIElement from "../../UI/BaseUIElement"; export default abstract class ImageAttributionSource { - private _cache = new Map>() GetAttributionFor(url: string): UIEventSource { @@ -22,6 +21,7 @@ export default abstract class ImageAttributionSource { public abstract SourceIcon(backlinkSource?: string) : BaseUIElement; protected abstract DownloadAttribution(url: string): UIEventSource; + /*Converts a value to a URL. Can return null if not applicable*/ public PrepareUrl(value: string): string{ return value; } diff --git a/Logic/Web/Imgur.ts b/Logic/ImageProviders/Imgur.ts similarity index 100% rename from Logic/Web/Imgur.ts rename to Logic/ImageProviders/Imgur.ts diff --git a/Logic/Web/ImgurUploader.ts b/Logic/ImageProviders/ImgurUploader.ts similarity index 100% rename from Logic/Web/ImgurUploader.ts rename to Logic/ImageProviders/ImgurUploader.ts diff --git a/Logic/Web/Mapillary.ts b/Logic/ImageProviders/Mapillary.ts similarity index 100% rename from Logic/Web/Mapillary.ts rename to Logic/ImageProviders/Mapillary.ts diff --git a/Logic/Web/Wikimedia.ts b/Logic/ImageProviders/Wikimedia.ts similarity index 82% rename from Logic/Web/Wikimedia.ts rename to Logic/ImageProviders/Wikimedia.ts index 4668f3511..9d74b7769 100644 --- a/Logic/Web/Wikimedia.ts +++ b/Logic/ImageProviders/Wikimedia.ts @@ -4,6 +4,7 @@ import BaseUIElement from "../../UI/BaseUIElement"; import Svg from "../../Svg"; import {UIEventSource} from "../UIEventSource"; import Link from "../../UI/Base/Link"; +import {Utils} from "../../Utils"; /** * This module provides endpoints for wikipedia/wikimedia and others @@ -138,21 +139,28 @@ export class Wikimedia extends ImageAttributionSource { "api.php?action=query&prop=imageinfo&iiprop=extmetadata&" + "titles=" + filename + "&format=json&origin=*"; - console.log("Getting attribution at ", url) - $.getJSON(url, function (data) { - const licenseInfo = new LicenseInfo(); - const license = data.query.pages[-1].imageinfo[0].extmetadata; + Utils.downloadJson(url).then( + data =>{ + const licenseInfo = new LicenseInfo(); + const license = (data.query.pages[-1].imageinfo ?? [])[0]?.extmetadata; + if(license === undefined){ + console.error("This file has no usable metedata or license attached... Please fix the license info file yourself!") + source.setData(null) + return; + } - licenseInfo.artist = license.Artist?.value; - licenseInfo.license = license.License?.value; - licenseInfo.copyrighted = license.Copyrighted?.value; - licenseInfo.attributionRequired = license.AttributionRequired?.value; - licenseInfo.usageTerms = license.UsageTerms?.value; - licenseInfo.licenseShortName = license.LicenseShortName?.value; - licenseInfo.credit = license.Credit?.value; - licenseInfo.description = license.ImageDescription?.value; - source.setData(licenseInfo); - }); + licenseInfo.artist = license.Artist?.value; + licenseInfo.license = license.License?.value; + licenseInfo.copyrighted = license.Copyrighted?.value; + licenseInfo.attributionRequired = license.AttributionRequired?.value; + licenseInfo.usageTerms = license.UsageTerms?.value; + licenseInfo.licenseShortName = license.LicenseShortName?.value; + licenseInfo.credit = license.Credit?.value; + licenseInfo.description = license.ImageDescription?.value; + source.setData(licenseInfo); + } + ) + return source; } diff --git a/UI/Image/AttributedImage.ts b/UI/Image/AttributedImage.ts index 260c5306f..43bfc3ea2 100644 --- a/UI/Image/AttributedImage.ts +++ b/UI/Image/AttributedImage.ts @@ -1,7 +1,7 @@ import Combine from "../Base/Combine"; import Attribution from "./Attribution"; import Img from "../Base/Img"; -import ImageAttributionSource from "../../Logic/Web/ImageAttributionSource"; +import ImageAttributionSource from "../../Logic/ImageProviders/ImageAttributionSource"; export class AttributedImage extends Combine { diff --git a/UI/Image/Attribution.ts b/UI/Image/Attribution.ts index 514bc475a..0fddcc6f3 100644 --- a/UI/Image/Attribution.ts +++ b/UI/Image/Attribution.ts @@ -3,7 +3,7 @@ import Translations from "../i18n/Translations"; import BaseUIElement from "../BaseUIElement"; import {VariableUiElement} from "../Base/VariableUIElement"; import {UIEventSource} from "../../Logic/UIEventSource"; -import {LicenseInfo} from "../../Logic/Web/Wikimedia"; +import {LicenseInfo} from "../../Logic/ImageProviders/Wikimedia"; export default class Attribution extends VariableUiElement { diff --git a/UI/Image/ImageCarousel.ts b/UI/Image/ImageCarousel.ts index a28d89a2c..7c5d90335 100644 --- a/UI/Image/ImageCarousel.ts +++ b/UI/Image/ImageCarousel.ts @@ -6,10 +6,9 @@ import {AttributedImage} from "./AttributedImage"; import BaseUIElement from "../BaseUIElement"; import Img from "../Base/Img"; import Toggle from "../Input/Toggle"; -import ImageAttributionSource from "../../Logic/Web/ImageAttributionSource"; -import {Wikimedia} from "../../Logic/Web/Wikimedia"; -import {Mapillary} from "../../Logic/Web/Mapillary"; -import {Imgur} from "../../Logic/Web/Imgur"; +import {Wikimedia} from "../../Logic/ImageProviders/Wikimedia"; +import {Imgur} from "../../Logic/ImageProviders/Imgur"; +import {Mapillary} from "../../Logic/ImageProviders/Mapillary"; export class ImageCarousel extends Toggle { diff --git a/UI/Image/ImageUploadFlow.ts b/UI/Image/ImageUploadFlow.ts index cf5881f64..58d9a3760 100644 --- a/UI/Image/ImageUploadFlow.ts +++ b/UI/Image/ImageUploadFlow.ts @@ -8,7 +8,7 @@ import BaseUIElement from "../BaseUIElement"; import LicensePicker from "../BigComponents/LicensePicker"; import Toggle from "../Input/Toggle"; import FileSelectorButton from "../Input/FileSelectorButton"; -import ImgurUploader from "../../Logic/Web/ImgurUploader"; +import ImgurUploader from "../../Logic/ImageProviders/ImgurUploader"; import UploadFlowStateUI from "../BigComponents/UploadFlowStateUI"; import LayerConfig from "../../Customizations/JSON/LayerConfig"; diff --git a/Utils.ts b/Utils.ts index 111939b1a..fa3e55ef8 100644 --- a/Utils.ts +++ b/Utils.ts @@ -1,5 +1,4 @@ import * as colors from "./assets/colors.json" -import {Util} from "leaflet"; export class Utils { @@ -324,6 +323,34 @@ export class Utils { } return result; } + + public static externalDownloadFunction: (url: string) => Promise; + + public static downloadJson(url: string): Promise{ + if(this.externalDownloadFunction !== undefined){ + return this.externalDownloadFunction(url) + } + + return new Promise( + (resolve, reject) => { + try{ + const xhr = new XMLHttpRequest(); + xhr.onload = () => { + if (xhr.status == 200) { + resolve(JSON.parse(xhr.response)) + } else { + reject(xhr.statusText) + } + }; + xhr.open('GET', url); + xhr.send(); + }catch(e){ + reject(e) + } + } + ) + + } /** * Triggers a 'download file' popup which will download the contents diff --git a/scripts/ScriptUtils.ts b/scripts/ScriptUtils.ts index 220c9e280..86c358414 100644 --- a/scripts/ScriptUtils.ts +++ b/scripts/ScriptUtils.ts @@ -2,11 +2,20 @@ import {lstatSync, readdirSync, readFileSync} from "fs"; import * as https from "https"; import {LayerConfigJson} from "../Customizations/JSON/LayerConfigJson"; import {LayoutConfigJson} from "../Customizations/JSON/LayoutConfigJson"; +import * as fs from "fs"; +import {Utils} from "../Utils"; export default class ScriptUtils { + + + public static fixUtils() { + Utils.externalDownloadFunction = ScriptUtils.DownloadJSON + } + + public static readDirRecSync(path, maxDepth = 999): string[] { const result = [] - if(maxDepth <= 0){ + if (maxDepth <= 0) { return [] } for (const entry of readdirSync(path)) { @@ -23,6 +32,20 @@ export default class ScriptUtils { return result; } + public static DownloadFileTo(url, targetFilePath: string): void { + console.log("Downloading ", url, "to", targetFilePath) + https.get(url, (res) => { + const filePath = fs.createWriteStream(targetFilePath); + res.pipe(filePath); + filePath.on('finish', () => { + filePath.close(); + console.log('Download Completed'); + }) + + + }) + } + public static DownloadJSON(url): Promise { return new Promise((resolve, reject) => { try { @@ -77,7 +100,7 @@ export default class ScriptUtils { }) } - public static getThemeFiles() : {parsed: LayoutConfigJson, path: string}[] { + public static getThemeFiles(): { parsed: LayoutConfigJson, path: string }[] { return ScriptUtils.readDirRecSync("./assets/themes") .filter(path => path.endsWith(".json")) .filter(path => path.indexOf("license_info.json") < 0) diff --git a/scripts/fixTheme.ts b/scripts/fixTheme.ts index b3d040469..e11bce26e 100644 --- a/scripts/fixTheme.ts +++ b/scripts/fixTheme.ts @@ -1,14 +1,18 @@ - /* * This script attempt to automatically fix some basic issues when a theme from the custom generator is loaded */ import {Utils} from "../Utils" Utils.runningFromConsole = true; + import {readFileSync, writeFileSync} from "fs"; import {LayoutConfigJson} from "../Customizations/JSON/LayoutConfigJson"; -import {Layer} from "leaflet"; import LayerConfig from "../Customizations/JSON/LayerConfig"; import SmallLicense from "../Models/smallLicense"; +import ScriptUtils from "./ScriptUtils"; +import AllImageProviders from "../Logic/ImageProviders/AllImageProviders"; + + +ScriptUtils.fixUtils() if(process.argv.length == 2){ console.log("USAGE: ts-node scripts/fixTheme ") @@ -22,7 +26,6 @@ console.log("Fixing up ", path) const themeConfigJson : LayoutConfigJson = JSON.parse(readFileSync(path, "UTF8")) -const linuxHints = [] const licenses : SmallLicense[] = [] const replacements: {source: string, destination: string}[] = [] @@ -40,15 +43,32 @@ for (const layerConfigJson of themeConfigJson.layers) { const layerConfig = new LayerConfig(layerConfigJson, true) const images : string[] = Array.from(layerConfig.ExtractImages()) const remoteImages = images.filter(img => img.startsWith("http")) + for (const remoteImage of remoteImages) { - linuxHints.push("wget " + remoteImage) + + + const filename = remoteImage.substring(remoteImage.lastIndexOf("/")) + ScriptUtils.DownloadFileTo(remoteImage, dir + "/" + filename) + + const imgPath = remoteImage.substring(remoteImage.lastIndexOf("/") + 1) - licenses.push({ - path: imgPath, - license: "", - authors: [], - sources: [remoteImage] - }) + + for (const attributionSrc of AllImageProviders.ImageAttributionSource) { + try { + attributionSrc.GetAttributionFor(remoteImage).addCallbackAndRun(license => { + console.log("Downloaded an attribution!") + licenses.push({ + path: imgPath, + license: license?.license ?? "", + authors:Utils.NoNull([license?.artist]), + sources: [remoteImage] + }) + }) + }catch(e){ + // Hush hush + } + } + replacements.push({source: remoteImage, destination: `${dir}/${imgPath}`}) } } @@ -58,13 +78,9 @@ for (const replacement of replacements) { fixedThemeJson = fixedThemeJson.replace(new RegExp(replacement.source, "g"), replacement.destination) } -const fixScriptPath = dir + "/fix_script_"+path.replace(/\//g,"_")+".sh" writeFileSync(dir + "/generated.license_info.json", JSON.stringify(licenses, null, " ")) -writeFileSync(fixScriptPath, linuxHints.join("\n")) writeFileSync(path+".autofixed.json", fixedThemeJson) console.log(`IMPORTANT: - 1) run ${fixScriptPath} - 2) Copy generated.license_info.json over into license_info.json and add the missing attributions and authors - 3) Verify ${path}.autofixed.json as theme, and rename it to ${path} - 4) Delete the fix script and other unneeded files`) \ No newline at end of file + 1) Copy generated.license_info.json over into license_info.json and add the missing attributions and authors + 2) Verify ${path}.autofixed.json as theme, and rename it to ${path}`) \ No newline at end of file diff --git a/test.ts b/test.ts index 1736e2aa9..cdc00c93f 100644 --- a/test.ts +++ b/test.ts @@ -9,7 +9,7 @@ import {SlideShow} from "./UI/Image/SlideShow"; import {FixedUiElement} from "./UI/Base/FixedUiElement"; import Img from "./UI/Base/Img"; import {AttributedImage} from "./UI/Image/AttributedImage"; -import {Imgur} from "./Logic/Web/Imgur"; +import {Imgur} from "./Logic/ImageProviders/Imgur"; import ReviewForm from "./UI/Reviews/ReviewForm"; import {OsmConnection} from "./Logic/Osm/OsmConnection";