midgard/mapcomplete
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Move imageAttributionSources around, improve fixTheme script

Browse source
This commit is contained in:
pietervdvn 2021-06-22 14:21:32 +02:00
parent 8bca83d708
commit aaaf876257
15 changed files with 125 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Logic/Actors/ImageSearcher.ts
View file

@ -1,4 +1,4 @@
import {ImagesInCategory, Wikidata, Wikimedia} from "../Web/Wikimedia";
import {ImagesInCategory, Wikidata, Wikimedia} from "../ImageProviders/Wikimedia";
import {UIEventSource} from "../UIEventSource";
/**

9
Logic/ImageProviders/AllImageProviders.ts Normal file
View file

@ -0,0 +1,9 @@
import {Mapillary} from "./Mapillary";
import {Wikimedia} from "./Wikimedia";
import {Imgur} from "./Imgur";
export default class AllImageProviders{
public static ImageAttributionSource = [Imgur.singleton, Mapillary.singleton, Wikimedia.singleton]
}

2
Logic/Web/ImageAttributionSource.ts → Logic/ImageProviders/ImageAttributionSource.ts
View file

@ -5,7 +5,6 @@ import BaseUIElement from "../../UI/BaseUIElement";
export default abstract class ImageAttributionSource {
private _cache = new Map<string, UIEventSource<LicenseInfo>>()
GetAttributionFor(url: string): UIEventSource<LicenseInfo> {
@ -22,6 +21,7 @@ export default abstract class ImageAttributionSource {
public abstract SourceIcon(backlinkSource?: string) : BaseUIElement;
protected abstract DownloadAttribution(url: string): UIEventSource<LicenseInfo>;
/*Converts a value to a URL. Can return null if not applicable*/
public PrepareUrl(value: string): string{
return value;
}

0
Logic/Web/Imgur.ts → Logic/ImageProviders/Imgur.ts
View file

0
Logic/Web/ImgurUploader.ts → Logic/ImageProviders/ImgurUploader.ts
View file

0
Logic/Web/Mapillary.ts → Logic/ImageProviders/Mapillary.ts
View file

36
Logic/Web/Wikimedia.ts → Logic/ImageProviders/Wikimedia.ts
View file

@ -4,6 +4,7 @@ import BaseUIElement from "../../UI/BaseUIElement";
import Svg from "../../Svg";
import {UIEventSource} from "../UIEventSource";
import Link from "../../UI/Base/Link";
import {Utils} from "../../Utils";
/**
* This module provides endpoints for wikipedia/wikimedia and others
@ -138,21 +139,28 @@ export class Wikimedia extends ImageAttributionSource {
"api.php?action=query&prop=imageinfo&iiprop=extmetadata&" +
"titles=" + filename +
"&format=json&origin=*";
console.log("Getting attribution at ", url)
$.getJSON(url, function (data) {
const licenseInfo = new LicenseInfo();
const license = data.query.pages[-1].imageinfo[0].extmetadata;
Utils.downloadJson(url).then(
data =>{
const licenseInfo = new LicenseInfo();
const license = (data.query.pages[-1].imageinfo ?? [])[0]?.extmetadata;
if(license === undefined){
console.error("This file has no usable metedata or license attached... Please fix the license info file yourself!")
source.setData(null)
return;
}
licenseInfo.artist = license.Artist?.value;
licenseInfo.license = license.License?.value;
licenseInfo.copyrighted = license.Copyrighted?.value;
licenseInfo.attributionRequired = license.AttributionRequired?.value;
licenseInfo.usageTerms = license.UsageTerms?.value;
licenseInfo.licenseShortName = license.LicenseShortName?.value;
licenseInfo.credit = license.Credit?.value;
licenseInfo.description = license.ImageDescription?.value;
source.setData(licenseInfo);
});
licenseInfo.artist = license.Artist?.value;
licenseInfo.license = license.License?.value;
licenseInfo.copyrighted = license.Copyrighted?.value;
licenseInfo.attributionRequired = license.AttributionRequired?.value;
licenseInfo.usageTerms = license.UsageTerms?.value;
licenseInfo.licenseShortName = license.LicenseShortName?.value;
licenseInfo.credit = license.Credit?.value;
licenseInfo.description = license.ImageDescription?.value;
source.setData(licenseInfo);
}
)
return source;
}

2
UI/Image/AttributedImage.ts
View file

@ -1,7 +1,7 @@
import Combine from "../Base/Combine";
import Attribution from "./Attribution";
import Img from "../Base/Img";
import ImageAttributionSource from "../../Logic/Web/ImageAttributionSource";
import ImageAttributionSource from "../../Logic/ImageProviders/ImageAttributionSource";
export class AttributedImage extends Combine {

2
UI/Image/Attribution.ts
View file

@ -3,7 +3,7 @@ import Translations from "../i18n/Translations";
import BaseUIElement from "../BaseUIElement";
import {VariableUiElement} from "../Base/VariableUIElement";
import {UIEventSource} from "../../Logic/UIEventSource";
import {LicenseInfo} from "../../Logic/Web/Wikimedia";
import {LicenseInfo} from "../../Logic/ImageProviders/Wikimedia";
export default class Attribution extends VariableUiElement {

7
UI/Image/ImageCarousel.ts
View file

@ -6,10 +6,9 @@ import {AttributedImage} from "./AttributedImage";
import BaseUIElement from "../BaseUIElement";
import Img from "../Base/Img";
import Toggle from "../Input/Toggle";
import ImageAttributionSource from "../../Logic/Web/ImageAttributionSource";
import {Wikimedia} from "../../Logic/Web/Wikimedia";
import {Mapillary} from "../../Logic/Web/Mapillary";
import {Imgur} from "../../Logic/Web/Imgur";
import {Wikimedia} from "../../Logic/ImageProviders/Wikimedia";
import {Imgur} from "../../Logic/ImageProviders/Imgur";
import {Mapillary} from "../../Logic/ImageProviders/Mapillary";
export class ImageCarousel extends Toggle {

2
UI/Image/ImageUploadFlow.ts
View file

@ -8,7 +8,7 @@ import BaseUIElement from "../BaseUIElement";
import LicensePicker from "../BigComponents/LicensePicker";
import Toggle from "../Input/Toggle";
import FileSelectorButton from "../Input/FileSelectorButton";
import ImgurUploader from "../../Logic/Web/ImgurUploader";
import ImgurUploader from "../../Logic/ImageProviders/ImgurUploader";
import UploadFlowStateUI from "../BigComponents/UploadFlowStateUI";
import LayerConfig from "../../Customizations/JSON/LayerConfig";

29
Utils.ts
View file

@ -1,5 +1,4 @@
import * as colors from "./assets/colors.json"
import {Util} from "leaflet";
export class Utils {
@ -324,6 +323,34 @@ export class Utils {
}
return result;
}
public static externalDownloadFunction: (url: string) => Promise<any>;
public static downloadJson(url: string): Promise<any>{
if(this.externalDownloadFunction !== undefined){
return this.externalDownloadFunction(url)
}
return new Promise(
(resolve, reject) => {
try{
const xhr = new XMLHttpRequest();
xhr.onload = () => {
if (xhr.status == 200) {
resolve(JSON.parse(xhr.response))
} else {
reject(xhr.statusText)
}
};
xhr.open('GET', url);
xhr.send();
}catch(e){
reject(e)
}
}
)
}
/**
* Triggers a 'download file' popup which will download the contents

27
scripts/ScriptUtils.ts
View file

@ -2,11 +2,20 @@ import {lstatSync, readdirSync, readFileSync} from "fs";
import * as https from "https";
import {LayerConfigJson} from "../Customizations/JSON/LayerConfigJson";
import {LayoutConfigJson} from "../Customizations/JSON/LayoutConfigJson";
import * as fs from "fs";
import {Utils} from "../Utils";
export default class ScriptUtils {
public static fixUtils() {
Utils.externalDownloadFunction = ScriptUtils.DownloadJSON
}
public static readDirRecSync(path, maxDepth = 999): string[] {
const result = []
if(maxDepth <= 0){
if (maxDepth <= 0) {
return []
}
for (const entry of readdirSync(path)) {
@ -23,6 +32,20 @@ export default class ScriptUtils {
return result;
}
public static DownloadFileTo(url, targetFilePath: string): void {
console.log("Downloading ", url, "to", targetFilePath)
https.get(url, (res) => {
const filePath = fs.createWriteStream(targetFilePath);
res.pipe(filePath);
filePath.on('finish', () => {
filePath.close();
console.log('Download Completed');
})
})
}
public static DownloadJSON(url): Promise<any> {
return new Promise((resolve, reject) => {
try {
@ -77,7 +100,7 @@ export default class ScriptUtils {
})
}
public static getThemeFiles() : {parsed: LayoutConfigJson, path: string}[] {
public static getThemeFiles(): { parsed: LayoutConfigJson, path: string }[] {
return ScriptUtils.readDirRecSync("./assets/themes")
.filter(path => path.endsWith(".json"))
.filter(path => path.indexOf("license_info.json") < 0)

48
scripts/fixTheme.ts
View file

@ -1,14 +1,18 @@
/*
* This script attempt to automatically fix some basic issues when a theme from the custom generator is loaded
*/
import {Utils} from "../Utils"
Utils.runningFromConsole = true;
import {readFileSync, writeFileSync} from "fs";
import {LayoutConfigJson} from "../Customizations/JSON/LayoutConfigJson";
import {Layer} from "leaflet";
import LayerConfig from "../Customizations/JSON/LayerConfig";
import SmallLicense from "../Models/smallLicense";
import ScriptUtils from "./ScriptUtils";
import AllImageProviders from "../Logic/ImageProviders/AllImageProviders";
ScriptUtils.fixUtils()
if(process.argv.length == 2){
console.log("USAGE: ts-node scripts/fixTheme <path to theme>")
@ -22,7 +26,6 @@ console.log("Fixing up ", path)
const themeConfigJson : LayoutConfigJson = JSON.parse(readFileSync(path, "UTF8"))
const linuxHints = []
const licenses : SmallLicense[] = []
const replacements: {source: string, destination: string}[] = []
@ -40,15 +43,32 @@ for (const layerConfigJson of themeConfigJson.layers) {
const layerConfig = new LayerConfig(layerConfigJson, true)
const images : string[] = Array.from(layerConfig.ExtractImages())
const remoteImages = images.filter(img => img.startsWith("http"))
for (const remoteImage of remoteImages) {
linuxHints.push("wget " + remoteImage)
const filename = remoteImage.substring(remoteImage.lastIndexOf("/"))
ScriptUtils.DownloadFileTo(remoteImage, dir + "/" + filename)
const imgPath = remoteImage.substring(remoteImage.lastIndexOf("/") + 1)
licenses.push({
path: imgPath,
license: "",
authors: [],
sources: [remoteImage]
})
for (const attributionSrc of AllImageProviders.ImageAttributionSource) {
try {
attributionSrc.GetAttributionFor(remoteImage).addCallbackAndRun(license => {
console.log("Downloaded an attribution!")
licenses.push({
path: imgPath,
license: license?.license ?? "",
authors:Utils.NoNull([license?.artist]),
sources: [remoteImage]
})
})
}catch(e){
// Hush hush
}
}
replacements.push({source: remoteImage, destination: `${dir}/${imgPath}`})
}
}
@ -58,13 +78,9 @@ for (const replacement of replacements) {
fixedThemeJson = fixedThemeJson.replace(new RegExp(replacement.source, "g"), replacement.destination)
}
const fixScriptPath = dir + "/fix_script_"+path.replace(/\//g,"_")+".sh"
writeFileSync(dir + "/generated.license_info.json", JSON.stringify(licenses, null, " "))
writeFileSync(fixScriptPath, linuxHints.join("\n"))
writeFileSync(path+".autofixed.json", fixedThemeJson)
console.log(`IMPORTANT:
1) run ${fixScriptPath}
2) Copy generated.license_info.json over into license_info.json and add the missing attributions and authors
3) Verify ${path}.autofixed.json as theme, and rename it to ${path}
4) Delete the fix script and other unneeded files`)
1) Copy generated.license_info.json over into license_info.json and add the missing attributions and authors
2) Verify ${path}.autofixed.json as theme, and rename it to ${path}`)

2
test.ts
View file

@ -9,7 +9,7 @@ import {SlideShow} from "./UI/Image/SlideShow";
import {FixedUiElement} from "./UI/Base/FixedUiElement";
import Img from "./UI/Base/Img";
import {AttributedImage} from "./UI/Image/AttributedImage";
import {Imgur} from "./Logic/Web/Imgur";
import {Imgur} from "./Logic/ImageProviders/Imgur";
import ReviewForm from "./UI/Reviews/ReviewForm";
import {OsmConnection} from "./Logic/Osm/OsmConnection";