import Script from "./Script" import { Overpass } from "../src/Logic/Osm/Overpass" import { RegexTag } from "../src/Logic/Tags/RegexTag" import Constants from "../src/Models/Constants" import { BBox } from "../src/Logic/BBox" import { existsSync, readFileSync, writeFileSync } from "fs" import { PanoramaxUploader } from "../src/Logic/ImageProviders/Panoramax" import { Feature } from "geojson" import { LicenseInfo } from "../src/Logic/ImageProviders/LicenseInfo" import { GeoOperations } from "../src/Logic/GeoOperations" import { Tag } from "../src/Logic/Tags/Tag" import { Utils } from "../src/Utils" import ChangeTagAction from "../src/Logic/Osm/Actions/ChangeTagAction" import { And } from "../src/Logic/Tags/And" import { Changes } from "../src/Logic/Osm/Changes" import { ChangeDescription } from "../src/Logic/Osm/Actions/ChangeDescription" import OsmObjectDownloader from "../src/Logic/Osm/OsmObjectDownloader" import { OsmObject } from "../src/Logic/Osm/OsmObject" import { createReadStream } from "node:fs" import { File } from 'buffer'; import { open } from 'node:fs/promises'; import { UploadableTag } from "../src/Logic/Tags/TagTypes" export class ImgurToPanoramax extends Script { private readonly panoramax = new PanoramaxUploader(Constants.panoramax.url, Constants.panoramax.token) private _imageDirectory: string private _licenseDirectory: string private readonly sequenceIds = { test: "7f34cf53-27ff-46c9-ac22-78511fa8457a", cc0: "f0d6f78a-ff95-4db1-8494-6eb44a17bb37", ccby: "288a8052-b475-422c-811a-4f6f1a00015e", ccbysa: "f3d02893-b4c1-4cd6-8b27-e27ab57eb59a", } as const constructor() { super( "Queries OSM for 'imgur'-images, uploads them to Panoramax and creates a changeset to update OSM", ) } async uploadImage(key: string, feat: Feature, sequences: ({ id: string; "stats:items": { count: number } })[]): Promise { const v = feat.properties[key] if (!v) { return undefined } const imageHash = v.split("/").at(-1).split(".").at(0) let path: string = undefined if (existsSync(this._imageDirectory + "/" + imageHash + ".jpg")) { path = this._imageDirectory + "/" + imageHash + ".jpg" } else if (existsSync(this._imageDirectory + "/" + imageHash + ".jpeg")) { path = this._imageDirectory + "/" + imageHash + ".jpeg" } if (!path) { return undefined } const licensePath = this._licenseDirectory + "/" + v.replaceAll(/[^a-zA-Z0-9]/g, "_") + ".json" if (!existsSync(licensePath)) { return undefined } const licenseText: LicenseInfo = JSON.parse(readFileSync(licensePath, "utf8")) if (!licenseText.licenseShortName) { console.log("No license found for", path, licenseText) return undefined } const license = licenseText.licenseShortName.toLowerCase().split(" ")[0].replace(/-/g, "") const sequence = this.sequenceIds[license] const author = licenseText.artist const handle = await open(path); const stat = await handle.stat(); class MyFile extends File { // we should set correct size // otherwise we will encounter UND_ERR_REQ_CONTENT_LENGTH_MISMATCH size = stat.size; stream = undefined } const file = new MyFile([], path) file.stream = function() { return handle.readableWebStream(); }; console.log("Uploading", imageHash, sequence) const result = await this.panoramax.uploadImage( file, GeoOperations.centerpointCoordinates(feat), author, true, sequence) await handle.close() return new And([new Tag(key.replace("image", result.key), result.value), new Tag(key,"")]) } async main(args: string[]): Promise { this._imageDirectory = args[0] ?? "/home/pietervdvn/data/imgur-image-backup" this._licenseDirectory = args[1] ?? "/home/pietervdvn/git/MapComplete-data/ImageLicenseInfo" const bounds = new BBox([[3.6984301050112833, 51.06715570450848], [3.7434328399847914, 51.039379568816145]]) const maxcount = 100 const filter = new RegexTag("image", /^https:\/\/i.imgur.com\/.*/) const overpass = new Overpass(filter, [], Constants.defaultOverpassUrls[0]) const features = (await overpass.queryGeoJson(bounds))[0].features let converted = 0 const pano = this.panoramax.panoramax const sequences = await pano.mySequences() const changes: ChangeDescription[] = [] do { const f = features.shift() if (!f) { break } const changedTags: (UploadableTag | undefined)[] = [] for (const k of ["image", "image:menu", "image:streetsign"]) { changedTags.push(await this.uploadImage(k, f, sequences)) for (let i = 0; i < 20; i++) { changedTags.push( await this.uploadImage(k + ":" + i, f, sequences), ) } } const action = new ChangeTagAction(f.properties.id, new And(Utils.NoNull(changedTags)), f.properties, { theme: "image-mover", changeType: "link-image", }, ) changes.push(...await action.CreateChangeDescriptions()) converted++ } while (converted < maxcount) const modif: string[] = Utils.Dedup(changes.map(ch => ch.type + "/" + ch.id)) const modifiedObjectsFresh = (await Promise.all(modif.map(id => new OsmObjectDownloader().DownloadObjectAsync(id)))) .filter(m => m !== "deleted") const modifiedObjects = Changes.createChangesetObjectsStatic( changes, modifiedObjectsFresh,false, []) const cs = Changes.buildChangesetXML("0", modifiedObjects) writeFileSync("imgur_to_panoramax.osc", cs, "utf8") } } new ImgurToPanoramax().run()