From 3079bbb934e0688319290b0253fb6b519c914deb Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Sat, 24 Aug 2024 01:50:55 +0200 Subject: [PATCH] Improve script to handle errors --- scripts/handleErrors.ts | 164 +++++++++++++++++++++++---------------- src/Logic/Osm/Changes.ts | 6 +- 2 files changed, 101 insertions(+), 69 deletions(-) diff --git a/scripts/handleErrors.ts b/scripts/handleErrors.ts index ac9b2c8f0..0546b3334 100644 --- a/scripts/handleErrors.ts +++ b/scripts/handleErrors.ts @@ -1,5 +1,5 @@ import Script from "./Script" -import { readFileSync, writeFileSync } from "fs" +import { appendFileSync, readFileSync, writeFile, writeFileSync } from "fs" import { ChangeDescription } from "../src/Logic/Osm/Actions/ChangeDescription" import { Changes } from "../src/Logic/Osm/Changes" import { OsmObject } from "../src/Logic/Osm/OsmObject" @@ -28,6 +28,91 @@ class HandleErrors extends Script { constructor() { super("Inspects the errors made on a given day. Argument: path to errors") } + + private readonly ignoreUsers = new Set([]) + + private async handleError(parsed: ErrorMessage, changesObj: Changes, downloader: OsmObjectDownloader, createdChangesets: Set, refusedFiles: Set) { + console.log( + parsed.message.username, + parsed.message.layout, + parsed.message.message, + parsed.date, + ) + + const e = parsed.message + const neededIds = Changes.GetNeededIds(e.pendingChanges) + // We _do not_ pass in the Changes object itself - we want the data from OSM directly in order to apply the changes + const osmObjects: { id: string; osmObj: OsmObject | "deleted" }[] = await Promise.all<{ + id: string + osmObj: OsmObject | "deleted" + }>( + neededIds.map(async (id) => { + try { + + const osmObj = await downloader.DownloadObjectAsync(id) + return ({ + id, + osmObj, + }) + } catch (e) { + console.error("COULD NOT DOWNLOAD OBJECT", id) + return { + id, + osmObj: "deleted", + } + } + }), + ) + + const objects = osmObjects + .filter((obj) => obj.osmObj !== "deleted") + .map((obj) => obj.osmObj) + + const { toUpload, refused } = changesObj.fragmentChanges(e.pendingChanges, objects) + + const changes: { + newObjects: OsmObject[] + modifiedObjects: OsmObject[] + deletedObjects: OsmObject[] + } = changesObj.CreateChangesetObjects(toUpload, objects, true) + + const changeset = Changes.createChangesetFor("", changes) + const path = + "error_changeset_" + parsed.index + "_" + e.layout + "_" + e.username + ".osc" + if ( + changeset === + `` + ) { + /*console.log( + "Changes for " + parsed.index + ": empty changeset, not creating a file for it" + )*/ + } else if (createdChangesets.has(changeset)) { + /* console.log( + "Changeset " + + parsed.index + + " is identical to previously seen changeset, not writing to file" + )*/ + } else { + const changesetWithMsg = ` +${changeset}` + writeFileSync(path, changesetWithMsg, "utf8") + createdChangesets.add(changeset) + console.log("Written", path, "with " + e.pendingChanges.length + " changes") + } + const refusedContent = JSON.stringify(refused) + if (refusedFiles.has(refusedContent)) { + /* console.log( + "Refused changes for " + + parsed.index + + " is identical to previously seen changeset, not writing to file" + )*/ + } else { + writeFileSync(path + ".refused.json", refusedContent, "utf8") + refusedFiles.add(refusedContent) + console.log("Written refused", path) + } + } + async main(args: string[]): Promise { const osmConnection = new OsmConnection() const downloader = new OsmObjectDownloader(osmConnection.Backend(), undefined) @@ -45,7 +130,7 @@ class HandleErrors extends Script { osmConnection, }, false, - (err) => console.error(err) + (err) => console.error(err), ) const all: ErrorMessage[] = [] @@ -60,11 +145,14 @@ class HandleErrors extends Script { console.log("Skipping GRB ") continue } + if (this.ignoreUsers.has(e.username)) { + continue + } for (const pendingChange of e.pendingChanges) { console.log( "\t https://osm.org/" + pendingChange.type + "/" + pendingChange.id, pendingChange.meta.changeType, - pendingChange.doDelete ? "DELETE" : "" + pendingChange.doDelete ? "DELETE" : "", ) } all.push(parsed) @@ -74,72 +162,12 @@ class HandleErrors extends Script { } for (const parsed of all) { - console.log( - parsed.message.username, - parsed.message.layout, - parsed.message.message, - parsed.date - ) + try { - const e = parsed.message - const neededIds = Changes.GetNeededIds(e.pendingChanges) - // We _do not_ pass in the Changes object itself - we want the data from OSM directly in order to apply the changes - const osmObjects: { id: string; osmObj: OsmObject | "deleted" }[] = await Promise.all<{ - id: string - osmObj: OsmObject | "deleted" - }>( - neededIds.map(async (id) => ({ - id, - osmObj: await downloader.DownloadObjectAsync(id), - })) - ) - - const objects = osmObjects - .filter((obj) => obj.osmObj !== "deleted") - .map((obj) => obj.osmObj) - - const { toUpload, refused } = changesObj.fragmentChanges(e.pendingChanges, objects) - - const changes: { - newObjects: OsmObject[] - modifiedObjects: OsmObject[] - deletedObjects: OsmObject[] - } = changesObj.CreateChangesetObjects(toUpload, objects) - - const changeset = Changes.createChangesetFor("", changes) - const path = - "error_changeset_" + parsed.index + "_" + e.layout + "_" + e.username + ".osc" - if ( - changeset === - `` - ) { - /*console.log( - "Changes for " + parsed.index + ": empty changeset, not creating a file for it" - )*/ - } else if (createdChangesets.has(changeset)) { - /* console.log( - "Changeset " + - parsed.index + - " is identical to previously seen changeset, not writing to file" - )*/ - } else { - const changesetWithMsg = ` -${changeset}` - writeFileSync(path, changesetWithMsg, "utf8") - createdChangesets.add(changeset) - console.log("Written", path, "with " + e.pendingChanges.length + " changes") - } - const refusedContent = JSON.stringify(refused) - if (refusedFiles.has(refusedContent)) { - /* console.log( - "Refused changes for " + - parsed.index + - " is identical to previously seen changeset, not writing to file" - )*/ - } else { - writeFileSync(path + ".refused.json", refusedContent, "utf8") - refusedFiles.add(refusedContent) - console.log("Written refused", path) + await this.handleError(parsed, changesObj, downloader, createdChangesets, refusedFiles) + } catch (e) { + console.error("ERROR: could not handle ", parsed, " due to", e) + writeFileSync("ERRORS."+parsed.index, "ERROR: due to " + e + ": could not handle\n" + JSON.stringify(parsed), "utf8") } } } diff --git a/src/Logic/Osm/Changes.ts b/src/Logic/Osm/Changes.ts index a13d1e03e..8c303fbbb 100644 --- a/src/Logic/Osm/Changes.ts +++ b/src/Logic/Osm/Changes.ts @@ -266,7 +266,8 @@ export class Changes { public CreateChangesetObjects( changes: ChangeDescription[], - downloadedOsmObjects: OsmObject[] + downloadedOsmObjects: OsmObject[], + ignoreNoCreate: boolean = false ): { newObjects: OsmObject[] modifiedObjects: OsmObject[] @@ -314,6 +315,9 @@ export class Changes { } if (change.changes === undefined) { // This object is a change to a newly created object. However, we have not seen the creation changedescription yet! + if(ignoreNoCreate){ + continue + } throw "Not a creation of the object: " + JSON.stringify(change) } // This is a new object that should be created