diff --git a/package.json b/package.json index c289219f1..2597f4c84 100644 --- a/package.json +++ b/package.json @@ -111,7 +111,7 @@ "housekeeping": "git pull && npx update-browserslist-db@latest && npm run weblate-fix-heavy && npm run generate && npm run generate:docs && npm run generate:contributor-list && vite-node scripts/fetchLanguages.ts && vite-node scripts/generateSunnyUnlabeled.ts && npm run format && git add assets/ langs/ Docs/ **/*.ts Docs/* src/* && git commit -m 'chore: automated housekeeping...'", "reuse-compliance": "reuse lint", "backup:images": "vite-node scripts/generateImageAnalysis.ts -- ~/data/imgur-image-backup/", - "downloadNsiLogos": "vite-node scripts/downloadNsiLogos.ts", + "downloadNsiLogos": "vite-node scripts/downloadNsiLogos.ts || npm run downloadNsiLogos # This script crashes often without the possibility to correct - hence the auto retry with OR", "dloadVelopark": "vite-node scripts/velopark/veloParkToGeojson.ts ", "compareVelopark": "vite-node scripts/velopark/compare.ts -- velopark_nonsynced_.geojson ~/Projecten/OSM/Fietsberaad/2024-02-02\\ Fietsenstallingen_OSM_met_velopark_ref.geojson\n", "scrapeWebsites": "vite-node scripts/importscripts/compareWebsiteData.ts -- ~/Downloads/ShopsWithWebsiteNodes.csv ~/data/scraped_websites/", diff --git a/scripts/downloadNsiLogos.ts b/scripts/downloadNsiLogos.ts index 7253ddac4..7413039cd 100644 --- a/scripts/downloadNsiLogos.ts +++ b/scripts/downloadNsiLogos.ts @@ -15,7 +15,7 @@ class DownloadNsiLogos extends Script { return await this.downloadLogoUnsafe(nsiItem, type, basePath) } catch (e) { console.error("Could not download", nsiItem.displayName, "due to", e) - return false + return "error" } } @@ -39,7 +39,7 @@ class DownloadNsiLogos extends Script { return false } if (logos.facebook) { - // Facebook logo's are generally better and square + // Facebook's logos are generally better and square await ScriptUtils.DownloadFileTo(logos.facebook, path) return true } @@ -88,19 +88,32 @@ class DownloadNsiLogos extends Script { const items = NameSuggestionIndex.allPossible(type) const basePath = "./public/assets/data/nsi/logos/" let downloadCount = 0 - const stepcount = 100 + const stepcount = 5 for (let i = 0; i < items.length; i += stepcount) { - if (i % 100 === 0) { + if (downloadCount > 0 || i % 200 === 0) { console.log(i + "/" + items.length, "downloaded " + downloadCount) } - await Promise.all( + + const results = await Promise.all( Utils.TimesT(stepcount, (j) => j).map(async (j) => { const downloaded = await this.downloadLogo(items[i + j], type, basePath) if (downloaded) { downloadCount++ } + return downloaded }) ) + for (let j = 0; j < results.length; j++) { + let didDownload = results[j] + if(didDownload !== "error"){ + continue + } + console.log("Retrying", items[i + j].id, type) + didDownload = await this.downloadLogo(items[i + j], type, basePath) + if(didDownload === "error"){ + console.log("Failed again:", items[i + j].id) + } + } } } }