From e16def1464141dd957569a97bac99f91775cf273 Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Sat, 10 Apr 2021 03:18:32 +0200 Subject: [PATCH] Add validation scripts --- scripts/ScriptUtils.ts | 20 ++++++ scripts/generateLayerOverview.ts | 110 +++++++++++++++++++++++++++++++ scripts/generateLayouts.ts | 2 +- scripts/generateLicenseInfo.ts | 43 ++++-------- 4 files changed, 145 insertions(+), 30 deletions(-) create mode 100644 scripts/ScriptUtils.ts create mode 100644 scripts/generateLayerOverview.ts diff --git a/scripts/ScriptUtils.ts b/scripts/ScriptUtils.ts new file mode 100644 index 0000000..36ecc24 --- /dev/null +++ b/scripts/ScriptUtils.ts @@ -0,0 +1,20 @@ +import {lstatSync, readdirSync} from "fs"; + +export default class ScriptUtils { + public static readDirRecSync(path): string[] { + const result = [] + for (const entry of readdirSync(path)) { + const fullEntry = path + "/" + entry + const stats = lstatSync(fullEntry) + if (stats.isDirectory()) { + // Subdirectory + // @ts-ignore + result.push(...ScriptUtils.readDirRecSync(fullEntry)) + } else { + result.push(fullEntry) + } + } + return result; + } + +} diff --git a/scripts/generateLayerOverview.ts b/scripts/generateLayerOverview.ts new file mode 100644 index 0000000..66dc6ef --- /dev/null +++ b/scripts/generateLayerOverview.ts @@ -0,0 +1,110 @@ +import ScriptUtils from "./ScriptUtils"; +import {Utils} from "../Utils"; +import {lstatSync, readdirSync, readFileSync, writeFileSync} from "fs"; + +Utils.runningFromConsole = true +import LayerConfig from "../Customizations/JSON/LayerConfig"; +import {error} from "util"; +import * as licenses from "../assets/generated/license_info.json" +import SmallLicense from "../Models/smallLicense"; +import LayoutConfig from "../Customizations/JSON/LayoutConfig"; +// This scripts scans 'assets/layers/*.json' for layer definition files and 'assets/themes/*.json' for theme definition files. +// It spits out an overview of those to be used to load them + + +// First, remove the old file. It might be buggy! +writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({ + "layers": [], + "themes": [] +})) +const layerFiles = ScriptUtils.readDirRecSync("./assets/layers") + .filter(path => path.indexOf(".json") > 0) + .filter(path => path.indexOf("license_info.json") < 0) + .map(path => { + try { + const parsed = JSON.parse(readFileSync(path, "UTF8")); + return parsed + } catch (e) { + console.error("Could not parse file ", path, "due to ", e) + } + }) +const themeFiles : any[] = ScriptUtils.readDirRecSync("./assets/themes") + .filter(path => path.indexOf(".json") > 0) + .filter(path => path.indexOf("license_info.json") < 0) + .map(path => { + return JSON.parse(readFileSync(path, "UTF8")); + }) +writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({ + "layers": layerFiles, + "themes": themeFiles +})) + + +console.log("Discovered ", layerFiles.length, "layers and ", themeFiles.length, "themes\n") +console.log(" ---------- VALIDATING ---------") +// ------------- VALIDATION -------------- +const licensePaths = [] +for (const i in licenses) { + licensePaths.push(licenses[i].path) +} +const knownPaths = new Set(licensePaths) + +function validateLayer(layer: LayerConfig, context?:string) : number{ + let errorCount = 0; + const images = Array.from(layer.ExtractImages()) + const remoteImages = images.filter(img => img.indexOf("http") == 0) + for (const remoteImage of remoteImages) { + console.error("Found a remote image:", remoteImage, "in layer", layer.id) + errorCount++ + } + for (const image of images) { + if (!knownPaths.has(image)) { + console.error("Image with path", image, "not found or not attributed; it is used in", layer.id, context === undefined ? "" : ` in a layer defined in the theme ${context}`) + errorCount++ + } + } + return errorCount +} +let layerErrorCount = 0 +const knownLayerIds = new Set(); +for (const layerFile of layerFiles) { + knownLayerIds.add(layerFile.id) + try { + const layer = new LayerConfig(layerFile, "test", true) + layerErrorCount += validateLayer(layer) + } catch (e) { + console.error("Layer ", layerFile.id, " is invalid: ", e) + layerErrorCount++ + } + if (layerFile.overpassTags !== undefined) { + layerErrorCount++ + console.warn("Layer ", layerFile.id, "still uses the old 'overpassTags'-format. Please use 'source: {osmTags: }' instead") + } +} + +let themeErrorCount = 0 +for (const themeFile of themeFiles) { + + for (const layer of themeFile.layers) { + if(typeof layer === "string"){ + if(!knownLayerIds.has(layer)){ + console.error("Unknown layer id: ", layer) + themeErrorCount++ + } + } + } + + themeFile.layers = themeFile.layers.filter(l => typeof l != "string") + + try { + const layout = new LayoutConfig(themeFile, true, "test") + for (const layer of layout.layers) { + layerErrorCount += validateLayer(layer, layout.id) + } + } catch (e) { + console.error("Could not parse theme", themeFile["id"], "due to", e) + themeErrorCount++; + } +} + +console.log("Found ", layerErrorCount, "errors in the layers; "+themeErrorCount+" errors in the themes") \ No newline at end of file diff --git a/scripts/generateLayouts.ts b/scripts/generateLayouts.ts index b51b251..4cfa1c1 100644 --- a/scripts/generateLayouts.ts +++ b/scripts/generateLayouts.ts @@ -234,7 +234,7 @@ if (!existsSync(generatedDir)) { } const blacklist = ["", "test", ".", "..", "manifest", "index", "land", "preferences", "account", "openstreetmap", "custom"] -const all = AllKnownLayouts.allSets; +const all = AllKnownLayouts.allKnownLayouts; for (const layoutName in all) { if (blacklist.indexOf(layoutName.toLowerCase()) >= 0) { diff --git a/scripts/generateLicenseInfo.ts b/scripts/generateLicenseInfo.ts index 700ee05..204b276 100644 --- a/scripts/generateLicenseInfo.ts +++ b/scripts/generateLicenseInfo.ts @@ -1,6 +1,7 @@ import {Utils} from "../Utils"; import {lstatSync, readdirSync, readFileSync, writeFileSync} from "fs"; import SmallLicense from "../Models/smallLicense"; +import ScriptUtils from "./ScriptUtils"; Utils.runningFromConsole = true; @@ -9,22 +10,6 @@ Utils.runningFromConsole = true; * Checks that the license info is included for each of them and generates a compiles license_info.json for those */ -function readDirRecSync(path): string[] { - const result = [] - for (const entry of readdirSync(path)) { - const fullEntry = path + "/" + entry - const stats = lstatSync(fullEntry) - if (stats.isDirectory()) { - // Subdirectory - // @ts-ignore - result.push(...readDirRecSync(fullEntry)) - } else { - result.push(fullEntry) - } - } - return result; -} - function generateLicenseInfos(paths: string[]): SmallLicense[] { const licenses = [] for (const path of paths) { @@ -69,28 +54,28 @@ const prompt = require('prompt-sync')(); const knownLicenses = new Map() knownLicenses.set("cf", { - authors: ["Pieter Fiers", "Thibault Declercq", "Pierre Barban", "Joost Schouppe","Pieter Vander Vennet"], + authors: ["Pieter Fiers", "Thibault Declercq", "Pierre Barban", "Joost Schouppe", "Pieter Vander Vennet"], path: undefined, license: "CC-BY-SA", sources: ["https://osoc.be/editions/2020/cyclofix"] }) knownLicenses.set("me", { authors: ["Pieter Vander Vennet"], - path : undefined, + path: undefined, license: "CC0", sources: [] }) knownLicenses.set("t", { authors: [], - path : undefined, + path: undefined, license: "CC0; trivial", sources: [] }) knownLicenses.set("na", { authors: [], - path : undefined, + path: undefined, license: "CC0", sources: [] }) @@ -103,21 +88,21 @@ function promptLicenseFor(path): SmallLicense { }) const author = prompt("What is the author for artwork " + path + "? (or: [Q]uit, [S]kip) > ") path = path.substring(path.lastIndexOf("/") + 1) - - if(knownLicenses.has(author)){ + + if (knownLicenses.has(author)) { const license = knownLicenses.get(author); license.path = path; return license; } - if(author == "s"){ + if (author == "s") { return null; } if (author == "Q" || author == "q" || author == "") { throw "Quitting now!" } let authors = author.split(";") - if(author.toLowerCase() == "none"){ + if (author.toLowerCase() == "none") { authors = [] } return { @@ -130,7 +115,7 @@ function promptLicenseFor(path): SmallLicense { function createLicenseInfoFor(path): void { const li = promptLicenseFor(path); - if(li == null){ + if (li == null) { return; } writeFileSync(path + ".license_info.json", JSON.stringify(li, null, " ")) @@ -156,7 +141,7 @@ function shuffle(array) { } console.log("Checking and compiling license info") -const contents = readDirRecSync("./assets") +const contents = ScriptUtils.readDirRecSync("./assets") .filter(entry => entry.indexOf("./assets/generated") != 0) const licensePaths = contents.filter(entry => entry.indexOf("license_info.json") >= 0) const licenseInfos = generateLicenseInfos(licensePaths); @@ -169,7 +154,7 @@ console.log(`There are ${missingLicenses.length} licenses missing.`, missingLice // shuffle(missingLicenses) -process.on('SIGINT', function() { +process.on('SIGINT', function () { console.log("Aborting... Bye!"); process.exit(); }); @@ -178,8 +163,8 @@ let i = 1; for (const missingLicens of missingLicenses) { console.log(i + " / " + missingLicenses.length) i++; - if(i < missingLicenses.length - 5){ - // continue + if (i < missingLicenses.length - 5) { + // continue } createLicenseInfoFor(missingLicens) }