Add validation scripts

scripts/ScriptUtils.ts

@@ -0,0 +1,20 @@
+import {lstatSync, readdirSync} from "fs";
+
+export default class ScriptUtils {
+    public static readDirRecSync(path): string[] {
+        const result = []
+        for (const entry of readdirSync(path)) {
+            const fullEntry = path + "/" + entry
+            const stats = lstatSync(fullEntry)
+            if (stats.isDirectory()) {
+                // Subdirectory
+                // @ts-ignore
+                result.push(...ScriptUtils.readDirRecSync(fullEntry))
+            } else {
+                result.push(fullEntry)
+            }
+        }
+        return result;
+    }
+
+}

scripts/generateLayerOverview.ts

@@ -0,0 +1,110 @@
+import ScriptUtils from "./ScriptUtils";
+import {Utils} from "../Utils";
+import {lstatSync, readdirSync, readFileSync, writeFileSync} from "fs";
+
+Utils.runningFromConsole = true
+import LayerConfig from "../Customizations/JSON/LayerConfig";
+import {error} from "util";
+import * as licenses from "../assets/generated/license_info.json"
+import SmallLicense from "../Models/smallLicense";
+import LayoutConfig from "../Customizations/JSON/LayoutConfig";
+// This scripts scans 'assets/layers/*.json' for layer definition files and 'assets/themes/*.json' for theme definition files.
+// It spits out an overview of those to be used to load them
+
+
+// First, remove the old file. It might be buggy!
+writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({
+    "layers": [],
+    "themes": []
+}))
+const layerFiles = ScriptUtils.readDirRecSync("./assets/layers")
+    .filter(path => path.indexOf(".json") > 0)
+    .filter(path => path.indexOf("license_info.json") < 0)
+    .map(path => {
+        try {
+            const parsed = JSON.parse(readFileSync(path, "UTF8"));
+            return parsed
+        } catch (e) {
+            console.error("Could not parse file ", path, "due to ", e)
+        }
+    })
+const themeFiles : any[] = ScriptUtils.readDirRecSync("./assets/themes")
+    .filter(path => path.indexOf(".json") > 0)
+    .filter(path => path.indexOf("license_info.json") < 0)
+    .map(path => {
+        return JSON.parse(readFileSync(path, "UTF8"));
+    })
+writeFileSync("./assets/generated/known_layers_and_themes.json", JSON.stringify({
+    "layers": layerFiles,
+    "themes": themeFiles
+}))
+
+
+console.log("Discovered ", layerFiles.length, "layers and ", themeFiles.length, "themes\n")
+console.log("   ---------- VALIDATING ---------")
+// ------------- VALIDATION --------------
+const licensePaths = []
+for (const i in licenses) {
+    licensePaths.push(licenses[i].path)
+}
+const knownPaths = new Set<string>(licensePaths)
+
+function validateLayer(layer: LayerConfig, context?:string) : number{
+    let errorCount = 0;
+    const images = Array.from(layer.ExtractImages())
+    const remoteImages = images.filter(img => img.indexOf("http") == 0)
+    for (const remoteImage of remoteImages) {
+        console.error("Found a remote image:", remoteImage, "in layer", layer.id)
+        errorCount++
+    }
+    for (const image of images) {
+        if (!knownPaths.has(image)) {
+            console.error("Image with path", image, "not found or not attributed; it is used in", layer.id, context === undefined ? "" : ` in a layer defined in the theme ${context}`)
+            errorCount++
+        }
+    }
+    return errorCount
+}
+let layerErrorCount = 0
+const knownLayerIds = new Set<string>();
+for (const layerFile of layerFiles) {
+    knownLayerIds.add(layerFile.id)
+    try {
+        const layer = new LayerConfig(layerFile, "test", true)
+       layerErrorCount += validateLayer(layer)
+    } catch (e) {
+        console.error("Layer ", layerFile.id, " is invalid: ", e)
+        layerErrorCount++
+    }
+    if (layerFile.overpassTags !== undefined) {
+        layerErrorCount++
+        console.warn("Layer ", layerFile.id, "still uses the old 'overpassTags'-format. Please use 'source: {osmTags: <tags>}' instead")
+    }
+}
+
+let themeErrorCount = 0
+for (const themeFile of themeFiles) {
+
+    for (const layer of themeFile.layers) {
+        if(typeof layer === "string"){
+            if(!knownLayerIds.has(layer)){
+                console.error("Unknown layer id: ", layer)
+                themeErrorCount++
+            }
+        }
+    }
+    
+    themeFile.layers = themeFile.layers.filter(l => typeof l != "string")
+    
+    try {
+        const layout = new LayoutConfig(themeFile, true, "test")
+        for (const layer of layout.layers) {
+            layerErrorCount += validateLayer(layer, layout.id)
+        }
+    } catch (e) {
+        console.error("Could not parse theme", themeFile["id"], "due to", e)
+        themeErrorCount++;
+    }
+}
+
+console.log("Found ", layerErrorCount, "errors in the layers; "+themeErrorCount+" errors in the themes")

scripts/generateLayouts.ts

@@ -234,7 +234,7 @@ if (!existsSync(generatedDir)) {
 }
 
 const blacklist = ["", "test", ".", "..", "manifest", "index", "land", "preferences", "account", "openstreetmap", "custom"]
-const all = AllKnownLayouts.allSets;
+const all = AllKnownLayouts.allKnownLayouts;
 
 for (const layoutName in all) {
     if (blacklist.indexOf(layoutName.toLowerCase()) >= 0) {

scripts/generateLicenseInfo.ts

@@ -1,6 +1,7 @@
 import {Utils} from "../Utils";
 import {lstatSync, readdirSync, readFileSync, writeFileSync} from "fs";
 import SmallLicense from "../Models/smallLicense";
+import ScriptUtils from "./ScriptUtils";
 
 Utils.runningFromConsole = true;
 
@@ -9,22 +10,6 @@ Utils.runningFromConsole = true;
  * Checks that the license info is included for each of them and generates a compiles license_info.json for those
  */
 
-function readDirRecSync(path): string[] {
-    const result = []
-    for (const entry of readdirSync(path)) {
-        const fullEntry = path + "/" + entry
-        const stats = lstatSync(fullEntry)
-        if (stats.isDirectory()) {
-            // Subdirectory
-            // @ts-ignore
-            result.push(...readDirRecSync(fullEntry))
-        } else {
-            result.push(fullEntry)
-        }
-    }
-    return result;
-}
-
 function generateLicenseInfos(paths: string[]): SmallLicense[] {
     const licenses = []
     for (const path of paths) {
@@ -69,28 +54,28 @@ const prompt = require('prompt-sync')();
 
 const knownLicenses = new Map<string, SmallLicense>()
 knownLicenses.set("cf", {
-    authors: ["Pieter Fiers", "Thibault Declercq", "Pierre Barban", "Joost Schouppe","Pieter Vander Vennet"],
+    authors: ["Pieter Fiers", "Thibault Declercq", "Pierre Barban", "Joost Schouppe", "Pieter Vander Vennet"],
     path: undefined,
     license: "CC-BY-SA",
     sources: ["https://osoc.be/editions/2020/cyclofix"]
 })
 knownLicenses.set("me", {
     authors: ["Pieter Vander Vennet"],
-    path : undefined,
+    path: undefined,
     license: "CC0",
     sources: []
 })
 
 knownLicenses.set("t", {
     authors: [],
-    path : undefined,
+    path: undefined,
     license: "CC0; trivial",
     sources: []
 })
 
 knownLicenses.set("na", {
     authors: [],
-    path : undefined,
+    path: undefined,
     license: "CC0",
     sources: []
 })
@@ -103,21 +88,21 @@ function promptLicenseFor(path): SmallLicense {
     })
     const author = prompt("What is the author for artwork " + path + "? (or: [Q]uit, [S]kip)  > ")
     path = path.substring(path.lastIndexOf("/") + 1)
- 
-    if(knownLicenses.has(author)){
+
+    if (knownLicenses.has(author)) {
         const license = knownLicenses.get(author);
         license.path = path;
         return license;
     }
 
-    if(author == "s"){
+    if (author == "s") {
         return null;
     }
     if (author == "Q" || author == "q" || author == "") {
         throw "Quitting now!"
     }
     let authors = author.split(";")
-    if(author.toLowerCase() == "none"){
+    if (author.toLowerCase() == "none") {
         authors = []
     }
     return {
@@ -130,7 +115,7 @@ function promptLicenseFor(path): SmallLicense {
 
 function createLicenseInfoFor(path): void {
     const li = promptLicenseFor(path);
-    if(li == null){
+    if (li == null) {
         return;
     }
     writeFileSync(path + ".license_info.json", JSON.stringify(li, null, "  "))
@@ -156,7 +141,7 @@ function shuffle(array) {
 }
 
 console.log("Checking and compiling license info")
-const contents = readDirRecSync("./assets")
+const contents = ScriptUtils.readDirRecSync("./assets")
     .filter(entry => entry.indexOf("./assets/generated") != 0)
 const licensePaths = contents.filter(entry => entry.indexOf("license_info.json") >= 0)
 const licenseInfos = generateLicenseInfos(licensePaths);
@@ -169,7 +154,7 @@ console.log(`There are ${missingLicenses.length} licenses missing.`, missingLice
 
 // shuffle(missingLicenses)
 
-process.on('SIGINT', function() {
+process.on('SIGINT', function () {
     console.log("Aborting... Bye!");
     process.exit();
 });
@@ -178,8 +163,8 @@ let i = 1;
 for (const missingLicens of missingLicenses) {
     console.log(i + " / " + missingLicenses.length)
     i++;
-    if(i < missingLicenses.length - 5){
-    //    continue
+    if (i < missingLicenses.length - 5) {
+        //    continue
     }
     createLicenseInfoFor(missingLicens)
 }