Add support for smaller theme encodings

InitUiElements.ts

@@ -36,6 +36,8 @@ import Translations from "./UI/i18n/Translations";
 import MapControlButton from "./UI/MapControlButton";
 import Combine from "./UI/Base/Combine";
 import SelectedFeatureHandler from "./Logic/Actors/SelectedFeatureHandler";
+import LZString from "lz-string";
+import {LayoutConfigJson} from "./Customizations/JSON/LayoutConfigJson";
 
 export class InitUiElements {
 
@@ -209,7 +211,17 @@ export class InitUiElements {
                 hashFromLocalStorage.setData(hash);
                 dedicatedHashFromLocalStorage.setData(hash);
             }
-            const layoutToUse = new LayoutConfig(JSON.parse(atob(hash)), false);
+            
+            let json = {}
+            try{
+               json = JSON.parse(atob(hash));
+            } catch (e) {
+                // We try to decode with lz-string
+                json = JSON.parse( Utils.UnMinify(LZString.decompressFromBase64(hash)))
+            }
+           
+            // @ts-ignore
+            const layoutToUse = new LayoutConfig(json, false);
             userLayoutParam.setData(layoutToUse.id);
             return layoutToUse;
         } catch (e) {

UI/CustomGenerator/CustomGeneratorPanel.ts

@@ -17,7 +17,8 @@ import {LocalStorageSource} from "../../Logic/Web/LocalStorageSource";
 import HelpText from "./HelpText";
 import Svg from "../../Svg";
 import Constants from "../../Models/Constants";
-
+import LZString from "lz-string";
+import {Utils} from "../../Utils";
 
 export default class CustomGeneratorPanel extends UIElement {
     private mainPanel: UIElement;
@@ -40,7 +41,7 @@ export default class CustomGeneratorPanel extends UIElement {
 
     private InitMainPanel(layout: LayoutConfigJson, userDetails: UserDetails, connection: OsmConnection) {
         const es = new UIEventSource(layout);
-        const encoded = es.map(config => btoa(JSON.stringify(config)));
+        const encoded = es.map(config => LZString.compressToBase64(Utils.MinifyJSON(JSON.stringify(config, null, 0))));
         encoded.addCallback(encoded => LocalStorageSource.Get("last-custom-theme"))
         const liveUrl = encoded.map(encoded => `./index.html?userlayout=${es.data.id}#${encoded}`)
         const testUrl = encoded.map(encoded => `./index.html?test=true&userlayout=${es.data.id}#${encoded}`)

Utils.ts

@@ -10,6 +10,8 @@ export class Utils {
     public static runningFromConsole = false;
 
     public static readonly assets_path = "./assets/svg/";
+    private static knownKeys = ["addExtraTags", "and", "calculatedTags", "changesetmessage", "clustering", "color", "condition", "customCss", "dashArray", "defaultBackgroundId", "description", "descriptionTail", "doNotDownload", "enableAddNewPoints", "enableBackgroundLayerSelection", "enableGeolocation", "enableLayers", "enableMoreQuests", "enableSearch", "enableShareScreen", "enableUserBadge", "freeform", "hideFromOverview", "hideInAnswer", "icon", "iconOverlays", "iconSize", "id", "if", "ifnot", "isShown", "key", "language", "layers", "lockLocation", "maintainer", "mappings", "maxzoom", "maxZoom", "minNeededElements", "minzoom", "multiAnswer", "name", "or", "osmTags", "passAllFeatures", "presets", "question", "render", "roaming", "roamingRenderings", "rotation", "shortDescription", "socialImage", "source", "startLat", "startLon", "startZoom", "tagRenderings", "tags", "then", "title", "titleIcons", "type", "version", "wayHandling", "widenFactor", "width"]
+    private static extraKeys = ["nl", "en", "fr", "de", "pt", "es", "name", "phone", "email", "amenity", "leisure", "highway", "building", "yes", "no", "true", "false"]
 
     static EncodeXmlValue(str) {
         return str.replace(/&/g, '&')
@@ -202,6 +204,42 @@ export class Utils {
         return {x: Utils.lon2tile(lon, z), y: Utils.lat2tile(lat, z), z: z}
     }
 
+    public static MinifyJSON(stringified: string): string {
+        stringified = stringified.replace(/\|/g, "||");
+
+        const keys = Utils.knownKeys.concat(Utils.extraKeys);
+        for (let i = 0; i < keys.length; i++) {
+            const knownKey = keys[i];
+            let code = i;
+            if (i >= 124) {
+                code += 1; // Character 127 is our 'escape' character |
+            }
+            let replacement = "|" + String.fromCharCode(code)
+            stringified = stringified.replace(new RegExp(`\"${knownKey}\":`, "g"), replacement);
+        }
+
+        return stringified;
+    }
+
+    public static UnMinify(minified: string): string {
+
+        const parts = minified.split("|");
+        let result = parts.shift();
+        const keys = Utils.knownKeys.concat(Utils.extraKeys);
+
+        for (const part of parts) {
+            if (part == "") {
+                // Empty string => this was a || originally
+                result += "|"
+                continue
+            }
+            const i = part.charCodeAt(0);
+            result += "\"" + keys[i] + "\":" + part.substring(1)
+        }
+
+        return result;
+    }
+
     private static tile2long(x, z) {
         return (x / Math.pow(2, z) * 360 - 180);
     }

package-lock.json

@@ -4018,6 +4018,11 @@
         "@types/leaflet": "*"
       }
     },
+    "@types/lz-string": {
+      "version": "1.3.34",
+      "resolved": "https://registry.npmjs.org/@types/lz-string/-/lz-string-1.3.34.tgz",
+      "integrity": "sha512-j6G1e8DULJx3ONf6NdR5JiR2ZY3K3PaaqiEuKYkLQO0Czfi1AzrtjfnfCROyWGeDd5IVMKCwsgSmMip9OWijow=="
+    },
     "@types/node": {
       "version": "7.10.14",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-7.10.14.tgz",
@@ -7943,6 +7948,11 @@
       "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.7.3.tgz",
       "integrity": "sha1-bUUk6LlV+V1PW1iFHOId1y+06VI="
     },
+    "lz-string": {
+      "version": "1.4.4",
+      "resolved": "https://registry.npmjs.org/lz-string/-/lz-string-1.4.4.tgz",
+      "integrity": "sha1-wNjq82BZ9wV5bh40SBHPTEmNOiY="
+    },
     "magic-string": {
       "version": "0.22.5",
       "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.22.5.tgz",

package.json

@@ -37,6 +37,7 @@
     "@types/leaflet-markercluster": "^1.0.3",
     "@types/leaflet-providers": "^1.2.0",
     "@types/leaflet.markercluster": "^1.4.3",
+    "@types/lz-string": "^1.3.34",
     "autoprefixer": "^9.8.6",
     "country-language": "^0.1.7",
     "email-validator": "^2.0.4",
@@ -49,6 +50,7 @@
     "leaflet.markercluster": "^1.4.1",
     "libphonenumber": "0.0.10",
     "libphonenumber-js": "^1.7.55",
+    "lz-string": "^1.4.4",
     "mangrove-reviews": "^0.1.3",
     "moment": "^2.29.0",
     "opening_hours": "^3.5.0",

scripts/genKeys.sh

@@ -0,0 +1,12 @@
+#! /bin/bash
+
+# Generates all the keys that are frequently used in the JSON in order to compress them
+touch keys.csv
+for f in ../Customizations/JSON/*Json.ts
+do
+    echo "$f"
+    cat $f | tr -d "[]{}," | sed "s/^[ \t]*//" | grep -v "^/\?\*" | grep -v "import \.*" | grep -v "^export" | sed "s/?\?:.*//" >> keys.csv
+done
+cat keys.csv | wc -l
+cat keys.csv | sort | uniq | sed "s/^\(.*\)$/\"\1\",/" | tr -d "\n"
+rm keys.csv

test/Utils.spec.ts

@@ -0,0 +1,50 @@
+import T from "./TestHelper";
+import {Utils} from "../Utils";
+import {equal} from "assert";
+import {existsSync, mkdirSync, readFileSync, writeFile, writeFileSync} from "fs";
+import LZString from "lz-string";
+new T("Utils",[
+    ["Minify-json",() => {
+        const str = JSON.stringify({title: "abc", "and":"xyz", "render":"somevalue"}, null, 0);
+        const minified = Utils.MinifyJSON(str);
+        console.log(minified)
+        console.log("Minified version has ", minified.length, "chars")
+        const restored = Utils.UnMinify(minified)
+        console.log(restored)
+        console.log("Restored version has ", restored.length, "chars")
+        equal(str, restored)
+
+    }],
+    ["Minify-json of the bookcases",() => {
+        let str = readFileSync("/home/pietervdvn/git/MapComplete/assets/layers/public_bookcases/public_bookcases.json", "UTF8")
+        str = JSON.stringify(JSON.parse(str), null, 0)
+        const minified = Utils.MinifyJSON(str);
+        console.log("Minified version has ", minified.length, "chars")
+        const restored = Utils.UnMinify(minified)
+        console.log("Restored version has ", restored.length, "chars")
+        equal(str, restored)
+
+    }],
+    ["Minify-json with LZ-string of the bookcases",() => {
+        let str = readFileSync("/home/pietervdvn/git/MapComplete/assets/layers/public_bookcases/public_bookcases.json", "UTF8")
+        str = JSON.stringify(JSON.parse(str), null, 0)
+        const minified =LZString.compressToBase64(Utils.MinifyJSON(str));
+        console.log("Minified version has ", minified.length, "chars")
+        const restored = Utils.UnMinify(LZString.decompressFromBase64(minified))
+        console.log("Restored version has ", restored.length, "chars")
+        equal(str, restored)
+
+    }],
+    ["Minify-json with only LZ-string of the bookcases",() => {
+        let str = readFileSync("/home/pietervdvn/git/MapComplete/assets/layers/public_bookcases/public_bookcases.json", "UTF8")
+        str = JSON.stringify(JSON.parse(str), null, 0)
+        const minified =LZString.compressToBase64(str);
+        console.log("Minified version has ", minified.length, "chars")
+        const restored = LZString.decompressFromBase64(minified)
+        console.log("Restored version has ", restored.length, "chars")
+        equal(str, restored)
+
+    }]
+    
+    
+])