From 14e8606ad6fd8282402e17603b78cecc128ec31b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20K=C3=BCrten?= Date: Thu, 6 May 2021 18:11:17 +0200 Subject: [PATCH 1/6] Fix a few typos in the README --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 02d65c663..e75ed3704 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ A typical user journey would be: * Note that _all messages_ must be read before being able to add a point. * In other words, sending a message to a misbehaving MapComplete user acts as having a **zero-day-block**. This is added deliberately to make sure new users _have_ to read feedback from the community. -4. At 50 changesets, the [personal layout](https://pietervdvn.github.io/MapComplete/personal.html) is advertised. The personal theme is a theme where contributors can pick layers from all the offical themes. Note that the personal theme is always available. +4. At 50 changesets, the [personal layout](https://pietervdvn.github.io/MapComplete/personal.html) is advertised. The personal theme is a theme where contributors can pick layers from all the official themes. Note that the personal theme is always available. 5. At 200 changesets, the tags become visible when answering questions and when adding a new point from a preset. This is to give more control to power users and to teach new users the tagging scheme @@ -131,7 +131,7 @@ Images are fetched from: Images are uploaded to Imgur, as their API was way easier to handle. The URL is written into the changes. -The idea is that once in a while, the images are transfered to wikipedia or that we hook up wikimedia directly (but I need some help in getting their API working). +The idea is that once in a while, the images are transferred to wikipedia or that we hook up wikimedia directly (but I need some help in getting their API working). ### Uploading changes @@ -149,7 +149,7 @@ All documentation can be found in [here](Docs/) Privacy is important, we try to leak as little information as possible. All major personal information is handled by OSM. -Geolocation is available on mobile only throught hte device's GPS location (so no geolocation is sent of to Google). +Geolocation is available on mobile only through the device's GPS location (so no geolocation is sent of to Google). TODO: erase cookies of third party websites and API's From 55eedb4185302c96744c01d37b2224a17c9d59e5 Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Fri, 7 May 2021 02:06:08 +0200 Subject: [PATCH 2/6] Fix initial setup scripts --- scripts/generateLicenseInfo.ts | 60 ++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/scripts/generateLicenseInfo.ts b/scripts/generateLicenseInfo.ts index c34d73ecc..db8e818bd 100644 --- a/scripts/generateLicenseInfo.ts +++ b/scripts/generateLicenseInfo.ts @@ -1,5 +1,5 @@ import {Utils} from "../Utils"; -import {lstatSync, readdirSync, readFileSync, writeFileSync, unlinkSync} from "fs"; +import {lstatSync, readdirSync, readFileSync, writeFileSync, unlinkSync, existsSync, mkdir, mkdirSync} from "fs"; import SmallLicense from "../Models/smallLicense"; import ScriptUtils from "./ScriptUtils"; @@ -13,28 +13,29 @@ Utils.runningFromConsole = true; function generateLicenseInfos(paths: string[]): SmallLicense[] { const licenses = [] for (const path of paths) { - try{ - - - const parsed = JSON.parse(readFileSync(path, "UTF-8")) - if (Array.isArray(parsed)) { - const l: SmallLicense[] = parsed - for (const smallLicens of l) { - smallLicens.path = path.substring(0, path.length - "license_info.json".length) + smallLicens.path - } - licenses.push(...l) - } else { - const smallLicens: SmallLicense = parsed; - /*if(parsed.license === "CC-BY"){ - console.log("Rewriting ", path) - parsed.license === "CC-BY 4.0" - writeFileSync(path, JSON.stringify(smallLicens, null, " ")) - }*/ + try { - smallLicens.path = path.substring(0, 1 + path.lastIndexOf("/")) + smallLicens.path - licenses.push(smallLicens) - }}catch(e){ - console.error("Error: ",e, "while handling",path) + + const parsed = JSON.parse(readFileSync(path, "UTF-8")) + if (Array.isArray(parsed)) { + const l: SmallLicense[] = parsed + for (const smallLicens of l) { + smallLicens.path = path.substring(0, path.length - "license_info.json".length) + smallLicens.path + } + licenses.push(...l) + } else { + const smallLicens: SmallLicense = parsed; + /*if(parsed.license === "CC-BY"){ + console.log("Rewriting ", path) + parsed.license === "CC-BY 4.0" + writeFileSync(path, JSON.stringify(smallLicens, null, " ")) + }*/ + + smallLicens.path = path.substring(0, 1 + path.lastIndexOf("/")) + smallLicens.path + licenses.push(smallLicens) + } + } catch (e) { + console.error("Error: ", e, "while handling", path) } } @@ -195,6 +196,9 @@ const contents = ScriptUtils.readDirRecSync("./assets") const licensePaths = contents.filter(entry => entry.indexOf("license_info.json") >= 0) const licenseInfos = generateLicenseInfos(licensePaths); +if (!existsSync("./assets/generated")) { + mkdirSync("./assets/generated") +} writeFileSync("./assets/generated/license_info.json", JSON.stringify(licenseInfos, null, " ")) @@ -203,13 +207,13 @@ const missingLicenses = missingLicenseInfos(licenseInfos, artwork) const invalidLicenses = licenseInfos.filter(l => (l.license ?? "") === "").map(l => `License for artwork ${l.path} is empty string or undefined`) for (const licenseInfo of licenseInfos) { for (const source of licenseInfo.sources) { - if(source == ""){ - invalidLicenses.push("Invalid license: empty string in "+JSON.stringify(licenseInfo)) + if (source == "") { + invalidLicenses.push("Invalid license: empty string in " + JSON.stringify(licenseInfo)) } - try{ + try { new URL(source); - }catch{ - invalidLicenses.push("Not a valid URL: "+source) + } catch { + invalidLicenses.push("Not a valid URL: " + source) } } } @@ -218,7 +222,7 @@ if (process.argv.indexOf("--prompt") >= 0 || process.argv.indexOf("--query") >= } if (missingLicenses.length > 0) { const msg = `There are ${missingLicenses.length} licenses missing and ${invalidLicenses.length} invalid licenses.` - console.log( missingLicenses.concat(invalidLicenses).join("\n")) + console.log(missingLicenses.concat(invalidLicenses).join("\n")) console.error(msg) if (process.argv.indexOf("--report") >= 0) { console.log("Writing report!") From e47e23701ea5f6d744556d7eab6f96c000ebc8cb Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Fri, 7 May 2021 02:08:03 +0200 Subject: [PATCH 3/6] Fix init script --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9ac25f6f3..7e88630fc 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "increase-memory": "export NODE_OPTIONS=--max_old_space_size=4096", "start": "ts-node scripts/generateLayerOverview.ts --no-fail && npm run increase-memory && parcel *.html UI/** Logic/** assets/** assets/**/** assets/**/**/** vendor/* vendor/*/*", "test": "ts-node test/Tag.spec.ts && ts-node test/TagQuestion.spec.ts && ts-node test/ImageSearcher.spec.ts && ts-node test/ImageAttribution.spec.ts && ts-node test/Theme.spec.ts", - "init": "npm run generate:editor-layer-index && npm run generate:licenses && npm run generate:layeroverview && npm run generate:layouts && npm run clean", + "init": "npm run generate:editor-layer-index && npm run generate:licenses && npm run reset:layeroverview && npm run generate:layeroverview && npm run generate:layouts && npm run clean", "generate:editor-layer-index": "cd assets/ && wget https://osmlab.github.io/editor-layer-index/imagery.geojson --output-document=editor-layer-index.json", "generate:images": "ts-node scripts/generateIncludedImages.ts", "generate:translations": "ts-node scripts/generateTranslations.ts", From be01c55dd14e679ed462ba61b68a75c2ddb67677 Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Fri, 7 May 2021 02:15:56 +0200 Subject: [PATCH 4/6] Cleanup of package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7e88630fc..c20ae305a 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "increase-memory": "export NODE_OPTIONS=--max_old_space_size=4096", "start": "ts-node scripts/generateLayerOverview.ts --no-fail && npm run increase-memory && parcel *.html UI/** Logic/** assets/** assets/**/** assets/**/**/** vendor/* vendor/*/*", "test": "ts-node test/Tag.spec.ts && ts-node test/TagQuestion.spec.ts && ts-node test/ImageSearcher.spec.ts && ts-node test/ImageAttribution.spec.ts && ts-node test/Theme.spec.ts", - "init": "npm run generate:editor-layer-index && npm run generate:licenses && npm run reset:layeroverview && npm run generate:layeroverview && npm run generate:layouts && npm run clean", + "init": "npm run generate && npm run generate:layouts && npm run clean", "generate:editor-layer-index": "cd assets/ && wget https://osmlab.github.io/editor-layer-index/imagery.geojson --output-document=editor-layer-index.json", "generate:images": "ts-node scripts/generateIncludedImages.ts", "generate:translations": "ts-node scripts/generateTranslations.ts", From 286ca5776d3797d50f2c3c75fbbb271149b67903 Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Fri, 7 May 2021 02:17:46 +0200 Subject: [PATCH 5/6] Documentation updates, fix 262 --- README.md | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/README.md b/README.md index e75ed3704..7dc4b3c09 100644 --- a/README.md +++ b/README.md @@ -77,21 +77,8 @@ I love it to see where the project ends up. You are free to reuse the software ( ## Dev -To develop: +To develop or deploy a version of MapComplete, have a look [to the guide](Docs/Development_deployment.md). -0. Make sure you have a recent version of nodejs - at least 12.0, preferably 15 -0. Make a fork and clone the repository. -1. Install `npm`. Linux: `sudo apt install npm` (or your favourite package manager), Windows: install nodeJS: https://nodejs.org/en/download/ -2. Run `npm install` to install the package dependencies -3. Run `npm run generate` and `npm run generate:editor-layer-index` to generate some additional dependencies -4. Run `npm run start` to host a local testversion at http://localhost:1234/index.html -5. By default, the 'bookcases'-theme is loaded. In order to load another theme, use `layout=themename` or `userlayout=true#`. Note that the custom URLs (e.g. `bookcases.html`, `aed.html`, ...) _don't_ exist on the development version. (These are automatically generated from a template on the server). - -To deploy: - -0. `rm -rf dist/` to remove the local build -1. `npm run build` -2. Copy the entire `dist` folder to where you host your website. Visiting `index.html` gives you the website ## Translating MapComplete From dfe3af5be107818b85989fa2297a526fc3483123 Mon Sep 17 00:00:00 2001 From: pietervdvn Date: Fri, 7 May 2021 02:18:52 +0200 Subject: [PATCH 6/6] Add development information --- Docs/Development_deployment.md | 74 ++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 Docs/Development_deployment.md diff --git a/Docs/Development_deployment.md b/Docs/Development_deployment.md new file mode 100644 index 000000000..01f01e39c --- /dev/null +++ b/Docs/Development_deployment.md @@ -0,0 +1,74 @@ + + Development and deployment + ========================== + + There are various scripts to help setup MapComplete for deployment and develop-deployment. + + This documents attempts to shed some light on these scripts. + + Note: these scripts change every now and then - if the documentation here is incorrect or you run into troubles, do leave a message in [the issue tracker](https://github.com/pietervdvn/MapComplete/issues) + + Architecture overview + --------------------- + + At its core, MapComplete is a static (!) website. There are no servers to host. + + The data is fetched from Overpass/OSM/Wikidata/Wikipedia/Mapillary/... and written there directly. This means that any static file server will do to create a self-hosted version of MapComplete. + + Development + ----------- + + To develop and build MapComplete, yo + +0. Make sure you have a recent version of nodejs - at least 12.0, preferably 15 +0. Make a fork and clone the repository. +1. Install `npm`. Linux: `sudo apt install npm` (or your favourite package manager), Windows: install nodeJS: https://nodejs.org/en/download/ +2. Run `npm install` to install the package dependencies +3. Run `npm run init` and generate some additional dependencies and generated files +4. Run `npm run start` to host a local testversion at http://localhost:1234/index.html +5. By default, a landing page with available themes is served. In order to load a single theme, use `layout=themename` or `userlayout=true#` as [Query parameter](URL_Parameters.md). Note that the shorter URLs (e.g. `bookcases.html`, `aed.html`, ...) _don't_ exist on the development version. + + + Deploying a fork + ---------------- + + A script creates a webpage for every theme automatically, with some customizations in order to: + + - to have shorter urls + - have individual social images + - have individual web manifests + + + This script can be invoked with `npm run prepare-deploy` + +If you want to deploy your fork: + +0. `npm run prepare-deploy` +1. `npm run build` +2. Copy the entire `dist` folder to where you host your website. Visiting `index.html` gives you the landing page, visiting `yourwebsite/` should bring you to the appropriate theme. + + Overview of package.json-scripts + -------------------------------- + + - `increase-memory`: this is a big (and memory-intensive) project to build and run, so we give nodejs some more RAM. + - `start`: start a development server. + - `test`: run the unit tests + - `init`: Generates and downloads various assets which are needed to compile + - `generate:editor-layer-index`: downloads the editor-layer-index-json from osmlab.github.io + - `generate:images`: compiles the SVG's into an asset + - `generate:translations`: compiles the translation file into a javascript file + - `generate:layouts`: uses `index.html` as template to create all the theme index pages. You'll want to run `clean` when done + - `generate:docs`: generates various documents, such as information about available metatags, information to put on the [OSM-wiki](https://wiki.openstreetmap.org/wiki/MapComplete),... + - `generate:cache:speelplekken`: creates an offline copy of all the data required for one specific (paid for) theme + - `generate:layeroverview`: reads all the theme- and layerconfigurations, compiles them into a single JSON. + - `reset:layeroverview`: if something is wrong with the layeroverview, creates an empty one + - `generate:licenses`: compiles all the license info of images into a single json + - `optimize:images`: attempts to make smaller pngs - optional to run before a deployment + - `generate`: run all the necesary generate-scripts + - `build`: actually bundle all the files into a single `dist/`-folder + - `prepare-deploy`: create the layouts + - `deploy:staging`,`deploy:pietervdvn`, `deploy:production`: deploy the latest code on various locations + - `lint`: get depressed by the amount of warnings + - `clean`: remove some generated files which are annoying in the repo + +