midgard/mapcomplete
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Refactoring: automatically generate code files from layer/theme files to avoid using 'Eval'

Browse source
This commit is contained in:
Pieter Vander Vennet 2023-09-22 11:20:22 +02:00
parent 865b0bc44f
commit 39944a01fb
17 changed files with 269 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
404.html
View file

@ -3,6 +3,7 @@
<head>
<meta charset="UTF-8">
<meta content="width=device-width, initial-scale=1.0, user-scalable=no" name="viewport">
<meta http-equiv="Content-Security-Policy" content="default-src 'self'; script-src 'self' https://gc.zgo.at/; img-src *; connect-src 'self' https://www.openstreetmap.org/ https://api.openstreetmap.org/;">
<link href="./css/mobile.css" rel="stylesheet"/>
<link href="./css/tagrendering.css" rel="stylesheet"/>
<link href="./css/index-tailwind-output.css" rel="stylesheet"/>

7
index.html
View file

@ -3,6 +3,7 @@
<head>
<meta charset="UTF-8">
<meta content="width=device-width, initial-scale=1.0, user-scalable=no" name="viewport">
<meta http-equiv="Content-Security-Policy" content="default-src 'self'; script-src 'self' https://gc.zgo.at/; img-src *; connect-src 'self' https://www.openstreetmap.org/ https://api.openstreetmap.org/;">
<link href="./css/mobile.css" rel="stylesheet"/>
<link href="./css/openinghourstable.css" rel="stylesheet"/>
<link href="./css/tagrendering.css" rel="stylesheet"/>
@ -16,8 +17,6 @@
<title>MapComplete</title>
<link href="./index.webmanifest" rel="manifest">
<!-- Mastodon link verification: https://docs.joinmastodon.org/user/profile/#Link%20verification -->
<a rel="me" href="https://en.osm.town/@MapComplete" style="display: none">Mastodon</a>
<link href="./assets/svg/add.svg" rel="icon" sizes="any" type="image/svg+xml">
<meta content="./assets/SocialImage.png" property="og:image">
<meta content="MapComplete - editable, thematic maps with OpenStreetMap" property="og:title">
@ -48,10 +47,12 @@
</head>
<body>
<!-- Mastodon link verification: https://docs.joinmastodon.org/user/profile/#Link%20verification -->
<a rel="me" href="https://en.osm.town/@MapComplete" class="hidden">Mastodon</a>
<div id="main"></div>
<script type="module" src="./src/all_themes_index.ts"></script>
<script async data-goatcounter="https://pietervdvn.goatcounter.com/count" src="//gc.zgo.at/count.js" crossorigin="anonymous"
<script async data-goatcounter="https://pietervdvn.goatcounter.com/count" src="https://gc.zgo.at/count.js" crossorigin="anonymous"
integrity="sha384-gtO6vSydQeOAGGK19NHrlVLNtaDSJjN4aGMWschK+dwAZOdPQWbjXgL+FM5XsgFJ"></script>
<script>

22
public/css/index-tailwind-output.css
View file

@ -761,6 +761,10 @@ video {
isolation: auto;
}
.-z-10 {
z-index: -10;
}
.float-right {
float: right;
}
@ -1096,10 +1100,6 @@ video {
height: 2.75rem;
}
.h-10 {
height: 2.5rem;
}
.h-48 {
height: 12rem;
}
@ -1108,6 +1108,10 @@ video {
height: 10rem;
}
.h-10 {
height: 2.5rem;
}
.h-80 {
height: 20rem;
}
@ -1632,16 +1636,16 @@ video {
background-color: rgb(248 113 113 / var(--tw-bg-opacity));
}
.bg-white {
--tw-bg-opacity: 1;
background-color: rgb(255 255 255 / var(--tw-bg-opacity));
}
.bg-black {
--tw-bg-opacity: 1;
background-color: rgb(0 0 0 / var(--tw-bg-opacity));
}
.bg-white {
--tw-bg-opacity: 1;
background-color: rgb(255 255 255 / var(--tw-bg-opacity));
}
.bg-gray-200 {
--tw-bg-opacity: 1;
background-color: rgb(229 231 235 / var(--tw-bg-opacity));

4
scripts/build.sh
View file

@ -12,8 +12,8 @@ mkdir dist/assets 2> /dev/null
export NODE_OPTIONS="--max-old-space-size=8192"
# This script ends every line with '&&' to chain everything. A failure will thus stop the build
npm run generate:editor-layer-index &&
npm run generate &&
# npm run generate:editor-layer-index &&
# npm run generate &&
npm run generate:layouts
if [ $? -ne 0 ]; then

124
scripts/generateLayerOverview.ts
View file

@ -21,6 +21,7 @@ import { Utils } from "../src/Utils"
import Script from "./Script"
import { AllSharedLayers } from "../src/Customizations/AllSharedLayers"
import { parse as parse_html } from "node-html-parser"
import { ExtraFunctions } from "../src/Logic/ExtraFunctions"
// This scripts scans 'src/assets/layers/*.json' for layer definition files and 'src/assets/themes/*.json' for theme definition files.
// It spits out an overview of those to be used to load them
@ -395,10 +396,129 @@ class LayerOverviewUtils extends Script {
skippedLayers.length +
" layers"
)
// We always need the calculated tags of 'usersettings', so we export them separately
this.extractJavascriptCodeForLayer(
state.sharedLayers.get("usersettings"),
"./src/Logic/State/UserSettingsMetaTagging.ts"
)
return sharedLayers
}
/**
* Given: a fully expanded themeConfigJson
*
* Will extract a dictionary of the special code and write it into a javascript file which can be imported.
* This removes the need for _eval_, allowing for a correct CSP
* @param themeFile
* @private
*/
private extractJavascriptCode(themeFile: LayoutConfigJson) {
const allCode = [
"import {Feature} from 'geojson'",
'import { ExtraFuncType } from "../../../Logic/ExtraFunctions";',
'import { Utils } from "../../../Utils"',
"export class ThemeMetaTagging {",
" public static readonly themeName = " + JSON.stringify(themeFile.id),
"",
]
for (const layer of themeFile.layers) {
const l = <LayerConfigJson>layer
const code = l.calculatedTags ?? []
allCode.push(
" public metaTaggging_for_" +
l.id +
"(feat: Feature, helperFunctions: Record<ExtraFuncType, (feature: Feature) => Function>) {"
)
allCode.push(" const {" + ExtraFunctions.types.join(", ") + "} = helperFunctions")
for (const line of code) {
const firstEq = line.indexOf("=")
let attributeName = line.substring(0, firstEq).trim()
const expression = line.substring(firstEq + 1)
const isStrict = attributeName.endsWith(":")
if (!isStrict) {
allCode.push(
" Utils.AddLazyProperty(feat.properties, '" +
attributeName +
"', () => " +
expression +
" ) "
)
} else {
attributeName = attributeName.substring(0, attributeName.length - 2).trim()
allCode.push(" feat.properties['" + attributeName + "'] = " + expression)
}
}
allCode.push(" }")
}
const targetDir = "./src/assets/generated/metatagging/"
if (!existsSync(targetDir)) {
mkdirSync(targetDir)
}
allCode.push("}")
writeFileSync(targetDir + themeFile.id + ".ts", allCode.join("\n"))
}
private extractJavascriptCodeForLayer(l: LayerConfigJson, targetPath?: string) {
let importPath = "../../../"
if (targetPath) {
const l = targetPath.split("/")
if (l.length == 1) {
importPath = "./"
} else {
importPath = ""
for (let i = 0; i < l.length - 3; i++) {
const _ = l[i]
importPath += "../"
}
}
}
const allCode = [
`import { Utils } from "${importPath}Utils"`,
`/** This code is autogenerated - do not edit. Edit ./assets/layers/${l.id}/${l.id}.json instead */`,
"export class ThemeMetaTagging {",
" public static readonly themeName = " + JSON.stringify(l.id),
"",
]
const code = l.calculatedTags ?? []
allCode.push(
" public metaTaggging_for_" + l.id + "(feat: {properties: Record<string, string>}) {"
)
for (const line of code) {
const firstEq = line.indexOf("=")
let attributeName = line.substring(0, firstEq).trim()
const expression = line.substring(firstEq + 1)
const isStrict = attributeName.endsWith(":")
if (!isStrict) {
allCode.push(
" Utils.AddLazyProperty(feat.properties, '" +
attributeName +
"', () => " +
expression +
" ) "
)
} else {
attributeName = attributeName.substring(0, attributeName.length - 2).trim()
allCode.push(" feat.properties['" + attributeName + "'] = " + expression)
}
}
allCode.push(" }")
allCode.push("}")
const targetDir = "./src/assets/generated/metatagging/"
if (!targetPath) {
if (!existsSync(targetDir)) {
mkdirSync(targetDir)
}
}
writeFileSync(targetPath ?? targetDir + "layer_" + l.id + ".ts", allCode.join("\n"))
}
private buildThemeIndex(
licensePaths: Set<string>,
sharedLayers: Map<string, LayerConfigJson>,
@ -436,6 +556,7 @@ class LayerOverviewUtils extends Script {
})
const skippedThemes: string[] = []
for (let i = 0; i < themeFiles.length; i++) {
const themeInfo = themeFiles[i]
const themePath = themeInfo.path
@ -443,6 +564,7 @@ class LayerOverviewUtils extends Script {
const targetPath =
LayerOverviewUtils.themePath + "/" + themePath.substring(themePath.lastIndexOf("/"))
const usedLayers = Array.from(
LayerOverviewUtils.extractLayerIdsFrom(themeFile, false)
).map((id) => LayerOverviewUtils.layerPath + id + ".json")
@ -504,6 +626,8 @@ class LayerOverviewUtils extends Script {
this.writeTheme(themeFile)
fixed.set(themeFile.id, themeFile)
this.extractJavascriptCode(themeFile)
} catch (e) {
console.error("ERROR: could not prepare theme " + themePath + " due to " + e)
throw e

28
scripts/generateLayouts.ts
View file

@ -200,6 +200,26 @@ function asLangSpan(t: Translation, tag = "span"): string {
return values.join("\n")
}
let cspCached: string = undefined
function generateCsp(): string {
if (cspCached !== undefined) {
return cspCached
}
const csp = {
"default-src": "'self'",
"script-src": "'self'",
"img-src": "*",
"connect-src": "*",
}
const content = Object.keys(csp)
.map((k) => k + ": " + csp[k])
.join("; ")
cspCached = `<meta http-equiv="Content-Security-Policy" content="${content}">`
return cspCached
}
async function createLandingPage(layout: LayoutConfig, manifest, whiteIcons, alreadyWritten) {
Locale.language.setData(layout.language[0])
const targetLanguage = layout.language[0]
@ -279,6 +299,7 @@ async function createLandingPage(layout: LayoutConfig, manifest, whiteIcons, alr
Translations.t.general.poweredByOsm.textFor(targetLanguage)
)
.replace(/<!-- THEME-SPECIFIC -->.*<!-- THEME-SPECIFIC-END-->/s, themeSpecific)
.replace(/<!-- CSP -->/, generateCsp())
.replace(
/<!-- DESCRIPTION START -->.*<!-- DESCRIPTION END -->/s,
asLangSpan(layout.shortDescription)
@ -298,7 +319,12 @@ async function createLandingPage(layout: LayoutConfig, manifest, whiteIcons, alr
async function createIndexFor(theme: LayoutConfig) {
const filename = "index_" + theme.id + ".ts"
writeFileSync(filename, `import layout from "./src/assets/generated/themes/${theme.id}.json"\n`)
const imports = [
`import layout from "./src/assets/generated/themes/${theme.id}.json"`,
`import { ThemeMetaTagging } from "./src/assets/generated/metatagging/${theme.id}"`,
]
writeFileSync(filename, imports.join("\n") + "\n")
appendFileSync(filename, codeTemplate)
}

6
src/Logic/ExtraFunctions.ts
View file

@ -454,12 +454,16 @@ export class ExtraFunctions {
"To enable this feature, add a field `calculatedTags` in the layer object, e.g.:",
"````",
'"calculatedTags": [',
' "_someKey=javascript-expression",',
' "_someKey=javascript-expression (lazy execution)",',
' "_some_other_key:=javascript expression (strict execution)',
' "name=feat.properties.name ?? feat.properties.ref ?? feat.properties.operator",',
" \"_distanceCloserThen3Km=distanceTo(feat)( some_lon, some_lat) < 3 ? 'yes' : 'no'\" ",
" ]",
"````",
"",
"By using `:=` as separator, the attribute will be calculated as soone as the data is loaded (strict evaluation)",
"The default behaviour, using `=` as separator, is lazy loading",
"",
"The above code will be executed for every feature in the layer. The feature is accessible as `feat` and is an amended geojson object:",
new List([

51
src/Logic/MetaTagging.ts
View file

@ -9,7 +9,6 @@ import { IndexedFeatureSource } from "./FeatureSource/FeatureSource"
import OsmObjectDownloader from "./Osm/OsmObjectDownloader"
import { Utils } from "../Utils"
import { Store, UIEventSource } from "./UIEventSource"
import { SpecialVisualizationState } from "../UI/SpecialVisualization"
/**
* Metatagging adds various tags to the elements, e.g. lat, lon, surface area, ...
@ -19,6 +18,7 @@ import { SpecialVisualizationState } from "../UI/SpecialVisualization"
export default class MetaTagging {
private static errorPrintCount = 0
private static readonly stopErrorOutputAt = 10
private static metataggingObject: any = undefined
private static retaggingFuncCache = new Map<
string,
((feature: Feature, propertiesStore: UIEventSource<any>) => void)[]
@ -77,6 +77,23 @@ export default class MetaTagging {
})
}
// noinspection JSUnusedGlobalSymbols
/**
* The 'metaTagging'-object is an object which contains some functions.
* Those functions are named `metaTaggging_for_<layer_name>` and are constructed based on the 'calculatedField' for this layer.
*
* If they are set, those functions will be used instead of parsing them at runtime.
*
* This means that we can avoid using eval, resulting in faster and safer code (at the cost of more complexity) - at least for official themes.
*
* Note: this function might appear unused while developing, it is used in the generated `index_<themename>.ts` files.
*
* @param metatagging
*/
public static setThemeMetatagging(metatagging: any) {
MetaTagging.metataggingObject = metatagging
}
/**
* This method (re)calculates all metatags and calculated tags on every given feature.
* The given features should be part of the given layer
@ -298,6 +315,38 @@ export default class MetaTagging {
layer: LayerConfig,
helpers: Record<ExtraFuncType, (feature: Feature) => Function>
): (feature: Feature, tags: UIEventSource<Record<string, any>>) => boolean {
if (MetaTagging.metataggingObject) {
const funcName = "metaTaggging_for_" + layer.id
if (typeof MetaTagging.metataggingObject[funcName] !== "function") {
console.log(MetaTagging.metataggingObject)
throw (
"Error: metatagging-object for this theme does not have an entry at " +
funcName +
" (or it is not a function)"
)
}
// public metaTaggging_for_walls_and_buildings(feat: Feature, helperFunctions: Record<ExtraFuncType, (feature: Feature) => Function>) {
//
const func: (feat: Feature, helperFunctions: Record<string, any>) => void =
MetaTagging.metataggingObject[funcName]
return (feature: Feature) => {
const tags = feature.properties
if (tags === undefined) {
return
}
try {
func(feature, helpers)
} catch (e) {
console.error("Could not calculate calculated tags in exported class: ", e)
}
return true // Something changed
}
}
console.warn(
"Static MetataggingObject for theme is not set; using `new Function` (aka `eval`) to get calculated tags. This might trip up the CSP"
)
const calculatedTags: [string, string, boolean][] = layer.calculatedTags
if (calculatedTags === undefined || calculatedTags.length === 0) {
return undefined

8
src/Logic/State/UserRelatedState.ts
View file

@ -16,6 +16,7 @@ import LinkToWeblate from "../../UI/Base/LinkToWeblate"
import FeatureSwitchState from "./FeatureSwitchState"
import Constants from "../../Models/Constants"
import { QueryParameters } from "../Web/QueryParameters"
import { ThemeMetaTagging } from "./UserSettingsMetaTagging"
/**
* The part of the state which keeps track of user-related stuff, e.g. the OSM-connection,
@ -326,12 +327,15 @@ export default class UserRelatedState {
},
[translationMode]
)
const usersettingMetaTagging = new ThemeMetaTagging()
osmConnection.userDetails.addCallback((userDetails) => {
for (const k in userDetails) {
amendedPrefs.data["_" + k] = "" + userDetails[k]
}
for (const [name, code, _] of usersettingsConfig.calculatedTags) {
usersettingMetaTagging.metaTaggging_for_usersettings({ properties: amendedPrefs.data })
/*for (const [name, code, _] of usersettingsConfig.calculatedTags) {
try {
let result = new Function("feat", "return " + code + ";")({
properties: amendedPrefs.data,
@ -349,7 +353,7 @@ export default class UserRelatedState {
e
)
}
}
}*/
const simplifiedName = userDetails.name.toLowerCase().replace(/\s+/g, "")
const isTranslator = translators.contributors.find(

13
src/Logic/State/UserSettingsMetaTagging.ts Normal file
View file

@ -0,0 +1,13 @@
import { Utils } from "../../Utils"
/** This code is autogenerated - do not edit. Edit ./assets/layers/usersettings/usersettings.json instead */
export class ThemeMetaTagging {
public static readonly themeName = "usersettings"
public metaTaggging_for_usersettings(feat: {properties: Record<string, string>}) {
Utils.AddLazyProperty(feat.properties, '_mastodon_candidate_md', () => feat.properties._description.match(/\[[^\]]*\]\((.*(mastodon|en.osm.town).*)\).*/)?.at(1) )
Utils.AddLazyProperty(feat.properties, '_d', () => feat.properties._description?.replace(/&lt;/g,'<')?.replace(/&gt;/g,'>') ?? '' )
Utils.AddLazyProperty(feat.properties, '_mastodon_candidate_a', () => (feat => {const e = document.createElement('div');e.innerHTML = feat.properties._d;return Array.from(e.getElementsByTagName("a")).filter(a => a.href.match(/mastodon|en.osm.town/) !== null)[0]?.href }) (feat) )
Utils.AddLazyProperty(feat.properties, '_mastodon_link', () => (feat => {const e = document.createElement('div');e.innerHTML = feat.properties._d;return Array.from(e.getElementsByTagName("a")).filter(a => a.getAttribute("rel")?.indexOf('me') >= 0)[0]?.href})(feat) )
Utils.AddLazyProperty(feat.properties, '_mastodon_candidate', () => feat.properties._mastodon_candidate_md ?? feat.properties._mastodon_candidate_a )
}
}

9
src/Models/ThemeConfig/Conversion/Validation.ts
View file

@ -838,6 +838,15 @@ export class ValidateLayer extends DesugaringStep<LayerConfigJson> {
}
}
const layerConfig = new LayerConfig(json, "validation", true)
for (const [attribute, code, isStrict] of layerConfig.calculatedTags ?? []) {
try {
new Function("feat", "return " + code + ";")
} catch (e) {
throw `Invalid function definition: the custom javascript is invalid:${e} (at ${context}). The offending javascript code is:\n ${code}`
}
}
if (json.source === "special") {
if (!Constants.priviliged_layers.find((x) => x == json.id)) {
errors.push(

6
src/Models/ThemeConfig/LayerConfig.ts
View file

@ -204,12 +204,6 @@ export default class LayerConfig extends WithContextLoader {
}
const code = kv.substring(index + 1)
try {
new Function("feat", "return " + code + ";")
} catch (e) {
throw `Invalid function definition: the custom javascript is invalid:${e} (at ${context}). The offending javascript code is:\n ${code}`
}
this.calculatedTags.push([key, code, isStrict])
}
}

2
src/UI/SpecialVisualizations.ts
View file

@ -1171,7 +1171,7 @@ export default class SpecialVisualizations {
new Link(
Utils.SubstituteKeys(text, tags),
Utils.SubstituteKeys(href, tags),
download === undefined,
download === undefined && !href.startsWith("#"),
Utils.SubstituteKeys(download, tags)
).SetClass(classnames)
)

7
src/Utils.ts
View file

@ -152,7 +152,8 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
}
DOMPurify.addHook("afterSanitizeAttributes", function (node) {
// set all elements owning target to target=_blank + add noopener noreferrer
if ("target" in node) {
const target = node.getAttribute("target")
if (target) {
node.setAttribute("target", "_blank")
node.setAttribute("rel", "noopener noreferrer")
}
@ -307,10 +308,12 @@ In the case that MapComplete is pointed to the testing grounds, the edit will be
}
/**
* Adds a property to the given object, but the value will _only_ be calculated when it is actually requested
* Adds a property to the given object, but the value will _only_ be calculated when it is actually requested.
* This calculation will run once
* @param object
* @param name
* @param init
* @param whenDone: called when the value is updated. Note that this will be called at most once
* @constructor
*/
public static AddLazyProperty(

6
src/assets/editor-layer-index.json
View file

File diff suppressed because one or more lines are too long

3
src/index_theme.ts.template
View file

@ -2,7 +2,10 @@ import ThemeViewState from "./src/Models/ThemeViewState"
import SvelteUIElement from "./src/UI/Base/SvelteUIElement"
import ThemeViewGUI from "./src/UI/ThemeViewGUI.svelte"
import LayoutConfig from "./src/Models/ThemeConfig/LayoutConfig";
import MetaTagging from "./src/Logic/MetaTagging";
MetaTagging.setThemeMetatagging(new ThemeMetaTagging())
const state = new ThemeViewState(new LayoutConfig(<any> layout))
const main = new SvelteUIElement(ThemeViewGUI, { state })
main.AttachTo("maindiv")

3
theme.html
View file

@ -4,6 +4,7 @@
<head>
<meta charset="UTF-8">
<meta content="width=device-width, initial-scale=1.0, user-scalable=no" name="viewport">
<!-- CSP // disabled -->
<link href="./css/mobile.css" rel="stylesheet"/>
<link href="./css/openinghourstable.css" rel="stylesheet"/>
<link href="./css/tagrendering.css" rel="stylesheet"/>
@ -63,7 +64,7 @@
</div>
</div>
</div>
<div id="belowmap" class="absolute top-0 left-0" style="z-index: -1;">Below</div>
<div id="belowmap" class="absolute top-0 left-0 -z-10">Below</div>
<script>