Merge branch 'develop' of github.com:pietervdvn/MapComplete into develop

Docs/Tools/GenPlot.py

@@ -35,7 +35,10 @@ def createBar(options):
     keys = genKeys(data, options["interpetKeysAs"])
     values = list(map(lambda kv: kv["value"], data))
 
-    pyplot.bar(keys, values, label=options["name"])
+    color = None
+    if "color" in options["plot"]:
+    	color = options["plot"]["color"] 
+    pyplot.bar(keys, values, label=options["name"], color=color)
     pyplot.legend()
 
 

Docs/Tools/GenerateSeries.ts

@@ -15,7 +15,6 @@ class StatsDownloader {
     private readonly _targetDirectory: string;
 
 
-
     constructor(targetDirectory = ".") {
         this._targetDirectory = targetDirectory;
     }
@@ -177,6 +176,11 @@ class ChangesetDataTools {
         }
         if (cs.properties.modify + cs.properties.delete + cs.properties.create == 0) {
             cs.properties.metadata.theme = "EMPTY CS"
+        }
+        try {
+            cs.properties.metadata.host = new URL(cs.properties.metadata.host).host
+        } catch (e) {
+
         }
         return cs
     }
@@ -193,17 +197,17 @@ interface PlotSpec {
         type: "stacked-bar"
         count: {
             label: string,
-            values: { key: string | Date, value: number }[]
+            values: { key: string | Date, value: number }[],
+            color?: string
         }[]
     },
 
-    render()
+    render(): Promise<void>
 }
 
-
 function createGraph(
     title: string,
-    ...options: PlotSpec[]) {
+    ...options: PlotSpec[]): Promise<void> {
     console.log("Creating graph", title, "...")
     const process = exec("python3 GenPlot.py \"graphs/" + title + "\"", ((error, stdout, stderr) => {
         console.log("Python: ", stdout)
@@ -221,6 +225,11 @@ function createGraph(
     }
     process.stdin._write("\n", "utf-8", undefined)
 
+
+    return new Promise((resolve) => {
+        process.on("exit", () => resolve())
+    })
+
 }
 
 class Histogram<K> {
@@ -414,16 +423,18 @@ class Histogram<K> {
             },
             render: undefined
         }
-        graph.render = () => createGraph(graph.name, graph)
+        graph.render = async () => await createGraph(graph.name, graph)
         return graph;
     }
 
     public asBar(options: {
         name: string
-        compare?: (a: K, b: K) => number
+        compare?: (a: K, b: K) => number,
+        color?: string
     }): PlotSpec {
         const spec = this.asPie(options)
         spec.plot.type = "bar"
+        spec.plot["color"] = options.color
         return spec;
     }
 
@@ -438,6 +449,10 @@ class Histogram<K> {
 
 }
 
+/**
+ * A group keeps track of a matrix of changes, e.g.
+ * 'All contributors per day'. This will be stored internally, e.g. as {'2022-03-16' --> ['Pieter Vander Vennet', 'Pieter Vander Vennet', 'Joost Schouppe', 'Pieter Vander Vennet', 'dentonny', ...]}
+ */
 class Group<K, V> {
 
     public groups: Map<K, V[]> = new Map<K, V[]>()
@@ -497,6 +512,11 @@ class Group<K, V> {
         return hist
     }
 
+    /**
+     * Given a group, creates a kind of histogram.
+     * E.g: if the Group is {'2022-03-16' --> ['Pieter Vander Vennet', 'Pieter Vander Vennet', 'Seppe Santens']}, the resulting 'groupedHists' will be:
+     * [['Pieter Vander Vennet', {'2022-03-16' --> 2}],['Seppe Santens', {'2022-03-16' --> 1}]]
+     */
     asGroupedHists(): [V, Histogram<K>][] {
 
         const allHists = new Map<V, Histogram<K>>()
@@ -520,6 +540,10 @@ class Group<K, V> {
     }
 }
 
+/**
+ *
+ * @param hists
+ */
 function stackHists<K, V>(hists: [V, Histogram<K>][]): [V, Histogram<K>][] {
     const runningTotals = new Histogram<K>()
     const result: [V, Histogram<K>][] = []
@@ -534,17 +558,85 @@ function stackHists<K, V>(hists: [V, Histogram<K>][]): [V, Histogram<K>][] {
     return result
 }
 
+/**
+ * Given histograms which should be shown as bars on top of each other, creates a new list of histograms with adjusted heights in order to create a coherent sum
+ * e.g.: for a given day, there are 2 deletions, 3 additions and 5 answers, this will be ordered as 2, 5 and 10 in order to mimic a coherent bar
+ * @param hists
+ */
+function stackHistsSimple<K>(hists: Histogram<K>[]): Histogram<K>[] {
+    const runningTotals = new Histogram<K>()
+    const result: Histogram<K>[] = []
+    for (const hist of hists) {
+        const clone = hist.Clone()
+        clone.bumpHist(runningTotals) // "Copies" one histogram into the other
+        runningTotals.bumpHist(hist)
+        result.push(clone)
+    }
+    result.reverse(/* Changes in place, safe copy*/)
+    return result
+}
 
-function createGraphs(allFeatures: ChangeSetData[], appliedFilterDescription: string, cutoff = undefined) {
+function createActualChangesGraph(allFeatures: ChangeSetData[], appliedFilterDescription: string) {
+    const metadataOptions = {
+        "answer": "#5b5bdc",
+        "create": "#46ea46",
+        "move": "#ffa600",
+        "deletion": "#ff0000",
+        "soft-delete": "#ff8888",
+        "add-image": "#8888ff",
+        "import": "#00ff00",
+        "conflation": "#ffff00",
+        "split": "#000000",
+        "relation-fix": "#cccccc",
+        "delete-image": "#ff00ff"
+    }
+
+    const metadataKeys: string[] = Object.keys(metadataOptions)
+    const histograms: Map<string, Histogram<string>> = new Map<string, Histogram<string>>() // {metakey --> Histogram<date>}
+    allFeatures.forEach(f => {
+        const day = f.properties.date.substr(0, 10)
+
+        for (const key of metadataKeys) {
+            const v = f.properties.metadata[key]
+            if (v === undefined) {
+                continue
+            }
+            const count = Number(v)
+            if (isNaN(count)) {
+                continue
+            }
+            if (!histograms.has(key)) {
+                histograms.set(key, new Histogram<string>())
+            }
+            histograms.get(key).bump(day, count)
+        }
+
+    })
+
+
+    const entries = stackHists(Array.from(histograms.entries()))
+
+    const allGraphs = entries.map(([name, stackedHist]) => {
+            const hist = histograms.get(name)
+            return stackedHist
+                .keyToDate(true)
+                .asBar({name: `${name} (${hist.total()})`, color: metadataOptions[name]});
+        }
+    )
+
+    createGraph("Actual changes" + appliedFilterDescription, ...allGraphs)
+}
+
+async function createGraphs(allFeatures: ChangeSetData[], appliedFilterDescription: string, cutoff = undefined) {
     const hist = new Histogram<string>(allFeatures.map(f => f.properties.metadata.theme))
-    hist
+    await hist
         .createOthersCategory("other", cutoff ?? 20)
         .addCountToName()
         .asBar({name: "Changesets per theme (bar)" + appliedFilterDescription})
         .render()
 
 
-    new Histogram<string>(allFeatures.map(f => f.properties.user))
+    await new Histogram<string>(allFeatures.map(f => f.properties.user))
         .binPerCount()
         .stringifyName()
         .createOthersCategory("25 or more", (key, _) => Number(key) >= (cutoff ?? 25)).asBar(
@@ -572,7 +664,7 @@ function createGraphs(allFeatures: ChangeSetData[], appliedFilterDescription: st
         }
         return keys
     })
-        .keyToDate()
+        .keyToDate(true)
         .asLine({
             compare: (a, b) => a.getTime() - b.getTime(),
             name: "Rolling 7 day average" + appliedFilterDescription
@@ -592,14 +684,15 @@ function createGraphs(allFeatures: ChangeSetData[], appliedFilterDescription: st
             name: "Rolling 31 day average" + appliedFilterDescription
         })
 
-    createGraph("Changesets per day (line)" + appliedFilterDescription, perDayLine, perDayAvg, perDayAvgMonth)
+    await createGraph("Changesets per day (line)" + appliedFilterDescription, perDayLine, perDayAvg, perDayAvgMonth)
 
-    new Histogram<string>(allFeatures.map(f => f.properties.metadata.host))
+
+    await new Histogram<string>(allFeatures.map(f => f.properties.metadata.host))
         .asPie({
             name: "Changesets per host" + appliedFilterDescription
         }).render()
 
-    new Histogram<string>(allFeatures.map(f => f.properties.metadata.theme))
+    await new Histogram<string>(allFeatures.map(f => f.properties.metadata.theme))
         .createOthersCategory("< 25 changesets", (cutoff ?? 25))
         .addCountToName()
         .asPie({
@@ -613,6 +706,7 @@ function createGraphs(allFeatures: ChangeSetData[], appliedFilterDescription: st
         cutoff ?? 25
     )
 
+
     Group.createStackedBarChartPerDay(
         "Changesets per version number" + appliedFilterDescription,
         allFeatures,
@@ -654,7 +748,7 @@ function createGraphs(allFeatures: ChangeSetData[], appliedFilterDescription: st
             }
         })
         const total = new Set(allFeatures.map(f => f.properties.user)).size
-        createGraph(
+        await createGraph(
             `Contributors per day${appliedFilterDescription}`,
             contributorCountPerDay
                 .asHist(true)
@@ -670,14 +764,14 @@ function createGraphs(allFeatures: ChangeSetData[], appliedFilterDescription: st
                 }),
         )
 
-
+        await createActualChangesGraph(allFeatures, appliedFilterDescription);
     }
 
 
 }
 
-function createMiscGraphs(allFeatures: ChangeSetData[], emptyCS: ChangeSetData[]) {
+async function createMiscGraphs(allFeatures: ChangeSetData[], emptyCS: ChangeSetData[]) {
-    new Histogram(emptyCS.map(f => f.properties.date)).keyToDate().asBar({
+    await new Histogram(emptyCS.map(f => f.properties.date)).keyToDate().asBar({
         name: "Empty changesets by date"
     }).render()
     const geojson = {
@@ -726,12 +820,17 @@ async function main(): Promise<void>{
     if (process.argv.indexOf("--no-graphs") >= 0) {
         return
     }
-    createMiscGraphs(allFeatures, emptyCS)
+    await createMiscGraphs(allFeatures, emptyCS)
-    createGraphs(allFeatures, "")
+
-    // createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2020")), " in 2020")
+    const grbOnly = allFeatures.filter(f => f.properties.metadata.theme === "grb")
-    // createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2021")), " in 2021")
+    allFeatures = allFeatures.filter(f => f.properties.metadata.theme !== "grb")
-    createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2022")), " in 2022"),
+    await createGraphs(allFeatures, "")
-    createGraphs(allFeatures.filter(f => f.properties.metadata.theme==="toerisme_vlaanderen"), " met pin je punt", 0)
+    await createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2020")), " in 2020")
+    await createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2021")), " in 2021")
+    await createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2022")), " in 2022")
+    await createGraphs(allFeatures.filter(f => f.properties.metadata.theme === "toerisme_vlaanderen"), " met pin je punt", 0)
+    await createGraphs(grbOnly, " with the GRB import tool", 0)
+
 }
 
 main().then(_ => console.log("All done!"))