midgard/mapcomplete
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch 'develop' of github.com:pietervdvn/MapComplete into develop

Browse source
This commit is contained in:
pietervdvn 2022-03-17 23:15:47 +01:00
commit 225f14ec06
2 changed files with 161 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
Docs/Tools/GenPlot.py
View file

@ -35,7 +35,10 @@ def createBar(options):
keys = genKeys(data, options["interpetKeysAs"]) keys = genKeys(data, options["interpetKeysAs"])
values = list(map(lambda kv: kv["value"], data)) values = list(map(lambda kv: kv["value"], data))
pyplot.bar(keys, values, label=options["name"]) color = None
if "color" in options["plot"]:
color = options["plot"]["color"]
pyplot.bar(keys, values, label=options["name"], color=color)
pyplot.legend() pyplot.legend()

199
Docs/Tools/GenerateSeries.ts
View file

@ -15,7 +15,6 @@ class StatsDownloader {
private readonly _targetDirectory: string; private readonly _targetDirectory: string;
constructor(targetDirectory = ".") { constructor(targetDirectory = ".") {
this._targetDirectory = targetDirectory; this._targetDirectory = targetDirectory;
} }
@ -177,6 +176,11 @@ class ChangesetDataTools {
} }
if (cs.properties.modify + cs.properties.delete + cs.properties.create == 0) { if (cs.properties.modify + cs.properties.delete + cs.properties.create == 0) {
cs.properties.metadata.theme = "EMPTY CS" cs.properties.metadata.theme = "EMPTY CS"
}
try {
cs.properties.metadata.host = new URL(cs.properties.metadata.host).host
} catch (e) {
} }
return cs return cs
} }
@ -193,18 +197,18 @@ interface PlotSpec {
type: "stacked-bar" type: "stacked-bar"
count: { count: {
label: string, label: string,
values: { key: string | Date, value: number }[] values: { key: string | Date, value: number }[],
color?: string
}[] }[]
}, },
render() render(): Promise<void>
} }
function createGraph( function createGraph(
title: string, title: string,
...options: PlotSpec[]) { ...options: PlotSpec[]): Promise<void> {
console.log("Creating graph",title,"...") console.log("Creating graph", title, "...")
const process = exec("python3 GenPlot.py \"graphs/" + title + "\"", ((error, stdout, stderr) => { const process = exec("python3 GenPlot.py \"graphs/" + title + "\"", ((error, stdout, stderr) => {
console.log("Python: ", stdout) console.log("Python: ", stdout)
if (error !== null) { if (error !== null) {
@ -221,6 +225,11 @@ function createGraph(
} }
process.stdin._write("\n", "utf-8", undefined) process.stdin._write("\n", "utf-8", undefined)
return new Promise((resolve) => {
process.on("exit", () => resolve())
})
} }
class Histogram<K> { class Histogram<K> {
@ -414,16 +423,18 @@ class Histogram<K> {
}, },
render: undefined render: undefined
} }
graph.render = () => createGraph(graph.name, graph) graph.render = async () => await createGraph(graph.name, graph)
return graph; return graph;
} }
public asBar(options: { public asBar(options: {
name: string name: string
compare?: (a: K, b: K) => number compare?: (a: K, b: K) => number,
color?: string
}): PlotSpec { }): PlotSpec {
const spec = this.asPie(options) const spec = this.asPie(options)
spec.plot.type = "bar" spec.plot.type = "bar"
spec.plot["color"] = options.color
return spec; return spec;
} }
@ -438,6 +449,10 @@ class Histogram<K> {
} }
/**
* A group keeps track of a matrix of changes, e.g.
* 'All contributors per day'. This will be stored internally, e.g. as {'2022-03-16' --> ['Pieter Vander Vennet', 'Pieter Vander Vennet', 'Joost Schouppe', 'Pieter Vander Vennet', 'dentonny', ...]}
*/
class Group<K, V> { class Group<K, V> {
public groups: Map<K, V[]> = new Map<K, V[]>() public groups: Map<K, V[]> = new Map<K, V[]>()
@ -497,6 +512,11 @@ class Group<K, V> {
return hist return hist
} }
/**
* Given a group, creates a kind of histogram.
* E.g: if the Group is {'2022-03-16' --> ['Pieter Vander Vennet', 'Pieter Vander Vennet', 'Seppe Santens']}, the resulting 'groupedHists' will be:
* [['Pieter Vander Vennet', {'2022-03-16' --> 2}],['Seppe Santens', {'2022-03-16' --> 1}]]
*/
asGroupedHists(): [V, Histogram<K>][] { asGroupedHists(): [V, Histogram<K>][] {
const allHists = new Map<V, Histogram<K>>() const allHists = new Map<V, Histogram<K>>()
@ -520,6 +540,10 @@ class Group<K, V> {
} }
} }
/**
*
* @param hists
*/
function stackHists<K, V>(hists: [V, Histogram<K>][]): [V, Histogram<K>][] { function stackHists<K, V>(hists: [V, Histogram<K>][]): [V, Histogram<K>][] {
const runningTotals = new Histogram<K>() const runningTotals = new Histogram<K>()
const result: [V, Histogram<K>][] = [] const result: [V, Histogram<K>][] = []
@ -534,25 +558,93 @@ function stackHists<K, V>(hists: [V, Histogram<K>][]): [V, Histogram<K>][] {
return result return result
} }
/**
* Given histograms which should be shown as bars on top of each other, creates a new list of histograms with adjusted heights in order to create a coherent sum
* e.g.: for a given day, there are 2 deletions, 3 additions and 5 answers, this will be ordered as 2, 5 and 10 in order to mimic a coherent bar
* @param hists
*/
function stackHistsSimple<K>(hists: Histogram<K>[]): Histogram<K>[] {
const runningTotals = new Histogram<K>()
const result: Histogram<K>[] = []
for (const hist of hists) {
const clone = hist.Clone()
clone.bumpHist(runningTotals) // "Copies" one histogram into the other
runningTotals.bumpHist(hist)
result.push(clone)
}
result.reverse(/* Changes in place, safe copy*/)
return result
}
function createGraphs(allFeatures: ChangeSetData[], appliedFilterDescription: string, cutoff = undefined) { function createActualChangesGraph(allFeatures: ChangeSetData[], appliedFilterDescription: string) {
const metadataOptions = {
"answer": "#5b5bdc",
"create": "#46ea46",
"move": "#ffa600",
"deletion": "#ff0000",
"soft-delete": "#ff8888",
"add-image": "#8888ff",
"import": "#00ff00",
"conflation": "#ffff00",
"split": "#000000",
"relation-fix": "#cccccc",
"delete-image": "#ff00ff"
}
const metadataKeys: string[] = Object.keys(metadataOptions)
const histograms: Map<string, Histogram<string>> = new Map<string, Histogram<string>>() // {metakey --> Histogram<date>}
allFeatures.forEach(f => {
const day = f.properties.date.substr(0, 10)
for (const key of metadataKeys) {
const v = f.properties.metadata[key]
if (v === undefined) {
continue
}
const count = Number(v)
if (isNaN(count)) {
continue
}
if (!histograms.has(key)) {
histograms.set(key, new Histogram<string>())
}
histograms.get(key).bump(day, count)
}
})
const entries = stackHists(Array.from(histograms.entries()))
const allGraphs = entries.map(([name, stackedHist]) => {
const hist = histograms.get(name)
return stackedHist
.keyToDate(true)
.asBar({name: `${name} (${hist.total()})`, color: metadataOptions[name]});
}
)
createGraph("Actual changes" + appliedFilterDescription, ...allGraphs)
}
async function createGraphs(allFeatures: ChangeSetData[], appliedFilterDescription: string, cutoff = undefined) {
const hist = new Histogram<string>(allFeatures.map(f => f.properties.metadata.theme)) const hist = new Histogram<string>(allFeatures.map(f => f.properties.metadata.theme))
hist await hist
.createOthersCategory("other", cutoff ?? 20) .createOthersCategory("other", cutoff ?? 20)
.addCountToName() .addCountToName()
.asBar({name: "Changesets per theme (bar)" + appliedFilterDescription}) .asBar({name: "Changesets per theme (bar)" + appliedFilterDescription})
.render() .render()
new Histogram<string>(allFeatures.map(f => f.properties.user)) await new Histogram<string>(allFeatures.map(f => f.properties.user))
.binPerCount() .binPerCount()
.stringifyName() .stringifyName()
.createOthersCategory("25 or more", (key, _) => Number(key) >=(cutoff ?? 25)).asBar( .createOthersCategory("25 or more", (key, _) => Number(key) >= (cutoff ?? 25)).asBar(
{ {
compare: (a, b) => Number(a) - Number(b), compare: (a, b) => Number(a) - Number(b),
name: "Contributors per changeset count" + appliedFilterDescription name: "Contributors per changeset count" + appliedFilterDescription
}) })
.render() .render()
const csPerDay = new Histogram<string>(allFeatures.map(f => f.properties.date.substr(0, 10))) const csPerDay = new Histogram<string>(allFeatures.map(f => f.properties.date.substr(0, 10)))
@ -572,11 +664,11 @@ function createGraphs(allFeatures: ChangeSetData[], appliedFilterDescription: st
} }
return keys return keys
}) })
.keyToDate() .keyToDate(true)
.asLine({ .asLine({
compare: (a, b) => a.getTime() - b.getTime(), compare: (a, b) => a.getTime() - b.getTime(),
name: "Rolling 7 day average" + appliedFilterDescription name: "Rolling 7 day average" + appliedFilterDescription
}) })
const perDayAvgMonth = csPerDay.asRunningAverages(key => { const perDayAvgMonth = csPerDay.asRunningAverages(key => {
const keys = [] const keys = []
@ -588,18 +680,19 @@ function createGraphs(allFeatures: ChangeSetData[], appliedFilterDescription: st
}) })
.keyToDate() .keyToDate()
.asLine({ .asLine({
compare: (a, b) => a.getTime() - b.getTime(), compare: (a, b) => a.getTime() - b.getTime(),
name: "Rolling 31 day average" + appliedFilterDescription name: "Rolling 31 day average" + appliedFilterDescription
}) })
createGraph("Changesets per day (line)" + appliedFilterDescription, perDayLine, perDayAvg, perDayAvgMonth) await createGraph("Changesets per day (line)" + appliedFilterDescription, perDayLine, perDayAvg, perDayAvgMonth)
new Histogram<string>(allFeatures.map(f => f.properties.metadata.host))
await new Histogram<string>(allFeatures.map(f => f.properties.metadata.host))
.asPie({ .asPie({
name: "Changesets per host" + appliedFilterDescription name: "Changesets per host" + appliedFilterDescription
}).render() }).render()
new Histogram<string>(allFeatures.map(f => f.properties.metadata.theme)) await new Histogram<string>(allFeatures.map(f => f.properties.metadata.theme))
.createOthersCategory("< 25 changesets", (cutoff ?? 25)) .createOthersCategory("< 25 changesets", (cutoff ?? 25))
.addCountToName() .addCountToName()
.asPie({ .asPie({
@ -613,6 +706,7 @@ function createGraphs(allFeatures: ChangeSetData[], appliedFilterDescription: st
cutoff ?? 25 cutoff ?? 25
) )
Group.createStackedBarChartPerDay( Group.createStackedBarChartPerDay(
"Changesets per version number" + appliedFilterDescription, "Changesets per version number" + appliedFilterDescription,
allFeatures, allFeatures,
@ -624,12 +718,12 @@ function createGraphs(allFeatures: ChangeSetData[], appliedFilterDescription: st
"Changesets per minor version number" + appliedFilterDescription, "Changesets per minor version number" + appliedFilterDescription,
allFeatures, allFeatures,
f => { f => {
const base = f.properties.editor?.substr("MapComplete ".length)?.replace(/[a-zA-Z-/]/g, '') ?? "UNKNOWN" const base = f.properties.editor?.substr("MapComplete ".length)?.replace(/[a-zA-Z-/]/g, '') ?? "UNKNOWN"
const [major, minor, patch] = base.split(".") const [major, minor, patch] = base.split(".")
return major+"."+minor return major + "." + minor
}, },
cutoff ??1 cutoff ?? 1
) )
Group.createStackedBarChartPerDay( Group.createStackedBarChartPerDay(
@ -654,7 +748,7 @@ function createGraphs(allFeatures: ChangeSetData[], appliedFilterDescription: st
} }
}) })
const total = new Set(allFeatures.map(f => f.properties.user)).size const total = new Set(allFeatures.map(f => f.properties.user)).size
createGraph( await createGraph(
`Contributors per day${appliedFilterDescription}`, `Contributors per day${appliedFilterDescription}`,
contributorCountPerDay contributorCountPerDay
.asHist(true) .asHist(true)
@ -670,14 +764,14 @@ function createGraphs(allFeatures: ChangeSetData[], appliedFilterDescription: st
}), }),
) )
await createActualChangesGraph(allFeatures, appliedFilterDescription);
} }
} }
function createMiscGraphs(allFeatures: ChangeSetData[], emptyCS: ChangeSetData[]) { async function createMiscGraphs(allFeatures: ChangeSetData[], emptyCS: ChangeSetData[]) {
new Histogram(emptyCS.map(f => f.properties.date)).keyToDate().asBar({ await new Histogram(emptyCS.map(f => f.properties.date)).keyToDate().asBar({
name: "Empty changesets by date" name: "Empty changesets by date"
}).render() }).render()
const geojson = { const geojson = {
@ -697,41 +791,46 @@ function createMiscGraphs(allFeatures: ChangeSetData[], emptyCS: ChangeSetData[]
console.error("Could not create center point: ", e, f) console.error("Could not create center point: ", e, f)
return undefined return undefined
} }
})) }))
} }
writeFileSync("centerpoints.geojson", JSON.stringify(geojson, undefined, 2)) writeFileSync("centerpoints.geojson", JSON.stringify(geojson, undefined, 2))
} }
async function main(): Promise<void>{ async function main(): Promise<void> {
if(!existsSync("graphs")){ if (!existsSync("graphs")) {
mkdirSync("graphs") mkdirSync("graphs")
} }
if(process.argv.indexOf("--no-download") < 0){ if (process.argv.indexOf("--no-download") < 0) {
await new StatsDownloader("stats").DownloadStats() await new StatsDownloader("stats").DownloadStats()
} }
const allPaths = readdirSync("stats") const allPaths = readdirSync("stats")
.filter(p => p.startsWith("stats.") && p.endsWith(".json")); .filter(p => p.startsWith("stats.") && p.endsWith(".json"));
let allFeatures: ChangeSetData[] = [].concat(...allPaths let allFeatures: ChangeSetData[] = [].concat(...allPaths
.map(path => JSON.parse(readFileSync("stats/" + path, "utf-8")).features .map(path => JSON.parse(readFileSync("stats/" + path, "utf-8")).features
.map(cs => ChangesetDataTools.cleanChangesetData(cs)))); .map(cs => ChangesetDataTools.cleanChangesetData(cs))));
allFeatures = allFeatures.filter(f => f.properties.editor === null || f.properties.editor.toLowerCase().startsWith("mapcomplete")) allFeatures = allFeatures.filter(f => f.properties.editor === null || f.properties.editor.toLowerCase().startsWith("mapcomplete"))
const emptyCS = allFeatures.filter(f => f.properties.metadata.theme === "EMPTY CS") const emptyCS = allFeatures.filter(f => f.properties.metadata.theme === "EMPTY CS")
allFeatures = allFeatures.filter(f => f.properties.metadata.theme !== "EMPTY CS") allFeatures = allFeatures.filter(f => f.properties.metadata.theme !== "EMPTY CS")
const noEditor = allFeatures.filter(f => f.properties.editor === null).map(f =>"https://www.osm.org/changeset/"+ f.id) const noEditor = allFeatures.filter(f => f.properties.editor === null).map(f => "https://www.osm.org/changeset/" + f.id)
writeFileSync("missing_editor.json", JSON.stringify(noEditor, null, " ")); writeFileSync("missing_editor.json", JSON.stringify(noEditor, null, " "));
if(process.argv.indexOf("--no-graphs") >= 0){ if (process.argv.indexOf("--no-graphs") >= 0) {
return return
} }
createMiscGraphs(allFeatures, emptyCS) await createMiscGraphs(allFeatures, emptyCS)
createGraphs(allFeatures, "")
// createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2020")), " in 2020") const grbOnly = allFeatures.filter(f => f.properties.metadata.theme === "grb")
// createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2021")), " in 2021") allFeatures = allFeatures.filter(f => f.properties.metadata.theme !== "grb")
createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2022")), " in 2022"), await createGraphs(allFeatures, "")
createGraphs(allFeatures.filter(f => f.properties.metadata.theme==="toerisme_vlaanderen"), " met pin je punt", 0) await createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2020")), " in 2020")
await createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2021")), " in 2021")
await createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2022")), " in 2022")
await createGraphs(allFeatures.filter(f => f.properties.metadata.theme === "toerisme_vlaanderen"), " met pin je punt", 0)
await createGraphs(grbOnly, " with the GRB import tool", 0)
} }
main().then(_ => console.log("All done!")) main().then(_ => console.log("All done!"))