midgard/mapcomplete
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch 'develop' of github.com:pietervdvn/MapComplete into develop

Browse source
This commit is contained in:
pietervdvn 2022-03-17 23:15:47 +01:00
commit 225f14ec06
2 changed files with 161 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
Docs/Tools/GenPlot.py
View file

@ -35,7 +35,10 @@ def createBar(options):
keys = genKeys(data, options["interpetKeysAs"])
values = list(map(lambda kv: kv["value"], data))
pyplot.bar(keys, values, label=options["name"])
color = None
if "color" in options["plot"]:
color = options["plot"]["color"]
pyplot.bar(keys, values, label=options["name"], color=color)
pyplot.legend()
@ -72,4 +75,4 @@ while (True):
createLine(options)
else:
print("Unkown type: " + options.type)
print("Plot generated")
print("Plot generated")

213
Docs/Tools/GenerateSeries.ts
View file

@ -14,7 +14,6 @@ class StatsDownloader {
private readonly _targetDirectory: string;
constructor(targetDirectory = ".") {
this._targetDirectory = targetDirectory;
@ -177,6 +176,11 @@ class ChangesetDataTools {
}
if (cs.properties.modify + cs.properties.delete + cs.properties.create == 0) {
cs.properties.metadata.theme = "EMPTY CS"
}
try {
cs.properties.metadata.host = new URL(cs.properties.metadata.host).host
} catch (e) {
}
return cs
}
@ -193,18 +197,18 @@ interface PlotSpec {
type: "stacked-bar"
count: {
label: string,
values: { key: string | Date, value: number }[]
values: { key: string | Date, value: number }[],
color?: string
}[]
},
render()
render(): Promise<void>
}
function createGraph(
title: string,
...options: PlotSpec[]) {
console.log("Creating graph",title,"...")
...options: PlotSpec[]): Promise<void> {
console.log("Creating graph", title, "...")
const process = exec("python3 GenPlot.py \"graphs/" + title + "\"", ((error, stdout, stderr) => {
console.log("Python: ", stdout)
if (error !== null) {
@ -221,6 +225,11 @@ function createGraph(
}
process.stdin._write("\n", "utf-8", undefined)
return new Promise((resolve) => {
process.on("exit", () => resolve())
})
}
class Histogram<K> {
@ -414,16 +423,18 @@ class Histogram<K> {
},
render: undefined
}
graph.render = () => createGraph(graph.name, graph)
graph.render = async () => await createGraph(graph.name, graph)
return graph;
}
public asBar(options: {
name: string
compare?: (a: K, b: K) => number
compare?: (a: K, b: K) => number,
color?: string
}): PlotSpec {
const spec = this.asPie(options)
spec.plot.type = "bar"
spec.plot["color"] = options.color
return spec;
}
@ -438,6 +449,10 @@ class Histogram<K> {
}
/**
* A group keeps track of a matrix of changes, e.g.
* 'All contributors per day'. This will be stored internally, e.g. as {'2022-03-16' --> ['Pieter Vander Vennet', 'Pieter Vander Vennet', 'Joost Schouppe', 'Pieter Vander Vennet', 'dentonny', ...]}
*/
class Group<K, V> {
public groups: Map<K, V[]> = new Map<K, V[]>()
@ -497,6 +512,11 @@ class Group<K, V> {
return hist
}
/**
* Given a group, creates a kind of histogram.
* E.g: if the Group is {'2022-03-16' --> ['Pieter Vander Vennet', 'Pieter Vander Vennet', 'Seppe Santens']}, the resulting 'groupedHists' will be:
* [['Pieter Vander Vennet', {'2022-03-16' --> 2}],['Seppe Santens', {'2022-03-16' --> 1}]]
*/
asGroupedHists(): [V, Histogram<K>][] {
const allHists = new Map<V, Histogram<K>>()
@ -520,6 +540,10 @@ class Group<K, V> {
}
}
/**
*
* @param hists
*/
function stackHists<K, V>(hists: [V, Histogram<K>][]): [V, Histogram<K>][] {
const runningTotals = new Histogram<K>()
const result: [V, Histogram<K>][] = []
@ -534,25 +558,93 @@ function stackHists<K, V>(hists: [V, Histogram<K>][]): [V, Histogram<K>][] {
return result
}
/**
* Given histograms which should be shown as bars on top of each other, creates a new list of histograms with adjusted heights in order to create a coherent sum
* e.g.: for a given day, there are 2 deletions, 3 additions and 5 answers, this will be ordered as 2, 5 and 10 in order to mimic a coherent bar
* @param hists
*/
function stackHistsSimple<K>(hists: Histogram<K>[]): Histogram<K>[] {
const runningTotals = new Histogram<K>()
const result: Histogram<K>[] = []
for (const hist of hists) {
const clone = hist.Clone()
clone.bumpHist(runningTotals) // "Copies" one histogram into the other
runningTotals.bumpHist(hist)
result.push(clone)
}
result.reverse(/* Changes in place, safe copy*/)
return result
}
function createGraphs(allFeatures: ChangeSetData[], appliedFilterDescription: string, cutoff = undefined) {
function createActualChangesGraph(allFeatures: ChangeSetData[], appliedFilterDescription: string) {
const metadataOptions = {
"answer": "#5b5bdc",
"create": "#46ea46",
"move": "#ffa600",
"deletion": "#ff0000",
"soft-delete": "#ff8888",
"add-image": "#8888ff",
"import": "#00ff00",
"conflation": "#ffff00",
"split": "#000000",
"relation-fix": "#cccccc",
"delete-image": "#ff00ff"
}
const metadataKeys: string[] = Object.keys(metadataOptions)
const histograms: Map<string, Histogram<string>> = new Map<string, Histogram<string>>() // {metakey --> Histogram<date>}
allFeatures.forEach(f => {
const day = f.properties.date.substr(0, 10)
for (const key of metadataKeys) {
const v = f.properties.metadata[key]
if (v === undefined) {
continue
}
const count = Number(v)
if (isNaN(count)) {
continue
}
if (!histograms.has(key)) {
histograms.set(key, new Histogram<string>())
}
histograms.get(key).bump(day, count)
}
})
const entries = stackHists(Array.from(histograms.entries()))
const allGraphs = entries.map(([name, stackedHist]) => {
const hist = histograms.get(name)
return stackedHist
.keyToDate(true)
.asBar({name: `${name} (${hist.total()})`, color: metadataOptions[name]});
}
)
createGraph("Actual changes" + appliedFilterDescription, ...allGraphs)
}
async function createGraphs(allFeatures: ChangeSetData[], appliedFilterDescription: string, cutoff = undefined) {
const hist = new Histogram<string>(allFeatures.map(f => f.properties.metadata.theme))
hist
await hist
.createOthersCategory("other", cutoff ?? 20)
.addCountToName()
.asBar({name: "Changesets per theme (bar)" + appliedFilterDescription})
.render()
.render()
new Histogram<string>(allFeatures.map(f => f.properties.user))
await new Histogram<string>(allFeatures.map(f => f.properties.user))
.binPerCount()
.stringifyName()
.createOthersCategory("25 or more", (key, _) => Number(key) >=(cutoff ?? 25)).asBar(
{
compare: (a, b) => Number(a) - Number(b),
name: "Contributors per changeset count" + appliedFilterDescription
})
.render()
.createOthersCategory("25 or more", (key, _) => Number(key) >= (cutoff ?? 25)).asBar(
{
compare: (a, b) => Number(a) - Number(b),
name: "Contributors per changeset count" + appliedFilterDescription
})
.render()
const csPerDay = new Histogram<string>(allFeatures.map(f => f.properties.date.substr(0, 10)))
@ -572,12 +664,12 @@ function createGraphs(allFeatures: ChangeSetData[], appliedFilterDescription: st
}
return keys
})
.keyToDate()
.keyToDate(true)
.asLine({
compare: (a, b) => a.getTime() - b.getTime(),
name: "Rolling 7 day average" + appliedFilterDescription
})
compare: (a, b) => a.getTime() - b.getTime(),
name: "Rolling 7 day average" + appliedFilterDescription
})
const perDayAvgMonth = csPerDay.asRunningAverages(key => {
const keys = []
for (let i = 0; i < 31; i++) {
@ -588,18 +680,19 @@ function createGraphs(allFeatures: ChangeSetData[], appliedFilterDescription: st
})
.keyToDate()
.asLine({
compare: (a, b) => a.getTime() - b.getTime(),
name: "Rolling 31 day average" + appliedFilterDescription
})
createGraph("Changesets per day (line)" + appliedFilterDescription, perDayLine, perDayAvg, perDayAvgMonth)
compare: (a, b) => a.getTime() - b.getTime(),
name: "Rolling 31 day average" + appliedFilterDescription
})
new Histogram<string>(allFeatures.map(f => f.properties.metadata.host))
await createGraph("Changesets per day (line)" + appliedFilterDescription, perDayLine, perDayAvg, perDayAvgMonth)
await new Histogram<string>(allFeatures.map(f => f.properties.metadata.host))
.asPie({
name: "Changesets per host" + appliedFilterDescription
}).render()
new Histogram<string>(allFeatures.map(f => f.properties.metadata.theme))
await new Histogram<string>(allFeatures.map(f => f.properties.metadata.theme))
.createOthersCategory("< 25 changesets", (cutoff ?? 25))
.addCountToName()
.asPie({
@ -613,23 +706,24 @@ function createGraphs(allFeatures: ChangeSetData[], appliedFilterDescription: st
cutoff ?? 25
)
Group.createStackedBarChartPerDay(
"Changesets per version number" + appliedFilterDescription,
allFeatures,
f => f.properties.editor?.substr("MapComplete ".length, 6)?.replace(/[a-zA-Z-/]/g, '') ?? "UNKNOWN",
cutoff ?? 1
)
Group.createStackedBarChartPerDay(
"Changesets per minor version number" + appliedFilterDescription,
allFeatures,
f => {
const base = f.properties.editor?.substr("MapComplete ".length)?.replace(/[a-zA-Z-/]/g, '') ?? "UNKNOWN"
const [major, minor, patch] = base.split(".")
return major+"."+minor
const base = f.properties.editor?.substr("MapComplete ".length)?.replace(/[a-zA-Z-/]/g, '') ?? "UNKNOWN"
const [major, minor, patch] = base.split(".")
return major + "." + minor
},
cutoff ??1
cutoff ?? 1
)
Group.createStackedBarChartPerDay(
@ -654,7 +748,7 @@ function createGraphs(allFeatures: ChangeSetData[], appliedFilterDescription: st
}
})
const total = new Set(allFeatures.map(f => f.properties.user)).size
createGraph(
await createGraph(
`Contributors per day${appliedFilterDescription}`,
contributorCountPerDay
.asHist(true)
@ -670,14 +764,14 @@ function createGraphs(allFeatures: ChangeSetData[], appliedFilterDescription: st
}),
)
await createActualChangesGraph(allFeatures, appliedFilterDescription);
}
}
function createMiscGraphs(allFeatures: ChangeSetData[], emptyCS: ChangeSetData[]) {
new Histogram(emptyCS.map(f => f.properties.date)).keyToDate().asBar({
async function createMiscGraphs(allFeatures: ChangeSetData[], emptyCS: ChangeSetData[]) {
await new Histogram(emptyCS.map(f => f.properties.date)).keyToDate().asBar({
name: "Empty changesets by date"
}).render()
const geojson = {
@ -691,47 +785,52 @@ function createMiscGraphs(allFeatures: ChangeSetData[], emptyCS: ChangeSetData[]
for (const key in f.properties.metadata) {
point.properties[key] = f.properties.metadata[key]
}
return point
} catch (e) {
console.error("Could not create center point: ", e, f)
return undefined
}
}))
}))
}
writeFileSync("centerpoints.geojson", JSON.stringify(geojson, undefined, 2))
}
async function main(): Promise<void>{
if(!existsSync("graphs")){
async function main(): Promise<void> {
if (!existsSync("graphs")) {
mkdirSync("graphs")
}
if(process.argv.indexOf("--no-download") < 0){
if (process.argv.indexOf("--no-download") < 0) {
await new StatsDownloader("stats").DownloadStats()
}
const allPaths = readdirSync("stats")
.filter(p => p.startsWith("stats.") && p.endsWith(".json"));
let allFeatures: ChangeSetData[] = [].concat(...allPaths
.map(path => JSON.parse(readFileSync("stats/" + path, "utf-8")).features
.map(path => JSON.parse(readFileSync("stats/" + path, "utf-8")).features
.map(cs => ChangesetDataTools.cleanChangesetData(cs))));
allFeatures = allFeatures.filter(f => f.properties.editor === null || f.properties.editor.toLowerCase().startsWith("mapcomplete"))
const emptyCS = allFeatures.filter(f => f.properties.metadata.theme === "EMPTY CS")
allFeatures = allFeatures.filter(f => f.properties.metadata.theme !== "EMPTY CS")
const noEditor = allFeatures.filter(f => f.properties.editor === null).map(f =>"https://www.osm.org/changeset/"+ f.id)
const noEditor = allFeatures.filter(f => f.properties.editor === null).map(f => "https://www.osm.org/changeset/" + f.id)
writeFileSync("missing_editor.json", JSON.stringify(noEditor, null, " "));
if(process.argv.indexOf("--no-graphs") >= 0){
return
if (process.argv.indexOf("--no-graphs") >= 0) {
return
}
createMiscGraphs(allFeatures, emptyCS)
createGraphs(allFeatures, "")
// createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2020")), " in 2020")
// createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2021")), " in 2021")
createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2022")), " in 2022"),
createGraphs(allFeatures.filter(f => f.properties.metadata.theme==="toerisme_vlaanderen"), " met pin je punt", 0)
await createMiscGraphs(allFeatures, emptyCS)
const grbOnly = allFeatures.filter(f => f.properties.metadata.theme === "grb")
allFeatures = allFeatures.filter(f => f.properties.metadata.theme !== "grb")
await createGraphs(allFeatures, "")
await createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2020")), " in 2020")
await createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2021")), " in 2021")
await createGraphs(allFeatures.filter(f => f.properties.date.startsWith("2022")), " in 2022")
await createGraphs(allFeatures.filter(f => f.properties.metadata.theme === "toerisme_vlaanderen"), " met pin je punt", 0)
await createGraphs(grbOnly, " with the GRB import tool", 0)
}
main().then(_ => console.log("All done!"))