mapcomplete/Docs/Tools/csvGrapher.py

import csv
import string
from datetime import datetime

from matplotlib import pyplot
import re

useLegend = True

def counts(lst):
    counts = {}
    for v in lst:
        if not v in counts:
            counts[v] = 0
        counts[v] += 1
    return counts


class Hist:

    def __init__(self, firstcolumn):
        self.key = "\"" + firstcolumn + "\""
        self.dictionary = {}
        self.key = ""

    def add(self, key, value):
        if not key in self.dictionary:
            self.dictionary[key] = []
        self.dictionary[key].append(value)

    def values(self):
        allV = []
        for v in self.dictionary.values():
            allV += list(set(v))
        return list(set(allV))
    
    def keys(self):
        return self.dictionary.keys()

    def get(self, key):
        if key in self.dictionary:
            return self.dictionary[key]
        return None

    # Returns values.map(f). 
    def map(self, f):
        vals = []
        keys = self.keys()
        for key in keys:
            vals.append(f(self.get(key)))
        return vals

    def mapcumul(self, f, add, zero):
        vals = []
        running_value = zero
        keys = self.keys()
        for key in keys:
            v = f(self.get(key))
            running_value = add(running_value, v)
            vals.append(running_value)
        return vals

    # Returns [(key, flatten(values))] To be used with e.g. pyplot.plot
    def flatten(self, flatten):
        result = []
        keys = self.keys()
        for key in keys:
            v = flatten(self.get(key))
            result.append((key, v))
        return result

    def csv(self):
        csv = self.key + "," + ",".join(self.values())
        header = self.values()
        for k in self.dictionary.keys():
            csv += k
            values = counts(self.dictionary[k])
            for head in header:
                if head in values:
                    csv += "," + str(values[head])
                else:
                    csv += ",0"
            csv += "\n"
        return csv

    def __str__(self):
        return str(self.dictionary)


def build_hist(stats, keyIndex, valueIndex):
    hist = Hist("date")
    c = 0
    for row in stats:
        c += 1
        hist.add(row[keyIndex], row[valueIndex])
    return hist


def as_date(str):
    return datetime.strptime(str, "%Y-%m-%d")


def cumulative_users(stats):
    users_hist = build_hist(stats, 0, 1)
    all_users_per_day = users_hist.mapcumul(
        lambda users: set(users),
        lambda a, b: a.union(b),
        set([])
    )
    cumul_uniq = list(map(len, all_users_per_day))
    unique_per_day = users_hist.map(lambda users: len(set(users)))
    new_users = [0]
    for i in range(len(cumul_uniq) - 1):
        new_users.append(cumul_uniq[i + 1] - cumul_uniq[i])
    dates = map(as_date, users_hist.keys())
    return list(dates), cumul_uniq, list(unique_per_day), list(new_users)


def pyplot_init():
    pyplot.figure(figsize=(14, 8), dpi=200)
    pyplot.xticks(rotation='vertical')
    pyplot.grid()


def create_usercount_graphs(stats, extra_text=""):
    print("Creating usercount graphs " + extra_text)
    dates, cumul_uniq, unique_per_day, new_users = cumulative_users(stats)
    total = cumul_uniq[-1]

    pyplot_init()
    pyplot.bar(dates, unique_per_day, label='Unique contributors')
    pyplot.bar(dates, new_users, label='First time contributor via MapComplete')
    if (useLegend):
        pyplot.legend()
    pyplot.title("Unique contributors" + extra_text + ' with MapComplete (' + str(total) + ' contributors)')
    pyplot.ylabel("Number of unique contributors")
    pyplot.xlabel("Date")
    pyplot.savefig("Contributors" + extra_text + ".png", dpi=400, facecolor='w', edgecolor='w')

    pyplot_init()
    pyplot.plot(dates, cumul_uniq, label='Cumulative unique contributors')
    if (useLegend):
        pyplot.legend()
    pyplot.title("Cumulative unique contributors" + extra_text + " with MapComplete - " + str(total) + " contributors")
    pyplot.ylabel("Number of unique contributors")
    pyplot.xlabel("Date")
    pyplot.savefig("CumulativeContributors" + extra_text + ".png", dpi=400, facecolor='w', edgecolor='w')


def create_contributors_per_total_cs(contents, extra_text = "", cutoff=25, per_day=False):
    hist = Hist("contributor")
    for cs in contents:
        hist.add(cs[1], cs[0])
        
    count_per_contributor = hist.map(lambda dates : len(set(dates))) if per_day else hist.map(len)
    
    per_count = Hist("per cs count")
    for cs_count in count_per_contributor:
        per_count.add(min(cs_count, cutoff), 1)            

    to_plot = per_count.flatten(len)
    to_plot.sort(key=lambda a: a[0])
    to_plot[ - 1] = (str(cutoff)+ " or more", to_plot[-1][1])
    pyplot_init()
    pyplot.bar(list(map(lambda a : str(a[0]), to_plot)), list(map(lambda a: a[1], to_plot)) )
    pyplot.title("Contributors per total number of changesets"+extra_text)
    pyplot.ylabel("Number of contributors")
    pyplot.xlabel("Mapping days with MapComplete" if per_day else "Number of changesets with MapComplete")
    pyplot.savefig("Contributors per total number of "+("mapping days" if per_day else "changesets")+extra_text+".png", dpi=400)



def create_theme_breakdown(stats, fileExtra="", cutoff=15):
    print("Creating theme breakdown " + fileExtra)
    themeCounts = {}
    for row in stats:
        theme = row[3].lower()
        if theme in theme_remappings:
            theme = theme_remappings[theme]
        if theme in themeCounts:
            themeCounts[theme] += 1
        else:
            themeCounts[theme] = 1
    themes = list(themeCounts.items())
    if len(themes) == 0:
        print("No entries found for theme breakdown (extra: " + str(fileExtra) + ")")
        return
    themes.sort(key=lambda kv: kv[1], reverse=True)
    other_count = sum([theme[1] for theme in themes if theme[1] < cutoff])
    themes_filtered = [theme for theme in themes if theme[1] >= cutoff]
    keys = list(map(lambda kv: kv[0] + " (" + str(kv[1]) + ")", themes_filtered))
    values = list(map(lambda kv: kv[1], themes_filtered))
    total = sum(map(lambda kv: kv[1], themes))
    first_pct = themes[0][1] / total;
    if other_count > 0:
        keys.append("other")
        values.append(other_count)
    pyplot_init()
    pyplot.pie(values, labels=keys, startangle=(90 - 360 * first_pct / 2))
    pyplot.title("MapComplete changes per theme" + fileExtra + " - " + str(total) + " total changes")
    pyplot.savefig("Theme distribution" + fileExtra + ".png", dpi=400, facecolor='w', edgecolor='w',
                   bbox_inches='tight')
    return themes

def summed_changes_per(contents, extraText, sum_column=5):
    newPerDay = build_hist(contents, 0, 5)
    kv = newPerDay.flatten(sum)
    keysNew = list(map(lambda kv: as_date(kv[0]), kv))
    valuesNew = list(map(lambda kv: kv[1], kv))
    changedPerDay = build_hist(contents, 0, 6)
    kv = changedPerDay.flatten(sum)
    keysChanged = list(map(lambda kv: as_date(kv[0]), kv))
    valuesChanged = list(map(lambda kv: kv[1], kv))
    if len(keysChanged) == 0 and len(keysNew) == 0:
        return

    pyplot_init()
    text = "New and changed nodes per day "+extraText
    pyplot.title(text)
    if len(keysChanged) > 0:
        pyplot.bar(keysChanged, valuesChanged, label="Changed")
    if len(keysNew) > 0:
        pyplot.bar(keysNew, valuesNew, label="New")
    if (useLegend):
        pyplot.legend()
    pyplot.savefig(text)

def cumulative_changes_per(contents, index, subject, filenameextra="", cutoff=5, cumulative=True, sort=True):
    print("Creating graph about " + subject + filenameextra)
    themes = Hist("date")
    dates_per_theme = Hist("theme")
    all_themes = set()
    for row in contents:
        th = row[index]
        all_themes.add(th)
        themes.add(as_date(row[0]), th)
        dates_per_theme.add(th, row[0])
    per_theme_count = list(zip(dates_per_theme.keys(), dates_per_theme.map(len)))
    # PerThemeCount gives the most popular theme first
    if sort == True:
        per_theme_count.sort(key=lambda kv: kv[1], reverse=False)
    elif sort is not None:
        per_theme_count.sort(key=sort)
    values_to_show = []  # (theme name, value to fill between - this is stacked, with the first layer to print last)
    running_totals = None
    other_total = 0
    other_theme_count = 0
    other_cumul = None

    for kv in per_theme_count:
        theme = kv[0]
        total_for_this_theme = kv[1]
        if cumulative:
            edits_per_day_cumul = themes.mapcumul(
                lambda themes_for_date: len([x for x in themes_for_date if theme == x]),
                lambda a, b: a + b, 0)
        else:
            edits_per_day_cumul = themes.map(lambda themes_for_date: len([x for x in themes_for_date if theme == x]))

        if (not cumulative) or (running_totals is None):
            running_totals = edits_per_day_cumul
        else:
            running_totals = list(map(lambda ab: ab[0] + ab[1], zip(running_totals, edits_per_day_cumul)))

        if total_for_this_theme >= cutoff:
            values_to_show.append((theme, running_totals))
        else:
            other_total += total_for_this_theme
            other_theme_count += 1
            if other_cumul is None:
                other_cumul = edits_per_day_cumul
            else:
                other_cumul = list(map(lambda ab: ab[0] + ab[1], zip(other_cumul, edits_per_day_cumul)))

    keys = list(themes.keys())
    values_to_show.reverse()
    values_to_show.append(("other", other_cumul))
    totals = dict(per_theme_count)
    total = sum(totals.values())
    totals["other"] = other_total

    pyplot_init()
    for kv in values_to_show:
        if kv[1] is None:
            continue  # No 'other' graph
        msg = kv[0] + " (" + str(totals[kv[0]]) + ")"
        if kv[0] == "other":
            msg = str(other_theme_count) + " small " + subject + "s (" + str(other_total) + " changes)"
        if cumulative:
            pyplot.fill_between(keys, kv[1], label=msg)
        else:
            pyplot.plot(keys, kv[1], label=msg)

    if cumulative:
        cumulative_txt = "Cumulative changesets"
    else:
        cumulative_txt = "Changesets"
    pyplot.title(cumulative_txt + " per " + subject + filenameextra + " (" + str(total) + " changesets)")
    if (useLegend):
        pyplot.legend(loc="upper left", ncol=3)
    pyplot.savefig(cumulative_txt + " per " + subject + filenameextra + ".png")


def contents_where(contents, index, starts_with, invert=False):
    for row in contents:
        if row[index].startswith(starts_with) is not invert:
            yield row


def sortable_user_number(kv):
    str = kv[0]
    ls = list(map(lambda str : "0"+str if len(str) < 2 else str, re.findall("[0-9]+", str)))
    return ".".join(ls)


def create_graphs(contents):
    summed_changes_per(contents, "")
    create_contributors_per_total_cs(contents)
    create_contributors_per_total_cs(contents, per_day=True)
    
    cumulative_changes_per(contents, 4, "version number", cutoff=1, sort=sortable_user_number)
    create_usercount_graphs(contents)
    create_theme_breakdown(contents)
    cumulative_changes_per(contents, 3, "created element", cutoff=10)
    cumulative_changes_per(contents, 3, "theme", cutoff=10)
    cumulative_changes_per(contents, 3, "theme", cutoff=10, cumulative=False)
    cumulative_changes_per(contents, 1, "contributor", cutoff=15)
    cumulative_changes_per(contents, 2, "language", cutoff=1)
    cumulative_changes_per(contents, 8, "host", cutoff=1)

    currentYear = datetime.now().year
    for year in range(2020, currentYear + 1):
        contents_filtered = list(contents_where(contents, 0, str(year)))
        extratext = " in " + str(year)
        create_contributors_per_total_cs(contents_filtered, extratext)
        create_contributors_per_total_cs(contents_filtered, extratext, per_day=True)
        create_usercount_graphs(contents_filtered, extratext)
        create_theme_breakdown(contents_filtered, extratext)
        cumulative_changes_per(contents_filtered, 3, "theme", extratext, cutoff=5)
        cumulative_changes_per(contents_filtered, 3, "theme", extratext, cutoff=5, cumulative=False)
        cumulative_changes_per(contents_filtered, 1, "contributor", extratext, cutoff=10)
        cumulative_changes_per(contents_filtered, 2, "language", extratext, cutoff=1)
        cumulative_changes_per(contents_filtered, 4, "version number", extratext, cutoff=1, cumulative=False,
                               sort=sortable_user_number)
        cumulative_changes_per(contents_filtered, 4, "version number", extratext, cutoff=1, sort=sortable_user_number)
        cumulative_changes_per(contents_filtered, 8, "host", extratext, cutoff=1)
        summed_changes_per(contents_filtered, "for year "+str(year))



def create_per_theme_graphs(contents, cutoff=10):
    all_themes = set(map(lambda row: row[3], contents))
    for theme in all_themes:
        filtered = list(contents_where(contents, 3, theme))
        if len(filtered) < cutoff:
            # less then 10 changesets - we do not map it
            continue
        contributors = set(map(lambda row: row[1], filtered))
        if len(contributors) >= 2:
            cumulative_changes_per(filtered, 1, "contributor", " for theme " + theme, cutoff=1)
        if len(filtered) > 25:
            summed_changes_per(filtered, "for theme "+theme)




def create_per_contributor_graphs(contents, least_needed_changesets):
    all_contributors = set(map(lambda row: row[1], contents))
    for contrib in all_contributors:
        filtered = list(contents_where(contents, 1, contrib))
        if len(filtered) < least_needed_changesets:
            print("Skipping "+contrib+" - too little changesets");
            continue
        themes = set(map(lambda row: row[3], filtered))
        if len(themes) >= 2:
            cumulative_changes_per(filtered, 3, "theme", " for contributor " + contrib, cutoff=1)
        if len(filtered) > 25:
            summed_changes_per(filtered, "for contributor "+contrib)


theme_remappings = {
    "metamap": "maps",
    "groen": "buurtnatuur",
    "updaten van metadata met mapcomplete": "buurtnatuur",
    "Toevoegen of dit natuurreservaat toegangkelijk is":"buurtnatuur",
    "wiki:mapcomplete/fritures": "fritures",
    "wiki:MapComplete/Fritures": "fritures",
    "lits": "lit",
    "pomp": "cyclofix",
    "wiki:user:joost_schouppe/campersite": "campersite",
    "wiki-user-joost_schouppe-geveltuintjes": "geveltuintjes",
    "wiki-user-joost_schouppe-campersite": "campersite",
    "wiki-User-joost_schouppe-campersite": "campersite",
    "wiki-User-joost_schouppe-geveltuintjes": "geveltuintjes",
    "wiki:User:joost_schouppe/campersite": "campersite",
    "arbres":"arbres_llefia",
    "aed_brugge": "aed",
     "https://llefia.org/arbres/mapcomplete.json":"arbres_llefia",
    "toevoegen of dit natuurreservaat toegangkelijk is":"buurtnatuur",
    "testing mapcomplete 0.0.0":"buurtnatuur",
    "https://raw.githubusercontent.com/osmbe/play/master/mapcomplete/geveltuinen/geveltuinen.json": "geveltuintjes"
}


def clean_input(contents):
    for row in contents:
        theme = row[3].strip().strip("\"").lower()
        if theme == "null":
            # The theme metadata has only been set later on - we fetch this from the comment
            i = row[7].rfind("#")
            theme = row[7][i + 1:-1].lower()
        if theme in theme_remappings:
            theme = theme_remappings[theme]
        row[3] = theme
        row[4] = row[4].strip().strip("\"")[len("MapComplete "):]
        row[4] = re.findall("[0-9]*\.[0-9]*\.[0-9]*", row[4])[0]
        row = [data.strip().strip("\"") for data in row]
        row[5] = int(row[5])
        row[6] = int(row[6])
        yield row


def contributor_count(stats):
    seen_contributors = set()
    for line in stats:
        contributor = line[1]
        if(contributor in seen_contributors):
            continue
        print("New contributor " + str(len(seen_contributors) + 1) + ": "+contributor)
        seen_contributors.add(contributor)
        print(line)

def main():
    print("Creating graphs...")
    with open('stats.csv', newline='') as csvfile:
        stats = list(clean_input(csv.reader(csvfile, delimiter=',', quotechar='"')))
        print("Found " + str(len(stats)) + " changesets")
        
        # contributor_count(stats)
        create_graphs(stats)
        # create_per_theme_graphs(stats, 15)
        # create_per_contributor_graphs(stats, 25)
    print("All done!")


main()