New graphs, improved script

Docs/Tools/.~lock.stats.csv#

@@ -0,0 +1 @@
+,pietervdvn,pietervdvn-Latitude-5591,28.03.2021 22:46,file:///home/pietervdvn/.config/libreoffice/4;

Docs/Tools/csvGrapher.py

@@ -32,7 +32,7 @@ class Hist:
         for v in self.dictionary.values():
             allV += list(set(v))
         return list(set(allV))
-
+    
     def keys(self):
         return self.dictionary.keys()
 
@@ -59,6 +59,15 @@ class Hist:
             vals.append(running_value)
         return vals
 
+    # Returns [(key, flatten(values))]
+    def flatten(self, flatten):
+        result = []
+        keys = self.keys()
+        for key in keys:
+            v = flatten(self.get(key))
+            result.append((key, v))
+        return result
+
     def csv(self):
         csv = self.key + "," + ",".join(self.values())
         header = self.values()
@@ -117,8 +126,8 @@ def create_usercount_graphs(stats, extra_text=""):
     total = cumul_uniq[-1]
 
     pyplot_init()
-    pyplot.fill_between(dates, unique_per_day, label='Unique contributors')
+    pyplot.bar(dates, unique_per_day, label='Unique contributors')
-    pyplot.fill_between(dates, new_users, label='First time contributor via MapComplete')
+    pyplot.bar(dates, new_users, label='First time contributor via MapComplete')
     pyplot.legend()
     pyplot.title("Unique contributors" + extra_text + ' with MapComplete (' + str(total) + ' contributors)')
     pyplot.ylabel("Number of unique contributors")
@@ -166,6 +175,27 @@ def create_theme_breakdown(stats, fileExtra="", cutoff=5):
                    bbox_inches='tight')
     return themes
 
+def summed_changes_per(contents, extraText, sum_column=5):
+    newPerDay = build_hist(contents, 0, 5)
+    kv = newPerDay.flatten(sum)
+    keysNew = list(map(lambda kv: as_date(kv[0]), kv))
+    valuesNew = list(map(lambda kv: kv[1], kv))
+    changedPerDay = build_hist(contents, 0, 6)
+    kv = changedPerDay.flatten(sum)
+    keysChanged = list(map(lambda kv: as_date(kv[0]), kv))
+    valuesChanged = list(map(lambda kv: kv[1], kv))
+    if len(keysChanged) == 0 and len(keysNew) == 0:
+        return
+
+    pyplot_init()
+    text = "New and changed nodes per day "+extraText
+    pyplot.title(text)
+    if len(keysChanged) > 0:
+        pyplot.bar(keysChanged, valuesChanged, label="Changed")
+    if len(keysNew) > 0:
+        pyplot.bar(keysNew, valuesNew, label="New")
+    pyplot.legend()
+    pyplot.savefig(text)
 
 def cumulative_changes_per(contents, index, subject, filenameextra="", cutoff=5, cumulative=True, sort=True):
     print("Creating graph about " + subject + filenameextra)
@@ -255,9 +285,11 @@ def sortable_user_number(kv):
 
 
 def create_graphs(contents):
+    summed_changes_per(contents, "")
     cumulative_changes_per(contents, 4, "version number", cutoff=1, sort=sortable_user_number)
     create_usercount_graphs(contents)
     create_theme_breakdown(contents)
+    cumulative_changes_per(contents, 3, "created element", cutoff=10)
     cumulative_changes_per(contents, 3, "theme", cutoff=10)
     cumulative_changes_per(contents, 3, "theme", cutoff=10, cumulative=False)
     cumulative_changes_per(contents, 1, "contributor", cutoff=15)
@@ -278,6 +310,8 @@ def create_graphs(contents):
                                sort=sortable_user_number)
         cumulative_changes_per(contents_filtered, 4, "version number", extratext, cutoff=1, sort=sortable_user_number)
         cumulative_changes_per(contents_filtered, 8, "host", extratext, cutoff=1)
+        summed_changes_per(contents_filtered, "for year "+str(year))
+
 
 
 def create_per_theme_graphs(contents, cutoff=10):
@@ -288,9 +322,10 @@ def create_per_theme_graphs(contents, cutoff=10):
             # less then 10 changesets - we do not map it
             continue
         contributors = set(map(lambda row: row[1], filtered))
-        if len(contributors) < 2:
+        if len(contributors) >= 2:
-            continue  # one contributor makes a boring graph
+            cumulative_changes_per(filtered, 1, "contributor", " for theme " + theme, cutoff=1)
-        cumulative_changes_per(filtered, 1, "contributor", " for theme " + theme, cutoff=1)
+        if len(filtered) > 25:
+            summed_changes_per(filtered, "for theme "+theme)
 
 
 def create_per_contributor_graphs(contents, least_needed_changesets):
@@ -301,15 +336,17 @@ def create_per_contributor_graphs(contents, least_needed_changesets):
             print("Skipping "+contrib+" - too little changesets");
             continue
         themes = set(map(lambda row: row[3], filtered))
-        if len(themes) < 2:
+        if len(themes) >= 2:
-            print("Skipping "+contrib+" - only one theme edited");
+            cumulative_changes_per(filtered, 3, "theme", " for contributor " + contrib, cutoff=1)
-            continue  # one theme makes a boring graph
+        if len(filtered) > 25:
-        cumulative_changes_per(filtered, 3, "theme", " for contributor " + contrib, cutoff=1)
+            summed_changes_per(filtered, "for contributor "+contrib)
 
 
 theme_remappings = {
     "metamap": "maps",
     "groen": "buurtnatuur",
+    "updaten van metadata met mapcomplete": "buurtnatuur",
+    "Toevoegen of dit natuurreservaat toegangkelijk is":"buurtnatuur",
     "wiki:mapcomplete/fritures": "fritures",
     "wiki:MapComplete/Fritures": "fritures",
     "lits": "lit",
@@ -336,7 +373,10 @@ def clean_input(contents):
         row[3] = theme
         row[4] = row[4].strip().strip("\"")[len("MapComplete "):]
         row[4] = re.findall("[0-9]*\.[0-9]*\.[0-9]*", row[4])[0]
-        yield [data.strip().strip("\"") for data in row]
+        row = [data.strip().strip("\"") for data in row]
+        row[5] = int(row[5])
+        row[6] = int(row[6])
+        yield row
 
 
 def main():
@@ -350,6 +390,4 @@ def main():
     print("All done!")
 
 
-# pyplot.fill_between(range(0,5),  [1,2,3,3,2],)
-# pyplot.show()
 main()

Docs/Tools/stats.csv

@@ -3139,3 +3139,45 @@
 "2021-03-24", "WinstonSmith", "en", "drinking_water", "MapComplete 0.6.0", 1, 1, "Adding data with #MapComplete for theme #drinking_water", "mapcomplete.osm.be"
 "2021-03-25", "PeeWee32", "en", "benchesandpicnictables", "MapComplete 0.6.0", 0, 1, "Adding data with #MapComplete for theme #benchesandpicnictables", "mapcomplete.osm.be"
 "2021-03-25", "PeeWee32", "en", "benches", "MapComplete 0.6.0", 1, 0, "Adding data with #MapComplete for theme #benches", "mapcomplete.osm.be"
+"2021-03-25", "Peter Elderson", "en", "HailHydrant", "MapComplete 0.6.0", 0, 8, "Adding data with #MapComplete for theme #HailHydrant", "mapcomplete.osm.be"
+"2021-03-26", "adrigrillo", "es", "1roadAlllanes", "MapComplete 0.6.2", 0, 3, "Adding data with #MapComplete for theme #1roadAlllanes", "mapcomplete.osm.be"
+"2021-03-26", "BS97n", "en", "maps", "MapComplete 0.6.2", 1, 2, "Adding data with #MapComplete for theme #maps", "mapcomplete.osm.be"
+"2021-03-26", "Erin76", "nl", "surveillance", "MapComplete 0.6.2", 1, 1, "Adding data with #MapComplete for theme #surveillance", "mapcomplete.osm.be"
+"2021-03-26", "jayjay420", "en", "surveillance", "MapComplete 0.6.2", 1, 2, "Adding data with #MapComplete for theme #surveillance", "mapcomplete.osm.be"
+"2021-03-26", "Kinjkajh", "nl", "surveillance", "MapComplete 0.6.3", 7, 21, "Adding data with #MapComplete for theme #surveillance", "pietervdvn.github.io"
+"2021-03-26", "Koen Rijnsent", "en", "personal", "MapComplete 0.6.2", 0, 2, "Adding data with #MapComplete for theme #personal", "mapcomplete.osm.be"
+"2021-03-26", "Koen Rijnsent", "en", "personal", "MapComplete 0.6.2", 0, 2, "Adding data with #MapComplete for theme #personal", "mapcomplete.osm.be"
+"2021-03-26", "matissevdberg", "en", "surveillance", "MapComplete 0.6.3", 11, 31, "Adding data with #MapComplete for theme #surveillance", "pietervdvn.github.io"
+"2021-03-26", "n1kn0k", "en", "cyclofix", "MapComplete 0.6.2", 1, 4, "Adding data with #MapComplete for theme #cyclofix", "mapcomplete.osm.be"
+"2021-03-26", "PeeWee32", "en", "benchesandpicnictables", "MapComplete 0.6.2", 1, 5, "Adding data with #MapComplete for theme #benchesandpicnictables", "mapcomplete.osm.be"
+"2021-03-26", "Peter Elderson", "en", "stolpersteine", "MapComplete 0.6.2", 1, 4, "Adding data with #MapComplete for theme #stolpersteine", "mapcomplete.osm.be"
+"2021-03-26", "Pieter Vander Vennet", "nl", "artworks", "MapComplete 0.6.2", 1, 2, "Adding data with #MapComplete for theme #artworks", "mapcomplete.osm.be"
+"2021-03-26", "Pieter Vander Vennet", "nl", "cyclofix", "MapComplete 0.6.2", 0, 2, "Adding data with #MapComplete for theme #cyclofix", "mapcomplete.osm.be"
+"2021-03-26", "Pieter Vander Vennet", "nl", "surveillance", "MapComplete 0.6.2", 1, 1, "Adding data with #MapComplete for theme #surveillance", "mapcomplete.osm.be"
+"2021-03-26", "Pieter Vander Vennet", "nl", "surveillance", "MapComplete 0.6.2", 2, 4, "Adding data with #MapComplete for theme #surveillance", "mapcomplete.osm.be"
+"2021-03-26", "Stinus_Clasius", "nl", "sport_pitches", "MapComplete 0.6.2", 0, 2, "Adding data with #MapComplete for theme #sport_pitches", "mapcomplete.osm.be"
+"2021-03-27", "Awo", "en", "artworks", "MapComplete 0.6.3", 0, 2, "Adding data with #MapComplete for theme #artworks", "pietervdvn.github.io"
+"2021-03-27", "BS97n", "de", "personal", "MapComplete 0.6.2", 0, 2, "Adding data with #MapComplete for theme #personal", "mapcomplete.osm.be"
+"2021-03-27", "BS97n", "en", "personal", "MapComplete 0.6.2", 0, 4, "Adding data with #MapComplete for theme #personal", "mapcomplete.osm.be"
+"2021-03-27", "klimaanvzw", "nl", "facadegardens", "MapComplete 0.6.2", 1, 4, "Adding data with #MapComplete for theme #facadegardens", "mapcomplete.osm.be"
+"2021-03-27", "Koen Rijnsent", "en", "personal", "MapComplete 0.6.2", 0, 1, "Adding data with #MapComplete for theme #personal", "mapcomplete.osm.be"
+"2021-03-27", "Koen Rijnsent", "en", "personal", "MapComplete 0.6.2", 5, 9, "Adding data with #MapComplete for theme #personal", "mapcomplete.osm.be"
+"2021-03-27", "Thierry1030", "nl", "fietsstraten", "MapComplete 0.6.3", 0, 1, "Adding data with #MapComplete for theme #fietsstraten", "pietervdvn.github.io"
+"2021-03-28", "bbigras", "en", "artworks", "MapComplete 0.6.2", 0, 4, "Adding data with #MapComplete for theme #artworks", "mapcomplete.osm.be"
+"2021-03-28", "Devolved", "en", "campersite", "MapComplete 0.6.2", 0, 4, "Adding data with #MapComplete for theme #campersite", "mapcomplete.osm.be"
+"2021-03-28", "dkf2010", "en", "benches", "MapComplete 0.6.2", 0, 3, "Adding data with #MapComplete for theme #benches", "mapcomplete.osm.be"
+"2021-03-28", "grouper", "en", "shops", "MapComplete 0.6.2", 1, 0, "Adding data with #MapComplete for theme #shops", "mapcomplete.osm.be"
+"2021-03-28", "JackGilmore", "en", "aed", "MapComplete 0.6.2", 1, 1, "Adding data with #MapComplete for theme #aed", "mapcomplete.osm.be"
+"2021-03-28", "Jakka", "nl", "ghostbikes", "MapComplete 0.6.2", 1, 0, "Adding data with #MapComplete for theme #ghostbikes", "mapcomplete.osm.be"
+"2021-03-28", "Jez Nicholson", "en", "benches", "MapComplete 0.6.2", 0, 25, "Adding data with #MapComplete for theme #benches", "mapcomplete.osm.be"
+"2021-03-28", "Pieter Vander Vennet", "nl", "grb", "MapComplete 0.6.2", 0, 12, "Adding data with #MapComplete for theme #grb", "mapcomplete.osm.be"
+"2021-03-28", "Pieter Vander Vennet", "nl", "surveillance", "MapComplete 0.6.2", 3, 9, "Adding data with #MapComplete for theme #surveillance", "mapcomplete.osm.be"
+"2021-03-28", "ptjamp", "en", "surveillance", "MapComplete 0.6.3", 3, 4, "Adding data with #MapComplete for theme #surveillance", "pietervdvn.github.io"
+"2021-03-28", "ptjamp", "en", "surveillance", "MapComplete 0.6.3", 39, 67, "Adding data with #MapComplete for theme #surveillance", "pietervdvn.github.io"
+"2021-03-28", "schoka", "de", "cyclofix", "MapComplete 0.6.2", 0, 11, "Adding data with #MapComplete for theme #cyclofix", "mapcomplete.osm.be"
+"2021-03-28", "Sharwin_F", "en", "wherethesidewalkshavenoname", "MapComplete 0.6.2", 0, 4, "Adding data with #MapComplete for theme #wherethesidewalkshavenoname", "mapcomplete.osm.be"
+"2021-03-28", "sjokomoeske", "en", "benches", "MapComplete 0.6.2", 4, 5, "Adding data with #MapComplete for theme #benches", "mapcomplete.osm.be"
+"2021-03-28", "tastrax", "en", "charging_stations", "MapComplete 0.6.2", 0, 2, "Adding data with #MapComplete for theme #charging_stations", "mapcomplete.osm.be"
+"2021-03-28", "Thierry1030", "nl", "fietsstraten", "MapComplete 0.6.3", 0, 1, "Adding data with #MapComplete for theme #fietsstraten", "pietervdvn.github.io"
+"2021-03-28", "WinstonSmith", "en", "drinking_water", "MapComplete 0.6.2", 0, 2, "Adding data with #MapComplete for theme #drinking_water", "mapcomplete.osm.be"
+"2021-03-28", "WinstonSmith", "en", "drinking_water", "MapComplete 0.6.2", 1, 0, "Adding data with #MapComplete for theme #drinking_water", "mapcomplete.osm.be"