Add numberdealers-ng error-checking style and change report layout

numberdealers/parse_numberdealers.py

@@ -3,6 +3,7 @@
 import re
 import json
 from dataclasses import dataclass
+from enum import Enum
 from typing import Optional, List
 from .users import USERS
 
@@ -49,7 +50,11 @@ NUMBER_EMOJI = {
 
 URL_PREFIX = "https://mattermost.zeus.gent/zeus/pl/"
 
-def parse(message_json_lines):
+class ErrorStyle(Enum):
+	NUMBERDEALERS = 1
+	NUMBERDEALERS_NG = 2
+
+def parse(message_json_lines, error_style: ErrorStyle):
 	second_last_number = None
 	second_last_message = None
 	last_number = None
@@ -57,6 +62,7 @@ def parse(message_json_lines):
 	numbers = []
 	errors = []
 	start_number = None
+	expected = None
 	for line in message_json_lines:
 		line = json.loads(line)
 		# Ignore non-message posts (e.g. join/leave)
@@ -119,33 +125,44 @@ def parse(message_json_lines):
 				start_number = number
 				last_number = number - 1
 				second_last_number = number - 2
+				if error_style == ErrorStyle.NUMBERDEALERS_NG:
+					expected = number
 
 			numbers.append(message_obj)
-			if number != last_number + 1:
+			if error_style == ErrorStyle.NUMBERDEALERS_NG:
-				if number == second_last_number + 2 and last_number != second_last_number + 1:
+				if number == expected:
-					errors.pop()
+					expected = number + 1
-					errors.append(
-						ShouldHaveBeen(last_message, second_last_message, number-1)
-					)
-				elif number == last_number:
-					errors.append(
-						Duplicate(message_obj, last_message, last_number+1)
-					)
-				elif number == second_last_number + 1 and last_number != second_last_number + 1:
-					errors.pop()
-					errors.append(
-						Stray(last_message, second_last_message, last_number+1)
-					)
-				elif last_number == second_last_number + 1 and number == last_number + 2:
-					errors.pop()
-					errors.append(
-						Skipped(last_message, second_last_message, number-1)
-					)
 				else:
 					errors.append(
-						Jump(message_obj, last_message, last_number+1)
+						ShouldHaveBeen(message_obj, last_message, expected)
 					)
 
+			else:
+				if number != last_number + 1:
+					if number == second_last_number + 2 and last_number != second_last_number + 1:
+						errors.pop()
+						errors.append(
+							ShouldHaveBeen(last_message, second_last_message, number-1)
+						)
+					elif number == last_number:
+						errors.append(
+							Duplicate(message_obj, last_message, last_number+1)
+						)
+					elif number == second_last_number + 1 and last_number != second_last_number + 1:
+						errors.pop()
+						errors.append(
+							Stray(last_message, second_last_message, last_number+1)
+						)
+					elif last_number == second_last_number + 1 and number == last_number + 2:
+						errors.pop()
+						errors.append(
+							Skipped(last_message, second_last_message, number-1)
+						)
+					else:
+						errors.append(
+							Jump(message_obj, last_message, last_number+1)
+						)
+
 			second_last_number = last_number
 			second_last_message = last_message
 			last_number = number
@@ -158,7 +175,7 @@ def main():
 	import sys
 	from datetime import datetime, timezone
 
-	numbers, _errors = parse(sys.stdin)
+	numbers, _errors = parse(sys.stdin, ErrorStyle.NUMBERDEALERS_NG)
 	for number in numbers:
 		moment = datetime.fromtimestamp(number.create_at / 1000, timezone.utc)
 		moment_str = str(moment).replace("+00:00", "")

numberdealers/report_errors.py

@@ -14,21 +14,21 @@ def mention(message: parse_numberdealers.Message):
 
 def str_from_error(err):
 	if isinstance(err, parse_numberdealers.UnrecognizedNumber):
-		msg = f"- Unrecognized post {link(err.message.message, err.message)}"
+		msg = f"🚨 Unrecognized post {link(err.message.message, err.message)}"
 	elif isinstance(err, parse_numberdealers.EditedMessage):
-		msg = f"- Edited post {link(err.message.message, err.message)}"
+		msg = f"🚨 Edited post {link(err.message.message, err.message)}"
 	elif isinstance(err, parse_numberdealers.NonNumberMessage):
-		msg = f"- Non-number message {link(err.message.message, err.message)}"
+		msg = f"🚨 Non-number message {link(err.message.message, err.message)}"
 	elif isinstance(err, parse_numberdealers.ShouldHaveBeen):
-		msg = f"- {link(err.message.recognized_number, err.message)} should have been {err.expected_number}"
+		msg = f"🚨 {link(err.message.recognized_number, err.message)} should have been {err.expected_number}"
 	elif isinstance(err, parse_numberdealers.Duplicate):
-		msg = f"- Duplicate {link(err.message.recognized_number, err.message)}"
+		msg = f"🚨 Duplicate {link(err.message.recognized_number, err.message)}"
 	elif isinstance(err, parse_numberdealers.Stray):
-		msg = f"- Stray {link(err.message.recognized_number, err.message)}"
+		msg = f"🚨 Stray {link(err.message.recognized_number, err.message)}"
 	elif isinstance(err, parse_numberdealers.Skipped):
-		msg = f"- {link('Skipped', err.message)} {err.expected_number}"
+		msg = f"🚨 {link('Skipped', err.message)} {err.expected_number}"
 	elif isinstance(err, parse_numberdealers.Jump):
-		msg = f"- Going from {link(err.previous_message.recognized_number, err.previous_message)}" \
+		msg = f"🚨 Going from {link(err.previous_message.recognized_number, err.previous_message)}" \
 				f" to {link(err.message.recognized_number, err.message)}"
 
 	return msg + mention(err.message)
@@ -36,15 +36,20 @@ def str_from_error(err):
 
 def report_errors(errors):
 	if errors:
-		print("🚨 Errors: 🚨")
+		return list(map(str_from_error, errors))
-		print("\n".join(map(str_from_error, errors)))
 	else:
-		print("No errors! 🎉")
+		return []
 
 
 def main():
 	import sys
-	numbers, errors = parse_numberdealers.parse(sys.stdin)
+
+	error_style = {
+		"--numberdealers": parse_numberdealers.ErrorStyle.NUMBERDEALERS,
+		"--numberdealers-ng": parse_numberdealers.ErrorStyle.NUMBERDEALERS_NG
+	}[sys.argv[1]]
+
+	numbers, errors = parse_numberdealers.parse(sys.stdin, error_style)
 
 	if numbers == [] and errors == []:
 		print("No input data")
@@ -53,7 +58,10 @@ def main():
 			print("No valid number messages!")
 		else:
 			print(f"Checked from {numbers[0].recognized_number} up to {numbers[-1].recognized_number}")
-			report_errors(errors)
+			if not errors:
+				print("No errors! 🎉")
+			else:
+				print("\n".join(report_errors(errors)))
 
 
 if __name__ == "__main__":

report_numberdealers.py

@@ -1,34 +1,110 @@
 #!/usr/bin/env python3
 
 import sys
+from itertools import zip_longest
 from numberdealers import parse_numberdealers, times, report_errors, numbers_per_user
+from numberdealers.times import format_time
+from mdtables import Table, Column
 
 
 def main():
-	channel = sys.argv[1]
+	numberdealers_tail = sys.argv[1]
-	numbers, errors = parse_numberdealers.parse(sys.stdin)
+	numberdealers_history = sys.argv[2]
+	numberdealers_ng = sys.argv[3]
 
-	if numbers == [] and errors == []:
+	with \
-		print("No input data")
+			open(numberdealers_tail, "r") as nd_tail, \
-		return
+			open(numberdealers_history, "r") as nd_history, \
-	elif numbers == []:
+			open(numberdealers_ng, "r") as nd_ng:
-		print("No valid number messages!")
+		numbers_tail, errors_tail = parse_numberdealers.parse(nd_tail, parse_numberdealers.ErrorStyle.NUMBERDEALERS)
-		return
+		numbers_history, _errors = parse_numberdealers.parse(nd_history, parse_numberdealers.ErrorStyle.NUMBERDEALERS)
+		numbers_ng, errors_ng = parse_numberdealers.parse(nd_ng, parse_numberdealers.ErrorStyle.NUMBERDEALERS_NG)
+
+	assert numbers_tail
+	assert numbers_history
+	assert numbers_ng
+
+	table = Table(
+		Column('', alignment='right'),
+		Column('~NumberDealers'),
+		Column('~numberdealers-ng')
+	)
+
+	table.row(
+		"Stats for",
+		f"{numbers_history[0].recognized_number} up to {numbers_history[-1].recognized_number}",
+		f"{numbers_ng[0].recognized_number} up to {numbers_ng[-1].recognized_number}",
+	)
+
+	label = "Errors"
+	for error_line, error_line_ng in zip(
+			report_errors.report_errors(errors_tail) or [f"No errors (after {numbers_tail[-1].recognized_number})! 🎉"],
+			report_errors.report_errors(errors_ng) or ["No errors! 🎉"]
+	):
+		table.row(
+			label,
+			error_line,
+			error_line_ng
+		)
+		label = ""
+
+	table.row("", "", "")
+
+	a = times.analyze_times(numbers_history)
+	b = times.analyze_times(numbers_ng)
+	table.row(     "μ", format_time(a.avg),    format_time(b.avg))
+	table.row(     "σ", format_time(a.stdev),  format_time(b.stdev))
+	table.row("", "", "")
+	table.row(   "min", format_time(a.min),    format_time(b.min))
+	table.row(    "P5", format_time(a.perc5),  format_time(b.perc5))
+	table.row("median", format_time(a.med),    format_time(b.med))
+	table.row(   "P95", format_time(a.perc95), format_time(b.perc95))
+	table.row(   "max", format_time(a.max),    format_time(b.max))
+
+	print(table)
+	print()
+
+	print("```")
+	_stats_history = numbers_per_user.analyze_users(numbers_history)
+	stats_history = sorted(_stats_history.items(), key=lambda x: x[1], reverse=True)
+	_stats_ng = numbers_per_user.analyze_users(numbers_ng)
+	stats_ng = sorted(_stats_ng.items(), key=lambda x: x[1], reverse=True)
+	print(f"{'~NumberDealers':31s}  ~numberdealers-ng")
+	i = 0
+	others_count_hist = 0
+	others_names_hist = 0
+	others_count_ng = 0
+	others_names_ng = 0
+	for hist, ng in zip_longest(stats_history, stats_ng):
+		if hist is not None:
+			username_hist, count_hist = hist
+		else:
+			username_hist, count_hist = "", ""
+		if ng is not None:
+			username_ng, count_ng = ng
+		else:
+			username_ng, count_ng = "", ""
+
+		if i < 7:
+			print(f"{str(count_hist):>5s} {username_hist:25s}  {str(count_ng):>5s} {username_ng}")
+		else:
+			if hist is not None:
+				others_count_hist += count_hist
+				others_names_hist += 1
+			if ng is not None:
+				others_count_ng += count_ng
+				others_names_ng += 1
+		i += 1
+
+	if others_names_hist > 0 or others_names_ng > 0:
+		if others_names_hist > 0:
+			others_hist = f"{others_count_hist: 5d} [{others_names_hist} others]"
+		if others_names_ng > 0:
+			others_ng = f"{others_count_ng: 5d} [{others_names_ng} others]"
+		print(f"{others_hist:31s}  {others_ng}")
+
+	print("```")
 
-	print(f"##### Checked ~{channel} from {numbers[0].recognized_number} up to {numbers[-1].recognized_number}")
-	if "--no-errors" not in sys.argv[2:]:
-		print()
-		report_errors.report_errors(errors)
-	if "--no-times" not in sys.argv[2:]:
-		print()
-		print("```")
-		times.report_times(numbers)
-		print("```")
-	if "--no-users" not in sys.argv[2:]:
-		print()
-		print("```")
-		numbers_per_user.report_users(numbers)
-		print("```")
 
 
 if __name__ == "__main__":