From 66aea79de9e3813d278a9557b274179b692d101b Mon Sep 17 00:00:00 2001 From: Pieter Vander Vennet Date: Fri, 24 Feb 2023 20:17:31 +0100 Subject: [PATCH] Add statistics to import viewer --- Docs/Reasonings/PinJePunt_One_Year_Later.md | 93 ++++++++++ UI/ImportFlow/ImportViewerGui.ts | 177 ++++++++++++++++++-- 2 files changed, 258 insertions(+), 12 deletions(-) create mode 100644 Docs/Reasonings/PinJePunt_One_Year_Later.md diff --git a/Docs/Reasonings/PinJePunt_One_Year_Later.md b/Docs/Reasonings/PinJePunt_One_Year_Later.md new file mode 100644 index 000000000..9859b5d0b --- /dev/null +++ b/Docs/Reasonings/PinJePunt_One_Year_Later.md @@ -0,0 +1,93 @@ +# 'Pin je Punt' - one year later + +About a year ago, we launched a mapping campaign at the request from [_Visit Flanders_ (Toerisme Vlaanderen)](toerismevlaanderen.be/pin-je-punt). (The project is explained below) + +A part of the campaign involved a guided import. The agencies had many datasets lying around (e.g. about benches or picnic tables) which they wanted to have imported in OSM. As doing a data import is hard - and the data was sometimes outdated, we opted for another approach: for every possible feature, a map note was created containing a friendly explanation and instructions to open MapComplete - which would close the map note on behalf of the contributor, marking them "imported", "not found" or "not valid" depending on what the contributor chose. + +Most map notes are closed by now, but the central question in this analysis today is: _should remaining map notes be closed in batch, or do we leave them open for longer_? Note that input of the local community will be gathered as well. + +## + + +## The project in a nutshell + +> This is what I sent to the Belgian Mailing list on the 5th of february. Note that the actual launch was later, around the 4th of March + +Toerisme Vlaanderen will be launching an OpenStreetMap-based project on +the 14th of February. This is a rather big project which I'd like to +introduce to you with this email. The project consists of a few parts +which might have some impact: + +- A MapComplete theme with focus on some touristical POI's will be launched +- A guided survey/data import will be started +- Toerisme Vlaanderen will ask their partners to start mapping, so + hopefully we'll welcome a group of new mappers + + +This project will focus on the following POI's: + +- Benches and picnic tables +- (Public) toilets +- Playgrounds, +- electrical charging stations (with a focus on charging stations for + electrical bicycles) +- Bicycle pumps and repair stations +- and observation towers + + + *What is this project about?* + +/*Toerisme Vlaanderen*/ is a Flemish state agency which promotes tourism +in Flanders, together with the 5 provincial touristical offices and some +other organisations. +Historically, these 5 offices have each held their own small set of +geodata for typical items such as benches, public toilets, picnic +tables, playgrounds, ... to put those items on their maps. + +And of course, these offices have kept this data in their own +spreadsheets, in their own formats (except for one - which has been +using OSM for years now). + +Toerisme Vlaanderen would like to unify all these databases into +OpenStreetMap, increase the data quality of the items already there and +improve the surveying flow. + +This is where MapComplete comes in. *MapComplete* +is a web-app where one can show information about POI, can answer +questions about these POI and can add new points. Depending on the +chosen map, some categories of POI are shown. + +For this project, a theme showing (and asking information about) +benches, picnictables, playgrounds, charging stations, ... has been +created and will be launched on 14 februari/. (If you look around a bit, +you can already find a link to the theme, but another email will follow +when the project is live.)/ + + + A slow data import: methodology + +Of course, there is quite a bit of geodata laying around with the +provincial offices, which ideally ends up in OpenStreetMap too. + +For this, a slow data import has been started. Instead of dumping all +the data into OSM, a *map note* is created for every item that should be +checked. + +This map note is structured in such a way that a contributor can use it, +but MapComplete can also pick this up to show this to a contributor. +This contributor can then quickly add/import the new feature if they +found it, or they can mark the note as a duplicate or not existing +anymore - closing the note in at the same time. These notes also link to +both the wiki page and to the mapcomplete page where they can be easily +added. + +As with all things in life, this method has advantages and drawbacks: + +- The biggest advantage is exposure of the import to experienced + contributors. (Case in point: I've already had valuable response on a + few notes within 24hours of them going live) +- The import is tracked in OSM itself, containing a lot of information + and providing a flexible forum +- It is quick and easy to setup +- Others can make a similar note, which will be picked up by MapComplete + too! diff --git a/UI/ImportFlow/ImportViewerGui.ts b/UI/ImportFlow/ImportViewerGui.ts index 0b7e75e44..5a79da8ff 100644 --- a/UI/ImportFlow/ImportViewerGui.ts +++ b/UI/ImportFlow/ImportViewerGui.ts @@ -22,6 +22,7 @@ import { LoginToggle } from "../Popup/LoginButton" import { QueryParameters } from "../../Logic/Web/QueryParameters" import Lazy from "../Base/Lazy" import { Button } from "../Base/Button" +import ChartJs from "../Base/ChartJs" interface NoteProperties { id: number @@ -207,6 +208,138 @@ class MassAction extends Combine { } } +class Statistics extends Combine { + constructor(noteStates: NoteState[]) { + if (noteStates.length === 0) { + super([]) + return + } + // We assume all notes are created at the same time + let dateOpened = new Date(noteStates[0].dateStr) + for (const noteState of noteStates) { + const openDate = new Date(noteState.dateStr) + if (openDate < dateOpened) { + dateOpened = openDate + } + } + const today = new Date() + const daysBetween = (today.getTime() - dateOpened.getTime()) / (1000 * 60 * 60 * 24) + const ranges = { + dates: [], + is_open: [], + } + const closed_by: Record = {} + + for (const noteState of noteStates) { + const closing_user = noteState.props.comments.at(-1).user + if (closed_by[closing_user] === undefined) { + closed_by[closing_user] = [] + } + } + + for (let i = -1; i < daysBetween; i++) { + const dt = new Date(dateOpened.getTime() + 24 * 60 * 60 * 1000 * i) + let open_count = 0 + + for (const closing_user in closed_by) { + closed_by[closing_user].push(0) + } + + for (const noteState of noteStates) { + const openDate = new Date(noteState.dateStr) + if (openDate > dt) { + // Not created at this point + continue + } + if (noteState.props.closed_at === undefined) { + open_count++ + } else if ( + new Date(noteState.props.closed_at.substring(0, 10)).getTime() > dt.getTime() + ) { + open_count++ + } else { + const closing_user = noteState.props.comments.at(-1).user + const user_count = closed_by[closing_user] + user_count[user_count.length - 1] += 1 + } + } + + ranges.dates.push( + new Date(dateOpened.getTime() + i * 1000 * 60 * 60 * 24) + .toISOString() + .substring(0, 10) + ) + ranges.is_open.push(open_count) + } + + const labels = ranges.dates.map((i) => "" + i) + const data = { + labels: labels, + datasets: [ + { + label: "Total open", + data: ranges.is_open, + fill: false, + borderColor: "rgb(75, 192, 192)", + tension: 0.1, + }, + ], + } + + function r() { + return Math.floor(Math.random() * 256) + } + + for (const closing_user in closed_by) { + if (closed_by[closing_user].at(-1) <= 10) { + continue + } + data.datasets.push({ + label: "Closed by " + closing_user, + data: closed_by[closing_user], + fill: false, + borderColor: "rgba(" + r() + "," + r() + "," + r() + ")", + tension: 0.1, + }) + } + const importers = Object.keys(closed_by) + importers.sort((a, b) => closed_by[b].at(-1) - closed_by[a].at(-1)) + super([ + new ChartJs({ + type: "line", + data, + options: { + scales: { + yAxes: [ + { + ticks: { + beginAtZero: true, + }, + }, + ], + }, + }, + }), + new ChartJs({ + type: "doughnut", + data: { + labels: importers, + datasets: [ + { + label: "Closed by", + data: importers.map((k) => closed_by[k].at(-1)), + backgroundColor: importers.map( + (_) => "rgba(" + r() + "," + r() + "," + r() + ")" + ), + }, + ], + }, + }).SetClass("h-16"), + ]) + this.SetClass("block w-full h-64 border border-red") + } +} + class NoteTable extends Combine { private static individualActions: [() => BaseUIElement, string][] = [ [Svg.not_found_svg, "This feature does not exist"], @@ -381,22 +514,24 @@ class BatchView extends Toggleable { badges.push(toggle) }) - const fullTable = new NoteTable(noteStates, state) + const fullTable = new Combine([ + new NoteTable(noteStates, state), + new Statistics(noteStates), + ]) super( new Combine([ new Title(theme + ": " + intro, 2), new Combine(badges).SetClass("flex flex-wrap"), ]), + new VariableUiElement( filterOn.map((filter) => { if (filter === undefined) { return fullTable } - return new NoteTable( - noteStates.filter((ns) => ns.status === filter), - state - ) + const notes = noteStates.filter((ns) => ns.status === filter) + return new Combine([new NoteTable(notes, state), new Statistics(notes)]) }) ), { @@ -422,10 +557,13 @@ class ImportInspector extends VariableUiElement { url = "https://api.openstreetmap.org/api/0.6/notes/search.json?display_name=" + encodeURIComponent(userDetails["display_name"]) + - "&limit=10000&closed=730&sort=created_at&q=" + - encodeURIComponent(userDetails["search"] ?? "#import") + "&limit=10000&closed=730&sort=created_at&q=" + if (userDetails["search"] !== "") { + url += userDetails["search"] + } else { + url += "#import" + } } - const notes: UIEventSource< { error: string } | { success: { features: { properties: NoteProperties }[] } } > = UIEventSource.FromPromiseWithErr(Utils.downloadJson(url)) @@ -444,6 +582,11 @@ class ImportInspector extends VariableUiElement { if (userDetails["uid"]) { props = props.filter((n) => n.comments[0].uid === userDetails["uid"]) } + if (userDetails["display_name"] !== undefined) { + const display_name = userDetails["display_name"] + props = props.filter((n) => n.comments[0].user === display_name) + } + const perBatch: NoteState[][] = Array.from( ImportInspector.SplitNotesIntoBatches(props).values() ) @@ -462,6 +605,12 @@ class ImportInspector extends VariableUiElement { ] } contents.push(accordeon) + contents.push( + new Combine([ + new Title("Statistics for all notes"), + new Statistics([].concat(...perBatch)), + ]) + ) const content = new Combine(contents) return new LeftIndex( [ @@ -516,9 +665,13 @@ class ImportInspector extends VariableUiElement { ) { status = "invalid" } else if ( - ["imported", "erbij", "toegevoegd", "added"].some( - (keyword) => lastComment.toLowerCase().indexOf(keyword) >= 0 - ) + [ + "imported", + "erbij", + "toegevoegd", + "added", + "openstreetmap.org/changeset", + ].some((keyword) => lastComment.toLowerCase().indexOf(keyword) >= 0) ) { status = "imported" } else { @@ -559,7 +712,7 @@ class ImportViewerGui extends LoginToggle { (ud) => { const display_name = displayNameParam.data const search = searchParam.data - if (display_name !== "" && search !== "") { + if (display_name !== "" || search !== "") { return new ImportInspector({ display_name, search }, undefined) } return new ImportInspector(ud, state)