From a0a212e72520ad9dfa19f36422f2715c4160775f Mon Sep 17 00:00:00 2001 From: Londo <109172537+Londopy@users.noreply.github.com> Date: Mon, 3 Aug 2026 18:04:23 -0700 Subject: [PATCH 01/37] Stop database contention from blanking the board, and fix the legend --- .gitignore | 5 ++ CHANGELOG.md | 3 + README.md | 8 +-- app.py | 74 ++++++++++++++++++++-- docs/architecture.md | 6 +- docs/devpost.md | 4 +- research/literature.md | 95 ++++++++++++++++++++++++++++ static/scripts/map.js | 27 +++++++- static/styles/map.css | 47 ++++++++++---- tests/test_app.py | 136 +++++++++++++++++++++++++++++++++++++++++ 10 files changed, 377 insertions(+), 28 deletions(-) create mode 100644 research/literature.md diff --git a/.gitignore b/.gitignore index 94f9c26..ac9f7ef 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,11 @@ env/ # Database *.db +# WAL leaves two sidecar files beside the database. They are as much a local +# artefact as the .db itself, and a committed -wal is a committed transaction +# log from somebody else's machine. +*.db-wal +*.db-shm *.sqlite *.sqlite3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d8d156..4572bde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,9 @@ Disaster response tracker. Logs the volunteers going in, not just where the disa ### Fixed +- The board, the map and the feed could return an error page under load. All three run the silence check before answering, the check writes to the database, and SQLite lets one writer lock out every reader — so with a few people watching a board that refreshes every three seconds, a write could wait past its timeout and fail the whole response. The pages that exist to be watched during an emergency were the ones that broke when watched. Three changes: the database now uses a journal mode where readers and the writer stop blocking each other, the check runs at most once every thirty seconds instead of once per visitor per refresh, and a failed check is caught rather than allowed to become an error page. It deliberately does not record itself as having run when it fails, because a board showing green while nothing is checking is the worst outcome available. +- The map legend used the same red for *nobody going* and for *responder*, so two of its four keys said different things in the same colour. Colour was already carrying meaning — red, blue and green say whether anyone is coming — which left nothing for it to say about a person. The legend now uses shape the way the map already does: reports are teardrops, responders are a hollow ring. Colour means one thing on each. +- The map stopped telling the difference between having no signal and the server returning an error, so a failure looked like being offline. It now checks the response before believing it, and the warning appears above the map rather than underneath the statistics, where a phone screen never reached it. - The stamp that records when the silence check ran could take down the page it is displayed on. It runs before every read of the board, which polls every three seconds, so something that had been read-only started writing on every request — and a write that fails inside a before-request hook returns an error page instead of a board. It now fails quietly: the timestamp goes stale and the board reports that in amber, which is true, because a check we could not record is not a check we can claim. ## [1.0.2] - 2026-08-03 diff --git a/README.md b/README.md index 3ebc445..098c9b4 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ DiresQ tracks the people going into it.** [![CI](https://github.com/Skythe7/DiresQ/actions/workflows/ci.yml/badge.svg)](https://github.com/Skythe7/DiresQ/actions/workflows/ci.yml) [![Security](https://github.com/Skythe7/DiresQ/actions/workflows/security.yml/badge.svg)](https://github.com/Skythe7/DiresQ/actions/workflows/security.yml) -[![Tests](https://img.shields.io/badge/tests-551%20passing-brightgreen)](tests/test_app.py) +[![Tests](https://img.shields.io/badge/tests-560%20passing-brightgreen)](tests/test_app.py) [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) [![Limitations](https://img.shields.io/badge/limitations-written_down-f38ba8)](docs/limits.md) @@ -67,8 +67,8 @@ board turns red; the report on the right filed itself. That table is the night itself and does not move. It has kept growing since: -**25,151 lines written** — **19,033 lines of code** and 6,118 of -documentation — across **551 test functions**, 696 cases after +**25,400 lines written** — **19,279 lines of code** and 6,121 of +documentation — across **560 test functions**, 705 cases after parametrisation. Every one of those is checked by a test, so unlike the snapshot they cannot @@ -396,7 +396,7 @@ pip install -r requirements-dev.txt pytest -q ``` -551 test functions, which parametrisation expands into 696 cases, covering +560 test functions, which parametrisation expands into 705 cases, covering every route, the permission rules, feed ordering, staffing resolution, ETA parsing, overdue calculation, packet signing, the offline queues for both check-ins and reports, arrival-time duplicate detection, the auth guardrails diff --git a/app.py b/app.py index b64a0a1..a0ed83b 100644 --- a/app.py +++ b/app.py @@ -75,6 +75,13 @@ DATABASE = os.environ.get("DIRESQ_DB", "diresq.db") +# How long a statement waits for a lock before giving up. SQLite's own default +# is zero — it returns "database is locked" on the first attempt — and Python's +# driver default is five seconds. Named here because it is a real deadline: two +# gunicorn workers of four threads each are eight things contending, and this +# is how long the eighth is willing to queue. +SQLITE_BUSY_SECONDS = 5.0 + # How long a responder has to check in when they didn't give an ETA. DEFAULT_CHECKIN_MINUTES = 30 @@ -165,8 +172,32 @@ def note_login_failure(username: str) -> None: def get_db() -> sqlite3.Connection: if "db" not in g: - g.db = sqlite3.connect(DATABASE, detect_types=sqlite3.PARSE_DECLTYPES) + g.db = sqlite3.connect(DATABASE, detect_types=sqlite3.PARSE_DECLTYPES, + timeout=SQLITE_BUSY_SECONDS) g.db.row_factory = sqlite3.Row + + # busy_timeout first, so the journal_mode switch below waits its turn + # rather than failing outright on a database somebody else is using. + g.db.execute(f"PRAGMA busy_timeout = {int(SQLITE_BUSY_SECONDS * 1000)}") + + # WAL, because this app writes on reads. The board polls + # /api/responders every three seconds and that request runs the + # silence sweep and stamps the clock, so a read endpoint is also the + # busiest writer. In the default rollback journal a writer locks out + # every reader and a reader locks out the writer, so two people with + # the board open is enough to start returning "database is locked" — + # which Flask serves as a 500 on the exact endpoint the map needs. + # + # In WAL readers never block the writer and the writer never blocks + # readers, which is the shape of this workload. It is a property of + # the file, not the connection, so this is a no-op after the first. + g.db.execute("PRAGMA journal_mode = WAL") + + # Don't fsync on every commit. The durability being traded away is + # "the last few commits survive an OS crash"; on an instance whose + # whole filesystem is discarded when it sleeps, that was never true. + g.db.execute("PRAGMA synchronous = NORMAL") + g.db.execute("PRAGMA foreign_keys = ON") return g.db @@ -1459,13 +1490,46 @@ def security_headers(response): return response +# How often the sweep is actually worth running. The board polls +# /api/responders every three seconds *per viewer*, and the sweep is a join +# across every open assignment plus a write. Running it on each poll made the +# cost of the alarm scale with the number of people watching it, which is +# backwards: the sweep is about elapsed time, and elapsed time does not care +# how many browsers are open. +# +# Thirty seconds is thirty times inside the fifteen-minute escalation it +# guards, so nothing is ever noticed late, and it takes roughly nine writes +# in ten off the hot path. +SWEEP_EVERY_SECONDS = 30 + + @app.before_request def escalate_silence(): - if (request.method == "GET" - and request.endpoint in SWEEP_ON - and current_user() is not None): + if (request.method != "GET" + or request.endpoint not in SWEEP_ON + or current_user() is None): + return + + since = last_swept()["seconds"] + if since is not None and since < SWEEP_EVERY_SECONDS: + return + + try: sweep_silent_responders() - record_sweep() + except sqlite3.Error: + # Same reasoning as record_sweep below, and higher stakes. This runs in + # before_request, so an exception here does not fail the sweep — it + # fails the whole response, and the endpoints it is attached to are the + # board, the map and the feed. A lock contention blanking the three + # screens somebody uses during a flood is a far worse outcome than a + # sweep that runs thirty seconds later instead. + # + # Deliberately does not stamp: a sweep that did not finish must not + # claim it did, or the board shows green while nothing is checking. + get_db().rollback() + return + + record_sweep() def record_sweep() -> None: diff --git a/docs/architecture.md b/docs/architecture.md index a4a4525..df221a8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -12,8 +12,8 @@ something in the code, or take it apart. ## Shape Flask, SQLite through the `sqlite3` module, Jinja templates rendered on the -server, and a small amount of JavaScript that layers on top. Roughly 4,000 -lines of Python across six modules, 33 routes, six tables, 551 test +server, and a small amount of JavaScript that layers on top. Roughly 4,250 +lines of Python across six modules, 33 routes, six tables, 560 test functions. ``` @@ -290,7 +290,7 @@ by report and by responder, check-ins by `(responder, created_at DESC)`. ## Testing -551 test functions, which parametrisation expands into over six hundred +560 test functions, which parametrisation expands into over six hundred cases. Each gets a throwaway database via `tmp_path`, so order never matters and a failure can't poison the next test. diff --git a/docs/devpost.md b/docs/devpost.md index 05abaff..6ac582e 100644 --- a/docs/devpost.md +++ b/docs/devpost.md @@ -97,7 +97,7 @@ The fix was a lexicon of the categories a triage protocol calls immediate, writt It still gets one report in four wrong. That's survivable because it lands in a dropdown you control, next to the words that caused it — and it would be unacceptable if it were deciding anything. -**551 tests**, including 31 adversarial ones, a WCAG 2.1 AA audit, and a suite that reads our own documentation and fails the build when the numbers in it go stale. +**560 tests**, including 31 adversarial ones, a WCAG 2.1 AA audit, and a suite that reads our own documentation and fails the build when the numbers in it go stale. ## What we learned @@ -147,7 +147,7 @@ github-actions, ruff, wcag, start-triage, ics-214 ## Development tools -Sublime Text 4, Git and GitHub, GitHub Actions for CI, pytest (551 tests), +Sublime Text 4, Git and GitHub, GitHub Actions for CI, pytest (560 tests), ruff, bandit, pip-audit, gitleaks, Leaflet and OpenStreetMap, Astro for the documentation site, Adobe Premiere Pro for the demo video, Discord for team coordination. diff --git a/research/literature.md b/research/literature.md new file mode 100644 index 0000000..672a1b4 --- /dev/null +++ b/research/literature.md @@ -0,0 +1,95 @@ +# Literature matrix + +Working notes for a possible paper. **This file is not a claim about DiresQ.** +It exists to find out whether there is a contribution here, and it is allowed +to conclude that there is not. + +One row per source: what it claims, what evidence backs it, and what it does +not cover. The third column is the only one that becomes a paper. + +Rule set before starting, so it cannot be bent later: **if three sources +already make our argument, we change the argument rather than the wording.** + +--- + +## Status: two searches in, the claim has narrowed twice + +The Devpost story says *"we couldn't find one that keeps a list of the people +walking into it."* That is not going to survive review. Two rounds of +searching have found: + +1. **Spontaneous volunteer management is established doctrine.** FEMA has a + publication on it. ASPR TRACIE has a topic collection. Volunteer Reception + Centers, credentialing, and deployment logging are standard practice, and + commercial software does it. +2. **Responder accountability is a mature technical field.** Fire services run + PAR (Personnel Accountability Report) processes. Commercial accountability + software tracks responder location in real time. There are granted US + patents on interlinking electronic identities for personnel tracking at an + incident scene. +3. **The self-deployment problem is already named.** Practitioner literature + describes uncontrolled self-deployment — volunteers going straight to a + damaged neighbourhood without accountability — as a known hazard. + +So the problem is real *and already recognised*. That is the honest position. + +### What might still be left + +Every system found so far assumes **a command structure exists**: an incident +commander running accountability, a reception centre issuing credentials, an +organisation accepting the volunteer. The Harvey boat owner has none of those, +and is the person the doctrine describes as the problem to be prevented rather +than the user to be served. + +The candidate gap, stated narrowly enough to be defensible: + +> Accountability mechanisms for disaster response assume an organisational +> structure — incident command, credentialing, a reception centre. Volunteers +> who self-deploy bypass all three by definition. Can accountability be made +> *peer-to-peer and self-service*, so that it functions with no dispatcher and +> no command structure at all? + +The dead man's switch is the mechanism for that: escalation triggered by +**silence** rather than by a coordinator noticing. + +**This is still unverified.** It needs the searches below run before anyone +writes it down as fact. + +--- + +## Sources read + +| Source | Claim | Evidence | Does not cover | +| --- | --- | --- | --- | +| Starbird & Palen, *Voluntweeters* (CHI 2011) | | | | +| Starbird, *Crowdwork, Crisis and Convergence* (2012 diss.) | | | | +| Liu, *Crisis Crowdsourcing Framework* (CSCW 2014) | | | | +| FEMA, *Managing Spontaneous Volunteers in Times of Disaster* | | | | +| ASPR TRACIE, Volunteer Management topic collection | | | | +| ERHMS framework (responder health monitoring) | | | | +| Fire service PAR / accountability software | | | | +| US 8995946 / 9497610 (personnel accountability patents) | | | | + +## Searches still to run + +- [ ] ISCRAM proceedings: "spontaneous volunteer", "convergence", + "accountability", "self-deployment" +- [ ] Does anything cover accountability **without** an incident commander? +- [ ] Fritz & Mathewson (1957) on convergence — the origin of the term, and + whether physical convergence is still being studied or whether the field + moved to informational convergence after 2010 +- [ ] Crowdsource Rescue and similar Harvey-era tools: what did they actually + track, and is there any write-up of it? +- [ ] Prior art search on the two patents above — how close do they get? + +## Honest outcomes, ranked by likelihood + +1. **The gap is real but small.** A short design paper at a student venue or + ISCRAM, positioned as peer-to-peer accountability for the unaffiliated. +2. **The gap is already occupied.** Somebody has built or studied this. The + project stays a good system and a good story, and is not a paper. +3. **The gap is real and open.** Worth a proper venue — and would need user or + expert evaluation before submission, which does not exist yet. + +Outcome 2 is not a failure. Finding out early is the entire point of doing +this step first. diff --git a/static/scripts/map.js b/static/scripts/map.js index a770a78..5e42d8f 100644 --- a/static/scripts/map.js +++ b/static/scripts/map.js @@ -284,7 +284,16 @@ function responderPopup(responder) { } fetch("/api/responders") - .then(res => res.json()) + .then(res => { + // fetch only rejects on a network failure. A 500 or a redirect to the + // login page resolves normally, and .json() then throws a parse error + // on the HTML — same banner, but nothing anywhere says which of the + // three happened. Reading the status is the difference between "no + // signal" and "the server is broken", and only one of those is fixed + // by moving somewhere with bars. + if (!res.ok) throw new Error(`/api/responders returned ${res.status}`); + return res.json(); + }) .then(responders => { responders.forEach(responder => { @@ -311,11 +320,23 @@ fetch("/api/responders") // when the network is bad — which is when somebody is most likely to // be staring at this screen. Failing quietly leaves a map that looks // complete and is missing every responder on it, so say so. - .catch(() => { + .catch(err => { + console.warn("responder positions unavailable:", err); + const warning = document.createElement("div"); warning.className = "map-warning"; warning.setAttribute("role", "status"); warning.textContent = "Could not load responder positions. Reports are still shown."; - document.body.appendChild(warning); + + // Above the map, not at the end of the document. Appending to body put + // it below the statistics cards, off the bottom of a phone screen — + // the map looked complete, was missing every responder, and the notice + // saying so was somewhere you had to scroll to find. + const canvas = document.getElementById("map"); + if (canvas && canvas.parentNode) { + canvas.parentNode.insertBefore(warning, canvas); + } else { + document.body.appendChild(warning); + } }); \ No newline at end of file diff --git a/static/styles/map.css b/static/styles/map.css index 7d44066..bb09659 100644 --- a/static/styles/map.css +++ b/static/styles/map.css @@ -172,17 +172,9 @@ border-left:6px solid var(--green); border-radius:50%; -} - -.legend-marker.report{ - - background:var(--blue); - -} - -.legend-marker.responder{ - - background:var(--red); + /* Four items that wrap. Without this the swatches squash before the + labels do, and a legend of ovals explains nothing. */ + flex:none; } @@ -335,10 +327,43 @@ border-left:6px solid var(--green); 100% { box-shadow:0 2px 6px rgba(0,0,0,.5), 0 0 0 0 rgba(243,139,168,0); } } +/* The legend has to carry the same distinction the map does, and colour + cannot carry it. A responder's dot is coloured by *their* status, out of + the same red / blue / green the report pins use — an overdue responder is + red and so is a report nobody is going to. Two swatches that were both + --red made "Nobody going" and "Responder" indistinguishable, which is not a + palette mistake so much as the legend admitting the colour axis is already + full. + So the legend says it in shape, as the map already does: reports are + teardrops, responders are circles. Colour then means one thing per shape — + on a teardrop, whether anybody is going; on a circle, what that person is + doing. */ + +.legend-marker.nobody, +.legend-marker.coming, +.legend-marker.there{ + + /* Same geometry as .pin, at legend size. */ + border-radius:50% 50% 50% 0; + transform:rotate(-45deg); + +} + .legend-marker.nobody { background:var(--red, #f38ba8); } .legend-marker.coming { background:var(--blue, #89b4fa); } .legend-marker.there { background:var(--green, #a6e3a1); } +/* The neutral "available" grey getResponderColor() uses, deliberately not one + of the three above: no single swatch is honest about a marker whose colour + is read off the person. The ring is what makes it a circle at 12px, where + a bare dot and a rotated teardrop are hard to tell apart. */ +.legend-marker.responder{ + + background:transparent; + border:3px solid #a6adc8; + +} + .gaps-toggle{ /* 48px tall: this gets pressed on a phone, in the rain. */ min-height:48px; diff --git a/tests/test_app.py b/tests/test_app.py index 37e89ed..b93934c 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -5288,3 +5288,139 @@ def test_the_model_card_admits_it(self, client): def test_the_browser_copy_gets_the_threshold(self): model = json.loads(diresq.MODEL_FILE.read_text(encoding="utf-8")) assert model["min_known_share"] == classify.MIN_KNOWN_SHARE + + +class TestTheBoardSurvivesADatabaseUnderLoad: + """The board polls /api/responders every three seconds, per viewer, and + that request writes. Everything here is about the same failure: a write + that cannot get the lock must not take down the screens the writing exists + to make trustworthy. + """ + + def test_the_database_is_in_wal_mode(self, client): + # The default rollback journal has a writer lock out every reader. + # This app writes on its busiest read, so that mode makes contention + # between two viewers, which is not a load level worth failing at. + with diresq.app.app_context(): + mode = diresq.get_db().execute( + "PRAGMA journal_mode").fetchone()[0] + assert mode.lower() == "wal", f"journal_mode is {mode!r}, not WAL" + + def test_a_locked_database_does_not_take_down_the_board( + self, client, monkeypatch): + import sqlite3 + + def jammed(): + raise sqlite3.OperationalError("database is locked") + + monkeypatch.setattr(diresq, "sweep_silent_responders", jammed) + + # Every page the sweep hook is attached to. A 500 on any of these is + # the alarm's plumbing breaking the thing it was protecting. + for path in ("/", "/board", "/map", "/api/reports", "/api/responders"): + assert client.get(path).status_code == 200, path + + def test_a_sweep_that_failed_does_not_claim_it_ran( + self, client, monkeypatch): + import sqlite3 + + with diresq.app.app_context(): + diresq.get_db().execute("UPDATE system SET last_swept_at = NULL") + diresq.get_db().commit() + + monkeypatch.setattr(diresq, "sweep_silent_responders", + lambda: (_ for _ in ()).throw( + sqlite3.OperationalError("database is locked"))) + + client.get("/board") + + with diresq.app.app_context(): + stamp = diresq.get_db().execute( + "SELECT last_swept_at FROM system").fetchone()[0] + assert stamp is None, ( + "the stamp was written for a sweep that never finished — the board " + "would show green while nothing was checking") + + def test_the_sweep_does_not_run_on_every_poll(self, client, monkeypatch): + runs = [] + real = diresq.sweep_silent_responders + monkeypatch.setattr(diresq, "sweep_silent_responders", + lambda: (runs.append(1), real())[1]) + + for _ in range(10): + client.get("/api/responders") + + # Ten polls is about thirty seconds of one viewer watching the board. + # Before the throttle this was ten sweeps and ten writes. + assert len(runs) == 1, ( + f"{len(runs)} sweeps for 10 polls; the cost of the alarm is " + "scaling with the number of people watching it") + + def test_the_throttle_is_well_inside_the_escalation_it_guards(self): + assert diresq.SWEEP_EVERY_SECONDS * 4 < \ + diresq.SILENT_ESCALATE_MINUTES * 60, ( + "the sweep runs too rarely to notice silence on time") + + +class TestTheLegendDistinguishesAPersonFromAPlace: + """A responder marker is coloured by *their* status, out of the same + red/blue/green the report pins use. So colour cannot separate the two, and + a legend that tries reads as two identical swatches. + """ + + CSS = Path(__file__).resolve().parents[1] / "static/styles/map.css" + + def swatch(self, name): + """Every declaration that lands on .legend-marker.. + + Rules are collected across grouped selectors, not just the one that + names the state on its own — the shape is set for all three report + states in a single rule, and a helper that only read the individual + ones would report a teardrop as a circle. + """ + css = re.sub(r"/\*.*?\*/", "", self.CSS.read_text(encoding="utf-8"), + flags=re.S) + # `[^{}]` on both sides keeps the match inside one rule, so an @media + # wrapper cannot swallow the block after it. + blocks = re.findall( + r"([^{}]*\.legend-marker\.%s\b[^{}]*)\{([^{}]*)\}" % name, css) + assert blocks, f"no rule targets .legend-marker.{name}" + return " ".join(body for _, body in blocks) + + def colour(self, name): + """The colour a swatch actually paints, normalised. + + `var(--red)` and `var(--red, #f38ba8)` are the same colour written two + ways, and comparing the raw text would call them different — which is + exactly how two identical red dots shipped. + """ + decls = self.swatch(name) + tokens = re.findall(r"var\(\s*(--[\w-]+)|#[0-9a-fA-F]{3,8}", decls) + return {t for t in tokens if t} or set( + re.findall(r"#[0-9a-fA-F]{3,8}", decls)) + + def test_nobody_going_and_responder_are_not_the_same_swatch(self): + assert self.colour("nobody") != self.colour("responder"), ( + "'Nobody going' and 'Responder' paint the same colour — on the " + "map they are a report nobody is going to and an overdue person") + + def test_the_responder_swatch_is_not_one_of_the_report_colours(self): + responder = self.swatch("responder") + for state, colour in (("nobody", "--red"), ("coming", "--blue"), + ("there", "--green")): + assert colour not in responder, ( + f"the responder swatch reuses {colour}, which already means " + f"{state!r} on a report pin") + + def test_reports_are_teardrops_and_responders_are_circles(self): + # The shape is what carries it, here and on the map itself. + for state in ("nobody", "coming", "there"): + assert "50% 50% 50% 0" in self.swatch(state), ( + f"the {state!r} swatch is not the teardrop the pin is") + assert "50% 50% 50% 0" not in self.swatch("responder") + + def test_every_legend_item_in_the_template_has_a_rule(self): + html = (Path(__file__).resolve().parents[1] + / "templates/map.html").read_text(encoding="utf-8") + for name in re.findall(r'legend-marker (\w+)"', html): + self.swatch(name) From d40966cf4cafadba9502ab08812faa854c1b6f04 Mon Sep 17 00:00:00 2001 From: Londo <109172537+Londopy@users.noreply.github.com> Date: Mon, 3 Aug 2026 18:20:20 -0700 Subject: [PATCH 02/37] Put seeded responders where they can be seen --- CHANGELOG.md | 2 ++ README.md | 8 ++--- app.py | 39 ++++++++++++++++++++--- docs/architecture.md | 6 ++-- docs/devpost.md | 4 +-- tests/test_app.py | 73 ++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 119 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4572bde..314f1d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ Disaster response tracker. Logs the volunteers going in, not just where the disa ### Fixed - The board, the map and the feed could return an error page under load. All three run the silence check before answering, the check writes to the database, and SQLite lets one writer lock out every reader — so with a few people watching a board that refreshes every three seconds, a write could wait past its timeout and fail the whole response. The pages that exist to be watched during an emergency were the ones that broke when watched. Three changes: the database now uses a journal mode where readers and the writer stop blocking each other, the check runs at most once every thirty seconds instead of once per visitor per refresh, and a failed check is caught rather than allowed to become an error page. It deliberately does not record itself as having run when it fails, because a board showing green while nothing is checking is the worst outcome available. +- Seeded responders were on the map and could not be seen. They were placed within ninety metres of the report they had joined — close enough that the circle marking a person sat underneath the teardrop marking the place, and Leaflet draws places above people. Nothing was missing from the data, which is exactly why nothing looked wrong with it. They now stand off by about two hundred metres, so the thing this project is an argument about — several people converging on one address — is visible rather than inferred. +- The demo moved every time the server restarted. Responder positions were derived from Python's built-in string hashing, which is deliberately salted differently for each process, so a seed built on the promise that every visitor arrives at the same incident produced a different incident on every boot. It now uses a checksum that gives the same answer on every machine and every run. - The map legend used the same red for *nobody going* and for *responder*, so two of its four keys said different things in the same colour. Colour was already carrying meaning — red, blue and green say whether anyone is coming — which left nothing for it to say about a person. The legend now uses shape the way the map already does: reports are teardrops, responders are a hollow ring. Colour means one thing on each. - The map stopped telling the difference between having no signal and the server returning an error, so a failure looked like being offline. It now checks the response before believing it, and the warning appears above the map rather than underneath the statistics, where a phone screen never reached it. - The stamp that records when the silence check ran could take down the page it is displayed on. It runs before every read of the board, which polls every three seconds, so something that had been read-only started writing on every request — and a write that fails inside a before-request hook returns an error page instead of a board. It now fails quietly: the timestamp goes stale and the board reports that in amber, which is true, because a check we could not record is not a check we can claim. diff --git a/README.md b/README.md index 098c9b4..08692ed 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ DiresQ tracks the people going into it.** [![CI](https://github.com/Skythe7/DiresQ/actions/workflows/ci.yml/badge.svg)](https://github.com/Skythe7/DiresQ/actions/workflows/ci.yml) [![Security](https://github.com/Skythe7/DiresQ/actions/workflows/security.yml/badge.svg)](https://github.com/Skythe7/DiresQ/actions/workflows/security.yml) -[![Tests](https://img.shields.io/badge/tests-560%20passing-brightgreen)](tests/test_app.py) +[![Tests](https://img.shields.io/badge/tests-566%20passing-brightgreen)](tests/test_app.py) [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) [![Limitations](https://img.shields.io/badge/limitations-written_down-f38ba8)](docs/limits.md) @@ -67,8 +67,8 @@ board turns red; the report on the right filed itself. That table is the night itself and does not move. It has kept growing since: -**25,400 lines written** — **19,279 lines of code** and 6,121 of -documentation — across **560 test functions**, 705 cases after +**25,506 lines written** — **19,383 lines of code** and 6,123 of +documentation — across **566 test functions**, 711 cases after parametrisation. Every one of those is checked by a test, so unlike the snapshot they cannot @@ -396,7 +396,7 @@ pip install -r requirements-dev.txt pytest -q ``` -560 test functions, which parametrisation expands into 705 cases, covering +566 test functions, which parametrisation expands into 711 cases, covering every route, the permission rules, feed ordering, staffing resolution, ETA parsing, overdue calculation, packet signing, the offline queues for both check-ins and reports, arrival-time duplicate detection, the auth guardrails diff --git a/app.py b/app.py index a0ed83b..3073dd9 100644 --- a/app.py +++ b/app.py @@ -44,6 +44,8 @@ import base64 import binascii import csv +import math +import zlib import io import os import re @@ -2653,6 +2655,36 @@ def seed_minimal() -> tuple[int, int]: return len(accounts), len(reports) +def responder_position(username: str, lat: float, + lng: float) -> tuple[float, float]: + """Where to put a seeded responder relative to the report they joined. + + Two bugs lived in the one line this replaces. + + It used `hash(username)`, and Python salts string hashing per process — + so the demo moved every time the server booted. A seed whose whole point + is that every visitor arrives at the same incident cannot be built on a + number that changes at import. crc32 is stable across runs and versions. + + And it offset by at most ninety metres, which put the responder under the + report. Leaflet draws markers in a pane above circles, so the responder + was not missing from the map — it was painted underneath the teardrop and + invisible at any useful zoom. Standing them off on a ring is what makes + "six people on one street" a thing you can see rather than infer. + """ + n = zlib.crc32(username.encode()) + + bearing = math.radians(n % 360) + metres = 150 + (n // 360) % 120 + + # Degrees per metre. Longitude narrows with latitude, so the correction + # keeps the ring circular rather than an ellipse. + d_lat = metres * math.cos(bearing) / 111_320 + d_lng = metres * math.sin(bearing) / (111_320 * math.cos(math.radians(lat))) + + return round(lat + d_lat, 6), round(lng + d_lng, 6) + + def seed_data() -> tuple[int, int]: """Load a disaster already in progress. @@ -2731,17 +2763,16 @@ def ago(minutes): "AND status = 'unassigned'", (report_id,)) if checkin is not None: - # Scatter positions near the report so the map has something. report = db.execute( "SELECT lat, lng FROM reports WHERE id = ?", (report_id,) ).fetchone() - jitter = (hash(username) % 9 - 4) / 5000 + lat, lng = responder_position(username, report["lat"], + report["lng"]) db.execute(""" INSERT INTO checkins (responder, lat, lng, created_at, received_at) VALUES (?, ?, ?, ?, ?) - """, (who[username], report["lat"] + jitter, - report["lng"] - jitter, ago(checkin), ago(checkin))) + """, (who[username], lat, lng, ago(checkin), ago(checkin))) db.commit() return len(SEED_ACCOUNTS), len(SEED_REPORTS) diff --git a/docs/architecture.md b/docs/architecture.md index df221a8..5d5dad1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -13,7 +13,7 @@ something in the code, or take it apart. Flask, SQLite through the `sqlite3` module, Jinja templates rendered on the server, and a small amount of JavaScript that layers on top. Roughly 4,250 -lines of Python across six modules, 33 routes, six tables, 560 test +lines of Python across six modules, 33 routes, six tables, 566 test functions. ``` @@ -290,7 +290,7 @@ by report and by responder, check-ins by `(responder, created_at DESC)`. ## Testing -560 test functions, which parametrisation expands into over six hundred +566 test functions, which parametrisation expands into over six hundred cases. Each gets a throwaway database via `tmp_path`, so order never matters and a failure can't poison the next test. @@ -328,7 +328,7 @@ happens. | File | Lines | What it owns | | --- | --- | --- | -| `app.py` | 2,774 | Routes, queries, the rules, CLI commands | +| `app.py` | 2,869 | Routes, queries, the rules, CLI commands | | `classify.py` | 805 | The classifier, its corpus, and the browser export | | `transport.py` | 204 | The radio packet: layout, signing, verification | | `eta.py` | 175 | Free-text ETA parsing behind a confidence gate | diff --git a/docs/devpost.md b/docs/devpost.md index 6ac582e..90dec63 100644 --- a/docs/devpost.md +++ b/docs/devpost.md @@ -97,7 +97,7 @@ The fix was a lexicon of the categories a triage protocol calls immediate, writt It still gets one report in four wrong. That's survivable because it lands in a dropdown you control, next to the words that caused it — and it would be unacceptable if it were deciding anything. -**560 tests**, including 31 adversarial ones, a WCAG 2.1 AA audit, and a suite that reads our own documentation and fails the build when the numbers in it go stale. +**566 tests**, including 31 adversarial ones, a WCAG 2.1 AA audit, and a suite that reads our own documentation and fails the build when the numbers in it go stale. ## What we learned @@ -147,7 +147,7 @@ github-actions, ruff, wcag, start-triage, ics-214 ## Development tools -Sublime Text 4, Git and GitHub, GitHub Actions for CI, pytest (560 tests), +Sublime Text 4, Git and GitHub, GitHub Actions for CI, pytest (566 tests), ruff, bandit, pip-audit, gitleaks, Leaflet and OpenStreetMap, Astro for the documentation site, Adobe Premiere Pro for the demo video, Discord for team coordination. diff --git a/tests/test_app.py b/tests/test_app.py index b93934c..1f716c6 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -5362,6 +5362,79 @@ def test_the_throttle_is_well_inside_the_escalation_it_guards(self): "the sweep runs too rarely to notice silence on time") +class TestASeededResponderCanActuallyBeSeen: + """The demo seeds responder positions. Two things stopped them showing. + + The offset was at most ninety metres, and Leaflet draws markers in a pane + above circles — so the responder circle sat under the report teardrop and + was invisible. Nothing was missing from the data, which is why it looked + like nothing was wrong with the data. + + And the offset came from `hash(username)`, which Python salts per process, + so the positions moved on every boot of a seed built to be identical for + every visitor. + """ + + REPORT = (29.7834, -95.8321) + + @staticmethod + def metres(a, b): + import math + d_lat = (a[0] - b[0]) * 111_320 + d_lng = (a[1] - b[1]) * 111_320 * math.cos(math.radians(a[0])) + return math.hypot(d_lat, d_lng) + + def test_the_same_name_always_lands_in_the_same_place(self): + # `hash()` on a str is salted per process. A demo that claims every + # visitor sees the same incident cannot be built on it. + first = diresq.responder_position("londo", *self.REPORT) + assert first == diresq.responder_position("londo", *self.REPORT) + assert first == (29.784012, -95.830353), ( + "the seeded position moved — is this still crc32?") + + def test_the_source_does_not_use_pythons_salted_hash(self): + # Strip comments first. The docstring above the fix names the old + # call to explain it, and a check that matches its own explanation + # is the bug it is trying to prevent, one level up. + source = (diresq.SCHEMA.parent / "app.py").read_text(encoding="utf-8") + code = "\n".join(line for line in source.split("\n") + if not line.lstrip().startswith("#")) + assert "= hash(" not in code + assert "hash(username) %" not in code + + def test_a_responder_stands_clear_of_the_report(self): + # Under about a hundred metres it disappears beneath the teardrop. + for name in ("londo", "s.reyes", "j.okafor", "m.torres", "d.nguyen"): + away = self.metres( + diresq.responder_position(name, *self.REPORT), self.REPORT) + assert 120 <= away <= 300, f"{name} is {away:.0f} m from the report" + + def test_two_responders_on_one_report_do_not_stack(self): + names = ("londo", "s.reyes", "j.okafor", "m.torres", "d.nguyen") + spots = [diresq.responder_position(n, *self.REPORT) for n in names] + for i in range(len(spots)): + for j in range(i + 1, len(spots)): + apart = self.metres(spots[i], spots[j]) + assert apart > 40, ( + f"{names[i]} and {names[j]} are {apart:.0f} m apart") + + def test_the_ring_is_round_rather_than_an_ellipse(self): + # Longitude degrees narrow with latitude. Without the correction the + # ring stretches east-west and the spacing stops being what it says. + near = self.metres( + diresq.responder_position("londo", 0.0, 0.0), (0.0, 0.0)) + far = self.metres( + diresq.responder_position("londo", 60.0, 0.0), (60.0, 0.0)) + assert abs(near - far) < 5, "the offset changes with latitude" + + def test_the_seed_gives_somebody_a_position_to_show(self, client): + with diresq.app.app_context(): + diresq.seed_data() + placed = [r for r in diresq.fetch_responders() + if r.get("last_position")] + assert len(placed) >= 3, "the map has no responder circles to draw" + + class TestTheLegendDistinguishesAPersonFromAPlace: """A responder marker is coloured by *their* status, out of the same red/blue/green the report pins use. So colour cannot separate the two, and From a71b99fd26d916939bf2fe0ab7970edb084547d4 Mon Sep 17 00:00:00 2001 From: Londo <109172537+Londopy@users.noreply.github.com> Date: Mon, 3 Aug 2026 18:29:45 -0700 Subject: [PATCH 03/37] Say on the report page who has stopped answering --- CHANGELOG.md | 2 + README.md | 8 +-- app.py | 24 ++++++++- docs/architecture.md | 6 +-- docs/devpost.md | 4 +- static/styles/actions.css | 30 +++++++++++ static/styles/homepage.css | 12 +++++ templates/homepage.html | 9 ++++ templates/report.html | 17 +++++- tests/test_app.py | 106 +++++++++++++++++++++++++++++++++++++ 10 files changed, 207 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 314f1d9..1d847b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ Disaster response tracker. Logs the volunteers going in, not just where the disa ### Fixed - The board, the map and the feed could return an error page under load. All three run the silence check before answering, the check writes to the database, and SQLite lets one writer lock out every reader — so with a few people watching a board that refreshes every three seconds, a write could wait past its timeout and fail the whole response. The pages that exist to be watched during an emergency were the ones that broke when watched. Three changes: the database now uses a journal mode where readers and the writer stop blocking each other, the check runs at most once every thirty seconds instead of once per visitor per refresh, and a failed check is caught rather than allowed to become an error page. It deliberately does not record itself as having run when it fails, because a board showing green while nothing is checking is the worst outcome available. +- A report could say somebody was on scene while the board had them forty-five minutes overdue, and the report page gave no sign of it. Status is what a responder last told us; whether they are still answering is a different fact, and only one of the two pages knew it. Anybody opening a report to decide whether the address needed more help was counting a person who had gone silent as help. The page now says how many have stopped answering, marks each of them, and says plainly not to count them. Somebody who has cleared is not chased, because going home is not going quiet. +- On a phone, the only way to file a report was to open a menu labelled "Filter". The button lived in the sidebar, which slides off-screen below 768 pixels, so the one thing this app exists to do was behind the one control nobody would press to do it. There is now a Create Report button in the page itself on small screens, and exactly one of the two is ever present so a screen reader never hears it twice. - Seeded responders were on the map and could not be seen. They were placed within ninety metres of the report they had joined — close enough that the circle marking a person sat underneath the teardrop marking the place, and Leaflet draws places above people. Nothing was missing from the data, which is exactly why nothing looked wrong with it. They now stand off by about two hundred metres, so the thing this project is an argument about — several people converging on one address — is visible rather than inferred. - The demo moved every time the server restarted. Responder positions were derived from Python's built-in string hashing, which is deliberately salted differently for each process, so a seed built on the promise that every visitor arrives at the same incident produced a different incident on every boot. It now uses a checksum that gives the same answer on every machine and every run. - The map legend used the same red for *nobody going* and for *responder*, so two of its four keys said different things in the same colour. Colour was already carrying meaning — red, blue and green say whether anyone is coming — which left nothing for it to say about a person. The legend now uses shape the way the map already does: reports are teardrops, responders are a hollow ring. Colour means one thing on each. diff --git a/README.md b/README.md index 08692ed..8c8a656 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ DiresQ tracks the people going into it.** [![CI](https://github.com/Skythe7/DiresQ/actions/workflows/ci.yml/badge.svg)](https://github.com/Skythe7/DiresQ/actions/workflows/ci.yml) [![Security](https://github.com/Skythe7/DiresQ/actions/workflows/security.yml/badge.svg)](https://github.com/Skythe7/DiresQ/actions/workflows/security.yml) -[![Tests](https://img.shields.io/badge/tests-566%20passing-brightgreen)](tests/test_app.py) +[![Tests](https://img.shields.io/badge/tests-575%20passing-brightgreen)](tests/test_app.py) [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) [![Limitations](https://img.shields.io/badge/limitations-written_down-f38ba8)](docs/limits.md) @@ -67,8 +67,8 @@ board turns red; the report on the right filed itself. That table is the night itself and does not move. It has kept growing since: -**25,506 lines written** — **19,383 lines of code** and 6,123 of -documentation — across **566 test functions**, 711 cases after +**25,702 lines written** — **19,577 lines of code** and 6,125 of +documentation — across **575 test functions**, 720 cases after parametrisation. Every one of those is checked by a test, so unlike the snapshot they cannot @@ -396,7 +396,7 @@ pip install -r requirements-dev.txt pytest -q ``` -566 test functions, which parametrisation expands into 711 cases, covering +575 test functions, which parametrisation expands into 720 cases, covering every route, the permission rules, feed ordering, staffing resolution, ETA parsing, overdue calculation, packet signing, the offline queues for both check-ins and reports, arrival-time duplicate detection, the auth guardrails diff --git a/app.py b/app.py index 3073dd9..def35bf 100644 --- a/app.py +++ b/app.py @@ -591,13 +591,35 @@ def fetch_report(report_id: int) -> dict | None: item["responders"] = [dict(x) for x in get_db().execute(""" SELECT asg.id, asg.status, asg.eta, asg.staffing_vote, asg.joined_at, asg.position_mismatch, - acc.username, acc.capabilities, acc.id AS account_id + acc.username, acc.capabilities, acc.id AS account_id, + (SELECT MAX(created_at) FROM checkins c + WHERE c.responder = asg.responder) AS last_checkin FROM assignments asg JOIN accounts acc ON acc.id = asg.responder WHERE asg.report_id = ? ORDER BY asg.joined_at """, (report_id,)).fetchall()] + # Whether each of them is still answering. + # + # This page listed status and nothing else, and status is what somebody + # last *told* us — not whether they are still there to tell us anything. + # So a responder could read "on scene" here while the board had them + # forty-five minutes overdue, and a coordinator deciding whether this + # address needs more help would count them as coverage. + # + # That is this project's own argument failing on its own page. The feed + # groups duplicates so that six people on one incident cannot read as two + # comfortable rows; the same honesty has to apply to one person who has + # stopped answering. An unresponsive responder is not coverage. + for responder in item["responders"]: + responder["overdue"] = ( + responder["status"] != "cleared" + and is_overdue(responder["joined_at"], responder["eta"], + responder["last_checkin"])) + + item["overdue_here"] = sum(1 for r in item["responders"] if r["overdue"]) + # What the person looking at this page is allowed to press. Working it out # here keeps the permission rules in one place instead of scattered # through the template. diff --git a/docs/architecture.md b/docs/architecture.md index 5d5dad1..7563a88 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -13,7 +13,7 @@ something in the code, or take it apart. Flask, SQLite through the `sqlite3` module, Jinja templates rendered on the server, and a small amount of JavaScript that layers on top. Roughly 4,250 -lines of Python across six modules, 33 routes, six tables, 566 test +lines of Python across six modules, 33 routes, six tables, 575 test functions. ``` @@ -290,7 +290,7 @@ by report and by responder, check-ins by `(responder, created_at DESC)`. ## Testing -566 test functions, which parametrisation expands into over six hundred +575 test functions, which parametrisation expands into over six hundred cases. Each gets a throwaway database via `tmp_path`, so order never matters and a failure can't poison the next test. @@ -328,7 +328,7 @@ happens. | File | Lines | What it owns | | --- | --- | --- | -| `app.py` | 2,869 | Routes, queries, the rules, CLI commands | +| `app.py` | 2,891 | Routes, queries, the rules, CLI commands | | `classify.py` | 805 | The classifier, its corpus, and the browser export | | `transport.py` | 204 | The radio packet: layout, signing, verification | | `eta.py` | 175 | Free-text ETA parsing behind a confidence gate | diff --git a/docs/devpost.md b/docs/devpost.md index 90dec63..c8c0a56 100644 --- a/docs/devpost.md +++ b/docs/devpost.md @@ -97,7 +97,7 @@ The fix was a lexicon of the categories a triage protocol calls immediate, writt It still gets one report in four wrong. That's survivable because it lands in a dropdown you control, next to the words that caused it — and it would be unacceptable if it were deciding anything. -**566 tests**, including 31 adversarial ones, a WCAG 2.1 AA audit, and a suite that reads our own documentation and fails the build when the numbers in it go stale. +**575 tests**, including 31 adversarial ones, a WCAG 2.1 AA audit, and a suite that reads our own documentation and fails the build when the numbers in it go stale. ## What we learned @@ -147,7 +147,7 @@ github-actions, ruff, wcag, start-triage, ics-214 ## Development tools -Sublime Text 4, Git and GitHub, GitHub Actions for CI, pytest (566 tests), +Sublime Text 4, Git and GitHub, GitHub Actions for CI, pytest (575 tests), ruff, bandit, pip-audit, gitleaks, Leaflet and OpenStreetMap, Astro for the documentation site, Adobe Premiere Pro for the demo video, Discord for team coordination. diff --git a/static/styles/actions.css b/static/styles/actions.css index eafb466..e44ff4d 100644 --- a/static/styles/actions.css +++ b/static/styles/actions.css @@ -234,3 +234,33 @@ quietly undone. */ color:#a6adc8; } + + +/* Somebody on this report who has stopped answering. + Peach rather than red: red on this page already means HIGH priority, and + a second red would read as another way of saying the same thing. This is a + different fact — the incident's severity has not changed, our contact with + a person has. */ +.overdue-warning{ + margin:0 0 14px; + padding:10px 12px; + border-left:3px solid var(--peach, #fab387); + background:rgba(250,179,135,.10); + color:var(--peach, #fab387); + font-weight:600; + font-size:0.9rem; +} + +.responder.is-overdue{ + border-left-color:var(--peach, #fab387); +} + +.overdue-tag{ + font-size:0.7rem; + letter-spacing:0.08em; + font-weight:700; + color:var(--peach, #fab387); + border:1px solid var(--peach, #fab387); + border-radius:4px; + padding:2px 6px; +} diff --git a/static/styles/homepage.css b/static/styles/homepage.css index f9c02d1..9431465 100644 --- a/static/styles/homepage.css +++ b/static/styles/homepage.css @@ -154,6 +154,8 @@ gap:30px; } +.create-btn-mobile{ display:none; } + .create-btn{ background:var(--blue); @@ -252,6 +254,16 @@ gap:10px; @media(max-width:768px){ + /* Show the in-page button and hide the drawer's, so only one exists at + a time — display:none takes it out of the accessibility tree too. */ + .create-btn-mobile{ + display:block; + margin:0 16px 16px; + } + + aside .create-btn{ display:none; } + + .container{ grid-template-columns:1fr; diff --git a/templates/homepage.html b/templates/homepage.html index c89273d..4602aac 100644 --- a/templates/homepage.html +++ b/templates/homepage.html @@ -196,6 +196,15 @@

+ + + + Create Report + +