diff --git a/CHANGELOG.md b/CHANGELOG.md index 80840dc..5a060f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,32 @@ ## Mobile (not released yet) +- Analysers subtract the sender calibration: when the AVS4 QR carries one, the + AV offset shown and saved is the measured value minus the sender's own + screen-vs-speaker offset. Measure and Results show the raw value and the + sender calibration in a small line; saved results keep `rawOffsetMs` and + `senderCalibrationMs`. Done once in core C++, so iOS and Android behave the + same; this phone's own camera-vs-mic calibration is still subtracted on top, + once. "check" in Results also marks a different sender calibration. +- Debug → Clear calibration now asks first and says it clears both: the + analyser calibration and the sender calibration. +- Results: a sender's default name is its Title from the QR (the newest one), + then the short ID. A nickname you set always wins. +- iPhone test signal: Stop now clears the whole picture (the scale and the + marker stayed on screen before) and shows a large "Stopped". +- Calibrator wizard (iPhone): Settings → With calibrator. Step 2 measures + camera vs microphone with the calibrator board's flashes and clicks and saves + it like a clap calibration (method `calibrator-pulse`). Step 3 shows the + generator's calibration pattern (halves swap every 500 ms, a beep 100 ms + later) and saves screen vs speaker as the phone's sender calibration; the test + signal then sends it in the AVS4 QR (method `transponder`) instead of "not + calibrated". A TV fed by the container's calibration pattern can be measured + too (not saved). New camera view method `beginCalibrator(mode)`, generator + option `mode: 'calibration'` and `senderCalibration*` options. Not tested + with a real board yet (`docs/calibration.md`, `docs/calibrator.md`). +- Container generator: `AVSYNC_PATTERN=calibration` (live; Signal API + `pattern`, web form "Picture") sends the calibration pattern instead of the + test signal, for a phone with the calibrator on a TV or monitor. - iPhone generator: the source ID stays the same when the generator is stopped and started again. It changes only after a reinstall, a new app version or build, or another event rate; name, subtitle and image do not change it @@ -59,6 +85,12 @@ ## Container (not released yet) +- Analyzer (live and file): subtracts the sender calibration from the AVS4 QR + from `av_offset_ms` (and `deviation_ms`, grade, metrics). Events keep + `raw_av_offset_ms` and `sender_calibration_ms`; `/api/v1/measurement` has + `raw_offset_ms` and `sender_calibration_ms`; the web UI Input card shows + "AV sync raw"; the file result `summary` has `sender_calibration_ms` and + `median_raw_av_offset_ms`. Not calibrated: no change. - Generator: the default `AVSYNC_TEST_ID` is now `-` plus a short hash of the seed, the app version and the timing settings (event rate, audio timecode, video format, AV offset, audio/video lead, source emit offsets), diff --git a/CMakeLists.txt b/CMakeLists.txt index 42bb2fc..7fcd3c7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,6 +59,7 @@ add_library(avsync-core STATIC core/src/phone_generator.cpp core/src/picture_reader.cpp core/src/two_way.cpp + core/src/calibrator.cpp third_party/nayuki/qrcodegen.cpp ) target_include_directories(avsync-core @@ -70,7 +71,7 @@ set_source_files_properties(core/src/text.cpp PROPERTIES OBJECT_DEPENDS "${DEJAV target_link_libraries(avsync-core PUBLIC blend2d::blend2d) target_compile_options(avsync-core PRIVATE -Wall -Wextra -Wno-unused-parameter) -add_executable(avsync-core-tests tests/unit/test_core.cpp tests/unit/test_stage23.cpp tests/unit/test_timecode.cpp) +add_executable(avsync-core-tests tests/unit/test_core.cpp tests/unit/test_stage23.cpp tests/unit/test_timecode.cpp tests/unit/test_calibrator.cpp) target_include_directories(avsync-core-tests PRIVATE third_party/doctest) target_link_libraries(avsync-core-tests PRIVATE avsync-core) target_compile_options(avsync-core-tests PRIVATE -Wall -Wextra -Wno-unused-parameter) @@ -182,8 +183,11 @@ if(FFMPEG_FOUND AND NOT AVSYNC_CORE_ONLY) add_executable(avsync-phone-fixture tests/tools/phone_fixture.cpp) target_link_libraries(avsync-phone-fixture PRIVATE avsync-mxlio) target_compile_options(avsync-phone-fixture PRIVATE -Wall -Wextra -Wno-unused-parameter) + add_executable(avsync-calibration-flow tests/tools/calibration_flow.cpp) + target_link_libraries(avsync-calibration-flow PRIVATE avsync-mxlio) + target_compile_options(avsync-calibration-flow PRIVATE -Wall -Wextra -Wno-unused-parameter) if(MXL_RUNTIME_LIB_DIRS) - set_target_properties(avsync-sample-flow avsync-pattern-writer avsync-phone-fixture PROPERTIES BUILD_RPATH "${MXL_RUNTIME_LIB_DIRS}") + set_target_properties(avsync-sample-flow avsync-pattern-writer avsync-phone-fixture avsync-calibration-flow PROPERTIES BUILD_RPATH "${MXL_RUNTIME_LIB_DIRS}") endif() message(STATUS "libmxl found: live reader and writer are linked into mxl-avsync") else() diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md index 9d83ab0..32bb522 100644 --- a/IMPLEMENTATION_PLAN.md +++ b/IMPLEMENTATION_PLAN.md @@ -1352,3 +1352,57 @@ only calibration is compared for them. The 100-result limit stays, but a reference result is never dropped by it. `sourceId` 0 (a QR without a timecode ID) counts as unknown sender. + +## Calibrator wizard (2026-11-08) + +Maintainer design (2026-11-08): Settings → With calibrator, a three-step wizard +(plug in, camera, screen) for the calibrator board (contract +`docs/calibrator.md`, hardware on PR #69). The clap calibration is unchanged. + +- Analysis in core C++ (`calibrator.hpp`): sound onsets (click/beep), flash + onsets from row bands, camera pairing, display pairing, robust median and + uncertainty; the calibration pattern (frame based) and the beep. Unit tests + on synthetic signals (noise, missing pulses, stray sound, rolling shutter + with known, estimated and fitted readout, long exposure, slow screen, + sensor mismatch, external TV, failures). `PhoneAnalyzer::begin_calibrator` + runs the camera take (and an audio-only take for an external screen). +- Screen calibration is a mode of the generators, not a React view: iPhone + `generateStyled(..., {mode: 'calibration'})` (same display link, layers, + audio engine and player node), container `AVSYNC_PATTERN=calibration` + (live key, Signal API `pattern`, web form "Picture"). Android has no + generator (`supportsGenerator` false): skipped; its analyser could call + `begin_calibrator` through JNI later (not wired). +- Results: camera → analyser calibration record (method `calibrator-pulse`, + same validity as the clap); screen → `senderCalibration` in the session + state (method `transponder`, 90 days, same OS version, sent only on the same + audio output) → AVS4 QR of the phone generator. + +Deviations and choices: + +- AVS4 method: `transponder` for the sender calibration (the board answers + each screen swap with a click). `pulse-train` stays unused: the flash train + of camera mode calibrates the analyser, not a sender. +- Contract additions, no firmware change (listed in `docs/calibrator.md`): + the generator's sound is a 4 ms 1.5 kHz beep; camera mode uses every paired + pulse incl. the start pulses; display mode does not need the swap times + (beep 0–450 ms after the click). +- iOS rolling shutter: no readout time from the camera, full-frame exposure; + the readout is fitted over the pulses (the board's clock drifts against the + camera). The camera take locks exposure at the lowest ISO. +- The screen calibration needs the microphone, so the generator runs the + audio session in play-and-record (built-in speaker) instead of playback. + Same output node and latency compensation; to verify on a device (open + point in `docs/calibrator.md`). +- Constants still 0: buzzer delay, display path difference. Camera mode + subtracts 1.5 cm sound travel. +- Decided (Adrian, 2026-11-08): the analysers subtract the sender calibration + once in core (`Pairer::set_sender_calibration`), keep the raw value + (`raw_av_offset_ms` / `rawOffsetMs`) and the calibration next to it. The + phone's analyser calibration stays in the app (`liveView`), so nothing is + counted twice. +- The wizard and the calibrator take are iOS only (`beginCalibrator` is not in + the Android camera view yet); the Settings entry is hidden on Android. +- Debug → Clear calibration clears the sender calibration too (decided; with + a confirm that says both are cleared). +- The iPhone pattern size assumes about 6 pt per mm (current iPhones); the + outline is a placing guide, the pattern line itself is exact. diff --git a/SPECIFICATION.md b/SPECIFICATION.md index 0dcbc2e..fb642b7 100644 --- a/SPECIFICATION.md +++ b/SPECIFICATION.md @@ -113,7 +113,7 @@ The readers MUST NOT copy whole grains. The marker detector reads only its regio | Quantity | Definition | Needs | | --- | --- | --- | -| **AV offset** | `av_offset_ms = audio_event_time_ms − video_event_time_ms`, both at the measuring point | nothing beyond the signal | +| **AV offset** | `av_offset_ms = audio_event_time_ms − video_event_time_ms`, both at the measuring point; minus the sender calibration when the AVS4 QR carries one (§5.6; the measured value is then `raw_av_offset_ms`) | nothing beyond the signal | | **Deviation** | `deviation_ms = av_offset_ms − expected_av_offset_ms` (the generator's set offset) | the event's expected offset (QR or manifest) | | **Video latency** | time the video event is seen at the measuring point − time it was emitted | a common clock | | **Audio latency** | same for audio | a common clock | @@ -368,6 +368,7 @@ The rule of mxl-scopes §6, for the same reasons: a stuck reader takes down one | `test_id`, `event_id`, `seq`, `mode` | | | `video_event_time`, `audio_event_time` | ms (FILE) or TAI ns (LIVE), index times | | `av_offset_ms`, `expected_av_offset_ms`, `deviation_ms` | §3.1 | +| `raw_av_offset_ms`, `sender_calibration_ms` | only when the AVS4 QR carries a sender calibration: the measured offset before it was subtracted, and the calibration (§5.6) | | `av_offset_arrival_ms` | LIVE only | | `video_latency_ms`, `audio_latency_ms`, `video_index_shift_ms`, `audio_index_shift_ms` | LIVE with a reference only; else absent | | `video_source_emit_offset_ms`, `audio_source_emit_offset_ms`, `video_chain_latency_ms`, `audio_chain_latency_ms` | with a latency and a source emit offset (entered, or the flow's NMOS tag; §3.3); else absent | @@ -515,6 +516,7 @@ Precedence: environment > `AVSYNC_CONFIG_FILE` (flat JSON, keys are the variable | `AVSYNC_VIDEO_FORMAT` | `1080p50` | flow | §5.4 | | `AVSYNC_AUDIO_CHANNELS` | 2 | flow | 1–16 | | `AVSYNC_AV_OFFSET_MS` | 0 | live | −500…+500, set offset | +| `AVSYNC_PATTERN` | `test` | live | `test` (the AV sync test signal) or `calibration`: the calibrator board's pattern (halves swap every 500 ms, 1.5 kHz beep 100 ms after each swap at the chirp level; docs/calibrator.md). The flows and test id stay the same | | `AVSYNC_CHIRP_LEVEL_DBFS` | −12 | live | −40…−1 | | `AVSYNC_AUDIO_ENABLE` | `true` | live | `false` mutes the chirps (silence, flows keep running) | | `AVSYNC_AUDIO_LEAD_MS` | 20 | restart | how far ahead of TAI audio is written | @@ -667,6 +669,7 @@ The camera and microphone have different capture latencies (OS buffers, exposure - The app measures the rig, stores the capture offset per device model and OS version with the date, the method, the number of events and the spread, and subtracts it from later measurements. - A calibration expires after an OS update or 90 days. Uncalibrated measurements are allowed and shown as **uncalibrated** in orange, with no grade colour. - Expected accuracy is unknown until measured on real devices; the app shows the measured spread of its own calibration as its uncertainty. +- **Calibrator board** (maintainer design, 2026-11-08; contract docs/calibrator.md): Settings → With calibrator. Camera mode (flash + click on the rear camera) gives the capture calibration above (method `calibrator-pulse`, same record and validity as the clap). Display mode (the generator's calibration pattern, the board clicks at each swap, the generator beeps D = 100 ms later) gives the phone's **sender calibration** (screen vs speaker, AVS4 method `transponder`), which the phone generator then sends in the QR. The pattern is a mode of the generators themselves (same display and audio path); the container has it too (`AVSYNC_PATTERN`). Analysis in `core/…/calibrator.hpp`. Every analyser (phone, container, file) subtracts the sender calibration from the measured AV offset once, in core (`Pairer`), and keeps the raw value and the calibration next to it; "not calibrated" changes nothing. The phone's own capture calibration is subtracted separately (app), so neither is counted twice. Debug → Clear calibration clears both. --- @@ -785,10 +788,10 @@ Family look: header with label, role, state and registration pills, version; tab | Method | Path | Role | Purpose | | --- | --- | --- | --- | | GET | `/api/v1/info` | both | version, core version, MXL revision, role, label | -| GET | `/api/v1/status` (also `/statusz`) | both | readers or writers, NMOS summary; analyzer: `signal`, the source the picture QR names (`source_id`, `test_id`, `name`, `live`, `rate`, `format`, `estimated_source`, `sender_calibration` {`method`, and when calibrated `offset_ms`, `uncertainty_ms`, `date`}), null without a QR | +| GET | `/api/v1/status` (also `/statusz`) | both | readers or writers, NMOS summary; analyzer: `signal`, the source the picture QR names (`source_id`, `test_id`, `name`, `live`, `rate`, `format`, `estimated_source`, `sender_calibration` {`method`, and when calibrated `offset_ms`, `uncertainty_ms`, `date`}; analysers subtract it), null without a QR | | GET/PATCH | `/api/v1/generator` | gen | offset, level, mute, texts, rate; GET also shows `test_id` | | POST | `/api/v1/generator/stop`, `/start` | gen | §11 | -| GET | `/api/v1/measurement` | ana | current grade, offset, latencies, window statistics | +| GET | `/api/v1/measurement` | ana | current grade, offset, latencies, window statistics; with a sender calibration also `raw_offset_ms` and `sender_calibration_ms` | | GET | `/api/v1/events?since=&limit=` | ana | the newest matching event records, newest first, plus `total` | | GET | `/api/v1/statistics` | ana | window and since-start statistics (§8.2) | | POST | `/api/v1/statistics/reset` | ana | new statistics epoch (the event log keeps everything) | @@ -1057,7 +1060,7 @@ integers big-endian: | 5 | event sequence k (40 bits) | | 8 | video send time T_k in ns (TAI for LIVE, media timeline for FILE) | | 3 | source AV offset in 1/16 ms, signed (±524 s) | -| 6, only with bit 3 | sender calibration: method u8 (1 clap with a second phone, 2 transponder calibrator, 3 pulse-train), offset i16 in 0.1 ms (sound output minus picture output, positive: sound later; same sign as `av_offset_ms`, §3.2), uncertainty u8 in 0.5 ms, day u16 since 2020-01-01 (0 unknown) | +| 6, only with bit 3 | sender calibration: method u8 (1 clap with a second phone, 2 transponder calibrator, 3 pulse-train), offset i16 in 0.1 ms (sound output minus picture output, positive: sound later; same sign as `av_offset_ms`, §3.2; analysers subtract it from the measured offset), uncertainty u8 in 0.5 ms, day u16 since 2020-01-01 (0 unknown) | | 1 + n | test id: n characters (1–64) of `0-9a-zA-Z-_`, 6 bits each; the source id is CRC-32 of the id. n = 0: the 4-byte source id alone follows (the id did not fit or does not give that source id); the analyzer then shows `source-<8 hex>` | | rest | generator name, UTF-8, 0–24 bytes; cosmetic, not part of the id | diff --git a/app/config.cpp b/app/config.cpp index 5f2c42e..89d0bd5 100644 --- a/app/config.cpp +++ b/app/config.cpp @@ -125,7 +125,7 @@ bool is_restart_key(const std::string& key) bool is_live_key(const std::string& key) { return key == "AVSYNC_AV_OFFSET_MS" || key == "AVSYNC_CHIRP_LEVEL_DBFS" || key == "AVSYNC_AUDIO_ENABLE" - || key == "AVSYNC_EVENT_RATE" || key == "AVSYNC_AUDIO_TIMECODE" + || key == "AVSYNC_EVENT_RATE" || key == "AVSYNC_AUDIO_TIMECODE" || key == "AVSYNC_PATTERN" || key == "AVSYNC_TEST_ID" || key == "AVSYNC_TITLE" || key == "AVSYNC_SOURCE" || key == "AVSYNC_SUBTITLE" || key == "AVSYNC_NOTES" || key == "AVSYNC_AUDIO_CHANNEL" || key == "AVSYNC_AUDIO_THRESHOLD" || key == "AVSYNC_QR_RATE" || key == "AVSYNC_PASS_MIN_MS" || key == "AVSYNC_PASS_MAX_MS" || key == "AVSYNC_WARN_MIN_MS" || key == "AVSYNC_WARN_MAX_MS" @@ -172,6 +172,7 @@ Config load_config(const std::map& extra) }; config.role = pick("AVSYNC_ROLE", ""); config.audio_timecode = pick("AVSYNC_AUDIO_TIMECODE", "1200"); + config.pattern = pick("AVSYNC_PATTERN", "test"); config.event_rate = event_rate_value(pick("AVSYNC_EVENT_RATE", "2")); config.chirp_ms = static_cast(num(pick("AVSYNC_CHIRP_MS", "30"), "AVSYNC_CHIRP_MS")); config.chirp_low_hz = num(pick("AVSYNC_CHIRP_LOW_HZ", "700"), "AVSYNC_CHIRP_LOW_HZ"); @@ -235,6 +236,7 @@ Config load_config(const std::map& extra) config.log_format = pick("LOG_FORMAT", "json"); config.shutdown_timeout_s = static_cast(num(pick("SHUTDOWN_TIMEOUT_S", "10"), "SHUTDOWN_TIMEOUT_S")); require(config.audio_timecode == "legacy" || config.audio_timecode == "600" || config.audio_timecode == "1200", "AVSYNC_AUDIO_TIMECODE must be legacy, 600 or 1200"); + require(config.pattern == "test" || config.pattern == "calibration", "AVSYNC_PATTERN must be test or calibration"); if (config.audio_timecode != "legacy") { require(config.event_rate == 1 || config.event_rate == 2, "timecode supports 1 or 2 events/s"); require(config.audio_timecode != "600" || config.event_rate == 1, "600 baud requires 1 event/s"); @@ -264,6 +266,7 @@ nlohmann::json config_settings(const Config& config) return { {"AVSYNC_ROLE", config.role}, {"AVSYNC_AUDIO_TIMECODE", config.audio_timecode}, + {"AVSYNC_PATTERN", config.pattern}, {"AVSYNC_EVENT_RATE", std::to_string(config.event_rate)}, {"AVSYNC_CHIRP_MS", std::to_string(config.chirp_ms)}, {"AVSYNC_CHIRP_LOW_HZ", std::to_string(config.chirp_low_hz)}, @@ -314,6 +317,7 @@ void apply_settings(Config& config, const nlohmann::json& flat, std::vector(num(value, key.c_str())); else if (key == "AVSYNC_CHIRP_LOW_HZ") next.chirp_low_hz = num(value, key.c_str()); @@ -351,6 +355,7 @@ void apply_settings(Config& config, const nlohmann::json& flat, std::vector= -500 && next.offset_ms <= 500, "AVSYNC_AV_OFFSET_MS"); require(next.audio_timecode == "legacy" || next.audio_timecode == "600" || next.audio_timecode == "1200", "AVSYNC_AUDIO_TIMECODE must be legacy, 600 or 1200"); + require(next.pattern == "test" || next.pattern == "calibration", "AVSYNC_PATTERN must be test or calibration"); if (next.audio_timecode != "legacy") { require(next.event_rate == 1 || next.event_rate == 2, "timecode supports 1 or 2 events/s"); require(next.audio_timecode != "600" || next.event_rate == 1, "600 baud requires 1 event/s"); diff --git a/app/config.hpp b/app/config.hpp index 184760b..0a6db1d 100644 --- a/app/config.hpp +++ b/app/config.hpp @@ -13,6 +13,7 @@ struct Config { std::string role; int event_rate = 2; std::string audio_timecode = "1200"; // legacy, 600, 1200 + std::string pattern = "test"; // generator output: test (the AV sync test signal) or calibration (calibrator board) int chirp_ms = 30; double chirp_low_hz = 700; double chirp_high_hz = 2500; diff --git a/app/file_tool.cpp b/app/file_tool.cpp index 8cb083f..c8f7c17 100644 --- a/app/file_tool.cpp +++ b/app/file_tool.cpp @@ -773,6 +773,7 @@ int run_analyze(const AnalyzeRequest& request, std::string& error) reference.sync_state = "unknown"; const std::string test_id = have_manifest ? manifest.value("test_id", qr ? qr->test_id : "TEST") : (qr ? qr->test_id : "TEST"); Pairer pairer(detect.period_ms, GradeWindows{}, reference, "FILE", test_id); + if (qr) pairer.set_sender_calibration(qr->calibration); for (const auto& video : videos) { double expected = 0; if (have_manifest) { diff --git a/app/report.cpp b/app/report.cpp index 6669b85..53f18f4 100644 --- a/app/report.cpp +++ b/app/report.cpp @@ -45,6 +45,10 @@ nlohmann::json measurement_to_json(const Measurement& m) } j["av_offset_ms"] = opt_num(m.av_offset_ms); j["deviation_ms"] = opt_num(m.deviation_ms); + if (m.sender_calibration_ms) { + j["raw_av_offset_ms"] = opt_num(m.raw_av_offset_ms); + j["sender_calibration_ms"] = *m.sender_calibration_ms; + } if (m.av_offset_arrival_ms) { j["av_offset_arrival_ms"] = *m.av_offset_arrival_ms; } @@ -156,6 +160,15 @@ nlohmann::json result_document(const std::string& test_id, const std::string& mo if (bundle.offset.median) { sentence = describe_offset(*bundle.offset.median).line; } + nlohmann::json summary = {{"sentence", sentence}, {"valid", bundle.offset.valid}, {"discarded", bundle.offset.discarded}, + {"median_av_offset_ms", opt_num(bundle.offset.median)}}; + // AVS4: the sender calibration the events have subtracted (one QR, so one value), and the median before it. + for (auto it = events.rbegin(); it != events.rend(); ++it) { + if (!it->sender_calibration_ms) continue; + summary["sender_calibration_ms"] = *it->sender_calibration_ms; + if (bundle.offset.median) summary["median_raw_av_offset_ms"] = *bundle.offset.median + *it->sender_calibration_ms; + break; + } return { {"schema", kResultSchema}, {"test_id", test_id}, @@ -164,8 +177,7 @@ nlohmann::json result_document(const std::string& test_id, const std::string& mo {"reference", {{"type", reference.type}, {"domain", reference.domain}, {"sync_state", reference.sync_state}, {"accuracy_ns", reference.accuracy}}}, {"stream", stream}, - {"summary", {{"sentence", sentence}, {"valid", bundle.offset.valid}, {"discarded", bundle.offset.discarded}, - {"median_av_offset_ms", opt_num(bundle.offset.median)}}}, + {"summary", summary}, {"statistics", {{"window", statistics_to_json(bundle.offset)}, {"deviation", statistics_to_json(bundle.deviation)}, {"video_latency", statistics_to_json(bundle.video_latency)}, {"audio_latency", statistics_to_json(bundle.audio_latency)}}}, @@ -242,6 +254,15 @@ std::string human_report(const nlohmann::json& result) } os << "\n"; } + if (result.contains("summary") && result["summary"].contains("sender_calibration_ms")) { + const double c = result["summary"]["sender_calibration_ms"].get(); + os << " sender calibration " << (c >= 0 ? "+" : "") << c << " ms subtracted (QR)"; + if (result["summary"]["median_raw_av_offset_ms"].is_number()) { + const double raw = result["summary"]["median_raw_av_offset_ms"].get(); + os << ", raw median " << (raw >= 0 ? "+" : "") << raw << " ms"; + } + os << "\n"; + } double expected = 0; if (!result["events"].empty() && result["events"][0].contains("expected_av_offset_ms")) { expected = result["events"][0]["expected_av_offset_ms"].get(); diff --git a/core/include/avsync/avsync.hpp b/core/include/avsync/avsync.hpp index 60c083d..0266589 100644 --- a/core/include/avsync/avsync.hpp +++ b/core/include/avsync/avsync.hpp @@ -557,6 +557,10 @@ struct Measurement { std::optional av_offset_ms; double expected_av_offset_ms = 0; std::optional deviation_ms; + // AVS4 sender calibration (the generator's own sound-minus-picture output offset), only when the + // QR names one: av_offset_ms and deviation_ms have it subtracted, raw_av_offset_ms is the value before. + std::optional raw_av_offset_ms; + std::optional sender_calibration_ms; std::optional av_offset_arrival_ms; std::optional video_latency_ms; std::optional audio_latency_ms; @@ -764,6 +768,8 @@ class Pairer { Pairer(double period_ms, GradeWindows windows, ReferenceInfo reference, std::string mode, std::string test_id); void set_expected_offset(double ms) { default_offset_ms_ = ms; } void set_reference(ReferenceInfo reference) { reference_ = std::move(reference); } + // AVS4: the sender calibration from the QR; events paired from now on have it subtracted. + void set_sender_calibration(SenderCalibration calibration) { sender_calibration_ = std::move(calibration); } // Live: also report audio events no video claimed ("missing_video"), and drop closed events and // old observations instead of keeping the whole history (results() then stays empty). void set_live_stream(bool on) { live_stream_ = on; } @@ -788,6 +794,7 @@ class Pairer { std::string mode_; std::string test_id_; double default_offset_ms_ = 0; + SenderCalibration sender_calibration_; std::vector videos_; std::vector audios_; std::vector audio_used_; diff --git a/core/include/avsync/calibrator.hpp b/core/include/avsync/calibrator.hpp new file mode 100644 index 0000000..822fafe --- /dev/null +++ b/core/include/avsync/calibrator.hpp @@ -0,0 +1,98 @@ +#pragma once + +// The AV Sync calibrator board (docs/calibrator.md, signal contract v1): what the +// apps do with its flashes and clicks, and the calibration pattern every generator +// can show. Pure maths on samples and frames; no I/O. Times are milliseconds on one +// capture clock (camera and microphone of the same phone). + +#include +#include +#include +#include + +namespace avsync { + +// Numbers from the contract ("Timing summary"). +struct CalibratorContract { + static constexpr double click_hz = 2700; // buzzer: 3 cycles of a square wave + static constexpr double led_pulse_ms = 1.5; // camera mode LED on time + static constexpr double swap_period_ms = 500; // display mode: pattern halves swap + static constexpr double sound_delay_ms = 100; // D: generator sound after each swap + static constexpr double beep_hz = 1500; // the generator's own sound (not the buzzer's 2.7 kHz) + static constexpr double beep_ms = 4; // 6 cycles at 1.5 kHz + static constexpr double sound_ms_per_cm = 0.0291; // speed of sound, 343 m/s +}; + +// A short sound in a recording: a calibrator click or the generator's beep. +struct SoundOnset { + double time_ms = 0; // where the envelope first reaches 25 % of its peak + double peak = 0; // envelope peak (RMS over 0.25 ms, full scale 1) + double click_share = 0; // energy at 2.7 kHz / (2.7 kHz + 1.5 kHz) in the 3 ms after the onset + bool click() const { return click_share >= 0.5; } +}; +std::vector find_sound_onsets(const float* mono, size_t count, int sample_rate, double start_ms); + +// One camera frame, reduced to the mean luma of equal row bands (top to bottom, 0..255). +struct FlashFrame { + double time_ms = 0; // exposure start of the first row + double exposure_ms = 0; // 0: unknown + double readout_ms = 0; // first to last row; 0: unknown + std::vector rows; +}; +struct FlashOnset { + double time_ms = 0; + bool edge_in_frame = false; // the start of the flash lies inside the frame (rolling shutter edge seen) + double row = 0; // where that edge is, 0 top .. 1 bottom + bool row_timed = false; // time_ms includes the row's readout delay (readout known or estimated) +}; +// Start of each LED flash. When no frame knows its readout time, it is estimated from +// the height of the lit band (max(exposure, LED time) / band height); *readout_ms +// returns what was used (0: none, then camera_calibration fits it). +std::vector find_flash_onsets(const std::vector& frames, double led_ms = CalibratorContract::led_pulse_ms, + double* readout_ms = nullptr); + +struct CalibratorResult { + bool ok = false; + std::string error; // for the user when !ok + double offset_ms = 0; // camera: microphone minus camera; display: sound output minus picture output + double uncertainty_ms = 0; // standard error of the median of the pairs kept + double spread_ms = 0; // 10-90 % range of the pairs kept + int pairs = 0; // pairs kept + int rejected = 0; // pairs dropped as outliers + double direction_ms = 0; // display: median of one swap direction minus the other (sensor mismatch) + double readout_ms = 0; // camera: readout fitted from the pulses (0 when the frames knew it or too few edges) + std::vector values; // per kept pair +}; + +struct CameraCalibrationOptions { + double buzzer_delay_ms = 0; // coil on to sound; still to be measured per buzzer type (contract open point) + double mic_distance_cm = 1.5; // buzzer to microphone + int min_pairs = 8; +}; +// Camera mode: flash and click start together; offset = click - flash per pulse. When the +// flash times are not row-timed, the readout is fitted: the flash phase against the frames +// is random, so (click - flash) grows linearly with the edge row by the readout time. +CalibratorResult camera_calibration(const std::vector& flashes, const std::vector& sounds, + const CameraCalibrationOptions& options = {}); + +struct DisplayCalibrationOptions { + double sound_delay_ms = CalibratorContract::sound_delay_ms; + double buzzer_delay_ms = 0; + double click_minus_speaker_path_cm = 0; // calibrator-to-mic distance minus speaker-to-mic distance + int min_pairs = 8; +}; +// Display mode: each swap gives a click (when the screen changes) and then the generator's +// beep D after the swap (when the speaker plays it). offset = (beep - click) - D, so the +// microphone delay cancels. Both swap directions count equally. +CalibratorResult display_calibration(const std::vector& sounds, const DisplayCalibrationOptions& options = {}); + +// The calibration pattern, the same for all generators. Frames count from the pattern +// start at num/den frames per second; period k (swap k at its first frame) lasts 500 ms +// on the frame grid. In even periods the left half is white. +uint64_t calibration_period(uint64_t frame, int64_t num, int64_t den); +bool calibration_left_white(uint64_t frame, int64_t num, int64_t den); +uint64_t calibration_swap_frame(uint64_t k, int64_t num, int64_t den); +// The generator's beep (mono, starts at sample 0, peak 0.5), played D after each swap. +std::vector calibration_beep(int sample_rate); + +} // namespace avsync diff --git a/core/include/avsync/phone_analyzer.hpp b/core/include/avsync/phone_analyzer.hpp index 44706e6..3bfb852 100644 --- a/core/include/avsync/phone_analyzer.hpp +++ b/core/include/avsync/phone_analyzer.hpp @@ -6,6 +6,7 @@ // the device UTC clock; the app adds its NTP/GNSS correction. #include "avsync/avsync.hpp" +#include "avsync/calibrator.hpp" #include #include @@ -55,6 +56,15 @@ class PhoneAnalyzer { std::optional calibration_error() const { return calibration_error_; } std::optional take_calibration(); + // Calibrator board (docs/calibrator.md). Camera: LED flash + click on the rear camera, + // this phone's camera against its microphone. Display: an external screen shows the + // calibration pattern (container generator); only the microphone counts. + enum class CalibratorMode { Camera, Display }; + void begin_calibrator(CalibratorMode mode); + bool calibrator_active() const { return calibrator_.has_value(); } + // About once a second a progress report (JSON), then once the result or an error; else empty. + std::string poll_calibrator(); + // The detected picture (for the preview outline), normalized corners. bool picture(std::array& x, std::array& y) const; @@ -76,6 +86,11 @@ class PhoneAnalyzer { double cal_audio_origin_ = -1, spike_ = -1, floor_ = .002; int rate_ = 48000; void finish_calibration(double now); + // Calibrator take. + std::optional calibrator_; + std::vector flash_frames_; + double calibrator_poll_ = -1; + bool calibrator_done_ = false; }; } // namespace avsync diff --git a/core/src/calibrator.cpp b/core/src/calibrator.cpp new file mode 100644 index 0000000..2b01024 --- /dev/null +++ b/core/src/calibrator.cpp @@ -0,0 +1,344 @@ +#include "avsync/calibrator.hpp" + +#include +#include +#include + +namespace avsync { +namespace { + +constexpr double kPi = 3.14159265358979323846; + +double median(std::vector v) +{ + if (v.empty()) return 0; + std::sort(v.begin(), v.end()); + const size_t n = v.size(); + return n % 2 ? v[n / 2] : (v[n / 2 - 1] + v[n / 2]) / 2; +} + +// Energy of one frequency over a stretch of samples (Goertzel). +double tone_energy(const std::vector& x, size_t from, size_t to, double hz, int rate) +{ + const double k = 2 * std::cos(2 * kPi * hz / rate); + double s1 = 0, s2 = 0; + for (size_t i = from; i < to; ++i) { + const double s = x[i] + k * s1 - s2; + s2 = s1, s1 = s; + } + return s1 * s1 + s2 * s2 - k * s1 * s2; +} + +// Median and robust outlier rejection: keep values within max(3 sigma, 0.5 ms) of the +// median (sigma from the median absolute deviation), then the median of the rest. +struct Summary { + double median = 0, uncertainty = 0, spread = 0; + std::vector kept; + int rejected = 0; +}; +Summary summarize(const std::vector& values) +{ + Summary s; + if (values.empty()) return s; + const double m = median(values); + std::vector dev; + for (double v : values) dev.push_back(std::abs(v - m)); + const double limit = std::max(3 * 1.4826 * median(dev), 0.5); + for (double v : values) (std::abs(v - m) <= limit ? s.kept.push_back(v) : void(++s.rejected)); + s.median = median(s.kept); + const double n = double(s.kept.size()); + double var = 0; + for (double v : s.kept) var += (v - s.median) * (v - s.median); + const double sigma = n > 1 ? std::sqrt(var / (n - 1)) : 0; + s.uncertainty = 1.2533 * sigma / std::sqrt(n); // standard error of a median + auto sorted = s.kept; + std::sort(sorted.begin(), sorted.end()); + auto at = [&](double q) { return sorted[size_t(std::lround(q * (n - 1)))]; }; + s.spread = sorted.size() < 10 ? sorted.back() - sorted.front() : at(.9) - at(.1); + return s; +} + +CalibratorResult fail(const std::string& error) +{ + CalibratorResult r; + r.error = error; + return r; +} + +} // namespace + +std::vector find_sound_onsets(const float* x, size_t n, int rate, double start_ms) +{ + std::vector out; + if (!x || rate < 8000 || n < size_t(rate / 20)) return out; + // High-pass at 300 Hz: handling noise and room rumble out, clicks and beeps in. + std::vector hp(n); + const double rc = 1 / (2 * kPi * 300), a = rc / (rc + 1.0 / rate); + for (size_t i = 1; i < n; ++i) hp[i] = float(a * (hp[i - 1] + x[i] - x[i - 1])); + // Envelope: RMS over the last 0.25 ms (trailing, so it never rises before the sound). + const size_t w = std::max(4, size_t(rate / 4000)); + std::vector sum(n + 1, 0); + for (size_t i = 0; i < n; ++i) sum[i + 1] = sum[i] + double(hp[i]) * hp[i]; + std::vector e(n); + for (size_t i = 0; i < n; ++i) { + const size_t lo = i + 1 >= w ? i + 1 - w : 0; + e[i] = std::sqrt((sum[i + 1] - sum[lo]) / double(w)); + } + // Noise floor: the median envelope (the sounds are short and sparse). + std::vector sample; + for (size_t i = 0; i < n; i += 16) sample.push_back(e[i]); + const double floor = std::max(median(sample), 1e-6); + const double threshold = std::max(8 * floor, 3e-4); + const size_t ms = size_t(rate / 1000); + size_t i = 0; + while (i < n) { + if (e[i] < threshold) { ++i; continue; } + // The peak within 10 ms, then back to where the envelope first reaches 25 % of it. + size_t ip = i; + for (size_t j = i; j < std::min(n, i + 10 * ms); ++j) if (e[j] > e[ip]) ip = j; + const double peak = e[ip], level = .25 * peak; + size_t k = ip; + while (k > 0 && e[k] >= level) --k; + const double frac = e[k + 1] > e[k] ? (level - e[k]) / (e[k + 1] - e[k]) : 0; + const double onset = double(k) + std::clamp(frac, 0.0, 1.0); + const size_t from = size_t(onset), to = std::min(n, from + 3 * ms); + const double click = tone_energy(hp, from, to, CalibratorContract::click_hz, rate); + const double beep = tone_energy(hp, from, to, CalibratorContract::beep_hz, rate); + out.push_back({start_ms + onset * 1000.0 / rate, peak, click + beep > 0 ? click / (click + beep) : 0}); + // The sound ends when the envelope stays below max(threshold, 10 % of the peak) for 2 ms + // (ringing and room echo then do not count as new sounds); at least 8 ms. + const double quiet = std::max(threshold, .1 * peak); + size_t j = std::max(ip, from + 8 * ms), calm = 0; + while (j < n && calm < 2 * ms) calm = e[j++] < quiet ? calm + 1 : 0; + i = j; + } + return out; +} + +std::vector find_flash_onsets(const std::vector& frames, double led_ms, double* readout_out) +{ + std::vector out; + if (readout_out) *readout_out = 0; + if (frames.size() < 3 || frames[0].rows.empty()) return out; + const size_t bands = frames[0].rows.size(); + for (const auto& f : frames) if (f.rows.size() != bands) return out; + // Dark level of each band: its median (a flash lights only a few frames). + std::vector base(bands); + for (size_t b = 0; b < bands; ++b) { + std::vector v; + for (const auto& f : frames) v.push_back(f.rows[b]); + base[b] = median(v); + } + auto rise = [&](const FlashFrame& f, size_t b) { return std::max(0.0, double(f.rows[b]) - base[b]); }; + auto lit = [&](const FlashFrame& f) { + double top = 0; + for (size_t b = 0; b < bands; ++b) top = std::max(top, rise(f, b)); + return top; + }; + constexpr double kLit = 30; // 8-bit levels above dark + // Runs of lit frames are flashes. + struct Run { size_t first, last; double peak; }; + std::vector runs; + for (size_t i = 0; i < frames.size(); ++i) { + const double level = lit(frames[i]); + if (level < kLit) continue; + if (!runs.empty() && runs.back().last + 1 == i) runs.back().last = i, runs.back().peak = std::max(runs.back().peak, level); + else runs.push_back({i, i, level}); + } + // Share of the flash each band saw: undo the camera tone curve (gamma 2.2), 0..1. + auto share = [&](const FlashFrame& f, size_t b, double peak) { return std::pow(std::clamp(rise(f, b) / peak, 0.0, 1.0), 2.2); }; + // Readout when no frame knows it: a band lit with both edges inside one frame is + // max(exposure, LED) high at half level. + double readout = 0; + bool known = false; + for (const auto& f : frames) known |= f.readout_ms > 0; + if (!known) { + std::vector estimates; + for (const auto& run : runs) + for (size_t i = run.first; i <= run.last; ++i) { + const auto& f = frames[i]; + if (f.exposure_ms <= 0 || bands < 8) continue; + std::vector s(bands); + for (size_t b = 0; b < bands; ++b) s[b] = share(f, b, run.peak); + if (s.front() >= .5 || s.back() >= .5) continue; + size_t lo = 0, hi = bands - 1; + while (lo < bands && s[lo] < .5) ++lo; + while (hi > lo && s[hi] < .5) --hi; + if (lo >= bands || lo == 0 || hi + 1 >= bands) continue; + const double y1 = double(lo) - 1 + (.5 - s[lo - 1]) / std::max(1e-9, s[lo] - s[lo - 1]); + const double y2 = double(hi) + (s[hi] - .5) / std::max(1e-9, s[hi] - s[hi + 1]); + if (y2 - y1 >= .5) estimates.push_back(std::max(f.exposure_ms, led_ms) / (y2 - y1) * double(bands)); + } + if (estimates.size() >= 3) readout = median(estimates); + } + if (readout_out) *readout_out = known ? frames[0].readout_ms : readout; + for (const auto& run : runs) { + const auto& f = frames[run.first]; + const double r = known ? f.readout_ms : readout; + const double e = std::max(0.0, f.exposure_ms), m = std::min(e, led_ms); + std::vector s(bands); + for (size_t b = 0; b < bands; ++b) s[b] = share(f, b, run.peak); + // Row time of a band position y (in bands, 0 = top edge), as the analyzer times rows. + auto row_time = [&](double y) { return f.time_ms + r * std::clamp(y / double(bands), 0.0, 1.0); }; + size_t b = 0; + while (b < bands && s[b] < .5) ++b; + FlashOnset onset; + onset.row_timed = r > 0; + if (b < bands && b > 0) { + // Rolling-shutter edge: the half-level row's exposure ended half a flash after the start. + const double y = double(b) - .5 + (.5 - s[b - 1]) / std::max(1e-9, s[b] - s[b - 1]); + onset.time_ms = row_time(y) + e - m / 2; + onset.edge_in_frame = true; + onset.row = std::clamp(y / double(bands), 0.0, 1.0); + } else { + // No edge inside the frame: the share of the flash inside the exposure. + if (b >= bands) b = size_t(std::max_element(s.begin(), s.end()) - s.begin()); + onset.time_ms = row_time(double(b) + .5) + e - s[b] * m; + onset.row = (double(b) + .5) / double(bands); + } + out.push_back(onset); + } + return out; +} + +CalibratorResult camera_calibration(const std::vector& flashes, const std::vector& sounds, const CameraCalibrationOptions& o) +{ + if (flashes.empty()) return fail("No flashes seen. Lay the calibrator LED-down on the main rear camera."); + if (sounds.empty()) return fail("No clicks heard. Check that the calibrator is plugged in."); + // Rough offset: the most common click - flash difference (2 ms bins, counted over +/-3 ms). + std::vector diffs; + for (const auto& f : flashes) + for (const auto& s : sounds) { + const double d = s.time_ms - f.time_ms; + if (d > -250 && d < 350) diffs.push_back(d); + } + if (diffs.empty()) return fail("Flashes and clicks do not match. Try again."); + double rough = 0; + size_t best = 0; + for (double centre = -250; centre < 350; centre += 2) { + const size_t count = size_t(std::count_if(diffs.begin(), diffs.end(), [&](double d) { return std::abs(d - centre) <= 3; })); + if (count > best) best = count, rough = centre; + } + // Pair each flash with the nearest unused click within 12 ms of the rough offset. + std::vector used(sounds.size(), false); + std::vector values, rows; + std::vector edges; + bool timed = false; + const double fixed = o.buzzer_delay_ms + o.mic_distance_cm * CalibratorContract::sound_ms_per_cm; + for (const auto& f : flashes) { + long pick = -1; + double near = 12; + for (size_t i = 0; i < sounds.size(); ++i) { + const double d = std::abs(sounds[i].time_ms - f.time_ms - rough); + if (!used[i] && d <= near) near = d, pick = long(i); + } + if (pick < 0) continue; + used[size_t(pick)] = true; + values.push_back(sounds[size_t(pick)].time_ms - f.time_ms - fixed); + rows.push_back(f.row), edges.push_back(f.edge_in_frame), timed |= f.row_timed; + } + if (int(values.size()) < o.min_pairs) return fail("Too few flashes with a click (" + std::to_string(values.size()) + "). Keep the calibrator still on the lens."); + double readout = 0; + if (!timed) { + // Fit value = offset + readout * row over the edge pulses (twice: the second time + // without the outliers of the first), when their rows spread over at least 30 %. + std::vector use; + for (size_t i = 0; i < values.size(); ++i) if (edges[i]) use.push_back(i); + for (int pass = 0; pass < 2 && use.size() >= size_t(o.min_pairs); ++pass) { + double sr = 0, sv = 0, srr = 0, srv = 0, lo = 1, hi = 0; + for (size_t i : use) sr += rows[i], sv += values[i], srr += rows[i] * rows[i], srv += rows[i] * values[i], lo = std::min(lo, rows[i]), hi = std::max(hi, rows[i]); + const double n = double(use.size()), var = srr / n - (sr / n) * (sr / n); + if (hi - lo < .3 || var <= 0) break; + const double slope = (srv / n - (sr / n) * (sv / n)) / var; + if (!(slope >= 0 && slope <= 50)) break; + readout = slope; + std::vector residual; + for (size_t i : use) residual.push_back(values[i] - slope * rows[i]); + const auto fit = summarize(residual); + std::vector keep; + for (size_t k = 0; k < use.size(); ++k) + if (std::find(fit.kept.begin(), fit.kept.end(), residual[k]) != fit.kept.end()) keep.push_back(use[k]); + use = keep; + } + for (size_t i = 0; i < values.size(); ++i) values[i] -= readout * rows[i]; + } + const auto s = summarize(values); + CalibratorResult r; + r.ok = true; + r.readout_ms = readout; + r.offset_ms = s.median, r.uncertainty_ms = s.uncertainty, r.spread_ms = s.spread; + r.pairs = int(s.kept.size()), r.rejected = s.rejected, r.values = s.kept; + return r; +} + +CalibratorResult display_calibration(const std::vector& sounds, const DisplayCalibrationOptions& o) +{ + size_t clicks = 0; + for (const auto& s : sounds) clicks += s.click(); + if (!clicks) return fail("No clicks heard. Lay the calibrator on the marked spot and check the cable."); + if (clicks == sounds.size()) return fail("No test sound heard. Turn the volume up."); + // Each click with the next beep (not a click) less than one swap period later. + struct Pair { double click, value; }; + std::vector pairs; + const double fixed = o.buzzer_delay_ms + o.click_minus_speaker_path_cm * CalibratorContract::sound_ms_per_cm; + for (size_t i = 0; i < sounds.size(); ++i) { + if (!sounds[i].click()) continue; + for (size_t j = i + 1; j < sounds.size(); ++j) { + const double gap = sounds[j].time_ms - sounds[i].time_ms; + if (gap >= CalibratorContract::swap_period_ms - 50 || sounds[j].click()) break; + pairs.push_back({sounds[i].time_ms, gap - o.sound_delay_ms + fixed}); + break; + } + } + if (int(pairs.size()) < o.min_pairs) return fail("Too few clicks with a test sound (" + std::to_string(pairs.size()) + "). Check the placement and the volume."); + std::vector all, even, odd; + for (const auto& p : pairs) { + all.push_back(p.value); + // Swap direction: every other swap, counted from the first click. + const long k = std::lround((p.click - pairs[0].click) / CalibratorContract::swap_period_ms); + (k % 2 ? odd : even).push_back(p.value); + } + const auto s = summarize(all); + CalibratorResult r; + r.ok = true; + r.offset_ms = s.median, r.uncertainty_ms = s.uncertainty, r.spread_ms = s.spread; + r.pairs = int(s.kept.size()), r.rejected = s.rejected, r.values = s.kept; + if (even.size() >= 3 && odd.size() >= 3) { + // A sensor mismatch moves one direction early and the other late: use both equally. + const auto a = summarize(even), b = summarize(odd); + r.offset_ms = (a.median + b.median) / 2; + r.uncertainty_ms = std::hypot(a.uncertainty, b.uncertainty) / 2; + r.direction_ms = a.median - b.median; + } + return r; +} + +uint64_t calibration_period(uint64_t frame, int64_t num, int64_t den) +{ + // t = frame * den / num seconds; period = floor(t / 0.5 s), in integers. + return num > 0 && den > 0 ? frame * 2 * uint64_t(den) / uint64_t(num) : 0; +} + +bool calibration_left_white(uint64_t frame, int64_t num, int64_t den) { return calibration_period(frame, num, den) % 2 == 0; } + +uint64_t calibration_swap_frame(uint64_t k, int64_t num, int64_t den) +{ + // The first frame whose period is k: ceil(k * num / (2 * den)). + return num > 0 && den > 0 ? (k * uint64_t(num) + 2 * uint64_t(den) - 1) / (2 * uint64_t(den)) : 0; +} + +std::vector calibration_beep(int rate) +{ + // A sine that starts at full slope (sharp onset), fades out over its last cycle. + const size_t n = size_t(std::lround(CalibratorContract::beep_ms * rate / 1000)); + const double cycle = rate / CalibratorContract::beep_hz; + std::vector out(n); + for (size_t i = 0; i < n; ++i) { + const double left = double(n - i); + const double fade = left < cycle ? .5 - .5 * std::cos(kPi * left / cycle) : 1; + out[i] = float(.5 * fade * std::sin(2 * kPi * CalibratorContract::beep_hz * double(i) / rate)); + } + return out; +} + +} // namespace avsync diff --git a/core/src/grade.cpp b/core/src/grade.cpp index a330767..664c5ff 100644 --- a/core/src/grade.cpp +++ b/core/src/grade.cpp @@ -347,6 +347,12 @@ Measurement Pairer::pair_one(const PendingVideo& pending, const AudioObservation m.audio_tai_ns = audio->tai_ns; m.expected_av_offset_ms = audio->timecode ? (audio->audio_send_ns - audio->event_time_ns) / 1e6 : pending.expected_offset_ms; m.av_offset_ms = audio->time_ms - pending.video.time_ms; + if (sender_calibration_.calibrated()) { + // The generator's own output offset is not part of the chain under test. + m.raw_av_offset_ms = m.av_offset_ms; + m.sender_calibration_ms = sender_calibration_.offset_ms; + m.av_offset_ms = *m.av_offset_ms - sender_calibration_.offset_ms; + } m.deviation_ms = *m.av_offset_ms - m.expected_av_offset_ms; m.correlation = audio->correlation; m.audio_level_dbfs = audio->level_dbfs; diff --git a/core/src/phone.cpp b/core/src/phone.cpp index 2e65a00..f6fd9ba 100644 --- a/core/src/phone.cpp +++ b/core/src/phone.cpp @@ -318,6 +318,7 @@ PhoneReport PhoneSession::finish(const std::string& test_id) // Latency is against the device clock; the app checks that clock by NTP and hides it otherwise. reference.sync_state = "locked"; Pairer pairer(detect.period_ms, windows_, reference, "LIVE", test_id); + if (signal_) pairer.set_sender_calibration(signal_->calibration); const bool coded = (signal_ && signal_->timecode) || !report.audio_timecodes.empty(); for (auto video : videos_) { if (signal_ && !video.timecode) identify_video(video, *signal_); diff --git a/core/src/phone_analyzer.cpp b/core/src/phone_analyzer.cpp index ea01380..7b54375 100644 --- a/core/src/phone_analyzer.cpp +++ b/core/src/phone_analyzer.cpp @@ -93,7 +93,7 @@ void PhoneAnalyzer::set_device_utc_offset(double offset) void PhoneAnalyzer::observe_qr(const std::string& text, const std::array& x, const std::array& y, double capture_ms) { auto payload = parse_qr_payload(text); - if (!payload || !payload->optical_code || !std::isfinite(capture_ms) || calibration_) return; + if (!payload || !payload->optical_code || !std::isfinite(capture_ms) || calibration_ || calibrator_) return; qr_ = payload; qr_ms_ = capture_ms; QrDetection q; @@ -102,7 +102,7 @@ void PhoneAnalyzer::observe_qr(const std::string& text, const std::array 2 && frame_times_.front() < pts - 3000) frame_times_.pop_front(); + if (calibrator_) { + if (*calibrator_ != CalibratorMode::Camera || calibrator_done_ || flash_frames_.size() > 12000) return; + // Mean luma of 32 row bands (every 8th pixel of every 4th row). + constexpr int kBands = 32; + FlashFrame f{pts, exposure, readout, std::vector(kBands)}; + for (int b = 0; b < kBands; ++b) { + double sum = 0; + int count = 0; + for (int yy = b * h / kBands; yy < (b + 1) * h / kBands; yy += 4) + for (int xx = 0; xx < w; xx += 8) sum += luma[size_t(yy) * stride + size_t(xx)], ++count; + f.rows[size_t(b)] = count ? float(sum / count) : 0; + } + flash_frames_.push_back(std::move(f)); + return; + } if (calibration_) { if (calibration_done_) return; // Keep a small grey copy (at most 640 px) for the review. @@ -135,14 +150,18 @@ void PhoneAnalyzer::push_audio(const float* mono, size_t n, double pts, int rate if (origin_ < 0) origin_ = pts; rate_ = rate; last_audio_ = pts + n * 1000.0 / rate; - if (!calibration_) { + if (!calibration_ && !calibrator_) { session_.push_audio(mono, n, pts, rate, true); return; } - if (calibration_done_) return; + if (calibration_done_ || calibrator_done_) return; if (cal_audio_origin_ < 0) cal_audio_origin_ = pts; const size_t at = size_t(std::max(0.0, std::round((pts - cal_audio_origin_) * rate / 1000))); - if (at > cal_audio_.size() && at < size_t(18 * rate)) cal_audio_.resize(at, 0); + if (at > cal_audio_.size() && at < size_t((calibrator_ ? 40 : 18) * rate)) cal_audio_.resize(at, 0); + if (calibrator_) { + if (cal_audio_.size() < size_t(40 * rate)) cal_audio_.insert(cal_audio_.end(), mono, mono + n); + return; + } cal_audio_.insert(cal_audio_.end(), mono, mono + n); // A sharp sound: a peak well above the running floor, after a quiet start. float peak = 0; @@ -198,6 +217,58 @@ void PhoneAnalyzer::finish_calibration(double now) take_ = std::move(take); } +void PhoneAnalyzer::begin_calibrator(CalibratorMode mode) +{ + reset(); + calibrator_ = mode; +} + +std::string PhoneAnalyzer::poll_calibrator() +{ + constexpr int kWanted = 20; // pulses (camera) or swaps (display) for a result + constexpr double kLimitMs = 30000; // then the result with what there is, or an error + if (!calibrator_ || calibrator_done_ || cal_audio_origin_ < 0) return {}; + const double now = std::max(last_video_, last_audio_); + if (calibrator_poll_ >= 0 && now - calibrator_poll_ < 1000) return {}; + calibrator_poll_ = now; + const bool camera = *calibrator_ == CalibratorMode::Camera; + const auto sounds = find_sound_onsets(cal_audio_.data(), cal_audio_.size(), rate_, cal_audio_origin_); + double readout = 0; + std::vector flashes; + if (camera) flashes = find_flash_onsets(flash_frames_, CalibratorContract::led_pulse_ms, &readout); + auto run = [&](int min_pairs) { + if (camera) { + CameraCalibrationOptions o; + o.min_pairs = min_pairs; + return camera_calibration(flashes, sounds, o); + } + DisplayCalibrationOptions o; + o.min_pairs = min_pairs; + return display_calibration(sounds, o); + }; + const auto progress = run(1); + const double elapsed = now - origin_; + Json out; + out.str("mode", camera ? "camera" : "display"); + if (progress.pairs < kWanted && elapsed < kLimitMs) { + size_t clicks = 0; + for (const auto& s : sounds) clicks += s.click(); + out.str("kind", "calibrator-progress").integer("pairs", progress.pairs).integer("wanted", kWanted) + .integer("flashes", long(flashes.size())).integer("clicks", long(clicks)).num("elapsedMs", elapsed); + return out.text(); + } + calibrator_done_ = true; + const auto r = run(8); + if (!r.ok) return Json().str("kind", "error").str("message", r.error).text(); + const double fps = frames_ * 1000.0 / std::max(1.0, last_video_ - origin_); + out.str("kind", "calibrator").str("method", camera ? "calibrator-pulse" : "calibrator-display") + .num("offsetMs", r.offset_ms).num("uncertaintyMs", r.uncertainty_ms).num("spreadMs", r.spread_ms) + .integer("pairs", r.pairs).integer("rejected", r.rejected).num("directionMs", r.direction_ms) + .num("readoutMs", readout > 0 ? readout : r.readout_ms); + if (camera) out.num("fps", fps); + return out.text(); +} + std::optional PhoneAnalyzer::take_calibration() { auto out = std::move(take_); @@ -208,7 +279,7 @@ std::optional PhoneAnalyzer::take_calibration() bool PhoneAnalyzer::picture(std::array& x, std::array& y) const { const auto& pose = session_.current_picture(); - if (calibration_ || !pose.found || video_width_ <= 0 || last_audio_ - last_video_ > 300) return false; + if (calibration_ || calibrator_ || !pose.found || video_width_ <= 0 || last_audio_ - last_video_ > 300) return false; for (int i = 0; i < 4; ++i) x[i] = pose.corner_x[i] / video_width_, y[i] = pose.corner_y[i] / video_height_; return true; } @@ -216,7 +287,7 @@ bool PhoneAnalyzer::picture(std::array& x, std::array& y) bool PhoneAnalyzer::report_due() const { const double now = std::max(last_video_, last_audio_); - return !calibration_ && origin_ >= 0 && now - origin_ >= 1500 && now - last_report_ >= 1000; + return !calibration_ && !calibrator_ && origin_ >= 0 && now - origin_ >= 1500 && now - last_report_ >= 1000; } PhoneAnalyzer PhoneAnalyzer::snapshot_for_poll() @@ -237,20 +308,20 @@ void PhoneAnalyzer::merge_audio(const PhoneAnalyzer& analyzed) std::string PhoneAnalyzer::poll() { - if (calibration_ || origin_ < 0) return {}; + if (calibration_ || calibrator_ || origin_ < 0) return {}; const double now = std::max(last_video_, last_audio_); last_report_ = now; const auto report = session_.finish("PHONE"); session_.trim_before(now - 12000); // The window: the newest valid pairs. - struct Pair { uint64_t seq; double offset; std::optional video, audio; double time; }; + struct Pair { uint64_t seq; double offset; std::optional video, audio; double time; double raw; std::optional sender; }; std::vector pairs; for (const auto& e : report.events) { if (!e.av_offset_ms || e.grade == "invalid" || !std::isfinite(*e.av_offset_ms)) continue; const double t = std::max(e.video_time_ms, e.audio_time_ms); if (t < now - kWindowMs) continue; - Pair p{e.seq, *e.av_offset_ms, std::nullopt, std::nullopt, t}; + Pair p{e.seq, *e.av_offset_ms, std::nullopt, std::nullopt, t, e.raw_av_offset_ms.value_or(*e.av_offset_ms), e.sender_calibration_ms}; // Latency vs device UTC: capture time on the device UTC clock minus the source time (TAI). if (have_wall_ && qr_ && qr_->mode == "LIVE") { if (e.have_set_time) p.video = e.video_time_ms + wall_offset_ + tai_utc_ * 1000 - double(e.set_video_ns) / 1e6; @@ -287,14 +358,18 @@ std::string PhoneAnalyzer::poll() .raw("senderCalibration", calibration.text()).text()); } if (pairs.size() >= 3) { - std::vector offsets, video, audio; + std::vector offsets, raw, video, audio; for (const auto& p : pairs) { offsets.push_back(p.offset); + raw.push_back(p.raw); if (p.video) video.push_back(*p.video); if (p.audio) audio.push_back(*p.audio); } Json reading; reading.num("offsetMs", median(offsets)).num("spreadMs", spread(offsets)).integer("events", long(offsets.size())); + // offsetMs has the sender calibration (AVS4) subtracted; rawOffsetMs is as measured. + reading.num("rawOffsetMs", median(raw)); + if (pairs.back().sender) reading.num("senderCalibrationMs", *pairs.back().sender); if (video.size() >= 3) reading.num("videoLatencyMs", median(video)).num("videoSpreadMs", spread(video)); if (audio.size() >= 3) reading.num("audioLatencyMs", median(audio)).num("audioSpreadMs", spread(audio)); out.raw("reading", reading.text()); diff --git a/docs/calibration.md b/docs/calibration.md index 856f754..1577c7e 100644 --- a/docs/calibration.md +++ b/docs/calibration.md @@ -5,3 +5,67 @@ The phone measures only AV offset (`reference.type = none`). Its clock is not a `PhoneSession` in `avsync-core` locates the test picture with the fiducials, times the marker on the rolling-shutter row, detects the chirps, and subtracts a calibration offset. A record is valid for one model and OS version for 90 days. Otherwise the result is uncalibrated: the offset is shown and the grade colour is withheld. The rig is an LED and a piezo on one microcontroller pin. Parts and firmware are in `mobile-app/calibration/`. The pulse has not been checked with an oscilloscope here. The Expo UI is in `mobile-app/`; capturing on a device needs Xcode or the Android SDK. + +## With the calibrator + +The calibrator is a small board with an LED, a buzzer and two light sensors +([signal contract](calibrator.md)). The phone powers it over a USB-C cable. +In the app: **Settings → With calibrator** (iPhone for now; the clap +calibration stays as it is). Three steps: + +1. **Plug in** the calibrator. +2. **Camera and microphone.** Lay the calibrator LED-down on the main rear + camera and the phone screen-down on a table. The calibrator flashes and + clicks at the same moment, 23 times in about 13 s. The app records them and + shows **Camera vs mic**, e.g. `+12.3 ms ± 0.4`. **Save** stores it as this + phone's calibration, exactly like a clap calibration (same record, method + `calibrator-pulse`, valid for this model, OS version and camera mode for + 90 days; the ± is the spread of the pulses). +3. **Screen and speaker.** The phone opens its generator in calibration mode: + two halves at the bottom of the screen swap black and white every 500 ms, + and the speaker beeps 100 ms after each swap. Turn the calibrator over and + lay it on the red outline. It clicks when the screen swaps; the microphone + hears the click and then the beep, so its own delay cancels. The app shows + **Screen vs speaker**, e.g. `−38.0 ms ± 0.3` (positive: the sound comes + out after the picture). **Save** stores it as this phone's **sender + calibration**: from then on its test signal sends it in the AVS4 QR + (method `transponder`, offset in 0.1 ms, uncertainty rounded up to 0.5 ms, + the date), so an analyser can take the phone's own output offset into + account. It is valid for this OS version for 90 days and is only sent on the + audio output it was measured on (the built-in speaker). + +Why through the generator: the calibration picture and sound use the same +screen layers, display clock, audio engine and speaker as the real test signal, +so the offset belongs to the real signal. + +**Analysers subtract it.** When the QR carries a sender calibration, every +analyser (phone, container, file) takes it off the measured AV offset. The +result is the corrected value; the raw value and the calibration stay visible: + +- phone: Measure and Results show "raw +50.5 ms · sender calibration +38.0 ms" + under the result; the report `reading` has `offsetMs` (corrected), + `rawOffsetMs` and `senderCalibrationMs`; saved results keep `rawOffsetMs` and + `senderCalibrationMs`; +- container: `/api/v1/measurement` `offset_ms` (corrected), `raw_offset_ms`, + `sender_calibration_ms`; each event has `raw_av_offset_ms` and + `sender_calibration_ms`; web UI Input card ("AV sync raw"); +- file analyzer: the same event fields, and in `summary` + `sender_calibration_ms` and `median_raw_av_offset_ms`. + +"Not calibrated" changes nothing. The subtraction is in core C++ (`Pairer`), +so iOS, Android, container and file do it the same way. On the phone two +calibrations are in play, each subtracted once: the sender calibration (the +other phone's screen vs speaker, from the QR) in core, and this phone's own +analyser calibration (camera vs microphone) in the app. Raw means neither is +subtracted. Debug → Clear calibration clears both on this phone (after a +confirm). + +**A TV or monitor.** The container generator can send the same pattern +(`AVSYNC_PATTERN=calibration`, web UI → Signal → Picture). Lay the calibrator +on the centre line of the screen and choose **Measure a TV or monitor instead** +in step 3: the phone only listens and shows the screen-vs-speaker offset of +that chain. It is not saved. + +The analysis is shared C++ (`core/include/avsync/calibrator.hpp`) with unit +tests on synthetic signals. Not yet tested with a real board: see the open +points in [calibrator.md](calibrator.md#open-points). diff --git a/docs/calibrator.md b/docs/calibrator.md new file mode 100644 index 0000000..0a4f5bb --- /dev/null +++ b/docs/calibrator.md @@ -0,0 +1,247 @@ +# AV Sync calibrator: signal contract + +**v1 proposal. The app side is built (iOS app and container generator, see [App side](#app-side-built)), but only tested on synthetic signals. The board is not built or tested yet.** + +The calibrator is a small board (66 × 22 mm) that the phone powers over a USB-C to USB-C cable. +The cable is not included: the user brings their own. +It has a white LED, a buzzer and two light sensors. It picks its mode by itself from +what the sensors see. There is no button and no battery. + +Hardware files: [`hardware/calibrator/`](../hardware/calibrator/README.md). +Firmware: [`hardware/calibrator/firmware/`](../hardware/calibrator/firmware/README.md). +All numbers below are defaults in `cal_default_cfg()` (`firmware/src/cal_logic.c`). + +## Words + +- **Bottom side**: the flat side with no parts. It touches the phone or the screen. + The LED and both sensors look out through holes in the board on this side. +- **Sensor A / B**: the two light sensors, 9 mm apart, marked `A` and `B` on both sides. +- **Split line**: the line half-way between A and B. It runs along the board and ends in + the half-round notch at the LED end. It is printed on both sides; on the top it shows + next to the LED and again behind the buzzer, so it can be lined up while the board lies + face down. +- **Click**: 3 cycles of a 2.7 kHz square wave on the buzzer (about 1.1 ms). The first + cycle starts with the buzzer coil switched on. + +## Power + +- Only USB-C phones. The board is a USB-C *sink*: 5.1 kΩ on CC1 and CC2. D+ and D− go to + the MCU but are only used for firmware updates: the firmware never turns USB on, so the + phone does not see an audio or data device and keeps its own speaker and microphone. +- Firmware update (not with a phone): hold the board's BOOT button while plugging it into + a PC; see `hardware/calibrator/README.md`. +- The phone must supply 5 V on USB-C (iPhone 15 and later, most Android phones with OTG). +- The board needs about 10 mA, plus short peaks of about 100 mA during a click. +- It starts in **Idle** about 1 ms after power-up. + +## Modes + +| Mode | Enters when | Does | +| --- | --- | --- | +| Idle | power-up; sensors see light but nothing changes | nothing (silent, LED off) | +| Camera | both sensors dark (below 150 mV) for 1000 ms | pulse train (below) | +| Display | a valid swap is seen (below) | one click on every swap | + +- Camera → Idle: the sensors see light for 300 ms (the board was lifted). +- Display → Idle: no valid swap for 1500 ms. +- Idle or Camera → Display: at the first valid swap. That swap already gets a click. + A camera train stops at once. + +## Camera mode (calibrates a phone's camera against its microphone) + +**Placement**: the phone lies screen down. The board lies on the camera bump, bottom side +down, with the LED circle on the main rear camera lens. The buzzer sits next to the LED, +so it is close to the rear microphone on most phones. The sensors then face the phone's +back and see dark, which starts camera mode after 1 s. + +**Pulse**: the LED turns on and the click starts with **one register write** (LED and +buzzer pins on the same port, same store), so they start less than 1 µs apart. +LED on time: 1.5 ms (`led_pulse_us`, can be 0.5–5 ms). The buzzer needs some time to move +air after its coil is switched on; that delay is the same for every board of a type and +still has to be measured once (open point). + +**Train** (times from the start of the train): + +| Pulses | Times | +| --- | --- | +| 3 start pulses | 0, 125, 250 ms | +| 20 timing pulses | 1000, 1500, …, 10500 ms (every 500 ms) | +| pause | 2000 ms after the last pulse (until 12500 ms) | + +Then the train starts again with the 3 start pulses, as long as the board stays in +camera mode. The 125 ms spacing never happens elsewhere, so the app can find the start. + +**What the app must do** + +1. Record video (rear camera) and audio (built-in microphone) for at least one train (~13 s). +2. Find each flash: the LED floods the whole picture (no focus needed). With a short + exposure only some rows see it (rolling shutter); use the row timing as the existing + calibration does (`makeCalibration`, `docs/calibration.md`). Use the **start** of the + flash. +3. Find each click onset in the audio. +4. Pair flashes and clicks by the train pattern. `offset = click time − flash time`, + per pulse; use the median of the 20 timing pulses. +5. Subtract the fixed buzzer delay and the sound travel time from buzzer to microphone + (about 0.03 ms per cm; the buzzer is about 1–2 cm from the mic on most phones). +6. Save it like today's calibration record (per model and OS version). + +The MCU clock is only good to about 2 %, so the 500 ms spacing is not a time reference. +Only "flash and click start together" is. + +## Display mode (calibrates a sender screen against its speaker; "transponder") + +**Pattern** (shown by the sending phone, on its own screen): + +``` + ┌──────────────┬──────────────┐ + │ │ │ + │ half L │ half R │ each half at least 20 × 20 mm + │ │ │ (the board needs 2 mm margin around each sensor) + │ │ │ + └──────────────┴──────────────┘ + ^ pattern line = where the board's split line goes +``` + +- Two rectangles side by side, sharing one straight edge (the **pattern line**). +- State 1: L white, R black. State 2: L black, R white. Pure white (full brightness) and + pure black, nothing else on the halves. +- **Swap** between state 1 and state 2 every **500 ms** (allowed: 300–1000 ms). Both halves + change in the same frame. +- The pattern line must run **across the display's scan rows** so that both sensors are on + the same row: for a phone in portrait, put the halves left and right of each other + (vertical pattern line). Otherwise one sensor updates later and the crossing moves. +- Draw a marker where the notch goes, e.g. a short tick on the pattern line. +- Set the screen to full brightness while calibrating (this also keeps OLED flicker low). + +**Placement**: bottom side down on the screen. The split line lies on the pattern line +(the notch on the line), sensor A over one half and sensor B over the other. Up to ±2 mm +off is fine. + +**How the board decides**: the two sensors drive the two inputs of the MCU comparator. +The comparator flips at the moment the two sensors see the same light, i.e. half-way +through the swap, where the rising half and the falling half cross. Slow LCD pixels, +backlight changes, OLED flicker and room light hit both sensors the same way and mostly +cancel. A flip counts as a **valid swap** only if, in the 100 ms before, the ADC saw the +two sensors clearly different (at least 80 mV and 25 % apart, not dark) the *other* way +round. Noise around zero therefore never clicks. + +**Click**: at a valid swap the board clicks **at once** (a few µs after the comparator +flips, no timed delay). Then it ignores the sensors for **200 ms** (lock-out). + +**What the app must do** + +1. Run the pattern. For each swap, plan it on the phone's display clock (CADisplayLink / + Choreographer) and note its planned presentation time `t_swap`. +2. Play a short sound on the built-in speaker at exactly `t_swap + 100 ms` on the same + precise clock (100 ms = `D`; any fixed value 50–200 ms works, it must be longer than the + expected display delay and shorter than the lock-out gap to the next swap). +3. Record the phone's own microphone. For every swap it hears the board's click first, + then the speaker sound. +4. Measure `gap = speaker sound onset − click onset` per swap (both through the same + microphone, so the microphone delay cancels). +5. `screen-vs-speaker offset = gap − D` = (speaker output delay) − (display light delay). + Positive = the sound comes out later than the picture. Use the median. +6. Use both swap directions equally (state 1→2 and 2→1). A small mismatch between the two + sensors moves the crossing early in one direction and late in the other, so the + average cancels it. +7. Correct for the fixed buzzer delay and the sound path difference: the board sits on + the screen, a few cm from the microphone, while the speaker is in the phone body. Put + the pattern near the microphone end of the phone to keep that path short; about + 0.03 ms per cm. +8. If no click comes for 1.5 s, the board has fallen back to idle: tell the user to + check the placement. + +## Timing summary + +| Item | Value | +| --- | --- | +| LED on time | 1.5 ms | +| Click | 3 × 2.7 kHz (≈1.1 ms), first half-cycle = coil on | +| LED to click start | < 1 µs (same register write) | +| Camera train | 3 start pulses (0/125/250 ms), 20 pulses every 500 ms from 1000 ms, 2 s pause | +| Swap period (app) | 500 ms (300–1000 ms) | +| Light change to comparator flip | ~15 µs filter (22 kΩ + 1 nF) plus sensor response (estimate) | +| Comparator flip to click | a few µs (estimate) | +| Lock-out after click | 200 ms | +| Valid swap needs | ADC difference ≥ 80 mV and ≥ 25 %, opposite sign, within 100 ms | +| Dark | both sensors < 150 mV | +| Enter camera | dark for 1000 ms | +| Leave camera | light for 300 ms | +| Leave display | no valid swap for 1500 ms | +| Ignore sensors after a camera pulse | 5 ms | + +## App side (built) + +Code: `core/include/avsync/calibrator.hpp` (all analysis, shared by every app), +`PhoneAnalyzer::begin_calibrator` (camera take), the iPhone generator's +calibration mode and the container's `AVSYNC_PATTERN=calibration`. User guide: +[calibration.md](calibration.md#with-the-calibrator). + +**Finding the sounds.** High-pass at 300 Hz, then the RMS over the last 0.25 ms. +A sound starts where this envelope first reaches 25 % of its peak. The same +detector times the board's click and the generator's beep, so in display mode +its small delay cancels. + +**Click or beep?** First by time order (click, then the beep about D later), +then by sound: the energy at 2.7 kHz (click) against 1.5 kHz (beep) in the +3 ms after the onset. The generator's beep is 6 cycles of 1.5 kHz (4 ms), with a +sharp start and a soft end, peak at the chirp level. + +**Camera mode.** The analyser keeps the mean brightness of 32 row bands per +frame. The flash start is where a band reaches half its flash level, on the +row time the analyser already uses (`frame start + readout × row`): that row's +exposure ended half an LED pulse after the flash started. iOS gives no readout +time and the analyser exposes for the whole frame interval; then the readout +is fitted over the pulses: the board's clock drifts against the camera, so the +flash lands on different rows, and click − flash grows with the row by exactly +the readout time. Flashes and clicks are paired by the most common difference +(±12 ms around it). Offset = median of the pairs after dropping outliers +(beyond 3 σ, at least 0.5 ms); uncertainty = standard error of the median. +For the camera take the iPhone locks the exposure at the lowest ISO. + +**Display mode.** Every click is paired with the next beep less than 450 ms +later. Offset = beep − click − D, the median per swap direction (every other +swap), and the mean of the two directions. The app does not need the swap +times, so the same analysis works for a TV fed by the container generator. + +**Pattern.** Every generator uses the same core functions: period k lasts +500 ms on its frame grid (for 59.94 fps the swap is the first frame at or +after k × 500 ms), left half white in even periods, the beep D after each +swap. The iPhone draws the halves in portrait near the bottom microphone, +about 62 mm tall, with a red outline of the board (sensor end down, notch +tick). The container fills the whole picture and draws a grey tick at both +ends of the pattern line. + +### Changes from the v1 proposal + +None needs a firmware change. + +- The generator's sound is fixed: a 4 ms beep at 1.5 kHz (the proposal only + said "a short sound"). Click and beep are told apart by frequency and order. +- Camera mode uses every paired pulse, the 3 start pulses too; it does not need + the start pattern (pairing works on the click − flash difference). The + firmware may keep it. +- Display mode does not need the swap times. The rule "D longer than the + display delay" becomes: the beep must come 0–450 ms after the click, so + sound minus picture must be between −100 ms and +350 ms. That also covers a + TV chain behind the container generator. +- Constants still 0 until measured: buzzer delay (both modes) and, in display + mode, the path difference board→microphone vs speaker→microphone. Camera + mode subtracts 1.5 cm of sound travel. + +## Open points + +- The buzzer's delay from coil-on to sound must be measured once (lab: reference + microphone + photodiode). It enters both modes as a constant. +- Whether every phone powers a passive USB-C sink without asking the user. Expected yes + for iPhone 15 and later and most Android phones; to check on real phones. +- Sensor signal on dim screens and room light leaking into the clear sensor bodies from + above: to check on the first boards. +- App: the screen calibration runs the generator with the audio session in + play-and-record (to hear the microphone), the test signal in playback. Both use + the built-in speaker and subtract the reported output latency; whether the + real output delay differs between the two must be checked once on an iPhone + (calibrate, then measure the test signal with a second calibrated phone). +- App: in camera mode a flash that saturates the camera makes the half-level + edge a little early (up to half an LED pulse, 0.75 ms). The lowest ISO should + avoid it; to check on the first boards. diff --git a/docs/mobile-native.md b/docs/mobile-native.md index 1bb4b1b..5bb1454 100644 --- a/docs/mobile-native.md +++ b/docs/mobile-native.md @@ -17,12 +17,15 @@ the core, so both platforms give the same numbers. | `synchronizeClock(server)` | Promise of `{source:'ntp', server, referenceUnixMs, referenceMonotonicMs, uncertaintyMs, roundTripMs, offsetMs, clockId, timebase}` (SNTP, 8 samples, smallest round trip) | | `setScreenAwake(on)` | keeps the display on | | `generate(rate, taiUtcSeconds, options)` | Promise; shows the fullscreen AVS3 source until closed, resolves `{skippedDisplayFrames}` | +| `generateStyled(rate, taiUtcSeconds, options)` | the same with string options; `mode: 'calibration'` shows the calibration pattern and resolves with `calibration` or `calibrationError`; `senderCalibrationMethod/OffsetMs/UncertaintyMs/Day/Route` put the sender calibration in the QR (iOS; [phone-generator.md](phone-generator.md#calibration-mode)) | ## Camera view `Avsync` Props: `active` (bool), `captureFps` (0 = automatic best rate), `taiUtcOffset` (seconds, default 37). Methods: `beginCalibration()`, `resumeAnalysis()`, -`focusAt(x, y)` (tap to focus; x and y are 0–1 across the view, see below). +`focusAt(x, y)` (tap to focus; x and y are 0–1 across the view, see below), +`beginCalibrator(mode)` (iOS only so far; `camera` or `display`, see +[calibrator.md](calibrator.md#app-side-built)). Event `onReport` with one of: - **status** — `PhoneAnalyzer::poll()` parsed, plus `captureContext` @@ -30,11 +33,19 @@ Event `onReport` with one of: `inputGain, inputGainSettable`; Android adds `capturePath` `normal` or `high_speed`, and `captureStats.gpuReadMaxMs` on the high-speed path), `configuredFps` and `clockMapping {monotonicMs, deviceUnixMs, clockId, timebase}` for the - report's `timeMs`. + report's `timeMs`. `reading.offsetMs` has the sender calibration from the + AVS4 QR subtracted; `reading.rawOffsetMs` is as measured and + `reading.senderCalibrationMs` is present when the QR carries one. The + phone's own analyser calibration is not in the report (the app subtracts it). - **calibration** — `{frames:[{timeMs, uri}], spikeTimeMs, fps, uri (WAV), waveform (256 peaks), waveformStartMs, waveformBinDurationMs, waveformDurationMs, audioStartMs, captureContext}` from `PhoneAnalyzer::take_calibration()`; frames as JPEG files. +- **calibrator-progress** — `{mode, pairs, wanted, flashes, clicks, elapsedMs}` + about once a second during a calibrator take. +- **calibrator** — `{mode, method, offsetMs, uncertaintyMs, spreadMs, pairs, + rejected, directionMs, readoutMs, fps, captureContext}` once at its end, from + `PhoneAnalyzer::poll_calibrator()`. - **error** — `{message}`. ## Capture rules diff --git a/docs/phone-generator.md b/docs/phone-generator.md index b30f1e7..1f47449 100644 --- a/docs/phone-generator.md +++ b/docs/phone-generator.md @@ -49,12 +49,24 @@ photodiode measurement. All timing and synthesis run outside JavaScript. The screen reports skipped logical display frames and the reported audio lead. It prevents automatic screen lock and raises brightness while active, restoring -both when stopped. Stop/Close cancel queued sound and remove the timing picture. +both when stopped. Stop/Close cancel queued sound and clear the whole timing picture; a large "Stopped" shows instead. App inactivity, audio interruptions, output/configuration changes, a wall-clock jump over 100 ms, missed scheduling deadlines and the 23-hour session limit stop the source. Restart creates a fresh clock mapping. Bluetooth and other external output paths have unverified delays; use the built-in speaker. +## Calibration mode + +`generateStyled(rate, taiUtc, {mode: 'calibration'})` opens the same screen +with the calibration pattern instead of the test picture +([calibrator.md](calibrator.md), display mode). It uses the same display link, +layers, audio engine and player node; the beep (1.5 kHz, 4 ms) is scheduled +100 ms after each swap on the host clock, minus the reported output latency, +like the chirps. The audio session is play-and-record on the built-in speaker +so the microphone can listen. The result comes back when the screen closes: +`calibration` (`offsetMs`, `uncertaintyMs`, `pairs`, `directionMs`, +`audioRoute`, `outputLatencyMs`) or `calibrationError`. + ## Source ID The source ID (the QR test id; the audio packet carries its CRC-32) stays the @@ -81,14 +93,22 @@ Besides event and time it carries: change the ID; - the **sender calibration** of the phone's own screen-vs-speaker offset (method, offset, uncertainty, date). The offset has the same sign as every - AV offset here: sound minus picture, positive = sound later. This calibration - is not built yet, so the phone sends "not calibrated" (method `none`). + AV offset here: sound minus picture, positive = sound later. It comes from + the calibrator wizard (Settings → With calibrator, step 3, method + `transponder`; [calibration.md](calibration.md#with-the-calibrator)). Without a + valid one (none saved, another OS version, older than 90 days) the phone sends + "not calibrated" (method `none`). It is not sent when the audio output differs + from the one it was measured on; the status line then says so. Analysers + subtract it from the measured AV offset and show the raw value next to the + result ([calibration.md](calibration.md#with-the-calibrator)). The QR keeps its size (version 4, 33 × 33 modules); it now uses error correction M instead of Q to make room. The phone analyzer report shows these fields in `signal`: `testId`, `name` and `senderCalibration` (`method`, and when calibrated `offsetMs`, `uncertaintyMs`, `date`). The container analyzer shows -them in `/api/v1/status` `signal` and in the web UI's Input card. Old AVS2/AVS3 +them in `/api/v1/status` `signal` and in the web UI's Input card. In the app's +Results the Title is the default name of a sender until you set a nickname +(pencil); a nickname you set always wins. Old AVS2/AVS3 phones and analyzers cannot read AVS4: update all of them together. The live reader first tries the inexpensive full-raster marker path, then a diff --git a/mobile-app/App.js b/mobile-app/App.js index 00fc16c..bc28e74 100644 --- a/mobile-app/App.js +++ b/mobile-app/App.js @@ -6,6 +6,7 @@ import Ionicons from '@expo/vector-icons/Ionicons'; import { useFonts } from 'expo-font'; import native from './modules/avsync'; import CalibrationReview from './components/CalibrationReview'; +import CalibratorWizard from './components/CalibratorWizard'; import Measure from './components/Measure'; import Generate from './components/Generate'; import Results from './components/Results'; @@ -15,6 +16,7 @@ import { liveView, offsetText, savedResult, formatWhen, clockText } from './src/ import { clockCheckDetails, clockSettings, generatorClockOptions, referenceAt, synchronizeClock, CLOCK_MAX_AGE_MS } from './src/time-reference.mjs'; import { calibrationSummary, makeCalibration, validCalibration } from './src/workflow.mjs'; import { generatorOptions } from './src/generator.mjs'; +import { makePulseCalibration, makeSenderCalibration, senderCalibrationLabel, senderCalibrationOptions } from './src/calibrator.mjs'; import { screenAwakeEnabled } from './src/screen-awake.mjs'; import { addResult, liveDelta, loadSessionState, setNickname, toggleReference } from './src/results.mjs'; @@ -75,10 +77,16 @@ export default function App() { const calibrationState = calibrationSummary(session.calibration, os, Date.now(), report?.captureContext); const calibrationLabel = calibrationState.state === 'active' ? `${offsetText(calibrationState.appliedOffsetMs)} · ${formatWhen(calibrationState.savedAt)}` : calibrationState.state === 'inactive' ? 'Not valid for this camera mode' : 'Not calibrated'; + const senderLabel = senderCalibrationLabel(session.senderCalibration, os, Date.now(), formatWhen); const [flow, setFlow] = useState(null), [take, setTake] = useState(null); + // The calibrator wizard: null or {step, progress, result, error} (components/CalibratorWizard.js). + const [cal, setCal] = useState(null); const onReport = event => { const value = event.nativeEvent; + if (value.kind === 'calibrator-progress') { setCal(c => c && { ...c, progress: value }); return; } + if (value.kind === 'calibrator') { setCal(c => c && { step: c.step.replace(/-run$/, '-done'), result: value }); return; } + if (value.kind === 'error' && cal?.step.endsWith('-run')) { setCal(c => c && { step: c.step.replace(/-run$/, ''), error: value.message }); return; } if (value.kind === 'calibration') { setTake(value); setFlow('review'); return; } if (value.kind === 'error') { setNotice(value.message); if (flow === 'listening') setFlow('aim'); return; } setReport(value); setReportAt(performance.now()); @@ -105,6 +113,26 @@ export default function App() { setTake(null); setFlow(null); restart(); }; + // With calibrator: step 1 is a camera take (this phone's camera vs microphone), step 2 the + // generator's calibration pattern (this phone's screen vs speaker, sent in the AVS4 QR). + const supportsCalibrator = Platform.OS === 'ios' && Boolean(NativeCamera) && Boolean(native?.supportsGenerator?.()); + const startCalibrator = () => { setSheet(null); setTab('measure'); setNotice(null); setFlow(null); setCal({ step: 'plug' }); }; + const closeCalibrator = message => { setCal(null); restart(); setNotice(message ?? null); }; + const calibratorStart = async base => { + if (base !== 'display') { setCal({ step: `${base}-run`, progress: null }); camera.current?.beginCalibrator?.(base === 'tv' ? 'display' : 'camera'); return; } + setCal({ step: 'display-run' }); + try { + const result = await native.generateStyled(2, session.taiUtcSeconds ?? 37, { mode: 'calibration', keepAwake: 'true' }); + setCal(result?.calibration ? { step: 'display-done', result: result.calibration } : { step: 'display', error: result?.calibrationError || null }); + } catch (error) { setCal({ step: 'display', error: error.message }); } + }; + const calibratorSave = base => { + try { + if (base === 'camera') { update({ calibration: makePulseCalibration(cal.result, os, Date.now()) }); setCal({ step: 'display' }); } + else { update({ senderCalibration: makeSenderCalibration({ calibration: cal.result }, os, Date.now()) }); closeCalibrator('Sender calibration saved'); } + } catch (error) { setCal(c => ({ ...c, error: error.message })); } + }; + // Test signal. const [generating, setGenerating] = useState(false), [picking, setPicking] = useState(false); const image = useMemo(() => session.generatorImageId && native?.generatorImageInfo ? native.generatorImageInfo(session.generatorImageId) : null, [session.generatorImageId]); @@ -112,7 +140,8 @@ export default function App() { setGenerating(true); setNotice(null); try { const rate = session.generatorRate === 1 ? 1 : 2; - const options = { ...generatorOptions(session), keepAwake: screenAwakeEnabled(session) ? 'true' : 'false', ...generatorClockOptions(reference, native.clockNow?.(), limits.maxUncertaintyMs) }; + const options = { ...generatorOptions(session), keepAwake: screenAwakeEnabled(session) ? 'true' : 'false', ...generatorClockOptions(reference, native.clockNow?.(), limits.maxUncertaintyMs), + ...senderCalibrationOptions(session.senderCalibration, os, Date.now()) }; await (native.generateStyled || native.generate)(rate, session.taiUtcSeconds ?? 37, options); } catch (error) { setNotice(error.message); } finally { setGenerating(false); } @@ -135,7 +164,7 @@ export default function App() { const cameraView = !NativeCamera ? This build has no live camera measurement. : granted - ? + ? : AVsync needs the camera and the microphone to measure.