From 21b68269b7fa5b7be938401ec6dff6e8b758e43d Mon Sep 17 00:00:00 2001 From: Adrian Hilber Date: Sun, 8 Nov 2026 15:30:17 +0100 Subject: [PATCH 01/21] Core: calibrator analysis and calibration pattern Shared C++ for the AV Sync calibrator board (docs/calibrator.md, contract v1): - find_sound_onsets: clicks and beeps in a recording (trailing 0.25 ms RMS, 25 % of peak), told apart by 2.7 kHz vs 1.5 kHz energy. - find_flash_onsets: LED flash start from row-band means, on the rolling-shutter row the analyzer uses; readout from the band height or, with full-frame exposure, fitted over the pulses in camera_calibration. - camera_calibration: pair flashes with clicks, robust median, uncertainty. - display_calibration: click then beep D = 100 ms later, offset = gap - D, both swap directions weighted equally. - calibration pattern for all generators: halves swap every 500 ms, beep. Unit tests on synthetic signals (noise, missing pulses, stray sound, rolling shutter with known/unknown readout, slow screen, sensor mismatch). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017ybuzgBaGxuB5UiaE8dRa4 --- CMakeLists.txt | 3 +- core/include/avsync/calibrator.hpp | 96 +++++ core/src/calibrator.cpp | 334 ++++++++++++++++++ .../modules/avsync/android/CMakeLists.txt | 2 +- tests/unit/test_calibrator.cpp | 249 +++++++++++++ 5 files changed, 682 insertions(+), 2 deletions(-) create mode 100644 core/include/avsync/calibrator.hpp create mode 100644 core/src/calibrator.cpp create mode 100644 tests/unit/test_calibrator.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 42bb2fc..5354b87 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,6 +59,7 @@ add_library(avsync-core STATIC core/src/phone_generator.cpp core/src/picture_reader.cpp core/src/two_way.cpp + core/src/calibrator.cpp third_party/nayuki/qrcodegen.cpp ) target_include_directories(avsync-core @@ -70,7 +71,7 @@ set_source_files_properties(core/src/text.cpp PROPERTIES OBJECT_DEPENDS "${DEJAV target_link_libraries(avsync-core PUBLIC blend2d::blend2d) target_compile_options(avsync-core PRIVATE -Wall -Wextra -Wno-unused-parameter) -add_executable(avsync-core-tests tests/unit/test_core.cpp tests/unit/test_stage23.cpp tests/unit/test_timecode.cpp) +add_executable(avsync-core-tests tests/unit/test_core.cpp tests/unit/test_stage23.cpp tests/unit/test_timecode.cpp tests/unit/test_calibrator.cpp) target_include_directories(avsync-core-tests PRIVATE third_party/doctest) target_link_libraries(avsync-core-tests PRIVATE avsync-core) target_compile_options(avsync-core-tests PRIVATE -Wall -Wextra -Wno-unused-parameter) diff --git a/core/include/avsync/calibrator.hpp b/core/include/avsync/calibrator.hpp new file mode 100644 index 0000000..78e344d --- /dev/null +++ b/core/include/avsync/calibrator.hpp @@ -0,0 +1,96 @@ +#pragma once + +// The AV Sync calibrator board (docs/calibrator.md, signal contract v1): what the +// apps do with its flashes and clicks, and the calibration pattern every generator +// can show. Pure maths on samples and frames; no I/O. Times are milliseconds on one +// capture clock (camera and microphone of the same phone). + +#include +#include +#include +#include + +namespace avsync { + +// Numbers from the contract ("Timing summary"). +struct CalibratorContract { + static constexpr double click_hz = 2700; // buzzer: 3 cycles of a square wave + static constexpr double led_pulse_ms = 1.5; // camera mode LED on time + static constexpr double swap_period_ms = 500; // display mode: pattern halves swap + static constexpr double sound_delay_ms = 100; // D: generator sound after each swap + static constexpr double beep_hz = 1500; // the generator's own sound (not the buzzer's 2.7 kHz) + static constexpr double beep_ms = 4; // 6 cycles at 1.5 kHz + static constexpr double sound_ms_per_cm = 0.0291; // speed of sound, 343 m/s +}; + +// A short sound in a recording: a calibrator click or the generator's beep. +struct SoundOnset { + double time_ms = 0; // where the envelope first reaches 25 % of its peak + double peak = 0; // envelope peak (RMS over 0.25 ms, full scale 1) + double click_share = 0; // energy at 2.7 kHz / (2.7 kHz + 1.5 kHz) in the 3 ms after the onset + bool click() const { return click_share >= 0.5; } +}; +std::vector find_sound_onsets(const float* mono, size_t count, int sample_rate, double start_ms); + +// One camera frame, reduced to the mean luma of equal row bands (top to bottom, 0..255). +struct FlashFrame { + double time_ms = 0; // exposure start of the first row + double exposure_ms = 0; // 0: unknown + double readout_ms = 0; // first to last row; 0: unknown + std::vector rows; +}; +struct FlashOnset { + double time_ms = 0; + bool edge_in_frame = false; // the start of the flash lies inside the frame (rolling shutter edge seen) + double row = 0; // where that edge is, 0 top .. 1 bottom + bool row_timed = false; // time_ms includes the row's readout delay (readout known or estimated) +}; +// Start of each LED flash. When no frame knows its readout time, it is estimated from +// the height of the lit band (max(exposure, LED time) / band height); *readout_ms +// returns what was used (0: none, then camera_calibration fits it). +std::vector find_flash_onsets(const std::vector& frames, double led_ms = CalibratorContract::led_pulse_ms, + double* readout_ms = nullptr); + +struct CalibratorResult { + bool ok = false; + std::string error; // for the user when !ok + double offset_ms = 0; // camera: microphone minus camera; display: sound output minus picture output + double uncertainty_ms = 0; // standard error of the median of the pairs kept + double spread_ms = 0; // 10-90 % range of the pairs kept + int pairs = 0; // pairs kept + int rejected = 0; // pairs dropped as outliers + double direction_ms = 0; // display: median of one swap direction minus the other (sensor mismatch) + double readout_ms = 0; // camera: readout fitted from the pulses (0 when the frames knew it or too few edges) + std::vector values; // per kept pair +}; + +struct CameraCalibrationOptions { + double buzzer_delay_ms = 0; // coil on to sound; still to be measured per buzzer type (contract open point) + double mic_distance_cm = 1.5; // buzzer to microphone + int min_pairs = 8; +}; +// Camera mode: flash and click start together; offset = click - flash per pulse. When the +// flash times are not row-timed, the readout is fitted: the flash phase against the frames +// is random, so (click - flash) grows linearly with the edge row by the readout time. +CalibratorResult camera_calibration(const std::vector& flashes, const std::vector& sounds, + const CameraCalibrationOptions& options = {}); + +struct DisplayCalibrationOptions { + double sound_delay_ms = CalibratorContract::sound_delay_ms; + double buzzer_delay_ms = 0; + double click_minus_speaker_path_cm = 0; // calibrator-to-mic distance minus speaker-to-mic distance + int min_pairs = 8; +}; +// Display mode: each swap gives a click (when the screen changes) and then the generator's +// beep D after the swap (when the speaker plays it). offset = (beep - click) - D, so the +// microphone delay cancels. Both swap directions count equally. +CalibratorResult display_calibration(const std::vector& sounds, const DisplayCalibrationOptions& options = {}); + +// The calibration pattern, the same for all generators. Time is from the pattern start; +// swap k happens at k * 500 ms. In even periods the left half is white. +bool calibration_left_white(double ms_since_start); +uint64_t calibration_period(double ms_since_start); +// The generator's beep (mono, starts at sample 0, peak 0.5), played D after each swap. +std::vector calibration_beep(int sample_rate); + +} // namespace avsync diff --git a/core/src/calibrator.cpp b/core/src/calibrator.cpp new file mode 100644 index 0000000..6570532 --- /dev/null +++ b/core/src/calibrator.cpp @@ -0,0 +1,334 @@ +#include "avsync/calibrator.hpp" + +#include +#include +#include + +namespace avsync { +namespace { + +constexpr double kPi = 3.14159265358979323846; + +double median(std::vector v) +{ + if (v.empty()) return 0; + std::sort(v.begin(), v.end()); + const size_t n = v.size(); + return n % 2 ? v[n / 2] : (v[n / 2 - 1] + v[n / 2]) / 2; +} + +// Energy of one frequency over a stretch of samples (Goertzel). +double tone_energy(const std::vector& x, size_t from, size_t to, double hz, int rate) +{ + const double k = 2 * std::cos(2 * kPi * hz / rate); + double s1 = 0, s2 = 0; + for (size_t i = from; i < to; ++i) { + const double s = x[i] + k * s1 - s2; + s2 = s1, s1 = s; + } + return s1 * s1 + s2 * s2 - k * s1 * s2; +} + +// Median and robust outlier rejection: keep values within max(3 sigma, 0.5 ms) of the +// median (sigma from the median absolute deviation), then the median of the rest. +struct Summary { + double median = 0, uncertainty = 0, spread = 0; + std::vector kept; + int rejected = 0; +}; +Summary summarize(const std::vector& values) +{ + Summary s; + if (values.empty()) return s; + const double m = median(values); + std::vector dev; + for (double v : values) dev.push_back(std::abs(v - m)); + const double limit = std::max(3 * 1.4826 * median(dev), 0.5); + for (double v : values) (std::abs(v - m) <= limit ? s.kept.push_back(v) : void(++s.rejected)); + s.median = median(s.kept); + const double n = double(s.kept.size()); + double var = 0; + for (double v : s.kept) var += (v - s.median) * (v - s.median); + const double sigma = n > 1 ? std::sqrt(var / (n - 1)) : 0; + s.uncertainty = 1.2533 * sigma / std::sqrt(n); // standard error of a median + auto sorted = s.kept; + std::sort(sorted.begin(), sorted.end()); + auto at = [&](double q) { return sorted[size_t(std::lround(q * (n - 1)))]; }; + s.spread = sorted.size() < 10 ? sorted.back() - sorted.front() : at(.9) - at(.1); + return s; +} + +CalibratorResult fail(const std::string& error) +{ + CalibratorResult r; + r.error = error; + return r; +} + +} // namespace + +std::vector find_sound_onsets(const float* x, size_t n, int rate, double start_ms) +{ + std::vector out; + if (!x || rate < 8000 || n < size_t(rate / 20)) return out; + // High-pass at 300 Hz: handling noise and room rumble out, clicks and beeps in. + std::vector hp(n); + const double rc = 1 / (2 * kPi * 300), a = rc / (rc + 1.0 / rate); + for (size_t i = 1; i < n; ++i) hp[i] = float(a * (hp[i - 1] + x[i] - x[i - 1])); + // Envelope: RMS over the last 0.25 ms (trailing, so it never rises before the sound). + const size_t w = std::max(4, size_t(rate / 4000)); + std::vector sum(n + 1, 0); + for (size_t i = 0; i < n; ++i) sum[i + 1] = sum[i] + double(hp[i]) * hp[i]; + std::vector e(n); + for (size_t i = 0; i < n; ++i) { + const size_t lo = i + 1 >= w ? i + 1 - w : 0; + e[i] = std::sqrt((sum[i + 1] - sum[lo]) / double(w)); + } + // Noise floor: the median envelope (the sounds are short and sparse). + std::vector sample; + for (size_t i = 0; i < n; i += 16) sample.push_back(e[i]); + const double floor = std::max(median(sample), 1e-6); + const double threshold = std::max(8 * floor, 3e-4); + const size_t ms = size_t(rate / 1000); + size_t i = 0; + while (i < n) { + if (e[i] < threshold) { ++i; continue; } + // The peak within 10 ms, then back to where the envelope first reaches 25 % of it. + size_t ip = i; + for (size_t j = i; j < std::min(n, i + 10 * ms); ++j) if (e[j] > e[ip]) ip = j; + const double peak = e[ip], level = .25 * peak; + size_t k = ip; + while (k > 0 && e[k] >= level) --k; + const double frac = e[k + 1] > e[k] ? (level - e[k]) / (e[k + 1] - e[k]) : 0; + const double onset = double(k) + std::clamp(frac, 0.0, 1.0); + const size_t from = size_t(onset), to = std::min(n, from + 3 * ms); + const double click = tone_energy(hp, from, to, CalibratorContract::click_hz, rate); + const double beep = tone_energy(hp, from, to, CalibratorContract::beep_hz, rate); + out.push_back({start_ms + onset * 1000.0 / rate, peak, click + beep > 0 ? click / (click + beep) : 0}); + // The sound ends when the envelope stays below max(threshold, 10 % of the peak) for 2 ms + // (ringing and room echo then do not count as new sounds); at least 8 ms. + const double quiet = std::max(threshold, .1 * peak); + size_t j = std::max(ip, from + 8 * ms), calm = 0; + while (j < n && calm < 2 * ms) calm = e[j++] < quiet ? calm + 1 : 0; + i = j; + } + return out; +} + +std::vector find_flash_onsets(const std::vector& frames, double led_ms, double* readout_out) +{ + std::vector out; + if (readout_out) *readout_out = 0; + if (frames.size() < 3 || frames[0].rows.empty()) return out; + const size_t bands = frames[0].rows.size(); + for (const auto& f : frames) if (f.rows.size() != bands) return out; + // Dark level of each band: its median (a flash lights only a few frames). + std::vector base(bands); + for (size_t b = 0; b < bands; ++b) { + std::vector v; + for (const auto& f : frames) v.push_back(f.rows[b]); + base[b] = median(v); + } + auto rise = [&](const FlashFrame& f, size_t b) { return std::max(0.0, double(f.rows[b]) - base[b]); }; + auto lit = [&](const FlashFrame& f) { + double top = 0; + for (size_t b = 0; b < bands; ++b) top = std::max(top, rise(f, b)); + return top; + }; + constexpr double kLit = 30; // 8-bit levels above dark + // Runs of lit frames are flashes. + struct Run { size_t first, last; double peak; }; + std::vector runs; + for (size_t i = 0; i < frames.size(); ++i) { + const double level = lit(frames[i]); + if (level < kLit) continue; + if (!runs.empty() && runs.back().last + 1 == i) runs.back().last = i, runs.back().peak = std::max(runs.back().peak, level); + else runs.push_back({i, i, level}); + } + // Share of the flash each band saw: undo the camera tone curve (gamma 2.2), 0..1. + auto share = [&](const FlashFrame& f, size_t b, double peak) { return std::pow(std::clamp(rise(f, b) / peak, 0.0, 1.0), 2.2); }; + // Readout when no frame knows it: a band lit with both edges inside one frame is + // max(exposure, LED) high at half level. + double readout = 0; + bool known = false; + for (const auto& f : frames) known |= f.readout_ms > 0; + if (!known) { + std::vector estimates; + for (const auto& run : runs) + for (size_t i = run.first; i <= run.last; ++i) { + const auto& f = frames[i]; + if (f.exposure_ms <= 0 || bands < 8) continue; + std::vector s(bands); + for (size_t b = 0; b < bands; ++b) s[b] = share(f, b, run.peak); + if (s.front() >= .5 || s.back() >= .5) continue; + size_t lo = 0, hi = bands - 1; + while (lo < bands && s[lo] < .5) ++lo; + while (hi > lo && s[hi] < .5) --hi; + if (lo >= bands || lo == 0 || hi + 1 >= bands) continue; + const double y1 = double(lo) - 1 + (.5 - s[lo - 1]) / std::max(1e-9, s[lo] - s[lo - 1]); + const double y2 = double(hi) + (s[hi] - .5) / std::max(1e-9, s[hi] - s[hi + 1]); + if (y2 - y1 >= .5) estimates.push_back(std::max(f.exposure_ms, led_ms) / (y2 - y1) * double(bands)); + } + if (estimates.size() >= 3) readout = median(estimates); + } + if (readout_out) *readout_out = known ? frames[0].readout_ms : readout; + for (const auto& run : runs) { + const auto& f = frames[run.first]; + const double r = known ? f.readout_ms : readout; + const double e = std::max(0.0, f.exposure_ms), m = std::min(e, led_ms); + std::vector s(bands); + for (size_t b = 0; b < bands; ++b) s[b] = share(f, b, run.peak); + // Row time of a band position y (in bands, 0 = top edge), as the analyzer times rows. + auto row_time = [&](double y) { return f.time_ms + r * std::clamp(y / double(bands), 0.0, 1.0); }; + size_t b = 0; + while (b < bands && s[b] < .5) ++b; + FlashOnset onset; + onset.row_timed = r > 0; + if (b < bands && b > 0) { + // Rolling-shutter edge: the half-level row's exposure ended half a flash after the start. + const double y = double(b) - .5 + (.5 - s[b - 1]) / std::max(1e-9, s[b] - s[b - 1]); + onset.time_ms = row_time(y) + e - m / 2; + onset.edge_in_frame = true; + onset.row = std::clamp(y / double(bands), 0.0, 1.0); + } else { + // No edge inside the frame: the share of the flash inside the exposure. + if (b >= bands) b = size_t(std::max_element(s.begin(), s.end()) - s.begin()); + onset.time_ms = row_time(double(b) + .5) + e - s[b] * m; + onset.row = (double(b) + .5) / double(bands); + } + out.push_back(onset); + } + return out; +} + +CalibratorResult camera_calibration(const std::vector& flashes, const std::vector& sounds, const CameraCalibrationOptions& o) +{ + if (flashes.empty()) return fail("No flashes seen. Lay the calibrator LED-down on the main rear camera."); + if (sounds.empty()) return fail("No clicks heard. Check that the calibrator is plugged in."); + // Rough offset: the most common click - flash difference (2 ms bins, counted over +/-3 ms). + std::vector diffs; + for (const auto& f : flashes) + for (const auto& s : sounds) { + const double d = s.time_ms - f.time_ms; + if (d > -250 && d < 350) diffs.push_back(d); + } + if (diffs.empty()) return fail("Flashes and clicks do not match. Try again."); + double rough = 0; + size_t best = 0; + for (double centre = -250; centre < 350; centre += 2) { + const size_t count = size_t(std::count_if(diffs.begin(), diffs.end(), [&](double d) { return std::abs(d - centre) <= 3; })); + if (count > best) best = count, rough = centre; + } + // Pair each flash with the nearest unused click within 12 ms of the rough offset. + std::vector used(sounds.size(), false); + std::vector values, rows; + std::vector edges; + bool timed = false; + const double fixed = o.buzzer_delay_ms + o.mic_distance_cm * CalibratorContract::sound_ms_per_cm; + for (const auto& f : flashes) { + long pick = -1; + double near = 12; + for (size_t i = 0; i < sounds.size(); ++i) { + const double d = std::abs(sounds[i].time_ms - f.time_ms - rough); + if (!used[i] && d <= near) near = d, pick = long(i); + } + if (pick < 0) continue; + used[size_t(pick)] = true; + values.push_back(sounds[size_t(pick)].time_ms - f.time_ms - fixed); + rows.push_back(f.row), edges.push_back(f.edge_in_frame), timed |= f.row_timed; + } + if (int(values.size()) < o.min_pairs) return fail("Too few flashes with a click (" + std::to_string(values.size()) + "). Keep the calibrator still on the lens."); + double readout = 0; + if (!timed) { + // Fit value = offset + readout * row over the edge pulses (twice: the second time + // without the outliers of the first), when their rows spread over at least 30 %. + std::vector use; + for (size_t i = 0; i < values.size(); ++i) if (edges[i]) use.push_back(i); + for (int pass = 0; pass < 2 && use.size() >= size_t(o.min_pairs); ++pass) { + double sr = 0, sv = 0, srr = 0, srv = 0, lo = 1, hi = 0; + for (size_t i : use) sr += rows[i], sv += values[i], srr += rows[i] * rows[i], srv += rows[i] * values[i], lo = std::min(lo, rows[i]), hi = std::max(hi, rows[i]); + const double n = double(use.size()), var = srr / n - (sr / n) * (sr / n); + if (hi - lo < .3 || var <= 0) break; + const double slope = (srv / n - (sr / n) * (sv / n)) / var; + if (!(slope >= 0 && slope <= 50)) break; + readout = slope; + std::vector residual; + for (size_t i : use) residual.push_back(values[i] - slope * rows[i]); + const auto fit = summarize(residual); + std::vector keep; + for (size_t k = 0; k < use.size(); ++k) + if (std::find(fit.kept.begin(), fit.kept.end(), residual[k]) != fit.kept.end()) keep.push_back(use[k]); + use = keep; + } + for (size_t i = 0; i < values.size(); ++i) values[i] -= readout * rows[i]; + } + const auto s = summarize(values); + CalibratorResult r; + r.ok = true; + r.readout_ms = readout; + r.offset_ms = s.median, r.uncertainty_ms = s.uncertainty, r.spread_ms = s.spread; + r.pairs = int(s.kept.size()), r.rejected = s.rejected, r.values = s.kept; + return r; +} + +CalibratorResult display_calibration(const std::vector& sounds, const DisplayCalibrationOptions& o) +{ + size_t clicks = 0; + for (const auto& s : sounds) clicks += s.click(); + if (!clicks) return fail("No clicks heard. Lay the calibrator on the marked spot and check the cable."); + if (clicks == sounds.size()) return fail("No test sound heard. Turn the volume up."); + // Each click with the next beep (not a click) less than one swap period later. + struct Pair { double click, value; }; + std::vector pairs; + const double fixed = o.buzzer_delay_ms + o.click_minus_speaker_path_cm * CalibratorContract::sound_ms_per_cm; + for (size_t i = 0; i < sounds.size(); ++i) { + if (!sounds[i].click()) continue; + for (size_t j = i + 1; j < sounds.size(); ++j) { + const double gap = sounds[j].time_ms - sounds[i].time_ms; + if (gap >= CalibratorContract::swap_period_ms - 50 || sounds[j].click()) break; + pairs.push_back({sounds[i].time_ms, gap - o.sound_delay_ms + fixed}); + break; + } + } + if (int(pairs.size()) < o.min_pairs) return fail("Too few clicks with a test sound (" + std::to_string(pairs.size()) + "). Check the placement and the volume."); + std::vector all, even, odd; + for (const auto& p : pairs) { + all.push_back(p.value); + // Swap direction: every other swap, counted from the first click. + const long k = std::lround((p.click - pairs[0].click) / CalibratorContract::swap_period_ms); + (k % 2 ? odd : even).push_back(p.value); + } + const auto s = summarize(all); + CalibratorResult r; + r.ok = true; + r.offset_ms = s.median, r.uncertainty_ms = s.uncertainty, r.spread_ms = s.spread; + r.pairs = int(s.kept.size()), r.rejected = s.rejected, r.values = s.kept; + if (even.size() >= 3 && odd.size() >= 3) { + // A sensor mismatch moves one direction early and the other late: use both equally. + const auto a = summarize(even), b = summarize(odd); + r.offset_ms = (a.median + b.median) / 2; + r.uncertainty_ms = std::hypot(a.uncertainty, b.uncertainty) / 2; + r.direction_ms = a.median - b.median; + } + return r; +} + +uint64_t calibration_period(double ms) { return ms <= 0 ? 0 : uint64_t(std::floor(ms / CalibratorContract::swap_period_ms + 1e-9)); } + +bool calibration_left_white(double ms) { return calibration_period(ms) % 2 == 0; } + +std::vector calibration_beep(int rate) +{ + // A sine that starts at full slope (sharp onset), fades out over its last cycle. + const size_t n = size_t(std::lround(CalibratorContract::beep_ms * rate / 1000)); + const double cycle = rate / CalibratorContract::beep_hz; + std::vector out(n); + for (size_t i = 0; i < n; ++i) { + const double left = double(n - i); + const double fade = left < cycle ? .5 - .5 * std::cos(kPi * left / cycle) : 1; + out[i] = float(.5 * fade * std::sin(2 * kPi * CalibratorContract::beep_hz * double(i) / rate)); + } + return out; +} + +} // namespace avsync diff --git a/mobile-app/modules/avsync/android/CMakeLists.txt b/mobile-app/modules/avsync/android/CMakeLists.txt index 0c099dd..04752aa 100644 --- a/mobile-app/modules/avsync/android/CMakeLists.txt +++ b/mobile-app/modules/avsync/android/CMakeLists.txt @@ -3,7 +3,7 @@ project(avsync-android CXX) # The shared phone analyzer, compiled from the repository's core like the iOS Core_*.cpp wrappers. set(REPO ${CMAKE_CURRENT_SOURCE_DIR}/../../../..) -set(CORE_SOURCES chirp detect g711 grade locate marker optical_code phone phone_analyzer plan qr sync_scale timecode two_way) +set(CORE_SOURCES chirp detect g711 grade locate marker optical_code phone phone_analyzer plan qr sync_scale timecode two_way calibrator) list(TRANSFORM CORE_SOURCES PREPEND ${REPO}/core/src/) list(TRANSFORM CORE_SOURCES APPEND .cpp) diff --git a/tests/unit/test_calibrator.cpp b/tests/unit/test_calibrator.cpp new file mode 100644 index 0000000..fa87825 --- /dev/null +++ b/tests/unit/test_calibrator.cpp @@ -0,0 +1,249 @@ +#include "doctest.h" + +#include "avsync/calibrator.hpp" + +#include +#include +#include +#include + +using namespace avsync; + +namespace { + +constexpr double kPi = 3.14159265358979323846; +constexpr int kRate = 48000; + +struct Noise { + uint32_t state = 12345; + double uniform() { state = state * 1664525u + 1013904223u; return (state >> 8) / 16777216.0; } + double gauss() { return std::sqrt(-2 * std::log(uniform() + 1e-12)) * std::cos(2 * kPi * uniform()); } +}; + +// The buzzer: 3 cycles of 2.7 kHz (coil on at the start), then a fast decay. +void add_click(std::vector& audio, double at_ms, double level = .3) +{ + const size_t start = size_t(std::lround(at_ms * kRate / 1000)); + for (size_t i = 0; i < size_t(kRate / 200) && start + i < audio.size(); ++i) { + const double t = double(i) / kRate, on = 3 / CalibratorContract::click_hz; + const double envelope = t < on ? 1 : std::exp(-(t - on) / 0.0006); + audio[start + i] += float(level * envelope * std::sin(2 * kPi * CalibratorContract::click_hz * t)); + } +} + +void add_beep(std::vector& audio, double at_ms, double level = .6) +{ + const auto beep = calibration_beep(kRate); + const size_t start = size_t(std::lround(at_ms * kRate / 1000)); + for (size_t i = 0; i < beep.size() && start + i < audio.size(); ++i) audio[start + i] += float(level * beep[i]); +} + +std::vector noise_track(double seconds, double rms, Noise& noise) +{ + std::vector audio(size_t(seconds * kRate)); + for (auto& v : audio) v = float(rms * noise.gauss()); + return audio; +} + +// The camera train: 3 start pulses, 20 timing pulses every 500 ms, on the board's own +// clock (1.3 % fast here; the contract allows about 2 %), so the flashes drift against the frames. +std::vector train(double start_ms) +{ + std::vector t{0, 125, 250}; + for (int k = 0; k < 20; ++k) t.push_back(1000 + 500 * k); + for (auto& v : t) v = start_ms + v / 1.013; + return t; +} + +// Rolling-shutter frames: row y (0..1) exposes [t + readout * y, + exposure]; the LED adds light +// in proportion to the overlap with each flash, through a gamma 1/2.2 tone curve. +std::vector camera(const std::vector& flashes, double fps, double exposure, double readout, double report_readout, + Noise& noise, double seconds = 13, const std::set& hidden = {}) +{ + std::vector frames; + const int bands = 32, sub = 8; + const double led = CalibratorContract::led_pulse_ms, full = std::min(exposure, led); + for (double t = 3.1; t < seconds * 1000; t += 1000 / fps) { + FlashFrame f{t, exposure, report_readout, std::vector(bands)}; + for (int b = 0; b < bands; ++b) { + double sum = 0; + for (int s = 0; s < sub; ++s) { + const double start = t + readout * (b * sub + s + .5) / (bands * sub); + double light = 0; + for (size_t i = 0; i < flashes.size(); ++i) { + if (hidden.count(i)) continue; + light += std::max(0.0, std::min(start + exposure, flashes[i] + led) - std::max(start, flashes[i])); + } + sum += 16 + 239 * std::pow(std::min(1.0, light / full), 1 / 2.2); + } + f.rows[size_t(b)] = float(sum / sub + 2 * noise.gauss()); + } + frames.push_back(f); + } + return frames; +} + +} // namespace + +TEST_CASE("calibrator sounds: clicks and beeps are found and told apart") +{ + Noise noise; + auto audio = noise_track(1, .002, noise); + add_click(audio, 200); + add_beep(audio, 300); + add_click(audio, 700); + const auto onsets = find_sound_onsets(audio.data(), audio.size(), kRate, 1000); + REQUIRE(onsets.size() == 3); + CHECK(onsets[0].click()); + CHECK(!onsets[1].click()); + CHECK(onsets[2].click()); + CHECK(std::abs(onsets[0].time_ms - 1200) < .15); + CHECK(std::abs(onsets[1].time_ms - 1300) < .15); + CHECK(onsets[2].time_ms - onsets[0].time_ms == doctest::Approx(500).epsilon(1e-4)); + CHECK(find_sound_onsets(audio.data(), 100, kRate, 0).empty()); +} + +TEST_CASE("calibration pattern: halves swap every 500 ms, the beep starts at once") +{ + CHECK(calibration_left_white(0)); + CHECK(calibration_left_white(499.9)); + CHECK(!calibration_left_white(500)); + CHECK(calibration_left_white(1000)); + CHECK(calibration_period(1250) == 2); + const auto beep = calibration_beep(kRate); + REQUIRE(beep.size() == 192); + CHECK(beep[0] == 0); + CHECK(std::abs(beep[1]) > .05); + float peak = 0; + for (float v : beep) peak = std::max(peak, std::abs(v)); + CHECK(peak <= .5f); + CHECK(std::abs(beep.back()) < .01); +} + +TEST_CASE("camera mode: flash start on the rolling-shutter row against the click") +{ + Noise noise; + const auto flashes = train(400); + const double truth = 12.3, travel = 1.5 * CalibratorContract::sound_ms_per_cm; + auto audio = noise_track(13, .003, noise); + for (double t : flashes) add_click(audio, t + truth + travel); + const auto sounds = find_sound_onsets(audio.data(), audio.size(), kRate, 0); + REQUIRE(sounds.size() == flashes.size()); + + SUBCASE("readout known") + { + const auto frames = camera(flashes, 240, 1.0, 3.0, 3.0, noise); + double readout = 0; + const auto onsets = find_flash_onsets(frames, CalibratorContract::led_pulse_ms, &readout); + CHECK(readout == 3.0); + const auto r = camera_calibration(onsets, sounds); + REQUIRE(r.ok); + CHECK(r.pairs >= 20); + CHECK(std::abs(r.offset_ms - truth) < .25); + CHECK(r.uncertainty_ms < .3); + } + SUBCASE("readout unknown: estimated from the lit band") + { + const auto frames = camera(flashes, 240, 0.5, 3.0, 0, noise); + double readout = 0; + const auto onsets = find_flash_onsets(frames, CalibratorContract::led_pulse_ms, &readout); + CHECK(std::abs(readout - 3.0) < .5); + const auto r = camera_calibration(onsets, sounds); + REQUIRE(r.ok); + CHECK(std::abs(r.offset_ms - truth) < .4); + } + SUBCASE("full-frame exposure, readout unknown: fitted over the pulses") + { + // The analyzer locks exposure to the frame interval; iOS gives no readout time. + const auto frames = camera(flashes, 240, 1000 / 240.0, 3.5, 0, noise); + double readout = -1; + const auto onsets = find_flash_onsets(frames, CalibratorContract::led_pulse_ms, &readout); + CHECK(readout == 0); + const auto r = camera_calibration(onsets, sounds); + REQUIRE(r.ok); + CHECK(std::abs(r.readout_ms - 3.5) < .5); + CHECK(std::abs(r.offset_ms - truth) < .4); + } + SUBCASE("long exposure at 60 fps") + { + const auto frames = camera(flashes, 60, 4.0, 14.0, 14.0, noise); + const auto r = camera_calibration(find_flash_onsets(frames), sounds); + REQUIRE(r.ok); + CHECK(std::abs(r.offset_ms - truth) < .5); + } +} + +TEST_CASE("camera mode: missing pulses, a stray sound and nothing at all") +{ + Noise noise; + const auto flashes = train(250); + auto audio = noise_track(13, .01, noise); + for (size_t i = 0; i < flashes.size(); ++i) + if (i % 6 != 2) add_click(audio, flashes[i] + 30); + add_click(audio, 5100, .5); // a tap on the table + const auto frames = camera(flashes, 120, 1.0, 6.0, 6.0, noise, 13, {4, 9, 15}); + const auto onsets = find_flash_onsets(frames); + CHECK(onsets.size() == flashes.size() - 3); + const auto r = camera_calibration(onsets, find_sound_onsets(audio.data(), audio.size(), kRate, 0)); + REQUIRE(r.ok); + CHECK(r.pairs >= 14); + CHECK(std::abs(r.offset_ms - (30 - 1.5 * CalibratorContract::sound_ms_per_cm)) < .3); + + CHECK(!camera_calibration({}, {}).ok); + CHECK(!camera_calibration(onsets, {}).ok); + const std::vector few(onsets.begin(), onsets.begin() + 3); + const auto short_take = camera_calibration(few, find_sound_onsets(audio.data(), audio.size(), kRate, 0)); + CHECK(!short_take.ok); + CHECK(short_take.error.find("Too few") == 0); +} + +TEST_CASE("display mode: click then beep, both swap directions, slow screen") +{ + Noise noise; + const double screen = 45, speaker = 7; // light after the planned swap; sound after its planned time + auto audio = noise_track(13, .003, noise); + std::vector swaps; + for (int k = 0; k < 24; ++k) swaps.push_back(150 + 500 * k); + for (size_t k = 0; k < swaps.size(); ++k) { + // A sensor mismatch: one direction crosses 1.5 ms early, the other 1.5 ms late. + const double mismatch = k % 2 ? -1.5 : 1.5; + if (k != 5 && k != 6 && k != 13) add_click(audio, swaps[k] + screen + mismatch + .2 * noise.gauss()); + add_beep(audio, swaps[k] + CalibratorContract::sound_delay_ms + speaker); + } + const auto sounds = find_sound_onsets(audio.data(), audio.size(), kRate, 0); + const auto r = display_calibration(sounds); + REQUIRE(r.ok); + CHECK(r.pairs == 21); + CHECK(std::abs(r.offset_ms - (speaker - screen)) < .3); + CHECK(std::abs(std::abs(r.direction_ms) - 3) < .5); + CHECK(r.uncertainty_ms < .5); + + // The board 4 cm nearer the microphone than the speaker; 0.3 ms buzzer delay. + DisplayCalibrationOptions o; + o.buzzer_delay_ms = .3; + o.click_minus_speaker_path_cm = -4; + CHECK(std::abs(display_calibration(sounds, o).offset_ms - (r.offset_ms + .3 - 4 * CalibratorContract::sound_ms_per_cm)) < 1e-9); +} + +TEST_CASE("display mode: an external screen with picture later than sound, and failures") +{ + Noise noise; + auto audio = noise_track(8, .003, noise); + for (int k = 0; k < 14; ++k) { + add_click(audio, 300 + 500 * k + 120); // a TV: picture 120 ms late + add_beep(audio, 300 + 500 * k + 100 + 80, .4); // its sound 80 ms late + } + const auto r = display_calibration(find_sound_onsets(audio.data(), audio.size(), kRate, 0)); + REQUIRE(r.ok); + CHECK(std::abs(r.offset_ms - (80 - 120)) < .3); + + auto only_beeps = noise_track(4, .003, noise); + for (int k = 0; k < 8; ++k) add_beep(only_beeps, 200 + 500 * k); + const auto none = display_calibration(find_sound_onsets(only_beeps.data(), only_beeps.size(), kRate, 0)); + CHECK(!none.ok); + CHECK(none.error.find("No clicks") == 0); + + auto only_clicks = noise_track(4, .003, noise); + for (int k = 0; k < 8; ++k) add_click(only_clicks, 200 + 500 * k); + CHECK(display_calibration(find_sound_onsets(only_clicks.data(), only_clicks.size(), kRate, 0)).error.find("No test sound") == 0); +} From 6d752d9d9643f46c202f114bb5f602f41e510d32 Mon Sep 17 00:00:00 2001 From: Adrian Hilber Date: Sun, 8 Nov 2026 15:32:03 +0100 Subject: [PATCH 02/21] Phone analyzer: calibrator take (camera and external display) PhoneAnalyzer::begin_calibrator keeps 32 row-band means per frame and the microphone, and poll_calibrator reports progress about once a second, then the result (20 pairs or 30 s) or an error. No live report or QR scan while it runs. iOS: beginCalibrator(mode) on the camera view; the camera take first re-locks exposure (frame interval) at the lowest ISO, because the LED floods the lens. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017ybuzgBaGxuB5UiaE8dRa4 --- core/include/avsync/phone_analyzer.hpp | 15 ++++ core/src/phone_analyzer.cpp | 87 +++++++++++++++++-- .../modules/avsync/ios/AVSyncPhoneAnalyzer.h | 3 + .../modules/avsync/ios/AVSyncPhoneAnalyzer.mm | 8 ++ .../modules/avsync/ios/AvsyncCameraView.swift | 21 ++++- .../modules/avsync/ios/AvsyncModule.swift | 1 + .../modules/avsync/ios/Core_calibrator.cpp | 1 + tests/unit/test_calibrator.cpp | 44 ++++++++++ 8 files changed, 169 insertions(+), 11 deletions(-) create mode 100644 mobile-app/modules/avsync/ios/Core_calibrator.cpp diff --git a/core/include/avsync/phone_analyzer.hpp b/core/include/avsync/phone_analyzer.hpp index 44706e6..3bfb852 100644 --- a/core/include/avsync/phone_analyzer.hpp +++ b/core/include/avsync/phone_analyzer.hpp @@ -6,6 +6,7 @@ // the device UTC clock; the app adds its NTP/GNSS correction. #include "avsync/avsync.hpp" +#include "avsync/calibrator.hpp" #include #include @@ -55,6 +56,15 @@ class PhoneAnalyzer { std::optional calibration_error() const { return calibration_error_; } std::optional take_calibration(); + // Calibrator board (docs/calibrator.md). Camera: LED flash + click on the rear camera, + // this phone's camera against its microphone. Display: an external screen shows the + // calibration pattern (container generator); only the microphone counts. + enum class CalibratorMode { Camera, Display }; + void begin_calibrator(CalibratorMode mode); + bool calibrator_active() const { return calibrator_.has_value(); } + // About once a second a progress report (JSON), then once the result or an error; else empty. + std::string poll_calibrator(); + // The detected picture (for the preview outline), normalized corners. bool picture(std::array& x, std::array& y) const; @@ -76,6 +86,11 @@ class PhoneAnalyzer { double cal_audio_origin_ = -1, spike_ = -1, floor_ = .002; int rate_ = 48000; void finish_calibration(double now); + // Calibrator take. + std::optional calibrator_; + std::vector flash_frames_; + double calibrator_poll_ = -1; + bool calibrator_done_ = false; }; } // namespace avsync diff --git a/core/src/phone_analyzer.cpp b/core/src/phone_analyzer.cpp index ea01380..15ab30b 100644 --- a/core/src/phone_analyzer.cpp +++ b/core/src/phone_analyzer.cpp @@ -93,7 +93,7 @@ void PhoneAnalyzer::set_device_utc_offset(double offset) void PhoneAnalyzer::observe_qr(const std::string& text, const std::array& x, const std::array& y, double capture_ms) { auto payload = parse_qr_payload(text); - if (!payload || !payload->optical_code || !std::isfinite(capture_ms) || calibration_) return; + if (!payload || !payload->optical_code || !std::isfinite(capture_ms) || calibration_ || calibrator_) return; qr_ = payload; qr_ms_ = capture_ms; QrDetection q; @@ -102,7 +102,7 @@ void PhoneAnalyzer::observe_qr(const std::string& text, const std::array 2 && frame_times_.front() < pts - 3000) frame_times_.pop_front(); + if (calibrator_) { + if (*calibrator_ != CalibratorMode::Camera || calibrator_done_ || flash_frames_.size() > 12000) return; + // Mean luma of 32 row bands (every 8th pixel of every 4th row). + constexpr int kBands = 32; + FlashFrame f{pts, exposure, readout, std::vector(kBands)}; + for (int b = 0; b < kBands; ++b) { + double sum = 0; + int count = 0; + for (int yy = b * h / kBands; yy < (b + 1) * h / kBands; yy += 4) + for (int xx = 0; xx < w; xx += 8) sum += luma[size_t(yy) * stride + size_t(xx)], ++count; + f.rows[size_t(b)] = count ? float(sum / count) : 0; + } + flash_frames_.push_back(std::move(f)); + return; + } if (calibration_) { if (calibration_done_) return; // Keep a small grey copy (at most 640 px) for the review. @@ -135,14 +150,18 @@ void PhoneAnalyzer::push_audio(const float* mono, size_t n, double pts, int rate if (origin_ < 0) origin_ = pts; rate_ = rate; last_audio_ = pts + n * 1000.0 / rate; - if (!calibration_) { + if (!calibration_ && !calibrator_) { session_.push_audio(mono, n, pts, rate, true); return; } - if (calibration_done_) return; + if (calibration_done_ || calibrator_done_) return; if (cal_audio_origin_ < 0) cal_audio_origin_ = pts; const size_t at = size_t(std::max(0.0, std::round((pts - cal_audio_origin_) * rate / 1000))); - if (at > cal_audio_.size() && at < size_t(18 * rate)) cal_audio_.resize(at, 0); + if (at > cal_audio_.size() && at < size_t(40 * rate)) cal_audio_.resize(at, 0); + if (calibrator_) { + if (cal_audio_.size() < size_t(40 * rate)) cal_audio_.insert(cal_audio_.end(), mono, mono + n); + return; + } cal_audio_.insert(cal_audio_.end(), mono, mono + n); // A sharp sound: a peak well above the running floor, after a quiet start. float peak = 0; @@ -198,6 +217,58 @@ void PhoneAnalyzer::finish_calibration(double now) take_ = std::move(take); } +void PhoneAnalyzer::begin_calibrator(CalibratorMode mode) +{ + reset(); + calibrator_ = mode; +} + +std::string PhoneAnalyzer::poll_calibrator() +{ + constexpr int kWanted = 20; // pulses (camera) or swaps (display) for a result + constexpr double kLimitMs = 30000; // then the result with what there is, or an error + if (!calibrator_ || calibrator_done_ || cal_audio_origin_ < 0) return {}; + const double now = std::max(last_video_, last_audio_); + if (calibrator_poll_ >= 0 && now - calibrator_poll_ < 1000) return {}; + calibrator_poll_ = now; + const bool camera = *calibrator_ == CalibratorMode::Camera; + const auto sounds = find_sound_onsets(cal_audio_.data(), cal_audio_.size(), rate_, cal_audio_origin_); + double readout = 0; + std::vector flashes; + if (camera) flashes = find_flash_onsets(flash_frames_, CalibratorContract::led_pulse_ms, &readout); + auto run = [&](int min_pairs) { + if (camera) { + CameraCalibrationOptions o; + o.min_pairs = min_pairs; + return camera_calibration(flashes, sounds, o); + } + DisplayCalibrationOptions o; + o.min_pairs = min_pairs; + return display_calibration(sounds, o); + }; + const auto progress = run(1); + const double elapsed = now - origin_; + Json out; + out.str("mode", camera ? "camera" : "display"); + if (progress.pairs < kWanted && elapsed < kLimitMs) { + size_t clicks = 0; + for (const auto& s : sounds) clicks += s.click(); + out.str("kind", "calibrator-progress").integer("pairs", progress.pairs).integer("wanted", kWanted) + .integer("flashes", long(flashes.size())).integer("clicks", long(clicks)).num("elapsedMs", elapsed); + return out.text(); + } + calibrator_done_ = true; + const auto r = run(8); + if (!r.ok) return Json().str("kind", "error").str("message", r.error).text(); + const double fps = frames_ * 1000.0 / std::max(1.0, last_video_ - origin_); + out.str("kind", "calibrator").str("method", camera ? "calibrator-pulse" : "calibrator-display") + .num("offsetMs", r.offset_ms).num("uncertaintyMs", r.uncertainty_ms).num("spreadMs", r.spread_ms) + .integer("pairs", r.pairs).integer("rejected", r.rejected).num("directionMs", r.direction_ms) + .num("readoutMs", readout > 0 ? readout : r.readout_ms); + if (camera) out.num("fps", fps); + return out.text(); +} + std::optional PhoneAnalyzer::take_calibration() { auto out = std::move(take_); @@ -208,7 +279,7 @@ std::optional PhoneAnalyzer::take_calibration() bool PhoneAnalyzer::picture(std::array& x, std::array& y) const { const auto& pose = session_.current_picture(); - if (calibration_ || !pose.found || video_width_ <= 0 || last_audio_ - last_video_ > 300) return false; + if (calibration_ || calibrator_ || !pose.found || video_width_ <= 0 || last_audio_ - last_video_ > 300) return false; for (int i = 0; i < 4; ++i) x[i] = pose.corner_x[i] / video_width_, y[i] = pose.corner_y[i] / video_height_; return true; } @@ -216,7 +287,7 @@ bool PhoneAnalyzer::picture(std::array& x, std::array& y) bool PhoneAnalyzer::report_due() const { const double now = std::max(last_video_, last_audio_); - return !calibration_ && origin_ >= 0 && now - origin_ >= 1500 && now - last_report_ >= 1000; + return !calibration_ && !calibrator_ && origin_ >= 0 && now - origin_ >= 1500 && now - last_report_ >= 1000; } PhoneAnalyzer PhoneAnalyzer::snapshot_for_poll() @@ -237,7 +308,7 @@ void PhoneAnalyzer::merge_audio(const PhoneAnalyzer& analyzed) std::string PhoneAnalyzer::poll() { - if (calibration_ || origin_ < 0) return {}; + if (calibration_ || calibrator_ || origin_ < 0) return {}; const double now = std::max(last_video_, last_audio_); last_report_ = now; const auto report = session_.finish("PHONE"); diff --git a/mobile-app/modules/avsync/ios/AVSyncPhoneAnalyzer.h b/mobile-app/modules/avsync/ios/AVSyncPhoneAnalyzer.h index b7d8f27..7391263 100644 --- a/mobile-app/modules/avsync/ios/AVSyncPhoneAnalyzer.h +++ b/mobile-app/modules/avsync/ios/AVSyncPhoneAnalyzer.h @@ -7,6 +7,8 @@ NS_ASSUME_NONNULL_BEGIN @interface AVSyncPhoneAnalyzer : NSObject - (void)reset; - (void)beginCalibration; +// Calibrator board take: "camera" (flash + click) or "display" (external screen, microphone only). +- (void)beginCalibrator:(NSString *)mode; - (void)setTaiUtcOffset:(double)seconds; - (void)setDeviceUtcOffset:(double)unixMinusCaptureMs; - (void)observeQRDetection:(NSDictionary *)detection timeMs:(double)timeMs; @@ -18,6 +20,7 @@ NS_ASSUME_NONNULL_BEGIN - (nullable NSDictionary *)poll; - (void)mergeAudio:(AVSyncPhoneAnalyzer *)analyzed; // nil while a take runs; then once an error report or a calibration report with written files. +// A calibrator take also gives a progress report about once a second. - (nullable NSDictionary *)pollCalibration; - (NSDictionary *)pictureObservation; @end diff --git a/mobile-app/modules/avsync/ios/AVSyncPhoneAnalyzer.mm b/mobile-app/modules/avsync/ios/AVSyncPhoneAnalyzer.mm index c61d239..6edb7e0 100644 --- a/mobile-app/modules/avsync/ios/AVSyncPhoneAnalyzer.mm +++ b/mobile-app/modules/avsync/ios/AVSyncPhoneAnalyzer.mm @@ -32,6 +32,7 @@ @implementation AVSyncPhoneAnalyzer { std::unique_ptr cor - (instancetype)init { if((self=[super init])){core=std::make_unique();} return self; } - (void)reset { core->reset(); } - (void)beginCalibration { core->begin_calibration(); } +- (void)beginCalibrator:(NSString *)mode { core->begin_calibrator([mode isEqualToString:@"display"]?avsync::PhoneAnalyzer::CalibratorMode::Display:avsync::PhoneAnalyzer::CalibratorMode::Camera); } - (void)setTaiUtcOffset:(double)seconds { core->set_tai_utc_seconds(seconds); } - (void)setDeviceUtcOffset:(double)offset { core->set_device_utc_offset(offset); } - (void)observeQRDetection:(NSDictionary *)detection timeMs:(double)timeMs { @@ -84,6 +85,13 @@ - (NSDictionary *)poll { } - (void)mergeAudio:(AVSyncPhoneAnalyzer *)analyzed { if(analyzed)core->merge_audio(*analyzed->core); } - (NSDictionary *)pollCalibration { + if(core->calibrator_active()){ + const auto text=core->poll_calibrator(); + if(text.empty())return nil; + NSDictionary *report=[NSJSONSerialization JSONObjectWithData:[NSData dataWithBytes:text.data() length:text.size()] options:0 error:nil]; + if(![report[@"kind"] isEqual:@"calibrator-progress"])core->reset(); // done: live analysis resumes + return report; + } if(!core->calibrating())return nil; if(auto error=core->calibration_error()){NSString *message=[NSString stringWithUTF8String:error->c_str()];core->reset();return @{@"kind":@"error",@"message":message};} auto take=core->take_calibration(); diff --git a/mobile-app/modules/avsync/ios/AvsyncCameraView.swift b/mobile-app/modules/avsync/ios/AvsyncCameraView.swift index 2948956..e53e0bb 100644 --- a/mobile-app/modules/avsync/ios/AvsyncCameraView.swift +++ b/mobile-app/modules/avsync/ios/AvsyncCameraView.swift @@ -24,6 +24,7 @@ final class AvsyncCameraView: ExpoView, AVCaptureVideoDataOutputSampleBufferDele // samples queue private var analysisBusy = false, qrBusy = false, lastQrScan = -10.0, lastPoll = 0.0, generation = 0 private var focusing = false // the 1.5 s auto step: frames are dropped until the lock + private var pendingCalibrator: String? // samples queue: starts at the next restart (the exposure lock) private var targetFps = 0.0, captureMode = "unknown", capturedSampleRate = 0.0, exposureMs = 0.0 // main queue private var active = true, background = false @@ -84,6 +85,17 @@ final class AvsyncCameraView: ExpoView, AVCaptureVideoDataOutputSampleBufferDele DispatchQueue.main.async { [weak self] in self?.clearOutline() } samples.async { [weak self] in guard let self else { return }; self.generation += 1; self.focusing = false; self.processor.beginCalibration() } } + // Calibrator board take ("camera" or "display", docs/calibrator.md). For the camera the LED + // floods the lens: lock exposure again at the lowest gain first; the take starts at the lock. + func beginCalibrator(_ mode: String) { + DispatchQueue.main.async { [weak self] in self?.clearOutline() } + guard mode == "camera" else { + samples.async { [weak self] in guard let self else { return }; self.generation += 1; self.focusing = false; self.processor.beginCalibrator(mode) } + return + } + samples.async { [weak self] in self?.pendingCalibrator = mode } + control.async { [weak self] in self?.scheduleExposureLock(lowGain: true) } + } // Tap to focus (x, y: 0…1 across the view): re-run the auto step metered on that spot. func focus(atX x: Double, y: Double) { DispatchQueue.main.async { [weak self] in @@ -103,7 +115,10 @@ final class AvsyncCameraView: ExpoView, AVCaptureVideoDataOutputSampleBufferDele // MARK: session - private func restart() { generation += 1; focusing = false; processor.reset(); telemetry.reset(fps: targetFps) } + private func restart() { + generation += 1; focusing = false; processor.reset(); telemetry.reset(fps: targetFps) + if let mode = pendingCalibrator { pendingCalibrator = nil; processor.beginCalibrator(mode) } + } private func updateRunning() { let shouldRun = window != nil && active && !background control.async { [weak self] in @@ -206,7 +221,7 @@ final class AvsyncCameraView: ExpoView, AVCaptureVideoDataOutputSampleBufferDele // Focus and exposure are metered at the tap point, else at the centre (device coordinates). // The analysis holds during the auto step and starts fresh at the lock (also on camera start // and Restart), so no frame from the auto step reaches a reading. - private func scheduleExposureLock(at point: CGPoint? = nil) { + private func scheduleExposureLock(at point: CGPoint? = nil, lowGain: Bool = false) { lockGeneration += 1 let ticket = lockGeneration samples.async { [weak self] in self?.generation += 1; self?.focusing = true } @@ -227,7 +242,7 @@ final class AvsyncCameraView: ExpoView, AVCaptureVideoDataOutputSampleBufferDele defer { device.unlockForConfiguration() } let frame = device.activeVideoMinFrameDuration, longest = device.activeFormat.maxExposureDuration let duration = CMTimeCompare(frame, longest) <= 0 ? frame : longest - let iso = min(max(device.iso * Float(device.exposureDuration.seconds / duration.seconds), device.activeFormat.minISO), device.activeFormat.maxISO) + let iso = lowGain ? device.activeFormat.minISO : min(max(device.iso * Float(device.exposureDuration.seconds / duration.seconds), device.activeFormat.minISO), device.activeFormat.maxISO) if device.isExposureModeSupported(.custom) { device.setExposureModeCustom(duration: duration, iso: iso, completionHandler: nil) } if device.isFocusModeSupported(.locked) { device.focusMode = .locked } if device.isWhiteBalanceModeSupported(.locked) { device.whiteBalanceMode = .locked } diff --git a/mobile-app/modules/avsync/ios/AvsyncModule.swift b/mobile-app/modules/avsync/ios/AvsyncModule.swift index 26e05d8..c92a0b8 100644 --- a/mobile-app/modules/avsync/ios/AvsyncModule.swift +++ b/mobile-app/modules/avsync/ios/AvsyncModule.swift @@ -59,6 +59,7 @@ public class AvsyncModule: Module { Prop("captureFps") { (view: AvsyncCameraView, fps: Double) in view.setRequestedFps(fps) } Prop("taiUtcOffset") { (view: AvsyncCameraView, seconds: Double) in view.setTaiUtcOffset(seconds) } AsyncFunction("beginCalibration") { (view: AvsyncCameraView) in view.beginCalibration() } + AsyncFunction("beginCalibrator") { (view: AvsyncCameraView, mode: String) in view.beginCalibrator(mode) } AsyncFunction("resumeAnalysis") { (view: AvsyncCameraView) in view.resumeAnalysis() } AsyncFunction("focusAt") { (view: AvsyncCameraView, x: Double, y: Double) in view.focus(atX: x, y: y) } } diff --git a/mobile-app/modules/avsync/ios/Core_calibrator.cpp b/mobile-app/modules/avsync/ios/Core_calibrator.cpp new file mode 100644 index 0000000..12f095d --- /dev/null +++ b/mobile-app/modules/avsync/ios/Core_calibrator.cpp @@ -0,0 +1 @@ +#include "../../../../core/src/calibrator.cpp" diff --git a/tests/unit/test_calibrator.cpp b/tests/unit/test_calibrator.cpp index fa87825..3dd73c9 100644 --- a/tests/unit/test_calibrator.cpp +++ b/tests/unit/test_calibrator.cpp @@ -247,3 +247,47 @@ TEST_CASE("display mode: an external screen with picture later than sound, and f for (int k = 0; k < 8; ++k) add_click(only_clicks, 200 + 500 * k); CHECK(display_calibration(find_sound_onsets(only_clicks.data(), only_clicks.size(), kRate, 0)).error.find("No test sound") == 0); } + +#include "avsync/phone_analyzer.hpp" +#include + +TEST_CASE("phone analyzer: calibrator camera take reports progress, then the result") +{ + Noise noise; + const auto flashes = train(300); + auto audio = noise_track(14, .003, noise); + for (double t : flashes) add_click(audio, t + 20 + 1.5 * CalibratorContract::sound_ms_per_cm); + // 128 x 128 luma at 240 fps, readout 3 ms, exposure 1 ms; 10 ms audio blocks. + const int w = 128, h = 128; + const double fps = 240, exposure = 1, readout = 3, led = CalibratorContract::led_pulse_ms; + PhoneAnalyzer analyzer; + analyzer.begin_calibrator(PhoneAnalyzer::CalibratorMode::Camera); + CHECK(analyzer.calibrator_active()); + std::vector luma(size_t(w) * h); + std::string last; + int progress = 0; + size_t sample = 0; + for (double t = 1; t < 14000 && last.find("\"kind\":\"calibrator\"") == std::string::npos; t += 1000 / fps) { + for (int y = 0; y < h; ++y) { + const double start = t + readout * (y + .5) / h; + double light = 0; + for (double f : flashes) light += std::max(0.0, std::min(start + exposure, f + led) - std::max(start, f)); + std::fill_n(luma.begin() + long(y) * w, w, uint8_t(std::lround(16 + 239 * std::pow(std::min(1.0, light / std::min(exposure, led)), 1 / 2.2)))); + } + analyzer.push_video(luma.data(), w, h, w, t, exposure, readout); + while (sample * 1000.0 / kRate < t && sample + 480 <= audio.size()) { + analyzer.push_audio(audio.data() + sample, 480, sample * 1000.0 / kRate, kRate); + sample += 480; + } + const auto report = analyzer.poll_calibrator(); + if (report.find("calibrator-progress") != std::string::npos) ++progress; + if (!report.empty()) last = report; + } + CHECK(progress >= 5); + REQUIRE(last.find("\"kind\":\"calibrator\"") != std::string::npos); + CHECK(last.find("\"method\":\"calibrator-pulse\"") != std::string::npos); + const auto at = last.find("\"offsetMs\":"); + REQUIRE(at != std::string::npos); + CHECK(std::abs(std::stod(last.substr(at + 11)) - 20) < .4); + CHECK(analyzer.poll().empty()); // no live report while calibrating +} From 2df391fcb2038a5d18d464b9ea944a07d00e705d Mon Sep 17 00:00:00 2001 From: Adrian Hilber Date: Sun, 8 Nov 2026 15:35:34 +0100 Subject: [PATCH 03/21] iPhone generator: calibration pattern mode and sender calibration in the QR generateStyled(..., {mode: 'calibration'}) shows the calibration pattern in the generator's own screen: the same display link, layers, audio engine and player node as the test signal. Portrait halves left and right near the bottom microphone, a red outline of the board, halves swap every 500 ms, a 1.5 kHz beep 100 ms after each swap (scheduled on the host clock minus the reported output latency). The session is play-and-record on the built-in speaker; the microphone take is analysed with the shared core (display_calibration) and the result comes back when the screen closes. The test signal sends a saved sender calibration in the AVS4 QR (senderCalibration* options), only on the audio output it was measured on. Core pattern helpers are now frame based (also 59.94 fps). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017ybuzgBaGxuB5UiaE8dRa4 --- core/include/avsync/calibrator.hpp | 10 +- core/src/calibrator.cpp | 14 +- .../modules/avsync/ios/AVSyncGenerator.h | 9 + .../modules/avsync/ios/AVSyncGenerator.mm | 28 ++- .../ios/AvsyncGeneratorController.swift | 162 +++++++++++++++++- .../modules/avsync/ios/AvsyncModule.swift | 4 +- mobile-app/tests/native/generator-runner.mm | 24 +++ tests/unit/test_calibrator.cpp | 22 ++- 8 files changed, 251 insertions(+), 22 deletions(-) diff --git a/core/include/avsync/calibrator.hpp b/core/include/avsync/calibrator.hpp index 78e344d..822fafe 100644 --- a/core/include/avsync/calibrator.hpp +++ b/core/include/avsync/calibrator.hpp @@ -86,10 +86,12 @@ struct DisplayCalibrationOptions { // microphone delay cancels. Both swap directions count equally. CalibratorResult display_calibration(const std::vector& sounds, const DisplayCalibrationOptions& options = {}); -// The calibration pattern, the same for all generators. Time is from the pattern start; -// swap k happens at k * 500 ms. In even periods the left half is white. -bool calibration_left_white(double ms_since_start); -uint64_t calibration_period(double ms_since_start); +// The calibration pattern, the same for all generators. Frames count from the pattern +// start at num/den frames per second; period k (swap k at its first frame) lasts 500 ms +// on the frame grid. In even periods the left half is white. +uint64_t calibration_period(uint64_t frame, int64_t num, int64_t den); +bool calibration_left_white(uint64_t frame, int64_t num, int64_t den); +uint64_t calibration_swap_frame(uint64_t k, int64_t num, int64_t den); // The generator's beep (mono, starts at sample 0, peak 0.5), played D after each swap. std::vector calibration_beep(int sample_rate); diff --git a/core/src/calibrator.cpp b/core/src/calibrator.cpp index 6570532..2b01024 100644 --- a/core/src/calibrator.cpp +++ b/core/src/calibrator.cpp @@ -313,9 +313,19 @@ CalibratorResult display_calibration(const std::vector& sounds, cons return r; } -uint64_t calibration_period(double ms) { return ms <= 0 ? 0 : uint64_t(std::floor(ms / CalibratorContract::swap_period_ms + 1e-9)); } +uint64_t calibration_period(uint64_t frame, int64_t num, int64_t den) +{ + // t = frame * den / num seconds; period = floor(t / 0.5 s), in integers. + return num > 0 && den > 0 ? frame * 2 * uint64_t(den) / uint64_t(num) : 0; +} + +bool calibration_left_white(uint64_t frame, int64_t num, int64_t den) { return calibration_period(frame, num, den) % 2 == 0; } -bool calibration_left_white(double ms) { return calibration_period(ms) % 2 == 0; } +uint64_t calibration_swap_frame(uint64_t k, int64_t num, int64_t den) +{ + // The first frame whose period is k: ceil(k * num / (2 * den)). + return num > 0 && den > 0 ? (k * uint64_t(num) + 2 * uint64_t(den) - 1) / (2 * uint64_t(den)) : 0; +} std::vector calibration_beep(int rate) { diff --git a/mobile-app/modules/avsync/ios/AVSyncGenerator.h b/mobile-app/modules/avsync/ios/AVSyncGenerator.h index 35ab8a0..db3b20b 100644 --- a/mobile-app/modules/avsync/ios/AVSyncGenerator.h +++ b/mobile-app/modules/avsync/ios/AVSyncGenerator.h @@ -13,6 +13,15 @@ NS_ASSUME_NONNULL_BEGIN - (double)hostTimeForSequence:(uint64_t)sequence NS_SWIFT_NAME(hostTime(sequence:)); - (NSData *)audioForSequence:(uint64_t)sequence NS_SWIFT_NAME(audio(sequence:)); - (UIImage *)imageForSequence:(uint64_t)sequence NS_SWIFT_NAME(image(sequence:)); +// AVS4 sender calibration (this phone's screen vs speaker); method none until set. +- (void)setSenderCalibrationMethod:(NSString *)method offsetMs:(double)offsetMs uncertaintyMs:(double)uncertaintyMs day:(NSInteger)day NS_SWIFT_NAME(setSenderCalibration(method:offsetMs:uncertaintyMs:day:)); + +// Calibration pattern (docs/calibrator.md display mode; core/include/avsync/calibrator.hpp). ++ (BOOL)calibrationLeftWhiteAtFrame:(uint64_t)frame fps:(NSInteger)fps NS_SWIFT_NAME(calibrationLeftWhite(frame:fps:)); ++ (uint64_t)calibrationSwapFrame:(uint64_t)swap fps:(NSInteger)fps NS_SWIFT_NAME(calibrationSwapFrame(swap:fps:)); ++ (NSData *)calibrationBeepWithSampleRate:(double)sampleRate NS_SWIFT_NAME(calibrationBeep(sampleRate:)); +// Clicks and beeps in mono float samples: pairs, clicks and, with enough pairs, the result. ++ (NSDictionary *)displayCalibrationForSamples:(NSData *)samples sampleRate:(double)sampleRate minimumPairs:(NSInteger)minimumPairs NS_SWIFT_NAME(displayCalibration(samples:sampleRate:minimumPairs:)); @end NS_ASSUME_NONNULL_END diff --git a/mobile-app/modules/avsync/ios/AVSyncGenerator.mm b/mobile-app/modules/avsync/ios/AVSyncGenerator.mm index c0b0c57..0b33548 100644 --- a/mobile-app/modules/avsync/ios/AVSyncGenerator.mm +++ b/mobile-app/modules/avsync/ios/AVSyncGenerator.mm @@ -1,8 +1,10 @@ #import "AVSyncGenerator.h" #include "avsync/phone_generator.hpp" +#include "avsync/calibrator.hpp" #include "../../../../generator/picture.hpp" #include #include +#include namespace { void rgb(uint16_t y,uint16_t cb,uint16_t cr,uint8_t *out) { @@ -44,6 +46,7 @@ @implementation AVSyncGenerator { avsync::PictureInfo _info; avsync::SourceStill _sourceImage; avsync::SourceStill _appIcon; + avsync::SenderCalibration _calibration; } + (NSString *)testIDWithPrefix:(NSString *)prefix installID:(NSString *)installID version:(NSString *)version settings:(NSArray *)settings { std::vector values; @@ -91,11 +94,32 @@ - (NSDictionary *)frameAt:(double)target { - (double)hostTimeForSequence:(uint64_t)seq { return _plan->host_seconds(seq); } - (NSData *)audioForSequence:(uint64_t)seq { const auto samples=_plan->audio(seq);return [NSData dataWithBytes:samples.data() length:samples.size()*sizeof(float)]; } - (UIImage *)imageForSequence:(uint64_t)seq { - // AVS4 carries the shown name; the output offset is not calibrated yet (method none). - auto qr=_plan->qr(seq);qr.name=_info.source; + // AVS4 carries the shown name and this phone's sender calibration (method none until saved). + auto qr=_plan->qr(seq);qr.name=_info.source;qr.calibration=_calibration; _info.qr_json=avsync::qr_payload_text(qr);_info.counter="Event "+std::to_string(seq);_info.time_line="Scheduled source output"; avsync::Yuv422p10 frame;frame.resize(1280,720); avsync::render_picture(frame,_info,seq,0,_plan->period_frames(),false); return image_from_frame(frame); } +- (void)setSenderCalibrationMethod:(NSString *)method offsetMs:(double)offset uncertaintyMs:(double)uncertainty day:(NSInteger)day { + _calibration={}; + const std::string m=method.UTF8String?:""; + // The QR encoder throws on values it cannot carry: keep method none then. + if((m!="clap"&&m!="transponder"&&m!="pulse-train")||!std::isfinite(offset)||std::abs(offset)>3276||!std::isfinite(uncertainty)||uncertainty<0||uncertainty>127||day<0||day>0xFFFF)return; + _calibration.method=m;_calibration.offset_ms=offset;_calibration.uncertainty_ms=uncertainty;_calibration.day=int(day); +} ++ (BOOL)calibrationLeftWhiteAtFrame:(uint64_t)frame fps:(NSInteger)fps { return avsync::calibration_left_white(frame,fps,1); } ++ (uint64_t)calibrationSwapFrame:(uint64_t)swap fps:(NSInteger)fps { return avsync::calibration_swap_frame(swap,fps,1); } ++ (NSData *)calibrationBeepWithSampleRate:(double)rate { + const auto beep=avsync::calibration_beep(int(std::lround(rate))); + return [NSData dataWithBytes:beep.data() length:beep.size()*sizeof(float)]; +} ++ (NSDictionary *)displayCalibrationForSamples:(NSData *)samples sampleRate:(double)rate minimumPairs:(NSInteger)minimum { + const auto sounds=avsync::find_sound_onsets((const float*)samples.bytes,samples.length/sizeof(float),int(std::lround(rate)),0); + avsync::DisplayCalibrationOptions options;options.min_pairs=int(std::max(1,minimum)); + const auto r=avsync::display_calibration(sounds,options); + NSInteger clicks=0;for(const auto& sound:sounds)clicks+=sound.click(); + if(!r.ok)return @{@"ok":@NO,@"error":@(r.error.c_str()),@"clicks":@(clicks),@"sounds":@(sounds.size()),@"pairs":@0}; + return @{@"ok":@YES,@"offsetMs":@(r.offset_ms),@"uncertaintyMs":@(r.uncertainty_ms),@"spreadMs":@(r.spread_ms),@"pairs":@(r.pairs),@"rejected":@(r.rejected),@"directionMs":@(r.direction_ms),@"clicks":@(clicks),@"sounds":@(sounds.size())}; +} @end diff --git a/mobile-app/modules/avsync/ios/AvsyncGeneratorController.swift b/mobile-app/modules/avsync/ios/AvsyncGeneratorController.swift index f2f292b..95867e2 100644 --- a/mobile-app/modules/avsync/ios/AvsyncGeneratorController.swift +++ b/mobile-app/modules/avsync/ios/AvsyncGeneratorController.swift @@ -2,6 +2,8 @@ import AVFoundation import UIKit // Output scheduling stays entirely native. JS only opens/closes this screen. +// With `calibrate` it shows the calibration pattern instead (docs/calibrator.md, display mode): +// the same display link, layers, audio engine and player node, plus the microphone. final class AvsyncGeneratorController: UIViewController { var onClose: (([String: Any]) -> Void)? private let rate: Int @@ -42,10 +44,26 @@ final class AvsyncGeneratorController: UIViewController { private var utcMinusHost = 0.0 private let clockReference: [String:String] private var usingReference = false + private let senderCalibration: [String:String] + private var senderCalibrationNote = "" + // Calibration pattern: two halves, the calibrator outline, the microphone take. + private let calibrate: Bool + private let leftHalf = CALayer(), rightHalf = CALayer(), outline = CAShapeLayer() + private let recordQueue = DispatchQueue(label: "avsync.generator.record") + private var recorded: [Float] = [] // recordQueue + private var recordRate = 48000.0 + private var nextBeep: UInt64 = 1 + private var analysing = false, lastAnalysis = 0.0 + private var calibrationResult: [String: Any]? + private var calibrationError: String? + private static let wantedPairs = 20, calibrationLimit = 30.0, pointsPerMm: CGFloat = 6 // about 6 pt per mm on current iPhones - init(rate: Int, taiUtc: Double, name: String = "iPhone", subtitle: String = "", image: UIImage? = nil, clockReference: [String:String] = [:], keepScreenAwake: Bool = true) { + init(rate: Int, taiUtc: Double, name: String = "iPhone", subtitle: String = "", image: UIImage? = nil, clockReference: [String:String] = [:], keepScreenAwake: Bool = true, + senderCalibration: [String:String] = [:], calibrate: Bool = false) { self.keepScreenAwake = keepScreenAwake self.clockReference = clockReference + self.senderCalibration = senderCalibration + self.calibrate = calibrate self.rate = rate; self.taiUtc = taiUtc; self.sourceName = name; self.subtitle = subtitle; self.sourceImage = image super.init(nibName: nil, bundle: nil) modalPresentationStyle = .fullScreen @@ -75,6 +93,13 @@ final class AvsyncGeneratorController: UIViewController { toggle.accessibilityIdentifier = "generatorToggle"; view.addSubview(toggle) close.setTitle("Close", for: .normal); close.addTarget(self, action: #selector(closeScreen), for: .touchUpInside) close.accessibilityIdentifier = "generatorClose"; view.addSubview(close) + if calibrate { + canvas.isHidden = true + leftHalf.backgroundColor = UIColor.white.cgColor; rightHalf.backgroundColor = UIColor.black.cgColor + outline.strokeColor = UIColor.systemRed.cgColor; outline.fillColor = nil; outline.lineWidth = 2 + for layer in [leftHalf, rightHalf, outline] { view.layer.addSublayer(layer) } + toggle.setTitle("Stop", for: .normal) + } engine.attach(player) engine.connect(player, to: engine.mainMixerNode, format: AVAudioFormat(standardFormatWithSampleRate: 48000, channels: 1)!) oldBrightness = UIScreen.main.brightness @@ -95,6 +120,7 @@ final class AvsyncGeneratorController: UIViewController { override func viewDidAppear(_ animated: Bool) { super.viewDidAppear(animated); start() } override func viewDidLayoutSubviews() { super.viewDidLayoutSubviews() + if calibrate { layoutCalibration(); return } // Keep capture screens portrait. Only this test raster rotates; holding the // phone sideways gives a large landscape target without changing camera UI. let inset = view.safeAreaInsets @@ -111,10 +137,13 @@ final class AvsyncGeneratorController: UIViewController { guard link == nil, !finishing else { return } do { let session = AVAudioSession.sharedInstance() - try session.setCategory(.playback, mode: .default, options: []) + // Calibration also records: play-and-record, still on the built-in speaker (not the receiver). + if calibrate { try session.setCategory(.playAndRecord, mode: .default, options: [.defaultToSpeaker]) } + else { try session.setCategory(.playback, mode: .default, options: []) } try session.setPreferredSampleRate(48000) try session.setPreferredIOBufferDuration(0.005) try session.setActive(true) + if calibrate { try startRecording() } engine.prepare(); try engine.start() outputLatency = player.outputPresentationLatency guard outputLatency.isFinite, outputLatency >= 0, outputLatency < 1 else { @@ -127,8 +156,9 @@ final class AvsyncGeneratorController: UIViewController { if #available(iOS 15.0, *) { display.preferredFrameRateRange = CAFrameRateRange(minimum: 60, maximum: 60, preferred: 60) } else { display.preferredFramesPerSecond = 60 } display.add(to: .main, forMode: .common); link = display - toggle.setTitle("Stop signal", for: .normal) - status.text = "Turn phone sideways · audible test signal starts shortly\n1200 baud · \(rate)/s · estimated source clock / output timing" + toggle.setTitle(calibrate ? "Stop" : "Stop signal", for: .normal) + status.text = calibrate ? "Lay the calibrator on the red outline, sensors over the line. Keep the volume up." + : "Turn phone sideways · audible test signal starts shortly\n1200 baud · \(rate)/s · estimated source clock / output timing" } catch { stop("Cannot start: \(error.localizedDescription)") } } private func beginPlan(target: Double) throws { @@ -151,6 +181,7 @@ final class AvsyncGeneratorController: UIViewController { plan = AVSyncGenerator(testID: sourceID, rate: rate, epochTaiNs: epochTai, epochHost: epochHost) guard let plan else { return } plan.setSource(name: sourceName, subtitle: subtitle, image: sourceImage) + applySenderCalibration(plan) syncForeground.contents = plan.syncForeground().cgImage let geometry = plan.geometry() func n(_ key: String) -> CGFloat { CGFloat((geometry[key] as? NSNumber)?.doubleValue ?? 0) } @@ -207,6 +238,7 @@ final class AvsyncGeneratorController: UIViewController { @objc private func tick(_ display: CADisplayLink) { guard link != nil else { return } guard engine.isRunning else { stop("Audio engine stopped. Tap Start to restart."); return } + if calibrate { calibrationTick(display); return } if plan == nil { do { try beginPlan(target: display.targetTimestamp) } catch { stop(error.localizedDescription); return } } @@ -249,11 +281,12 @@ final class AvsyncGeneratorController: UIViewController { if abs(Date().timeIntervalSince1970 - now - utcMinusHost) > 0.1 { stop("Phone clock changed. Tap Start to create a fresh clock mapping."); return } - status.text = "TEST SIGNAL ACTIVE · 1200 baud · \(rate)/s · frame skips \(lateFrames)\nEstimated output timing · \(usingReference ? "NTP UTC" : "device UTC") · audio lead \(Int((outputLatency*1000).rounded())) ms" + status.text = "TEST SIGNAL ACTIVE · 1200 baud · \(rate)/s · frame skips \(lateFrames)\nEstimated output timing · \(usingReference ? "NTP UTC" : "device UTC") · audio lead \(Int((outputLatency*1000).rounded())) ms\(senderCalibrationNote)" } } private func stop(_ message: String) { link?.invalidate(); link = nil; renderGeneration += 1 + if calibrate { engine.inputNode.removeTap(onBus: 0) } player.stop(); engine.stop(); plan = nil; images.removeAll() CATransaction.begin(); CATransaction.setDisableActions(true) canvas.layer.contents = nil; syncDot.isHidden = true; markerA.backgroundColor = UIColor.black.cgColor; markerB.backgroundColor = UIColor.white.cgColor @@ -261,13 +294,15 @@ final class AvsyncGeneratorController: UIViewController { CATransaction.commit() screenAwake.setActive(false); UIScreen.main.brightness = oldBrightness try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) - status.text = message; toggle.setTitle("Start signal", for: .normal) + status.text = message; toggle.setTitle(calibrate ? "Start" : "Start signal", for: .normal) } @objc private func toggleSignal() { if link == nil { start() } else { stop("Signal stopped. Tap Start for a fresh source session.") } } func shutdown() { closeScreen() } @objc private func closeScreen() { guard !finishing else { return }; finishing = true - let result: [String: Any] = ["source": sourceID, "eventRate": rate, "skippedDisplayFrames": lateFrames, "audioPresentationLatencyMs": outputLatency*1000, "estimated": true, "clockReference": clockReference, "clockApplied": usingReference] + var result: [String: Any] = ["source": sourceID, "eventRate": rate, "skippedDisplayFrames": lateFrames, "audioPresentationLatencyMs": outputLatency*1000, "estimated": true, "clockReference": clockReference, "clockApplied": usingReference] + if let calibrationResult { result["calibration"] = calibrationResult } + if let calibrationError { result["calibrationError"] = calibrationError } stop("Closed") for token in notices { NotificationCenter.default.removeObserver(token) }; notices.removeAll() // Playback has stopped and cleanup is complete. Resolve independently of @@ -277,5 +312,118 @@ final class AvsyncGeneratorController: UIViewController { callback?(result) } override func viewDidDisappear(_ animated: Bool) { super.viewDidDisappear(animated); if !finishing { stop("Signal stopped") } } + // MARK: sender calibration and calibration pattern + + static func outputRoute() -> String { + AVAudioSession.sharedInstance().currentRoute.outputs.map { $0.portType.rawValue }.sorted().joined(separator: "+") + } + // The saved screen-vs-speaker calibration goes into the AVS4 QR, but only on the audio output it was measured on. + private func applySenderCalibration(_ plan: AVSyncGenerator) { + senderCalibrationNote = "" + guard let method = senderCalibration["senderCalibrationMethod"], let offset = Double(senderCalibration["senderCalibrationOffsetMs"] ?? ""), + let uncertainty = Double(senderCalibration["senderCalibrationUncertaintyMs"] ?? ""), let day = Int(senderCalibration["senderCalibrationDay"] ?? "") else { return } + if let route = senderCalibration["senderCalibrationRoute"], route != Self.outputRoute() { + senderCalibrationNote = " · sender calibration not sent (other audio output)"; return + } + plan.setSenderCalibration(method: method, offsetMs: offset, uncertaintyMs: uncertainty, day: day) + senderCalibrationNote = String(format: " · sender calibration %+.1f ms", offset) + } + private func startRecording() throws { + let input = engine.inputNode, format = input.outputFormat(forBus: 0) + guard format.sampleRate > 0, format.channelCount > 0 else { + throw NSError(domain: "Avsync", code: 22, userInfo: [NSLocalizedDescriptionKey: "The microphone is not available. Allow microphone access in Settings."]) + } + recordRate = format.sampleRate + recordQueue.sync { recorded.removeAll() } + input.removeTap(onBus: 0) + input.installTap(onBus: 0, bufferSize: 4800, format: format) { [weak self] buffer, _ in + guard let data = buffer.floatChannelData?[0] else { return } + let samples = Array(UnsafeBufferPointer(start: data, count: Int(buffer.frameLength))) + self?.recordQueue.async { self?.recorded.append(contentsOf: samples) } + } + } + // Portrait, halves left and right (the pattern line crosses the display rows), near the + // microphone end at the bottom. The outline is the 66 x 22 mm board, sensor end down. + private func layoutCalibration() { + let inset = view.safeAreaInsets, w = view.bounds.width, h = view.bounds.height, mm = Self.pointsPerMm + toggle.frame = CGRect(x: 16, y: inset.top + 8, width: w/2 - 24, height: 40) + close.frame = CGRect(x: w/2 + 8, y: toggle.frame.minY, width: w/2 - 24, height: 40) + status.frame = CGRect(x: 12, y: toggle.frame.maxY + 4, width: w - 24, height: 52) + let top = max(status.frame.maxY + 8, h - 62 * mm) + CATransaction.begin(); CATransaction.setDisableActions(true) + leftHalf.frame = CGRect(x: 0, y: top, width: w/2, height: h - top) + rightHalf.frame = CGRect(x: w/2, y: top, width: w - w/2, height: h - top) + // Sensors (5 mm from the notch end) at the middle of the pattern. + let notch = (top + h) / 2 + 5 * mm + let board = UIBezierPath(roundedRect: CGRect(x: w/2 - 11 * mm, y: notch - 66 * mm, width: 22 * mm, height: 66 * mm), cornerRadius: 1.5 * mm) + board.move(to: CGPoint(x: w/2 - 1.5 * mm, y: notch + 2 * mm)); board.addLine(to: CGPoint(x: w/2 + 1.5 * mm, y: notch + 2 * mm)) // notch tick + outline.path = board.cgPath + CATransaction.commit() + } + private func swapHost(_ k: UInt64) -> Double { epochHost + Double(AVSyncGenerator.calibrationSwapFrame(swap: k, fps: 60)) / 60 } + private func calibrationTick(_ display: CADisplayLink) { + let now = CACurrentMediaTime() + if plan == nil { + // The plan only marks a running session here (route and configuration changes stop it). + epochHost = display.targetTimestamp + 1.0 + plan = AVSyncGenerator(testID: "calibration", rate: 2, epochTaiNs: 0, epochHost: epochHost) + nextBeep = 1; lateFrames = 0; lastFrame = nil; lastAnalysis = now; calibrationResult = nil; calibrationError = nil + } + // The beep: D = 100 ms after each swap on the same host clock, minus the reported output latency. + while swapHost(nextBeep) + 0.1 < now + 2.5 { + let host = swapHost(nextBeep) + 0.1 - outputLatency + guard host > now + 0.03 else { stop("Scheduling deadline missed. Tap Start to try again."); return } + let data = AVSyncGenerator.calibrationBeep(sampleRate: 48000) + let format = AVAudioFormat(standardFormatWithSampleRate: 48000, channels: 1)! + let count = AVAudioFrameCount(data.count / MemoryLayout.size) + let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: count)! + buffer.frameLength = count + _ = data.withUnsafeBytes { bytes in memcpy(buffer.floatChannelData![0], bytes.baseAddress!, data.count) } + player.scheduleBuffer(buffer, at: AVAudioTime(hostTime: AVAudioTime.hostTime(forSeconds: host)), options: []) + nextBeep += 1 + } + let position = (display.targetTimestamp - epochHost) * 60 + let frame = UInt64(max(0, position.rounded())) + if position > -0.5 { + if let previous = lastFrame, frame > previous + 1 { lateFrames += Int(frame - previous - 1) } + lastFrame = frame + } + let white = position < -0.5 || AVSyncGenerator.calibrationLeftWhite(frame: frame, fps: 60) + CATransaction.begin(); CATransaction.setDisableActions(true) + leftHalf.backgroundColor = (white ? UIColor.white : UIColor.black).cgColor + rightHalf.backgroundColor = (white ? UIColor.black : UIColor.white).cgColor + CATransaction.commit() + guard !analysing, now - lastAnalysis >= 1, now > epochHost + 1.5 else { return } + lastAnalysis = now; analysing = true + let running = now - epochHost, rate = recordRate, skips = lateFrames, latency = outputLatency + recordQueue.async { [weak self] in + guard let self else { return } + let data = self.recorded.withUnsafeBufferPointer { Data(buffer: $0) } + let progress = AVSyncGenerator.displayCalibration(samples: data, sampleRate: rate, minimumPairs: 1) + let pairs = (progress["pairs"] as? NSNumber)?.intValue ?? 0, clicks = (progress["clicks"] as? NSNumber)?.intValue ?? 0 + let done = pairs >= Self.wantedPairs || running >= Self.calibrationLimit + let final = done ? AVSyncGenerator.displayCalibration(samples: data, sampleRate: rate, minimumPairs: 8) : progress + DispatchQueue.main.async { [weak self] in + guard let self else { return } + self.analysing = false + guard self.link != nil else { return } + if !done { + self.status.text = clicks == 0 && running > 3 ? "No click yet. Lay the calibrator on the red outline, sensors over the line, and check the cable." + : "Measuring · \(pairs) of \(Self.wantedPairs) swaps · frame skips \(skips)" + return + } + if (final["ok"] as? Bool) == true { + var result = final + result["method"] = "calibrator-display"; result["audioRoute"] = Self.outputRoute(); result["outputLatencyMs"] = latency * 1000 + result["skippedDisplayFrames"] = skips; result["sampleRate"] = rate + self.calibrationResult = result + self.closeScreen() + } else { + self.calibrationError = final["error"] as? String + self.stop(self.calibrationError ?? "Calibration failed. Tap Start to try again.") + } + } + } + } deinit { link?.invalidate(); for token in notices { NotificationCenter.default.removeObserver(token) } } } diff --git a/mobile-app/modules/avsync/ios/AvsyncModule.swift b/mobile-app/modules/avsync/ios/AvsyncModule.swift index c92a0b8..d8c0cd9 100644 --- a/mobile-app/modules/avsync/ios/AvsyncModule.swift +++ b/mobile-app/modules/avsync/ios/AvsyncModule.swift @@ -95,7 +95,9 @@ public class AvsyncModule: Module { } do { let image = try GeneratorImageStore.load(id: options["imageId"] ?? "") - let controller = AvsyncGeneratorController(rate: rate, taiUtc: taiUtc, name: label("name", limit: 64), subtitle: label("subtitle", limit: 128), image: image, clockReference: options.filter { $0.key.hasPrefix("clock") }, keepScreenAwake: options["keepAwake"] != "false") + // mode "calibration": the calibration pattern for the calibrator board instead of the test signal. + let controller = AvsyncGeneratorController(rate: rate, taiUtc: taiUtc, name: label("name", limit: 64), subtitle: label("subtitle", limit: 128), image: image, clockReference: options.filter { $0.key.hasPrefix("clock") }, keepScreenAwake: options["keepAwake"] != "false", + senderCalibration: options.filter { $0.key.hasPrefix("senderCalibration") }, calibrate: options["mode"] == "calibration") generator = controller; controller.onClose = { result in promise.resolve(result) } parent.view.endEditing(true) parent.present(controller, animated: true) diff --git a/mobile-app/tests/native/generator-runner.mm b/mobile-app/tests/native/generator-runner.mm index 1dddebc..65333fd 100644 --- a/mobile-app/tests/native/generator-runner.mm +++ b/mobile-app/tests/native/generator-runner.mm @@ -142,5 +142,29 @@ int main(int argc, char **argv) { @autoreleasepool { assert(packets[i].video_send_ns==quantize_timecode_ns(1700000037000000000LL+i*1000000000LL/rate)); } } + // Sender calibration in the QR, and the calibration pattern helpers. + { + AVSyncGenerator *source=[[AVSyncGenerator alloc] initWithTestID:@"phone-native-test" rate:2 epochTaiNs:1700000037000000000LL epochHost:100]; + [source setSenderCalibrationMethod:@"transponder" offsetMs:-38.04 uncertaintyMs:1.0 day:2503]; + CVPixelBufferRef pixel=nullptr; + assert(CVPixelBufferCreate(kCFAllocatorDefault,width,height,kCVPixelFormatType_32BGRA,(__bridge CFDictionaryRef)@{(id)kCVPixelBufferIOSurfacePropertiesKey:@{}},&pixel)==kCVReturnSuccess); + CVPixelBufferLockBaseAddress(pixel,0); + CGColorSpaceRef space=CGColorSpaceCreateDeviceRGB(); + CGContextRef context=CGBitmapContextCreate(CVPixelBufferGetBaseAddress(pixel),width,height,8,CVPixelBufferGetBytesPerRow(pixel),space,CGBitmapInfo(kCGBitmapByteOrder32Little)|CGBitmapInfo(kCGImageAlphaPremultipliedFirst)); + CGContextDrawImage(context,CGRectMake(0,0,width,height),[source imageForSequence:1].CGImage); + CVPixelBufferUnlockBaseAddress(pixel,0); + bool found=false; + for(NSString *payload in [AVSyncQRCodeReader readPixelBuffer:pixel][@"payloads"]) if(auto parsed=parse_qr_payload(payload.UTF8String)) { + assert(parsed->calibration.method=="transponder" && std::abs(parsed->calibration.offset_ms+38.0)<1e-9 && parsed->calibration.uncertainty_ms==1.0 && parsed->calibration.day==2503);found=true; + } + assert(found); + CGContextRelease(context);CGColorSpaceRelease(space);CVPixelBufferRelease(pixel); + [source setSenderCalibrationMethod:@"bogus" offsetMs:1 uncertaintyMs:1 day:1]; // ignored: method none + assert([AVSyncGenerator calibrationLeftWhiteAtFrame:29 fps:60] && ![AVSyncGenerator calibrationLeftWhiteAtFrame:30 fps:60]); + assert([AVSyncGenerator calibrationSwapFrame:2 fps:60]==60); + NSData *beep=[AVSyncGenerator calibrationBeepWithSampleRate:48000]; + assert(beep.length==192*sizeof(float)); + assert(![[AVSyncGenerator displayCalibrationForSamples:[NSMutableData dataWithLength:48000*sizeof(float)] sampleRate:48000 minimumPairs:8][@"ok"] boolValue]); + } std::cout<<"PASS: shared color renderer, DejaVu text, custom name/subtitle/image, native sync scale, QR geometry/flash onset and 1/2-Hz independent audio.\n"; } } diff --git a/tests/unit/test_calibrator.cpp b/tests/unit/test_calibrator.cpp index 3dd73c9..4c17e14 100644 --- a/tests/unit/test_calibrator.cpp +++ b/tests/unit/test_calibrator.cpp @@ -103,13 +103,23 @@ TEST_CASE("calibrator sounds: clicks and beeps are found and told apart") CHECK(find_sound_onsets(audio.data(), 100, kRate, 0).empty()); } -TEST_CASE("calibration pattern: halves swap every 500 ms, the beep starts at once") +TEST_CASE("calibration pattern: halves swap every 500 ms on the frame grid, the beep starts at once") { - CHECK(calibration_left_white(0)); - CHECK(calibration_left_white(499.9)); - CHECK(!calibration_left_white(500)); - CHECK(calibration_left_white(1000)); - CHECK(calibration_period(1250) == 2); + CHECK(calibration_left_white(0, 60, 1)); + CHECK(calibration_left_white(29, 60, 1)); + CHECK(!calibration_left_white(30, 60, 1)); + CHECK(calibration_left_white(60, 60, 1)); + CHECK(calibration_period(75, 60, 1) == 2); + CHECK(calibration_swap_frame(3, 60, 1) == 90); + CHECK(calibration_swap_frame(1, 25, 1) == 13); // 520 ms: the first frame at or after 500 ms + CHECK(calibration_period(12, 25, 1) == 0); + CHECK(calibration_period(13, 25, 1) == 1); + // 59.94 fps: every swap frame is the first of its period, and the period before it ends there. + for (uint64_t k = 1; k < 200; ++k) { + const auto f = calibration_swap_frame(k, 60000, 1001); + CHECK(calibration_period(f, 60000, 1001) == k); + CHECK(calibration_period(f - 1, 60000, 1001) == k - 1); + } const auto beep = calibration_beep(kRate); REQUIRE(beep.size() == 192); CHECK(beep[0] == 0); From 9ddbe93bde3b9e9224633e48308d3ab05ac2ddb9 Mon Sep 17 00:00:00 2001 From: Adrian Hilber Date: Sun, 8 Nov 2026 15:40:23 +0100 Subject: [PATCH 04/21] Container generator: AVSYNC_PATTERN=calibration Same feature as the phone: the generator can send the calibration pattern instead of the test signal, so a phone with the calibrator board can measure a TV/monitor chain. Left/right halves swap every 500 ms on the grain grid (also interlaced and 59.94), a 1.5 kHz beep 100 ms after each swap at the chirp level, a grey tick at the ends of the pattern line. Live key, also in the Signal API (pattern) and the web form (Picture). tests/tools/calibration_flow + tests/integration/calibration_pattern.py read both flows, check the timing and run a simulated calibrator through display_calibration (1080p50, 1080i25, 1080p59.94: 0.03 ms). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017ybuzgBaGxuB5UiaE8dRa4 --- CMakeLists.txt | 5 +- app/config.cpp | 7 +- app/config.hpp | 1 + mxlio/engine.cpp | 61 ++++++++++++- ops/service.cpp | 3 +- tests/integration/calibration_pattern.py | 94 ++++++++++++++++++++ tests/tools/calibration_flow.cpp | 107 +++++++++++++++++++++++ web/index.html | 14 +-- 8 files changed, 281 insertions(+), 11 deletions(-) create mode 100644 tests/integration/calibration_pattern.py create mode 100644 tests/tools/calibration_flow.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 5354b87..7fcd3c7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -183,8 +183,11 @@ if(FFMPEG_FOUND AND NOT AVSYNC_CORE_ONLY) add_executable(avsync-phone-fixture tests/tools/phone_fixture.cpp) target_link_libraries(avsync-phone-fixture PRIVATE avsync-mxlio) target_compile_options(avsync-phone-fixture PRIVATE -Wall -Wextra -Wno-unused-parameter) + add_executable(avsync-calibration-flow tests/tools/calibration_flow.cpp) + target_link_libraries(avsync-calibration-flow PRIVATE avsync-mxlio) + target_compile_options(avsync-calibration-flow PRIVATE -Wall -Wextra -Wno-unused-parameter) if(MXL_RUNTIME_LIB_DIRS) - set_target_properties(avsync-sample-flow avsync-pattern-writer avsync-phone-fixture PROPERTIES BUILD_RPATH "${MXL_RUNTIME_LIB_DIRS}") + set_target_properties(avsync-sample-flow avsync-pattern-writer avsync-phone-fixture avsync-calibration-flow PROPERTIES BUILD_RPATH "${MXL_RUNTIME_LIB_DIRS}") endif() message(STATUS "libmxl found: live reader and writer are linked into mxl-avsync") else() diff --git a/app/config.cpp b/app/config.cpp index 5f2c42e..89d0bd5 100644 --- a/app/config.cpp +++ b/app/config.cpp @@ -125,7 +125,7 @@ bool is_restart_key(const std::string& key) bool is_live_key(const std::string& key) { return key == "AVSYNC_AV_OFFSET_MS" || key == "AVSYNC_CHIRP_LEVEL_DBFS" || key == "AVSYNC_AUDIO_ENABLE" - || key == "AVSYNC_EVENT_RATE" || key == "AVSYNC_AUDIO_TIMECODE" + || key == "AVSYNC_EVENT_RATE" || key == "AVSYNC_AUDIO_TIMECODE" || key == "AVSYNC_PATTERN" || key == "AVSYNC_TEST_ID" || key == "AVSYNC_TITLE" || key == "AVSYNC_SOURCE" || key == "AVSYNC_SUBTITLE" || key == "AVSYNC_NOTES" || key == "AVSYNC_AUDIO_CHANNEL" || key == "AVSYNC_AUDIO_THRESHOLD" || key == "AVSYNC_QR_RATE" || key == "AVSYNC_PASS_MIN_MS" || key == "AVSYNC_PASS_MAX_MS" || key == "AVSYNC_WARN_MIN_MS" || key == "AVSYNC_WARN_MAX_MS" @@ -172,6 +172,7 @@ Config load_config(const std::map& extra) }; config.role = pick("AVSYNC_ROLE", ""); config.audio_timecode = pick("AVSYNC_AUDIO_TIMECODE", "1200"); + config.pattern = pick("AVSYNC_PATTERN", "test"); config.event_rate = event_rate_value(pick("AVSYNC_EVENT_RATE", "2")); config.chirp_ms = static_cast(num(pick("AVSYNC_CHIRP_MS", "30"), "AVSYNC_CHIRP_MS")); config.chirp_low_hz = num(pick("AVSYNC_CHIRP_LOW_HZ", "700"), "AVSYNC_CHIRP_LOW_HZ"); @@ -235,6 +236,7 @@ Config load_config(const std::map& extra) config.log_format = pick("LOG_FORMAT", "json"); config.shutdown_timeout_s = static_cast(num(pick("SHUTDOWN_TIMEOUT_S", "10"), "SHUTDOWN_TIMEOUT_S")); require(config.audio_timecode == "legacy" || config.audio_timecode == "600" || config.audio_timecode == "1200", "AVSYNC_AUDIO_TIMECODE must be legacy, 600 or 1200"); + require(config.pattern == "test" || config.pattern == "calibration", "AVSYNC_PATTERN must be test or calibration"); if (config.audio_timecode != "legacy") { require(config.event_rate == 1 || config.event_rate == 2, "timecode supports 1 or 2 events/s"); require(config.audio_timecode != "600" || config.event_rate == 1, "600 baud requires 1 event/s"); @@ -264,6 +266,7 @@ nlohmann::json config_settings(const Config& config) return { {"AVSYNC_ROLE", config.role}, {"AVSYNC_AUDIO_TIMECODE", config.audio_timecode}, + {"AVSYNC_PATTERN", config.pattern}, {"AVSYNC_EVENT_RATE", std::to_string(config.event_rate)}, {"AVSYNC_CHIRP_MS", std::to_string(config.chirp_ms)}, {"AVSYNC_CHIRP_LOW_HZ", std::to_string(config.chirp_low_hz)}, @@ -314,6 +317,7 @@ void apply_settings(Config& config, const nlohmann::json& flat, std::vector(num(value, key.c_str())); else if (key == "AVSYNC_CHIRP_LOW_HZ") next.chirp_low_hz = num(value, key.c_str()); @@ -351,6 +355,7 @@ void apply_settings(Config& config, const nlohmann::json& flat, std::vector= -500 && next.offset_ms <= 500, "AVSYNC_AV_OFFSET_MS"); require(next.audio_timecode == "legacy" || next.audio_timecode == "600" || next.audio_timecode == "1200", "AVSYNC_AUDIO_TIMECODE must be legacy, 600 or 1200"); + require(next.pattern == "test" || next.pattern == "calibration", "AVSYNC_PATTERN must be test or calibration"); if (next.audio_timecode != "legacy") { require(next.event_rate == 1 || next.event_rate == 2, "timecode supports 1 or 2 events/s"); require(next.audio_timecode != "600" || next.event_rate == 1, "600 baud requires 1 event/s"); diff --git a/app/config.hpp b/app/config.hpp index 184760b..0a6db1d 100644 --- a/app/config.hpp +++ b/app/config.hpp @@ -13,6 +13,7 @@ struct Config { std::string role; int event_rate = 2; std::string audio_timecode = "1200"; // legacy, 600, 1200 + std::string pattern = "test"; // generator output: test (the AV sync test signal) or calibration (calibrator board) int chirp_ms = 30; double chirp_low_hz = 700; double chirp_high_hz = 2500; diff --git a/mxlio/engine.cpp b/mxlio/engine.cpp index a2ca2f3..231bc0c 100644 --- a/mxlio/engine.cpp +++ b/mxlio/engine.cpp @@ -1,4 +1,5 @@ #include "avsync/picture_reader.hpp" +#include "avsync/calibrator.hpp" #include "avsync/optical_code.hpp" #include "engine.hpp" @@ -51,6 +52,49 @@ ChirpSpec chirp_from(const Config& config) return spec; } +// AVSYNC_PATTERN=calibration (docs/calibrator.md, display mode): the calibrator board lies on a +// screen showing this picture. Left half white or black, the right half the opposite, and a grey +// tick on the pattern line at the top and bottom edges. Two v210 grains, made once per size. +const std::vector& calibration_picture(int width, int rows, bool left_white) +{ + static thread_local std::vector packed[2]; + static thread_local int made_width = 0, made_rows = 0; + if (made_width != width || made_rows != rows) { + made_width = width, made_rows = rows; + std::vector luma(static_cast(width) * static_cast(rows)); + const int tick = std::max(2, width / 480), edge = std::max(1, rows / 16); + for (int i = 0; i < 2; ++i) { + for (int row = 0; row < rows; ++row) { + for (int x = 0; x < width; ++x) { + uint16_t v = ((x < width / 2) == (i == 0)) ? kYWhite : kYBlack; + if (std::abs(2 * x + 1 - width) < 2 * tick && (row < edge || row >= rows - edge)) v = 512; + luma[static_cast(row) * width + x] = v; + } + } + packed[i].assign(v210_line_bytes(width) * static_cast(rows), 0); + pack_v210(luma.data(), nullptr, nullptr, width, rows, width, packed[i].data()); + } + } + return packed[left_white ? 0 : 1]; +} + +// The calibration beep, D = 100 ms after each swap of the picture above (same grain grid), added +// to the samples [first, first + count) at the chirp level. +void calibration_audio(Rate pace, uint64_t first, float* mono, size_t count, double level_dbfs) +{ + static const std::vector beep = calibration_beep(48000); + const Rate audio{48000, 1}; + const float gain = static_cast(2 * std::pow(10.0, level_dbfs / 20)); // the beep peaks at 0.5 + const uint64_t start = time_of_index(audio, first), back = 200000000ULL; + for (uint64_t k = calibration_period(index_at(pace, start > back ? start - back : 0), pace.num, pace.den);; ++k) { + const uint64_t at = index_at(audio, time_of_index(pace, calibration_swap_frame(k, pace.num, pace.den)) + 100000000ULL); + if (at >= first + count) break; + for (size_t i = 0; i < beep.size(); ++i) { + if (at + i >= first && at + i < first + count) mono[at + i - first] += gain * beep[i]; + } + } +} + void run_generator(std::atomic& stop, LiveCallbacks& callbacks) { Config config = callbacks.copy_config(); @@ -173,7 +217,7 @@ void run_generator(std::atomic& stop, LiveCallbacks& callbacks) // onset - (lead + half a block): the published audio emit offset. The write before it stops at // the onset, and the thread wakes for the onset's own commit. std::optional onset; - if (!halted && audio_config.audio_enable) { + if (!halted && audio_config.audio_enable && audio_config.pattern != "calibration") { for (const auto& event : events_near(audio_config, audio_written)) { if (event.audio_sample >= 0 && static_cast(event.audio_sample) >= audio_written && (!onset || static_cast(event.audio_sample) < *onset)) { @@ -202,7 +246,15 @@ void run_generator(std::atomic& stop, LiveCallbacks& callbacks) if (target > audio_written && max_write > 0) { const int frames = static_cast(std::min(target - audio_written, 48000)); samples.assign(static_cast(frames * audio_config.audio_channels), 0.f); - if (!halted && audio_config.audio_enable) { + if (!halted && audio_config.audio_enable && audio_config.pattern == "calibration") { + std::vector mono(static_cast(frames), 0.f); + calibration_audio(pace, audio_written, mono.data(), mono.size(), audio_config.chirp_level_dbfs); + for (int frame = 0; frame < frames; ++frame) { + for (int channel = 0; channel < audio_config.audio_channels; ++channel) { + samples[static_cast(frame * audio_config.audio_channels + channel)] = mono[static_cast(frame)]; + } + } + } else if (!halted && audio_config.audio_enable) { PlanConfig plan = plan_from(audio_config, *format); const auto events = events_near(audio_config, audio_written); std::vector mono(static_cast(frames)); @@ -266,7 +318,10 @@ void run_generator(std::atomic& stop, LiveCallbacks& callbacks) if (slot.skipped > 0 && callbacks.count_missed) { callbacks.count_missed(true, slot.skipped); // grains never written } - if (!halted) { + if (!halted && config.pattern == "calibration") { + packed = &calibration_picture(format->width, grain_rows, calibration_left_white(index, pace.num, pace.den)); + qr_seq = ~0ULL; // the test picture starts fresh afterwards + } else if (!halted) { const auto render_start = std::chrono::steady_clock::now(); const Period period = plan_period(*format, config.event_rate); const uint64_t seq = index / static_cast(std::max(1, period.grains)); diff --git a/ops/service.cpp b/ops/service.cpp index 9ef05f3..ce0eae6 100644 --- a/ops/service.cpp +++ b/ops/service.cpp @@ -587,7 +587,7 @@ void install_web(HttpServer& http, Runtime& rt) {"stopped", rt.stopped}, {"title", rt.config.title}, {"source", rt.config.source}, {"subtitle", rt.config.subtitle}, {"notes", rt.config.notes}, {"test_id", rt.config.effective_test_id()}, {"timing_reference", rt.config.audio_timecode == "legacy" ? "centre" : "start"}, {"marker", rt.config.audio_timecode == "legacy" ? "toggle" : "flash"}, {"audio_timecode", rt.config.audio_timecode}, {"event_rate", rt.config.event_rate}, {"format", rt.config.video_format}, - {"source_image", !rt.source_image_path.empty()}, {"source_image_frames", rt.source_image_frames}}); + {"pattern", rt.config.pattern}, {"source_image", !rt.source_image_path.empty()}, {"source_image_frames", rt.source_image_frames}}); }); http.add("PATCH", "/api/v1/generator", [&](const HttpRequest& request) { std::lock_guard lock(rt.mu); @@ -612,6 +612,7 @@ void install_web(HttpServer& http, Runtime& rt) if (body.contains("muted")) flat["AVSYNC_AUDIO_ENABLE"] = body["muted"].get() ? "false" : "true"; if (body.contains("audio_timecode")) flat["AVSYNC_AUDIO_TIMECODE"] = body["audio_timecode"].get(); if (body.contains("event_rate")) flat["AVSYNC_EVENT_RATE"] = std::to_string(body["event_rate"].get()); + if (body.contains("pattern")) flat["AVSYNC_PATTERN"] = body["pattern"].get(); try { apply_settings(rt.config, flat, nullptr); } catch (const std::exception& ex) { diff --git a/tests/integration/calibration_pattern.py b/tests/integration/calibration_pattern.py new file mode 100644 index 0000000..34de147 --- /dev/null +++ b/tests/integration/calibration_pattern.py @@ -0,0 +1,94 @@ +"""Container generator with AVSYNC_PATTERN=calibration (docs/calibrator.md, display mode). + +Starts mxl-avsync as a generator on a tmpfs MXL domain, checks the calibration pattern with +avsync-calibration-flow (halves swap every 500 ms, beep 100 ms after each swap, a simulated +calibrator gives 0 ms), switches the pattern live through the Signal API and back. +Needs the MXL build. Set MXL_LIB_DIR/VCPKG_LIB_DIR when the loader needs them. +""" +import argparse +import json +import os +from pathlib import Path +import socket +import subprocess +import tempfile +import time +import urllib.request + +ROOT = Path(__file__).resolve().parents[2] + + +def free_port(): + with socket.socket() as listener: + listener.bind(('127.0.0.1', 0)) + return listener.getsockname()[1] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--binary', default=str(ROOT / 'build/mxl-avsync')) + parser.add_argument('--tool', default=None) + parser.add_argument('--format', default='1080p50') + args = parser.parse_args() + tool = args.tool or str(Path(args.binary).resolve().parent / 'avsync-calibration-flow') + env = {k: v for k, v in os.environ.items() if not k.startswith(('AVSYNC_', 'MXL_', 'NMOS_')) and k != 'WEB_PORT'} + libraries = [os.environ.get(name, '') for name in ('MXL_LIB_DIR', 'VCPKG_LIB_DIR', 'LD_LIBRARY_PATH')] + if any(libraries): + env['LD_LIBRARY_PATH'] = ':'.join(filter(None, libraries)) + port = free_port() + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + + def call(method, path, body=None): + data = json.dumps(body).encode() if body is not None else None + request = urllib.request.Request(f'http://127.0.0.1:{port}{path}', data=data, method=method, headers={'Content-Type': 'application/json'}) + with opener.open(request, timeout=3) as response: + return json.load(response) + + with tempfile.TemporaryDirectory(prefix='avsync-cal-', dir='/dev/shm') as directory: + work = Path(directory) + env.update({'AVSYNC_ROLE': 'generator', 'AVSYNC_PATTERN': 'calibration', 'AVSYNC_VIDEO_FORMAT': args.format, + 'NMOS_ENABLE': 'false', 'NMOS_SEED': 'calibration-pattern-test', 'MXL_DOMAIN_SCAN_PATH': str(work), + 'MXL_OUTPUT_DOMAIN_DIR': str(work / 'gen'), 'AVSYNC_STATE_DIR': str(work / 'state'), 'WEB_PORT': str(port), + 'AVSYNC_CLOCK_MAX_ERROR_US': '5000', 'AVSYNC_RESULT_ENABLE': 'false'}) + (work / 'state').mkdir() + log = (work / 'generator.log').open('w') + generator = subprocess.Popen([args.binary, 'serve'], env=env, stdout=log, stderr=subprocess.STDOUT) + try: + deadline = time.monotonic() + 20 + status = {} + while time.monotonic() < deadline: + try: + status = call('GET', '/api/v1/status') + if status.get('media_ready') and status.get('video_flow'): + break + except OSError: + pass + time.sleep(.2) + assert status.get('video_flow'), f'generator not ready: {status}' + assert call('GET', '/api/v1/generator')['pattern'] == 'calibration' + time.sleep(1) + check = subprocess.run([tool, str(work / 'gen'), status['video_flow'], status['audio_flow'], '5'], env=env, + capture_output=True, text=True, timeout=30) + print(check.stdout.strip() or check.stderr.strip()) + assert check.returncode == 0, check.stderr + # Live switch to the test signal: the tool must now fail (no swaps of the halves). + assert call('PATCH', '/api/v1/generator', {'pattern': 'test'})['ok'] + assert call('GET', '/api/v1/generator')['pattern'] == 'test' + time.sleep(.5) + other = subprocess.run([tool, str(work / 'gen'), status['video_flow'], status['audio_flow'], '2'], env=env, + capture_output=True, text=True, timeout=30) + assert other.returncode != 0, 'test signal must not pass as the calibration pattern' + try: + call('PATCH', '/api/v1/generator', {'pattern': 'bogus'}) + raise AssertionError('bogus pattern accepted') + except urllib.error.HTTPError as error: + assert error.code == 400 + print(f'PASS calibration pattern ({args.format})') + finally: + generator.terminate() + generator.wait(timeout=15) + log.close() + + +if __name__ == '__main__': + main() diff --git a/tests/tools/calibration_flow.cpp b/tests/tools/calibration_flow.cpp new file mode 100644 index 0000000..d2011d0 --- /dev/null +++ b/tests/tools/calibration_flow.cpp @@ -0,0 +1,107 @@ +#include "avsync/calibrator.hpp" +#include "domain.hpp" + +#include +#include +#include +#include +#include +#include + +// Reads a generator's video and audio flows with AVSYNC_PATTERN=calibration for some seconds and +// checks the calibration pattern: the halves swap every 500 ms, each beep starts 100 ms after its +// swap, and a simulated calibrator (a click at each swap) gives a display calibration of 0 ms. +// Prints one JSON line; exit 0 when all checks pass. +int main(int argc, char** argv) +{ + using namespace avsync; + if (argc < 4) { + std::cerr << "usage: avsync-calibration-flow [seconds]\n"; + return 78; + } + const double seconds = argc > 4 ? std::stod(argv[4]) : 5; + Domain domain; + std::string error; + FlowDesc desc; + Rate pace{}; + int rows = 0, channels = 0; + uint32_t grains = 0; + size_t max_read = 0; + if (!domain.open(argv[1], &error) || !domain.create_video_reader(argv[2], &desc, &pace, &rows, &grains, &error) + || !domain.create_audio_reader(argv[3], &channels, &max_read, &error)) { + std::cerr << error << "\n"; + return 1; + } + const Rate audio{48000, 1}; + uint64_t index = domain.video_head(); + const uint64_t first = index, last = index + uint64_t(seconds * double(pace.num) / double(pace.den)); + const double origin = double(time_of_index(pace, first)) / 1e6; + uint64_t sample = index_at(audio, time_of_index(pace, first)); + const uint64_t first_sample = sample; + std::vector mono, block; + std::vector swaps; + int previous = -1, missing = 0; + std::vector payload; + std::vector luma; + for (; index < last; ++index) { + bool invalid = false; + if (!domain.copy_grain(index, 500000000, rows, desc.width, grains, &payload, &invalid)) { + ++missing; + continue; + } + luma.assign(size_t(desc.width) * size_t(rows), 0); + unpack_v210_luma(payload.data(), desc.width, rows, luma.data(), desc.width); + const size_t middle = size_t(rows / 2) * size_t(desc.width); + const int state = luma[middle + size_t(desc.width / 4)] > 500 ? 1 : 0; + if ((luma[middle + size_t(3 * desc.width / 4)] > 500) == bool(state)) { + std::cerr << "halves are not opposite at grain " << index << "\n"; + return 1; + } + if (previous >= 0 && state != previous) swaps.push_back(double(time_of_index(pace, index)) / 1e6 - origin); + previous = state; + // Audio up to this grain's time (the writer commits it ahead, like the video). + const uint64_t until = index_at(audio, time_of_index(pace, index)); + if (until > sample) { + const int frames = int(until - sample); + if (!domain.copy_samples(until, frames, channels, 500000000, max_read, &block, &invalid)) { + std::cerr << "audio not readable\n"; + return 1; + } + for (int i = 0; i < frames; ++i) mono.push_back(block[size_t(i * channels)]); + sample = until; + } + } + const double audio_origin = double(time_of_index(audio, first_sample)) / 1e6 - origin; + const auto beeps = find_sound_onsets(mono.data(), mono.size(), 48000, audio_origin); + // Each beep 100 ms after the swap before it. + double worst = 0; + size_t matched = 0; + for (const auto& beep : beeps) { + const auto swap = std::upper_bound(swaps.begin(), swaps.end(), beep.time_ms); + if (swap == swaps.begin()) continue; + worst = std::max(worst, std::abs(beep.time_ms - *(swap - 1) - CalibratorContract::sound_delay_ms)); + ++matched; + } + double period = 0; + for (size_t i = 1; i < swaps.size(); ++i) period = std::max(period, std::abs(swaps[i] - swaps[i - 1] - CalibratorContract::swap_period_ms)); + // A calibrator with an instant screen: a click at every swap, through the same analysis as the phone. + auto mic = mono; + for (double swap : swaps) { + const long at = std::lround((swap - audio_origin) * 48); + for (long i = 0; i < 240 && at + i < long(mic.size()); ++i) { + if (at + i < 0) continue; + const double t = double(i) / 48000, on = 3 / CalibratorContract::click_hz; + mic[size_t(at + i)] += float(.3 * (t < on ? 1 : std::exp(-(t - on) / 0.0006)) * std::sin(2 * 3.14159265358979 * CalibratorContract::click_hz * t)); + } + } + DisplayCalibrationOptions options; + options.min_pairs = 4; + const auto r = display_calibration(find_sound_onsets(mic.data(), mic.size(), 48000, audio_origin), options); + // Grain timing: an interlaced field grid or 59.94 fps moves a swap by up to one grain. + const double grain_ms = 1000.0 * double(pace.den) / double(pace.num); + const bool ok = swaps.size() >= size_t(seconds * 2) - 2 && matched + 2 >= swaps.size() && worst < .15 && period <= grain_ms + .01 && r.ok + && std::abs(r.offset_ms) < .15 && missing == 0; + std::printf("{\"ok\":%s,\"swaps\":%zu,\"beeps\":%zu,\"matched\":%zu,\"worstBeepErrorMs\":%.3f,\"worstPeriodErrorMs\":%.3f,\"offsetMs\":%.3f,\"pairs\":%d,\"missingGrains\":%d}\n", + ok ? "true" : "false", swaps.size(), beeps.size(), matched, worst, period, r.offset_ms, r.pairs, missing); + return ok ? 0 : 1; +} diff --git a/web/index.html b/web/index.html index aba4613..9cd538f 100644 --- a/web/index.html +++ b/web/index.html @@ -156,6 +156,10 @@

Preview

v-memo="[generatorBaseline, signalDirty, signalSaving, signalError, imageFile, imageSet, imageFrames, imageUrl, audioOn, stopped]">

Unsaved changes

+ +

Halves swap every 500 ms with a beep 100 ms later. Lay the calibrator on the screen's centre line and use the phone app (Settings → Calibration → With calibrator).