Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,13 @@ jobs:
# the comment above says the hooks were verified by hand once, which is
# exactly the kind of check that goes stale without a gate. --check-deps
# loads OpenCV and onnxruntime and builds a real SessionOptions, needing
# no video and no ~110 MB weights download.
# no video and no ~110 MB weights download. --skip-ffmpeg because the
# release runner is not a render machine (ffmpeg is deliberately not
# bundled); a bare --check-deps now requires it, which is the point on a
# user's machine and noise here.
run: |
dist\redactcam.exe --help
dist\redactcam.exe --check-deps
dist\redactcam.exe --check-deps --skip-ffmpeg

- name: Install Inno Setup
# Installed, not assumed: a runner image that quietly drops a
Expand Down
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,16 @@ All notable changes to this project are documented here. The format follows
- **Stale-install detection.** `redactcam --identity`, `--expect-revision`,
`--expect-code-sha256`, `--expect-version` and `redactcam.check_current()`;
an install whose identity cannot be determined never passes.
- **`--check-deps` checks ffmpeg/ffprobe and supplied models.** A missing or broken
ffmpeg exits 1 with the install line (`--skip-ffmpeg` opts out; the Windows
release smoke test uses it). With `--model KIND=PATH` each file is hashed,
loaded and run on a blank frame.
- **Clean-machine qualification.** `redactcam --check-deps --receipt FILE` with all
four `--model` files renders a synthetic clip end to end and writes a versioned
receipt; exits 0 only if qualified. Never downloads weights.
- `redact_video()` now raises `FfmpegUnavailableError` (with install instructions)
before detection when ffmpeg or ffprobe is unusable, instead of failing at the
mask stage after the whole detection pass.

### Changed

Expand Down
34 changes: 32 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -252,9 +252,39 @@ session options intra=1 inter=1
inference CPUExecutionProvider
```

It also runs `ffmpeg -version` and `ffprobe -version`, and exits 1 with the
per-platform install line when either is missing or will not run (`redactcam` also
checks this at the top of every render, before detection starts, rather than
hours later at the mask stage). `--skip-ffmpeg` drops that check for a build smoke
test on a runner with no ffmpeg; it is not valid with `--receipt`.

It exits non-zero and names the module if one fails to load, exits 1 with the
repair when the GPU build is installed but CUDA is unavailable, and needs no model
weights, so it will not trigger the first-run download.
weights, so it will not trigger the first-run download. Add `--model KIND=PATH`
and it also hashes each supplied file, builds its detector and runs it on one
blank frame, so a file that is not a YOLO ONNX model fails here with the file and
the flag named, not on the first real clip.

### Qualify a clean machine

A successful download proves nothing about whether the machine can redact. On a
disposable install of the OS you intend to support, with the model files you have
qualified (nothing is downloaded, and the run refuses to start without all four):

```bash
redactcam --check-deps --receipt clean-machine.json \
--model face=/models/face.onnx --model plate=/models/plate.onnx \
--model vehicle=/models/coco.onnx --model person=/models/coco.onnx
```

It runs every check above, then one synthetic 2 s clip (ffmpeg `testsrc`, no real
footage) through the whole pipeline to an encoded H.264 file, and writes a
versioned receipt (`redactcam.clean-machine-receipt/v1`): launch details, the
installed identity, the diagnostics, each detector's status and file hash, and the
render's output hash. `"qualified": true` only when the diagnostics, all four
detectors and the render passed; every gap writes a receipt with `false` and
exits 1. Which OS and which model files count as qualified is your decision: the
receipt records what ran, it does not judge it.

### Which build is installed

Expand Down Expand Up @@ -440,7 +470,7 @@ Everything below is importable straight from `redactcam`.
`file_hash()`.

**Provenance** — `runtime_identity()`, `model_identity()`, `check_current()`,
`StaleInstallError`.
`StaleInstallError`, `require_ffmpeg()`, `FfmpegUnavailableError`.

**Detection** — `detect_and_track()` (the dense per-frame workhorse),
`detect_regions()` (sparse, no tracking), `detect_image()` (one still),
Expand Down
10 changes: 9 additions & 1 deletion redactcam/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,13 @@
``pipeline.redact_video`` wires all five together for the common case.
"""

from .apply import apply_blur, blur_filter_complex, blur_radius
from .apply import (
FfmpegUnavailableError,
apply_blur,
blur_filter_complex,
blur_radius,
require_ffmpeg,
)
from .coverage import (
CLASSES,
ClassCoverage,
Expand Down Expand Up @@ -127,6 +133,8 @@
"blur_radius",
"blur_filter_complex",
"apply_blur",
"require_ffmpeg",
"FfmpegUnavailableError",
# coverage
"CoverageReport",
"ClassCoverage",
Expand Down
23 changes: 23 additions & 0 deletions redactcam/apply.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,20 @@

logger = logging.getLogger(__name__)

FFMPEG_TOOLS = ("ffmpeg", "ffprobe")
FFMPEG_INSTALL_HINT = (
"Install FFmpeg and make sure both ffmpeg and ffprobe are on PATH "
"(Windows: `winget install -e --id Gyan.FFmpeg`; macOS: `brew install ffmpeg`; "
"Debian/Ubuntu: `apt install ffmpeg` as root), then open a new terminal."
)


class FfmpegUnavailableError(RuntimeError):
"""``ffmpeg`` or ``ffprobe`` is missing or does not run. Raised before any
detection starts: the mask render is the first ffmpeg call, so without this
a missing binary would surface only after the whole detection pass, hours
into a long video, as a bare ``FileNotFoundError``."""


def tool_status(name: str) -> dict:
"""Where ``name`` resolves on PATH and the first line of ``-version``.
Expand All @@ -50,6 +64,15 @@ def tool_status(name: str) -> dict:
return {"name": name, "path": path, "version": lines[0][:200], "error": None}


def require_ffmpeg() -> dict[str, dict]:
"""Confirm ``ffmpeg`` and ``ffprobe`` both run; raise ``FfmpegUnavailableError``
with the install instructions if not. Returns each tool's ``tool_status``."""
status = {name: tool_status(name) for name in FFMPEG_TOOLS}
problems = [s["error"] for s in status.values() if s["error"]]
if problems:
raise FfmpegUnavailableError("; ".join(problems) + ". " + FFMPEG_INSTALL_HINT)
return status

# boxblur luma radius at 1920 px wide. Scaled by actual width in blur_radius().
# 12 at 1080p reduces a face to an unrecognisable smear while leaving the scene
# legible; well below this and features start to survive at full-screen playback.
Expand Down
178 changes: 139 additions & 39 deletions redactcam/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@
import json
import logging
import sys
import tempfile
from pathlib import Path

from . import provenance
from . import provenance, qualify
from .apply import FFMPEG_INSTALL_HINT, FFMPEG_TOOLS, tool_status
from .coverage import CLASSES, RequiredClassError
from .models import DEFAULT_MODELS, ModelSpec
from .pipeline import CoverageError, redact_video
Expand Down Expand Up @@ -76,7 +78,26 @@ def _parser() -> argparse.ArgumentParser:
p.add_argument(
"--check-deps",
action="store_true",
help="load the native extensions, report their versions, and exit; takes no input",
help=(
"load the native extensions, check ffmpeg/ffprobe, verify any --model files "
"load, report versions, and exit; takes no input"
),
)
p.add_argument(
"--skip-ffmpeg",
action="store_true",
help="with --check-deps: do not require ffmpeg/ffprobe (for a build smoke test "
"on a runner that has none; not valid with --receipt)",
)
p.add_argument(
"--receipt",
type=Path,
metavar="FILE",
help=(
"with --check-deps: qualify this machine and write a versioned clean-machine "
"receipt. Needs all four --model KIND=PATH files (nothing is downloaded); runs "
"one synthetic clip through the whole pipeline; exits 0 only if qualified"
),
)
p.add_argument(
"--identity",
Expand Down Expand Up @@ -112,58 +133,116 @@ def _check_inference_providers_for_check() -> list[str]:
return check_inference_providers()


def _check_deps() -> int:
def _ffmpeg_tools_for_check() -> dict[str, dict]:
"""Indirection so the ffmpeg check can be driven in a test on any machine."""
return {name: tool_status(name) for name in FFMPEG_TOOLS}


def _check_deps(
models: dict[str, ModelSpec] | None = None,
*,
receipt: Path | None = None,
skip_ffmpeg: bool = False,
work_dir: Path | None = None,
) -> int:
"""Load the native extensions, use them, and say what loaded.

This is for the frozen Windows build. PyInstaller resolves OpenCV and
onnxruntime through hooks, and a miss surfaces only when someone runs a real
detection -- while the release smoke test was ``--help``, which never touches
onnxruntime, because detect.py imports it inside a function. The installer
could therefore ship an executable that could not detect anything with every
gate green.
This is for the frozen Windows build and for a first run on a clean machine.
PyInstaller resolves OpenCV and onnxruntime through hooks, and a miss surfaces
only when someone runs a real detection -- while the release smoke test was
``--help``, which never touches onnxruntime, because detect.py imports it
inside a function. The installer could therefore ship an executable that
could not detect anything with every gate green.

Importing alone would not prove much, so this builds a real SessionOptions
through the same helper the detectors call. It needs no video and no model
file on purpose: the weights are a ~110 MB download and a release runner has
neither.
through the same helper the detectors call. With no ``--model`` it needs no
video and no model file: the weights are a ~110 MB download and a release
runner has neither. ffmpeg and ffprobe are checked because the mask render
and the blur composite shell out to them; ``--model`` files are loaded and run
once, never downloaded.

With ``receipt`` it is a qualification: every check runs even after one
fails (so the receipt says everything that is wrong, not the first thing),
all four models are required, a synthetic clip is rendered, and the receipt
is written whatever happened. Exit 0 only if it qualified.
"""
from .detect import GpuProviderUnavailableError, _import_onnxruntime

models = models or {}
diag: dict = {"errors": []}
errors: list[str] = diag["errors"]

def fail(msg: str) -> None:
print(f"redactcam: {msg}", file=sys.stderr)
errors.append(msg)

try:
import cv2

opts = _session_options_for_check(1)
ort = _import_onnxruntime()
except ImportError as exc:
print(f"redactcam: a native dependency did not load: {exc}", file=sys.stderr)
return 1
fail(f"a native dependency did not load: {exc}")
else:
providers = list(ort.get_available_providers())
print(f"opencv {cv2.__version__}")
print(f"onnxruntime {ort.__version__}")
print(f"providers {', '.join(providers)}")
print(f"session options intra={opts.intra_op_num_threads} inter={opts.inter_op_num_threads}")
diag.update(opencv=cv2.__version__, onnxruntime=ort.__version__, providers=providers)
# The provider LIST above is what the wheel advertises; this line is the
# decision a detector makes from it — and it is where a GPU build that lost
# CUDA fails, instead of exiting 0 and running on CPU for a day.
try:
inference = _check_inference_providers_for_check()
except GpuProviderUnavailableError as exc:
fail(str(exc))
else:
print(f"inference {inference[0]}")
diag["inference"] = inference[0]

print(f"opencv {cv2.__version__}")
print(f"onnxruntime {ort.__version__}")
print(f"providers {', '.join(ort.get_available_providers())}")
print(f"session options intra={opts.intra_op_num_threads} inter={opts.inter_op_num_threads}")
# The provider LIST above is what the wheel advertises; this line is the
# decision a detector makes from it — and it is where a GPU build that lost
# CUDA fails, instead of exiting 0 and running on CPU for a day.
try:
inference = _check_inference_providers_for_check()
except GpuProviderUnavailableError as exc:
print(f"redactcam: {exc}", file=sys.stderr)
return 1
print(f"inference {inference[0]}")
return 0
if skip_ffmpeg:
print("ffmpeg skipped (--skip-ffmpeg)")
else:
tools = _ffmpeg_tools_for_check()
broken = False
for name, status in tools.items():
diag[name] = status
if status["error"]:
fail(status["error"])
broken = True
else:
print(f"{name:<15} {status['version']}")
if broken:
print(f"redactcam: {FFMPEG_INSTALL_HINT}", file=sys.stderr)

qualifying = receipt is not None
model_results: dict[str, dict] = {}
if models or qualifying:
model_results = qualify.check_models(models, require_all=qualifying)
for kind, m in model_results.items():
if m["status"] == "ok":
print(f"model {kind:<9} ok {m['sha256'][:12]} {m['provider']}")
else:
fail(m["error"])

def _models(pairs: list[str]) -> dict[str, ModelSpec]:
out: dict[str, ModelSpec] = {}
for pair in pairs:
kind, _, path = pair.partition("=")
if not path:
raise SystemExit(f"--model expects KIND=PATH, got {pair!r}")
if kind not in DEFAULT_MODELS:
raise SystemExit(f"unknown model kind {kind!r}; expected one of {sorted(DEFAULT_MODELS)}")
out[kind] = ModelSpec(name=kind, path=path)
return out
if not qualifying:
return 1 if errors else 0

render = {"attempted": False, "ok": False, "error": "skipped: an earlier check failed"}
if not errors:
with tempfile.TemporaryDirectory(prefix="redactcam-qualify-") as tmp:
render = qualify.synthetic_render(models, work_dir or Path(tmp))
if render.get("ok"):
print(f"render ok {render['frames']} frames {render['output_sha256'][:12]}")
else:
fail(f"synthetic render failed: {render.get('error')}")
result = qualify.build_clean_machine_receipt(
diagnostics=diag, models=model_results, render=render
)
provenance.write_receipt(receipt, result)
print(f"receipt {receipt} qualified={str(result['qualified']).lower()}")
return 0 if result["qualified"] else 1


def _check_expectations(args: argparse.Namespace) -> int:
Expand All @@ -181,6 +260,18 @@ def _check_expectations(args: argparse.Namespace) -> int:
return 0


def _models(pairs: list[str]) -> dict[str, ModelSpec]:
out: dict[str, ModelSpec] = {}
for pair in pairs:
kind, _, path = pair.partition("=")
if not path:
raise SystemExit(f"--model expects KIND=PATH, got {pair!r}")
if kind not in DEFAULT_MODELS:
raise SystemExit(f"unknown model kind {kind!r}; expected one of {sorted(DEFAULT_MODELS)}")
out[kind] = ModelSpec(name=kind, path=path)
return out


def _print_coverage(report) -> None:
"""One line per privacy class. UNCHECKED is printed as loudly as a failure:
it means unknown, and "coverage OK" must never stand in for it."""
Expand All @@ -201,11 +292,20 @@ def main(argv: list[str] | None = None) -> int:
level=logging.INFO if args.verbose else logging.WARNING,
format="%(levelname)s %(name)s: %(message)s",
)
if (args.receipt or args.skip_ffmpeg) and not args.check_deps:
parser.error("--receipt and --skip-ffmpeg only apply together with --check-deps")
if args.receipt and args.skip_ffmpeg:
parser.error("--receipt qualifies a machine to render, so it cannot skip ffmpeg")
expecting = args.expect_version or args.expect_code_sha256 or args.expect_revision
if args.check_deps or expecting or args.identity:
rc = 0
if args.check_deps:
rc = _check_deps()
rc = _check_deps(
_models(args.model),
receipt=args.receipt,
skip_ffmpeg=args.skip_ffmpeg,
work_dir=args.work_dir,
)
if expecting:
rc = max(rc, _check_expectations(args))
if args.identity:
Expand Down
Loading
Loading