From 944bf155fa795e2bb2e6d307406ae5f440ce784d Mon Sep 17 00:00:00 2001 From: Joe Petrucelli Date: Tue, 10 Nov 2026 10:23:11 -0500 Subject: [PATCH] feat(photos): star trail per night session in photos sort --- changelog.d/photo-sort-star-trails.md | 3 + docs/pipeline/178-photo-sort-groups.md | 7 +- gomoveshift/pipeline/photos/sort_groups.py | 103 ++++++++++++++++++--- tests/test_photo_sort_groups.py | 97 ++++++++++++++++++- 4 files changed, 191 insertions(+), 19 deletions(-) create mode 100644 changelog.d/photo-sort-star-trails.md diff --git a/changelog.d/photo-sort-star-trails.md b/changelog.d/photo-sort-star-trails.md new file mode 100644 index 0000000..84be80c --- /dev/null +++ b/changelog.d/photo-sort-star-trails.md @@ -0,0 +1,3 @@ +### Added + +- `photos sort` now makes one star-trail picture for every night session of 24 or more usable frames, into `trails/`, shown on the session's card between the stacks and the time-lapse. The session's frames are sorted into lights and lens-cap darks once for both outputs, and a resumed run builds only the trails that are missing. diff --git a/docs/pipeline/178-photo-sort-groups.md b/docs/pipeline/178-photo-sort-groups.md index 78b7f85..ef5f4b8 100644 --- a/docs/pipeline/178-photo-sort-groups.md +++ b/docs/pipeline/178-photo-sort-groups.md @@ -21,12 +21,13 @@ After the gallery is complete (so the page can be opened while this runs), one g 2. Panoramas to `pano/-.jpg` (`panorama.stitch_panorama`), stitched from the developed JPEGs this run wrote, never the RAW sources: the stitcher reads with `cv2.imread`, which cannot open a RAW. 3. Night stacks to `stacks/.tif` (the master) and `stacks/.jpg` (the preview), highest mean total first. The call sequence mirrors `photos astro`: the RAM and disk gate, `prepare_frames` with the run's `--workers`, then `stack_run`. No calibration frames. Stacking does not use the GPU. -4. Time-lapses to `timelapse/.mp4` with a same-stem poster `timelapse/.jpg` (the video's middle frame, at most 1080 px tall). One per night session, not per chunk: chunks join into a session while each starts within `astro_sequence.MAX_FRAME_GAP_S` of the previous chunk's last frame, so a session is one `detect_sequences` run, found again from `ranked.csv` alone. The session is named for its first chunk (`night-1`). Lens-cap darks are told from lights with the stack's own triage verdict (`exposure_class == "cap_on"`, #289) and are never rendered as lights; they go to `render_timelapse` as dark calibration when `build_validated_master_dark` accepts them against the lights, else they are dropped. Rendered by `astro_timelapse.render_timelapse` at the night product's fps and 2160 px height ceiling. A burst of `TIMELAPSE_MIN_FRAMES` (24) or more renders its developed JPEGs, in capture order, through `timelapse.render_stills_timelapse`. A session with fewer than 24 usable lights gets no time-lapse and is remembered, so `--resume` does not decode it again. The output kind is `timelapse` and the page matches it to its card by `group_id`. +4. Star trails to `trails/.tif` (the master) and `trails/.jpg` (the browser JPEG, `night_products.render_master_preview`), one per night session (sessions as in step 5). Rendered by `astro_trails.render_trail`: calibrated, unaligned, per-pixel maximum, so the sky's rotation draws arcs. The session's light/dark triage runs once and is shared with its time-lapse; darks calibrate only when `build_validated_master_dark` accepts them, since a maximum keeps an unsubtracted hot pixel forever. A session with fewer than `TRAILS_MIN_FRAMES` (24) usable lights, about 13 minutes at the measured 32 s interval, gets no trail and is remembered (`trails-skip:v1`). The output kind is `trails`; on the session's card it sits after the stacks and before the time-lapse. +5. Time-lapses to `timelapse/.mp4` with a same-stem poster `timelapse/.jpg` (the video's middle frame, at most 1080 px tall). One per night session, not per chunk: chunks join into a session while each starts within `astro_sequence.MAX_FRAME_GAP_S` of the previous chunk's last frame, so a session is one `detect_sequences` run, found again from `ranked.csv` alone. The session is named for its first chunk (`night-1`). Lens-cap darks are told from lights with the stack's own triage verdict (`exposure_class == "cap_on"`, #289) and are never rendered as lights; they go to `render_timelapse` as dark calibration when `build_validated_master_dark` accepts them against the lights, else they are dropped. Rendered by `astro_timelapse.render_timelapse` at the night product's fps and 2160 px height ceiling. A burst of `TIMELAPSE_MIN_FRAMES` (24) or more renders its developed JPEGs, in capture order, through `timelapse.render_stills_timelapse`. A session with fewer than 24 usable lights gets no time-lapse and is remembered, so `--resume` does not decode it again. The output kind is `timelapse` and the page matches it to its card by `group_id`. -`index.html` is rewritten after each output lands: stacks at the top of night-sky, HDR and panoramas as their own sections. `--no-stacks` skips the night stacks and night time-lapses; burst time-lapses still render. +`index.html` is rewritten after each output lands: stacks at the top of night-sky, HDR and panoramas as their own sections. `--no-stacks` skips the night stacks, star trails and night time-lapses; burst time-lapses still render. ## Failures and resume A group that fails (too few usable frames, a stitch that fails its seam gate, a RAM refusal) adds a `skipped.csv` row `group:` with the reason and the run goes on. -Resume marker: an output is done when its file exists and the ledger holds `put_score(, "stack:v1", 1)`, written after the output lands (a time-lapse uses the same marker under the key of kind `timelapse` and its session's sorted member paths). A crash between the two redoes that group. `photos sort --resume ` (`run_sort(..., resume=)`) reads `ranked.csv`, rebuilds the groups from `group_id`, and does only the unfinished ones; it does not re-develop or re-score. +Resume marker: an output is done when its file exists and the ledger holds `put_score(, "stack:v1", 1)`, written after the output lands (a time-lapse or star trail uses the same marker under the key of kind `timelapse` or `trails` and its session's sorted member paths). A crash between the two redoes that group. `photos sort --resume ` (`run_sort(..., resume=)`) reads `ranked.csv`, rebuilds the groups from `group_id`, and does only the unfinished ones; it does not re-develop or re-score. diff --git a/gomoveshift/pipeline/photos/sort_groups.py b/gomoveshift/pipeline/photos/sort_groups.py index 14899d3..ce4aca6 100644 --- a/gomoveshift/pipeline/photos/sort_groups.py +++ b/gomoveshift/pipeline/photos/sort_groups.py @@ -4,13 +4,15 @@ order a shot is claimed: ``night`` (an intervalometer chunk), ``bracket`` (exposure bracket), ``pano`` (a fast run of same-settings frames) and ``burst`` (look-alikes inside one category). Outputs: ``hdr/`` fusions, ``pano/`` stitches, then ``stacks/`` night stacks, -one at a time. Then ``timelapse/``: one MP4 (plus a same-stem poster JPEG) per night session +one at a time. Then, per night session, ``trails/``: one star-trail TIFF master plus a +same-stem JPEG; and ``timelapse/``: one MP4 (plus a same-stem poster JPEG) per night session and per long burst. A failed output is a ``skipped.csv`` row and never touches a member photo. Resume marker: a group output counts as done only when the output file exists AND the ledger holds ``put_score(, "stack:v1", 1)``, written after the output lands. A crash between the two leaves no marker, so the half-written output is redone. A time-lapse -uses the same marker under its own key (``group_key("timelapse", ...)``). +uses the same marker under its own key (``group_key("timelapse", ...)``); a star trail under +``group_key("trails", ...)``. """ from __future__ import annotations @@ -37,7 +39,14 @@ #: (``astro_timelapse.TIMELAPSE_FPS``), so 24 frames is one second of video: below that the #: clip is a flash, not motion, and the stack already shows the scene. TIMELAPSE_MIN_FRAMES = 24 +#: A session too short for a star trail, remembered like ``TIMELAPSE_SKIP_MARKER``. +TRAILS_SKIP_MARKER = "trails-skip:v1" +#: Fewest lights worth a star trail: 24 frames at the measured 32 s interval +#: (``astro_trails``) is about 13 minutes, about 3 degrees of sky rotation. Shorter arcs read +#: as dots, and the stack already shows that scene. +TRAILS_MIN_FRAMES = 24 HDR_DIR, PANO_DIR, STACK_DIR, TIMELAPSE_DIR = "hdr", "pano", "stacks", "timelapse" +TRAILS_DIR = "trails" _TIME_FORMATS = ("%Y:%m:%d %H:%M:%S", "%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S") @@ -139,12 +148,24 @@ def _poster(video, poster, frames, fps) -> None: raise RuntimeError(f"poster frame failed: exit {done.returncode}: {tail}") -def _night_timelapse(lights, darks, out_mp4, out_jpg) -> None: - """One session's lights to an MP4 with the night product's fps and height cap. Darks - calibrate only when they validate against the lights (same identity check as a stack); - a set that does not match is dropped, never subtracted blind.""" +def _validated_darks(lights, darks) -> list: + """``darks`` when they validate against the lights (same identity check as a stack); a + set that does not match is dropped, never subtracted blind.""" from .astro_calibrate import AstroCalibrationError from .astro_orchestrator import build_validated_master_dark + + if not darks: + return [] + try: + build_validated_master_dark(lights[:1], darks) + except AstroCalibrationError: + return [] + return list(darks) + + +def _night_timelapse(lights, darks, out_mp4, out_jpg) -> None: + """One session's lights to an MP4 with the night product's fps and height cap; darks + pass through ``_validated_darks``.""" from .astro_timelapse import ( TIMELAPSE_FPS, TIMELAPSE_HEIGHT_DEFAULT, @@ -152,11 +173,7 @@ def _night_timelapse(lights, darks, out_mp4, out_jpg) -> None: render_timelapse, ) - if darks: - try: - build_validated_master_dark(lights[:1], darks) - except AstroCalibrationError: - darks = [] + darks = _validated_darks(lights, darks) preflight_timelapse(out_mp4.parent, megapixels=_megapixels(lights[0]), dark_count=len(darks)) video, used, _rejected = render_timelapse( lights, out_mp4, darks, fps=TIMELAPSE_FPS, scale_height=TIMELAPSE_HEIGHT_DEFAULT @@ -164,6 +181,20 @@ def _night_timelapse(lights, darks, out_mp4, out_jpg) -> None: _poster(video, out_jpg, used, TIMELAPSE_FPS) +def _star_trails(lights, darks, out_tif, out_jpg) -> None: + """One session's lights to a star-trail master (``astro_trails.render_trail``: unaligned + per-pixel maximum) plus a browser JPEG from that master, as the night products export it. + Darks pass through ``_validated_darks``; a maximum keeps a hot pixel forever, so an + unmatched set is never subtracted.""" + from .astro_trails import preflight_trail, render_trail + from .night_products import render_master_preview + + darks = _validated_darks(lights, darks) + preflight_trail(out_tif.parent, megapixels=_megapixels(lights[0]), dark_count=len(darks)) + master, _used, _rejected = render_trail(lights, out_tif, darks) + render_master_preview(Path(master), out_jpg) + + def _still_timelapse(jpegs, out_mp4, out_jpg) -> None: """A burst's developed JPEGs to an MP4 through the shared still-frame core. The core orders stills by file name and the developed names are rank-first, so the frames go in @@ -207,6 +238,8 @@ class GroupOps: night_timelapse: Callable = _night_timelapse #: ``(developed_jpegs_in_capture_order, out_mp4, out_jpg)``: writes the video and poster. still_timelapse: Callable = _still_timelapse + #: ``(lights, darks, out_tif, out_jpg)``: writes the star-trail master and its JPEG. + star_trails: Callable = _star_trails def real_ops() -> GroupOps: @@ -399,7 +432,7 @@ def group_key(kind: str, members) -> str: @dataclass class Output: - kind: str # "hdr" | "pano" | "stacks" | "timelapse" + kind: str # "hdr" | "pano" | "stacks" | "trails" | "timelapse" group_id: str rel: str # path relative to the run folder (the image the page shows) @@ -414,7 +447,8 @@ class StageResult: def build_outputs( run_dir: Path, rows, ops: GroupOps, ledger, *, workers: int, no_stacks: bool, on_output ) -> StageResult: - """HDR, panoramas, night stacks (best session first), then time-lapses; one at a time.""" + """HDR, panoramas, night stacks (best session first), then star trails and time-lapses; + one at a time.""" result = StageResult() groups = groups_from_rows(rows) queue = [g for g in groups if g.kind == "bracket"] + [g for g in groups if g.kind == "pano"] @@ -451,7 +485,8 @@ def build_outputs( def _build_timelapses(run_dir, groups, ops, ledger, result, no_stacks, on_output) -> None: - """One time-lapse per night session (not under ``--no-stacks``) and per long burst.""" + """Per night session (not under ``--no-stacks``) a star trail then a time-lapse, sharing + one light/dark triage; per long burst a time-lapse.""" units = [] # (id, kind, member sources, developed JPEGs) if not no_stacks: for session in night_sessions(groups): @@ -462,6 +497,15 @@ def _build_timelapses(run_dir, groups, ops, ledger, result, no_stacks, on_output if g.kind == "burst" and len(g.members) >= TIMELAPSE_MIN_FRAMES: units.append((g.id, "burst", g.members, [o for o in g.outputs if o])) for gid, kind, members, jpegs in units: + triage: list = [] # the session's (lights, darks), decoded at most once + + def classify(members=members, triage=triage): + if not triage: + triage.append(ops.classify_night([Path(m) for m in members])) + return triage[0] + + if kind == "night": + _build_trail(run_dir, gid, members, classify, ops, ledger, result, on_output) mp4 = run_dir / TIMELAPSE_DIR / f"{gid}.mp4" poster = mp4.with_suffix(".jpg") key = group_key("timelapse", members) @@ -472,7 +516,7 @@ def _build_timelapses(run_dir, groups, ops, ledger, result, no_stacks, on_output else: try: if kind == "night": - lights, darks = ops.classify_night([Path(m) for m in members]) + lights, darks = classify() if len(lights) < TIMELAPSE_MIN_FRAMES: ledger.put_score(key, TIMELAPSE_SKIP_MARKER, 1) continue @@ -495,6 +539,35 @@ def _build_timelapses(run_dir, groups, ops, ledger, result, no_stacks, on_output on_output(result.outputs) +def _build_trail(run_dir, gid, members, classify, ops, ledger, result, on_output) -> None: + """One star trail for a night session: resume, skip and failure as a time-lapse.""" + tif = run_dir / TRAILS_DIR / f"{gid}.tif" + jpg = tif.with_suffix(".jpg") + key = group_key("trails", members) + if tif.is_file() and jpg.is_file() and ledger.get_score(key, STACK_MARKER) == 1: + result.stats["resumed"] += 1 + elif ledger.get_score(key, TRAILS_SKIP_MARKER) == 1: + return + else: + try: + lights, darks = classify() + if len(lights) < TRAILS_MIN_FRAMES: + ledger.put_score(key, TRAILS_SKIP_MARKER, 1) + return + tif.parent.mkdir(parents=True, exist_ok=True) + ops.star_trails(lights, darks, tif, jpg) + except Exception as exc: # a star trail must never abort the run + result.skipped.append((f"group:{gid}", f"trails failed: {exc}")) + result.stats["failed"] += 1 + for leftover in (tif, jpg): + leftover.unlink(missing_ok=True) + return + ledger.put_score(key, STACK_MARKER, 1) + result.stats["built"] += 1 + result.outputs.append(Output("trails", gid, f"{TRAILS_DIR}/{gid}.jpg")) + on_output(result.outputs) + + def _produce(group, ops, paths, main, shown, workers, out_dir) -> None: if group.kind == "bracket": ops.fuse(paths, main) diff --git a/tests/test_photo_sort_groups.py b/tests/test_photo_sort_groups.py index 0b4cada..1f98e4e 100644 --- a/tests/test_photo_sort_groups.py +++ b/tests/test_photo_sort_groups.py @@ -45,7 +45,7 @@ class Ops: def __init__(self, sequences=(), brackets=(), fail=(), hashes=None, chunk=None, darks=()): self.sequences, self.brackets, self.fail = list(sequences), list(brackets), set(fail) self.chunk, self.darks = chunk, {Path(d) for d in darks} - self.lapse = [] + self.lapse, self.trails = [], [] self.hashes = hashes or {} self.calls = [] @@ -100,6 +100,14 @@ def night_timelapse(self, lights, darks, out_mp4, out_jpg): raise RuntimeError("lapse boom") out_jpg.write_bytes(b"jpg") + def star_trails(self, lights, darks, out_tif, out_jpg): + self.calls.append(("star_trails", out_tif.name)) + self.trails.append((list(lights), list(darks))) + out_tif.write_bytes(b"tif") + if "star_trails" in self.fail: + raise RuntimeError("trail boom") + out_jpg.write_bytes(b"jpg") + def still_timelapse(self, jpegs, out_mp4, out_jpg): self.calls.append(("still_timelapse", out_mp4.name)) self.lapse.append((list(jpegs), [])) @@ -119,6 +127,7 @@ def ops(self): classify_night=self.classify_night, night_timelapse=self.night_timelapse, still_timelapse=self.still_timelapse, + star_trails=self.star_trails, ) @@ -522,7 +531,9 @@ def test_session_with_too_few_usable_lights_is_skipped_quietly(tmp_path, root): src, root, make_hooks(ops, records=night_records(seq), readings=night_readings(seq)) ) assert lapse_calls(ops) == [] and rows(result.run_dir / "skipped.csv") == [] + assert lapse_calls(ops, "star_trails") == [] assert not (result.run_dir / "timelapse").exists() + assert not (result.run_dir / "trails").exists() ops.calls.clear() run(src, root, make_hooks(ops), resume=result.run_dir) # the verdict is remembered assert lapse_calls(ops, "classify_night") == [] @@ -592,6 +603,90 @@ def test_no_stacks_skips_night_timelapses(tmp_path, root): assert ops.calls == [] +def test_night_session_renders_one_star_trail_beside_its_timelapse(tmp_path, root): + names = [f"n{i:02d}" for i in range(30)] + src = card(tmp_path, names) + seq = stems(src, names) + ops = Ops(sequences=[seq], chunk=15, darks=seq[-4:]) + hooks = make_hooks(ops, records=night_records(seq), readings=night_readings(seq)) + result = run(src, root, hooks) + assert lapse_calls(ops, "star_trails") == [("star_trails", "night-1.tif")] + assert ops.trails == [(seq[:-4], seq[-4:])] # lights in capture order; lens-cap darks apart + assert lapse_calls(ops, "classify_night") == [("classify_night", 30)] # one triage, shared + assert (result.run_dir / "trails" / "night-1.tif").is_file() + assert (result.run_dir / "trails" / "night-1.jpg").is_file() + page = (result.run_dir / "index.html").read_text() + stack, trail = page.index("stacks/night-1.jpg"), page.index("trails/night-1.jpg") + assert stack < trail < page.index("timelapse/night-1.mp4") + with Ledger(root / ps.LEDGER_NAME) as ledger: + key = sg.group_key("trails", [str(p) for p in seq]) + assert ledger.get_score(key, sg.STACK_MARKER) == 1 + + +def test_resume_skips_a_finished_trail_and_rebuilds_a_missing_one(tmp_path, root): + names = [f"n{i:02d}" for i in range(30)] + src = card(tmp_path, names) + seq = stems(src, names) + ops = Ops(sequences=[seq]) + hooks = make_hooks(ops, records=night_records(seq), readings=night_readings(seq)) + first = run(src, root, hooks) + ops.calls.clear() + run(src, root, hooks, resume=first.run_dir) + assert ops.calls == [] + (first.run_dir / "trails" / "night-1.jpg").unlink() + run(src, root, hooks, resume=first.run_dir) + assert [c[0] for c in ops.calls] == ["classify_night", "star_trails"] + + +def test_failed_trail_is_a_skipped_row_and_the_timelapse_still_lands(tmp_path, root): + names = [f"n{i:02d}" for i in range(30)] + src = card(tmp_path, names) + seq = stems(src, names) + ops = Ops(sequences=[seq], fail=["star_trails"]) + result = run( + src, root, make_hooks(ops, records=night_records(seq), readings=night_readings(seq)) + ) + skipped = rows(result.run_dir / "skipped.csv") + assert [(r["path"], r["reason"]) for r in skipped] == [ + ("group:night-1", "trails failed: trail boom") + ] + assert not list((result.run_dir / "trails").glob("night-1.*")) + assert (result.run_dir / "timelapse" / "night-1.mp4").is_file() + assert "trails/night-1.jpg" not in (result.run_dir / "index.html").read_text() + + +def test_default_trail_wrapper_uses_the_astro_trail_engine(tmp_path, monkeypatch): + import gomoveshift.pipeline.photos.astro_calibrate as ac + import gomoveshift.pipeline.photos.astro_orchestrator as ao + import gomoveshift.pipeline.photos.astro_trails as at + import gomoveshift.pipeline.photos.night_products as npx + + seen = {} + monkeypatch.setattr(sg, "_megapixels", lambda p: 20.0) + monkeypatch.setattr(at, "preflight_trail", lambda d, **k: seen.update(pre=(d, k))) + + def bad_darks(lights, darks): + raise ac.AstroCalibrationError("mismatch") + + monkeypatch.setattr(ao, "build_validated_master_dark", bad_darks) + monkeypatch.setattr( + at, + "render_trail", + lambda lights, out, darks: ( + seen.update(trail=(list(lights), list(darks), out)) or (out, 2, ()) + ), + ) + monkeypatch.setattr( + npx, "render_master_preview", lambda master, out: seen.update(preview=(master, out)) or out + ) + lights = [Path("l1"), Path("l2")] + tif, jpg = tmp_path / "t.tif", tmp_path / "t.jpg" + sg._star_trails(lights, [Path("d1")], tif, jpg) + assert seen["pre"] == (tmp_path, {"megapixels": 20.0, "dark_count": 0}) + assert seen["trail"] == (lights, [], tif) # unmatched darks dropped, never subtracted + assert seen["preview"] == (tif, jpg) + + def test_default_classify_night_splits_cap_on_frames_from_lights(monkeypatch): import gomoveshift.pipeline.photos.astro_orchestrator as ao import gomoveshift.pipeline.photos.astro_triage as at