From 2334b462f457a1d49e9c672d617cbeb80deb42e8 Mon Sep 17 00:00:00 2001 From: Viktor Mikhalkin <9029022+ViktorMikhalkin@users.noreply.github.com> Date: Fri, 23 Oct 2026 21:47:06 +0300 Subject: [PATCH 1/9] feat(domain): count only unpaired removals against the removal guard A removed source that pairs with an added file of the same content hash and size (the one-to-one pairing of ADR-0027) loses no content, so it no longer counts against the guard of its old root, whether or not its passages are reused. Unpaired removals count as before, empties_root included: a root whose sources all moved elsewhere is not emptied. The plan and the guard message show the paired and unpaired counts separately; the guard details and the plan JSON carry `paired` when it is not zero. Amends ADR-0027. Story: batch 28, #45. --- src/docgrep/api/cli.py | 10 ++++ src/docgrep/domain/update_plan.py | 24 ++++++--- src/docgrep/usecases/plan_update.py | 41 +++++++++++++-- src/docgrep/usecases/update_index.py | 18 +++++-- tests/integration/test_update_plan_cli.py | 48 ++++++++++++++++++ tests/unit/test_plan_update.py | 62 +++++++++++++++++++++++ tests/unit/test_reuse_by_hash.py | 5 +- tests/unit/test_update_plan_rules.py | 19 +++++++ 8 files changed, 212 insertions(+), 15 deletions(-) diff --git a/src/docgrep/api/cli.py b/src/docgrep/api/cli.py index a24681e..ac695ab 100644 --- a/src/docgrep/api/cli.py +++ b/src/docgrep/api/cli.py @@ -1437,6 +1437,8 @@ def _preview_json(preview: UpdatePreview, *, paths: bool) -> dict[str, Any]: "recorded": root.recorded, "guard_threshold": root.threshold, "guarded": root.name in guarded, + "unpaired_removals": root.unpaired_removals, + "paired_removals": root.paired_removals, **_plan_counts(root.plan), "paths": _preview_paths(root) if paths else None, } @@ -1453,6 +1455,7 @@ def _preview_json(preview: UpdatePreview, *, paths: bool) -> dict[str, Any]: "recorded": g.recorded, "threshold": g.threshold, "reason": g.reason, + **({"paired": g.paired} if g.paired else {}), } for g in preview.guarded ], @@ -1482,6 +1485,12 @@ def _format_preview(preview: UpdatePreview, *, paths: bool) -> str: lines = [f"Update plan. {PLAN_NOTE}\n"] for root in preview.roots: lines.append(_plan_line(f"Root {root.name}", root.plan)) + if root.paired_removals and root.name in {g.root for g in preview.guarded}: + lines.append( + f" Removals: {root.unpaired_removals} unpaired (counted by the " + f"removal guard), {root.paired_removals} paired with an added " + "file (not counted).\n" + ) if not paths: continue groups: tuple[tuple[str, Sequence[str]], ...] = ( @@ -1513,6 +1522,7 @@ def _format_preview(preview: UpdatePreview, *, paths: bool) -> str: if g.reason == "empties_root" else f"{g.removals} of {g.recorded} sources (at most {g.threshold} allowed)" ) + + (f" (not counted: {g.paired} paired with an added file)" if g.paired else "") + "; `docgrep update` would stop without changing anything unless run " "with --yes.\n" for g in preview.guarded diff --git a/src/docgrep/domain/update_plan.py b/src/docgrep/domain/update_plan.py index 4029387..8e98974 100644 --- a/src/docgrep/domain/update_plan.py +++ b/src/docgrep/domain/update_plan.py @@ -156,12 +156,16 @@ class GuardedRoot: recorded: int threshold: int reason: GuardReason = "threshold" + # Further removals of the root that the guard does not count because each + # pairs with an added file of the same content (``pair_moves``). + paired: int = 0 def guarded_roots( removals: Mapping[str, int], recorded: Mapping[str, int], guard: RemovalGuard, + paired: Mapping[str, int] | None = None, ) -> tuple[GuardedRoot, ...]: """The roots, by name, whose planned removals the guard refuses. @@ -170,10 +174,13 @@ def guarded_roots( point without its disk, passes the check that a root can be read), or else when they exceed the threshold (``threshold``). - ``removals`` counts the planned removals of each root of the run (moved - sources and sources no longer matched by the patterns included) and - ``recorded`` the sources the index records for it. Empty: the update may - proceed. + ``removals`` counts the planned removals of each root of the run that + lose content: sources no longer matched by the patterns included, moved + sources not, because a removed source that pairs with an added file of + the same content and size loses nothing (``pair_moves``); a root whose + sources all moved elsewhere is not emptied. ``recorded`` is the sources + the index records for the root and ``paired`` the paired removals, which + are only reported. Empty: the update may proceed. """ found = [] for root in sorted(removals): @@ -181,7 +188,12 @@ def guarded_roots( total = recorded.get(root, 0) limit = guard.threshold(total) if count > 0 and count == total: - found.append(GuardedRoot(root, count, total, limit, "empties_root")) + reason: GuardReason = "empties_root" elif count > limit: - found.append(GuardedRoot(root, count, total, limit)) + reason = "threshold" + else: + continue + found.append( + GuardedRoot(root, count, total, limit, reason, (paired or {}).get(root, 0)) + ) return tuple(found) diff --git a/src/docgrep/usecases/plan_update.py b/src/docgrep/usecases/plan_update.py index f263dc7..8be773a 100644 --- a/src/docgrep/usecases/plan_update.py +++ b/src/docgrep/usecases/plan_update.py @@ -165,6 +165,10 @@ class DetectedChanges: outside_run: tuple[tuple[str, int], ...] = () # The root of each recorded source of the run, by stored path. source_roots: Mapping[str, str] = field(default_factory=dict) + # The stored paths of the removed sources that pair with an added file of + # the same content and size (``pair_moves``), whether or not their + # passages are reused: they lose no content. + paired: frozenset[str] = frozenset() @property def moves(self) -> tuple[Move, ...]: @@ -176,15 +180,31 @@ def moves(self) -> tuple[Move, ...]: ) def removals(self) -> dict[str, int]: - """The planned removals of each root of the run, moves included.""" + """The planned removals of each root of the run that lose content: + the unpaired ones. A removed source that pairs with an added file of + the same content and size (a move) loses nothing and is counted by + ``paired_removals`` instead (ADR-0029).""" counts = dict.fromkeys(self.roots, 0) for change in self.changes: - if change.kind in ("removed", "moved"): + if change.kind == "removed" and change.source.path not in self.paired: + counts[change.root] = counts.get(change.root, 0) + 1 + return counts + + def paired_removals(self) -> dict[str, int]: + """The planned removals of each root of the run that pair with an + added file, moves included.""" + counts = dict.fromkeys(self.roots, 0) + for change in self.changes: + if change.kind == "moved" or ( + change.kind == "removed" and change.source.path in self.paired + ): counts[change.root] = counts.get(change.root, 0) + 1 return counts def guarded(self, guard: RemovalGuard) -> tuple[GuardedRoot, ...]: - return guarded_roots(self.removals(), self.recorded, guard) + return guarded_roots( + self.removals(), self.recorded, guard, self.paired_removals() + ) def plan(self, root: str | None = None) -> UpdatePlan: """The counts of one root, or of the whole run.""" @@ -229,6 +249,10 @@ class RootPreview: refreshed: tuple[str, ...] moved: tuple[Move, ...] copied: tuple[Copy, ...] = () + # The removals the guard counts, and those it does not because they pair + # with an added file of the same content (moves included). + unpaired_removals: int = 0 + paired_removals: int = 0 @dataclass(frozen=True, slots=True) @@ -259,6 +283,7 @@ def reused(kind: ChangeKind, root: str) -> tuple[tuple[str, str], ...]: if c.kind == kind and c.root == root and c.reuse is not None ) + unpaired, paired = detected.removals(), detected.paired_removals() roots = tuple( RootPreview( name=name, @@ -278,6 +303,8 @@ def reused(kind: ChangeKind, root: str) -> tuple[tuple[str, str], ...]: ), moved=tuple(Move(*pair) for pair in reused("moved", name)), copied=tuple(Copy(*pair) for pair in reused("copied", name)), + unpaired_removals=unpaired.get(name, 0), + paired_removals=paired.get(name, 0), ) for name in detected.roots ) @@ -470,6 +497,13 @@ def change( unchanged[name] += 1 for path in sorted(set(recorded) - seen): change("removed", recorded[path].source, recorded[path], root_of[path]) + paired = frozenset( + move.removed + for move in pair_moves( + (c.source for c in found["removed"]), + (c.source for c in found["added"]), + ) + ) if self.reuse_by_hash: self._plan_reuse(found, committed) changes = tuple( @@ -485,6 +519,7 @@ def change( unchanged=unchanged, outside_run=tuple(sorted(outside.items())), source_roots=root_of, + paired=paired, ) def _index_model(self) -> str | None: diff --git a/src/docgrep/usecases/update_index.py b/src/docgrep/usecases/update_index.py index de62329..a58f6bf 100644 --- a/src/docgrep/usecases/update_index.py +++ b/src/docgrep/usecases/update_index.py @@ -748,10 +748,13 @@ def _remove( def _guard_message(guarded: Sequence[GuardedRoot]) -> str: roots = "; ".join( - f"root {g.root!r} would lose all its {g.recorded} sources" - if g.reason == "empties_root" - else f"root {g.root!r} would lose {g.removals} of {g.recorded} sources " - f"(at most {g.threshold} allowed)" + ( + f"root {g.root!r} would lose all its {g.recorded} sources" + if g.reason == "empties_root" + else f"root {g.root!r} would lose {g.removals} of {g.recorded} sources " + f"(at most {g.threshold} allowed)" + ) + + _paired_note(g) for g in guarded ) return ( @@ -761,6 +764,12 @@ def _guard_message(guarded: Sequence[GuardedRoot]) -> str: ) +def _paired_note(guarded: GuardedRoot) -> str: + if not guarded.paired: + return "" + return f" (not counted: {guarded.paired} paired with an added file)" + + def _guard_details(guarded: Sequence[GuardedRoot]) -> dict[str, object]: return { "roots": [ @@ -770,6 +779,7 @@ def _guard_details(guarded: Sequence[GuardedRoot]) -> dict[str, object]: "recorded": g.recorded, "threshold": g.threshold, "reason": g.reason, + **({"paired": g.paired} if g.paired else {}), } for g in guarded ] diff --git a/tests/integration/test_update_plan_cli.py b/tests/integration/test_update_plan_cli.py index f848339..e9b83de 100644 --- a/tests/integration/test_update_plan_cli.py +++ b/tests/integration/test_update_plan_cli.py @@ -338,6 +338,54 @@ def test_the_guard_stops_removals_above_the_threshold( ] +def test_moving_every_source_of_a_root_to_another_passes_the_guard( + indexed: tuple[Path, Path, Path], capsys: pytest.CaptureFixture[str] +) -> None: + books, notes, data = indexed + for path in sorted(notes.iterdir()): + target = books / "from-notes" / path.name + target.parent.mkdir(exist_ok=True) + path.rename(target) + + code, out, _ = _cli(capsys, "update", "--plan", "--json") + body = json.loads(out) + root = next(r for r in body["roots"] if r["root"] == "notes") + + assert code == 0 and body["guarded_roots"] == [] + assert (root["unpaired_removals"], root["paired_removals"]) == (0, 2) + + code, _, err = _cli(capsys, "update") + + assert code == 0, err + manifest = SQLiteManifest(data, read_only=True) + assert manifest.get_source("notes/x.txt") is None + assert manifest.get_source("from-notes/x.txt") is not None + + +def test_the_guard_message_separates_paired_and_unpaired_removals( + indexed: tuple[Path, Path, Path], + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + books, _, _ = indexed + monkeypatch.setenv("DOCGREP_UPDATE_GUARD_COUNT", "1") + (books / "shelf" / "b.md").rename(books / "shelf" / "b2.md") + (books / "c.txt").unlink() + (books / "d.txt").unlink() + + code, _, err = _cli(capsys, "update") + + assert code == 1 + assert ( + "root 'books' would lose 2 of 8 sources (at most 1 allowed) " + "(not counted: 1 paired with an added file)" in err + ) + code, out, _ = _cli(capsys, "update", "--plan", "--json") + assert json.loads(out)["guarded_roots"][0]["paired"] == 1 + code, out, _ = _cli(capsys, "update", "--plan") + assert "Removals: 2 unpaired (counted by the removal guard), 1 paired" in out + + def test_removing_one_of_two_sources_is_applied( indexed: tuple[Path, Path, Path], capsys: pytest.CaptureFixture[str] ) -> None: diff --git a/tests/unit/test_plan_update.py b/tests/unit/test_plan_update.py index adddce2..9011805 100644 --- a/tests/unit/test_plan_update.py +++ b/tests/unit/test_plan_update.py @@ -438,3 +438,65 @@ def test_removing_one_of_two_sources_is_allowed() -> None: assert harness.planner().run().guarded == () assert harness.update().run().removed == 1 + + +def _move_all(harness: Harness, count: int) -> None: + for index in range(count): + harness.corpus.files[f"moved/doc{index:02d}.txt"] = harness.corpus.files.pop( + f"doc{index:02d}.txt" + ) + + +@pytest.mark.parametrize("reuse", [True, False]) +def test_moving_every_source_of_a_root_passes_the_guard(reuse: bool) -> None: + harness = Harness(_files(40)) + harness.build() + _move_all(harness, 40) + + preview = harness.planner(reuse_by_hash=reuse).run() + + # Whether or not the passages are reused, nothing is lost. + assert preview.guarded == () + assert preview.roots[0].paired_removals == 40 + assert preview.roots[0].unpaired_removals == 0 + summary = harness.update(reuse_by_hash=reuse).run() + assert summary.failed == 0 + assert {r.source.path for r in harness.manifest.committed_sources()} == { + f"moved/doc{i:02d}.txt" for i in range(40) + } + + +def test_deleting_as_many_sources_without_copies_still_trips_the_guard() -> None: + harness = _guarded_harness(40, 40) + + with pytest.raises(RemovalGuardTripped) as refused: + harness.update().run() + + assert "paired" not in refused.value.details["roots"][0] + + +def test_the_guard_counts_only_the_unpaired_part_of_a_mix() -> None: + harness = Harness(_files(60)) + harness.build() + # Threshold of 60 sources: 3. Two moved, four deleted. + _move_all(harness, 2) + for index in range(2, 6): + del harness.corpus.files[f"doc{index:02d}.txt"] + + with pytest.raises(RemovalGuardTripped) as refused: + harness.update().run() + + [root] = refused.value.details["roots"] + assert (root["removals"], root["paired"], root["reason"]) == (4, 2, "threshold") + assert "4 of 60" in str(refused.value) and "2 paired" in str(refused.value) + + +def test_a_mix_within_the_threshold_proceeds() -> None: + harness = Harness(_files(60)) + harness.build() + _move_all(harness, 10) + del harness.corpus.files["doc20.txt"] + + assert harness.planner(reuse_by_hash=True).run().guarded == () + summary = harness.update(reuse_by_hash=True).run() + assert (summary.moved, summary.removed) == (10, 1) diff --git a/tests/unit/test_reuse_by_hash.py b/tests/unit/test_reuse_by_hash.py index 65182c7..59c0bcd 100644 --- a/tests/unit/test_reuse_by_hash.py +++ b/tests/unit/test_reuse_by_hash.py @@ -582,14 +582,15 @@ def test_a_move_whose_old_source_stays_is_finished_by_the_next_update() -> None: assert harness.manifest.get_source("a.txt") is None -def test_moves_count_against_the_removal_guard() -> None: +def test_moves_do_not_count_against_the_removal_guard() -> None: harness = _library() _move_a(harness) detected = harness.planner().detect(retry_failed=True) assert detected.moves == (Move("a.txt", "shelf/a.txt"),) - assert detected.removals() == {"_legacy": 1} + assert detected.removals() == {"_legacy": 0} + assert detected.paired_removals() == {"_legacy": 1} def test_the_plan_takes_the_model_of_the_latest_build_by_default() -> None: diff --git a/tests/unit/test_update_plan_rules.py b/tests/unit/test_update_plan_rules.py index 2c84ce9..3aa3e85 100644 --- a/tests/unit/test_update_plan_rules.py +++ b/tests/unit/test_update_plan_rules.py @@ -153,3 +153,22 @@ def test_the_guard_settings_have_defaults_and_are_validated( patch.setenv(name, value) with pytest.raises(SettingsValidationError): Settings() + + +def test_paired_removals_are_reported_and_never_counted() -> None: + guard = RemovalGuard(ratio=0.05, count=3) + + # Only the unpaired part is compared with the threshold. + assert guarded_roots({"r": 3}, {"r": 100}, guard, {"r": 40}) == () + assert guarded_roots({"r": 4}, {"r": 100}, guard, {"r": 40}) == ( + GuardedRoot("r", 4, 100, 3, "threshold", 40), + ) + + +def test_a_root_whose_sources_all_moved_is_not_emptied() -> None: + guard = RemovalGuard(ratio=1, count=10_000) + + assert guarded_roots({"r": 0}, {"r": 5}, guard, {"r": 5}) == () + assert guarded_roots({"r": 5}, {"r": 5}, guard, {"r": 0}) == ( + GuardedRoot("r", 5, 5, 10_000, "empties_root"), + ) From cc4c3ce5caa374ecf60faacea0f10ff6156d9cbd Mon Sep 17 00:00:00 2001 From: Viktor Mikhalkin <9029022+ViktorMikhalkin@users.noreply.github.com> Date: Fri, 23 Oct 2026 21:49:53 +0300 Subject: [PATCH 2/9] feat(usecases): scope an update to a set of paths UpdateIndex.run and PlanUpdate.detect take an UpdateScope: only its stored paths are compared with the corpus and changed, every other recorded source counts as unchanged and no other file is added. A corpus that can stat single files (PathScanner, implemented by the corpus adapter as scan_paths) does not walk its roots, and a root that cannot be read raises instead of reporting its files gone. In a scoped update the roots need not all be readable, the removals of a root that trips the removal guard are held back and named in the summary instead of refusing the run (paired removals and all additions and modifications still apply), and the run kind recorded in the journal can be `sync`. Status and cancel treat a sync run like an update. Story: batch 28, #45. --- src/docgrep/adapters/corpus/__init__.py | 63 +++++++++++- src/docgrep/adapters/manifest/__init__.py | 2 +- src/docgrep/api/cli.py | 6 +- src/docgrep/api/container.py | 10 ++ src/docgrep/domain/protocols.py | 16 ++- src/docgrep/domain/run_state.py | 1 + src/docgrep/domain/update_plan.py | 9 ++ src/docgrep/usecases/inspect_status.py | 4 +- src/docgrep/usecases/plan_update.py | 31 +++++- src/docgrep/usecases/update_index.py | 74 ++++++++++++-- tests/integration/test_scan_paths.py | 54 ++++++++++ tests/unit/test_scoped_update.py | 114 ++++++++++++++++++++++ 12 files changed, 361 insertions(+), 23 deletions(-) create mode 100644 tests/integration/test_scan_paths.py create mode 100644 tests/unit/test_scoped_update.py diff --git a/src/docgrep/adapters/corpus/__init__.py b/src/docgrep/adapters/corpus/__init__.py index 34c3c6e..43b6364 100644 --- a/src/docgrep/adapters/corpus/__init__.py +++ b/src/docgrep/adapters/corpus/__init__.py @@ -1,7 +1,7 @@ """Local corpus enumeration and content hashing.""" import os -from collections.abc import Iterator, Sequence +from collections.abc import Collection, Iterator, Sequence from pathlib import Path from blake3 import blake3 @@ -270,6 +270,67 @@ def scan_corpus( return tuple(sorted(scanned, key=lambda item: item.path)) +def scan_paths( + corpus_root: CorpusRoots | Path, + paths: Collection[str], + include_globs: Sequence[str] = ("**/*",), + exclude_globs: Sequence[str] = (), + data_dir: Path | None = None, +) -> tuple[ScannedFile, ...]: + """The entries ``scan_corpus`` would give for these stored paths, without + walking any root: one stat per path. + + A path whose file is gone, is not a regular file, is a symlink, lies in a + ``.git`` or data directory, or whose root is not configured is left out. + A configured root that cannot be read raises, as a walk would, so that + its files are never taken for deleted ones. + """ + roots = as_roots(corpus_root) + readable: set[str] = set() + scanned = [] + for stored in sorted(set(paths)): + root = roots.owner(stored) + if root is None: + continue + if root.name not in readable: + try: + _require_readable_root(root.path) + except OSError as error: + raise _inaccessible(root.path, error, root) from error + readable.add(root.name) + relative = root.relative_path(stored) + path = root.path / relative + skipped = _skipped_data_dir(root.path, data_dir) + if ".git" in relative.split("/") or ( + skipped is not None and skipped in path.resolve().parents + ): + continue + try: + if path.is_symlink() or not path.is_file(): + continue + stat = path.stat() + except FileNotFoundError: + continue + except OSError as error: + raise _inaccessible(root.path, error, root) from error + if not _matches(relative, include_globs): + excluded_by: str | None = NOT_MATCHED_BY_INCLUDES + else: + excluded_by = _matching_pattern(relative, exclude_globs) + extension = path.suffix.lower() + scanned.append( + ScannedFile( + path=stored, + size=stat.st_size, + extension=extension, + file_type=_FILE_TYPES.get(extension), + excluded_by=excluded_by, + mtime=stat.st_mtime, + ) + ) + return tuple(scanned) + + def source_file(roots: CorpusRoots, path: str) -> tuple[Path, Path]: """The file on disk of a stored path and the folder of its root. diff --git a/src/docgrep/adapters/manifest/__init__.py b/src/docgrep/adapters/manifest/__init__.py index 39dfe9b..ea1458b 100644 --- a/src/docgrep/adapters/manifest/__init__.py +++ b/src/docgrep/adapters/manifest/__init__.py @@ -27,7 +27,7 @@ _LOG = logging.getLogger(__name__) # The kinds of run recorded in the runs table. -RUN_KINDS = ("build", "update", "ocr", "reencode_sparse") +RUN_KINDS = ("build", "update", "sync", "ocr", "reencode_sparse") # Columns added to the runs table after the first schema; added when missing. _RUN_COLUMNS = ( diff --git a/src/docgrep/api/cli.py b/src/docgrep/api/cli.py index ac695ab..217e5e9 100644 --- a/src/docgrep/api/cli.py +++ b/src/docgrep/api/cli.py @@ -870,7 +870,7 @@ def _format_cancel(result: CancelResult) -> tuple[str, int]: "reencode-sparse --yes` again to continue from its checkpoint.\n", 0, ) - if result.outcome == "interrupted" and result.kind == "update": + if result.outcome == "interrupted" and result.kind in ("update", "sync"): return ( f"Update run {result.run_id} was interrupted: its process is gone and " "nothing holds the index lock. What it applied stays live; run " @@ -1070,10 +1070,10 @@ def _format_status(report: dict[str, Any]) -> str: f"started {run['started_at']}, finished {run['finished_at'] or '-'}" + (f", resumed from {run['resumed_from']}" if run["resumed_from"] else "") ) - elif run["kind"] == "update": + elif run["kind"] in ("update", "sync"): totals = run["totals"] lines.append( - f"Latest run: {run['run_id']} (update) {run['state']}, started " + f"Latest run: {run['run_id']} ({run['kind']}) {run['state']}, started " f"{run['started_at']}, finished {run['finished_at'] or '-'}, " + ", ".join( f"{totals.get(key, 0)} {key}" diff --git a/src/docgrep/api/container.py b/src/docgrep/api/container.py index 77ed3a1..a151eda 100644 --- a/src/docgrep/api/container.py +++ b/src/docgrep/api/container.py @@ -16,6 +16,7 @@ require_available, root_readable, scan_corpus, + scan_paths, ) from docgrep.adapters.filelock import FileIndexLock from docgrep.adapters.manifest import SQLiteManifest, SQLiteOcrRegistry @@ -181,6 +182,15 @@ def scan(self) -> tuple[ScannedFile, ...]: self._settings.data_dir, ) + def scan_paths(self, paths: Collection[str]) -> tuple[ScannedFile, ...]: + return scan_paths( + self._roots, + paths, + self._settings.include_globs, + self._settings.exclude_globs, + self._settings.data_dir, + ) + def describe(self, path: str) -> Source: return describe_source(self._roots, path) diff --git a/src/docgrep/domain/protocols.py b/src/docgrep/domain/protocols.py index e38990b..a508317 100644 --- a/src/docgrep/domain/protocols.py +++ b/src/docgrep/domain/protocols.py @@ -2,7 +2,7 @@ from collections.abc import Callable, Collection, Mapping, Sequence from contextlib import AbstractContextManager -from typing import Literal, Protocol +from typing import Literal, Protocol, runtime_checkable from docgrep.domain.context import OutlineEntry from docgrep.domain.evaluation import ChannelLists @@ -147,6 +147,20 @@ def describe(self, path: str) -> Source: ... +@runtime_checkable +class PathScanner(Protocol): + """A corpus reader that can also stat chosen files without walking the + roots, so that a scoped update neither reads nor depends on the rest.""" + + def scan_paths(self, paths: Collection[str]) -> tuple[ScannedFile, ...]: + """The files among the stored ``paths`` that exist, from directory + entries alone, as ``CorpusReader.scan`` reports them (excluded files + carry their pattern). A path that is gone is left out. Raises + ``ValidationError`` when the root of a path cannot be read, so that a + missing root is never taken for deleted files.""" + ... + + class FolderReader(Protocol): """The files of a folder compared with a root, such as a backup copy. diff --git a/src/docgrep/domain/run_state.py b/src/docgrep/domain/run_state.py index 829ef95..bca4c87 100644 --- a/src/docgrep/domain/run_state.py +++ b/src/docgrep/domain/run_state.py @@ -132,6 +132,7 @@ def latest_root_prefixes(manifest: ManifestReader) -> frozenset[str]: for run in ( manifest.latest_run_of_kind("build"), manifest.latest_run_of_kind("update"), + manifest.latest_run_of_kind("sync"), ) if run is not None ] diff --git a/src/docgrep/domain/update_plan.py b/src/docgrep/domain/update_plan.py index 8e98974..a0c0ff7 100644 --- a/src/docgrep/domain/update_plan.py +++ b/src/docgrep/domain/update_plan.py @@ -22,6 +22,15 @@ GUARD_SMALL_ROOT = 20 +@dataclass(frozen=True, slots=True) +class UpdateScope: + """The stored paths an update is limited to (ADR-0029): only these are + compared with the corpus and changed; every other recorded source counts + as unchanged and no other file is added.""" + + paths: frozenset[str] + + @dataclass(frozen=True, slots=True) class Move: """A removed source and an added file with the same content and size. diff --git a/src/docgrep/usecases/inspect_status.py b/src/docgrep/usecases/inspect_status.py index 57825cf..f0d8818 100644 --- a/src/docgrep/usecases/inspect_status.py +++ b/src/docgrep/usecases/inspect_status.py @@ -216,7 +216,7 @@ def warn(code: str, message: str) -> None: latest.carried_sources, latest.kind, ) - if latest.kind == "update" and run_view.state in ( + if latest.kind in ("update", "sync") and run_view.state in ( "aborted", "cancelled", "interrupted", @@ -548,7 +548,7 @@ def _active( ) return ActiveOperation( kind=run.kind - if run.kind in ("update", "ocr", REENCODE_SPARSE) + if run.kind in ("update", "sync", "ocr", REENCODE_SPARSE) else ("resume" if run.resumed_from else "build"), run_id=run.run_id, started_at=run.started_at, diff --git a/src/docgrep/usecases/plan_update.py b/src/docgrep/usecases/plan_update.py index 8be773a..6b03ab1 100644 --- a/src/docgrep/usecases/plan_update.py +++ b/src/docgrep/usecases/plan_update.py @@ -2,7 +2,7 @@ and which files it can index by reusing stored passages (ADR-0028).""" from collections import Counter -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from dataclasses import dataclass, field, replace from typing import Literal @@ -10,6 +10,7 @@ DELEGATED_TO_OCR, FileType, ManifestSourceRecord, + ScannedFile, Source, ) from docgrep.domain.protocols import ( @@ -19,6 +20,7 @@ ManifestReader, OcrDelegations, OcrOutputOwners, + PathScanner, RootCatalog, parser_reprocess_reason, parser_version, @@ -30,6 +32,7 @@ GuardedRoot, Move, RemovalGuard, + UpdateScope, can_reuse, guarded_roots, pair_moves, @@ -422,12 +425,18 @@ def run(self, *, retry_failed: bool = True) -> UpdatePreview: return UpdatePreview.of(self.detect(retry_failed, roots), self.guard) def detect( - self, retry_failed: bool, roots: CorpusRoots | None = None + self, + retry_failed: bool, + roots: CorpusRoots | None = None, + scope: UpdateScope | None = None, ) -> DetectedChanges: """Compare the roots of the run with the committed sources. Only files whose size or modification time differ from the record, - and added files, are read and hashed. + and added files, are read and hashed. With a ``scope`` only its paths + are compared and can be added, changed or removed; the other + recorded sources count as unchanged, and a corpus that can stat + single files (``PathScanner``) does not walk its roots. """ default = CorpusRoots().legacy_name run_roots = (default,) if roots is None else roots.names @@ -458,7 +467,7 @@ def change( ) -> None: found[kind].append(PlannedChange(kind, source, record, name, reason)) - for item in self.corpus.scan(): + for item in self._items(scope): if item.excluded_by is not None: continue seen.add(item.path) @@ -495,7 +504,12 @@ def change( change("retried", current, record, name) else: unchanged[name] += 1 - for path in sorted(set(recorded) - seen): + gone = set(recorded) - seen + if scope is not None: + for path in set(recorded) - scope.paths: + unchanged[root_of[path]] += 1 + gone &= scope.paths + for path in sorted(gone): change("removed", recorded[path].source, recorded[path], root_of[path]) paired = frozenset( move.removed @@ -522,6 +536,13 @@ def change( paired=paired, ) + def _items(self, scope: UpdateScope | None) -> Iterable[ScannedFile]: + if scope is None: + return self.corpus.scan() + if isinstance(self.corpus, PathScanner): + return self.corpus.scan_paths(scope.paths) + return [item for item in self.corpus.scan() if item.path in scope.paths] + def _index_model(self) -> str | None: if self.embedding_model is not None: return self.embedding_model diff --git a/src/docgrep/usecases/update_index.py b/src/docgrep/usecases/update_index.py index a58f6bf..56e735e 100644 --- a/src/docgrep/usecases/update_index.py +++ b/src/docgrep/usecases/update_index.py @@ -4,8 +4,9 @@ import threading import time from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import UTC, datetime +from typing import Literal from docgrep.domain.errors import ( BuildFailed, @@ -43,7 +44,12 @@ unfinished_reencode, ) from docgrep.domain.sparse import SparseWeighting -from docgrep.domain.update_plan import GuardedRoot, RemovalGuard, reused_source +from docgrep.domain.update_plan import ( + GuardedRoot, + RemovalGuard, + UpdateScope, + reused_source, +) from docgrep.usecases.build_index import ( BuildFailure, Cancelled, @@ -103,6 +109,10 @@ class UpdateSummary: cancelled: bool = False moved: int = 0 copied: int = 0 + # A scoped update (sync): the roots whose removals the guard held back, + # and the stored paths of those removals, which were not applied. + held: tuple[GuardedRoot, ...] = () + held_removals: tuple[str, ...] = () @property def no_changes(self) -> bool: @@ -203,9 +213,19 @@ def run( on_plan: Callable[[UpdatePlan], None] | None = None, override_guard: bool = False, progress: ProgressOptions | None = None, + scope: UpdateScope | None = None, + run_kind: Literal["update", "sync"] = "update", ) -> UpdateSummary: """Update the index; raises before writing anything when it is refused. + With a ``scope`` (ADR-0029) only its paths are compared and changed. + The roots are not required to be readable as a whole, and the + removals of a root that trips the guard are held back, not refused: + they are left out and named in the summary, while the rest of the + scope is applied. ``run_kind`` is the kind recorded in the journal; + with ``wait_for_lock`` false a held index lock raises + ``IndexLockHeld`` at once. + The changes are found first, without writing anything; ``on_plan`` is then called with the work plan, before any change is applied. @@ -219,28 +239,32 @@ def run( with self.lock.hold("update"): metadata = self._check_preconditions() self._choose_sparse_encoder(metadata) - roots = self.run_roots() + roots = self.known_roots() if scope else self.run_roots() try: - detected = self.detect(retry_failed, roots) + detected = self.detect(retry_failed, roots, scope) except DocgrepError: raise except Exception as error: raise BuildFailed( "Index update aborted", details={"reason": str(error)} ) from error + held: tuple[GuardedRoot, ...] = () + held_paths: tuple[str, ...] = () + if scope is not None: + detected, held, held_paths = self._hold_guarded(detected) plan = detected.plan() if on_plan is not None: on_plan(plan) - if not override_guard: + if scope is None and not override_guard: self._check_guard(detected) self._close_dead_update() run_id = self.manifest.start_run( - {**self.run_config, **self.roots_config(roots)}, kind="update" + {**self.run_config, **self.roots_config(roots)}, kind=run_kind ) _LOG.info( "index_update_started", extra={"run_id": run_id, **self.log_context} ) - return self._execute( + summary = self._execute( run_id, started, detected.changes, @@ -248,6 +272,33 @@ def run( detected.refreshes, progress=progress, ) + return replace(summary, held=held, held_removals=held_paths) + + def _hold_guarded( + self, detected: DetectedChanges + ) -> tuple[DetectedChanges, tuple[GuardedRoot, ...], tuple[str, ...]]: + """Leave out the unpaired removals of the roots the guard refuses. + + Paired removals (moves) lose no content and stay, as do all the + additions and modifications. + """ + guarded = detected.guarded(self.guard) + if not guarded: + return detected, (), () + roots = {g.root for g in guarded} + held = tuple( + c.source.path + for c in detected.changes + if c.kind == "removed" + and c.root in roots + and c.source.path not in detected.paired + ) + kept = tuple(c for c in detected.changes if c.source.path not in held) + _LOG.warning( + "sync_removals_held", + extra={"roots": len(guarded), "removals": len(held)}, + ) + return replace(detected, changes=kept), guarded, held def forget_plan(self, name: str) -> tuple[ManifestSourceRecord, ...]: """The committed sources of a root, which ``forget_root`` removes. @@ -380,13 +431,16 @@ def _check_preconditions( def _close_dead_update(self) -> None: """Mark an update that lost its process as aborted; its work stays.""" latest = self.manifest.latest_run() - if latest is not None and latest.kind == "update": + if latest is not None and latest.kind in ("update", "sync"): if latest.status == "running": self.manifest.finish_run(latest.run_id, "aborted", dict(latest.totals)) _LOG.info("update_run_closed", extra={"run_id": latest.run_id}) def detect( - self, retry_failed: bool = True, roots: CorpusRoots | None = None + self, + retry_failed: bool = True, + roots: CorpusRoots | None = None, + scope: UpdateScope | None = None, ) -> DetectedChanges: """What an update with these roots would change; nothing is written.""" return PlanUpdate( @@ -398,7 +452,7 @@ def detect( ocr_outputs=self.ocr_outputs, reuse_by_hash=self.reuse_by_hash, embedding_model=self.embedder.model_name, - ).detect(retry_failed, roots) + ).detect(retry_failed, roots, scope) def _check_guard(self, detected: DetectedChanges) -> None: guarded = detected.guarded(self.guard) diff --git a/tests/integration/test_scan_paths.py b/tests/integration/test_scan_paths.py new file mode 100644 index 0000000..1e2a012 --- /dev/null +++ b/tests/integration/test_scan_paths.py @@ -0,0 +1,54 @@ +"""Stat-only scanning of chosen stored paths, as a scoped update reads them.""" + +from pathlib import Path + +import pytest + +from docgrep.adapters.corpus import scan_corpus, scan_paths +from docgrep.domain.errors import ValidationError +from docgrep.domain.roots import CorpusRoot, CorpusRoots + + +def _roots(tmp_path: Path) -> CorpusRoots: + books, notes = tmp_path / "books", tmp_path / "notes" + for folder, name in ((books, "a.txt"), (notes, "x.txt")): + folder.mkdir() + (folder / name).write_text(name, encoding="utf-8") + (books / "sub").mkdir() + (books / "sub" / "b.md").write_text("b", encoding="utf-8") + (books / "skip.tmp").write_text("t", encoding="utf-8") + return CorpusRoots( + (CorpusRoot("books", books, legacy=True), CorpusRoot("notes", notes)) + ) + + +def test_it_reports_what_a_full_scan_reports_for_the_same_paths( + tmp_path: Path, +) -> None: + roots = _roots(tmp_path) + full = {f.path: f for f in scan_corpus(roots, exclude_globs=("*.tmp",))} + wanted = ["a.txt", "sub/b.md", "notes/x.txt", "skip.tmp"] + + found = scan_paths(roots, wanted, exclude_globs=("*.tmp",)) + + assert {f.path: f for f in found} == {p: full[p] for p in wanted} + assert next(f for f in found if f.path == "skip.tmp").excluded_by == "*.tmp" + + +def test_missing_files_and_unknown_roots_are_left_out(tmp_path: Path) -> None: + roots = _roots(tmp_path) + + found = scan_paths(roots, ["gone.txt", "sub", "other/x.txt"] + ["a.txt"]) + + assert [f.path for f in found] == ["a.txt"] + + +def test_an_unreadable_root_raises_instead_of_reporting_files_gone( + tmp_path: Path, +) -> None: + roots = _roots(tmp_path) + (tmp_path / "notes").rename(tmp_path / "unmounted") + + with pytest.raises(ValidationError): + scan_paths(roots, ["notes/x.txt"]) + assert [f.path for f in scan_paths(roots, ["a.txt"])] == ["a.txt"] diff --git a/tests/unit/test_scoped_update.py b/tests/unit/test_scoped_update.py new file mode 100644 index 0000000..2f02966 --- /dev/null +++ b/tests/unit/test_scoped_update.py @@ -0,0 +1,114 @@ +"""An update limited to a set of stored paths, the unit of automatic sync +(ADR-0029), through domain ports and in-memory fakes.""" + +from test_plan_update import Harness, _files + +from docgrep.domain.update_plan import UpdateScope + + +def _scope(*paths: str) -> UpdateScope: + return UpdateScope(frozenset(paths)) + + +def test_only_the_paths_of_the_scope_change() -> None: + harness = Harness(_files(6)) + harness.build() + files = harness.corpus.files + files["doc00.txt"] = ("changed words of doc zero", 2.0) + files["doc01.txt"] = ("changed words of doc one", 2.0) + files["new_a.txt"] = ("a new document in scope", 2.0) + files["new_b.txt"] = ("a new document out of scope", 2.0) + del files["doc02.txt"] + del files["doc03.txt"] + + summary = harness.update().run( + scope=_scope("doc00.txt", "new_a.txt", "doc02.txt"), run_kind="sync" + ) + + assert (summary.added, summary.modified, summary.removed) == (1, 1, 1) + # Everything else counts as unchanged, whatever its state on disk. + assert summary.plan.unchanged == 4 + assert harness.parser.parsed == ["doc00.txt", "new_a.txt"] + paths = {r.source.path for r in harness.manifest.committed_sources()} + assert paths == { + "doc00.txt", + "doc01.txt", + "doc03.txt", + "doc04.txt", + "doc05.txt", + "new_a.txt", + } + stale = harness.manifest.get_source("doc01.txt") + assert stale is not None and stale.source.mtime == 1.0 + + +def test_a_scoped_update_is_recorded_as_a_sync_run() -> None: + harness = Harness(_files(3)) + harness.build() + harness.corpus.files["doc00.txt"] = ("changed words of doc zero", 2.0) + + summary = harness.update().run(scope=_scope("doc00.txt"), run_kind="sync") + + run = harness.manifest.latest_run() + assert run is not None and (run.kind, run.run_id) == ("sync", summary.run_id) + assert run.status == "completed" + + +def test_an_empty_scope_changes_nothing() -> None: + harness = Harness(_files(3)) + harness.build() + harness.corpus.files["extra.txt"] = ("not in scope", 2.0) + del harness.corpus.files["doc00.txt"] + + summary = harness.update().run(scope=_scope(), run_kind="sync") + + assert summary.plan.changes == 0 and summary.plan.unchanged == 3 + assert harness.parser.parsed == [] + + +def test_a_move_in_scope_reuses_the_passages_without_a_model_call() -> None: + harness = Harness(_files(4)) + harness.build() + harness.corpus.files["moved/doc00.txt"] = harness.corpus.files.pop("doc00.txt") + + summary = harness.update(reuse_by_hash=True).run( + scope=_scope("doc00.txt", "moved/doc00.txt"), run_kind="sync" + ) + + assert summary.moved == 1 and summary.failed == 0 + assert harness.parser.parsed == [] and harness.embedder.embedded == 0 + assert harness.manifest.get_source("doc00.txt") is None + assert harness.manifest.get_source("moved/doc00.txt") is not None + + +def test_scoped_removals_that_trip_the_guard_are_held_and_the_rest_applied() -> None: + harness = Harness(_files(40)) + harness.build() + files = harness.corpus.files + gone = [f"doc{i:02d}.txt" for i in range(30)] + for name in gone: + del files[name] + files["fresh.txt"] = ("a fresh document", 2.0) + + summary = harness.update().run(scope=_scope(*gone, "fresh.txt"), run_kind="sync") + + assert summary.removed == 0 and summary.added == 1 + assert [(g.root, g.removals) for g in summary.held] == [("_legacy", 30)] + assert set(summary.held_removals) == set(gone) + assert harness.manifest.get_source("fresh.txt") is not None + assert all(harness.manifest.get_source(name) is not None for name in gone) + + +def test_moves_in_scope_pass_the_guard_beside_held_deletions() -> None: + harness = Harness(_files(40)) + harness.build() + files = harness.corpus.files + moved = [f"doc{i:02d}.txt" for i in range(30)] + for name in moved: + files[f"to/{name}"] = files.pop(name) + + summary = harness.update(reuse_by_hash=True).run( + scope=_scope(*moved, *(f"to/{n}" for n in moved)), run_kind="sync" + ) + + assert summary.held == () and summary.moved == 30 From 879a467816cdcaf18e9f737c7792deada52900d1 Mon Sep 17 00:00:00 2001 From: Viktor Mikhalkin <9029022+ViktorMikhalkin@users.noreply.github.com> Date: Fri, 23 Oct 2026 21:55:11 +0300 Subject: [PATCH 3/9] feat(usecases): detect settled corpus changes by periodic scan SyncIndex scans every readable root from directory entries alone, keeps the changes it saw in memory until they have settled, and applies the settled ones with an update scoped to their paths, recorded as a `sync` run. A file is applied when its size and modification time were identical in two consecutive scans and its modification time is older than the settle time; a removal when the file was absent in two consecutive scans. Partial downloads, temporary, lock and swap files and the .obsidian, .git and .trash folders are never touched. A cycle that finds the index lock held ends and keeps its entries; a failed file is retried only when it changes or at a reconciliation. A reconciliation with the update plan runs at start-up, daily and for a root that is readable again; it settles additions and changes at once, and the targets of moves wait for the removal of their old path so that both are applied together. A root that cannot be read is paused, removals that trip the removal guard are held back (additions and changes still apply, and moves pass), a root without a completed manual update is left alone, and a configuration that drifted or an unfinished build or re-encode keeps sync idle with its reason. Story: batch 28, #45. --- src/docgrep/domain/models.py | 5 +- src/docgrep/domain/protocols.py | 19 + src/docgrep/domain/sync.py | 188 ++++++++ src/docgrep/usecases/sync_index.py | 625 +++++++++++++++++++++++++++ src/docgrep/usecases/update_index.py | 7 + tests/unit/test_sync_index.py | 602 ++++++++++++++++++++++++++ 6 files changed, 1444 insertions(+), 2 deletions(-) create mode 100644 src/docgrep/domain/sync.py create mode 100644 src/docgrep/usecases/sync_index.py create mode 100644 tests/unit/test_sync_index.py diff --git a/src/docgrep/domain/models.py b/src/docgrep/domain/models.py index 1776e4b..1ae046f 100644 --- a/src/docgrep/domain/models.py +++ b/src/docgrep/domain/models.py @@ -288,8 +288,9 @@ class ManifestRunRecord: ``swapped`` means the new index is published but the committed sources have not been replaced from the run's journal yet. ``kind`` is ``build`` for a - full build or its resume, ``update`` for an incremental update, ``ocr`` - for OCR processing and ``reencode_sparse`` for a re-encode of the sparse + full build or its resume, ``update`` for an incremental update, ``sync`` + for an update that follows the corpus by itself (ADR-0029), ``ocr`` for + OCR processing and ``reencode_sparse`` for a re-encode of the sparse vectors. """ diff --git a/src/docgrep/domain/protocols.py b/src/docgrep/domain/protocols.py index a508317..0b24b60 100644 --- a/src/docgrep/domain/protocols.py +++ b/src/docgrep/domain/protocols.py @@ -161,6 +161,25 @@ def scan_paths(self, paths: Collection[str]) -> tuple[ScannedFile, ...]: ... +class SyncCorpus(Protocol): + """The corpus as automatic sync reads it: one root at a time, so that a + root that is gone pauses itself alone (ADR-0029).""" + + def root_readable(self, name: str) -> bool: + """Whether the root's folder exists and can be listed.""" + ... + + def scan_root(self, name: str) -> tuple[ScannedFile, ...]: + """Every file of one root from directory entries alone, as + ``CorpusReader.scan`` reports them. Raises ``ValidationError`` when the + root cannot be read.""" + ... + + def describe(self, path: str) -> Source: + """The source record of one file, as ``CorpusReader.describe``.""" + ... + + class FolderReader(Protocol): """The files of a folder compared with a root, such as a backup copy. diff --git a/src/docgrep/domain/sync.py b/src/docgrep/domain/sync.py new file mode 100644 index 0000000..b55d098 --- /dev/null +++ b/src/docgrep/domain/sync.py @@ -0,0 +1,188 @@ +"""Pure rules of automatic sync: which names it never indexes, when a changed +file has settled, and the state it reports (ADR-0029).""" + +from collections.abc import Mapping +from dataclasses import dataclass +from fnmatch import fnmatchcase +from typing import Any + +# Defaults of DOCGREP_SYNC_INTERVAL_SECONDS, DOCGREP_SYNC_SETTLE_SECONDS and +# DOCGREP_SYNC_RECONCILE_HOURS. +DEFAULT_SYNC_INTERVAL_SECONDS = 120.0 +DEFAULT_SYNC_SETTLE_SECONDS = 60.0 +DEFAULT_SYNC_RECONCILE_HOURS = 24.0 +# A change that has not settled after this long is reported (and watched on). +UNSETTLED_REPORT_SECONDS = 30 * 60 +# A file or removal must be seen in this many consecutive scans. +SETTLE_SCANS = 2 + +# Names that sync never adds, changes or removes, on top of the root +# patterns: partial downloads, temporary files, lock and swap files. +IGNORED_NAMES = ("*.part", "*.crdownload", "*.tmp", "~$*", ".~lock.*#", ".#*", "*.swp") +# Folders it never enters: other tools' own data. +IGNORED_DIRS = (".obsidian", ".git", ".trash") + + +def is_ignored(path: str) -> bool: + """Whether a POSIX path (stored or root-relative) is one sync leaves + alone: its name matches ``IGNORED_NAMES`` or a folder in it is one of + ``IGNORED_DIRS``.""" + *folders, name = path.split("/") + return any(folder in IGNORED_DIRS for folder in folders) or any( + fnmatchcase(name, pattern) for pattern in IGNORED_NAMES + ) + + +@dataclass(slots=True) +class Pending: + """An added or modified file waiting to settle. + + ``scans`` counts the consecutive scans that saw the same size and + modification time. ``forced`` marks a change found by a reconciliation, + which settles without waiting for the modification time to age. + """ + + kind: str + size: int + mtime: float + scans: int = 1 + first_seen: float = 0.0 + forced: bool = False + reported: bool = False + + def observe(self, size: int, mtime: float) -> None: + """Count one more scan that saw the file; a different size or time + starts the wait again.""" + if (size, mtime) == (self.size, self.mtime): + self.scans += 1 + else: + self.size, self.mtime, self.scans, self.forced = size, mtime, 1, False + + def settled(self, wall_now: float, settle_seconds: float) -> bool: + """Whether the file may be applied: seen unchanged in two scans and, + unless a reconciliation found it, not modified for + ``settle_seconds``.""" + if self.scans < SETTLE_SCANS: + return False + return self.forced or wall_now - self.mtime >= settle_seconds + + +@dataclass(frozen=True, slots=True) +class HeldRoot: + """Removals of one root that the removal guard holds back.""" + + root: str + count: int + paths: tuple[str, ...] = () + + +@dataclass(frozen=True, slots=True) +class LastApplied: + """The latest run that sync applied.""" + + run_id: str + finished_at: str + added: int = 0 + modified: int = 0 + removed: int = 0 + moved: int = 0 + copied: int = 0 + failed: int = 0 + cancelled: bool = False + + +@dataclass(frozen=True, slots=True) +class SyncStatus: + """What sync reports about itself; paths only on request.""" + + enabled: bool + running: bool = False + interval_seconds: float = DEFAULT_SYNC_INTERVAL_SECONDS + settle_seconds: float = DEFAULT_SYNC_SETTLE_SECONDS + last_scan_at: str | None = None + last_applied: LastApplied | None = None + pending_added: int = 0 + pending_modified: int = 0 + pending_removed: int = 0 + unsettled: int = 0 + unsettled_over_limit: int = 0 + held: tuple[HeldRoot, ...] = () + paused_roots: tuple[str, ...] = () + awaiting_first_update: tuple[str, ...] = () + idle_reason: str | None = None + last_error: str | None = None + applying: tuple[int, int] | None = None + + def to_dict(self, *, paths: bool = False) -> dict[str, Any]: + """The ``sync`` section of the status bodies; ``paths`` adds a few + stored paths of each held root (``docgrep status`` only).""" + return { + "enabled": self.enabled, + "running": self.running, + "interval_seconds": self.interval_seconds, + "settle_seconds": self.settle_seconds, + "last_scan_at": self.last_scan_at, + "last_applied": None + if self.last_applied is None + else { + "run_id": self.last_applied.run_id, + "finished_at": self.last_applied.finished_at, + "added": self.last_applied.added, + "modified": self.last_applied.modified, + "removed": self.last_applied.removed, + "moved": self.last_applied.moved, + "copied": self.last_applied.copied, + "failed": self.last_applied.failed, + "cancelled": self.last_applied.cancelled, + }, + "pending": { + "added": self.pending_added, + "modified": self.pending_modified, + "removed": self.pending_removed, + }, + "unsettled": self.unsettled, + "unsettled_over_limit": self.unsettled_over_limit, + "held_removals": [ + {"root": held.root, "count": held.count} + | ({"paths": list(held.paths)} if paths else {}) + for held in self.held + ], + "paused_roots": list(self.paused_roots), + "awaiting_first_update": list(self.awaiting_first_update), + "idle_reason": self.idle_reason, + "last_error": self.last_error, + "applying": None + if self.applying is None + else {"done": self.applying[0], "total": self.applying[1]}, + } + + @classmethod + def from_dict(cls, body: Mapping[str, Any]) -> "SyncStatus": + """The status written by ``to_dict`` (with paths).""" + applied = body.get("last_applied") + pending = body.get("pending") or {} + applying = body.get("applying") + return cls( + enabled=bool(body["enabled"]), + running=bool(body.get("running", False)), + interval_seconds=float(body["interval_seconds"]), + settle_seconds=float(body["settle_seconds"]), + last_scan_at=body.get("last_scan_at"), + last_applied=None if applied is None else LastApplied(**applied), + pending_added=int(pending.get("added", 0)), + pending_modified=int(pending.get("modified", 0)), + pending_removed=int(pending.get("removed", 0)), + unsettled=int(body.get("unsettled", 0)), + unsettled_over_limit=int(body.get("unsettled_over_limit", 0)), + held=tuple( + HeldRoot(h["root"], int(h["count"]), tuple(h.get("paths", ()))) + for h in body.get("held_removals", ()) + ), + paused_roots=tuple(body.get("paused_roots", ())), + awaiting_first_update=tuple(body.get("awaiting_first_update", ())), + idle_reason=body.get("idle_reason"), + last_error=body.get("last_error"), + applying=None + if applying is None + else (int(applying["done"]), int(applying["total"])), + ) diff --git a/src/docgrep/usecases/sync_index.py b/src/docgrep/usecases/sync_index.py new file mode 100644 index 0000000..e127430 --- /dev/null +++ b/src/docgrep/usecases/sync_index.py @@ -0,0 +1,625 @@ +"""Follow the configured roots by periodic scan and apply what has settled, +safely enough to run unattended (ADR-0029). + +One cycle scans every readable root from directory entries alone, keeps the +changes it saw in memory until they have settled, and then applies the settled +ones with an update scoped to their paths. A root that cannot be read is +paused, removals are applied only when the removal guard allows them, and +nothing is written while another operation holds the index lock. +""" + +import json +import logging +import threading +import time +from collections.abc import Callable, Iterable +from dataclasses import dataclass, field +from datetime import UTC, datetime +from typing import Protocol + +from docgrep.domain.errors import DocgrepError, IndexLockHeld, ValidationError +from docgrep.domain.models import ManifestSourceRecord, ScannedFile, Source +from docgrep.domain.progress import ProgressEvent, ProgressOptions +from docgrep.domain.protocols import ( + CorpusReader, + ManifestReader, + RootCatalog, + SyncCorpus, +) +from docgrep.domain.roots import RUN_ROOTS_KEY, CorpusRoots +from docgrep.domain.sync import ( + DEFAULT_SYNC_INTERVAL_SECONDS, + DEFAULT_SYNC_RECONCILE_HOURS, + DEFAULT_SYNC_SETTLE_SECONDS, + SETTLE_SCANS, + UNSETTLED_REPORT_SECONDS, + HeldRoot, + LastApplied, + Pending, + SyncStatus, + is_ignored, +) +from docgrep.domain.update_plan import UpdateScope, pair_moves +from docgrep.usecases.plan_update import DetectedChanges, PlanUpdate +from docgrep.usecases.update_index import UpdateIndex, UpdateSummary + +_LOG = logging.getLogger(__name__) + +# How many stored paths of a held root the status names. +SAMPLE_PATHS = 5 +# The pause after each source of an applied run, which lets a search that is +# waiting for the models go first. +YIELD_SECONDS = 0.05 + + +class ScopedUpdates(Protocol): + """The update sync applies: scoped to paths, recorded as a ``sync`` run.""" + + def check_ready(self) -> None: + """Raise a ``DocgrepError`` when no update can run now.""" + ... + + def apply(self, scope: UpdateScope, progress: ProgressOptions) -> UpdateSummary: + """Apply the scope; raises ``IndexLockHeld`` without waiting when + another operation owns the index lock.""" + ... + + def request_cancel(self) -> None: + """Stop after the source being processed.""" + ... + + +class UpdateApplier: + """``ScopedUpdates`` over the update use case of the application.""" + + def __init__(self, update: UpdateIndex) -> None: + self._update = update + + def check_ready(self) -> None: + self._update.check_ready() + + def apply(self, scope: UpdateScope, progress: ProgressOptions) -> UpdateSummary: + return self._update.run(scope=scope, run_kind="sync", progress=progress) + + def request_cancel(self) -> None: + self._update.request_cancel() + + +class _ReconcileView: + """The readable roots as a corpus an update plan can read: the files sync + does not leave alone, root by root.""" + + def __init__(self, corpus: SyncCorpus, names: Iterable[str]) -> None: + self._corpus = corpus + self._names = tuple(names) + + def scan(self) -> tuple[ScannedFile, ...]: + return tuple( + item + for name in self._names + for item in self._corpus.scan_root(name) + if not is_ignored(item.path) + ) + + def describe(self, path: str) -> Source: + return self._corpus.describe(path) + + +@dataclass(frozen=True, slots=True) +class CycleResult: + """What one cycle did, in counts.""" + + scanned_roots: int = 0 + applied: UpdateSummary | None = None + skipped: str | None = None + held: int = 0 + pending: int = 0 + + +@dataclass(slots=True) +class _State: + pending: dict[str, Pending] = field(default_factory=dict) + # Consecutive scans that did not see a recorded path. + absent: dict[str, int] = field(default_factory=dict) + # Removals the guard held back, by root. + held: dict[str, set[str]] = field(default_factory=dict) + # The size and time a path had when it was last applied, so that a file + # that failed is retried only when it changes. + attempted: dict[str, tuple[int, float]] = field(default_factory=dict) + paused: set[str] = field(default_factory=set) + awaiting: set[str] = field(default_factory=set) + started: bool = False + last_reconcile: float | None = None + last_scan_at: str | None = None + last_applied: LastApplied | None = None + idle_reason: str | None = None + last_error: str | None = None + applying: tuple[int, int] | None = None + + +class SyncIndex: + """Keep the index following the configured roots (ADR-0029). + + ``run_cycle`` does one scan and, when something has settled, one scoped + update; the caller decides when. All state is in memory and is rebuilt by + the reconciliation of the first cycle, so a restart loses nothing. + """ + + def __init__( + self, + *, + corpus: SyncCorpus, + roots: RootCatalog, + manifest: ManifestReader, + updates: ScopedUpdates, + planner: Callable[[CorpusReader], PlanUpdate], + interval_seconds: float = DEFAULT_SYNC_INTERVAL_SECONDS, + settle_seconds: float = DEFAULT_SYNC_SETTLE_SECONDS, + reconcile_hours: float = DEFAULT_SYNC_RECONCILE_HOURS, + clock: Callable[[], float] = time.monotonic, + wall_clock: Callable[[], float] = time.time, + now: Callable[[], datetime] = lambda: datetime.now(UTC), + sleep: Callable[[float], None] = time.sleep, + yield_seconds: float = YIELD_SECONDS, + on_change: Callable[[SyncStatus], None] | None = None, + ) -> None: + """``planner`` builds the update plan over a corpus view, for the + reconciliation. ``on_change`` is called with the status after every + cycle and when an applied run starts or ends.""" + if interval_seconds <= 0: + raise ValueError("interval_seconds must be positive") + if settle_seconds < 0: + raise ValueError("settle_seconds must not be negative") + if reconcile_hours <= 0: + raise ValueError("reconcile_hours must be positive") + self.corpus = corpus + self.roots = roots + self.manifest = manifest + self.updates = updates + self.planner = planner + self.interval_seconds = interval_seconds + self.settle_seconds = settle_seconds + self.reconcile_seconds = reconcile_hours * 3600 + self.clock = clock + self.wall_clock = wall_clock + self.now = now + self.sleep = sleep + self.yield_seconds = yield_seconds + self.on_change = on_change + self._state = _State() + self._guard = threading.Lock() + self._stopping = threading.Event() + + # -- control --------------------------------------------------------- + + def request_stop(self) -> None: + """Start no new cycle, and stop an applied run after its current + source.""" + self._stopping.set() + self.updates.request_cancel() + + @property + def stopping(self) -> bool: + return self._stopping.is_set() + + # -- status ---------------------------------------------------------- + + def status(self, *, running: bool = True) -> SyncStatus: + """The state sync reports; a copy, safe to call during a cycle.""" + with self._guard: + state = self._state + kinds = [entry.kind for entry in state.pending.values()] + loose = [p for p in state.absent if not self._is_held(p)] + wall = self.wall_clock() + mono = self.clock() + waiting = [ + entry + for entry in state.pending.values() + if not entry.settled(wall, self.settle_seconds) + ] + return SyncStatus( + enabled=True, + running=running, + interval_seconds=self.interval_seconds, + settle_seconds=self.settle_seconds, + last_scan_at=state.last_scan_at, + last_applied=state.last_applied, + pending_added=kinds.count("added"), + pending_modified=kinds.count("modified"), + pending_removed=len(loose), + unsettled=len(waiting) + + sum(1 for p in loose if state.absent[p] < SETTLE_SCANS), + unsettled_over_limit=sum( + 1 + for entry in waiting + if mono - entry.first_seen >= UNSETTLED_REPORT_SECONDS + ), + held=tuple( + HeldRoot(root, len(paths), tuple(sorted(paths))[:SAMPLE_PATHS]) + for root, paths in sorted(state.held.items()) + if paths + ), + paused_roots=tuple(sorted(state.paused)), + awaiting_first_update=tuple(sorted(state.awaiting)), + idle_reason=state.idle_reason, + last_error=state.last_error, + applying=state.applying, + ) + + def _is_held(self, path: str) -> bool: + return any(path in paths for paths in self._state.held.values()) + + def _publish(self) -> None: + if self.on_change is not None: + try: + self.on_change(self.status()) + except Exception: + _LOG.warning("sync_status_not_published", exc_info=True) + + # -- one cycle ------------------------------------------------------- + + def run_cycle(self) -> CycleResult: + """Scan, settle and apply once; never raises for a corpus or an + index that is not ready, only reports why in the status.""" + if self._stopping.is_set(): + return CycleResult(skipped="stopping") + started = self.clock() + with self._guard: + self._state.last_scan_at = self.now().isoformat() + result = self._cycle() + _LOG.info( + "sync_cycle", + extra={ + "scanned_roots": result.scanned_roots, + "skipped": result.skipped or "", + "pending": result.pending, + "held_removals": result.held, + "applied": result.applied is not None, + **_counts(result.applied), + "duration_ms": round((self.clock() - started) * 1000, 1), + }, + ) + self._publish() + return result + + def _cycle(self) -> CycleResult: + try: + self.updates.check_ready() + except DocgrepError as error: + with self._guard: + self._state.idle_reason = f"{error.category}: {error}" + return CycleResult(skipped="not_ready") + with self._guard: + self._state.idle_reason = None + + roots = self.roots.roots() + committed = { + record.source.path: record for record in self.manifest.committed_sources() + } + awaiting = self._awaiting(roots, committed) + scanned, paused = self._scan(roots, awaiting) + with self._guard: + resumed = self._state.paused - paused + self._state.paused = paused + self._state.awaiting = awaiting + self._update_pending(roots, committed, scanned, paused | awaiting) + self._reconcile_if_due(roots, scanned, resumed) + return self._apply(roots, scanned) + + def _awaiting( + self, roots: CorpusRoots, committed: dict[str, ManifestSourceRecord] + ) -> set[str]: + """The roots whose first manual update has not completed: they have + no recorded source and no completed build or update visited them.""" + with_sources = {roots.root_of(path) for path in committed} + visited: set[str] = set() + for kind in ("build", "update"): + run = self.manifest.latest_run_of_kind(kind) + if run is not None and run.status == "completed": + visited |= _visited_roots(dict(run.config).get(RUN_ROOTS_KEY)) + return {name for name in roots.names if name not in with_sources | visited} + + def _scan( + self, roots: CorpusRoots, skip: set[str] + ) -> tuple[dict[str, dict[str, ScannedFile]], set[str]]: + """The files sync looks at in each readable root, and the roots it + pauses because they cannot be read.""" + scanned: dict[str, dict[str, ScannedFile]] = {} + paused: set[str] = set() + for name in roots.names: + if name in skip: + continue + try: + if not self.corpus.root_readable(name): + paused.add(name) + continue + files = self.corpus.scan_root(name) + except (ValidationError, OSError): + paused.add(name) + continue + scanned[name] = { + item.path: item + for item in files + if item.excluded_by is None and not is_ignored(item.path) + } + return scanned, paused + + def _update_pending( + self, + roots: CorpusRoots, + committed: dict[str, ManifestSourceRecord], + scanned: dict[str, dict[str, ScannedFile]], + inactive: set[str], + ) -> None: + """Fold one scan into the pending changes and absent counts.""" + state = self._state + mono = self.clock() + for table in (state.pending, state.absent, state.attempted): + for path in [p for p in table if roots.root_of(p) in inactive]: + del table[path] + state.held = { + root: paths for root, paths in state.held.items() if root not in inactive + } + for name, files in scanned.items(): + for path, item in files.items(): + state.absent.pop(path, None) + record = committed.get(path) + stat = (item.size, item.mtime) + entry = state.pending.get(path) + if record is not None and stat == ( + record.source.size, + record.source.mtime, + ): + # Unchanged on disk; only a reconciliation keeps it. + if entry is None or not entry.forced: + state.pending.pop(path, None) + state.attempted.pop(path, None) + continue + if state.attempted.get(path) == stat: + state.pending.pop(path, None) + continue + kind = "added" if record is None else "modified" + mtime = item.mtime if item.mtime is not None else 0.0 + if entry is None: + state.pending[path] = Pending( + kind, item.size, mtime, first_seen=mono + ) + else: + entry.kind = kind + entry.observe(item.size, mtime) + for path in [ + p for p in state.pending if roots.root_of(p) == name and p not in files + ]: + del state.pending[path] + for path, record in committed.items(): + root = roots.root_of(path) + if root not in scanned or is_ignored(path): + continue + if path not in scanned[root]: + state.absent[path] = state.absent.get(path, 0) + 1 + for path in [p for p in state.absent if p not in committed]: + del state.absent[path] + for root, paths in state.held.items(): + paths &= {p for p in paths if p in state.absent} + + # -- reconciliation -------------------------------------------------- + + def _reconcile_if_due( + self, + roots: CorpusRoots, + scanned: dict[str, dict[str, ScannedFile]], + resumed: set[str], + ) -> None: + """Compare whole roots with the manifest with the update plan: at + start-up, daily, and for a root that is readable again.""" + mono = self.clock() + state = self._state + everything = ( + not state.started + or state.last_reconcile is None + or mono - state.last_reconcile >= self.reconcile_seconds + ) + names = set(scanned) if everything else resumed & set(scanned) + if not names: + return + try: + view = _ReconcileView(self.corpus, sorted(names)) + detected = self.planner(view).detect(True, roots.only(names)) + except (DocgrepError, OSError) as error: + _LOG.warning( + "sync_reconcile_failed", + extra={"category": getattr(error, "category", "os_error")}, + ) + return + with self._guard: + if everything: + state.started = True + state.last_reconcile = mono + state.attempted.clear() + self._merge_plan(detected, mono) + + def _merge_plan(self, detected: DetectedChanges, mono: float) -> None: + """Mark what a reconciliation found as settled, except the targets of + moves, which wait for the removal of their old path so that both are + applied together.""" + state = self._state + removed = [c.source for c in detected.changes if c.kind == "removed"] + added = [c.source for c in detected.changes if c.kind == "added"] + delayed = {m.added for m in pair_moves(removed, added)} + delayed |= {c.source.path for c in detected.changes if c.kind == "moved"} + for change in detected.changes: + path = change.source.path + if change.kind in ("removed", "moved") or path in delayed: + continue + kind = "added" if change.kind in ("added", "copied") else "modified" + entry = state.pending.get(path) + if entry is None: + entry = Pending( + kind, + change.source.size or 0, + change.source.mtime or 0.0, + first_seen=mono, + ) + state.pending[path] = entry + entry.forced = True + entry.scans = max(entry.scans, SETTLE_SCANS) + + # -- applying -------------------------------------------------------- + + def _apply( + self, roots: CorpusRoots, scanned: dict[str, dict[str, ScannedFile]] + ) -> CycleResult: + mono, wall = self.clock(), self.wall_clock() + state = self._state + with self._guard: + files = sorted( + path + for path, entry in state.pending.items() + if entry.settled(wall, self.settle_seconds) + ) + removals = sorted( + path + for path, count in state.absent.items() + if count >= SETTLE_SCANS and not self._is_held(path) + ) + resubmit = { + path + for root in {roots.root_of(path) for path in removals} + for path in state.held.get(root, ()) + } + self._report_unsettled(mono, wall) + held_total = sum(len(paths) for paths in state.held.values()) + pending_total = len(state.pending) + len(state.absent) + base = CycleResult( + scanned_roots=len(scanned), held=held_total, pending=pending_total + ) + if not files and not removals: + return base + scope = UpdateScope(frozenset(files) | frozenset(removals) | resubmit) + try: + summary = self.updates.apply(scope, ProgressOptions(self._on_progress)) + except IndexLockHeld: + return CycleResult( + scanned_roots=len(scanned), + skipped="index_lock_held", + held=held_total, + pending=pending_total, + ) + except Exception as error: + category = ( + error.category if isinstance(error, DocgrepError) else "unexpected" + ) + _LOG.warning("sync_apply_failed", extra={"category": category}) + with self._guard: + state.applying = None + state.last_error = f"{category}: {error}" + return CycleResult( + scanned_roots=len(scanned), + skipped="apply_failed", + held=held_total, + pending=pending_total, + ) + with self._guard: + self._record_applied(roots, scope, summary) + held_now = sum(len(paths) for paths in state.held.values()) + return CycleResult( + scanned_roots=len(scanned), + applied=summary, + held=held_now, + pending=len(state.pending) + len(state.absent), + ) + + def _record_applied( + self, roots: CorpusRoots, scope: UpdateScope, summary: UpdateSummary + ) -> None: + state = self._state + held = set(summary.held_removals) + for path in scope.paths: + entry = state.pending.pop(path, None) + if entry is not None: + state.attempted[path] = (entry.size, entry.mtime) + if path not in held: + state.absent.pop(path, None) + touched = {roots.root_of(path) for path in scope.paths} + for root in touched: + state.held.pop(root, None) + for path in held: + state.held.setdefault(roots.root_of(path), set()).add(path) + state.applying = None + state.last_error = None + state.last_applied = LastApplied( + run_id=summary.run_id, + finished_at=self.now().isoformat(), + added=summary.added, + modified=summary.modified, + removed=summary.removed, + moved=summary.moved, + copied=summary.copied, + failed=summary.failed, + cancelled=summary.cancelled, + ) + + def _report_unsettled(self, mono: float, wall: float) -> None: + """Log once, with counts, the changes that never settle.""" + late = [ + entry + for entry in self._state.pending.values() + if not entry.settled(wall, self.settle_seconds) + and not entry.reported + and mono - entry.first_seen >= UNSETTLED_REPORT_SECONDS + ] + for entry in late: + entry.reported = True + if late: + _LOG.warning("sync_unsettled", extra={"files": len(late)}) + + def _on_progress(self, event: ProgressEvent) -> None: + """Note how far an applied run has got, and give a waiting search the + models between two sources.""" + with self._guard: + self._state.applying = ( + None if event.finished else (event.done, event.total or 0) + ) + if event.phase == "writing" and self.yield_seconds > 0: + self.sleep(self.yield_seconds) + if event.done == 0 and not event.finished and event.label is None: + self._publish() + + +def _counts(summary: UpdateSummary | None) -> dict[str, int]: + if summary is None: + return {} + return { + "added": summary.added, + "modified": summary.modified, + "removed": summary.removed, + "moved": summary.moved, + "copied": summary.copied, + "failed": summary.failed, + "held": len(summary.held_removals), + } + + +def _visited_roots(value: object) -> set[str]: + """The root names a run recorded as visited.""" + if not isinstance(value, str): + return set() + try: + entries = json.loads(value) + except ValueError: + return set() + if not isinstance(entries, list): + return set() + return { + entry["name"] + for entry in entries + if isinstance(entry, dict) and isinstance(entry.get("name"), str) + } + + +__all__ = [ + "CycleResult", + "ScopedUpdates", + "SyncIndex", + "UpdateApplier", +] diff --git a/src/docgrep/usecases/update_index.py b/src/docgrep/usecases/update_index.py index 56e735e..f898993 100644 --- a/src/docgrep/usecases/update_index.py +++ b/src/docgrep/usecases/update_index.py @@ -300,6 +300,13 @@ def _hold_guarded( ) return replace(detected, changes=kept), guarded, held + def check_ready(self) -> None: + """Raise what ``run`` would refuse before changing anything for a + reason other than the corpus: an unfinished build or re-encode, an + index that is not complete, or settings that drifted from it. Reads + only and takes no lock.""" + self._check_preconditions() + def forget_plan(self, name: str) -> tuple[ManifestSourceRecord, ...]: """The committed sources of a root, which ``forget_root`` removes. diff --git a/tests/unit/test_sync_index.py b/tests/unit/test_sync_index.py new file mode 100644 index 0000000..d3adee9 --- /dev/null +++ b/tests/unit/test_sync_index.py @@ -0,0 +1,602 @@ +"""Automatic sync: scan, settle and apply with an update scoped to the settled +paths (ADR-0029), through domain ports and in-memory fakes. The clock is +injected and a cycle is run explicitly; nothing sleeps.""" + +from collections.abc import Iterator +from contextlib import contextmanager +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import pytest +from test_plan_update import Harness, _files + +from docgrep.domain.errors import ConfigDrift, IndexLockHeld, ValidationError +from docgrep.domain.models import ScannedFile, Source +from docgrep.domain.roots import CorpusRoot, CorpusRoots +from docgrep.domain.sync import IGNORED_DIRS, IGNORED_NAMES, is_ignored +from docgrep.domain.update_plan import RemovalGuard +from docgrep.usecases.plan_update import PlanUpdate +from docgrep.usecases.sync_index import CycleResult, SyncIndex, UpdateApplier + +ROOT = CorpusRoots.single(Path("/corpus"), "docs") +WALL = 1_000_000.0 + + +class Clock: + def __init__(self) -> None: + self.mono = 0.0 + self.wall = WALL + + def advance(self, seconds: float) -> None: + self.mono += seconds + self.wall += seconds + + +class Catalog: + def __init__(self, roots: CorpusRoots = ROOT) -> None: + self._roots = roots + + def roots(self) -> CorpusRoots: + return self._roots + + +class FakeSyncCorpus: + """The files of a harness corpus, root by root.""" + + def __init__(self, harness: Harness, roots: CorpusRoots = ROOT) -> None: + self.harness = harness + self.roots = roots + self.unreadable: set[str] = set() + self.scans = 0 + + def root_readable(self, name: str) -> bool: + return name not in self.unreadable + + def scan_root(self, name: str) -> tuple[ScannedFile, ...]: + if name in self.unreadable: + raise ValidationError("corpus is not accessible") + self.scans += 1 + return tuple( + item + for item in self.harness.corpus.scan() + if self.roots.root_of(item.path) == name + ) + + def describe(self, path: str) -> Source: + return self.harness.corpus.describe(path) + + +class HeldLock: + """An index lock that another operation may hold.""" + + def __init__(self) -> None: + self.held = False + + @contextmanager + def hold(self, operation: str = "build") -> Iterator[None]: + if self.held: + raise IndexLockHeld("Another index operation is running") + yield + + def is_held(self) -> bool: + return self.held + + +class Rig: + def __init__( + self, + files: dict[str, tuple[str, float]] | None = None, + *, + reuse: bool = True, + guard: RemovalGuard | None = None, + settle: float = 60.0, + roots: CorpusRoots = ROOT, + ) -> None: + self.harness = Harness(files if files is not None else _files(6)) + self.harness.build() + self.lock = HeldLock() + self.harness.lock = self.lock # type: ignore[assignment] # test double + self.clock = Clock() + self.corpus = FakeSyncCorpus(self.harness, roots) + self.catalog = Catalog(roots) + self.update = self.harness.update( + roots=self.catalog, + reuse_by_hash=reuse, + guard=guard or RemovalGuard(), + ) + self.slept: list[float] = [] + self.statuses: list[Any] = [] + self.sync = SyncIndex( + corpus=self.corpus, + roots=self.catalog, + manifest=self.harness.manifest, + updates=UpdateApplier(self.update), + planner=lambda view: PlanUpdate( + corpus=view, + manifest=self.harness.manifest, + parsers=self.harness.parsers, + reuse_by_hash=reuse, + embedding_model=self.harness.embedder.model_name, + guard=guard or RemovalGuard(), + ), + interval_seconds=120, + settle_seconds=settle, + clock=lambda: self.clock.mono, + wall_clock=lambda: self.clock.wall, + now=lambda: datetime(2026, 10, 23, tzinfo=UTC), + sleep=self.slept.append, + on_change=self.statuses.append, + ) + + @property + def files(self) -> dict[str, tuple[str, float]]: + return self.harness.corpus.files + + def cycle(self, advance: float = 120) -> CycleResult: + self.clock.advance(advance) + return self.sync.run_cycle() + + def start(self) -> None: + """The first cycle reconciles; run it on an unchanged corpus.""" + assert self.cycle().applied is None + + def recorded(self) -> set[str]: + return {r.source.path for r in self.harness.manifest.committed_sources()} + + def runs(self, kind: str) -> int: + return sum(1 for run in self.harness.manifest._runs if run.kind == kind) + + def aged(self, text: str) -> tuple[str, float]: + """A file last written long before the settle time.""" + return text, self.clock.wall - 3600 + + +def test_the_ignore_list_matches_names_and_folders() -> None: + for name in ("a.part", "b.crdownload", "c.tmp", "~$d.docx", ".~lock.e.odt#"): + assert is_ignored(name) and is_ignored(f"sub/{name}") + assert is_ignored(".#f.txt") and is_ignored("g.swp") + for folder in IGNORED_DIRS: + assert is_ignored(f"books/{folder}/notes.md") + assert not is_ignored("notes.md") and not is_ignored("obsidian/notes.md") + assert len(IGNORED_NAMES) == 7 + + +def test_a_changed_file_is_applied_when_seen_unchanged_in_two_scans() -> None: + rig = Rig() + rig.start() + rig.files["new.txt"] = rig.aged("a brand new document") + rig.files["doc01.txt"] = rig.aged("rewritten words of doc one") + + first = rig.cycle() + assert first.applied is None and rig.recorded() == set(_files(6)) + status = rig.sync.status() + assert (status.pending_added, status.pending_modified, status.unsettled) == ( + 1, + 1, + 2, + ) + + second = rig.cycle() + + assert second.applied is not None + assert (second.applied.added, second.applied.modified) == (1, 1) + assert "new.txt" in rig.recorded() + assert rig.sync.status().pending_added == 0 + assert rig.runs("sync") == 1 and rig.runs("update") == 0 + + +def test_a_file_growing_across_scans_is_not_applied_until_stable() -> None: + rig = Rig() + rig.start() + for step in range(1, 4): + rig.files["big.txt"] = rig.aged("growing " * step) + assert rig.cycle().applied is None + assert "big.txt" not in rig.recorded() + + assert rig.cycle().applied is not None + assert "big.txt" in rig.recorded() + + +def test_a_file_written_a_moment_ago_waits_for_its_time_to_age() -> None: + rig = Rig(settle=60) + rig.start() + rig.files["fresh.txt"] = ("just written", rig.clock.wall - 30) + + # Seen twice, but modified less than 60 seconds before the second scan. + assert rig.cycle(advance=10).applied is None + assert rig.cycle(advance=10).applied is None + assert rig.cycle(advance=90).applied is not None + + +def test_a_file_deleted_before_settling_is_dropped() -> None: + rig = Rig() + rig.start() + rig.files["tmp.txt"] = rig.aged("here for a moment") + rig.cycle() + del rig.files["tmp.txt"] + + assert rig.cycle().applied is None + status = rig.sync.status() + assert (status.pending_added, status.unsettled) == (0, 0) + assert rig.runs("sync") == 0 + + +def test_ignored_names_are_never_applied_or_removed() -> None: + rig = Rig({**_files(6), ".obsidian/n.txt": ("obsidian", 1.0)}) + # Recorded before the ignore list existed; sync leaves it alone. + assert ".obsidian/n.txt" in rig.recorded() + rig.start() + for name in ("movie.part", "doc.tmp", "~$x.txt", "a.swp", ".git/c.txt"): + rig.files[name] = rig.aged("partial " + name) + rig.files["sub/.trash/old.txt"] = rig.aged("trash") + del rig.files[".obsidian/n.txt"] + + for _ in range(4): + assert rig.cycle().applied is None + + assert ".obsidian/n.txt" in rig.recorded() + assert rig.sync.status().pending_added == 0 and rig.runs("sync") == 0 + + +def test_a_removal_needs_two_scans_without_the_file() -> None: + rig = Rig() + rig.start() + del rig.files["doc02.txt"] + + assert rig.cycle().applied is None + assert rig.sync.status().pending_removed == 1 + assert "doc02.txt" in rig.recorded() + + result = rig.cycle() + + assert result.applied is not None and result.applied.removed == 1 + assert "doc02.txt" not in rig.recorded() + + +def test_a_file_that_comes_back_cancels_its_removal() -> None: + rig = Rig() + rig.start() + saved = rig.files.pop("doc02.txt") + rig.cycle() + rig.files["doc02.txt"] = saved + + assert rig.cycle().applied is None + assert rig.sync.status().pending_removed == 0 + assert "doc02.txt" in rig.recorded() and rig.runs("sync") == 0 + + +def test_a_held_lock_ends_the_cycle_and_keeps_the_entries() -> None: + rig = Rig() + rig.start() + rig.files["new.txt"] = rig.aged("waiting for the lock") + del rig.files["doc03.txt"] + rig.cycle() + rig.lock.held = True + + result = rig.cycle() + + assert result.skipped == "index_lock_held" and result.applied is None + assert rig.runs("sync") == 0 and "new.txt" not in rig.recorded() + status = rig.sync.status() + assert (status.pending_added, status.pending_removed) == (1, 1) + + rig.lock.held = False + result = rig.cycle() + + assert result.applied is not None + assert (result.applied.added, result.applied.removed) == (1, 1) + + +def test_a_failed_file_is_retried_only_when_it_changes_or_at_reconciliation() -> None: + rig = Rig() + rig.start() + rig.harness.parser.failing.add("bad.txt") + rig.files["bad.txt"] = rig.aged("this one fails to parse") + rig.cycle() + failed = rig.cycle() + assert failed.applied is not None and failed.applied.failed == 1 + attempts = rig.harness.parser.parsed.count("bad.txt") + + for _ in range(5): + assert rig.cycle().applied is None + assert rig.harness.parser.parsed.count("bad.txt") == attempts + + # A change of the file starts it again. + rig.files["bad.txt"] = rig.aged("this one still fails to parse, longer") + rig.cycle() + assert rig.cycle().applied is not None + assert rig.harness.parser.parsed.count("bad.txt") == attempts + 1 + + # The daily reconciliation retries it once more. + rig.harness.parser.failing.clear() + result = rig.cycle(advance=24 * 3600) + assert result.applied is not None and result.applied.failed == 0 + record = rig.harness.manifest.get_source("bad.txt") + assert record is not None and record.status == "indexed" + + +def test_start_up_reconciliation_catches_a_change_made_while_stopped() -> None: + rig = Rig() + # Written moments ago, so the settle time has not passed; and a deletion. + rig.files["doc00.txt"] = ("edited while docgrep was stopped", rig.clock.wall) + rig.files["added.txt"] = ("added while docgrep was stopped", rig.clock.wall) + del rig.files["doc05.txt"] + + first = rig.cycle() + + # Additions and changes settle at once; the removal waits for a second scan. + assert first.applied is not None + assert (first.applied.added, first.applied.modified, first.applied.removed) == ( + 1, + 1, + 0, + ) + second = rig.cycle() + assert second.applied is not None and second.applied.removed == 1 + + +def test_a_moved_file_reuses_its_passages_without_a_model_call() -> None: + rig = Rig() + rig.start() + rig.files["moved/doc00.txt"] = rig.files.pop("doc00.txt") + before = rig.harness.embedder.embedded + + rig.cycle() + result = rig.cycle() + + assert result.applied is not None and result.applied.moved == 1 + assert rig.harness.embedder.embedded == before + assert rig.harness.parser.parsed == [] + assert "doc00.txt" not in rig.recorded() and "moved/doc00.txt" in rig.recorded() + + +def test_emptying_a_root_holds_its_removals_and_applies_additions() -> None: + rig = Rig(_files(40)) + rig.start() + for path in list(rig.files): + del rig.files[path] + rig.files["fresh.txt"] = rig.aged("a fresh document") + rig.cycle() + + result = rig.cycle() + + assert result.applied is not None + assert (result.applied.added, result.applied.removed) == (1, 0) + assert len(result.applied.held_removals) == 40 + assert len(rig.recorded()) == 41 + status = rig.sync.status() + assert [(h.root, h.count) for h in status.held] == [("docs", 40)] + assert len(status.held[0].paths) == 5 and status.pending_removed == 0 + # Nothing more is applied while they stay held. + runs = rig.runs("sync") + for _ in range(3): + assert rig.cycle().applied is None + assert rig.runs("sync") == runs + + +def test_held_removals_are_applied_by_a_manual_update_with_yes() -> None: + rig = Rig(_files(40)) + rig.start() + rig.files.clear() + rig.cycle() + rig.cycle() + assert len(rig.sync.status().held[0].paths) == 5 + + summary = rig.update.run(override_guard=True) + + assert summary.removed == 40 + assert rig.cycle().applied is None + status = rig.sync.status() + assert status.held == () and status.pending_removed == 0 + + +def test_files_that_reappear_drop_their_held_removals() -> None: + rig = Rig(_files(40)) + rig.start() + saved = dict(rig.files) + rig.files.clear() + rig.cycle() + rig.cycle() + assert rig.sync.status().held + + rig.files.update(saved) + rig.cycle() + + status = rig.sync.status() + assert status.held == () and status.pending_removed == 0 + assert len(rig.recorded()) == 40 + + +def test_removals_beyond_the_threshold_in_later_cycles_stay_together() -> None: + rig = Rig(_files(60)) + rig.start() + for index in range(10): + del rig.files[f"doc{index:02d}.txt"] + rig.cycle() + rig.cycle() + assert len(rig.sync.status().held[0].paths) == 5 + held = rig.sync.status().held[0].count + assert held == 10 + + # One more deletion does not slip under the threshold on its own. + del rig.files["doc50.txt"] + rig.cycle() + result = rig.cycle() + + assert result.applied is not None and result.applied.removed == 0 + assert rig.sync.status().held[0].count == 11 + + +def test_moving_more_files_than_the_threshold_passes_the_guard() -> None: + rig = Rig(_files(40)) + rig.start() + for path in list(rig.files): + rig.files[f"archive/{path}"] = rig.files.pop(path) + rig.cycle() + + result = rig.cycle() + + assert result.applied is not None + assert (result.applied.moved, result.applied.held_removals) == (40, ()) + assert rig.recorded() == {f"archive/{p}" for p in _files(40)} + assert rig.sync.status().held == () + + +def test_the_same_number_of_deletions_without_copies_is_held() -> None: + rig = Rig(_files(40)) + rig.start() + rig.files.clear() + rig.cycle() + + result = rig.cycle() + + assert result.applied is not None and len(result.applied.held_removals) == 40 + assert len(rig.recorded()) == 40 + + +def test_moves_start_up_together_with_the_removal_of_their_old_path() -> None: + rig = Rig(_files(40)) + for path in list(rig.files): + rig.files[f"archive/{path}"] = rig.files.pop(path) + + first = rig.cycle() + # The targets wait for the second scan, which confirms the removals. + assert first.applied is None + + second = rig.cycle() + + assert second.applied is not None + assert (second.applied.moved, second.applied.held_removals) == (40, ()) + + +def test_sync_stays_idle_and_says_why_when_the_configuration_drifted() -> None: + rig = Rig() + rig.start() + + def drifted() -> None: + raise ConfigDrift("The configuration differs from the index") + + rig.update.check_ready = drifted # type: ignore[method-assign] # test double + rig.files["new.txt"] = rig.aged("never indexed") + scans = rig.corpus.scans + + result = rig.cycle() + + assert result.skipped == "not_ready" and rig.corpus.scans == scans + status = rig.sync.status() + assert status.idle_reason is not None + assert status.idle_reason.startswith("config_drift: ") + assert rig.runs("sync") == 0 + + +def test_a_run_gives_way_between_sources_and_reports_progress() -> None: + rig = Rig() + rig.start() + for index in range(3): + rig.files[f"new{index}.txt"] = rig.aged(f"new document number {index}") + rig.cycle() + + result = rig.cycle() + + assert result.applied is not None and result.applied.added == 3 + assert len(rig.slept) >= 3 and set(rig.slept) == {0.05} + assert rig.sync.status().applying is None + + +def test_stopping_starts_no_new_cycle_and_cancels_a_run() -> None: + rig = Rig() + rig.start() + rig.files["new.txt"] = rig.aged("after the stop") + rig.cycle() + + rig.sync.request_stop() + + assert rig.sync.stopping + result = rig.cycle() + assert result.skipped == "stopping" and "new.txt" not in rig.recorded() + + +def test_the_status_lists_its_sections_without_paths_unless_asked() -> None: + rig = Rig(_files(40)) + rig.start() + rig.files.clear() + rig.cycle() + rig.cycle() + + status = rig.sync.status() + body = status.to_dict() + with_paths = status.to_dict(paths=True) + + assert set(body) == { + "enabled", + "running", + "interval_seconds", + "settle_seconds", + "last_scan_at", + "last_applied", + "pending", + "unsettled", + "unsettled_over_limit", + "held_removals", + "paused_roots", + "awaiting_first_update", + "idle_reason", + "last_error", + "applying", + } + assert body["held_removals"] == [{"root": "docs", "count": 40}] + assert len(with_paths["held_removals"][0]["paths"]) == 5 + assert type(status).from_dict(with_paths) == status + + +def test_invalid_settings_are_refused() -> None: + rig = Rig() + for name, value in ( + ("interval_seconds", 0), + ("settle_seconds", -1), + ("reconcile_hours", 0), + ): + with pytest.raises(ValueError, match=name): + SyncIndex( + corpus=rig.corpus, + roots=rig.catalog, + manifest=rig.harness.manifest, + updates=UpdateApplier(rig.update), + planner=lambda view: None, # type: ignore[arg-type,return-value] # never called + **{name: value}, + ) + + +def test_paused_roots_keep_their_sources_and_are_reconciled_when_back() -> None: + books, notes = Path("/books"), Path("/notes") + roots = CorpusRoots( + (CorpusRoot("books", books, legacy=True), CorpusRoot("notes", notes)) + ) + files = { + **_files(5), + **{f"notes/n{i}.txt": (f"note number {i} here", 1.0) for i in range(5)}, + } + rig = Rig(files, roots=roots) + rig.start() + saved = {p: v for p, v in rig.files.items() if p.startswith("notes/")} + for path in saved: + del rig.files[path] + rig.corpus.unreadable.add("notes") + + for _ in range(3): + assert rig.cycle().applied is None + + assert rig.sync.status().paused_roots == ("notes",) + assert {p for p in rig.recorded() if p.startswith("notes/")} == set(saved) + + # The folder comes back with a change made while it was away. + rig.corpus.unreadable.clear() + rig.files.update(saved) + rig.files["notes/n0.txt"] = ("note zero was edited while away", rig.clock.wall) + + result = rig.cycle() + + assert rig.sync.status().paused_roots == () + assert result.applied is not None and result.applied.modified == 1 + assert "notes/n0.txt" in rig.recorded() From 7cd961aae024c000e21cf2f6da29c93c385690e4 Mon Sep 17 00:00:00 2001 From: Viktor Mikhalkin <9029022+ViktorMikhalkin@users.noreply.github.com> Date: Fri, 23 Oct 2026 21:58:47 +0300 Subject: [PATCH 4/9] feat(api): sync the index inside docgrep serve With DOCGREP_SYNC=true the lifespan of `docgrep serve` starts a background task that runs one sync cycle after the other on a worker thread, every DOCGREP_SYNC_INTERVAL_SECONDS, and stops it cleanly on shutdown: no new cycle, and an applied run finishes the source it is on. The task shares the container of search, so the models are loaded once, and asks for it on a worker thread, so the server answers while they load. Sync is off by default and cannot be enabled, triggered or configured over HTTP or MCP. New settings: DOCGREP_SYNC, DOCGREP_SYNC_INTERVAL_SECONDS (120), DOCGREP_SYNC_SETTLE_SECONDS (60) and DOCGREP_SYNC_RECONCILE_HOURS (24). The status sync reports is written to a small file in the data directory after every cycle, for other processes. Story: batch 28, #45. --- src/docgrep/adapters/syncstate/__init__.py | 73 ++++++ src/docgrep/api/container.py | 92 +++++-- src/docgrep/api/http.py | 5 + src/docgrep/api/sync_task.py | 70 +++++ src/docgrep/domain/protocols.py | 7 + src/docgrep/settings.py | 14 + src/docgrep/usecases/sync_index.py | 9 +- tests/integration/test_sync_real.py | 290 +++++++++++++++++++++ tests/integration/test_sync_serve.py | 195 ++++++++++++++ tests/sync_support.py | 103 ++++++++ tests/unit/test_sync_settings.py | 45 ++++ 11 files changed, 879 insertions(+), 24 deletions(-) create mode 100644 src/docgrep/adapters/syncstate/__init__.py create mode 100644 src/docgrep/api/sync_task.py create mode 100644 tests/integration/test_sync_real.py create mode 100644 tests/integration/test_sync_serve.py create mode 100644 tests/sync_support.py create mode 100644 tests/unit/test_sync_settings.py diff --git a/src/docgrep/adapters/syncstate/__init__.py b/src/docgrep/adapters/syncstate/__init__.py new file mode 100644 index 0000000..03326ac --- /dev/null +++ b/src/docgrep/adapters/syncstate/__init__.py @@ -0,0 +1,73 @@ +"""The state of automatic sync, shared with other processes through a file. + +`docgrep serve` runs sync and keeps its state in memory; `docgrep status` is +another process, so the server writes the status it reports to a small JSON +file in the data directory after every cycle (ADR-0029). +""" + +import json +import logging +import os +import tempfile +from datetime import UTC, datetime +from pathlib import Path + +from docgrep.domain.sync import SyncStatus + +_LOG = logging.getLogger(__name__) + +SYNC_STATE_FILE = "sync-status.json" + + +def process_alive(pid: int) -> bool: + """Whether a process with this id exists (signal 0 sends nothing).""" + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +class FileSyncState: + """Write and read the sync status of the server of one data directory.""" + + def __init__(self, data_dir: Path) -> None: + self.path = data_dir / SYNC_STATE_FILE + + def write(self, status: SyncStatus) -> None: + """Replace the file atomically; failures are logged, never raised, + so a full disk cannot stop sync.""" + body = { + "pid": os.getpid(), + "written_at": datetime.now(UTC).isoformat(), + "status": status.to_dict(paths=True), + } + try: + self.path.parent.mkdir(parents=True, exist_ok=True) + handle, name = tempfile.mkstemp(dir=self.path.parent, prefix=".sync-") + try: + with os.fdopen(handle, "w", encoding="utf-8") as stream: + json.dump(body, stream) + os.replace(name, self.path) + except BaseException: + Path(name).unlink(missing_ok=True) + raise + except OSError: + _LOG.warning("sync_state_not_written", exc_info=True) + + def read(self) -> SyncStatus | None: + """The status the server last wrote, with ``running`` true only + while that server process is alive; None without a readable file.""" + try: + body = json.loads(self.path.read_text(encoding="utf-8")) + status = SyncStatus.from_dict(body["status"]) + alive = process_alive(int(body["pid"])) + except (OSError, ValueError, KeyError, TypeError): + return None + if status.running and not alive: + return SyncStatus.from_dict( + {**status.to_dict(paths=True), "running": False} + ) + return status diff --git a/src/docgrep/api/container.py b/src/docgrep/api/container.py index a151eda..a65b67d 100644 --- a/src/docgrep/api/container.py +++ b/src/docgrep/api/container.py @@ -39,6 +39,7 @@ from docgrep.adapters.parsers.text import TextParser from docgrep.adapters.qdrant import QdrantPassageStore from docgrep.adapters.scans import FileScanProbe +from docgrep.adapters.syncstate import FileSyncState from docgrep.api.log_config import configure_logging from docgrep.api.roots import corpus_roots from docgrep.domain.context import ContextResult, OutlineEntry @@ -83,6 +84,7 @@ from docgrep.usecases.plan_update import PlanUpdate from docgrep.usecases.process_scans import ProcessScans from docgrep.usecases.reencode_sparse import ReencodeSparse +from docgrep.usecases.sync_index import SyncIndex, UpdateApplier from docgrep.usecases.update_index import UpdateIndex, UpdatePlan, UpdateSummary _LOG = logging.getLogger(__name__) @@ -156,6 +158,8 @@ class Services: update: UpdateOperation | None = None browse: BrowseOperation | None = None context: ContextOperation | None = None + # Automatic sync (ADR-0029), only with DOCGREP_SYNC=true. + sync: SyncIndex | None = None class _CorpusEnumerator: @@ -195,6 +199,30 @@ def describe(self, path: str) -> Source: return describe_source(self._roots, path) +class _SyncCorpus: + """The roots of this process one at a time, for automatic sync: a root + that is gone must pause itself without stopping the others.""" + + def __init__(self, settings: Settings, roots: CorpusRoots) -> None: + self._settings = settings + self._roots = roots + + def root_readable(self, name: str) -> bool: + root = self._roots.get(name) + return root is not None and root_readable(root.path) + + def scan_root(self, name: str) -> tuple[ScannedFile, ...]: + return scan_corpus( + self._roots.only({name}), + self._settings.include_globs, + self._settings.exclude_globs, + self._settings.data_dir, + ) + + def describe(self, path: str) -> Source: + return describe_source(self._roots, path) + + class _FolderFiles: """The files of any folder, enumerated as a root would be, with the include and exclude patterns; the data directory is skipped.""" @@ -829,6 +857,48 @@ def create_container( "corpus_root": _log_root_name(settings, roots), "backend": settings.backend, } + update = UpdateIndex( + lock=lock, + corpus=corpus, + parsers=parsers, + chunker=chunker, + embedder=embedder, + sparse_encoder=sparse, + index=store, + manifest=manifest, + config=config, + upsert_batch_size=settings.upsert_batch_size, + log_context=log_context, + delegations=delegations, + roots=catalog, + availability=corpus, + sparse_encoders=encoders, + guard=settings.removal_guard, + ocr_outputs=delegations, + reuse_by_hash=settings.reuse_by_hash, + ) + sync = None + if settings.sync: + sync = SyncIndex( + corpus=_SyncCorpus(settings, roots), + roots=catalog, + manifest=manifest, + updates=UpdateApplier(update), + planner=lambda view: PlanUpdate( + corpus=view, + manifest=manifest, + parsers=parsers, + delegations=delegations, + guard=settings.removal_guard, + ocr_outputs=delegations, + reuse_by_hash=settings.reuse_by_hash, + embedding_model=embedder.model_name, + ), + interval_seconds=settings.sync_interval_seconds, + settle_seconds=settings.sync_settle_seconds, + reconcile_hours=settings.sync_reconcile_hours, + on_change=FileSyncState(settings.data_dir).write, + ) return Services( search=FindPassages( embedder=embedder, @@ -869,26 +939,7 @@ def create_container( ocr_outputs=delegations, reuse_by_hash=settings.reuse_by_hash, ), - update=UpdateIndex( - lock=lock, - corpus=corpus, - parsers=parsers, - chunker=chunker, - embedder=embedder, - sparse_encoder=sparse, - index=store, - manifest=manifest, - config=config, - upsert_batch_size=settings.upsert_batch_size, - log_context=log_context, - delegations=delegations, - roots=catalog, - availability=corpus, - sparse_encoders=encoders, - guard=settings.removal_guard, - ocr_outputs=delegations, - reuse_by_hash=settings.reuse_by_hash, - ), + update=update, # The search store is shared, so a local store is never opened twice. browse=BrowseSources( manifest=manifest, store=store, ocr=delegations, roots=catalog @@ -896,4 +947,5 @@ def create_container( context=_expand_context( settings, store=store, manifest=manifest, ocr=delegations ), + sync=sync, ) diff --git a/src/docgrep/api/http.py b/src/docgrep/api/http.py index ce183a3..73aee86 100644 --- a/src/docgrep/api/http.py +++ b/src/docgrep/api/http.py @@ -43,6 +43,7 @@ warn_if_not_loopback, ) from docgrep.api.roots import corpus_roots +from docgrep.api.sync_task import SyncTask from docgrep.domain.context import ContextResult, PassageContext, PassageLocation from docgrep.domain.errors import ( ConfigDrift, @@ -898,6 +899,9 @@ async def lifespan(_app: FastAPI) -> AsyncIterator[None]: if count is not None: warn_if_local_mode_is_large(count) async with AsyncExitStack() as stack: + if effective.sync: + stack.push_async_callback(sync_task.stop) + sync_task.start() if mcp_endpoint is not None: await stack.enter_async_context(_serve_mcp(mcp_endpoint)) elif effective.mcp_http_enabled: @@ -913,6 +917,7 @@ async def lifespan(_app: FastAPI) -> AsyncIterator[None]: app = FastAPI(title="docgrep", version=__version__, lifespan=lifespan) initialization_lock = Lock() + sync_task = SyncTask(lambda: operations(), effective.sync_interval_seconds) def operations() -> Services: nonlocal initialized diff --git a/src/docgrep/api/sync_task.py b/src/docgrep/api/sync_task.py new file mode 100644 index 0000000..36ccedf --- /dev/null +++ b/src/docgrep/api/sync_task.py @@ -0,0 +1,70 @@ +"""The background task of `docgrep serve` that keeps the index following the +corpus (ADR-0029).""" + +import asyncio +import logging +from collections.abc import Callable + +from docgrep.api.container import Services +from docgrep.usecases.sync_index import SyncIndex + +_LOG = logging.getLogger(__name__) + + +class SyncTask: + """Run sync cycles on a worker thread, one after the other, until stopped. + + The task takes the container the server shares with search, so the models + are loaded once; it asks for it on a worker thread, so the server answers + while they load. Stopping starts no new cycle and lets an applied run + finish the source it is on. + """ + + def __init__(self, operations: Callable[[], Services], interval: float) -> None: + self._operations = operations + self._interval = interval + self._stop: asyncio.Event | None = None + self._task: asyncio.Task[None] | None = None + self.sync: SyncIndex | None = None + + def start(self) -> None: + self._stop = asyncio.Event() + self._task = asyncio.create_task(self._run(), name="docgrep-sync") + + async def stop(self) -> None: + """Stop after the source being processed and wait for the task.""" + if self._task is None or self._stop is None: + return + self._stop.set() + if self.sync is not None: + self.sync.request_stop() + await self._task + self._task = None + if self.sync is not None: + self.sync.publish(running=False) + + async def _run(self) -> None: + assert self._stop is not None + try: + services = await asyncio.to_thread(self._operations) + self.sync = services.sync + if self.sync is None: + _LOG.warning("sync_unavailable", extra={"reason": "not configured"}) + return + if self._stop.is_set(): + self.sync.request_stop() + _LOG.info("sync_started") + while not self._stop.is_set(): + try: + await asyncio.to_thread(self.sync.run_cycle) + except Exception: + # A cycle that fails is tried again at the next one. + _LOG.exception("sync_cycle_failed") + try: + await asyncio.wait_for(self._stop.wait(), self._interval) + except TimeoutError: + pass + except Exception: + _LOG.exception("sync_stopped_unexpectedly") + finally: + _LOG.info("sync_stopped") diff --git a/src/docgrep/domain/protocols.py b/src/docgrep/domain/protocols.py index 0b24b60..0169970 100644 --- a/src/docgrep/domain/protocols.py +++ b/src/docgrep/domain/protocols.py @@ -25,6 +25,7 @@ from docgrep.domain.roots import CorpusRoots from docgrep.domain.scans import PageSample from docgrep.domain.sparse import SparseWeighting +from docgrep.domain.sync import SyncStatus class DocumentParser(Protocol): @@ -180,6 +181,12 @@ def describe(self, path: str) -> Source: ... +class SyncStateReader(Protocol): + """The status of automatic sync as the server running it last wrote it.""" + + def read(self) -> SyncStatus | None: ... + + class FolderReader(Protocol): """The files of a folder compared with a root, such as a backup copy. diff --git a/src/docgrep/settings.py b/src/docgrep/settings.py index 7ee2514..9ef181c 100644 --- a/src/docgrep/settings.py +++ b/src/docgrep/settings.py @@ -14,6 +14,11 @@ default_rerank_pool, ) from docgrep.domain.sources import MAX_PAGE_SIZE +from docgrep.domain.sync import ( + DEFAULT_SYNC_INTERVAL_SECONDS, + DEFAULT_SYNC_RECONCILE_HOURS, + DEFAULT_SYNC_SETTLE_SECONDS, +) from docgrep.domain.update_plan import ( DEFAULT_GUARD_COUNT, DEFAULT_GUARD_RATIO, @@ -116,6 +121,15 @@ class Settings(BaseSettings): # by copying the stored passages and vectors (ADR-0028). reuse_by_hash: bool = True + # Automatic sync inside `docgrep serve` (ADR-0029): scan the roots every + # interval, apply a change once its size and time were the same in two + # scans and it is older than the settle time, and compare whole roots + # with the index at start-up and every reconcile_hours. + sync: bool = False + sync_interval_seconds: float = Field(default=DEFAULT_SYNC_INTERVAL_SECONDS, gt=0) + sync_settle_seconds: float = Field(default=DEFAULT_SYNC_SETTLE_SECONDS, ge=0) + sync_reconcile_hours: float = Field(default=DEFAULT_SYNC_RECONCILE_HOURS, gt=0) + # The roots this process uses, once chosen (the roots file, the # deprecated corpus_root or --root parameters); never read from the # environment. diff --git a/src/docgrep/usecases/sync_index.py b/src/docgrep/usecases/sync_index.py index e127430..092edba 100644 --- a/src/docgrep/usecases/sync_index.py +++ b/src/docgrep/usecases/sync_index.py @@ -249,10 +249,11 @@ def status(self, *, running: bool = True) -> SyncStatus: def _is_held(self, path: str) -> bool: return any(path in paths for paths in self._state.held.values()) - def _publish(self) -> None: + def publish(self, *, running: bool = True) -> None: + """Hand the current status to ``on_change``.""" if self.on_change is not None: try: - self.on_change(self.status()) + self.on_change(self.status(running=running)) except Exception: _LOG.warning("sync_status_not_published", exc_info=True) @@ -279,7 +280,7 @@ def run_cycle(self) -> CycleResult: "duration_ms": round((self.clock() - started) * 1000, 1), }, ) - self._publish() + self.publish() return result def _cycle(self) -> CycleResult: @@ -583,7 +584,7 @@ def _on_progress(self, event: ProgressEvent) -> None: if event.phase == "writing" and self.yield_seconds > 0: self.sleep(self.yield_seconds) if event.done == 0 and not event.finished and event.label is None: - self._publish() + self.publish() def _counts(summary: UpdateSummary | None) -> dict[str, int]: diff --git a/tests/integration/test_sync_real.py b/tests/integration/test_sync_real.py new file mode 100644 index 0000000..9a9035f --- /dev/null +++ b/tests/integration/test_sync_real.py @@ -0,0 +1,290 @@ +"""Automatic sync over the real local-mode store, manifest and corpus adapters +with the deterministic backend (ADR-0029). The clock is injected and each +cycle is run explicitly.""" + +import os +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import pytest + +from docgrep.adapters.filelock import FileIndexLock +from docgrep.adapters.syncstate import FileSyncState +from sync_support import BOOKS, Rig, write + + +@pytest.fixture +def rig(tmp_path: Path) -> Iterator[Rig]: + rig = Rig(tmp_path) + yield rig + rig.close() + + +def _search(rig: Rig, query: str) -> list[str]: + assert rig.services is not None + result = rig.services.search.run(query) + return [hit.source.path for hit in result.results] + + +def test_added_changed_removed_and_moved_files_are_applied(rig: Rig) -> None: + rig.open() + assert rig.cycle().applied is None # start-up: nothing changed + write(rig.books, {"fresh.txt": "A lighthouse stands above the northern cliffs."}) + write(rig.books, {"book01.txt": "Book one was rewritten about glaciers."}) + (rig.books / "book02.txt").unlink() + (rig.books / "book03.txt").rename(rig.books / "book03-renamed.txt") + embedder: Any = rig.services.update.embedder # type: ignore[union-attr] # container internals + embedded = [] + inner = embedder.embed + embedder.embed = lambda texts: embedded.append(len(texts)) or inner(texts) + + assert rig.cycle().applied is None + result = rig.cycle() + + assert result.applied is not None + applied = result.applied + assert (applied.added, applied.modified, applied.removed, applied.moved) == ( + 1, + 1, + 1, + 1, + ) + sources = rig.sources() + assert "fresh.txt" in sources and "book02.txt" not in sources + assert "book03.txt" not in sources and "book03-renamed.txt" in sources + # Two files were embedded (the new and the changed one); the move reused. + assert len(embedded) == 2 + assert "fresh.txt" in _search(rig, "lighthouse northern cliffs") + manifest: Any = rig.services.update.manifest # type: ignore[union-attr] # container internals + run = manifest.latest_run_of_kind("sync") + assert run is not None and run.status == "completed" + assert manifest.latest_run_of_kind("update") is None + + +def test_start_up_reconciliation_catches_a_change_made_while_stopped( + rig: Rig, +) -> None: + write( + rig.books, + {"book01.txt": "Rewritten while docgrep was stopped."}, + mtime=rig.clock.wall, + ) + (rig.books / "book05.txt").unlink() + rig.open() + + first = rig.cycle() + + # The change settles at once; the removal waits for the second scan. + assert first.applied is not None and first.applied.modified == 1 + assert first.applied.removed == 0 + second = rig.cycle() + assert second.applied is not None and second.applied.removed == 1 + + +def test_a_lock_held_by_another_operation_keeps_the_entries(rig: Rig) -> None: + rig.open() + rig.cycle() + write(rig.books, {"fresh.txt": "Waiting for the index lock."}) + rig.cycle() + + with FileIndexLock(rig.data / "index.lock").hold("update"): + blocked = rig.cycle() + assert blocked.skipped == "index_lock_held" and blocked.applied is None + assert "fresh.txt" not in rig.sources() + assert rig.sync.status().pending_added == 1 + + assert rig.cycle().applied is not None + assert "fresh.txt" in rig.sources() + + +def test_an_unmounted_root_keeps_its_sources_and_is_reconciled_when_back( + rig: Rig, tmp_path: Path +) -> None: + rig.open() + rig.cycle() + away = tmp_path / "away" + rig.notes.rename(away) + + for _ in range(3): + assert rig.cycle().applied is None + + assert rig.sync.status().paused_roots == ("notes",) + assert {"notes/x.txt", "notes/y.txt"} <= set(rig.sources()) + + away.rename(rig.notes) + write(rig.notes, {"y.txt": "Edited while the disk was away."}, mtime=rig.clock.wall) + result = rig.cycle() + + assert rig.sync.status().paused_roots == () + assert result.applied is not None and result.applied.modified == 1 + + +def test_an_unreadable_folder_of_a_root_is_paused_not_emptied( + rig: Rig, tmp_path: Path +) -> None: + rig.open() + rig.cycle() + # The root's folder is replaced by a plain file: not a directory. + rig.notes.rename(tmp_path / "away") + rig.notes.write_text("not a folder") + + for _ in range(3): + assert rig.cycle().applied is None + + assert rig.sync.status().paused_roots == ("notes",) + assert "notes/x.txt" in rig.sources() + + +def test_a_new_root_waits_for_its_first_manual_update(rig: Rig, tmp_path: Path) -> None: + from roots_support import save_roots + + extra = tmp_path / "extra" + write(extra, {"e1.txt": "Extra root: a map of the old canal.", "e2.txt": "More."}) + save_roots( + rig.data, + ("books", rig.books, True), + ("notes", rig.notes, False), + ("extra", extra, False), + ) + rig.open() + + for _ in range(3): + assert rig.cycle().applied is None + assert rig.sync.status().awaiting_first_update == ("extra",) + assert not any(path.startswith("extra/") for path in rig.sources()) + + summary = rig.services.update.run() # type: ignore[union-attr] # the first manual update + assert summary.added == 2 + + write(extra, {"e3.txt": "A third file after the first update."}) + rig.cycle() + result = rig.cycle() + assert rig.sync.status().awaiting_first_update == () + assert result.applied is not None and result.applied.added == 1 + + +def test_emptying_a_root_holds_its_removals_but_applies_additions(rig: Rig) -> None: + rig.open() + rig.cycle() + for name in BOOKS: + (rig.books / name).unlink() + write(rig.books, {"fresh.txt": "The only file left in the root."}) + rig.cycle() + + result = rig.cycle() + + assert result.applied is not None + assert (result.applied.added, result.applied.removed) == (1, 0) + assert len(result.applied.held_removals) == len(BOOKS) + status = rig.sync.status() + assert [(h.root, h.count) for h in status.held] == [("books", len(BOOKS))] + assert set(BOOKS) <= set(rig.sources()) + + # A manual update with --yes applies them. + summary = rig.services.update.run(override_guard=True) # type: ignore[union-attr] # container + assert summary.removed == len(BOOKS) + assert rig.cycle().applied is None + assert rig.sync.status().held == () + + +def test_held_removals_are_dropped_when_the_files_reappear(rig: Rig) -> None: + rig.open() + rig.cycle() + for name in BOOKS: + (rig.books / name).unlink() + rig.cycle() + rig.cycle() + assert rig.sync.status().held + + write(rig.books, BOOKS) + rig.cycle() + + assert rig.sync.status().held == () + assert set(BOOKS) <= set(rig.sources()) + + +def test_moving_a_whole_root_into_another_folder_passes_the_guard(rig: Rig) -> None: + rig.open() + rig.cycle() + target = rig.notes / "from-books" + target.mkdir() + for name in BOOKS: + (rig.books / name).rename(target / name) + rig.cycle() + + result = rig.cycle() + + assert result.applied is not None + assert (result.applied.moved, result.applied.held_removals) == (len(BOOKS), ()) + assert not any(path in rig.sources() for path in BOOKS) + assert rig.sync.status().held == () + + +def test_the_status_is_written_for_other_processes(rig: Rig) -> None: + rig.open() + rig.cycle() + + status = FileSyncState(rig.data).read() + + assert status is not None and status.enabled and status.running + assert status.last_scan_at is not None + assert status.interval_seconds == 120 + + +def test_ocr_outputs_are_picked_up_as_added_files(rig: Rig) -> None: + rig.open() + rig.cycle() + # What `docgrep ocr` writes next to an original is an ordinary new file. + write(rig.books, {"scan.ocr.txt": "Text recognised from a scanned page."}) + rig.cycle() + + result = rig.cycle() + + assert result.applied is not None and result.applied.added == 1 + assert "scan.ocr.txt" in rig.sources() + + +def test_files_the_settle_time_has_not_aged_wait(rig: Rig) -> None: + rig.open() + rig.cycle() + path = rig.books / "recent.txt" + path.write_text("Written a moment ago.", encoding="utf-8") + os.utime(path, (rig.clock.wall, rig.clock.wall)) + + assert rig.cycle(advance=10).applied is None + assert rig.cycle(advance=10).applied is None + assert rig.cycle(advance=120).applied is not None + + +def test_a_failed_file_is_not_retried_until_it_changes(rig: Rig) -> None: + from docgrep.domain.models import FileType + + rig.open() + rig.cycle() + update: Any = rig.services.update # type: ignore[union-attr] # container + real = update.parsers[FileType.TXT] + + class Failing: + version = real.version + attempts = 0 + + def parse(self, source: Any) -> Any: + if source.path == "broken.txt": + Failing.attempts += 1 + raise ValueError("injected parse failure") + return real.parse(source) + + update.parsers = {**update.parsers, FileType.TXT: Failing()} + write(rig.books, {"broken.txt": "This file cannot be parsed."}) + rig.cycle() + assert rig.cycle().applied is not None and Failing.attempts == 1 + + for _ in range(4): + assert rig.cycle().applied is None + assert Failing.attempts == 1 + + write(rig.books, {"broken.txt": "This file still cannot be parsed, but longer."}) + rig.cycle() + assert rig.cycle().applied is not None and Failing.attempts == 2 + assert rig.sources()["broken.txt"].startswith("failed:") diff --git a/tests/integration/test_sync_serve.py b/tests/integration/test_sync_serve.py new file mode 100644 index 0000000..b2dddb9 --- /dev/null +++ b/tests/integration/test_sync_serve.py @@ -0,0 +1,195 @@ +"""Automatic sync in the lifespan of `docgrep serve`: off by default, started +and stopped with the app, sharing its container, and finishing the source it +is on when the server stops (ADR-0029).""" + +import asyncio +import threading +from dataclasses import replace +from pathlib import Path +from typing import Any + +import pytest +from fastapi.testclient import TestClient + +from docgrep.api.container import Services, create_container +from docgrep.api.http import create_app +from docgrep.api.sync_task import SyncTask +from docgrep.usecases.sync_index import SyncIndex +from resume_support import close +from roots_support import settings +from sync_support import Rig, write + + +class StubSync: + """Stands for SyncIndex: counts cycles and stop requests.""" + + def __init__(self) -> None: + self.cycles = 0 + self.stopped = threading.Event() + self.first_cycle = threading.Event() + self.published: list[bool] = [] + + def run_cycle(self) -> None: + self.cycles += 1 + self.first_cycle.set() + + def request_stop(self) -> None: + self.stopped.set() + + def publish(self, *, running: bool = True) -> None: + self.published.append(running) + + +def _services(sync: Any) -> Services: + class Anything: + def __getattr__(self, name: str) -> Any: + raise AssertionError(f"unexpected use of {name}") + + return Services(search=Anything(), build=Anything(), sync=sync) # type: ignore[arg-type] # test double + + +def test_sync_is_off_by_default(tmp_path: Path) -> None: + stub = StubSync() + app = create_app(settings(tmp_path / "data"), services=_services(stub)) + + with TestClient(app, base_url="http://127.0.0.1") as client: + assert client.get("/health").status_code == 200 + + assert stub.cycles == 0 and not stub.stopped.is_set() + + +def test_sync_starts_and_stops_with_the_app(tmp_path: Path) -> None: + stub = StubSync() + app = create_app( + settings(tmp_path / "data", sync=True, sync_interval_seconds=3600), + services=_services(stub), + ) + + with TestClient(app, base_url="http://127.0.0.1") as client: + assert stub.first_cycle.wait(timeout=10) + assert client.get("/health").status_code == 200 + assert not stub.stopped.is_set() + + assert stub.stopped.is_set() + assert stub.cycles == 1 + # The last state written says it is no longer running. + assert stub.published == [False] + + +def test_a_cycle_that_fails_does_not_stop_the_task(tmp_path: Path) -> None: + class Failing(StubSync): + def run_cycle(self) -> None: + super().run_cycle() + raise RuntimeError("injected") + + stub = Failing() + app = create_app( + settings(tmp_path / "data", sync=True, sync_interval_seconds=3600), + services=_services(stub), + ) + + with TestClient(app, base_url="http://127.0.0.1"): + assert stub.first_cycle.wait(timeout=10) + + assert stub.stopped.is_set() + + +def test_the_task_shares_the_container_of_search(tmp_path: Path) -> None: + rig = Rig(tmp_path) + services = rig.open() + created: list[Services] = [] + + def operations() -> Services: + created.append(services) + return services + + async def run() -> None: + task = SyncTask(operations, interval=3600) + task.start() + # The first cycle has run once the task has a status to publish. + while rig.sync.status().last_scan_at is None: + await asyncio.sleep(0) + await asyncio.to_thread(lambda: None) + await task.stop() + assert task.sync is services.sync + + asyncio.run(run()) + rig.close() + + assert created == [services] + + +def test_shutdown_during_an_applied_run_finishes_the_current_source( + tmp_path: Path, +) -> None: + rig = Rig(tmp_path) + services = rig.open() + sync = services.sync + assert isinstance(sync, SyncIndex) + rig.cycle() + write(rig.books, {f"new{i}.txt": f"Brand new file number {i}." for i in range(5)}) + rig.cycle() # seen once; settles at the next scan + rig.clock.advance(120) + + in_run, release = threading.Event(), threading.Event() + calls: list[float] = [] + + def at_first_source(seconds: float) -> None: + # The first pause is before any source; the second is at the start + # of the first one, which is then in progress when the stop comes. + calls.append(seconds) + if len(calls) == 2: + in_run.set() + assert release.wait(timeout=10) + + sync.sleep = at_first_source + + async def run() -> Any: + task = SyncTask(lambda: replace(services, sync=sync), interval=3600) + # One cycle on a worker thread, as the task would run it. + cycle = asyncio.create_task(asyncio.to_thread(sync.run_cycle)) + task.sync = sync + task._stop = asyncio.Event() + task._task = asyncio.create_task(asyncio.sleep(0)) + await asyncio.to_thread(in_run.wait, 10) + stopping = asyncio.create_task(task.stop()) + await asyncio.sleep(0) + release.set() + await stopping + return await cycle + + result = asyncio.run(run()) + sources = rig.sources() + rig.close() + + assert result.applied is not None and result.applied.cancelled + # The source being processed was finished, no other was started. + assert result.applied.added == 1 + assert sum(1 for path in sources if path.startswith("new")) == 1 + + +@pytest.mark.parametrize("enabled", [False, True]) +def test_info_and_health_work_with_sync_on_or_off( + tmp_path: Path, enabled: bool +) -> None: + app = create_app( + settings(tmp_path / "data", sync=enabled, sync_interval_seconds=3600), + services=_services(StubSync()), + ) + + with TestClient(app, base_url="http://127.0.0.1") as client: + assert client.get("/info").status_code == 200 + + +def test_the_container_builds_sync_only_when_enabled(tmp_path: Path) -> None: + rig = Rig(tmp_path) + off = create_container(rig.settings()) + try: + assert off.sync is None + finally: + close(off) + on = create_container(rig.settings(sync=True)) + try: + assert isinstance(on.sync, SyncIndex) + finally: + close(on) diff --git a/tests/sync_support.py b/tests/sync_support.py new file mode 100644 index 0000000..b99437d --- /dev/null +++ b/tests/sync_support.py @@ -0,0 +1,103 @@ +"""A real container (fake backend, local Qdrant, SQLite) with automatic sync +under an injected clock, for the sync tests.""" + +import os +import time +from pathlib import Path +from typing import Any + +from docgrep.api.container import Services, create_container +from docgrep.settings import Settings +from docgrep.usecases.sync_index import SyncIndex +from resume_support import close +from roots_support import save_roots, settings + +OLD = 1_700_000_000 +BOOKS = { + f"book{i:02d}.txt": f"Book number {i}: a long tale of rivers, harbours, {i} lamps." + for i in range(12) +} +NOTES = { + "x.txt": "A yellow kite drifts over the quiet meadow.", + "y.txt": "Notes on a walk along the river after rain.", +} + + +def write(root: Path, files: dict[str, str], mtime: float = OLD) -> None: + for name, text in files.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + os.utime(path, (mtime, mtime)) + + +class Clock: + """Time that moves only when told to; wall time starts now.""" + + def __init__(self) -> None: + self.mono = 0.0 + self.wall = time.time() + + def advance(self, seconds: float = 120) -> None: + self.mono += seconds + self.wall += seconds + + +class Rig: + """Roots ``books`` (legacy) and ``notes``, built once, then synced.""" + + def __init__(self, tmp_path: Path, *, notes: bool = True, **overrides: Any) -> None: + self.books = tmp_path / "books" + self.notes = tmp_path / "notes" + self.data = tmp_path / "data" + write(self.books, BOOKS) + roots = [("books", self.books, True)] + if notes: + write(self.notes, NOTES) + roots.append(("notes", self.notes, False)) + save_roots(self.data, *roots) + self.overrides = overrides + self.clock = Clock() + self.services: Services | None = None + self.build() + + def settings(self, **extra: Any) -> Settings: + return settings(self.data, **{**self.overrides, **extra}) + + def build(self) -> None: + services = create_container(self.settings()) + services.build.run() + close(services) + + def open(self, **extra: Any) -> Services: + """A new process with sync on and the clock under control.""" + services = create_container(self.settings(sync=True, **extra)) + update: Any = services.update + update.sleep = lambda seconds: None + sync = services.sync + assert sync is not None + sync.clock = lambda: self.clock.mono + sync.wall_clock = lambda: self.clock.wall + sync.sleep = lambda seconds: None + self.services = services + return services + + @property + def sync(self) -> SyncIndex: + assert self.services is not None and self.services.sync is not None + return self.services.sync + + def cycle(self, advance: float = 120) -> Any: + self.clock.advance(advance) + return self.sync.run_cycle() + + def close(self) -> None: + if self.services is not None: + close(self.services) + self.services = None + + def sources(self) -> dict[str, str]: + """The committed sources, path to status.""" + assert self.services is not None + manifest: Any = self.services.update.manifest # type: ignore[attr-defined] # container internals + return {r.source.path: r.status for r in manifest.committed_sources()} diff --git a/tests/unit/test_sync_settings.py b/tests/unit/test_sync_settings.py new file mode 100644 index 0000000..11dda99 --- /dev/null +++ b/tests/unit/test_sync_settings.py @@ -0,0 +1,45 @@ +"""The settings of automatic sync (ADR-0029).""" + +import pytest +from pydantic import ValidationError + +from docgrep.settings import Settings + + +def test_sync_is_off_with_the_documented_defaults() -> None: + settings = Settings() + + assert settings.sync is False + assert settings.sync_interval_seconds == 120 + assert settings.sync_settle_seconds == 60 + assert settings.sync_reconcile_hours == 24 + + +def test_sync_is_read_from_the_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("DOCGREP_SYNC", "true") + monkeypatch.setenv("DOCGREP_SYNC_INTERVAL_SECONDS", "30") + monkeypatch.setenv("DOCGREP_SYNC_SETTLE_SECONDS", "0") + monkeypatch.setenv("DOCGREP_SYNC_RECONCILE_HOURS", "6") + + settings = Settings() + + assert settings.sync is True + assert (settings.sync_interval_seconds, settings.sync_settle_seconds) == (30, 0) + assert settings.sync_reconcile_hours == 6 + + +@pytest.mark.parametrize( + ("name", "value"), + [ + ("DOCGREP_SYNC_INTERVAL_SECONDS", "0"), + ("DOCGREP_SYNC_SETTLE_SECONDS", "-1"), + ("DOCGREP_SYNC_RECONCILE_HOURS", "0"), + ], +) +def test_invalid_sync_settings_are_refused( + monkeypatch: pytest.MonkeyPatch, name: str, value: str +) -> None: + monkeypatch.setenv(name, value) + + with pytest.raises(ValidationError): + Settings() From 36cf00086085e4c7f03e48d8b24993a6c92db4c5 Mon Sep 17 00:00:00 2001 From: Viktor Mikhalkin <9029022+ViktorMikhalkin@users.noreply.github.com> Date: Fri, 23 Oct 2026 22:10:21 +0300 Subject: [PATCH 5/9] feat(api): report sync state in status, info and MCP `docgrep status`, `GET /info` and the MCP `get_index_status` body gain a `sync` section: enabled, running, interval and settle time, the last scan, the last applied run, pending counts by kind, the number of changes still settling (and of those waiting over 30 minutes), held removals per root, paused roots, roots awaiting a first update, the reason sync is idle, the last error and the progress of an applied run. `docgrep status` reads the section from the file the server writes after every cycle, so it works from another process, and adds a few stored paths of each root with held removals. `GET /info` and the MCP tool served by `docgrep serve` read the live state and never include a path; the stdio MCP server reads the file and shows no path either. The latest-run line and the cancel message treat a sync run like an update. The README documents the new keys of the status report. Story: batch 28, #45. --- README.md | 3 +- src/docgrep/api/cli.py | 4 +- src/docgrep/api/container.py | 26 ++- src/docgrep/api/http.py | 89 ++++++++- src/docgrep/api/mcp.py | 4 + src/docgrep/domain/status.py | 17 +- src/docgrep/usecases/inspect_status.py | 7 +- tests/integration/test_corpus_roots_api.py | 3 + tests/integration/test_sync_status.py | 209 +++++++++++++++++++++ 9 files changed, 352 insertions(+), 10 deletions(-) create mode 100644 tests/integration/test_sync_status.py diff --git a/README.md b/README.md index b8ea5cc..8a7b99a 100644 --- a/README.md +++ b/README.md @@ -581,7 +581,7 @@ The `index_state` is one of: The JSON output has these keys (`schema_version` is 1; sections that do not apply are `null`): - `schema_version`, `index_state`. -- `latest_run`: the latest run of any kind. `run_id`, `kind` (`build`, including a resumed build, `update`, `ocr` or `reencode_sparse`), `status` (stored: `running`, `swapped`, `completed`, `aborted` or `cancelled`), `state` (the status, or `interrupted` for a `running` run without a lock holder), `started_at`, `finished_at`, `totals` (`indexed`, `skipped`, `failed`, and `copied` when a build copied passages; an update adds `added`, `modified`, `removed`, `retried`, `moved`, `copied` and `unchanged` source counts and `passages_written` and `passages_removed`; an OCR run records `processed`, `skipped`, `failed`, `cancelled` and `not_started` instead, a sparse re-encode `passages`), `total_sources` (passages for a sparse re-encode), `resumed_from` and `carried_sources` (sources taken over from the resumed run). +- `latest_run`: the latest run of any kind. `run_id`, `kind` (`build`, including a resumed build, `update`, `sync` for an update made by [automatic sync](#automatic-sync), `ocr` or `reencode_sparse`), `status` (stored: `running`, `swapped`, `completed`, `aborted` or `cancelled`), `state` (the status, or `interrupted` for a `running` run without a lock holder), `started_at`, `finished_at`, `totals` (`indexed`, `skipped`, `failed`, and `copied` when a build copied passages; an update adds `added`, `modified`, `removed`, `retried`, `moved`, `copied` and `unchanged` source counts and `passages_written` and `passages_removed`; an OCR run records `processed`, `skipped`, `failed`, `cancelled` and `not_started` instead, a sparse re-encode `passages`), `total_sources` (passages for a sparse re-encode), `resumed_from` and `carried_sources` (sources taken over from the resumed run). - `active_operation`: present while the index lock is held. `kind` (`build`, `resume`, `update`, `ocr` or `reencode_sparse`; for `ocr` the sources are the files to process and `passages_written` is 0; for `reencode_sparse` `sources_done`, `sources_total` and `passages_written` count passages), `run_id`, `started_at`, `sources_done` and `passages_written` (from the run's journal, so they include carried-over sources), `sources_total`, `elapsed_seconds`, `eta_seconds` (`null` until computable; the pace of this process, ignoring carried-over sources) and `cancel_requested_at`. - `vector_store`: `mode` (`local` or `server`), `status` (`reachable`, `unreachable`, `busy` or `absent` for a local store that does not exist yet) and `reason`, a short text for the degraded cases. - `collection`: when the index can be read. `passage_count`, `embedding_model`, `embedding_dimension`, `embedding_dtype`, `docgrep_version`, `completed_at` and `last_updated_at` (when the latest `docgrep update` completed, `null` before the first). @@ -589,6 +589,7 @@ The JSON output has these keys (`schema_version` is 1; sections that do not appl - `sources`: what the committed sources table says, or `null` when the manifest is missing or cannot be read. `total`; `by_state` with `indexed`, `skipped` and `failed`; `skipped_by_reason` with `requires_ocr` (no text layer), `garbled_text_layer` (a PDF whose text layer is garbled), `unsupported` (no parser for the type) and `other`, plus `delegated_to_ocr` (an original whose text is indexed from its OCR output) when any source has that reason; `skipped_by_extension`, the 10 most common extensions among the skipped sources, a list of `extension` (`(none)` without one) and `sources`; `failed`, the first 20 failed sources by path, each with `path`, `category` (for example `parse_error`, `embedding_error` or `store_error`) and `reason` (the stored error message, shortened to 200 characters); and `parser_changed`, the number of sources that the next `docgrep update` would extract again because their parser declares another version (the same rule as update, missing versions counting as `1`). The plain-text report shows a `Sources:` line, the skipped counts per reason and the failed sources with their paths. - `warnings`: a list of `code` and `message`, for example `store_unreachable`, `store_busy`, `manifest_unreadable`, `config_drift`, `resume_available`, `swap_pending_commit`, `update_unfinished` (the latest update was cancelled, aborted or interrupted; run `docgrep update` again), `reencode_unfinished` (the latest sparse re-encode stopped early; run `docgrep reencode-sparse --yes` again), `index_incomplete`, `lock_held_without_run`, `lock_unreadable`, `root_not_configured` (the index holds sources of roots that are not configured) and `root_unreadable` (a configured root cannot be read). - `roots`: the configured [corpus roots](#corpus-roots) in order, then the roots found only in the index, each with `name`, `legacy`, `configured`, `readable` (`null` for a root that is not configured) and the numbers of its committed sources that are `indexed`, `skipped` and `failed`. No path is shown. The plain-text report lists the roots when there is more than one or one needs attention. +- `sync`: [automatic sync](#automatic-sync), always present. `enabled` (`DOCGREP_SYNC`), `running` (a `docgrep serve` of this data directory is alive and runs it), `interval_seconds`, `settle_seconds`, `last_scan_at`, `last_applied` (`null` before the first applied run: `run_id`, `finished_at`, the counts `added`, `modified`, `removed`, `moved`, `copied` and `failed`, and `cancelled`), `pending` (`added`, `modified` and `removed` changes seen and not applied yet), `unsettled` (changes still waiting to settle) and `unsettled_over_limit` (of those, the ones waiting for over 30 minutes), `held_removals` (a list with one entry per root whose removals the [removal guard](#the-removal-guard) holds: `root` and `count`, and in `docgrep status` also `paths`, a few stored paths), `paused_roots`, `awaiting_first_update` (roots without a completed manual update), `idle_reason` (why sync does nothing, for example a configuration that drifted from the index; `null` when it works), `last_error` and `applying` (`null`, or `done` and `total` while a run is applied). `docgrep status` reads it from the file the server writes after every cycle; `GET /info` and the MCP tool read the live state of the server and never include `paths`. - `sparse`: the [sparse weighting](#sparse-weights-bm25) of the index, `null` without a completed index. `encoder` (`tf-v1` when the index records none, or `bm25-v2`), `bm25_k1`, `bm25_b` and `bm25_avgdl` (`null` for `tf-v1`), `reencode` (the latest sparse re-encode while it runs or after it stopped early, else `null`: `run_id`, `state`, `passages_done` and `passages_total`) and `hint` (what to run next, for a `tf-v1` index or an unfinished re-encode, else `null`). The plain-text report shows a `Sparse encoder:` line, `reencoding (n of N)` while a re-encode runs. ## Dry run diff --git a/src/docgrep/api/cli.py b/src/docgrep/api/cli.py index 217e5e9..9450d9e 100644 --- a/src/docgrep/api/cli.py +++ b/src/docgrep/api/cli.py @@ -51,6 +51,7 @@ format_filters, root_status_lines, run_search, + sync_status_lines, validation_problems, ) from docgrep.api.log_config import configure_logging @@ -1151,6 +1152,7 @@ def _format_status(report: dict[str, Any]) -> str: ) lines.extend(_sparse_status_lines(report.get("sparse"))) lines.extend(root_status_lines(report.get("roots"))) + lines.extend(sync_status_lines(report.get("sync"))) for entry in report["drift"]: lines.append( f"Drift: {entry['field']} is {entry['indexed']} in the index and " @@ -2288,7 +2290,7 @@ def main( return _ocr(args, ocr_factory(settings), register_signal, selection) if args.command == "status": - status_report = status_factory(settings).run().as_dict() + status_report = status_factory(settings).run().as_dict(sync_paths=True) if args.json: _emit(json.dumps(status_report, indent=2) + "\n") else: diff --git a/src/docgrep/api/container.py b/src/docgrep/api/container.py index a65b67d..6d8a17a 100644 --- a/src/docgrep/api/container.py +++ b/src/docgrep/api/container.py @@ -3,7 +3,7 @@ import logging from collections.abc import Callable, Collection, Iterator, Mapping, Sequence from contextlib import contextmanager -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path from typing import Protocol @@ -70,6 +70,7 @@ from docgrep.domain.run_state import latest_root_prefixes from docgrep.domain.search_effort import DenseSearchEffort, default_rerank_pool from docgrep.domain.sources import BrowseRequest, BrowseResult, SourceFilter +from docgrep.domain.sync import SyncStatus from docgrep.settings import Settings from docgrep.usecases.browse_sources import BrowseSources from docgrep.usecases.build_index import BuildIndex, BuildSummary @@ -404,9 +405,29 @@ def _vector_config(settings: Settings) -> dict[str, str | int | float]: } -def create_status(settings: Settings) -> InspectStatus: +def stored_sync_status(settings: Settings) -> SyncStatus: + """The state of automatic sync as the server of this data directory last + wrote it, for a process that is not that server; with no such server + running, what the settings say and the last state it left.""" + stored = FileSyncState(settings.data_dir).read() + if stored is not None and stored.running: + return stored + base = stored or SyncStatus( + enabled=settings.sync, + interval_seconds=settings.sync_interval_seconds, + settle_seconds=settings.sync_settle_seconds, + ) + return replace(base, enabled=settings.sync, running=False, applying=None) + + +def create_status( + settings: Settings, *, sync: Callable[[], SyncStatus] | None = None +) -> InspectStatus: """Wire the read-only status report. + ``sync`` gives the state of automatic sync; by default what the server + of the data directory wrote last (``stored_sync_status``). + It loads no model, takes no lock and creates nothing: the manifest is opened without migrating it, a local store that does not exist is not created, and a server is pinged with a short timeout. @@ -450,6 +471,7 @@ def open_store() -> StatusStore | None: }, roots=_RootCatalog(settings), root_readable=lambda root: root_readable(root.path), + sync=sync or (lambda: stored_sync_status(settings)), ) diff --git a/src/docgrep/api/http.py b/src/docgrep/api/http.py index 73aee86..094b7f0 100644 --- a/src/docgrep/api/http.py +++ b/src/docgrep/api/http.py @@ -4,6 +4,7 @@ import logging from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence from contextlib import AsyncExitStack, asynccontextmanager +from dataclasses import replace from threading import Lock from typing import Annotated, Any, Literal, Self from urllib.parse import urlsplit @@ -33,6 +34,7 @@ create_container, create_discover_scans, create_process_scans, + create_status, index_missing, open_ocr_registry_view, unindexed_context, @@ -74,6 +76,7 @@ distinct_texts, parse_file_types, ) +from docgrep.domain.sync import SyncStatus from docgrep.settings import Settings from docgrep.usecases.build_index import BuildSummary from docgrep.usecases.process_scans import OcrRequest @@ -455,6 +458,59 @@ def root_status_lines(roots: Sequence[Mapping[str, Any]] | None) -> list[str]: return [format_root_status(root) for root in roots] +def sync_status_lines(sync: Mapping[str, Any] | None) -> list[str]: + """The ``sync`` section of a status report as text; nothing when sync is + off. Stored paths are listed only when the section carries them.""" + if not sync or not sync["enabled"]: + return [] + state = "running" if sync["running"] else "enabled, but no server is running it" + lines = [ + f"Sync: {state}, scanning every {sync['interval_seconds']:g} s, " + f"settling {sync['settle_seconds']:g} s" + + (f"; last scan {sync['last_scan_at']}" if sync["last_scan_at"] else "") + ] + applied = sync["last_applied"] + if applied is not None: + lines.append( + f" Last applied: run {applied['run_id']} at {applied['finished_at']}: " + f"{applied['added']} added, {applied['modified']} modified, " + f"{applied['removed']} removed, {applied['moved']} moved, " + f"{applied['copied']} copied, {applied['failed']} failed" + + (" (cancelled)" if applied["cancelled"] else "") + ) + pending = sync["pending"] + lines.append( + f" Pending: {pending['added']} added, {pending['modified']} modified, " + f"{pending['removed']} removed; {sync['unsettled']} not yet settled" + + ( + f", {sync['unsettled_over_limit']} for over 30 minutes" + if sync["unsettled_over_limit"] + else "" + ) + ) + if sync["applying"] is not None: + lines.append( + f" Applying: {sync['applying']['done']} of {sync['applying']['total']}" + ) + for held in sync["held_removals"]: + lines.append( + f" Held removals in root {held['root']}: {held['count']} (the removal " + "guard; `docgrep update --yes` applies them)" + ) + lines.extend(f" {path}" for path in held.get("paths", ())) + if sync["paused_roots"]: + lines.append(" Paused roots: " + ", ".join(sync["paused_roots"])) + if sync["awaiting_first_update"]: + lines.append( + " Awaiting a first update: " + ", ".join(sync["awaiting_first_update"]) + ) + if sync["idle_reason"]: + lines.append(f" Idle: {sync['idle_reason']}") + if sync["last_error"]: + lines.append(f" Last error: {sync['last_error']}") + return lines + + def format_root_status(root: Mapping[str, Any]) -> str: """One root of the status report on one line: name, flags and counts.""" flags = ["legacy"] if root["legacy"] else [] @@ -742,11 +798,24 @@ class InfoResponse(BaseModel): # the search effort of the dense channel, hnsw_ef or "exact" (ADR-0026). rerank_pool: int dense_hnsw_ef: int | Literal["exact"] + # Automatic sync (ADR-0029): the same section as ``get_index_status``. + sync: dict[str, Any] = Field(default_factory=dict) -def service_info(settings: Settings) -> InfoResponse: +def _sync_off(settings: Settings) -> SyncStatus: + return SyncStatus( + enabled=settings.sync, + interval_seconds=settings.sync_interval_seconds, + settle_seconds=settings.sync_settle_seconds, + ) + + +def service_info( + settings: Settings, sync: Callable[[], SyncStatus] | None = None +) -> InfoResponse: """What ``GET /info`` returns; the index is not read.""" return InfoResponse( + sync=(sync or (lambda: _sync_off(settings)))().to_dict(), version=__version__, backend=settings.backend, rerank_pool=settings.effective_rerank_pool, @@ -919,6 +988,16 @@ async def lifespan(_app: FastAPI) -> AsyncIterator[None]: initialization_lock = Lock() sync_task = SyncTask(lambda: operations(), effective.sync_interval_seconds) + def sync_status() -> SyncStatus: + """The live state of sync; the server answers while the container, + and with it sync, is still being created.""" + if sync_task.sync is not None: + return sync_task.sync.status() + status = _sync_off(effective) + if effective.sync: + return replace(status, running=True, idle_reason="starting") + return status + def operations() -> Services: nonlocal initialized if initialized is None: @@ -934,7 +1013,11 @@ async def _serve_mcp(endpoint: McpEndpoint) -> AsyncIterator[None]: # Imported here, at startup: the module imports this one. from docgrep.api.mcp import create_mcp_server - server = create_mcp_server(effective, services=operations) + server = create_mcp_server( + effective, + services=operations, + status=lambda: create_status(effective, sync=sync_status), + ) mcp_app = server.streamable_http_app() async with server.session_manager.run(): endpoint.app = mcp_app @@ -996,7 +1079,7 @@ def health() -> dict[str, str]: def info() -> InfoResponse: """The version, the backend and the configured corpus roots with their absolute paths; the index is not read.""" - return service_info(effective) + return service_info(effective, sync_status) @app.post("/search", response_model=SearchResponse) def search(request: SearchRequest) -> SearchResponse: diff --git a/src/docgrep/api/mcp.py b/src/docgrep/api/mcp.py index 655ccfb..f06986e 100644 --- a/src/docgrep/api/mcp.py +++ b/src/docgrep/api/mcp.py @@ -51,6 +51,7 @@ format_filters, root_status_lines, run_search, + sync_status_lines, validation_problems, ) from docgrep.api.roots import corpus_roots @@ -202,6 +203,8 @@ class IndexStatusBody(BaseModel): # The corpus roots by name with their flags and source counts; never a # path. roots: list[dict[str, Any]] | None = None + # Automatic sync (ADR-0029): counts and names only, never a path. + sync: dict[str, Any] | None = None # The declared output schemas accept a success body or an error body: the SDK @@ -321,6 +324,7 @@ def render_status(report: Mapping[str, Any]) -> str: f" failed: {failed['path']}: {failed['category']}: {failed['reason']}" ) lines.extend(root_status_lines(report.get("roots"))) + lines.extend(sync_status_lines(report.get("sync"))) for warning in report["warnings"]: lines.append(f"Warning ({warning['code']}): {warning['message']}") return "\n".join(lines) + "\n" diff --git a/src/docgrep/domain/status.py b/src/docgrep/domain/status.py index 3838abb..5638a29 100644 --- a/src/docgrep/domain/status.py +++ b/src/docgrep/domain/status.py @@ -3,6 +3,8 @@ from dataclasses import dataclass from typing import Any +from docgrep.domain.sync import SyncStatus + IndexState = str # one of INDEX_STATES @@ -168,9 +170,15 @@ class StatusReport: roots: tuple[RootStatusView, ...] | None = None # None without a completed index (ADR-0025). sparse: SparseView | None = None + # Automatic sync (ADR-0029); None when the report was made without it. + sync: SyncStatus | None = None + + def as_dict(self, *, sync_paths: bool = False) -> dict[str, Any]: + """The JSON form; its key names are stable and documented in the README. - def as_dict(self) -> dict[str, Any]: - """The JSON form; its key names are stable and documented in the README.""" + ``sync_paths`` adds a few stored paths to each root with held + removals (``docgrep status`` only; never to a tool's client). + """ run = self.latest_run active = self.active_operation collection = self.collection @@ -228,6 +236,11 @@ def as_dict(self) -> dict[str, Any]: "sources": None if self.sources is None else _sources(self.sources), "sparse": None if self.sparse is None else _sparse(self.sparse), **({} if self.roots is None else {"roots": _roots(self.roots)}), + **( + {} + if self.sync is None + else {"sync": self.sync.to_dict(paths=sync_paths)} + ), } diff --git a/src/docgrep/usecases/inspect_status.py b/src/docgrep/usecases/inspect_status.py index f0d8818..cbfb5ea 100644 --- a/src/docgrep/usecases/inspect_status.py +++ b/src/docgrep/usecases/inspect_status.py @@ -52,6 +52,7 @@ StatusWarning, StoreView, ) +from docgrep.domain.sync import SyncStatus from docgrep.usecases.reencode_sparse import reencode_progress _LOG = logging.getLogger(__name__) @@ -104,11 +105,14 @@ def __init__( parser_versions: Mapping[FileType, str | None] | None = None, roots: RootCatalog | None = None, root_readable: Callable[[CorpusRoot], bool] | None = None, + sync: Callable[[], SyncStatus | None] | None = None, ) -> None: """``parser_versions`` maps each file type with a registered parser to the version it declares; without it the parser-version count is None. ``roots`` gives the configured roots, reported with their sources; - ``root_readable`` checks a root folder without walking it.""" + ``root_readable`` checks a root folder without walking it. ``sync`` + gives the state of automatic sync for the report (ADR-0029).""" + self.sync = sync self.roots = roots self.root_readable = root_readable self.lock = lock @@ -284,6 +288,7 @@ def warn(code: str, message: str) -> None: sources, roots, sparse, + None if self.sync is None else self.sync(), ) @staticmethod diff --git a/tests/integration/test_corpus_roots_api.py b/tests/integration/test_corpus_roots_api.py index 8ae449b..a5cebd3 100644 --- a/tests/integration/test_corpus_roots_api.py +++ b/tests/integration/test_corpus_roots_api.py @@ -54,6 +54,8 @@ def test_info_keeps_its_keys_and_lists_the_configured_roots(tmp_path: Path) -> N with _client(data) as client: body = client.get("/info").json() + # The sync section is checked in test_sync_status.py. + assert body.pop("sync")["enabled"] is False assert body == { "version": __version__, "backend": "fake", @@ -108,6 +110,7 @@ def test_the_openapi_schema_describes_info_and_the_filter(tmp_path: Path) -> Non "roots", "rerank_pool", "dense_hnsw_ef", + "sync", } assert set(components["InfoRootResponse"]["properties"]) == { "name", diff --git a/tests/integration/test_sync_status.py b/tests/integration/test_sync_status.py new file mode 100644 index 0000000..1b9fc68 --- /dev/null +++ b/tests/integration/test_sync_status.py @@ -0,0 +1,209 @@ +"""The `sync` section of `docgrep status`, `GET /info` and the MCP status tool +(ADR-0029): counts and names for every client, a few stored paths in +`docgrep status` only.""" + +import json +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import anyio +import pytest +from fastapi.testclient import TestClient + +from docgrep.api.cli import main +from docgrep.api.container import Services, create_status +from docgrep.api.http import create_app +from docgrep.api.mcp import create_mcp_server +from mcp_support import structured +from sync_support import BOOKS, Rig + +SECTION_KEYS = { + "enabled", + "running", + "interval_seconds", + "settle_seconds", + "last_scan_at", + "last_applied", + "pending", + "unsettled", + "unsettled_over_limit", + "held_removals", + "paused_roots", + "awaiting_first_update", + "idle_reason", + "last_error", + "applying", +} + + +@pytest.fixture +def held(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Rig]: + """A synced index whose books root had all its files removed: the + removals are held.""" + rig = Rig(tmp_path) + for name, value in { + "DOCGREP_DATA_DIR": str(rig.data), + "DOCGREP_BACKEND": "fake", + "DOCGREP_SYNC": "true", + "DOCGREP_CHUNK_SIZE": "10", + "DOCGREP_CHUNK_OVERLAP": "2", + "DOCGREP_LOG_LEVEL": "WARNING", + }.items(): + monkeypatch.setenv(name, value) + rig.open() + rig.cycle() + for name in BOOKS: + (rig.books / name).unlink() + rig.cycle() + rig.cycle() + assert rig.sync.status().held + yield rig + rig.close() + + +def _status_cli(capsys: pytest.CaptureFixture[str], *args: str) -> Any: + code = main(["status", *args]) + out, _ = capsys.readouterr() + assert code == 0 + return json.loads(out) if "--json" in args else out + + +def test_status_has_a_disabled_sync_section_by_default( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + rig = Rig(tmp_path) + monkeypatch.setenv("DOCGREP_DATA_DIR", str(rig.data)) + monkeypatch.setenv("DOCGREP_BACKEND", "fake") + + body = _status_cli(capsys, "--json") + text = _status_cli(capsys) + + assert set(body["sync"]) == SECTION_KEYS + assert (body["sync"]["enabled"], body["sync"]["running"]) == (False, False) + assert "Sync" not in text + + +def test_docgrep_status_shows_held_removals_with_a_few_paths( + held: Rig, capsys: pytest.CaptureFixture[str] +) -> None: + body = _status_cli(capsys, "--json")["sync"] + + assert set(body) == SECTION_KEYS + # The server of this data directory (this process) is alive. + assert (body["enabled"], body["running"]) == (True, True) + [entry] = body["held_removals"] + assert (entry["root"], entry["count"]) == ("books", len(BOOKS)) + assert 0 < len(entry["paths"]) <= 5 + assert set(entry["paths"]) <= set(BOOKS) + assert body["last_applied"] is not None and body["last_scan_at"] is not None + assert body["pending"] == {"added": 0, "modified": 0, "removed": 0} + + text = _status_cli(capsys) + assert "Sync: running, scanning every 120 s, settling 60 s" in text + assert f"Held removals in root books: {len(BOOKS)}" in text + assert f" {entry['paths'][0]}" in text + + +def test_docgrep_status_says_when_no_server_runs_sync( + held: Rig, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch +) -> None: + import docgrep.adapters.syncstate as syncstate + + monkeypatch.setattr(syncstate, "process_alive", lambda pid: False) + + body = _status_cli(capsys, "--json")["sync"] + + assert (body["enabled"], body["running"]) == (True, False) + assert body["held_removals"][0]["count"] == len(BOOKS) + assert "enabled, but no server is running it" in _status_cli(capsys) + + +def test_info_carries_the_live_section_without_paths(held: Rig) -> None: + assert held.services is not None + app = create_app(held.settings(sync=True), services=held.services) + + with TestClient(app, base_url="http://127.0.0.1") as client: + body = client.get("/info").json()["sync"] + + assert set(body) == SECTION_KEYS + assert body["held_removals"] == [{"root": "books", "count": len(BOOKS)}] + assert not any(name in json.dumps(body) for name in BOOKS) + + +def test_info_reports_sync_off_without_creating_a_container(tmp_path: Path) -> None: + rig = Rig(tmp_path) + + with TestClient(create_app(rig.settings()), base_url="http://127.0.0.1") as client: + body = client.get("/info").json()["sync"] + + assert set(body) == SECTION_KEYS and body["enabled"] is False + assert not (rig.data / "sync-status.json").exists() + + +def _mcp_body(rig: Rig, services: Services, **kwargs: Any) -> dict[str, Any]: + server = create_mcp_server( + rig.settings(sync=True), services=lambda: services, **kwargs + ) + return structured(anyio.run(server.call_tool, "get_index_status", {})) # type: ignore[no-any-return] # test helper + + +def test_the_mcp_status_body_has_the_section_and_never_a_path(held: Rig) -> None: + assert held.services is not None + sync = held.sync + + body = _mcp_body( + held, + held.services, + status=lambda: create_status(held.settings(sync=True), sync=sync.status), + ) + + assert set(body["sync"]) == SECTION_KEYS + assert body["sync"]["held_removals"] == [{"root": "books", "count": len(BOOKS)}] + dumped = json.dumps(body["sync"]) + assert not any(name in dumped for name in BOOKS) and str(held.books) not in dumped + + +def test_the_stdio_mcp_status_reads_the_file_of_the_server(held: Rig) -> None: + assert held.services is not None + + # No live sync is given: as in `docgrep mcp`, the file is read. + body = _mcp_body(held, held.services) + + assert body["sync"]["running"] is True + assert body["sync"]["held_removals"] == [{"root": "books", "count": len(BOOKS)}] + assert not any(name in json.dumps(body["sync"]) for name in BOOKS) + + +def test_the_live_state_shows_the_applying_progress(held: Rig) -> None: + seen: list[Any] = [] + sync = held.sync + inner = sync.sleep + sync.sleep = lambda seconds: (seen.append(sync.status().applying), inner(seconds))[ + 1 + ] + for index in range(3): + (held.books / f"new{index}.txt").write_text(f"new file {index}", "utf-8") + held.cycle() + held.cycle() + + totals = {a for a in seen if a is not None} + assert any(total == 3 for _, total in totals) + assert sync.status().applying is None + + +def test_idle_and_error_reasons_reach_the_status(held: Rig) -> None: + update: Any = held.services.update # type: ignore[union-attr] # container internals + + from docgrep.domain.errors import ConfigDrift + + def drifted() -> None: + raise ConfigDrift("The configuration differs from the index") + + update.check_ready = drifted + + held.cycle() + + body = held.sync.status().to_dict() + assert body["idle_reason"].startswith("config_drift: ") + assert held.sync.status().idle_reason == body["idle_reason"] From 8a2ced70b88a37784b390163dd4846d8ca1569d0 Mon Sep 17 00:00:00 2001 From: Viktor Mikhalkin <9029022+ViktorMikhalkin@users.noreply.github.com> Date: Fri, 23 Oct 2026 22:16:44 +0300 Subject: [PATCH 6/9] fix(adapters): serialize the calls of a local-mode Qdrant store Local mode keeps its collections in arrays that nothing synchronizes, so a search that reads while an upsert grows them failed with an index error (`index N is out of bounds`) and surfaced as `store_unavailable`. A server process searches from several threads, and automatic sync writes from another, which made it likely. Every call of the local client now takes one lock; a server client is unchanged. Story: batch 28, #45. --- src/docgrep/adapters/qdrant/__init__.py | 37 ++++++++++++- .../test_qdrant_local_concurrency.py | 44 +++++++++++++++ tests/unit/test_serialized_qdrant_client.py | 53 +++++++++++++++++++ 3 files changed, 132 insertions(+), 2 deletions(-) create mode 100644 tests/integration/test_qdrant_local_concurrency.py create mode 100644 tests/unit/test_serialized_qdrant_client.py diff --git a/src/docgrep/adapters/qdrant/__init__.py b/src/docgrep/adapters/qdrant/__init__.py index 21818b8..d5934f8 100644 --- a/src/docgrep/adapters/qdrant/__init__.py +++ b/src/docgrep/adapters/qdrant/__init__.py @@ -3,6 +3,7 @@ import logging import math import re +import threading import warnings from collections.abc import Callable, Collection, Iterator, Mapping, Sequence from contextlib import AbstractContextManager, contextmanager, nullcontext @@ -239,6 +240,38 @@ def _cosine( return dot / (stored_norm * query_norm) +class _SerializedClient: + """A local-mode client whose calls run one at a time. + + Local mode keeps its collections in arrays that nothing synchronizes: a + search that reads while an upsert grows them can fail with an index + error. A server process has several threads (searches, an update by + automatic sync), so every call of the client takes one lock. A server + client needs none. + """ + + def __init__(self, client: QdrantClient) -> None: + self._client = client + self._lock = threading.RLock() + + def __getattr__(self, name: str) -> Any: + attribute = getattr(self._client, name) + if not callable(attribute): + return attribute + + @wraps(attribute) + def serialized(*args: Any, **kwargs: Any) -> Any: + with self._lock: + return attribute(*args, **kwargs) + + return serialized + + +def _local_client(path: Path) -> QdrantClient: + client: QdrantClient = _SerializedClient(QdrantClient(path=str(path))) # type: ignore[assignment] # same calls, one at a time + return client + + def _domain_errors[**P, T](operation: Callable[P, T]) -> Callable[P, T]: @wraps(operation) def guarded(*args: P.args, **kwargs: P.kwargs) -> T: @@ -293,7 +326,7 @@ def __init__( elif path is not None: try: path.mkdir(parents=True, exist_ok=True) - self._client = QdrantClient(path=str(path)) + self._client = _local_client(path) except Exception as error: raise StoreUnavailable( "Cannot open Qdrant storage", @@ -793,7 +826,7 @@ def _reload_local(self) -> None: """ if self._local and self._path is not None: self._client.close() - self._client = QdrantClient(path=str(self._path)) + self._client = _local_client(self._path) @_domain_errors def passage_text_page( diff --git a/tests/integration/test_qdrant_local_concurrency.py b/tests/integration/test_qdrant_local_concurrency.py new file mode 100644 index 0000000..4cff152 --- /dev/null +++ b/tests/integration/test_qdrant_local_concurrency.py @@ -0,0 +1,44 @@ +"""A local-mode store used from several threads, as `docgrep serve` does +while sync writes: searches never fail while points are written.""" + +import threading +from pathlib import Path + +from docgrep.api.container import create_container +from resume_support import close +from roots_support import save_roots, settings +from sync_support import BOOKS, write + + +def test_searches_succeed_while_sources_are_written(tmp_path: Path) -> None: + books, data = tmp_path / "books", tmp_path / "data" + write(books, BOOKS) + save_roots(data, ("books", books, True)) + services = create_container(settings(data)) + services.build.run() + failures: list[BaseException] = [] + stop = threading.Event() + + def search() -> None: + while not stop.is_set(): + try: + services.search.run("harbours lamps tale") + except BaseException as error: # noqa: BLE001 # recorded and asserted below + failures.append(error) + return + + readers = [threading.Thread(target=search) for _ in range(3)] + for reader in readers: + reader.start() + try: + for index in range(40): + write(books, {f"new{index}.txt": f"New file {index} about glaciers."}) + assert services.update is not None + services.update.run() + finally: + stop.set() + for reader in readers: + reader.join(30) + close(services) + + assert failures == [] diff --git a/tests/unit/test_serialized_qdrant_client.py b/tests/unit/test_serialized_qdrant_client.py new file mode 100644 index 0000000..58e3b6f --- /dev/null +++ b/tests/unit/test_serialized_qdrant_client.py @@ -0,0 +1,53 @@ +"""Calls of a local-mode Qdrant client run one at a time.""" + +import threading + +from docgrep.adapters.qdrant import _SerializedClient + + +class Slow: + """A client whose `slow` call waits until the test lets it finish.""" + + attribute = "value" + + def __init__(self) -> None: + self.inside = threading.Event() + self.release = threading.Event() + self.order: list[str] = [] + + def slow(self) -> str: + self.order.append("slow:start") + self.inside.set() + assert self.release.wait(timeout=10) + self.order.append("slow:end") + return "slow" + + def fast(self) -> str: + self.order.append("fast") + return "fast" + + +def test_a_call_waits_for_the_one_in_progress() -> None: + inner = Slow() + client = _SerializedClient(inner) # type: ignore[arg-type] # a stand-in with the same calls + results: list[str] = [] + first = threading.Thread(target=lambda: results.append(client.slow())) + first.start() + assert inner.inside.wait(timeout=10) + second = threading.Thread(target=lambda: results.append(client.fast())) + second.start() + # `fast` cannot have run while `slow` holds the client. + assert inner.order == ["slow:start"] + + inner.release.set() + first.join(10) + second.join(10) + + assert inner.order == ["slow:start", "slow:end", "fast"] + assert sorted(results) == ["fast", "slow"] + + +def test_attributes_pass_through() -> None: + client = _SerializedClient(Slow()) # type: ignore[arg-type] # a stand-in with the same calls + + assert client.attribute == "value" From 85669b1da6a819df414f532ffe318bdeaf8d385c Mon Sep 17 00:00:00 2001 From: Viktor Mikhalkin <9029022+ViktorMikhalkin@users.noreply.github.com> Date: Fri, 23 Oct 2026 22:20:14 +0300 Subject: [PATCH 7/9] perf(usecases): pause once per source in an applied sync run An applied sync run gave a waiting search the models before and after every source, which made it about five times slower than needed. It now pauses once, after each finished source. Measured against a Qdrant server with 200 new files: the run takes 15 s instead of 27 s, search p50 rises 8% during it (33% when the run does not pause at all). Story: batch 28, #45. --- src/docgrep/usecases/sync_index.py | 14 +++++++++----- tests/integration/test_sync_serve.py | 22 ++++++++++++++-------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/docgrep/usecases/sync_index.py b/src/docgrep/usecases/sync_index.py index 092edba..4b60048 100644 --- a/src/docgrep/usecases/sync_index.py +++ b/src/docgrep/usecases/sync_index.py @@ -189,6 +189,9 @@ def __init__( self._state = _State() self._guard = threading.Lock() self._stopping = threading.Event() + # How many sources of the run in progress were already followed by + # a pause. + self._yielded = 0 # -- control --------------------------------------------------------- @@ -576,12 +579,13 @@ def _report_unsettled(self, mono: float, wall: float) -> None: def _on_progress(self, event: ProgressEvent) -> None: """Note how far an applied run has got, and give a waiting search the - models between two sources.""" + models after each source.""" with self._guard: - self._state.applying = ( - None if event.finished else (event.done, event.total or 0) - ) - if event.phase == "writing" and self.yield_seconds > 0: + state = self._state + state.applying = None if event.finished else (event.done, event.total or 0) + finished_source = event.done > self._yielded + self._yielded = 0 if event.finished else event.done + if finished_source and not event.finished and self.yield_seconds > 0: self.sleep(self.yield_seconds) if event.done == 0 and not event.finished and event.label is None: self.publish() diff --git a/tests/integration/test_sync_serve.py b/tests/integration/test_sync_serve.py index b2dddb9..130de8d 100644 --- a/tests/integration/test_sync_serve.py +++ b/tests/integration/test_sync_serve.py @@ -14,6 +14,7 @@ from docgrep.api.container import Services, create_container from docgrep.api.http import create_app from docgrep.api.sync_task import SyncTask +from docgrep.domain.sync import SyncStatus from docgrep.usecases.sync_index import SyncIndex from resume_support import close from roots_support import settings @@ -39,6 +40,9 @@ def request_stop(self) -> None: def publish(self, *, running: bool = True) -> None: self.published.append(running) + def status(self) -> SyncStatus: + return SyncStatus(enabled=True, running=True) + def _services(sync: Any) -> Services: class Anything: @@ -132,17 +136,19 @@ def test_shutdown_during_an_applied_run_finishes_the_current_source( rig.clock.advance(120) in_run, release = threading.Event(), threading.Event() - calls: list[float] = [] - - def at_first_source(seconds: float) -> None: - # The first pause is before any source; the second is at the start - # of the first one, which is then in progress when the stop comes. - calls.append(seconds) - if len(calls) == 2: + update: Any = services.update + embed = update.embedder.embed + calls: list[int] = [] + + def embed_first_source_slowly(texts: Any) -> Any: + # The first source is being embedded when the server is asked to stop. + calls.append(len(texts)) + if len(calls) == 1: in_run.set() assert release.wait(timeout=10) + return embed(texts) - sync.sleep = at_first_source + update.embedder.embed = embed_first_source_slowly async def run() -> Any: task = SyncTask(lambda: replace(services, sync=sync), interval=3600) From 0815ade55371bcb95037e2d782c70662fce4aa48 Mon Sep 17 00:00:00 2001 From: Viktor Mikhalkin <9029022+ViktorMikhalkin@users.noreply.github.com> Date: Fri, 23 Oct 2026 22:21:42 +0300 Subject: [PATCH 8/9] docs: describe automatic sync README: enabling sync and its settings, what it does and never does (OCR, new roots, held removals, paused roots), how to read its status and logs, and the removal guard section for unpaired removals. ADR-0029 records sync inside `docgrep serve` (polling with settling, the scoped update, the holds) and the amendment of ADR-0027: moves do not count against the removal guard. Closes #45 --- README.md | 57 +++++++++++++++++--- docs/adr/0029-automatic-sync-inside-serve.md | 55 +++++++++++++++++++ 2 files changed, 104 insertions(+), 8 deletions(-) create mode 100644 docs/adr/0029-automatic-sync-inside-serve.md diff --git a/README.md b/README.md index 8a7b99a..d33890a 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,8 @@ uv run docgrep update --plan # what would change; writes nothing uv run docgrep update # after adding, changing, moving or removing files ``` +To have `docgrep serve` do this for you while it runs, set `DOCGREP_SYNC=true` (see [Automatic sync](#automatic-sync)). + ### 5. Use it from other tools ```sh @@ -259,6 +261,8 @@ Failed sources: A file that was moved, renamed or copied, or a backup folder added as a root, is indexed by copying the passages already stored for its content, without parsing or embedding (see [Moved, renamed and copied files](#moved-renamed-and-copied-files)). +While `docgrep serve` runs, [Automatic sync](#automatic-sync) can apply these changes by itself. + See [Dry run](#dry-run), [Updating the index](#updating-the-index), [Seeing the plan first](#seeing-the-plan-first), [Comparing a folder with the index](#comparing-a-folder-with-the-index), [Status](#status) and [Cancelling a build](#cancelling-a-build) for the details. ### Finding files that need OCR @@ -467,7 +471,7 @@ Removal guard: root books would lose 2 of 32 sources (at most 1 allowed); `docgr The groups are those of `update`: `added`, `modified` (with its reasons), `removed`, `retried`, `refreshed` (same content, new size or time: only the recorded values are updated, nothing is parsed or embedded; such a source also counts as `unchanged`), `unchanged` and the roots outside the run, and `moved` and `copied` (see [Moved, renamed and copied files](#moved-renamed-and-copied-files)). A move is listed under the root of its removed source, a copy under the root of the copy, each as `old -> new`; `copied` is printed only when there is one. Without `--paths` only the counts are printed; `unchanged` sources are never listed. -`--json` prints one object with the keys `note`; `roots`, a list with one object per root of the run with `root`, `recorded` (the sources the index records for it), `guard_threshold`, `guarded`, the counts `added`, `modified`, `removed`, `retried`, `refreshed`, `unchanged`, `moved` and `copied`, `modified_by_reason` (`content_changed`, `parser_changed`, `newly_supported`, `delegated_to_ocr`, `delegation_ended`) and `paths` (`null` without `--paths`; otherwise `added`, `modified` as `path` and `reason`, `removed`, `retried`, `refreshed`, `moved` as `removed` and `added`, and `copied` as `original` and `copy`); `total`, the counts of the whole run; `outside_run`, a list of `root` and `sources`; and `guarded_roots`, a list of `root`, `removals`, `recorded`, `threshold` and `reason` (`empties_root` when every recorded source of the root would be removed, otherwise `threshold`). +`--json` prints one object with the keys `note`; `roots`, a list with one object per root of the run with `root`, `recorded` (the sources the index records for it), `guard_threshold`, `guarded`, `unpaired_removals` and `paired_removals` (see [The removal guard](#the-removal-guard)), the counts `added`, `modified`, `removed`, `retried`, `refreshed`, `unchanged`, `moved` and `copied`, `modified_by_reason` (`content_changed`, `parser_changed`, `newly_supported`, `delegated_to_ocr`, `delegation_ended`) and `paths` (`null` without `--paths`; otherwise `added`, `modified` as `path` and `reason`, `removed`, `retried`, `refreshed`, `moved` as `removed` and `added`, and `copied` as `original` and `copy`); `total`, the counts of the whole run; `outside_run`, a list of `root` and `sources`; and `guarded_roots`, a list of `root`, `removals` (the unpaired ones), `recorded`, `threshold` and `reason` (`empties_root` when every recorded source of the root would be removed, otherwise `threshold`), and `paired` when some removals of the root are paired with an added file. ### Moved, renamed and copied files @@ -478,7 +482,7 @@ A source is identified by its stored path, so a file that is moved, renamed or c - **Not reused**: the stored source must be indexed, with the same content hash, size and file type, the parser version the file would be parsed with now and the embedding model of the index. An original delegated to its OCR output, or an OCR output, is never reused (see [OCR](#ocr)). A failed source that is retried goes through the normal pipeline. Any of these is processed as an addition or a modification. - **A full build** reuses only within its own run: of two files with the same content, the first is indexed and the next ones are copies of it (the build summary says `N indexed (M copied from a file with the same content)`). It never reuses the index it replaces. -A move counts against the [removal guard](#the-removal-guard) like any removal of its root. +A move loses no content, so it does not count against the [removal guard](#the-removal-guard) of its old root. If a copy fails in the store, what was written of the new source is deleted, it is recorded as `failed:store_error` and the old source of a move stays; the next update retries it. A move whose copy is stored but whose old source could not be removed (or whose process stopped in between) leaves both indexed, and the next update removes the old one, whose file is gone. @@ -490,13 +494,13 @@ Passage ids follow the path (ADR-0023), so the passages of a moved or renamed fi ### The removal guard -Before applying anything, `docgrep update` checks the planned removals of each root of the run, sources that moved or that the include and exclude patterns no longer match included. When they exceed `min(DOCGREP_UPDATE_GUARD_RATIO × the sources the index records for the root, DOCGREP_UPDATE_GUARD_COUNT)` (defaults `0.05` and `50`; a root with fewer than 20 recorded sources is checked against the count alone), or when they would remove every source the index records for a root, whatever its size and the thresholds (an existing but empty folder, such as a mount point without its disk, can still be read), the update writes nothing at all, not even refreshed times, prints the plan, names the roots and exits with 1: +Before applying anything, `docgrep update` checks the planned removals of each root of the run, the sources that the include and exclude patterns no longer match included. **Only removals that lose content count.** A removed source that pairs with an added file of the same content hash and size (the one-to-one pairing of [Moved, renamed and copied files](#moved-renamed-and-copied-files)) is a move: it is not counted against its old root, whether or not its passages are reused (`DOCGREP_REUSE_BY_HASH=false` changes nothing here). The unpaired removals are compared with `min(DOCGREP_UPDATE_GUARD_RATIO × the sources the index records for the root, DOCGREP_UPDATE_GUARD_COUNT)` (defaults `0.05` and `50`; a root with fewer than 20 recorded sources is checked against the count alone), or, when they would remove every source the index records for a root, whatever its size and the thresholds (an existing but empty folder, such as a mount point without its disk, can still be read), the update writes nothing at all, not even refreshed times, prints the plan, names the roots and exits with 1. A root whose sources all moved to another folder or root is therefore not emptied, while the same number of deletions without new copies is. ```text removal_guard: Update stopped before any change: root 'books' would lose 2 of 32 sources (at most 1 allowed). Check that the root folder is mounted, complete and the right one; run `docgrep update --yes` to apply the removals anyway ``` -A root that would be emptied is reported as `root 'notes' would lose all its 2 sources`, in the error and in `docgrep update --plan`. +A root that would be emptied is reported as `root 'notes' would lose all its 2 sources`, in the error and in `docgrep update --plan`. When some removals of a guarded root are paired, the message adds `(not counted: N paired with an added file)` and `docgrep update --plan` shows the two counts apart (`Removals: 2 unpaired (counted by the removal guard), 1 paired with an added file (not counted).`); the JSON of the plan has `unpaired_removals` and `paired_removals` for each root. An unmounted disk, a sync client that briefly emptied a folder or a root saved with the wrong path would otherwise remove a large part of the index, and rebuilding it takes hours. Check the folder, or `docgrep update --plan --paths`, then run: @@ -504,7 +508,7 @@ An unmounted disk, a sync client that briefly emptied a folder or a root saved w uv run docgrep update --yes # apply the plan even when the guard would stop it ``` -`DOCGREP_UPDATE_GUARD_RATIO` takes a number from 0 to 1 and `DOCGREP_UPDATE_GUARD_COUNT` a non-negative integer; `0` for the count makes every removal need `--yes`. `docgrep forget-root` is not guarded: it has its own `--yes`. +`DOCGREP_UPDATE_GUARD_RATIO` takes a number from 0 to 1 and `DOCGREP_UPDATE_GUARD_COUNT` a non-negative integer; `0` for the count makes every removal need `--yes`. `docgrep forget-root` is not guarded: it has its own `--yes`. [Automatic sync](#automatic-sync) applies the same rule to every cycle and holds the removals of a guarded root instead of stopping. ### Comparing a folder with the index @@ -531,7 +535,44 @@ Folder /backup/library against root books, compared by size and modification tim `--paths` lists the paths of every group but `same`. `--json` prints one object with `folder`, `root`, `compared_by` (`stat` or `hash`), `hashed`, the counts `same`, `different`, `only_in_index` and `only_in_folder`, and `paths` (`null` without `--paths`; otherwise `different`, `only_in_index` and `only_in_folder`). A file whose content changed while its size and time stayed the same is found only with `--hash`. -The design of the plan, the guard and `diff` is recorded in [ADR-0027](docs/adr/0027-update-plan-removal-guard-and-diff.md). +The design of the plan, the guard and `diff` is recorded in [ADR-0027](docs/adr/0027-update-plan-removal-guard-and-diff.md); that moves do not count against the guard is recorded in [ADR-0029](docs/adr/0029-automatic-sync-inside-serve.md). + +### Automatic sync + +`docgrep serve` can keep the index following the configured roots, without a manual `docgrep update`: added, changed, removed, moved and renamed files are applied within minutes, safely enough to run unattended. It is off by default. + +```sh +# in ~/docgrep.env (the file you source before running docgrep) +export DOCGREP_SYNC=true +``` + +Restart `docgrep serve` and watch `docgrep status` (the `Sync:` lines, or the `sync` section of `docgrep status --json`). Sync lives only in the server: there is no `docgrep sync` command, and neither HTTP nor MCP can enable, trigger or configure it. It uses the same models as search, loaded once, and the server answers while they load. It reads the corpus roots when the server starts, so restart it after `docgrep roots add`. In local mode it takes turns with searches on the single store; a Qdrant server (`DOCGREP_QDRANT_URL`) avoids that wait. A manual `docgrep update` still works when the store allows a second process (a Qdrant server); local mode needs the server stopped, as before. + +Settings: + +- `DOCGREP_SYNC`: run sync in `docgrep serve` (default `false`). +- `DOCGREP_SYNC_INTERVAL_SECONDS`: the time between scans (default `120`). +- `DOCGREP_SYNC_SETTLE_SECONDS`: a file must be at least this old (default `60`). +- `DOCGREP_SYNC_RECONCILE_HOURS`: how often whole roots are compared with the index again (default `24`). + +**What it does.** Every interval it scans the roots from directory entries alone, with the same patterns as an update (stat first; a file is hashed only when its size or time changed). A change is applied when it has settled: an added or modified file when its size and modification time were identical in two consecutive scans and its modification time is older than the settle time (so a file that is still being copied or written waits), a removal when the file was absent in two consecutive scans of a readable root. Settled changes are applied by an update limited to those paths, recorded as a run of kind `sync` (`latest_run` in `docgrep status`), one source at a time with a short pause after each, so a search is not starved of the models. Moves and renames reuse the stored passages and embed nothing. When another operation holds the index lock the cycle ends and its changes wait for the next one. A file that never settles is logged after 30 minutes (`sync_unsettled`, counts only) and keeps being watched. Names that sync never adds, changes or removes, on top of the include and exclude patterns: `*.part`, `*.crdownload`, `*.tmp`, `~$*`, `.~lock.*#`, `.#*`, `*.swp`, and the folders `.obsidian/`, `.git/` and `.trash/`. + +**Start-up and the daily check.** When it starts, and every `DOCGREP_SYNC_RECONCILE_HOURS`, sync compares each readable root with the index as `docgrep update --plan` does, which also finds sources to retry, parsers that changed and OCR delegations. What it finds is applied at once, except removals, which still need the second scan. A change made while the server was stopped is therefore caught within one interval of the start. + +**What it never does, and when it holds back.** + +- A root that is missing, unmounted or unreadable is **paused**: nothing in it is scanned, added or removed. When it is readable again it is compared with the index in full on the next cycle. +- Removals that trip the [removal guard](#the-removal-guard) of their root in a cycle are **held**: additions and changes still apply, the removals stay in the index and in `docgrep status` (`held_removals`, with a few stored paths in `docgrep status` only). Once a root has held removals, its new removals join them. Only a manual `docgrep update --yes` applies them; they are dropped if the files reappear. Moves pass the guard, as in an update. +- A file that failed is retried only when its size or modification time changes, at the daily comparison and at start-up. +- A **newly configured root** is not synced until its first manual `docgrep update` has completed; it is listed under `awaiting_first_update`. +- When the configuration differs from the index in a setting that decides vectors or chunking, a build or a re-encode is unfinished, or no index exists, sync stays idle and says why (`idle_reason`). +- **OCR** is never started by sync. The `*.ocr.pdf` files that `docgrep ocr` writes are picked up as added files. + +**Reading its status.** `docgrep status` shows, when sync is enabled, a `Sync:` line with the interval, the settle time and the last scan, then the last applied run, the pending changes (seen but not applied), how many are still settling, the held removals per root, the paused roots, the roots awaiting a first update, what it is applying right now and why it is idle. The JSON keys are listed under [Status](#status). `docgrep status` is another process, so it reads the file that the server rewrites after every cycle (`sync-status.json` in the data directory) and says `enabled, but no server is running it` when that server is gone. `GET /info` and the MCP `get_index_status` give the same section from the live state of the server, with counts and root names only and never a path. + +Each cycle logs one `sync_cycle` record with counts only (roots scanned, pending, held removals, what was applied and how long it took), never a path; an applied run logs like an update (`index_update_started`, `index_update_finished`). Held removals log `sync_removals_held`. + +The design is recorded in [ADR-0029](docs/adr/0029-automatic-sync-inside-serve.md). ### Cancelling a build @@ -948,7 +989,7 @@ curl -X POST http://127.0.0.1:8000/search \ The body of `POST /search` takes `query`, an optional `top_n`, an optional `collapse_duplicates` (see [Duplicate passages](#duplicate-passages)) and the optional filter fields (see [Filtered search](#filtered-search)); unknown fields are rejected. -`POST /sources/browse` lists the indexed sources (see [Browse sources](#browse-sources)). `POST /passages/context` returns passages with the text around them (see [Passage context](#passage-context)). `POST /index/build` triggers the same full rebuild as `docgrep index`. `POST /scans/discover` runs the read-only scan discovery with an optional JSON body `{"path_glob": ..., "file_types": [...]}` and returns the same report as `docgrep discover-scans --json` (see [Scan discovery](#scan-discovery)). `POST /scans/process` runs an OCR batch synchronously, like `POST /index/build`, with a JSON body of `paths` (a list), `path_glob` or `all_discovered: true`, plus optional `languages` and `force`, and returns the same report as `docgrep ocr --json` (see [OCR](#ocr)); `docgrep cancel` stops it, missing OCR tools give 503 with category `ocr_tools_missing` and a held lock gives 409. `POST /scans/discover` also accepts `include_already_processed`. `GET /health` and `GET /info` provide basic service information: `GET /info` returns `version`, `backend` and `roots`, the configured [corpus roots](#corpus-roots) in order, each with `name`, `path` (absolute), `legacy` and `readable` (the folder exists and can be listed; nothing is walked). It does not read the index, so it gives no source counts (see `docgrep status`), and lists no root found only in the index. No endpoint adds or changes roots, and `POST /index/build` and `POST /scans/process` take no folder: they use the saved roots, and OCR `paths` must lie inside one of them. +`POST /sources/browse` lists the indexed sources (see [Browse sources](#browse-sources)). `POST /passages/context` returns passages with the text around them (see [Passage context](#passage-context)). `POST /index/build` triggers the same full rebuild as `docgrep index`. `POST /scans/discover` runs the read-only scan discovery with an optional JSON body `{"path_glob": ..., "file_types": [...]}` and returns the same report as `docgrep discover-scans --json` (see [Scan discovery](#scan-discovery)). `POST /scans/process` runs an OCR batch synchronously, like `POST /index/build`, with a JSON body of `paths` (a list), `path_glob` or `all_discovered: true`, plus optional `languages` and `force`, and returns the same report as `docgrep ocr --json` (see [OCR](#ocr)); `docgrep cancel` stops it, missing OCR tools give 503 with category `ocr_tools_missing` and a held lock gives 409. `POST /scans/discover` also accepts `include_already_processed`. `GET /health` and `GET /info` provide basic service information: `GET /info` returns `version`, `backend`, `rerank_pool`, `dense_hnsw_ef`, `sync` (the live section of [Automatic sync](#automatic-sync), without paths) and `roots`, the configured [corpus roots](#corpus-roots) in order, each with `name`, `path` (absolute), `legacy` and `readable` (the folder exists and can be listed; nothing is walked). It does not read the index, so it gives no source counts (see `docgrep status`), and lists no root found only in the index. No endpoint adds or changes roots, and `POST /index/build` and `POST /scans/process` take no folder: they use the saved roots, and OCR `paths` must lie inside one of them. When `rerank_skipped` is true, results are ordered by their stage-1 hybrid rank and `score` is derived from the dense similarity. It is not comparable with reranked scores and need not be monotonic with the returned order. @@ -957,7 +998,7 @@ When `rerank_skipped` is true, results are ordered by their stage-1 hybrid rank docgrep offers its search to MCP clients, such as Claude Code, as read-only tools over the same use cases as the command line and the HTTP API. Results are verbatim passages from your own documents, never summaries; clients are told to cite the path, the heading path and, when the passage has one, the page. Passage text is data from your documents, not instructions for the client. The design is recorded in [ADR-0019](docs/adr/0019-mcp-server-as-api-front-end.md). - `find_passages`: ranked verbatim passages for `query`, with an optional `top_n` (default 10, at most `DOCGREP_MCP_MAX_TOP_N`), `collapse_duplicates` and the filters of [Filtered search](#filtered-search), `roots` included; the structured result is the body of `POST /search`, and DOCX and EPUB passages have a null page number. Filters that match nothing give an empty result, not an error. -- `get_index_status`: the report of `docgrep status --json`, including the failed sources and their errors and the `roots` (names, flags and counts; never a path). +- `get_index_status`: the report of `docgrep status --json`, including the failed sources and their errors, the `roots` (names, flags and counts; never a path) and the `sync` section (counts and root names; never a path). - `browse_sources`: one page of indexed sources with their metadata and passage counts, filtered and sorted as described in [Browse sources](#browse-sources) (`page_size` default 50, at most `DOCGREP_MCP_MAX_PAGE_SIZE`); the structured result is the body of `POST /sources/browse`. A page too large for one tool result keeps its first sources and says so in `hint`. - `expand_passage_context`: the passages with the given `passage_ids` (at most `DOCGREP_MCP_MAX_CONTEXT_PASSAGES`) and the text around them, as described in [Passage context](#passage-context); the structured result is the body of `POST /passages/context`. A result too large for one tool result keeps its first entries and says so in `hint`. diff --git a/docs/adr/0029-automatic-sync-inside-serve.md b/docs/adr/0029-automatic-sync-inside-serve.md new file mode 100644 index 0000000..b585ee3 --- /dev/null +++ b/docs/adr/0029-automatic-sync-inside-serve.md @@ -0,0 +1,55 @@ +# ADR-0029: Automatic sync inside `docgrep serve` + +**Status:** Accepted +**Date:** 2026-10-23 +**Amends:** ADR-0027 (a removal that pairs with an added file does not count against the removal guard). + +## Context + +The index followed the corpus only when someone ran `docgrep update`. A library that changes every day (notes, downloads, a synced folder) is out of date until then, and an update that is forgotten is a search that misses the newest files. Issue #45 asks for the index to follow the configured roots by itself (UC-21). The maintainer decided that it runs inside `docgrep serve`. + +Running unattended changes what an update must be. Files are seen while they are still being written or downloaded. A root can disappear for a moment (a disk that is not mounted, a sync client that empties a folder). A failed file must not fail again every few minutes. Embedding shares the GPU with search. And the pieces to do it safely already exist: a read-only plan, the removal guard as a pure function (ADR-0027), reuse of stored passages for moves, renames and copies (ADR-0028), and a progress observer on long operations. + +## Decision + +**Where it runs.** With `DOCGREP_SYNC=true` (default `false`) the lifespan of `docgrep serve` starts one background task and stops it on shutdown: no new cycle, and an applied run finishes the source it is on. The task takes the container that search uses, so the models are loaded once, and asks for it on a worker thread, so the server answers while they load. There is no separate sync process and no sync command, and neither HTTP nor MCP can enable, trigger or configure it. Its state is in memory and is rebuilt by the first cycle, so a restart loses nothing. + +**Detection by periodic scan.** Every `DOCGREP_SYNC_INTERVAL_SECONDS` (default 120) the task scans the readable roots with the enumeration and patterns of the update plan, from directory entries alone: a file differs from the index when its size or modification time differs from the record, and it is hashed only later, by the update. No file-system notification library is used, which keeps the dependencies and works on network mounts. + +**Settling.** An added or modified file is applied when its size and modification time were identical in two consecutive scans and its modification time is older than `DOCGREP_SYNC_SETTLE_SECONDS` (default 60). A removal is applied when the file was absent in two consecutive scans of a readable root. A built-in ignore list applies on top of the root patterns: `*.part`, `*.crdownload`, `*.tmp`, `~$*`, `.~lock.*#`, `.#*`, `*.swp` and the folders `.obsidian/`, `.git/` and `.trash/`; sources already recorded under such a name are neither changed nor removed. A file that has not settled after 30 minutes is logged once and stays watched. + +**Applying.** A cycle with settled changes runs the update use case scoped to their paths (`UpdateScope`): only these paths are compared and changed, every other source counts as unchanged, no other file is added, and the corpus is asked for those files alone (one stat each) instead of being walked, so a root that is gone cannot make the rest look deleted. The run is recorded with the kind `sync`. Moves reuse stored passages (ADR-0028). The cycle only tries the index lock; when another operation holds it the cycle ends and the changes stay pending. Sources are processed one at a time with a short pause after each, so a waiting search gets the models. + +**Reconciliation.** When the task starts, every `DOCGREP_SYNC_RECONCILE_HOURS` (default 24) and for a root that is readable again, the readable roots are compared with the index with the update plan itself, which also finds what a scan of directory entries cannot: sources to retry, parsers whose version changed and OCR delegations. What it finds settles at once, except removals (second scan). The target of a move waits for the removal of its old path, so that both are applied in one run and pair up. + +**Safety holds.** + +- A root that is missing, unmounted or unreadable is paused: nothing in it is scanned, added or removed, and it is compared in full when it is readable again. +- Removals that trip the removal guard (ADR-0027, per root and cycle, as a pure function) are held back: the update leaves them out and names them, while additions, modifications and moves still apply. Once a root has held removals, its later removals are judged together with them. Only a manual `docgrep update --yes` applies held removals; they are dropped when the files reappear. +- A file that failed is retried only when its size or modification time changes, at the daily comparison and at start-up. +- A newly configured root is not synced until its first manual `docgrep update` has completed; sync reports it as awaiting a first update. +- A configuration that drifted from the index, an unfinished build or re-encode, or a missing index keep sync idle, and it says why. +- OCR is never started by sync; the OCR outputs that `docgrep ocr` writes are picked up as added files. + +**Moves do not count against the removal guard (amends ADR-0027).** A removed source that pairs with an added file of the same content hash and size (the one-to-one pairing of ADR-0027) loses no content. It is no longer counted against the guard of its old root, whether or not reuse (ADR-0028) applies to it. Removals without a pair count as before, including `empties_root`: a root whose sources all moved elsewhere is not emptied. Without this, sync would hold the removals of a large folder that was only moved, and `docgrep update` would ask for `--yes` for it. The plan and the guard message show the paired and the unpaired counts separately. ADR-0027 itself is unchanged. + +**Status.** `docgrep status`, `GET /info` and the MCP `get_index_status` body gain a `sync` section: enabled, running, interval, last scan, last applied run, pending counts by kind, changes still settling, held removals per root, paused roots, roots awaiting a first update, the idle reason, the last error and the progress of an applied run (through the progress observer). `docgrep status` is another process, so the server writes the section to `sync-status.json` in the data directory after every cycle and `docgrep status` reads it (with the process id, to tell a dead server from a live one); `docgrep status` adds a few stored paths of each root with held removals, and MCP never shows a path. One `sync_cycle` log record per cycle carries counts only. + +## Consequences + +An index of a changing library stays current within a few minutes of the files settling, at the cost of a scan of directory entries every interval (about a quarter of a second for 3,000 files). Changes made while the server is stopped are found at the start. + +A deletion that is real but large (a whole collection removed on purpose) is held until someone runs `docgrep update --yes`; that is the intended trade for running unattended. A file that is rewritten more often than the settle time never gets indexed until it rests, and an editor that writes a temporary file and renames it is seen after the second scan. + +Moving a large folder is applied as moves (a copy of stored passages each) without the guard asking for confirmation, because nothing is lost. A file that is moved and changed at the same time is an unpaired removal and an addition, and counts against the guard. + +The state of sync is in memory; a crash loses the pending list only, and the start-up comparison rebuilds it. The running `serve` holds the local-mode store, so a manual update in local mode needs the server stopped, as before; the calls of a local-mode store are now serialized, because a search and a sync write run in different threads of the server and the local client does not synchronize its arrays. + +## Alternatives considered + +- **File-system notifications:** miss events on network mounts and in containers, need a dependency per platform, and still need a periodic comparison to be trusted. +- **A separate `docgrep sync` process:** it cannot open the single-process local store and would load a second copy of the models. +- **Applying changes as soon as they are seen:** half-written and half-downloaded files would be indexed and then indexed again. +- **A guard that counts moves too, as in ADR-0027:** a moved folder would be held and need `--yes` in sync and in update, although it loses nothing. +- **Holding every removal until a person confirms:** unattended sync would never delete anything; the guard holds only the removals that look like an accident. +- **Keeping the sync status only in memory:** `docgrep status` could not show it. From a1773a258c082e7e9f69c36f6b31646dc2b6e0f0 Mon Sep 17 00:00:00 2001 From: Viktor Mikhalkin <9029022+ViktorMikhalkin@users.noreply.github.com> Date: Fri, 23 Oct 2026 22:22:50 +0300 Subject: [PATCH 9/9] fix(usecases): keep what a cancelled sync run did not reach A sync run that was cancelled dropped all its paths from the pending list and marked them as tried, so the files it had not reached were not applied until they changed. The paths now stay pending and are applied by the next cycle. An unexpected failure inside a cycle is recorded in the status by category only (a message can carry a path), and the MCP status scrubs paths from the idle reason and the last error. Story: batch 28, #45. --- src/docgrep/api/mcp.py | 5 +++ src/docgrep/usecases/sync_index.py | 27 ++++++++++++- tests/integration/test_sync_status.py | 16 ++++++++ tests/unit/test_sync_index.py | 55 +++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 2 deletions(-) diff --git a/src/docgrep/api/mcp.py b/src/docgrep/api/mcp.py index f06986e..a992254 100644 --- a/src/docgrep/api/mcp.py +++ b/src/docgrep/api/mcp.py @@ -859,6 +859,11 @@ def _scrub_report(report: dict[str, Any], paths: tuple[Path, ...]) -> dict[str, store = report["vector_store"] if store["reason"] is not None: store["reason"] = scrub_tool_output(store["reason"], paths) + sync = report.get("sync") + if sync is not None: + for key in ("idle_reason", "last_error"): + if sync[key] is not None: + sync[key] = scrub_tool_output(sync[key], paths) return report diff --git a/src/docgrep/usecases/sync_index.py b/src/docgrep/usecases/sync_index.py index 4b60048..8d271c2 100644 --- a/src/docgrep/usecases/sync_index.py +++ b/src/docgrep/usecases/sync_index.py @@ -270,7 +270,17 @@ def run_cycle(self) -> CycleResult: started = self.clock() with self._guard: self._state.last_scan_at = self.now().isoformat() - result = self._cycle() + try: + result = self._cycle() + except Exception as error: + category = ( + error.category if isinstance(error, DocgrepError) else "unexpected" + ) + _LOG.exception("sync_cycle_failed", extra={"category": category}) + with self._guard: + self._state.applying = None + self._state.last_error = category + result = CycleResult(skipped="cycle_failed") _LOG.info( "sync_cycle", extra={ @@ -516,7 +526,8 @@ def _apply( _LOG.warning("sync_apply_failed", extra={"category": category}) with self._guard: state.applying = None - state.last_error = f"{category}: {error}" + # The category only: a message may carry a path. + state.last_error = category return CycleResult( scanned_roots=len(scanned), skipped="apply_failed", @@ -538,6 +549,12 @@ def _record_applied( ) -> None: state = self._state held = set(summary.held_removals) + if summary.cancelled: + # Which paths were reached is not known here; the processed ones + # stop differing from the index at the next scan and the rest stay + # pending, so nothing is lost and nothing is taken for tried. + self._finish_applied(summary, roots, held) + return for path in scope.paths: entry = state.pending.pop(path, None) if entry is not None: @@ -547,6 +564,12 @@ def _record_applied( touched = {roots.root_of(path) for path in scope.paths} for root in touched: state.held.pop(root, None) + self._finish_applied(summary, roots, held) + + def _finish_applied( + self, summary: UpdateSummary, roots: CorpusRoots, held: set[str] + ) -> None: + state = self._state for path in held: state.held.setdefault(roots.root_of(path), set()).add(path) state.applying = None diff --git a/tests/integration/test_sync_status.py b/tests/integration/test_sync_status.py index 1b9fc68..1c30cc6 100644 --- a/tests/integration/test_sync_status.py +++ b/tests/integration/test_sync_status.py @@ -207,3 +207,19 @@ def drifted() -> None: body = held.sync.status().to_dict() assert body["idle_reason"].startswith("config_drift: ") assert held.sync.status().idle_reason == body["idle_reason"] + + +def test_the_mcp_status_scrubs_paths_from_the_idle_reason(held: Rig) -> None: + assert held.services is not None + sync = held.sync + # Internal state, to place a path where a reason could carry one. + sync._state.idle_reason = f"validation_error: cannot use {held.books}/x" + + body = _mcp_body( + held, + held.services, + status=lambda: create_status(held.settings(sync=True), sync=sync.status), + ) + + assert str(held.books) not in json.dumps(body["sync"]) + assert body["sync"]["idle_reason"].startswith("validation_error: ") diff --git a/tests/unit/test_sync_index.py b/tests/unit/test_sync_index.py index d3adee9..223fa02 100644 --- a/tests/unit/test_sync_index.py +++ b/tests/unit/test_sync_index.py @@ -600,3 +600,58 @@ def test_paused_roots_keep_their_sources_and_are_reconciled_when_back() -> None: assert rig.sync.status().paused_roots == () assert result.applied is not None and result.applied.modified == 1 assert "notes/n0.txt" in rig.recorded() + + +def test_a_cancelled_run_keeps_what_it_did_not_reach() -> None: + rig = Rig() + rig.start() + for index in range(3): + rig.files[f"new{index}.txt"] = rig.aged(f"new document number {index}") + rig.cycle() + parse = rig.harness.parser.parse + + def parse_and_cancel(source: Source) -> Any: + rig.update.request_cancel() + return parse(source) + + rig.harness.parser.parse = parse_and_cancel # type: ignore[method-assign] # test double + + cancelled = rig.cycle() + + assert cancelled.applied is not None and cancelled.applied.cancelled + done = {p for p in rig.recorded() if p.startswith("new")} + assert len(done) == 1 + assert rig.sync.status().last_applied is not None + assert rig.sync.status().last_applied.cancelled # type: ignore[union-attr] # checked just above + + rig.harness.parser.parse = parse # type: ignore[method-assign] # test double + again = rig.cycle() + + # The two files it did not reach are applied; the first is not repeated. + assert again.applied is not None and again.applied.added == 2 + assert {p for p in rig.recorded() if p.startswith("new")} == { + "new0.txt", + "new1.txt", + "new2.txt", + } + + +def test_an_unexpected_failure_is_reported_by_category_and_the_next_cycle_runs() -> ( + None +): + rig = Rig() + rig.start() + real = rig.harness.manifest.committed_sources + + def broken() -> Any: + raise RuntimeError("cannot read /srv/secret/path") + + rig.harness.manifest.committed_sources = broken # type: ignore[method-assign] # test double + + result = rig.cycle() + + assert result.skipped == "cycle_failed" + assert rig.sync.status().last_error == "unexpected" + + rig.harness.manifest.committed_sources = real # type: ignore[method-assign] # test double + assert rig.cycle().skipped is None