From 93f58360b218acac743884930f33d427cf6687cf Mon Sep 17 00:00:00 2001 From: plind-junior <59729252+plind-junior@users.noreply.github.com> Date: Tue, 30 Jun 2026 19:28:48 +0900 Subject: [PATCH] fix(storage): survive unreadable artifact files and pin utf-8 i/o MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `vouch pending` — and every bulk `list_*` path — crashed with a yaml ReaderError when a single proposal file held a control character: one unreadable file took down the whole listing. add a `_load_or_skip` helper that logs a warning and skips the bad file instead of aborting, applied across proposals, claims, entities, relations, evidence, sessions, sources. the control character traces back to vouch's own file i/o relying on the locale default encoding: on a non-utf-8 locale (e.g. latin-1) read_text / write_text / open mangle non-ascii into raw bytes the yaml loader rejects, and can even crash on write. pin encoding="utf-8" on all text-mode pathlib i/o under src/vouch. adds regression tests for the resilient listing. --- CHANGELOG.md | 8 +++ src/vouch/auto_pr.py | 4 +- src/vouch/cli.py | 2 +- src/vouch/context.py | 2 +- src/vouch/embeddings/scorer.py | 4 +- src/vouch/embeddings/similarity.py | 2 +- src/vouch/health.py | 2 +- src/vouch/http_server.py | 2 +- src/vouch/index_db.py | 2 +- src/vouch/jsonl_server.py | 2 +- src/vouch/migrations/_legacy.py | 9 +-- src/vouch/migrations/journal.py | 2 +- src/vouch/migrations/manifest.py | 2 +- src/vouch/migrations/runner.py | 8 +-- src/vouch/migrations/schema.py | 2 +- src/vouch/openclaw/context_engine.py | 2 +- src/vouch/page_kinds.py | 2 +- src/vouch/pr_cache.py | 4 +- src/vouch/proposals.py | 4 +- src/vouch/scoping.py | 2 +- src/vouch/server.py | 2 +- src/vouch/stats.py | 2 +- src/vouch/storage.py | 96 ++++++++++++++++++---------- src/vouch/sync.py | 2 +- src/vouch/volunteer_context.py | 2 +- src/vouch/web/server.py | 2 +- tests/test_storage.py | 29 +++++++++ 27 files changed, 134 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8023df2d..34a3c9c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ All notable changes to vouch are documented here. Format follows ## [Unreleased] +### Fixed +- `vouch pending` (and every bulk `list_*` path) no longer crashes when a + single artifact file is unreadable — a corrupt or mojibake yaml is skipped + with a warning instead of aborting the whole listing. +- all text-mode file i/o under `src/vouch/` now pins `encoding="utf-8"`, so a + non-utf-8 locale (e.g. latin-1) can no longer mangle non-ascii claim text + into raw control bytes that the yaml loader rejects, nor crash on write. + ### Docs - example KBs now carry their own screenshots: `examples/README.md` and the `tiny/` + `decision-log/` READMEs embed terminal renders of `vouch status`, diff --git a/src/vouch/auto_pr.py b/src/vouch/auto_pr.py index c9edd2ac..943fbb5a 100644 --- a/src/vouch/auto_pr.py +++ b/src/vouch/auto_pr.py @@ -319,10 +319,10 @@ def detect_or_bootstrap_guidance(ctx: RepoCtx, engine: Engine, "---\nname: auto-pr-contributing\ndescription: synthesized contribution " f"guide for {ctx.repo}, derived from merged PRs.\n---\n\n" ) - skill.write_text(front + guide) + skill.write_text(front + guide, encoding="utf-8") codex_mirror = ctx.clone_dir / ".codex" / "auto-pr-contributing.md" codex_mirror.parent.mkdir(parents=True, exist_ok=True) - codex_mirror.write_text(guide) + codex_mirror.write_text(guide, encoding="utf-8") return guide diff --git a/src/vouch/cli.py b/src/vouch/cli.py index ca465cae..e3fa1d5e 100644 --- a/src/vouch/cli.py +++ b/src/vouch/cli.py @@ -2797,7 +2797,7 @@ def review_ui( auth_note = " (Bearer auth on)" if token else "" if open_browser and is_loopback: # Lazy-import webbrowser; some CI envs (headless) don't have a default - # browser configured and webbrowser.open() returns False rather than + # browser configured and webbrowser.open(encoding="utf-8") returns False rather than # raising — that's fine, the URL is also printed to stdout. When auth # is on, hand the browser the token once via ?token= so it can stash it. import threading diff --git a/src/vouch/context.py b/src/vouch/context.py index 423c2fad..6e9b08ce 100644 --- a/src/vouch/context.py +++ b/src/vouch/context.py @@ -54,7 +54,7 @@ def _configured_backend(store: KBStore) -> str: falls back to "auto". """ try: - loaded = yaml.safe_load(store.config_path.read_text()) + loaded = yaml.safe_load(store.config_path.read_text(encoding="utf-8")) except (OSError, yaml.YAMLError): return "auto" if not isinstance(loaded, dict): diff --git a/src/vouch/embeddings/scorer.py b/src/vouch/embeddings/scorer.py index 48dcd782..c408d74d 100644 --- a/src/vouch/embeddings/scorer.py +++ b/src/vouch/embeddings/scorer.py @@ -57,7 +57,7 @@ def evaluate( raise ValueError(f"unknown metric(s): {sorted(unknown)}; known: {sorted(known)}") totals = {m: 0.0 for m in metrics} n = 0 - with queries_file.open() as f: + with queries_file.open(encoding="utf-8") as f: for line in f: if not line.strip(): continue @@ -78,4 +78,4 @@ def evaluate( def write_report(out: dict[str, float], path: Path) -> None: - path.write_text(json.dumps(out, indent=2)) + path.write_text(json.dumps(out, indent=2), encoding="utf-8") diff --git a/src/vouch/embeddings/similarity.py b/src/vouch/embeddings/similarity.py index 69cf9ade..911ea621 100644 --- a/src/vouch/embeddings/similarity.py +++ b/src/vouch/embeddings/similarity.py @@ -23,7 +23,7 @@ def similarity_threshold(store: KBStore) -> float: """Resolve `review.similarity_threshold` from config, else dedup default.""" try: - loaded = yaml.safe_load(store.config_path.read_text()) + loaded = yaml.safe_load(store.config_path.read_text(encoding="utf-8")) if isinstance(loaded, dict): review = loaded.get("review") if isinstance(review, dict) and review.get("similarity_threshold") is not None: diff --git a/src/vouch/health.py b/src/vouch/health.py index db2f1aa8..a22cef7e 100644 --- a/src/vouch/health.py +++ b/src/vouch/health.py @@ -97,7 +97,7 @@ def _load_claims_for_lint(store: KBStore) -> tuple[list[Claim], list[Finding]]: for p in sorted(cdir.glob("*.yaml")): cid = p.stem try: - valid.append(Claim.model_validate(_yaml_load(p.read_text()))) + valid.append(Claim.model_validate(_yaml_load(p.read_text(encoding="utf-8")))) except ValidationError as e: tail = str(e).splitlines()[-1].strip() if str(e) else "validation failed" findings.append( diff --git a/src/vouch/http_server.py b/src/vouch/http_server.py index 19ffdfcb..081cda24 100644 --- a/src/vouch/http_server.py +++ b/src/vouch/http_server.py @@ -122,7 +122,7 @@ def load_serve_config(path: Path) -> ServeConfig: if not path.exists(): return ServeConfig() try: - raw = yaml.safe_load(path.read_text()) or {} + raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {} except yaml.YAMLError as e: raise ServeConfigError(f"could not parse {path}: {e}") from e if not isinstance(raw, dict): diff --git a/src/vouch/index_db.py b/src/vouch/index_db.py index 24bfe9f1..146d9f52 100644 --- a/src/vouch/index_db.py +++ b/src/vouch/index_db.py @@ -173,7 +173,7 @@ def _snippet_for(kb_dir: Path, kind: str, eid: str) -> str: path = kb_dir / kind / f"{eid}.md" if not path.exists(): return eid - text = path.read_text() + text = path.read_text(encoding="utf-8") return text[:200].replace("\n", " ") diff --git a/src/vouch/jsonl_server.py b/src/vouch/jsonl_server.py index ec738c6b..5231cac4 100644 --- a/src/vouch/jsonl_server.py +++ b/src/vouch/jsonl_server.py @@ -164,7 +164,7 @@ def _h_search(p: dict) -> dict: def _load_cfg(store: KBStore) -> dict: try: - loaded = yaml.safe_load((store.kb_dir / "config.yaml").read_text()) + loaded = yaml.safe_load((store.kb_dir / "config.yaml").read_text(encoding="utf-8")) except Exception: return {} return loaded if isinstance(loaded, dict) else {} diff --git a/src/vouch/migrations/_legacy.py b/src/vouch/migrations/_legacy.py index 426b5ac9..6b9dab10 100644 --- a/src/vouch/migrations/_legacy.py +++ b/src/vouch/migrations/_legacy.py @@ -55,7 +55,7 @@ class MigrationResult: def read_config(store: KBStore) -> dict[str, Any]: if not store.config_path.exists(): return {} - loaded = yaml.safe_load(store.config_path.read_text()) + loaded = yaml.safe_load(store.config_path.read_text(encoding="utf-8")) if loaded is None: return {} if not isinstance(loaded, dict): @@ -64,7 +64,8 @@ def read_config(store: KBStore) -> dict[str, Any]: def write_config(store: KBStore, config: dict[str, Any]) -> None: - store.config_path.write_text(yaml.safe_dump(config, sort_keys=False, allow_unicode=True)) + store.config_path.write_text( + yaml.safe_dump(config, sort_keys=False, allow_unicode=True), encoding="utf-8") def detect_version(store: KBStore) -> int: @@ -162,14 +163,14 @@ def _migration_0_to_1(store: KBStore, dry_run: bool) -> list[str]: required_ignores = ("proposed/", "state.db", "state.db-*") existing_ignores: list[str] = [] if gitignore_path.exists(): - existing_ignores = gitignore_path.read_text().splitlines() + existing_ignores = gitignore_path.read_text(encoding="utf-8").splitlines() missing_ignores = [line for line in required_ignores if line not in existing_ignores] if missing_ignores: changes.append("ensure .gitignore excludes proposed/ and state.db") if not dry_run: gitignore_path.parent.mkdir(parents=True, exist_ok=True) lines = [*existing_ignores, *missing_ignores] - gitignore_path.write_text("\n".join(lines).rstrip() + "\n") + gitignore_path.write_text("\n".join(lines).rstrip() + "\n", encoding="utf-8") return changes diff --git a/src/vouch/migrations/journal.py b/src/vouch/migrations/journal.py index 13d28e53..7a152da2 100644 --- a/src/vouch/migrations/journal.py +++ b/src/vouch/migrations/journal.py @@ -60,7 +60,7 @@ def latest_journal(store: KBStore) -> Path | None: def read_journal(path: Path) -> tuple[dict[str, object], list[JournalEntry]]: header: dict[str, object] = {} entries: list[JournalEntry] = [] - for line in path.read_text().splitlines(): + for line in path.read_text(encoding="utf-8").splitlines(): if not line.strip(): continue rec = json.loads(line) diff --git a/src/vouch/migrations/manifest.py b/src/vouch/migrations/manifest.py index b67ddda5..171a4a4b 100644 --- a/src/vouch/migrations/manifest.py +++ b/src/vouch/migrations/manifest.py @@ -70,7 +70,7 @@ def _validate_transforms(transforms: Any, where: str) -> list[dict[str, Any]]: def parse_manifest(path: Path) -> Manifest: try: - data = yaml.safe_load(path.read_text()) + data = yaml.safe_load(path.read_text(encoding="utf-8")) except yaml.YAMLError as e: raise ManifestError(f"{path.name}: invalid yaml: {e}") from e if not isinstance(data, dict): diff --git a/src/vouch/migrations/runner.py b/src/vouch/migrations/runner.py index 1192fead..6db98357 100644 --- a/src/vouch/migrations/runner.py +++ b/src/vouch/migrations/runner.py @@ -88,7 +88,7 @@ def _changed_files(store: KBStore, manifest: Manifest) -> list[tuple[Path, str, """Files whose content the manifest would actually change: (path, old, new).""" out: list[tuple[Path, str, str]] = [] for path in artifact_files(store.kb_dir, manifest.artifact): - old = path.read_text() + old = path.read_text(encoding="utf-8") new = transform_text(old, manifest.artifact, manifest.transforms) if new != old: out.append((path, old, new)) @@ -311,13 +311,13 @@ def verify(store: KBStore) -> dict[str, Any]: for path in artifact_files(store.kb_dir, kind): checked += 1 try: - model.model_validate(_yaml_load(path.read_text())) + model.model_validate(_yaml_load(path.read_text(encoding="utf-8"))) except Exception as e: errors.append({"path": str(path.relative_to(store.kb_dir)), "error": str(e)}) for path in artifact_files(store.kb_dir, "pages"): checked += 1 try: - match = _FRONTMATTER_RE.match(path.read_text()) + match = _FRONTMATTER_RE.match(path.read_text(encoding="utf-8")) front = _yaml_load(match.group(1)) if match else {} Page.model_validate({**(front or {}), "body": match.group(2) if match else ""}) except Exception as e: @@ -325,7 +325,7 @@ def verify(store: KBStore) -> dict[str, Any]: for meta in sorted((store.kb_dir / "sources").glob("*/meta.yaml")): checked += 1 try: - Source.model_validate(_yaml_load(meta.read_text())) + Source.model_validate(_yaml_load(meta.read_text(encoding="utf-8"))) except Exception as e: errors.append({"path": str(meta.relative_to(store.kb_dir)), "error": str(e)}) return { diff --git a/src/vouch/migrations/schema.py b/src/vouch/migrations/schema.py index 7ef9681c..4660d941 100644 --- a/src/vouch/migrations/schema.py +++ b/src/vouch/migrations/schema.py @@ -25,7 +25,7 @@ def read_schema_version(store: KBStore) -> str: p = schema_version_path(store) if not p.exists(): return BASELINE_SCHEMA_VERSION - raw = p.read_text().strip() + raw = p.read_text(encoding="utf-8").strip() if not raw: return BASELINE_SCHEMA_VERSION semver.parse(raw) # validate; raises ValueError on garbage diff --git a/src/vouch/openclaw/context_engine.py b/src/vouch/openclaw/context_engine.py index 6647cd1b..323bfaa8 100644 --- a/src/vouch/openclaw/context_engine.py +++ b/src/vouch/openclaw/context_engine.py @@ -99,7 +99,7 @@ def resolve_kb_root(*, workspace_dir: Path | None, kb_path: str | None) -> Path: def load_cfg(store: KBStore) -> dict[str, Any]: try: - loaded = yaml.safe_load(store.config_path.read_text()) + loaded = yaml.safe_load(store.config_path.read_text(encoding="utf-8")) except Exception: return {} return loaded if isinstance(loaded, dict) else {} diff --git a/src/vouch/page_kinds.py b/src/vouch/page_kinds.py index c675ccde..2f51c506 100644 --- a/src/vouch/page_kinds.py +++ b/src/vouch/page_kinds.py @@ -224,7 +224,7 @@ def _read_page_kinds(store: KBStore) -> dict[str, Any]: if not path.exists(): return {} try: - loaded = yaml.safe_load(path.read_text()) + loaded = yaml.safe_load(path.read_text(encoding="utf-8")) except yaml.YAMLError: return {} if not isinstance(loaded, dict): diff --git a/src/vouch/pr_cache.py b/src/vouch/pr_cache.py index 9958221c..3f499eab 100644 --- a/src/vouch/pr_cache.py +++ b/src/vouch/pr_cache.py @@ -460,7 +460,7 @@ def _record_from_json(d: dict[str, Any]) -> PRRecord: def load_cache(path: Path) -> dict[str, PRRecord]: if not path.exists(): return {} - raw = json.loads(path.read_text()) + raw = json.loads(path.read_text(encoding="utf-8")) return {str(d["number"]): _record_from_json(d) for d in raw.get("prs", [])} @@ -475,7 +475,7 @@ def save_cache(path: Path, repo: RepoRef, records: dict[str, PRRecord]) -> None: "prs": [_record_to_json(r) for r in prs_sorted], } tmp = path.with_suffix(path.suffix + ".tmp") - tmp.write_text(json.dumps(payload, indent=2, sort_keys=False)) + tmp.write_text(json.dumps(payload, indent=2, sort_keys=False), encoding="utf-8") tmp.replace(path) diff --git a/src/vouch/proposals.py b/src/vouch/proposals.py index 22d71d4f..6f1dd25a 100644 --- a/src/vouch/proposals.py +++ b/src/vouch/proposals.py @@ -333,7 +333,7 @@ def _approval_block_reason( if approved_by == proposal.proposed_by: cfg: dict[str, Any] = {} try: - loaded = yaml.safe_load((store.kb_dir / "config.yaml").read_text()) + loaded = yaml.safe_load((store.kb_dir / "config.yaml").read_text(encoding="utf-8")) if isinstance(loaded, dict): cfg = loaded except Exception: @@ -577,7 +577,7 @@ def expire_pending_after_days(store: KBStore, *, override: int | None = None) -> if override is not None: return override try: - loaded = yaml.safe_load(store.config_path.read_text()) + loaded = yaml.safe_load(store.config_path.read_text(encoding="utf-8")) except Exception: return _DEFAULT_EXPIRE_PENDING_DAYS if not isinstance(loaded, dict): diff --git a/src/vouch/scoping.py b/src/vouch/scoping.py index c7aa9cf7..bef1cc41 100644 --- a/src/vouch/scoping.py +++ b/src/vouch/scoping.py @@ -64,7 +64,7 @@ def viewer_from( if config_path is not None and (resolved_project is None or resolved_agent is None): try: - loaded = yaml.safe_load(config_path.read_text()) + loaded = yaml.safe_load(config_path.read_text(encoding="utf-8")) except (OSError, yaml.YAMLError): loaded = None if isinstance(loaded, dict): diff --git a/src/vouch/server.py b/src/vouch/server.py index 6443b975..82c352b3 100644 --- a/src/vouch/server.py +++ b/src/vouch/server.py @@ -177,7 +177,7 @@ def _to_dicts(h: list[tuple[str, str, str, float]], used: str) -> dict[str, Any] def _load_cfg(store: KBStore) -> dict[str, Any]: try: - loaded = yaml.safe_load((store.kb_dir / "config.yaml").read_text()) + loaded = yaml.safe_load((store.kb_dir / "config.yaml").read_text(encoding="utf-8")) except Exception: return {} return loaded if isinstance(loaded, dict) else {} diff --git a/src/vouch/stats.py b/src/vouch/stats.py index 8f382e45..5e5ca35c 100644 --- a/src/vouch/stats.py +++ b/src/vouch/stats.py @@ -50,7 +50,7 @@ def _list_decided(store: KBStore) -> list[Proposal]: return [] out: list[Proposal] = [] for path in sorted(ddir.glob("*.yaml")): - out.append(Proposal.model_validate(_yaml_load(path.read_text()))) + out.append(Proposal.model_validate(_yaml_load(path.read_text(encoding="utf-8")))) return out diff --git a/src/vouch/storage.py b/src/vouch/storage.py index af21656c..ee697d69 100644 --- a/src/vouch/storage.py +++ b/src/vouch/storage.py @@ -30,9 +30,10 @@ import sqlite3 import stat from pathlib import Path -from typing import Any +from typing import Any, TypeVar import yaml +from pydantic import BaseModel, ValidationError from .models import ( Claim, @@ -144,6 +145,26 @@ def _yaml_load(text: str) -> Any: return yaml.safe_load(text) +_log = logging.getLogger("vouch.storage") + +_ModelT = TypeVar("_ModelT", bound=BaseModel) + + +def _load_or_skip(path: Path, model: type[_ModelT], kind: str) -> _ModelT | None: + """Parse one durable artifact file into ``model``. + + On a corrupt or unreadable file — e.g. a hand-edited yaml or mojibake + carrying a control character that pyyaml's loader rejects — log a warning + and return ``None`` instead of raising, so a single bad file cannot take + down a whole bulk listing (``vouch pending`` and friends). + """ + try: + return model.model_validate(_yaml_load(path.read_text(encoding="utf-8"))) + except (yaml.YAMLError, ValidationError, UnicodeDecodeError, OSError) as e: + _log.warning("skipping unreadable %s %s: %s", kind, path.name, e) + return None + + _FRONTMATTER_RE = re.compile(r"^---\n(.*?)\n---\n?(.*)$", re.DOTALL) @@ -215,14 +236,14 @@ def init(cls, root: Path) -> KBStore: for sub in SUBDIRS: (kb.kb_dir / sub).mkdir(exist_ok=True) if not kb.config_path.exists(): - kb.config_path.write_text(_yaml_dump(_starter_config())) + kb.config_path.write_text(_yaml_dump(_starter_config()), encoding="utf-8") schema_version_file = kb.kb_dir / SCHEMA_VERSION_FILENAME if not schema_version_file.exists(): - schema_version_file.write_text(SCHEMA_VERSION + "\n") + schema_version_file.write_text(SCHEMA_VERSION + "\n", encoding="utf-8") gi = kb.kb_dir / ".gitignore" if not gi.exists(): # state.db is derived; proposed/ is the agent's scratch space. - gi.write_text("proposed/\nstate.db\nstate.db-*\n") + gi.write_text("proposed/\nstate.db\nstate.db-*\n", encoding="utf-8") return kb # --- paths ------------------------------------------------------------- @@ -283,7 +304,7 @@ def put_source( content_path.write_bytes(content) meta_path = sdir / "meta.yaml" if meta_path.exists(): - return Source.model_validate(_yaml_load(meta_path.read_text())) + return Source.model_validate(_yaml_load(meta_path.read_text(encoding="utf-8"))) src = Source( id=sid, type=source_type, # type: ignore[arg-type] @@ -295,7 +316,7 @@ def put_source( tags=tags or [], metadata=metadata or {}, ) - meta_path.write_text(_yaml_dump(src.model_dump(mode="json"))) + meta_path.write_text(_yaml_dump(src.model_dump(mode="json")), encoding="utf-8") self._embed_and_store(kind="source", id=src.id, text=src.title or src.locator or "") return src @@ -303,7 +324,7 @@ def get_source(self, source_id: str) -> Source: meta_path = self._source_dir(source_id) / "meta.yaml" if not meta_path.exists(): raise ArtifactNotFoundError(f"source {source_id}") - return Source.model_validate(_yaml_load(meta_path.read_text())) + return Source.model_validate(_yaml_load(meta_path.read_text(encoding="utf-8"))) def read_source_content(self, source_id: str) -> bytes: p = self._source_dir(source_id) / "content" @@ -319,7 +340,9 @@ def list_sources(self) -> list[Source]: for sdir in sorted(sources_dir.iterdir()): meta = sdir / "meta.yaml" if meta.exists(): - out.append(Source.model_validate(_yaml_load(meta.read_text()))) + src = _load_or_skip(meta, Source, "source") + if src is not None: + out.append(src) return out # --- graph-integrity helpers ------------------------------------------ @@ -400,7 +423,7 @@ def put_claim(self, claim: Claim) -> Claim: ) self._validate_claim_refs(claim) try: - with self._claim_path(claim.id).open("x") as f: + with self._claim_path(claim.id).open("x", encoding="utf-8") as f: f.write(_yaml_dump(claim.model_dump(mode="json"))) except FileExistsError as e: raise ValueError( @@ -413,15 +436,16 @@ def get_claim(self, claim_id: str) -> Claim: p = self._claim_path(claim_id) if not p.exists(): raise ArtifactNotFoundError(f"claim {claim_id}") - return Claim.model_validate(_yaml_load(p.read_text())) + return Claim.model_validate(_yaml_load(p.read_text(encoding="utf-8"))) def list_claims(self) -> list[Claim]: cdir = self.kb_dir / "claims" if not cdir.is_dir(): return [] return [ - Claim.model_validate(_yaml_load(p.read_text())) + c for p in sorted(cdir.glob("*.yaml")) + if (c := _load_or_skip(p, Claim, "claim")) is not None ] def update_claim(self, claim: Claim) -> Claim: @@ -438,7 +462,8 @@ def update_claim(self, claim: Claim) -> Claim: # model validator can't catch (it has no KB access). Mirrors the # put_claim guard so the update path can't reintroduce the gap. self._validate_claim_refs(claim) - self._claim_path(claim.id).write_text(_yaml_dump(claim.model_dump(mode="json"))) + self._claim_path(claim.id).write_text( + _yaml_dump(claim.model_dump(mode="json")), encoding="utf-8") self._embed_and_store(kind="claim", id=claim.id, text=claim.text) # Keep the FTS5 row in sync with the on-disk claim so lifecycle # mutations (archive, supersede, contradict, confirm) are reflected @@ -523,7 +548,7 @@ def list_pages(self) -> list[Page]: def put_entity(self, entity: Entity) -> Entity: try: - with self._entity_path(entity.id).open("x") as f: + with self._entity_path(entity.id).open("x", encoding="utf-8") as f: f.write(_yaml_dump(entity.model_dump(mode="json"))) except FileExistsError as e: raise ValueError( @@ -539,14 +564,14 @@ def get_entity(self, eid: str) -> Entity: p = self._entity_path(eid) if not p.exists(): raise ArtifactNotFoundError(f"entity {eid}") - return Entity.model_validate(_yaml_load(p.read_text())) + return Entity.model_validate(_yaml_load(p.read_text(encoding="utf-8"))) def list_entities(self) -> list[Entity]: d = self.kb_dir / "entities" if not d.is_dir(): return [] - return [Entity.model_validate(_yaml_load(p.read_text())) - for p in sorted(d.glob("*.yaml"))] + return [e for p in sorted(d.glob("*.yaml")) + if (e := _load_or_skip(p, Entity, "entity")) is not None] # --- relations --------------------------------------------------------- @@ -572,7 +597,7 @@ def _validate_relation_refs(self, rel: Relation) -> None: def put_relation(self, rel: Relation) -> Relation: self._validate_relation_refs(rel) try: - with self._relation_path(rel.id).open("x") as f: + with self._relation_path(rel.id).open("x", encoding="utf-8") as f: f.write(_yaml_dump(rel.model_dump(mode="json"))) except FileExistsError as e: raise ValueError( @@ -605,7 +630,7 @@ def put_relation_idempotent(self, rel: Relation) -> Relation: # the linked claim was subsequently archived or retracted. self._validate_relation_refs(rel) try: - with path.open("x") as f: + with path.open("x", encoding="utf-8") as f: f.write(_yaml_dump(rel.model_dump(mode="json"))) except FileExistsError: self._embed_and_store( @@ -623,14 +648,14 @@ def get_relation(self, rid: str) -> Relation: p = self._relation_path(rid) if not p.exists(): raise ArtifactNotFoundError(f"relation {rid}") - return Relation.model_validate(_yaml_load(p.read_text())) + return Relation.model_validate(_yaml_load(p.read_text(encoding="utf-8"))) def list_relations(self) -> list[Relation]: d = self.kb_dir / "relations" if not d.is_dir(): return [] - return [Relation.model_validate(_yaml_load(p.read_text())) - for p in sorted(d.glob("*.yaml"))] + return [r for p in sorted(d.glob("*.yaml")) + if (r := _load_or_skip(p, Relation, "relation")) is not None] def relations_from(self, node_id: str) -> list[Relation]: return [r for r in self.list_relations() if r.source == node_id] @@ -644,7 +669,7 @@ def put_evidence(self, ev: Evidence) -> Evidence: if not (self._source_dir(ev.source_id) / "meta.yaml").exists(): raise ValueError(f"evidence {ev.id} cites unknown source {ev.source_id}") try: - with self._evidence_path(ev.id).open("x") as f: + with self._evidence_path(ev.id).open("x", encoding="utf-8") as f: f.write(_yaml_dump(ev.model_dump(mode="json"))) except FileExistsError as e: raise ValueError( @@ -657,20 +682,20 @@ def get_evidence(self, eid: str) -> Evidence: p = self._evidence_path(eid) if not p.exists(): raise ArtifactNotFoundError(f"evidence {eid}") - return Evidence.model_validate(_yaml_load(p.read_text())) + return Evidence.model_validate(_yaml_load(p.read_text(encoding="utf-8"))) def list_evidence(self) -> list[Evidence]: d = self.kb_dir / "evidence" if not d.is_dir(): return [] - return [Evidence.model_validate(_yaml_load(p.read_text())) - for p in sorted(d.glob("*.yaml"))] + return [ev for p in sorted(d.glob("*.yaml")) + if (ev := _load_or_skip(p, Evidence, "evidence")) is not None] # --- sessions ---------------------------------------------------------- def put_session(self, sess: Session) -> Session: try: - with self._session_path(sess.id).open("x") as f: + with self._session_path(sess.id).open("x", encoding="utf-8") as f: f.write(_yaml_dump(sess.model_dump(mode="json"))) except FileExistsError as e: raise ValueError( @@ -684,21 +709,22 @@ def update_session(self, sess: Session) -> Session: # guard against duplicate ids, so updates need a separate path. if not self._session_path(sess.id).exists(): raise ArtifactNotFoundError(f"session {sess.id}") - self._session_path(sess.id).write_text(_yaml_dump(sess.model_dump(mode="json"))) + self._session_path(sess.id).write_text( + _yaml_dump(sess.model_dump(mode="json")), encoding="utf-8") return sess def get_session(self, sid: str) -> Session: p = self._session_path(sid) if not p.exists(): raise ArtifactNotFoundError(f"session {sid}") - return Session.model_validate(_yaml_load(p.read_text())) + return Session.model_validate(_yaml_load(p.read_text(encoding="utf-8"))) def list_sessions(self) -> list[Session]: d = self.kb_dir / "sessions" if not d.is_dir(): return [] - return [Session.model_validate(_yaml_load(p.read_text())) - for p in sorted(d.glob("*.yaml"))] + return [s for p in sorted(d.glob("*.yaml")) + if (s := _load_or_skip(p, Session, "session")) is not None] # --- embedding hook ------------------------------------------------------ @@ -771,7 +797,7 @@ def _embed_and_store( def put_proposal(self, proposal: Proposal) -> Proposal: try: - with self._proposal_path(proposal.id).open("x") as f: + with self._proposal_path(proposal.id).open("x", encoding="utf-8") as f: f.write(_yaml_dump(proposal.model_dump(mode="json"))) except FileExistsError as e: raise ValueError( @@ -782,14 +808,16 @@ def put_proposal(self, proposal: Proposal) -> Proposal: def get_proposal(self, proposal_id: str) -> Proposal: for path in (self._proposal_path(proposal_id), self._decided_path(proposal_id)): if path.exists(): - return Proposal.model_validate(_yaml_load(path.read_text())) + return Proposal.model_validate(_yaml_load(path.read_text(encoding="utf-8"))) raise ArtifactNotFoundError(f"proposal {proposal_id}") def list_proposals(self, status: ProposalStatus | None = None) -> list[Proposal]: out: list[Proposal] = [] for sub in ("proposed", "decided"): for p in sorted((self.kb_dir / sub).glob("*.yaml")): - pr = Proposal.model_validate(_yaml_load(p.read_text())) + pr = _load_or_skip(p, Proposal, "proposal") + if pr is None: + continue if status is None or pr.status == status: out.append(pr) return out @@ -797,7 +825,7 @@ def list_proposals(self, status: ProposalStatus | None = None) -> list[Proposal] def move_proposal_to_decided(self, proposal: Proposal) -> None: src = self._proposal_path(proposal.id) dst = self._decided_path(proposal.id) - dst.write_text(_yaml_dump(proposal.model_dump(mode="json"))) + dst.write_text(_yaml_dump(proposal.model_dump(mode="json")), encoding="utf-8") if src.exists(): src.unlink() diff --git a/src/vouch/sync.py b/src/vouch/sync.py index de9abdc3..1a6636b5 100644 --- a/src/vouch/sync.py +++ b/src/vouch/sync.py @@ -282,7 +282,7 @@ def _write_conflict_report( report_path = report_dir / f"{check.source_id}.json" report = asdict(check) report["on_conflict"] = on_conflict - report_path.write_text(json.dumps(report, indent=2, sort_keys=True)) + report_path.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8") return str(report_path.relative_to(kb_dir)) diff --git a/src/vouch/volunteer_context.py b/src/vouch/volunteer_context.py index eef2f6bc..b1036e0f 100644 --- a/src/vouch/volunteer_context.py +++ b/src/vouch/volunteer_context.py @@ -66,7 +66,7 @@ def to_dict(self) -> dict[str, Any]: def load_config(store: KBStore) -> VolunteerConfig: """Read ``volunteer:`` from config.yaml; fall back to defaults.""" try: - loaded = yaml.safe_load(store.config_path.read_text()) + loaded = yaml.safe_load(store.config_path.read_text(encoding="utf-8")) except (OSError, yaml.YAMLError): return VolunteerConfig() if not isinstance(loaded, dict): diff --git a/src/vouch/web/server.py b/src/vouch/web/server.py index 2473fadb..ed51af1b 100644 --- a/src/vouch/web/server.py +++ b/src/vouch/web/server.py @@ -195,7 +195,7 @@ def _pending_page(store: KBStore, page: int, page_size: int proposals: list[Proposal] = [] for p in paths[lo:hi]: try: - proposals.append(Proposal.model_validate(_yaml_load(p.read_text()))) + proposals.append(Proposal.model_validate(_yaml_load(p.read_text(encoding="utf-8")))) except Exception as e: _log.warning("skipping unreadable proposal %s: %s", p.name, e) return proposals, page, pages, total diff --git a/tests/test_storage.py b/tests/test_storage.py index ce1d643f..03ce0fba 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -831,3 +831,32 @@ def test_cite_resolves_source_and_evidence(store: KBStore) -> None: for c in citations } assert "source" in kinds and "evidence" in kinds + + +# --- resilience: a single corrupt file must not break bulk listing -------- + + +def test_list_proposals_skips_unreadable_file(store: KBStore) -> None: + """One unparseable proposal file must not take down `vouch pending`.""" + src = store.put_source(b"evidence") + good = propose_claim(store, text="good", evidence=[src.id], + proposed_by="agent") + + # a raw U+0080 (C1 control byte) — pyyaml's loader rejects it even though + # its dumper would have escaped it. mirrors a hand-edited / mojibake file. + corrupt = store.kb_dir / "proposed" / "20990101-000000-corrupt.yaml" + corrupt.write_bytes(b"text: bad\xc2\x80value\n") + + pending = store.list_proposals(ProposalStatus.PENDING) + assert [p.id for p in pending] == [good.id] + + +def test_list_claims_skips_unreadable_file(store: KBStore) -> None: + """Same resilience for durable claim listing (vouch search/status).""" + src = store.put_source(b"e") + store.put_claim(Claim(id="c-ok", text="x", evidence=[src.id])) + (store.kb_dir / "claims" / "c-bad.yaml").write_bytes( + b"text: bad\xc2\x80value\n") + + claims = store.list_claims() + assert [c.id for c in claims] == ["c-ok"]