diff --git a/ion/analyze/__init__.py b/ion/analyze/__init__.py index 25dc87b..484685d 100644 --- a/ion/analyze/__init__.py +++ b/ion/analyze/__init__.py @@ -67,7 +67,7 @@ def compute_analysis(ctx, cache: dict | None = None) -> dict: r = cache[ck] if r and r["candidates"]: cohs.append(r) - arch = lifecycle.archive_candidates(live, pairs, cfg) + arch = lifecycle.archive_candidates(live, pairs, cfg, unread=set(live) - set(data)) result = {"lifecycle": lifecycle.stages(snaps, cfg), "boundaries": boundaries.violations(snaps, pairs, cfg), "gen": gen, "repos": sorted(snaps), "analyzed": sorted(data), "health": health_f, "policy": policy_f, "coupling": pairs, "cohesion": cohs, "archive": arch, "lagging_forks": lifecycle.lagging_forks(snaps)} return result diff --git a/ion/analyze/lifecycle.py b/ion/analyze/lifecycle.py index 9ca5bfc..538a26c 100644 --- a/ion/analyze/lifecycle.py +++ b/ion/analyze/lifecycle.py @@ -30,7 +30,8 @@ def stages(snaps: dict[str, dict], cfg, now=None) -> dict[str, str]: return {n: stage_of(n, s, cfg, now) for n, s in snaps.items() if not s.get("fork") and n not in cfg["exclude"]} -def archive_candidates(snaps: dict[str, dict], pairs: list[dict], cfg, now=None) -> list[dict]: +def archive_candidates(snaps: dict[str, dict], pairs: list[dict], cfg, now=None, unread: set | None = None) -> list[dict]: + """`unread`: live repos whose contents were not loaded (no mirror), so whether they reference a repo is unknown.""" th = cfg["thresholds"] dependents: dict[str, set] = {n: set() for n in snaps} for p in pairs: @@ -47,7 +48,12 @@ def archive_candidates(snaps: dict[str, dict], pairs: list[dict], cfg, now=None) continue age = days_since(s.get("pushed_at"), now) score, reasons = 0.5, [f"no push in {int(age)} days"] - if not dependents.get(name): + others_unread = (unread or set()) - {name} + if dependents.get(name): + pass + elif others_unread: + reasons.append(f"references unknown: {len(others_unread)} other repos were not analysed") + else: score += 0.2 reasons.append("no other repo references it") superseded = None diff --git a/ion/cli.py b/ion/cli.py index 4012015..c29bfca 100644 --- a/ion/cli.py +++ b/ion/cli.py @@ -157,8 +157,7 @@ def cmd_report(ctx, args, results=None): text, changed = build_digest(ctx, results or {}) if changed: errs = notify(ctx, text) - if len(errs) < len(ctx.cfg["notifiers"]): - mark_digest_sent(ctx, results or {}) + mark_digest_sent(ctx, results or {}) # a failed delivery is queued in the outbox and retried, not re-sent for e in errs: out(f"notify error (queued for retry): {e}") else: @@ -356,7 +355,10 @@ def main(argv=None) -> int: "execute": cmd_execute, "report": cmd_report, "run": cmd_run, "status": cmd_status, "explain": cmd_explain, "grep": cmd_grep, "autofix": cmd_autofix, "plan-config": cmd_plan_config, "dashboard": cmd_dashboard}[args.cmd] r = fn(ctx, args) - return r if isinstance(r, int) else 0 + if isinstance(r, int): + return r + # like `run`: a stage that reported errors (a breaker trip, a failed sync) exits non-zero + return 1 if args.cmd in ("propose", "sync-approvals", "execute") and isinstance(r, dict) and r.get("errors") else 0 except Exception as e: # noqa: BLE001 print(f"error: {e}", file=sys.stderr) return 1 diff --git a/ion/execute/autofix.py b/ion/execute/autofix.py index b4ab916..f260c1c 100644 --- a/ion/execute/autofix.py +++ b/ion/execute/autofix.py @@ -26,7 +26,7 @@ def plan(self, p: dict) -> Plan: base = self.default_branch(repo) def fix(repo=repo, w=w, url=url, base=base): - snap = {"default_branch": base, "description": self._description(repo)} + snap = {**self._snapshot(repo), "default_branch": base} # what the preview passed the fixer got = digest(diff_of(self.ctx, fixer, repo, snap, w, url)) want = s["diffs"].get(repo) if got != want: @@ -55,9 +55,9 @@ def pr(repo=repo, base=base): Step(f"{repo}.open-pr", f"{repo}: open the PR and label it `needs decision`", None, pr)] return Plan(pid, "autofix", "apply", steps) - def _description(self, repo: str) -> str: + def _snapshot(self, repo: str) -> dict: from ..collect.snapshot import latest_snapshots - return (latest_snapshots(self.db)[0].get(repo) or {}).get("description") or "" + return dict(latest_snapshots(self.db)[0].get(repo) or {}) def after_phase(self, p, plan): proposals.set_status(self.db, p["id"], "done") diff --git a/ion/execute/split.py b/ion/execute/split.py index 6cb4259..807d8d4 100644 --- a/ion/execute/split.py +++ b/ion/execute/split.py @@ -4,7 +4,7 @@ import json from .. import proposals -from ..util import Refused, SafetyError, filter_repo_path, safe_write +from ..util import Refused, SafetyError, filter_repo_path, redact, safe_write from ..writes import WriteRequest from .base import Executor, Plan, Step, filter_args from .provenance import origin_callback @@ -36,7 +36,10 @@ def filt(): self.g(["filter-repo", *fargs, "--commit-callback", origin_callback(self.cfg.org, A), "--force"], cwd=cdir) def has_refs() -> bool: - return bool(self.g([*self.auth(n_url), "ls-remote", n_url], check=False).stdout.strip()) + r = self.g([*self.auth(n_url), "ls-remote", n_url], check=False) + if r.returncode != 0: # fail closed: a remote we cannot list is not known to be empty + raise SafetyError(f"cannot list {N}'s refs to check that it is empty: {redact(r.stderr or '')[:160]}") + return bool(r.stdout.strip()) def create(): try: diff --git a/ion/fleet.py b/ion/fleet.py index fc79c67..f5e6414 100644 --- a/ion/fleet.py +++ b/ion/fleet.py @@ -173,6 +173,9 @@ class DependabotConfig(Autofixer): def find_repos(self, ctx, repos): out = [] for n, s in repos.items(): + ov = parse_repo_override(s.get("override_text"))[0] + if ov.get("ignore") or ov.get("enabled") is False: + continue tree = _tree(ctx, n, s) if tree is not None and ".github/dependabot.yml" not in tree and ".github/dependabot.yaml" not in tree \ and detect_ecosystems(tree): diff --git a/ion/planconfig.py b/ion/planconfig.py index cf93ee2..02a4a80 100644 --- a/ion/planconfig.py +++ b/ion/planconfig.py @@ -75,7 +75,8 @@ def brief(c): p = proposals.view(row) c = c_new.get(p["key"]) if c is None: - tracked.append({"id": p["id"], "key": p["key"], "effect": "no longer a candidate under the new config"}) + if p["key"] in c_old: # autofix and follow-up proposals are never analysis candidates + tracked.append({"id": p["id"], "key": p["key"], "effect": "no longer a candidate under the new config"}) elif c.get("blocked"): tracked.append({"id": p["id"], "key": p["key"], "effect": f"blocked: {c['blocked']}"}) else: diff --git a/ion/propose/scoring.py b/ion/propose/scoring.py index 2cc49ba..819d826 100644 --- a/ion/propose/scoring.py +++ b/ion/propose/scoring.py @@ -43,10 +43,10 @@ def cooldown_blocks(rejected_at: str | None, rejected_score: float | None, score return False if ((now or _now()) - t).days >= cfg["thresholds"]["cooldown_days"]: return False - return score < (rejected_score or 0.0) + cfg["thresholds"]["rescore_delta"] + return round(score - (rejected_score or 0.0), 9) < cfg["thresholds"]["rescore_delta"] # 0.7 - 0.55 is 0.1499... def replan_needed(old_hash: str | None, new_hash: str, epoch_score: float | None, score: float, cfg) -> bool: if old_hash is not None and old_hash != new_hash: return True - return epoch_score is not None and abs(score - epoch_score) >= cfg["thresholds"]["rescore_delta"] + return epoch_score is not None and round(abs(score - epoch_score), 9) >= cfg["thresholds"]["rescore_delta"] diff --git a/tests/test_execute.py b/tests/test_execute.py index 1a6634b..f819cd6 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -302,6 +302,28 @@ def test_split_refuses_non_empty_target(ctx, world): assert subjects(world, "mono-engine", "main") == ["pre"] +def test_split_refuses_when_it_cannot_tell_the_target_is_empty(ctx, world, monkeypatch): + """A transient `ls-remote` failure is not an empty repo: here the target already has a branch of its own.""" + sh("git", "init", "-q", "--bare", "-b", "main", str(world / "mono-engine.git")) + work = world / "_prefill" + sh("git", "init", "-q", "-b", "other", str(work)) + (work / "f").write_text("x") + sh("git", "add", "-A", cwd=work); sh("git", "commit", "-qm", "pre", cwd=work) + sh("git", "push", "-q", str(world / "mono-engine.git"), "other", cwd=work) + p = prep(ctx, "split", SPLIT) + ex = executor_for(ctx, "split") + real = ex.g + + def g(argv, cwd=None, **kw): + if "ls-remote" in argv: + return subprocess.CompletedProcess(argv, 128, "", "fatal: unable to access: Could not resolve host") + return real(argv, cwd=cwd, **kw) + monkeypatch.setattr(ex, "g", g) + with pytest.raises(SafetyError, match="cannot list"): + ex.apply(p, dry_run=False) + assert branches(world, "mono-engine") == ["other"] # nothing pushed + + def test_split_refuses_existing_public_repo_for_private_split(ctx, world): sh("git", "init", "-q", "--bare", "-b", "main", str(world / "mono-engine.git")) # someone pre-created it, public p = prep(ctx, "split", {**SPLIT, "private": True}) diff --git a/tests/test_fleet_planconfig.py b/tests/test_fleet_planconfig.py index 6fde02f..1d895d8 100644 --- a/tests/test_fleet_planconfig.py +++ b/tests/test_fleet_planconfig.py @@ -344,3 +344,44 @@ def test_plan_config_cli_with_set_overrides(ctx, capsys): def test_config_diff_is_flat_and_ignores_private_keys(): a, b = {"a": {"b": 1, "c": 2}, "_path": "x"}, {"a": {"b": 1, "c": 3}, "_path": "y"} assert config_diff(a, b) == [{"key": "a.c", "old": 2, "new": 3}] + + +def test_executed_autofix_gives_the_fixer_the_snapshot_the_preview_did(collected, monkeypatch): + """A plugin may read any snapshot field (here `languages`); the executor must recompute the same diff.""" + from ion import fleet + + class EditorConfig(Autofixer): + name = "editorconfig-by-language" + description = "add .editorconfig matching the primary language" + commit_message = pr_title = pr_body = "Add .editorconfig" + + def find_repos(self, ctx, repos): + return ["old-tool"] + + def apply(self, workdir, repo, snap): + size = 4 if "Python" in (snap.get("languages") or {}) else 2 + self.write(workdir, ".editorconfig", f"root = true\n\n[*]\nindent_size = {size}\n") + + monkeypatch.setitem(fleet.FIXERS, EditorConfig.name, EditorConfig) + cli.cmd_autofix(collected, cli.build_parser().parse_args(["autofix", EditorConfig.name, "--propose"])) + p = approve(collected, proposals.view(collected.db.one("select * from proposals where kind='autofix'"))) + executor_for(collected, "autofix").apply(p, dry_run=False) + assert proposals.get(collected.db, p["id"])["status"] == "done" + + +@pytest.mark.parametrize("text", ["ignore: true\n", "enabled: false\n"]) +def test_dependabot_config_honours_a_repo_opt_out(collected, text): + from helpers import set_override + from ion.fleet import fleet_repos + set_override(collected, "alpha-core", text) + assert "alpha-core" not in DependabotConfig().find_repos(collected, fleet_repos(collected)) + + +def test_plan_config_does_not_drop_proposals_analysis_never_makes(collected): + """An autofix proposal is not an analysis candidate under either config, so it cannot stop being one.""" + from ion.config import deep_merge + run_analyze(collected) + cli.cmd_autofix(collected, cli.build_parser().parse_args(["autofix", "dependabot-config", "--repo", "alpha-*", "--propose"])) + same = load_config(None, deep_merge({k: v for k, v in collected.cfg.items() if not k.startswith("_")}, {})) + r = plan_config(collected, same) + assert r["config_changes"] == [] and r["open_proposals"] == [] diff --git a/tests/test_lifecycle_boundaries.py b/tests/test_lifecycle_boundaries.py index b765e11..8876405 100644 --- a/tests/test_lifecycle_boundaries.py +++ b/tests/test_lifecycle_boundaries.py @@ -158,3 +158,23 @@ def test_cross_scope_merge_can_be_proposed_with_a_warning(ctx): m = proposals.view(ctx.db.one("select * from proposals where kind='merge'")) assert "different scopes" in m["subject"]["cross_scope"] assert any(r.startswith("BOUNDARY") for r in m["text"]["risks"]) and "crosses a boundary" in m["text"]["title"] + + +def test_no_dependents_is_claimed_only_when_every_repo_was_read(ctx, world, tmp_path): + """alpha-core depends on old-tool. Without alpha-core's mirror that reference cannot be seen, so the archive + evidence must not say "no other repo references it" (and must not score it).""" + import shutil + from conftest import sh + work = tmp_path / "w" + sh("git", "clone", "-q", str(world / "alpha-core.git"), str(work)) + (work / "pyproject.toml").write_text('[project]\nname = "alpha-core"\nversion = "0.1"\n' + 'dependencies = ["old-tool", "requests", "pyyaml", "click"]\n') + sh("git", "commit", "-qam", "depend on old-tool", cwd=work) + sh("git", "push", "-q", "origin", "main", cwd=work) + run_collect(ctx) + arch = {a["repo"]: a for a in run_analyze(ctx)["archive"]}["old-tool"] + assert arch["dependents"] == ["alpha-core"] and not arch["qualifies"] + shutil.rmtree(ctx.mirror_path("alpha-core")) + arch = {a["repo"]: a for a in run_analyze(ctx)["archive"]}["old-tool"] + assert not arch["qualifies"] and "no other repo references it" not in arch["reasons"] + assert any(r.startswith("references unknown") for r in arch["reasons"]) diff --git a/tests/test_misc.py b/tests/test_misc.py index afa9ab9..5813fd7 100644 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -230,3 +230,34 @@ def test_legacy_mimar_config_is_found_with_migration_note(tmp_path, monkeypatch, (tmp_path / ".mimar" / "mimar.yml").write_text("org: o\n") assert find_config_file() == tmp_path / ".mimar" / "mimar.yml" assert "deprecated" in capsys.readouterr().err + + +def test_digest_is_not_resent_while_a_notifier_backlog_is_queued(tmp_path, home, world): + """A failed delivery is queued and retried; it must not make every later run send the same digest again.""" + import argparse + bus = FakeBus() + bus.down = True + log = tmp_path / "digest.log" + ctx = make_ctx(tmp_path, world, {"notifiers": [{"type": "file", "path": str(log)}, + {"type": "divan_bus", "client": "/c/divan.py", "to": "ops"}]}, bus=bus) + args = argparse.Namespace(apply=False) + run_collect(ctx) + new_proposal(ctx, "merge", MERGE) + cli.cmd_report(ctx, args, {}) # file ok, bus down: queued + new_proposal(ctx, "archive", {"repo": "old-tool"}) + cli.cmd_report(ctx, args, {}) # something changed: sent, and queued for the bus + sent = log.read_text().count("awaiting approval (2)") + for _ in range(3): + cli.cmd_report(ctx, args, {}) # nothing changed + assert log.read_text().count("awaiting approval (2)") == sent + assert ctx.db.one("select count(*) n from outbox where sent=0")["n"] == 2 + + +def test_execute_exits_nonzero_when_it_reports_errors(ctx, monkeypatch, capsys): + from helpers import approve + run_collect(ctx) + for r in ("alpha-core", "alpha-cli", "mono"): + approve(ctx, new_proposal(ctx, "archive", {"repo": r})) # 3 archives > max_archives 2: the breaker trips + monkeypatch.setattr(cli, "make_ctx", lambda args, no_mirror=False: ctx) + assert cli.main(["execute", "--apply"]) == 1 + assert "error: circuit breaker" in capsys.readouterr().out diff --git a/tests/test_propose.py b/tests/test_propose.py index 862b331..c89fd89 100644 --- a/tests/test_propose.py +++ b/tests/test_propose.py @@ -275,3 +275,11 @@ def test_llm_call_cap_and_none_backend(): assert not NoneLLM(load_config(None, {"org": "o"})).enabled with pytest.raises(LLMError): NoneLLM(load_config(None, {"org": "o"})).complete_json("x") + + +def test_a_rise_of_exactly_rescore_delta_counts(): + """README: "the score moves by 0.15 or more". In floats 0.7 - 0.55 and 0.6 - 0.45 are just under 0.15.""" + from ion.util import parse_ts + c = cfg() + assert not scoring.cooldown_blocks("2026-10-01T00:00:00Z", 0.55, 0.7, c, now=parse_ts("2026-10-13T00:00:00Z")) + assert scoring.replan_needed("a", "a", 0.45, 0.6, c)