diff --git a/studies/nz-reconciliation/runner/src/nz_reconciliation/run_live_suite.py b/studies/nz-reconciliation/runner/src/nz_reconciliation/run_live_suite.py index e75dfe2a..df0c709d 100644 --- a/studies/nz-reconciliation/runner/src/nz_reconciliation/run_live_suite.py +++ b/studies/nz-reconciliation/runner/src/nz_reconciliation/run_live_suite.py @@ -49,11 +49,15 @@ def compare_live( domain = rs.get("domain") if domain == "income_tax": rs_out = { - "primary": rs.get("outputs", {}).get("individual_income_tax_before_credits") + "primary": rs.get("outputs", {}).get( + "individual_income_tax_before_credits" + ) or next(iter(rs.get("outputs", {}).values()), None) } of_out = { - "primary": of.get("outputs", {}).get("individual_income_tax_before_credits") + "primary": of.get("outputs", {}).get( + "individual_income_tax_before_credits" + ) } pair = compare_pair( {**rs, "outputs": {k: v for k, v in rs_out.items() if v}}, @@ -61,8 +65,12 @@ def compare_live( tolerance=tolerance, ) elif domain == "acc_earners_levy": - rs_val = rs.get("outputs", {}).get("acc_standard_earners_levy_including_gst") - of_val = of.get("outputs", {}).get("acc_standard_earners_levy_including_gst") + rs_val = rs.get("outputs", {}).get( + "acc_standard_earners_levy_including_gst" + ) + of_val = of.get("outputs", {}).get( + "acc_standard_earners_levy_including_gst" + ) pair = compare_pair( {**rs, "outputs": {"primary": rs_val} if rs_val else {}}, {**of, "outputs": {"primary": of_val} if of_val else {}}, @@ -84,18 +92,20 @@ def compare_live( def build_report( - rulespec_rows: list[dict], - openfisca_rows: list[dict], - comparison_rows: list[dict], + agreed: list[dict], + non_agreed: list[dict], + total_cases: int, + rulespec_ok: int, + openfisca_ok: int, ) -> str: - agreed = [r for r in comparison_rows if r.get("agreement")] + lines = [ "# NZ reconciliation — live dual-engine report", "", - f"Cases: **{len(comparison_rows)}**", + f"Cases: **{total_cases}**", f"Numeric agreements (≤$0.02): **{len(agreed)}**", - f"RuleSpec oracle ok: **{sum(1 for r in rulespec_rows if r.get('status')=='ok')}**", - f"OpenFisca live ok: **{sum(1 for r in openfisca_rows if r.get('status')=='ok')}**", + f"RuleSpec oracle ok: **{rulespec_ok}**", + f"OpenFisca live ok: **{openfisca_ok}**", "", "## Agreements", "", @@ -106,9 +116,7 @@ def build_report( else: lines.append("_None_") lines.extend(["", "## Non-agreements / gaps", ""]) - for row in comparison_rows: - if row.get("agreement"): - continue + for row in non_agreed: lines.append( f"- `{row.get('caseId')}`: `{row.get('classification')}` " f"(rs={row.get('rulespec', {}).get('status')}, " @@ -138,7 +146,9 @@ def build_report( def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description="Live dual-engine NZ reconciliation suite.") + parser = argparse.ArgumentParser( + description="Live dual-engine NZ reconciliation suite." + ) parser.add_argument("--results-dir", type=Path, default=DEFAULT_RESULTS) args = parser.parse_args(argv) results_dir = args.results_dir @@ -148,22 +158,41 @@ def main(argv: list[str] | None = None) -> int: try: openfisca_rows = run_openfisca_live_suite() except Exception as exc: # noqa: BLE001 - print(json.dumps({"ok": False, "error": f"{type(exc).__name__}: {exc}"}, indent=2)) + print( + json.dumps({"ok": False, "error": f"{type(exc).__name__}: {exc}"}, indent=2) + ) return 2 comparison_rows = compare_live(rulespec_rows, openfisca_rows) write_jsonl(results_dir / "rulespec-candidate-results.jsonl", rulespec_rows) write_jsonl(results_dir / "openfisca-aotearoa-live-results.jsonl", openfisca_rows) write_jsonl(results_dir / "comparison-live-results.jsonl", comparison_rows) - report = build_report(rulespec_rows, openfisca_rows, comparison_rows) + agreed = [] + non_agreed = [] + for r in comparison_rows: + if r.get("agreement"): + agreed.append(r) + else: + non_agreed.append(r) + + rulespec_ok = sum(1 for r in rulespec_rows if r.get("status") == "ok") + openfisca_live_ok = sum(1 for r in openfisca_rows if r.get("status") == "ok") + + report = build_report( + agreed, + non_agreed, + len(comparison_rows), + rulespec_ok, + openfisca_live_ok, + ) (results_dir / "LIVE_DUAL_ENGINE_REPORT.md").write_text(report, encoding="utf-8") summary = { "ok": True, "cases": len(comparison_rows), - "agreements": sum(1 for r in comparison_rows if r.get("agreement")), - "rulespec_ok": sum(1 for r in rulespec_rows if r.get("status") == "ok"), - "openfisca_live_ok": sum(1 for r in openfisca_rows if r.get("status") == "ok"), + "agreements": len(agreed), + "rulespec_ok": rulespec_ok, + "openfisca_live_ok": openfisca_live_ok, "report": str(results_dir / "LIVE_DUAL_ENGINE_REPORT.md"), } print(json.dumps(summary, indent=2))