From d54f539224a72622cbbe7e430e8b46d769dd5b01 Mon Sep 17 00:00:00 2001 From: MilindC Date: Sat, 2 May 2026 22:22:22 +0000 Subject: [PATCH 1/3] Update README and architecture documentation for ReasonBench--moving towards benchmarking LLMs in reasoning; enhance .gitignore for local data --- .gitignore | 2 + README.md | 124 ++++++++++++++++++++++++++++++------------- docs/architecture.md | 74 ++++++++++---------------- 3 files changed, 117 insertions(+), 83 deletions(-) diff --git a/.gitignore b/.gitignore index 83972fa..0b424a1 100644 --- a/.gitignore +++ b/.gitignore @@ -216,3 +216,5 @@ __marimo__/ # Streamlit .streamlit/secrets.toml +local_data/results/* +local_data/faiss_index/* diff --git a/README.md b/README.md index a18ac33..1867c00 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,34 @@ -# ArgumentLab +# ReasonBench (powered by ArgumentLab) -**A multi-agent reasoning system that conducts structured debates and quantitatively evaluates argument quality, consistency, and hallucination under adversarial conditions.** +**A multi-agent benchmark that conducts structured adversarial debates to quantitatively evaluate LLM reasoning quality, strategic consistency, and adaptability.** -Most AI systems are built to answer questions. ArgumentLab is built to *interrogate them* — forcing agents to construct structured arguments, defend them across debate rounds, cite real evidence, and withstand adversarial pressure. The goal is not a better chatbot. It is a rigorous framework for studying how AI systems reason, disagree, and fail. +Most AI benchmarks evaluate static question-answering. ReasonBench is built to *interrogate* models — forcing agents to construct structured strategies, defend them across debate rounds, explicitly state assumptions, and adapt to adversarial critiques. It is a rigorous framework for measuring how AI systems reason, disagree, and evolve their thinking. --- ## Why This Is Hard -Getting an LLM to argue a position is trivial. Getting it to: +Getting an LLM to argue a position is trivial. Getting it to any of the following: - maintain logical consistency across multiple rounds - cite grounded, verifiable evidence - respond specifically to an opponent's claims (not just re-assert its own) - detect when it is contradicting itself - converge toward a defensible conclusion under adversarial input -...is not. ArgumentLab treats each of these as a measurable engineering problem. + +ArgumentLab treats each of these as a measurable engineering problem. --- -## Architecture Overview - -Refer to [Architecture document](/docs/architecture.md) +## The ReasonBench Evaluation Suite + +ReasonBench currently evaluates reasoning across three core tasks, scored automatically by an LLM-as-Judge over a 3-round debate protocol: + +1. **Deterministic Logic (Constraint Puzzle):** Evaluates correctness, logical correctness, completeness, and responsiveness. +2. **Strategic Reasoning (Asymmetric Game):** Evaluates opponent modeling, strategic consistency, risk awareness, conditional reasoning, and responsiveness. +3. **Constrained Tradeoff Reasoning:** Evaluates constraint utilization, tradeoff specificity, explicit assumptions, risk analysis, and conditional reasoning. + +For full architectural details, see the [Architecture document](/docs/architecture.md). --- @@ -38,28 +45,29 @@ Four agents drive the system: | **Judge** | Evaluates argument quality and detects convergence | | **Moderator** *(optional)* | Enforces debate structure and prevents drift | -### Structured Argument Format +### Structured ReasonBench Format -Agents do not produce free text. Every argument is a structured object: +Agents do not produce free text. Every response is a structured object: ```json { - "claim": "...", - "evidence": ["source_1", "source_2"], - "assumptions": ["..."], - "counterpoints_addressed": ["..."], - "confidence_score": 0.82 + "strategy_or_answer": "Final answer or plan...", + "rationale": "Step-by-step reasoning...", + "assumptions": ["Explicit assumptions made..."], + "opponent_model": "What the model believes about the opponent...", + "risks": ["Failure modes or weaknesses..."], + "conditions": ["When the answer/strategy would change..."] } ``` -This eliminates the "chatty LLM" failure mode and makes every output machine-evaluable. +This eliminates the "chatty LLM" failure mode and makes every output strictly scorable against the benchmark rubrics. ### Iterative Debate Loop Debates run across three rounds with increasing specificity: - **Round 1** — Initial arguments, top-level claims -- **Round 2** — Targeted rebuttals; agents must address specific prior claims +- **Round 2** — Targeted rebuttals; agents must respond to specific prior claims - **Round 3** — Refinement; agents update positions based on accumulated evidence Each agent receives the full prior-round context and is penalized (in scoring) for ignoring it. @@ -132,7 +140,51 @@ Tracked across every debate session: - Contradiction frequency - Convergence round (or failure to converge) - Evidence citation rate -- Disagreement persistence across rounds +- Disagreement persistence through rounds +--- + +## Getting Started + +### Prerequisites + +Ensure you have Python 3.10+ installed and set your OpenAI API key: + +```bash +export OPENAI_API_KEY=sk-... +``` + +Install the dependencies: + +```bash +pip install -r requirements.txt +``` + +### 1. Ingest Data + +Before running a debate, the agents need a retrieval corpus (FAISS index). ArgumentLab includes a sample corpus to get started instantly: + +```bash +python setup/ingest_corpus.py --sample +``` + +You can also ingest your own `.txt` or `.pdf` documents: + +```bash +python setup/ingest_corpus.py --docs path/to/your/documents/ +``` + +### 2. Run a Debate + +Execute a full, structured debate by providing a proposition. The debate streams live to the console, printing argument blocks and judge scores round-by-round. + +```bash +python setup/debate.py \ + --proposition "Companies should replace legacy infrastructure with AI-driven systems." \ + --session-id my_debate_001 +``` + +Once finished, the debate state is automatically exported to `local_data/results/my_debate_001.json` and a human-readable `my_debate_001.md` report. + --- ## Demo Flow @@ -158,30 +210,28 @@ Tracked across every debate session: --- -## MVP Scope +## MVP Scope (ReasonBench) -**Must-have (v1):** +**Target Goal:** +Run all 3 benchmark tasks across 2 models and produce structured scores. + +**Current Features (Iteration 1):** - Proponent + Opponent + Judge agents -- Structured argument format (claim, evidence, confidence) +- Structured `ReasonBenchResponse` format - 3-round debate loop with context tracking -- Basic scoring — logical consistency + evidence support -- CLI or minimal web UI -**v2 additions:** -- Argument graph visualization -- Hallucination detection pipeline -- Metrics dashboard -- RAG-based evidence integration -**Stretch goals:** -- Human-in-the-loop intervention -- Adversarial injection testing suite -- Strategy modes (aggressive / evidence-first / exploratory) -- Multi-agent expansion (domain expert, skeptic, data-driven agents) -- Session history and longitudinal improvement tracking +- Task-specific scoring logic (0-2 scales mapping directly to the 3 task rubrics) +- `evaluate_reasonbench_round()` explicitly tracking **Responsiveness** across rounds. + +**Next Steps (Iteration 2+):** +- Migrate agent logic to output the new ReasonBench schema +- Wire the ReasonBench evaluator natively into the LangGraph state +- Add automated runner to benchmark multiple models at once +- Metrics dashboard / machine-readable score reports --- ## How This Differs from Kialo -[Kialo](https://www.kialo.com) is a platform for human-generated, community-refined argument trees — effectively structured Wikipedia for reasoning. It is a valuable tool for its purpose. +[Kialo](https://www.kialo.com) is a platform for human-generated, community-refined argument trees — effectively structured Wikipedia for reasoning. It is an effective tool for its purpose. ArgumentLab is a different category entirely: @@ -202,7 +252,7 @@ ArgumentLab is a different category entirely: ## Research Connections -ArgumentLab sits at the intersection of several active research directions: +ArgumentLab is positioned to be at the intersection of several active research directions: - **LLM-as-Judge** — using language models as evaluators of reasoning quality - **Multi-agent debate** — Du et al. (2023), *Improving Factuality and Reasoning in Language Models through Multiagent Debate* @@ -218,6 +268,6 @@ ArgumentLab sits at the intersection of several active research directions: ## Author -**Milind C** — MS Computer Science (Artifical Intelligence), Georgia Institute of Technology +**Milind C** — MS Computer Science (Artificial Intelligence), Georgia Institute of Technology [LinkedIn](https://linkedin.com/in/milind-chandramohan) · [GitHub](https://github.com/mildogrc) diff --git a/docs/architecture.md b/docs/architecture.md index d32a76a..4bcb276 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -30,7 +30,7 @@ Each layer is independently testable and builds on the outputs of the layer belo ## High-Level Architecture -![High-Level Architecture](/docs/images/highlevelarchitecture.png) +![High-Level Architecture](/docs/images/highlevelarch.png) --- @@ -49,34 +49,22 @@ The system uses four agents with distinct, non-overlapping roles: Each agent is implemented as a stateless function that receives a context object and returns a structured argument. State is held externally in the Shared Debate State Manager. -### Structured Argument Schema +### Structured ReasonBench Schema -Every agent output is a typed JSON object — no free-form prose: +Every agent output must strictly conform to the `ReasonBenchResponse` JSON schema — no free-form prose: ```json { - "round": 2, - "agent": "proponent", - "claim": "string — the primary assertion being made", - "evidence": [ - { - "source_id": "doc_42_chunk_7", - "excerpt": "string — relevant passage", - "reliability_score": 0.91 - } - ], - "assumptions": ["string — unstated premises this argument depends on"], - "counterpoints_addressed": ["claim_id_from_prior_round"], - "confidence_score": 0.82, - "metadata": { - "timestamp": "ISO 8601", - "tokens_used": 412, - "retrieval_latency_ms": 230 - } + "strategy_or_answer": "string — final answer or plan", + "rationale": "string — step-by-step reasoning", + "assumptions": ["string — explicit assumptions made"], + "opponent_model": "string — what the model believes about the opponent", + "risks": ["string — failure modes or weaknesses"], + "conditions": ["string — when the answer/strategy would change"] } ``` -This schema is enforced at the orchestrator level. Arguments that fail schema validation are rejected and the agent is re-prompted. +This schema is enforced at the orchestrator level using Pydantic. Responses that fail schema validation are rejected and the agent is re-prompted. ### Debate Loop @@ -159,18 +147,15 @@ At termination, the Judge outputs either a `ConsensusVerdict` or a `BestArgument ## Layer 3: Evaluation and Reliability -### Argument Quality Scorer +### ReasonBench Evaluator (LLM-as-Judge) -Each structured argument is scored independently by the Judge agent using a rubric-prompted LLM call: +Each round's responses are scored by the Judge agent using task-specific Pydantic schemas. Scores are given on a strict 0–2 integer scale for explicit dimensions tailored to the task: -| Dimension | Weight | What It Measures | -|---|---|---| -| **Logical Consistency** | 30% | Premises → conclusion validity; no internal contradictions | -| **Evidence Support** | 30% | Fraction of claims backed by retrieved sources | -| **Relevance** | 20% | Argument addresses the stated proposition, not a related but different claim | -| **Completeness** | 20% | Engages with the opponent's strongest prior point | +1. **Deterministic Logic:** Correctness, Logical Consistency, Completeness, Responsiveness +2. **Strategic Reasoning:** Opponent Modeling, Strategic Coherence, Risk Awareness, Conditional Reasoning, Responsiveness +3. **Constrained Tradeoff:** Constraint Utilization, Tradeoff Specificity, Assumptions Quality, Risk Analysis, Conditional Reasoning, Responsiveness -Scores are in [0, 1]. The weighted composite score is stored per agent per round and fed to the Metrics Dashboard. +The Judge is provided with the full prior-round context to accurately measure **Responsiveness** across iterations. ### Hallucination Detector @@ -241,15 +226,13 @@ Per-session: ### Core Types (Python / Pydantic) ```python -class Argument(BaseModel): - id: str # UUID - round: int - agent: Literal["proponent", "opponent", "judge"] - claim: str - evidence: List[EvidenceRef] - assumptions: List[str] - counterpoints_addressed: List[str] # List of claim IDs - confidence_score: float # [0, 1] +class ReasonBenchResponse(BaseModel): + strategy_or_answer: str + rationale: str + assumptions: list[str] + opponent_model: str + risks: list[str] + conditions: list[str] class EvidenceRef(BaseModel): source_id: str @@ -321,13 +304,12 @@ class Verdict(BaseModel): ## MVP vs. Future Scope -### v1 (MVP) +### v1 (ReasonBench MVP) - Proponent + Opponent + Judge agents -- 3-round debate loop with structured argument format -- Basic quality scoring (logical consistency + evidence support) -- Hallucination detection (source existence check) -- CLI or minimal web UI -- FAISS vector index over user-provided documents +- 3-round debate loop with `ReasonBenchResponse` output tracking +- Task-specific rubrics on 0-2 scales (Logic, Strategy, Tradeoffs) +- Per-round evaluation explicitly measuring "Responsiveness" +- Automated runner for evaluating multiple models ### v2 - Argument graph visualization (D3.js) From 6d9d188e8c8d3aba9d8a6a0700ce7eb43551c5c7 Mon Sep 17 00:00:00 2001 From: MilindC Date: Sat, 9 May 2026 12:48:59 +0000 Subject: [PATCH 2/3] Expanding the scope of ArgumentLab beyond standard proposition debate to formally benchmark model performance on logical puzzles, game theory, and constrained decision-making tasks --- .env.example | 19 + AGENTS.md | 21 +- CODEOWNERS | 1 + README.md | 65 +- docs/architecture.md | 654 ++++++++++---------- docs/design.md | 148 ++--- scripts/setup.sh | 11 + scripts/test.sh | 6 +- setup/debate.py | 5 +- setup/ingest_corpus.py | 2 +- src/argument_lab/core/agents.py | 50 +- src/argument_lab/core/evaluation.py | 50 +- src/argument_lab/core/exporter.py | 106 ++-- src/argument_lab/core/faiss_index.py | 1 - src/argument_lab/core/reasonbench_eval.py | 83 +++ src/argument_lab/core/reasonbench_models.py | 0 16 files changed, 676 insertions(+), 546 deletions(-) create mode 100644 .env.example create mode 100644 CODEOWNERS create mode 100644 src/argument_lab/core/reasonbench_eval.py create mode 100644 src/argument_lab/core/reasonbench_models.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..758c717 --- /dev/null +++ b/.env.example @@ -0,0 +1,19 @@ +# Copy this file to .env for local development. +# Never commit real API keys, tokens, credentials, or private endpoints. + +# Required for LLM-backed debate runs and embedding generation. +OPENAI_API_KEY= + +# Optional model overrides. Leave blank to use code defaults. +ARGUMENT_LAB_CHAT_MODEL= +ARGUMENT_LAB_QUERY_MODEL= +ARGUMENT_LAB_EMBEDDING_MODEL= + +# Local data paths used by setup scripts and CLI runs. +ARGUMENT_LAB_LOCAL_DATA_DIR=local_data +ARGUMENT_LAB_FAISS_INDEX_DIR=local_data/faiss_index +ARGUMENT_LAB_RESULTS_DIR=local_data/results + +# Test/runtime toggles for future harness work. +ARGUMENT_LAB_OFFLINE_MODE=true +ARGUMENT_LAB_LOG_LEVEL=INFO \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index 5dcf87b..5383c48 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,11 +12,12 @@ ## Tech stack -- Backend: Python, FastAPI -- Frontend: React, TypeScript -- Database: PostgreSQL +- Current implementation: Python, LangGraph, LangChain, FAISS, pytest +- Planned backend: FastAPI +- Planned frontend: React, TypeScript +- Planned database: PostgreSQL - LLM orchestration: LangGraph -- Tests: pytest, Vitest +- Tests: pytest; Vitest only after a frontend is added ## Rules for AI coding agents - Do not make large rewrites unless explicitly requested. @@ -29,9 +30,17 @@ ## Commands - Install backend: `pip install -r requirements.txt` - Run backend tests: `pytest` -- Run frontend tests: `npm test` +- Run frontend tests: `npm test` after `package.json` exists - Run full verification: `./scripts/verify.sh` +## Repository map +- Architecture overview: `docs/architecture.md` +- Current code design: `docs/design.md` +- Development environment: `docs/dev_environment.md` +- Testing and verification: `docs/testing.md` +- Agent workflow: `docs/agent_workflow.md` +- Harness readiness checklist: `docs/harness_readiness.md` + ## After making changes 1. Run ./scripts/verify.sh @@ -45,4 +54,4 @@ - Tests pass - Lint passes - New behavior is covered by tests -- Diff has been reviewed for risky changes \ No newline at end of file +- Diff has been reviewed for risky changes diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..3a6ba21 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +@Mildogrc \ No newline at end of file diff --git a/README.md b/README.md index 727156a..a1b79e1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ReasonBench (powered by ArgumentLab) -**A multi-agent benchmark that conducts structured adversarial debates to quantitatively evaluate LLM reasoning quality, strategic consistency, and adaptability.** +**A multi-agent benchmark that conducts structured adversarial debates to quantitatively evaluate LLM reasoning quality, strategic coherence, and adaptability.** Most AI benchmarks evaluate static question-answering. ReasonBench is built to *interrogate* models — forcing agents to construct structured strategies, defend them across debate rounds, explicitly state assumptions, and adapt to adversarial critiques. It is a rigorous framework for measuring how AI systems reason, disagree, and evolve their thinking. @@ -8,15 +8,14 @@ Most AI benchmarks evaluate static question-answering. ReasonBench is built to * ## Why This Is Hard -Getting an LLM to argue a position is trivial. Getting it to any of the following: +Getting an LLM to argue a position is trivial. Getting it to: - maintain logical consistency across multiple rounds - cite grounded, verifiable evidence - respond specifically to an opponent's claims (not just re-assert its own) - detect when it is contradicting itself - converge toward a defensible conclusion under adversarial input - -ArgumentLab treats each of these as a measurable engineering problem. +...is not. ArgumentLab treats each of these as a measurable engineering problem. --- @@ -24,8 +23,8 @@ ArgumentLab treats each of these as a measurable engineering problem. ReasonBench currently evaluates reasoning across three core tasks, scored automatically by an LLM-as-Judge over a 3-round debate protocol: -1. **Deterministic Logic (Constraint Puzzle):** Evaluates correctness, logical correctness, completeness, and responsiveness. -2. **Strategic Reasoning (Asymmetric Game):** Evaluates opponent modeling, strategic consistency, risk awareness, conditional reasoning, and responsiveness. +1. **Deterministic Logic (Constraint Puzzle):** Evaluates correctness, logical consistency, completeness, and responsiveness. +2. **Strategic Reasoning (Asymmetric Game):** Evaluates opponent modeling, strategic coherence, risk awareness, conditional reasoning, and responsiveness. 3. **Constrained Tradeoff Reasoning:** Evaluates constraint utilization, tradeoff specificity, explicit assumptions, risk analysis, and conditional reasoning. For full architectural details, see the [Architecture document](/docs/architecture.md). @@ -67,7 +66,7 @@ This eliminates the "chatty LLM" failure mode and makes every output strictly sc Debates run across three rounds with increasing specificity: - **Round 1** — Initial arguments, top-level claims -- **Round 2** — Targeted rebuttals; agents must respond to specific prior claims +- **Round 2** — Targeted rebuttals; agents must address specific prior claims - **Round 3** — Refinement; agents update positions based on accumulated evidence Each agent receives the full prior-round context and is penalized (in scoring) for ignoring it. @@ -140,51 +139,7 @@ Tracked across every debate session: - Contradiction frequency - Convergence round (or failure to converge) - Evidence citation rate -- Disagreement persistence through rounds ---- - -## Getting Started - -### Prerequisites - -Ensure you have Python 3.10+ installed and set your OpenAI API key: - -```bash -export OPENAI_API_KEY=sk-... -``` - -Install the dependencies: - -```bash -pip install -r requirements.txt -``` - -### 1. Ingest Data - -Before running a debate, the agents need a retrieval corpus (FAISS index). ArgumentLab includes a sample corpus to get started instantly: - -```bash -python setup/ingest_corpus.py --sample -``` - -You can also ingest your own `.txt` or `.pdf` documents: - -```bash -python setup/ingest_corpus.py --docs path/to/your/documents/ -``` - -### 2. Run a Debate - -Execute a full, structured debate by providing a proposition. The debate streams live to the console, printing argument blocks and judge scores round-by-round. - -```bash -python setup/debate.py \ - --proposition "Companies should replace legacy infrastructure with AI-driven systems." \ - --session-id my_debate_001 -``` - -Once finished, the debate state is automatically exported to `local_data/results/my_debate_001.json` and a human-readable `my_debate_001.md` report. - +- Disagreement persistence across rounds --- ## Getting Started @@ -275,7 +230,7 @@ Run all 3 benchmark tasks across 2 models and produce structured scores. ## How This Differs from Kialo -[Kialo](https://www.kialo.com) is a platform for human-generated, community-refined argument trees — effectively structured Wikipedia for reasoning. It is an effective tool for its purpose. +[Kialo](https://www.kialo.com) is a platform for human-generated, community-refined argument trees — effectively structured Wikipedia for reasoning. It is a valuable tool for its purpose. ArgumentLab is a different category entirely: @@ -296,7 +251,7 @@ ArgumentLab is a different category entirely: ## Research Connections -ArgumentLab is positioned to be at the intersection of several active research directions: +ArgumentLab sits at the intersection of several active research directions: - **LLM-as-Judge** — using language models as evaluators of reasoning quality - **Multi-agent debate** — Du et al. (2023), *Improving Factuality and Reasoning in Language Models through Multiagent Debate* @@ -312,6 +267,6 @@ ArgumentLab is positioned to be at the intersection of several active research d ## Author -**Milind C** — MS Computer Science (Artificial Intelligence), Georgia Institute of Technology +**Milind C** — MS Computer Science (Artifical Intelligence), Georgia Institute of Technology [LinkedIn](https://linkedin.com/in/milind-chandramohan) · [GitHub](https://github.com/mildogrc) diff --git a/docs/architecture.md b/docs/architecture.md index 4bcb276..83f3dfb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,327 +1,327 @@ -# ArgumentLab — Architecture Document - -## Table of Contents - -1. [System Overview](#system-overview) -2. [High-Level Architecture](#high-level-architecture) -3. [Layer 1: Core Reasoning Engine](#layer-1-core-reasoning-engine) -4. [Layer 2: Debate System](#layer-2-debate-system) -5. [Layer 3: Evaluation and Reliability](#layer-3-evaluation-and-reliability) -6. [Data Models](#data-models) -7. [Component Interaction & Data Flow](#component-interaction--data-flow) -8. [Tech Stack](#tech-stack) -9. [Deployment Architecture](#deployment-architecture) -10. [Key Design Decisions](#key-design-decisions) -11. [MVP vs. Future Scope](#mvp-vs-future-scope) - ---- - -## System Overview - -ArgumentLab is a multi-agent reasoning framework that runs structured AI-to-AI debates and quantitatively evaluates argument quality, consistency, and hallucination under adversarial conditions. - -The system is organized into three stacked layers: - -![System Overview](/docs/images/systemoverview.png) - -Each layer is independently testable and builds on the outputs of the layer below it. - ---- - -## High-Level Architecture - -![High-Level Architecture](/docs/images/highlevelarch.png) - ---- - -## Layer 1: Core Reasoning Engine - -### Agents - -The system uses four agents with distinct, non-overlapping roles: - -| Agent | Role | Required | -|---|---|---| -| **Proponent** | Argues FOR the proposition | Yes | -| **Opponent** | Argues AGAINST the proposition | Yes | -| **Judge** | Evaluates quality, detects convergence, produces verdict | Yes | -| **Moderator** | Enforces structure, prevents topic drift | Optional (v2+) | - -Each agent is implemented as a stateless function that receives a context object and returns a structured argument. State is held externally in the Shared Debate State Manager. - -### Structured ReasonBench Schema - -Every agent output must strictly conform to the `ReasonBenchResponse` JSON schema — no free-form prose: - -```json -{ - "strategy_or_answer": "string — final answer or plan", - "rationale": "string — step-by-step reasoning", - "assumptions": ["string — explicit assumptions made"], - "opponent_model": "string — what the model believes about the opponent", - "risks": ["string — failure modes or weaknesses"], - "conditions": ["string — when the answer/strategy would change"] -} -``` - -This schema is enforced at the orchestrator level using Pydantic. Responses that fail schema validation are rejected and the agent is re-prompted. - -### Debate Loop - -Debates run across three rounds with increasing specificity requirements: - -``` -Round 1 ──► Initial arguments; top-level claims; no rebuttal required -Round 2 ──► Targeted rebuttals; each agent must reference ≥1 prior claim by ID -Round 3 ──► Position refinement; agents may update confidence scores - and must acknowledge new evidence introduced in Round 2 -``` - -The orchestrator enforces sequencing: agents within a round may run in parallel, but Round N cannot begin until all agents have submitted valid Round N-1 outputs. - -### Shared Debate State Manager - -A central state object persists across rounds and is passed to each agent: - -```python -DebateState = { - "proposition": str, - "rounds": List[Round], # All prior round outputs - "claims_registry": Dict[str, Claim], # claim_id → claim object - "addressed_claims": Set[str], # Claims that received rebuttals - "ignored_claims": Set[str], # Claims that were not engaged - "agent_positions": Dict[str, List[float]], # Confidence drift per agent - "repetition_flags": List[str], # Claim IDs flagged as near-duplicates -} -``` - -Position drift (change in `confidence_score` across rounds) is tracked as a signal for convergence detection and contradiction analysis. - ---- - -## Layer 2: Debate System - -### Topic Framing - -Raw user input is pre-processed by the Topic Framer before any agent receives it: - -1. **Proposition normalization** — converts questions to binary propositions ("Should X?" → "X is preferable to Y under conditions Z") -2. **Scope constraints** — limits the proposition to a defined domain to prevent topic drift -3. **Ambiguity flagging** — identifies underspecified terms that could cause agents to argue past each other - -Output is a `DebateProposition` object shared with all agents. - -### Evidence Retrieval (RAG) - -Agents cannot assert facts without grounding them in retrieved evidence. The RAG pipeline: - -![Evidence Retrieval](/docs/images/evidenceretrieval.png) - -Source reliability is tracked as a first-class metric. Sources are scored on: domain authority (if web), internal consistency, and citation frequency across rounds. - -### Argument Graph - -The debate is represented as a directed graph, not a linear transcript. This is the primary data structure for the debate: - -![Argument Graph](/docs/images/argumentgraph.png) - -**Node types:** `claim`, `rebuttal`, `evidence`, `concession`, `agreement_zone` - -**Edge types:** `supports`, `challenged_by`, `ignored_by`, `conceded_by`, `cited_by` - -The graph is stored in NetworkX (backend) and serialized to JSON for frontend rendering with D3.js. The Evaluation Layer queries this graph directly for contradiction and convergence analysis. - -### Convergence Detector - -The Judge agent monitors three convergence signals after each round: - -| Signal | Definition | Action | -|---|---|---| -| **Agreement zone** | Both agents assign high confidence to the same claim | Mark as resolved; remove from active debate | -| **Unresolved conflict** | Both agents maintain opposing high-confidence positions | Flag for final verdict | -| **Stalemate** | No argument quality score improvement across a full round | Trigger early termination | - -At termination, the Judge outputs either a `ConsensusVerdict` or a `BestArgumentVerdict` with a structured explanation of what remained unresolved and why. - ---- - -## Layer 3: Evaluation and Reliability - -### ReasonBench Evaluator (LLM-as-Judge) - -Each round's responses are scored by the Judge agent using task-specific Pydantic schemas. Scores are given on a strict 0–2 integer scale for explicit dimensions tailored to the task: - -1. **Deterministic Logic:** Correctness, Logical Consistency, Completeness, Responsiveness -2. **Strategic Reasoning:** Opponent Modeling, Strategic Coherence, Risk Awareness, Conditional Reasoning, Responsiveness -3. **Constrained Tradeoff:** Constraint Utilization, Tradeoff Specificity, Assumptions Quality, Risk Analysis, Conditional Reasoning, Responsiveness - -The Judge is provided with the full prior-round context to accurately measure **Responsiveness** across iterations. - -### Hallucination Detector - -Runs as a post-processing step on every argument: - -``` -For each claim in argument: - 1. Check that every cited source_id exists in the vector index - 2. Retrieve the cited chunk; verify the claim is semantically entailed - 3. Extract named entities (names, numbers, dates) from the claim - 4. Verify each entity appears verbatim in the cited source - 5. Flag mismatches as hallucinations with a severity score -``` - -Hallucination rate is tracked as `hallucinations / total_claims` per agent per round. - -### Contradiction Detector - -Compares each agent's Round N output against all prior rounds for the same agent: - -``` -For each (Round N claim, Round K claim) where K < N: - 1. Embed both claims - 2. If cosine similarity > threshold_similar: - → Check for logical contradiction using LLM-as-judge - 3. If contradiction confirmed: - → Flag with round numbers, claim IDs, and contradiction type - → Apply penalty to agent's Round N quality score -``` - -Contradiction types: `direct_negation`, `weakened_commitment`, `shifted_evidence_basis`, `ignored_own_prior_claim`. - -### Adversarial Testing Mode - -A controlled evaluation mode that injects degraded inputs before the debate begins: - -| Injection Type | Implementation | Purpose | -|---|---|---| -| **Misleading data** | Replace k% of RAG chunks with plausible-but-false variants | Test hallucination resistance | -| **Incomplete context** | Remove chunks covering key sub-topics | Test inference under gaps | -| **Conflicting evidence** | Inject sources contradicting existing corpus | Test conflict resolution | - -Adversarial sessions produce a `BehavioralProfile` comparing quality scores, hallucination rates, and contradiction frequency against a clean baseline run on the same proposition. - -### Metrics Dashboard - -Tracked per session, surfaced in real-time (Streamlit or React + Recharts): - -``` -Per-agent, per-round: - - Argument quality score (composite + per-dimension breakdown) - - Hallucination rate - - Contradiction count - - Evidence citation rate - - Confidence score trajectory - -Per-session: - - Convergence round (or "did not converge") - - Disagreement persistence (% of claims still unresolved at termination) - - Argument graph topology (depth, branching factor, ignored-claim rate) - - Adversarial delta (if applicable) -``` - ---- - -## Data Models - -### Core Types (Python / Pydantic) - -```python -class ReasonBenchResponse(BaseModel): - strategy_or_answer: str - rationale: str - assumptions: list[str] - opponent_model: str - risks: list[str] - conditions: list[str] - -class EvidenceRef(BaseModel): - source_id: str - excerpt: str - reliability_score: float - -class DebateSession(BaseModel): - id: str - proposition: str - status: Literal["in_progress", "converged", "stalemate", "terminated"] - rounds: List[List[Argument]] # rounds[i] = all arguments in round i - argument_graph: GraphData - metrics: SessionMetrics - verdict: Optional[Verdict] - -class Verdict(BaseModel): - type: Literal["consensus", "best_argument"] - winner: Optional[Literal["proponent", "opponent"]] - consensus_claims: List[str] - unresolved_claims: List[str] - explanation: str -``` - ---- - -## Component Interaction & Data Flow - -![Component Interaction](/docs/images/dataflow.png) - ---- - -## Tech Stack - -| Component | Technology | Notes | -|---|---|---| -| Agent orchestration | LangGraph / custom loop | LangGraph for state management; custom loop for strict round enforcement | -| LLM backbone | Claude (Anthropic API) / GPT-4o | Swappable via provider abstraction | -| Embedding model | OpenAI `text-embedding-3-small` or equivalent | Used for RAG and contradiction detection | -| Vector index | FAISS (local) / ChromaDB (persistent) | FAISS for MVP; ChromaDB for v2 with session history | -| Argument graph | NetworkX (backend), D3.js (frontend) | Graph exported as JSON for frontend | -| Scoring | LLM-as-judge with rubric prompting | Judge agent uses structured output schema | -| Metrics dashboard | Streamlit (MVP) / React + Recharts (v2) | Streamlit for rapid iteration | -| Backend API | FastAPI | REST endpoints + WebSocket for real-time dashboard | -| Frontend | React (v2) | D3.js for argument graph; Recharts for metrics | - ---- - -## Deployment Architecture - -![Deployment Architecture](/docs/images/deployment.png) - ---- - -## Key Design Decisions - -**Structured output over free text.** All agent outputs are JSON objects validated against a schema. This prevents the "chatty LLM" failure mode and makes every output programmatically evaluable. - -**Claim IDs as the unit of discourse.** Every claim receives a UUID. Rebuttals must reference prior claim IDs. This enforces specificity and makes it possible to detect when agents ignore arguments rather than counter them. - -**External state management.** Agents are stateless functions. All debate context lives in the Shared Debate State Manager. This makes agents independently testable and allows replaying any round with a modified state. - -**Graph over transcript.** Representing the debate as a directed graph (rather than a linear conversation) exposes the actual reasoning topology — which arguments were addressed, which were ignored, and which drove convergence. - -**LLM-as-judge with rubric.** The Judge agent uses structured prompting with an explicit scoring rubric rather than free-form evaluation. This makes scores reproducible and auditable. - -**Hallucination as a first-class metric.** Most debate systems track persuasiveness. ArgumentLab treats factual grounding as equally important — an argument that cites non-existent evidence is penalized regardless of its logical structure. - ---- - -## MVP vs. Future Scope - -### v1 (ReasonBench MVP) -- Proponent + Opponent + Judge agents -- 3-round debate loop with `ReasonBenchResponse` output tracking -- Task-specific rubrics on 0-2 scales (Logic, Strategy, Tradeoffs) -- Per-round evaluation explicitly measuring "Responsiveness" -- Automated runner for evaluating multiple models - -### v2 -- Argument graph visualization (D3.js) -- Full hallucination pipeline (semantic entailment + entity grounding) -- Contradiction detector with cross-round comparison -- Metrics dashboard (React + Recharts) -- Convergence detection with stalemate handling -- Persistent sessions (ChromaDB + Postgres) - -### Stretch Goals -- Human-in-the-loop intervention (pause, redirect, inject evidence) -- Adversarial injection test suite -- Strategy modes: `aggressive`, `evidence-first`, `exploratory` -- Multi-agent expansion: domain expert, skeptic, data-driven agents -- Longitudinal session tracking (improvement across multiple debates on the same topic) +# ArgumentLab — Architecture Document + +## Table of Contents + +1. [System Overview](#system-overview) +2. [High-Level Architecture](#high-level-architecture) +3. [Layer 1: Core Reasoning Engine](#layer-1-core-reasoning-engine) +4. [Layer 2: Debate System](#layer-2-debate-system) +5. [Layer 3: Evaluation and Reliability](#layer-3-evaluation-and-reliability) +6. [Data Models](#data-models) +7. [Component Interaction & Data Flow](#component-interaction--data-flow) +8. [Tech Stack](#tech-stack) +9. [Deployment Architecture](#deployment-architecture) +10. [Key Design Decisions](#key-design-decisions) +11. [MVP vs. Future Scope](#mvp-vs-future-scope) + +--- + +## System Overview + +ArgumentLab is a multi-agent reasoning framework that runs structured AI-to-AI debates and quantitatively evaluates argument quality, consistency, and hallucination under adversarial conditions. + +The system is organized into three stacked layers: + +![System Overview](/docs/images/systemoverview.png) + +Each layer is independently testable and builds on the outputs of the layer below it. + +--- + +## High-Level Architecture + +![High-Level Architecture](/docs/images/highlevelarch.png) + +--- + +## Layer 1: Core Reasoning Engine + +### Agents + +The system uses four agents with distinct, non-overlapping roles: + +| Agent | Role | Required | +|---|---|---| +| **Proponent** | Argues FOR the proposition | Yes | +| **Opponent** | Argues AGAINST the proposition | Yes | +| **Judge** | Evaluates quality, detects convergence, produces verdict | Yes | +| **Moderator** | Enforces structure, prevents topic drift | Optional (v2+) | + +Each agent is implemented as a stateless function that receives a context object and returns a structured argument. State is held externally in the Shared Debate State Manager. + +### Structured ReasonBench Schema + +Every agent output must strictly conform to the `ReasonBenchResponse` JSON schema — no free-form prose: + +```json +{ + "strategy_or_answer": "string — final answer or plan", + "rationale": "string — step-by-step reasoning", + "assumptions": ["string — explicit assumptions made"], + "opponent_model": "string — what the model believes about the opponent", + "risks": ["string — failure modes or weaknesses"], + "conditions": ["string — when the answer/strategy would change"] +} +``` + +This schema is enforced at the orchestrator level using Pydantic. Responses that fail schema validation are rejected and the agent is re-prompted. + +### Debate Loop + +Debates run across three rounds with increasing specificity requirements: + +``` +Round 1 ──► Initial arguments; top-level claims; no rebuttal required +Round 2 ──► Targeted rebuttals; each agent must reference ≥1 prior claim by ID +Round 3 ──► Position refinement; agents may update confidence scores + and must acknowledge new evidence introduced in Round 2 +``` + +The orchestrator enforces sequencing: agents within a round may run in parallel, but Round N cannot begin until all agents have submitted valid Round N-1 outputs. + +### Shared Debate State Manager + +A central state object persists across rounds and is passed to each agent: + +```python +DebateState = { + "proposition": str, + "rounds": List[Round], # All prior round outputs + "claims_registry": Dict[str, Claim], # claim_id → claim object + "addressed_claims": Set[str], # Claims that received rebuttals + "ignored_claims": Set[str], # Claims that were not engaged + "agent_positions": Dict[str, List[float]], # Confidence drift per agent + "repetition_flags": List[str], # Claim IDs flagged as near-duplicates +} +``` + +Position drift (change in `confidence_score` across rounds) is tracked as a signal for convergence detection and contradiction analysis. + +--- + +## Layer 2: Debate System + +### Topic Framing + +Raw user input is pre-processed by the Topic Framer before any agent receives it: + +1. **Proposition normalization** — converts questions to binary propositions ("Should X?" → "X is preferable to Y under conditions Z") +2. **Scope constraints** — limits the proposition to a defined domain to prevent topic drift +3. **Ambiguity flagging** — identifies underspecified terms that could cause agents to argue past each other + +Output is a `DebateProposition` object shared with all agents. + +### Evidence Retrieval (RAG) + +Agents cannot assert facts without grounding them in retrieved evidence. The RAG pipeline: + +![Evidence Retrieval](/docs/images/evidenceretrieval.png) + +Source reliability is tracked as a first-class metric. Sources are scored on: domain authority (if web), internal consistency, and citation frequency across rounds. + +### Argument Graph + +The debate is represented as a directed graph, not a linear transcript. This is the primary data structure for the debate: + +![Argument Graph](/docs/images/argumentgraph.png) + +**Node types:** `claim`, `rebuttal`, `evidence`, `concession`, `agreement_zone` + +**Edge types:** `supports`, `challenged_by`, `ignored_by`, `conceded_by`, `cited_by` + +The graph is stored in NetworkX (backend) and serialized to JSON for frontend rendering with D3.js. The Evaluation Layer queries this graph directly for contradiction and convergence analysis. + +### Convergence Detector + +The Judge agent monitors three convergence signals after each round: + +| Signal | Definition | Action | +|---|---|---| +| **Agreement zone** | Both agents assign high confidence to the same claim | Mark as resolved; remove from active debate | +| **Unresolved conflict** | Both agents maintain opposing high-confidence positions | Flag for final verdict | +| **Stalemate** | No argument quality score improvement across a full round | Trigger early termination | + +At termination, the Judge outputs either a `ConsensusVerdict` or a `BestArgumentVerdict` with a structured explanation of what remained unresolved and why. + +--- + +## Layer 3: Evaluation and Reliability + +### ReasonBench Evaluator (LLM-as-Judge) + +Each round's responses are scored by the Judge agent using task-specific Pydantic schemas. Scores are given on a strict 0–2 integer scale for explicit dimensions tailored to the task: + +1. **Deterministic Logic:** Correctness, Logical Consistency, Completeness, Responsiveness +2. **Strategic Reasoning:** Opponent Modeling, Strategic Coherence, Risk Awareness, Conditional Reasoning, Responsiveness +3. **Constrained Tradeoff:** Constraint Utilization, Tradeoff Specificity, Assumptions Quality, Risk Analysis, Conditional Reasoning, Responsiveness + +The Judge is provided with the full prior-round context to accurately measure **Responsiveness** across iterations. + +### Hallucination Detector + +Runs as a post-processing step on every argument: + +``` +For each claim in argument: + 1. Check that every cited source_id exists in the vector index + 2. Retrieve the cited chunk; verify the claim is semantically entailed + 3. Extract named entities (names, numbers, dates) from the claim + 4. Verify each entity appears verbatim in the cited source + 5. Flag mismatches as hallucinations with a severity score +``` + +Hallucination rate is tracked as `hallucinations / total_claims` per agent per round. + +### Contradiction Detector + +Compares each agent's Round N output against all prior rounds for the same agent: + +``` +For each (Round N claim, Round K claim) where K < N: + 1. Embed both claims + 2. If cosine similarity > threshold_similar: + → Check for logical contradiction using LLM-as-judge + 3. If contradiction confirmed: + → Flag with round numbers, claim IDs, and contradiction type + → Apply penalty to agent's Round N quality score +``` + +Contradiction types: `direct_negation`, `weakened_commitment`, `shifted_evidence_basis`, `ignored_own_prior_claim`. + +### Adversarial Testing Mode + +A controlled evaluation mode that injects degraded inputs before the debate begins: + +| Injection Type | Implementation | Purpose | +|---|---|---| +| **Misleading data** | Replace k% of RAG chunks with plausible-but-false variants | Test hallucination resistance | +| **Incomplete context** | Remove chunks covering key sub-topics | Test inference under gaps | +| **Conflicting evidence** | Inject sources contradicting existing corpus | Test conflict resolution | + +Adversarial sessions produce a `BehavioralProfile` comparing quality scores, hallucination rates, and contradiction frequency against a clean baseline run on the same proposition. + +### Metrics Dashboard + +Tracked per session, surfaced in real-time (Streamlit or React + Recharts): + +``` +Per-agent, per-round: + - Argument quality score (composite + per-dimension breakdown) + - Hallucination rate + - Contradiction count + - Evidence citation rate + - Confidence score trajectory + +Per-session: + - Convergence round (or "did not converge") + - Disagreement persistence (% of claims still unresolved at termination) + - Argument graph topology (depth, branching factor, ignored-claim rate) + - Adversarial delta (if applicable) +``` + +--- + +## Data Models + +### Core Types (Python / Pydantic) + +```python +class ReasonBenchResponse(BaseModel): + strategy_or_answer: str + rationale: str + assumptions: list[str] + opponent_model: str + risks: list[str] + conditions: list[str] + +class EvidenceRef(BaseModel): + source_id: str + excerpt: str + reliability_score: float + +class DebateSession(BaseModel): + id: str + proposition: str + status: Literal["in_progress", "converged", "stalemate", "terminated"] + rounds: List[List[Argument]] # rounds[i] = all arguments in round i + argument_graph: GraphData + metrics: SessionMetrics + verdict: Optional[Verdict] + +class Verdict(BaseModel): + type: Literal["consensus", "best_argument"] + winner: Optional[Literal["proponent", "opponent"]] + consensus_claims: List[str] + unresolved_claims: List[str] + explanation: str +``` + +--- + +## Component Interaction & Data Flow + +![Component Interaction](/docs/images/dataflow.png) + +--- + +## Tech Stack + +| Component | Technology | Notes | +|---|---|---| +| Agent orchestration | LangGraph / custom loop | LangGraph for state management; custom loop for strict round enforcement | +| LLM backbone | Claude (Anthropic API) / GPT-4o | Swappable via provider abstraction | +| Embedding model | OpenAI `text-embedding-3-small` or equivalent | Used for RAG and contradiction detection | +| Vector index | FAISS (local) / ChromaDB (persistent) | FAISS for MVP; ChromaDB for v2 with session history | +| Argument graph | NetworkX (backend), D3.js (frontend) | Graph exported as JSON for frontend | +| Scoring | LLM-as-judge with rubric prompting | Judge agent uses structured output schema | +| Metrics dashboard | Streamlit (MVP) / React + Recharts (v2) | Streamlit for rapid iteration | +| Backend API | FastAPI | REST endpoints + WebSocket for real-time dashboard | +| Frontend | React (v2) | D3.js for argument graph; Recharts for metrics | + +--- + +## Deployment Architecture + +![Deployment Architecture](/docs/images/deployment.png) + +--- + +## Key Design Decisions + +**Structured output over free text.** All agent outputs are JSON objects validated against a schema. This prevents the "chatty LLM" failure mode and makes every output programmatically evaluable. + +**Claim IDs as the unit of discourse.** Every claim receives a UUID. Rebuttals must reference prior claim IDs. This enforces specificity and makes it possible to detect when agents ignore arguments rather than counter them. + +**External state management.** Agents are stateless functions. All debate context lives in the Shared Debate State Manager. This makes agents independently testable and allows replaying any round with a modified state. + +**Graph over transcript.** Representing the debate as a directed graph (rather than a linear conversation) exposes the actual reasoning topology — which arguments were addressed, which were ignored, and which drove convergence. + +**LLM-as-judge with rubric.** The Judge agent uses structured prompting with an explicit scoring rubric rather than free-form evaluation. This makes scores reproducible and auditable. + +**Hallucination as a first-class metric.** Most debate systems track persuasiveness. ArgumentLab treats factual grounding as equally important — an argument that cites non-existent evidence is penalized regardless of its logical structure. + +--- + +## MVP vs. Future Scope + +### v1 (ReasonBench MVP) +- Proponent + Opponent + Judge agents +- 3-round debate loop with `ReasonBenchResponse` output tracking +- Task-specific rubrics on 0-2 scales (Logic, Strategy, Tradeoffs) +- Per-round evaluation explicitly measuring "Responsiveness" +- Automated runner for evaluating multiple models + +### v2 +- Argument graph visualization (D3.js) +- Full hallucination pipeline (semantic entailment + entity grounding) +- Contradiction detector with cross-round comparison +- Metrics dashboard (React + Recharts) +- Convergence detection with stalemate handling +- Persistent sessions (ChromaDB + Postgres) + +### Stretch Goals +- Human-in-the-loop intervention (pause, redirect, inject evidence) +- Adversarial injection test suite +- Strategy modes: `aggressive`, `evidence-first`, `exploratory` +- Multi-agent expansion: domain expert, skeptic, data-driven agents +- Longitudinal session tracking (improvement across multiple debates on the same topic) diff --git a/docs/design.md b/docs/design.md index 6d45e36..5e087c0 100644 --- a/docs/design.md +++ b/docs/design.md @@ -1,74 +1,74 @@ -# ArgumentLab: Code Design & Implementation - -This document serves as a living guide to the actual codebase. It explains the current structure of the Python packages, what each file does, and how the logic is implemented. - ---- - -## Code Organization - -The application is modularized under `src/argument_lab/`, currently split into `core` data structures and the `orchestrator` graph logic. - -### 1. `core/models.py` (Pydantic Schemas) -This file defines the strict data schemas enforced throughout the system, primarily for structured LLM outputs. -- **`Argument`**: The core output format for agents. It strictly types the `agent` field (`"proponent"` or `"opponent"`) and enforces a `min_length=1` validator on `evidence`, meaning an agent cannot return an argument without at least one citation. -- **`EvidenceRef`** & **`Claim`**: Base objects for tracking retrieved evidence and registering claims. -- **`JudgeEvaluation` & `ArgumentScore`**: Defines the multi-dimensional scoring rubric (logical consistency, evidence support, relevance, completeness) as well as arrays for storing `hallucination_flags` and `contradiction_flags`. - -### 2. `core/state.py` (LangGraph State Management) -This file defines `DebateState`, the shared dictionary passed between all nodes in the LangGraph workflow. -Because nodes run in parallel, we implement custom **reducers** to prevent race conditions (where "last-write-wins" would corrupt the data): -- **`merge_dicts`**: Merges dictionary updates to `claims_registry` and `agent_positions`. -- **`union_sets`**: Merges sets of `addressed_claims` and `ignored_claims`. -- **`max_round`**: Ensures the `current_round` integer can only increase, preventing a lagging node from resetting the round number. -- **`merge_status`**: Resolves `status` updates by priority (e.g. if one node writes `"converged"` and another lazily writes `"in_progress"`, it resolves to `"converged"`). - -### 3. `core/agents.py` (Proponent & Opponent Logic) -Implements the core LangGraph agent nodes. Both agents follow a strict, deterministic pipeline instead of a chatty ReAct loop: -1. **Query Formulation**: A lightweight LLM call creates 1-3 targeted search queries based on the agent's stance and the debate history. -2. **Retrieval**: The queries are executed to fetch real `EvidenceRef` chunks. -3. **Generation**: The LLM uses `.with_structured_output(Argument)` to generate its argument, using the retrieved context. -4. **Counterpoint Enforcement**: In Rounds 2 and 3, the node explicitly re-prompts the LLM if it fails to populate `counterpoints_addressed` with an opponent's prior claim ID. - -### 4. `core/retriever.py` (RAG Interface) -A thin abstraction over the vector database (e.g. FAISS). It exposes `retrieve_multi()` which aggregates search results for multiple queries and deduplicates them by `source_id`, guaranteeing the best chunks are surfaced to the agent. - -### 5. `core/evaluation.py` (Parallel Evaluators) -Contains the three concurrent evaluation nodes that run after the agents: -- **`judge_node`**: Uses an LLM to score both arguments across four dimensions, detects convergence/stalemate, updates the debate `status`, and increments the round. -- **`hallucination_check`**: Validates that cited sources explicitly support the claims. Appends failing claim IDs to `hallucination_flags`. -- **`contradiction_check`**: Compares current arguments against the agent's historical claims to detect goalpost shifting. Appends offending claim IDs to `contradiction_flags`. - -### 6. `core/prompts.py` & `core/eval_prompts.py` -Isolate all LangChain `ChatPromptTemplate` strings. They handle formatting debate histories, chunk excerpts, and evaluation logic, making it easy to iterate on prompt wording without touching workflow logic. - -### 7. `orchestrator/graph.py` (Workflow Topology) -This file compiles the `StateGraph` that controls the execution flow. It is heavily parallelized to reduce latency: -- **Agent Fan-out**: The `start_round` node branches unconditionally to `proponent_node` and `opponent_node`, running them concurrently. -- **Evaluation Sync & Fan-out**: Both agents join at a dummy node (`start_evaluation`). From there, the graph fans out again to three concurrent evaluation nodes: `judge_node`, `hallucination_check`, and `contradiction_check`. -- **Graph Update & Routing**: The parallel evaluation nodes join at `graph_update`, which writes final states. The `route_round` conditional edge then reads the state's `status` to decide whether to loop back to `start_round` or terminate the debate (`END`). - -## Implementation Efficiencies - -1. **The 2-step retrieval pipeline**: Doing `_formulate_queries` -> `_retrieve_evidence` before entering the structured argument generator avoids the grounding problem. It gives you the benefits of tool use without the risk of the LLM abandoning the schema or crashing into infinite tool loops. -2. **The `.model_copy(update=...)` filter**: - ```python - "evidence": [e for e in argument.evidence if e.source_id in valid_source_ids] or evidence_refs[:1] - ``` - If the LLM hallucinates source IDs, they are filtered out. But because Pydantic demands `min_length=1`, replacing an empty list with `evidence_refs[:1]` guarantees that validation will pass, avoiding a potential failure state. -3. **Counterpoint Enforcement**: Using the `_enforce_counterpoint_rule` to re-prompt the LLM explicitly when it fails to address an opponent's claim handles Option 3. Raising an `AgentError` if it fails twice propagates the failure to the workflow and it will get caught by the judge, as a result the judge will lower the score for logical consistency. -4. **State updates**: Extracting the confidence trajectory and accurately mapping `newly_ignored` claims via set math. - - -## Testing - -Here is what was added: -1. **`tests/core/test_state.py`**: Tests all the custom reducers (`union_sets`, `merge_dicts`, `max_round`, `merge_status`) to ensure they handle `None` defaults properly and execute the right merge logic. -2. **`tests/core/test_retriever.py`**: Mocks the `VectorIndex` protocol to test `Retriever.retrieve()` and ensures that `retrieve_multi()` correctly deduplicates source chunks, keeping the highest score. -3. **`tests/core/test_prompts.py`**: Tests the formatting helpers (`format_debate_history` and `format_evidence_context`) for edge cases like empty histories. -4. **`tests/core/test_agents.py`**: Tests the pure Python state derivation logic (`_get_prior_opponent_claim_ids` and `_update_state_from_argument`). -5. **`tests/orchestrator/test_graph.py`**: Tests that the `build_graph()` factory can successfully compile the graph topologically. -6. **`tests/core/test_models.py`**: Kept your existing test verifying `min_length=1` for evidence. - ---- - -*Note: This document should be updated whenever significant structural changes, new node implementations, or data models are introduced.* +# ArgumentLab: Code Design & Implementation + +This document serves as a living guide to the actual codebase. It explains the current structure of the Python packages, what each file does, and how the logic is implemented. + +--- + +## Code Organization + +The application is modularized under `src/argument_lab/`, currently split into `core` data structures and the `orchestrator` graph logic. + +### 1. `core/models.py` (Pydantic Schemas) +This file defines the strict data schemas enforced throughout the system, primarily for structured LLM outputs. +- **`Argument`**: The core output format for agents. It strictly types the `agent` field (`"proponent"` or `"opponent"`) and enforces a `min_length=1` validator on `evidence`, meaning an agent cannot return an argument without at least one citation. +- **`EvidenceRef`** & **`Claim`**: Base objects for tracking retrieved evidence and registering claims. +- **`JudgeEvaluation` & `ArgumentScore`**: Defines the multi-dimensional scoring rubric (logical consistency, evidence support, relevance, completeness) as well as arrays for storing `hallucination_flags` and `contradiction_flags`. + +### 2. `core/state.py` (LangGraph State Management) +This file defines `DebateState`, the shared dictionary passed between all nodes in the LangGraph workflow. +Because nodes run in parallel, we implement custom **reducers** to prevent race conditions (where "last-write-wins" would corrupt the data): +- **`merge_dicts`**: Merges dictionary updates to `claims_registry` and `agent_positions`. +- **`union_sets`**: Merges sets of `addressed_claims` and `ignored_claims`. +- **`max_round`**: Ensures the `current_round` integer can only increase, preventing a lagging node from resetting the round number. +- **`merge_status`**: Resolves `status` updates by priority (e.g. if one node writes `"converged"` and another lazily writes `"in_progress"`, it resolves to `"converged"`). + +### 3. `core/agents.py` (Proponent & Opponent Logic) +Implements the core LangGraph agent nodes. Both agents follow a strict, deterministic pipeline instead of a chatty ReAct loop: +1. **Query Formulation**: A lightweight LLM call creates 1-3 targeted search queries based on the agent's stance and the debate history. +2. **Retrieval**: The queries are executed to fetch real `EvidenceRef` chunks. +3. **Generation**: The LLM uses `.with_structured_output(Argument)` to generate its argument, using the retrieved context. +4. **Counterpoint Enforcement**: In Rounds 2 and 3, the node explicitly re-prompts the LLM if it fails to populate `counterpoints_addressed` with an opponent's prior claim ID. + +### 4. `core/retriever.py` (RAG Interface) +A thin abstraction over the vector database (e.g. FAISS). It exposes `retrieve_multi()` which aggregates search results for multiple queries and deduplicates them by `source_id`, guaranteeing the best chunks are surfaced to the agent. + +### 5. `core/evaluation.py` (Parallel Evaluators) +Contains the three concurrent evaluation nodes that run after the agents: +- **`judge_node`**: Uses an LLM to score both arguments across four dimensions, detects convergence/stalemate, updates the debate `status`, and increments the round. +- **`hallucination_check`**: Validates that cited sources explicitly support the claims. Appends failing claim IDs to `hallucination_flags`. +- **`contradiction_check`**: Compares current arguments against the agent's historical claims to detect goalpost shifting. Appends offending claim IDs to `contradiction_flags`. + +### 6. `core/prompts.py` & `core/eval_prompts.py` +Isolate all LangChain `ChatPromptTemplate` strings. They handle formatting debate histories, chunk excerpts, and evaluation logic, making it easy to iterate on prompt wording without touching workflow logic. + +### 7. `orchestrator/graph.py` (Workflow Topology) +This file compiles the `StateGraph` that controls the execution flow. It is heavily parallelized to reduce latency: +- **Agent Fan-out**: The `start_round` node branches unconditionally to `proponent_node` and `opponent_node`, running them concurrently. +- **Evaluation Sync & Fan-out**: Both agents join at a dummy node (`start_evaluation`). From there, the graph fans out again to three concurrent evaluation nodes: `judge_node`, `hallucination_check`, and `contradiction_check`. +- **Graph Update & Routing**: The parallel evaluation nodes join at `graph_update`, which writes final states. The `route_round` conditional edge then reads the state's `status` to decide whether to loop back to `start_round` or terminate the debate (`END`). + +## Implementation Efficiencies + +1. **The 2-step retrieval pipeline**: Doing `_formulate_queries` -> `_retrieve_evidence` before entering the structured argument generator avoids the grounding problem. It gives you the benefits of tool use without the risk of the LLM abandoning the schema or crashing into infinite tool loops. +2. **The `.model_copy(update=...)` filter**: + ```python + "evidence": [e for e in argument.evidence if e.source_id in valid_source_ids] or evidence_refs[:1] + ``` + If the LLM hallucinates source IDs, they are filtered out. But because Pydantic demands `min_length=1`, replacing an empty list with `evidence_refs[:1]` guarantees that validation will pass, avoiding a potential failure state. +3. **Counterpoint Enforcement**: Using the `_enforce_counterpoint_rule` to re-prompt the LLM explicitly when it fails to address an opponent's claim handles Option 3. Raising an `AgentError` if it fails twice propagates the failure to the workflow and it will get caught by the judge, as a result the judge will lower the score for logical consistency. +4. **State updates**: Extracting the confidence trajectory and accurately mapping `newly_ignored` claims via set math. + + +## Testing + +Here is what was added: +1. **`tests/core/test_state.py`**: Tests all the custom reducers (`union_sets`, `merge_dicts`, `max_round`, `merge_status`) to ensure they handle `None` defaults properly and execute the right merge logic. +2. **`tests/core/test_retriever.py`**: Mocks the `VectorIndex` protocol to test `Retriever.retrieve()` and ensures that `retrieve_multi()` correctly deduplicates source chunks, keeping the highest score. +3. **`tests/core/test_prompts.py`**: Tests the formatting helpers (`format_debate_history` and `format_evidence_context`) for edge cases like empty histories. +4. **`tests/core/test_agents.py`**: Tests the pure Python state derivation logic (`_get_prior_opponent_claim_ids` and `_update_state_from_argument`). +5. **`tests/orchestrator/test_graph.py`**: Tests that the `build_graph()` factory can successfully compile the graph topologically. +6. **`tests/core/test_models.py`**: Kept your existing test verifying `min_length=1` for evidence. + +--- + +*Note: This document should be updated whenever significant structural changes, new node implementations, or data models are introduced.* diff --git a/scripts/setup.sh b/scripts/setup.sh index cd0c365..0006eb6 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -3,6 +3,17 @@ set -euo pipefail echo "Setting up project..." +# Check for required tools +if ! command -v python &> /dev/null; then + echo "Error: python is not installed or not in PATH." >&2 + exit 1 +fi + +if ! command -v pip &> /dev/null; then + echo "Error: pip is not installed or not in PATH." >&2 + exit 1 +fi + # Python setup if [ -f "requirements.txt" ]; then echo "Installing Python dependencies..." diff --git a/scripts/test.sh b/scripts/test.sh index ebbb614..1bd6d4a 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -4,9 +4,11 @@ set -euo pipefail echo "Running tests..." # Python tests -if command -v pytest &> /dev/null; then - pytest +if ! command -v pytest &> /dev/null; then + echo "Error: pytest is required but not installed." >&2 + exit 1 fi +pytest # Node tests if [ -f "package.json" ]; then diff --git a/setup/debate.py b/setup/debate.py index d50cf1a..551166a 100644 --- a/setup/debate.py +++ b/setup/debate.py @@ -27,7 +27,6 @@ import os import sys import textwrap -import uuid from datetime import datetime, timezone from pathlib import Path @@ -300,7 +299,7 @@ def main() -> None: retriever = Retriever(index=faiss_index, top_k=args.top_k) # ── Build graph ──────────────────────────────────────────────────────── - print(f" Building debate graph...\n") + print(" Building debate graph...\n") debate_graph = build_graph(retriever) # ── Initial state ────────────────────────────────────────────────────── @@ -361,7 +360,7 @@ def main() -> None: _print_summary(final_state) # ── Export ───────────────────────────────────────────────────────────── - print(f" Exporting results...") + print(" Exporting results...") json_path, md_path = export_debate( state=final_state, session_id=session_id, diff --git a/setup/ingest_corpus.py b/setup/ingest_corpus.py index 292b59d..f5ee4b5 100644 --- a/setup/ingest_corpus.py +++ b/setup/ingest_corpus.py @@ -266,7 +266,7 @@ def main() -> None: index.save(INDEX_OUTPUT_PATH) print(f"\n[ingest] Done. Index saved to: {INDEX_OUTPUT_PATH}") - print(f"[ingest] Run a debate with: python setup/debate.py --proposition \"...\"") + print("[ingest] Run a debate with: python setup/debate.py --proposition \"...\"") if __name__ == "__main__": diff --git a/src/argument_lab/core/agents.py b/src/argument_lab/core/agents.py index b19bbb0..a0542c8 100644 --- a/src/argument_lab/core/agents.py +++ b/src/argument_lab/core/agents.py @@ -12,16 +12,14 @@ before it ever reaches Pydantic validation. """ -import json import uuid from typing import Any from langchain_core.output_parsers import JsonOutputParser from langchain_core.prompts import ChatPromptTemplate -from langchain_openai import ChatOpenAI from argument_lab.core.models import Argument, Claim, EvidenceRef -from argument_lab.core.retriever import Retriever, RetrieverError +from argument_lab.core.retriever import Retriever from argument_lab.core.state import DebateState, MAX_ROUNDS from argument_lab.core.prompts import ( QUERY_FORMULATION_SYSTEM, @@ -43,8 +41,38 @@ import os -_llm = ChatOpenAI(model="gpt-4o", temperature=0.2, api_key=os.environ.get("OPENAI_API_KEY", "dummy")) -_query_llm = ChatOpenAI(model="gpt-4o-mini", temperature=0.0, api_key=os.environ.get("OPENAI_API_KEY", "dummy")) +_llm: Any | None = None +_query_llm: Any | None = None + + +def _make_chat_openai(*, model: str, temperature: float) -> Any: + try: + from langchain_openai import ChatOpenAI + except ModuleNotFoundError as exc: + raise AgentError( + "langchain_openai is required for LLM-backed agent execution. " + "Install project dependencies with `pip install -r requirements.txt`." + ) from exc + + return ChatOpenAI( + model=model, + temperature=temperature, + api_key=os.environ.get("OPENAI_API_KEY", "dummy"), + ) + + +def _get_generation_llm() -> Any: + global _llm + if _llm is None: + _llm = _make_chat_openai(model="gpt-4o", temperature=0.2) + return _llm + + +def _get_query_llm() -> Any: + global _query_llm + if _query_llm is None: + _query_llm = _make_chat_openai(model="gpt-4o-mini", temperature=0.0) + return _query_llm @@ -58,13 +86,14 @@ def _formulate_queries( stance: str, history: str, current_round: int, - llm: Any = _query_llm, + llm: Any | None = None, ) -> list[str]: """ Step 1: Ask a lightweight LLM to produce search queries for this agent's next argument. Returns a list of query strings, falling back to the proposition itself if the LLM output cannot be parsed. """ + llm = llm or _get_query_llm() prompt = ChatPromptTemplate.from_messages([ ("system", QUERY_FORMULATION_SYSTEM), ("user", QUERY_FORMULATION_USER), @@ -122,12 +151,13 @@ def _generate_argument( evidence_refs: list[EvidenceRef], evidence_context: str, argument_id: str, - llm: Any = _llm, + llm: Any | None = None, ) -> Argument: """ Step 2: Generate the structured Argument using the retrieved evidence injected into the system prompt. Uses .with_structured_output() to enforce schema compliance at the LangChain layer. """ + llm = llm or _get_generation_llm() structured_llm = llm.with_structured_output(Argument) system_prompt = AGENT_SYSTEM_TEMPLATE.format_map({ @@ -175,7 +205,7 @@ def _enforce_counterpoint_rule( current_round: int, prior_opponent_claim_ids: list[str], role: str, - llm: Any = _llm, + llm: Any | None = None, proposition: str = "", history: str = "", evidence_refs: list[EvidenceRef] = [], @@ -203,6 +233,7 @@ def _enforce_counterpoint_rule( f"{prior_opponent_claim_ids}. Revise your argument to address " f"at least one of these claims directly." ) + llm = llm or _get_generation_llm() structured_llm = llm.with_structured_output(Argument) prompt = ChatPromptTemplate.from_messages([ ("system", AGENT_SYSTEM_TEMPLATE.format_map({ @@ -262,7 +293,6 @@ def _update_state_from_argument( - marks addressed and ignored claims """ # Register the new claim - from argument_lab.core.models import Claim new_claim = Claim( id=argument.id, text=argument.claim, @@ -412,4 +442,4 @@ class AgentError(RuntimeError): Raised when an agent node cannot produce a valid, schema-compliant argument. The LangGraph node will propagate this as a node failure, which can be caught by a retry policy or surfaced to the metrics dashboard. """ - pass \ No newline at end of file + pass diff --git a/src/argument_lab/core/evaluation.py b/src/argument_lab/core/evaluation.py index da6877f..9ab95d9 100644 --- a/src/argument_lab/core/evaluation.py +++ b/src/argument_lab/core/evaluation.py @@ -21,7 +21,6 @@ from typing import Any from langchain_core.prompts import ChatPromptTemplate -from langchain_openai import ChatOpenAI from argument_lab.core.models import ( Argument, @@ -54,17 +53,38 @@ # false positives. # --------------------------------------------------------------------------- -_judge_llm = ChatOpenAI( - model="gpt-4o", - temperature=0.1, - api_key=os.environ.get("OPENAI_API_KEY", "dummy"), -) +_judge_llm: Any | None = None +_checker_llm: Any | None = None -_checker_llm = ChatOpenAI( - model="gpt-4o", - temperature=0.0, - api_key=os.environ.get("OPENAI_API_KEY", "dummy"), -) + +def _make_chat_openai(*, model: str, temperature: float) -> Any: + try: + from langchain_openai import ChatOpenAI + except ModuleNotFoundError as exc: + raise EvaluationError( + "langchain_openai is required for LLM-backed evaluation execution. " + "Install project dependencies with `pip install -r requirements.txt`." + ) from exc + + return ChatOpenAI( + model=model, + temperature=temperature, + api_key=os.environ.get("OPENAI_API_KEY", "dummy"), + ) + + +def _get_judge_llm() -> Any: + global _judge_llm + if _judge_llm is None: + _judge_llm = _make_chat_openai(model="gpt-4o", temperature=0.1) + return _judge_llm + + +def _get_checker_llm() -> Any: + global _checker_llm + if _checker_llm is None: + _checker_llm = _make_chat_openai(model="gpt-4o", temperature=0.0) + return _checker_llm # --------------------------------------------------------------------------- @@ -129,7 +149,7 @@ def judge_node(state: DebateState) -> dict: # JudgeEvaluation minus the `round` field — the LLM doesn't need to # infer it; we patch it in after. - structured_llm = _judge_llm.with_structured_output(JudgeEvaluation) + structured_llm = _get_judge_llm().with_structured_output(JudgeEvaluation) chain = prompt | structured_llm evaluation: JudgeEvaluation = chain.invoke({ @@ -191,7 +211,7 @@ def hallucination_check(state: DebateState) -> dict: def _check_hallucinations_for_arg( arg: Argument, proposition: str, - llm: Any = _checker_llm, + llm: Any | None = None, ) -> HallucinationReport: """ Runs the hallucination check for a single argument. Returns a @@ -201,6 +221,7 @@ def _check_hallucinations_for_arg( ("system", HALLUCINATION_SYSTEM), ("user", HALLUCINATION_USER), ]) + llm = llm or _get_checker_llm() structured_llm = llm.with_structured_output(HallucinationReport) chain = prompt | structured_llm @@ -263,7 +284,7 @@ def _check_contradictions_for_agent( prior_args: list[Argument], proposition: str, current_round: int, - llm: Any = _checker_llm, + llm: Any | None = None, ) -> ContradictionReport: """ Runs the contradiction check for a single agent's current argument @@ -273,6 +294,7 @@ def _check_contradictions_for_agent( ("system", CONTRADICTION_SYSTEM), ("user", CONTRADICTION_USER), ]) + llm = llm or _get_checker_llm() structured_llm = llm.with_structured_output(ContradictionReport) chain = prompt | structured_llm diff --git a/src/argument_lab/core/exporter.py b/src/argument_lab/core/exporter.py index 94c02bd..08df5a4 100644 --- a/src/argument_lab/core/exporter.py +++ b/src/argument_lab/core/exporter.py @@ -278,33 +278,33 @@ def _render_markdown(p: dict) -> list[str]: w = lines.append # shorthand # ── Header ──────────────────────────────────────────────────────────── - w(f"# ArgumentLab Debate Report") - w(f"") + w("# ArgumentLab Debate Report") + w("") w(f"**Session:** `{p['session_id']}` ") w(f"**Exported:** {p['exported_at']} ") w(f"**Status:** {status_emoji} ") w(f"**Rounds completed:** {p['rounds_completed']} ") - w(f"") - w(f"---") - w(f"") - w(f"## Proposition") - w(f"") + w("") + w("---") + w("") + w("## Proposition") + w("") w(f"> {p['proposition']}") - w(f"") + w("") # ── Score Summary ────────────────────────────────────────────────────── - w(f"---") - w(f"") - w(f"## Score Summary") - w(f"") + w("---") + w("") + w("## Score Summary") + w("") traj = p.get("score_trajectories", {}) rounds_list = traj.get("rounds", []) prop_composites = traj.get("proponent_composite", []) opp_composites = traj.get("opponent_composite", []) if rounds_list: - w(f"| Round | Proponent (composite) | Opponent (composite) | Verdict |") - w(f"|---|---|---|---|") + w("| Round | Proponent (composite) | Opponent (composite) | Verdict |") + w("|---|---|---|---|") for r, pc, oc, round_data in zip( rounds_list, prop_composites, @@ -318,39 +318,39 @@ def _render_markdown(p: dict) -> list[str]: elif judge.get("stalemate_detected"): verdict = "⚖️ Stalemate" w(f"| {r} | {pc:.3f} | {oc:.3f} | {verdict} |") - w(f"") + w("") # ── Evaluation flags ─────────────────────────────────────────────────── - w(f"---") - w(f"") - w(f"## Evaluation Flags") - w(f"") + w("---") + w("") + w("## Evaluation Flags") + w("") eval_data = p.get("evaluation", {}) - w(f"| Metric | Count |") - w(f"|---|---|") + w("| Metric | Count |") + w("|---|---|") w(f"| Hallucination flags | {eval_data.get('hallucination_count', 0)} |") w(f"| Contradiction flags | {eval_data.get('contradiction_count', 0)} |") w(f"| Ignored claims | {len(p.get('ignored_claims', []))} |") w(f"| Addressed claims | {len(p.get('addressed_claims', []))} |") - w(f"") + w("") if eval_data.get("hallucination_flags"): w(f"**Hallucinated claim IDs:** `{'`, `'.join(eval_data['hallucination_flags'])}`") - w(f"") + w("") if eval_data.get("contradiction_flags"): w(f"**Contradicted claim IDs:** `{'`, `'.join(eval_data['contradiction_flags'])}`") - w(f"") + w("") # ── Round transcripts ────────────────────────────────────────────────── - w(f"---") - w(f"") - w(f"## Debate Transcript") - w(f"") + w("---") + w("") + w("## Debate Transcript") + w("") for round_data in p.get("rounds", []): r = round_data["round"] w(f"### Round {r}") - w(f"") + w("") for role in ("proponent", "opponent"): arg = round_data.get(role) @@ -359,69 +359,69 @@ def _render_markdown(p: dict) -> list[str]: label = role.capitalize() confidence = arg["confidence_score"] w(f"#### {label}") - w(f"") + w("") w(f"**Claim** *(confidence: {confidence:.2f})*") w(f"> {arg['claim']}") - w(f"") + w("") if arg.get("evidence"): - w(f"**Evidence cited**") + w("**Evidence cited**") for e in arg["evidence"]: w(f"- `[{e['source_id']}]` (reliability: {e['reliability_score']:.2f})") w(f" > {e['excerpt']}") - w(f"") + w("") if arg.get("assumptions"): - w(f"**Assumptions**") + w("**Assumptions**") for assumption in arg["assumptions"]: w(f"- {assumption}") - w(f"") + w("") if arg.get("counterpoints_addressed"): w(f"**Counterpoints addressed:** `{'`, `'.join(arg['counterpoints_addressed'])}`") - w(f"") + w("") # Judge evaluation for this round judge = round_data.get("judge") if judge: w(f"#### Judge Evaluation — Round {r}") - w(f"") - w(f"| Dimension | Proponent | Opponent |") - w(f"|---|---|---|") + w("") + w("| Dimension | Proponent | Opponent |") + w("|---|---|---|") p_b = judge["proponent"] o_b = judge["opponent"] for dim in ("logical_consistency", "evidence_support", "relevance", "completeness"): label = dim.replace("_", " ").title() w(f"| {label} | {p_b[dim]:.2f} | {o_b[dim]:.2f} |") w(f"| **Composite** | **{p_b['composite']:.3f}** | **{o_b['composite']:.3f}** |") - w(f"") + w("") w(f"**Judge's note:** {judge['explanation']}") - w(f"") + w("") - w(f"---") - w(f"") + w("---") + w("") # ── Confidence trajectories ──────────────────────────────────────────── - w(f"## Agent Confidence Trajectories") - w(f"") + w("## Agent Confidence Trajectories") + w("") for agent, positions in p.get("agent_positions", {}).items(): trajectory = " → ".join(f"{v:.2f}" for v in positions) w(f"- **{agent.capitalize()}:** {trajectory}") - w(f"") + w("") # ── Claim graph summary ──────────────────────────────────────────────── graph = p.get("claim_graph", {}) node_count = len(graph.get("nodes", [])) edge_count = len(graph.get("edges", [])) - w(f"---") - w(f"") - w(f"## Argument Graph") - w(f"") + w("---") + w("") + w("## Argument Graph") + w("") w(f"- **Total claims (nodes):** {node_count}") w(f"- **Challenged-by edges:** {edge_count}") w(f"- **Ignored claims:** {', '.join(p.get('ignored_claims', [])) or 'none'}") - w(f"") - w(f"*Full interactive graph available in the ArgumentLab dashboard.*") - w(f"") + w("") + w("*Full interactive graph available in the ArgumentLab dashboard.*") + w("") return lines \ No newline at end of file diff --git a/src/argument_lab/core/faiss_index.py b/src/argument_lab/core/faiss_index.py index b1d7fe6..5b57df4 100644 --- a/src/argument_lab/core/faiss_index.py +++ b/src/argument_lab/core/faiss_index.py @@ -18,7 +18,6 @@ from __future__ import annotations -import json import os import pickle from dataclasses import dataclass diff --git a/src/argument_lab/core/reasonbench_eval.py b/src/argument_lab/core/reasonbench_eval.py new file mode 100644 index 0000000..081fc66 --- /dev/null +++ b/src/argument_lab/core/reasonbench_eval.py @@ -0,0 +1,83 @@ +import os +from typing import Any, Union +from langchain_core.prompts import ChatPromptTemplate +from langchain_openai import ChatOpenAI + +from argument_lab.core.reasonbench_models import ( + TaskType, + ReasonBenchResponse, + Task1Evaluation, + Task2Evaluation, + Task3Evaluation, +) + +REASONBENCH_JUDGE_SYSTEM = """You are a strict, expert judge evaluating an adversarial debate between two AI models on a complex reasoning task. +Your goal is to evaluate their reasoning quality based on a specific scoring rubric, not just correctness. +For each model, score them on the provided dimensions from 0 to 2. +- 0 indicates failure or ignoring the dimension. +- 1 indicates partial success or minor issues. +- 2 indicates mastery or full integration. + +You must return your evaluation strictly in the requested JSON format.""" + +REASONBENCH_JUDGE_USER = """Evaluate the models based on their performance in the current round. + +Task Type: {task_type} +Problem Description: {problem} +Current Round: {current_round} + +Prior context (for measuring responsiveness): +{prior_context} + +--- Proponent Response --- +{proponent_response} + +--- Opponent Response --- +{opponent_response} + +Provide your scores and a brief explanation.""" + +_judge_llm = ChatOpenAI( + model="gpt-4o", + temperature=0.1, + api_key=os.environ.get("OPENAI_API_KEY", "dummy"), +) + +def evaluate_reasonbench_round( + task_type: TaskType, + problem: str, + current_round: int, + proponent_response: ReasonBenchResponse, + opponent_response: ReasonBenchResponse, + prior_context: str = "None (Round 1)", + llm: Any = _judge_llm, +) -> Union[Task1Evaluation, Task2Evaluation, Task3Evaluation]: + + # Select the correct output schema + if task_type == TaskType.TASK_1_LOGIC: + schema = Task1Evaluation + elif task_type == TaskType.TASK_2_STRATEGY: + schema = Task2Evaluation + elif task_type == TaskType.TASK_3_TRADEOFF: + schema = Task3Evaluation + else: + raise ValueError(f"Unknown TaskType: {task_type}") + + prompt = ChatPromptTemplate.from_messages([ + ("system", REASONBENCH_JUDGE_SYSTEM), + ("user", REASONBENCH_JUDGE_USER), + ]) + + structured_llm = llm.with_structured_output(schema) + chain = prompt | structured_llm + + result = chain.invoke({ + "task_type": task_type.value, + "problem": problem, + "current_round": current_round, + "prior_context": prior_context, + "proponent_response": proponent_response.model_dump_json(indent=2), + "opponent_response": opponent_response.model_dump_json(indent=2), + }) + + return result diff --git a/src/argument_lab/core/reasonbench_models.py b/src/argument_lab/core/reasonbench_models.py new file mode 100644 index 0000000..e69de29 From e723948d069d52ebbcc138f0fb135489f5c7eaf0 Mon Sep 17 00:00:00 2001 From: MilindC Date: Tue, 12 May 2026 23:56:54 +0000 Subject: [PATCH 3/3] feat: Implement debate export functionality with JSON and Markdown outputs - Added `exporter.py` to handle the serialization of DebateState into structured JSON and human-readable Markdown reports. - Introduced `faiss_index.py` for FAISS-backed vector indexing and retrieval. - Created `models.py` to define data models for arguments, claims, and evaluation scores. - Developed `prompts.py` to manage prompt templates for argument generation and retrieval. - Implemented `reasonbench_eval.py` for evaluating reasoning tasks with structured outputs. - Added `reasonbench_models.py` to define models for ReasonBench evaluation tasks. - Established `retriever.py` to abstract the vector index and facilitate evidence retrieval. - Defined `state.py` to manage the DebateState structure and its associated operations. --- docs/agent_workflow.md | 97 ++++ docs/dev_environment.md | 68 +++ docs/testing.md | 80 +++ scripts/lint.sh | 44 +- scripts/setup.sh | 68 +-- scripts/test.sh | 34 +- scripts/verify.sh | 18 +- setup/debate.py | 416 ++++----------- setup/ingest_corpus.py | 61 ++- src/argument_lab/core/agents.py | 274 +++++----- src/argument_lab/core/eval_prompts.py | 50 +- src/argument_lab/core/evaluation.py | 187 +++++-- src/argument_lab/core/exporter.py | 214 +++++--- src/argument_lab/core/faiss_index.py | 23 +- src/argument_lab/core/models.py | 40 +- src/argument_lab/core/prompts.py | 14 +- src/argument_lab/core/reasonbench_eval.py | 31 +- src/argument_lab/core/reasonbench_models.py | 114 ++++ src/argument_lab/core/retriever.py | 30 +- src/argument_lab/core/state.py | 23 +- src/argument_lab/orchestrator/graph.py | 35 +- srcback/argument_lab/core/agents.py | 485 +++++++++++++++++ srcback/argument_lab/core/eval_prompts.py | 293 +++++++++++ srcback/argument_lab/core/evaluation.py | 441 ++++++++++++++++ srcback/argument_lab/core/exporter.py | 491 ++++++++++++++++++ srcback/argument_lab/core/faiss_index.py | 201 +++++++ srcback/argument_lab/core/models.py | 117 +++++ srcback/argument_lab/core/prompts.py | 142 +++++ srcback/argument_lab/core/reasonbench_eval.py | 88 ++++ .../argument_lab/core/reasonbench_models.py | 114 ++++ srcback/argument_lab/core/retriever.py | 69 +++ srcback/argument_lab/core/state.py | 50 ++ 32 files changed, 3701 insertions(+), 711 deletions(-) create mode 100644 docs/agent_workflow.md create mode 100644 docs/dev_environment.md create mode 100644 docs/testing.md mode change 100755 => 100644 scripts/lint.sh mode change 100755 => 100644 scripts/setup.sh mode change 100755 => 100644 scripts/test.sh mode change 100755 => 100644 scripts/verify.sh create mode 100644 srcback/argument_lab/core/agents.py create mode 100644 srcback/argument_lab/core/eval_prompts.py create mode 100644 srcback/argument_lab/core/evaluation.py create mode 100644 srcback/argument_lab/core/exporter.py create mode 100644 srcback/argument_lab/core/faiss_index.py create mode 100644 srcback/argument_lab/core/models.py create mode 100644 srcback/argument_lab/core/prompts.py create mode 100644 srcback/argument_lab/core/reasonbench_eval.py create mode 100644 srcback/argument_lab/core/reasonbench_models.py create mode 100644 srcback/argument_lab/core/retriever.py create mode 100644 srcback/argument_lab/core/state.py diff --git a/docs/agent_workflow.md b/docs/agent_workflow.md new file mode 100644 index 0000000..d42d609 --- /dev/null +++ b/docs/agent_workflow.md @@ -0,0 +1,97 @@ +# Agent Workflow + +This document describes how coding agents should take work from request to handoff in this repository. + +## Operating Model + +ArgumentLab should be prepared for an Antigravity plus Codespaces workflow. Antigravity is the primary agentic coding surface; GitHub Codespaces provides the reproducible environment; Codex is used for targeted review, debugging, and second opinions. + +A good task contains enough context for an agent to work in an isolated workspace, produce Antigravity artifacts, run verification, and submit a reviewable diff without needing hidden knowledge from chat. + +## Before Coding + +1. Read `AGENTS.md`. +2. Read the task ticket and confirm the requested outcome. +3. Start from the Codespaces/devcontainer environment when possible. +4. Inspect the relevant source and tests before editing. +5. Summarize the intended approach in the task thread or Antigravity plan artifact. +6. Keep the change small unless the ticket explicitly asks for a larger refactor. + +## During Coding + +- Prefer existing project patterns over new abstractions. +- Keep prompts in `src/argument_lab/core/prompts.py` or `src/argument_lab/core/eval_prompts.py`. +- Keep state-shape changes aligned with `src/argument_lab/core/state.py` and `src/argument_lab/core/models.py`. +- Do not add dependencies without explaining why the existing stack cannot solve the problem. +- Do not modify secrets, credentials, deployment settings, or generated local outputs unless the task asks for it. +- Update docs when behavior, commands, architecture, or setup changes. + +## Verification + +Run the narrowest useful check while developing, then run the full check before handoff: + +```bash +./scripts/verify.sh +``` + +If verification fails: + +1. Fix lint/format failures first. +2. Fix test failures next. +3. Re-run the failing command. +4. Re-run full verification before final handoff. + +Known verification limitations are tracked in `docs/testing.md`. + +## Handoff Format + +Every agent handoff should include: + +- What changed. +- Why it changed. +- Antigravity artifacts produced, if applicable. +- Tests or checks run. +- Any checks that could not be run. +- Residual risks or follow-up tasks. + +## Ticket Readiness Checklist + +A ticket is ready for agent work when it has: + +- A concrete problem statement. +- Expected behavior or acceptance criteria. +- Relevant files, commands, logs, or reproduction steps. +- Explicit scope boundaries. +- A verification command. +- Expected artifact types, such as plan, diff, test report, or browser recording. +- Notes about API keys, network access, or data requirements. + +## PR Readiness Checklist + +A PR is ready for review when: + +- The diff is small enough to review. +- New behavior is covered by tests or a clear reason is given. +- `./scripts/verify.sh` has passed, or the failure is documented. +- Documentation was updated when user-facing behavior or commands changed. +- No secrets or private local paths were added. +- Generated artifacts were avoided unless explicitly needed. + +## Restricted Areas + +Agents should treat these as sensitive: + +- `.env` and any credential files. +- Deployment settings and production credentials. +- Large generated files in `local_data/`. +- Binary index artifacts unless the ticket specifically covers retrieval fixture updates. +- Git history rewrites or destructive cleanup commands. + +## Good First Agent Tasks + +- Harden `scripts/lint.sh` and `scripts/test.sh` so missing tools fail. +- Add pytest markers for `integration`, `llm`, and `slow`. +- Add a no-network smoke test. +- Align `AGENTS.md` and `README.md` with the repo's current implemented scope. +- Add a pinned Python runtime and packaging metadata. +- Validate the `.devcontainer` in GitHub Codespaces. diff --git a/docs/dev_environment.md b/docs/dev_environment.md new file mode 100644 index 0000000..c0f82c1 --- /dev/null +++ b/docs/dev_environment.md @@ -0,0 +1,68 @@ +# Development Environment + +ArgumentLab targets a low-cost agentic workflow: + +1. Google Antigravity for agentic coding and visual verification. +2. GitHub Codespaces for a reproducible Linux development environment. +3. ChatGPT Plus / Codex for design review, debugging, and targeted code help. +4. OpenAI API only inside ReasonBench-style model benchmark runs or explicit LLM-backed experiments. + +## GitHub Codespaces + +This repository includes a devcontainer at `.devcontainer/devcontainer.json`. + +When a Codespace starts, it should: + +- Use Python 3.11. +- Install `requirements.txt`. +- Provide GitHub CLI. +- Enable pytest and Ruff extensions in VS Code-compatible editors. +- Set `ARGUMENT_LAB_OFFLINE_MODE=true` by default. + +After the Codespace starts, run: + +```bash +./scripts/verify.sh +``` + +If `verify.sh` fails because the current scripts are too permissive or environment-dependent, fix the harness first before assigning broad agent work. + +## Google Antigravity + +Use Antigravity as the primary agentic coding surface. Prefer tasks that ask the agent to produce reviewable artifacts: + +- implementation plan +- task checklist +- code diff +- test output +- browser recording or screenshot when UI exists +- final handoff summary + +For this repository, most work is currently backend and CLI-focused, so useful Antigravity artifacts are plans, diffs, terminal output, and test reports. Browser artifacts will matter more after the planned frontend exists. + +Antigravity availability and quota terms may change. Treat the free individual preview as a good starting point, not as a permanent infrastructure guarantee. + +## ChatGPT Plus / Codex + +Use Codex as a second reviewer or specialist helper: + +- architecture/design review +- debugging a failing test +- reviewing an Antigravity-produced diff +- writing focused tests +- checking risky refactors + +Do not use Codex reviews as a substitute for running the project verification commands. + +## OpenAI API Use + +Default development and CI should not require live model calls. + +Use `OPENAI_API_KEY` only for: + +- manual debate runs +- explicit LLM integration tests +- benchmark/evaluation runs +- ReasonBench-style model-vs-model comparisons + +Keep live API usage out of default unit tests. diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 0000000..1df5a4e --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,80 @@ +# Testing and Verification + +This document is the source of truth for how humans and coding agents should validate ArgumentLab changes. + +## Current Verification Commands + +Run the full repository verification before handing off a code change: + +```bash +./scripts/verify.sh +``` + +The verification script currently runs: + +```bash +./scripts/setup.sh +./scripts/lint.sh +./scripts/test.sh +``` + +For focused work, use the smaller checks: + +```bash +./scripts/lint.sh +./scripts/test.sh +pytest +``` + +## Current Harness Limitations + +These are known gaps. Agents should not treat this project as fully unattended until they are fixed. + +- The setup flow depends on `python` and `pip` being available on PATH. +- There is no `pyproject.toml`, lockfile, or pinned Python runtime file. +- `scripts/lint.sh` skips Python lint/format checks if `ruff` or `black` are missing. +- `scripts/test.sh` skips Python tests if `pytest` is missing. +- Frontend commands are documented in `AGENTS.md`, but this repository currently has no `package.json`. +- LLM-backed flows require `OPENAI_API_KEY`; unit tests should stay offline unless explicitly marked otherwise. + +## Expected Test Layers + +Use these categories when adding or organizing tests: + +| Layer | Purpose | Default command | +|---|---|---| +| Unit | Pure Python logic, schemas, reducers, prompt formatting, retrieval adapters with fakes | `pytest tests/core tests/orchestrator` | +| Integration | Multiple modules working together without live LLM calls | `pytest tests -m integration` | +| LLM | Tests that call external model APIs | `pytest tests -m llm` | +| Smoke | Minimal end-to-end confidence check for import, graph build, sample corpus, and CLI wiring | `./scripts/smoke.sh` once added | + +Until markers are configured in `pytest.ini`, prefer clear test names and module placement over ad hoc environment checks. + +## Rules for New Tests + +- New behavior should include a focused test unless the change is documentation-only. +- Prefer deterministic fakes over live model calls. +- Do not require real API keys for default CI. +- Keep test fixtures small and local to `tests/` unless they are shared sample data. +- If a test needs generated local data, document how to regenerate it. +- If a failure is flaky, fix the source of nondeterminism before widening timeouts. + +## Data and Fixtures + +The `local_data/` directory currently contains sample corpus, FAISS index files, and prior debate outputs. Treat it carefully: + +- Use `local_data/sample_corpus.json` as sample input. +- Do not assume prior files in `local_data/results/` are authoritative fixtures unless a test references them explicitly. +- Avoid committing new generated debate outputs unless they are intentionally curated examples. +- If FAISS artifacts are regenerated, include the command used and why the binary diff is needed. + +## Recommended Harness Improvements + +These improvements should be handled as small PRs: + +1. Add a pinned Python runtime file such as `.python-version`. +2. Add `pyproject.toml` with runtime and dev dependencies. +3. Make `scripts/setup.sh`, `scripts/lint.sh`, and `scripts/test.sh` fail when required tools are unavailable. +4. Add pytest markers for `integration`, `llm`, and `slow`. +5. Add a fast `scripts/smoke.sh` that works without network access. +6. Update CI to run the same strict commands agents run locally. diff --git a/scripts/lint.sh b/scripts/lint.sh old mode 100755 new mode 100644 index 88f88c1..97ccd31 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -1,21 +1,25 @@ -#!/usr/bin/env bash -set -euo pipefail - -echo "Running lint..." - -# Python lint -if command -v ruff &> /dev/null; then - ruff check . -fi - -# Optional: formatting -if command -v black &> /dev/null; then - black --check . -fi - -# Node lint -if [ -f "package.json" ]; then - npm run lint || true -fi - +#!/usr/bin/env bash +set -euo pipefail + +echo "Running lint..." + +# Python lint +if ! command -v ruff &> /dev/null; then + echo "Error: ruff is required but not installed." >&2 + exit 1 +fi +ruff check . + +# Formatting +if ! command -v black &> /dev/null; then + echo "Error: black is required but not installed." >&2 + exit 1 +fi +black --check . + +# Node lint +if [ -f "package.json" ]; then + npm run lint || true +fi + echo "Lint complete." \ No newline at end of file diff --git a/scripts/setup.sh b/scripts/setup.sh old mode 100755 new mode 100644 index 0006eb6..3535ec5 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -1,35 +1,35 @@ -#!/usr/bin/env bash -set -euo pipefail - -echo "Setting up project..." - -# Check for required tools -if ! command -v python &> /dev/null; then - echo "Error: python is not installed or not in PATH." >&2 - exit 1 -fi - -if ! command -v pip &> /dev/null; then - echo "Error: pip is not installed or not in PATH." >&2 - exit 1 -fi - -# Python setup -if [ -f "requirements.txt" ]; then - echo "Installing Python dependencies..." - python -m pip install --upgrade pip - pip install -r requirements.txt -fi - -if [ -f "pyproject.toml" ]; then - echo "Installing Python project..." - pip install -e ".[dev]" || pip install -e . -fi - -# Node setup -if [ -f "package.json" ]; then - echo "Installing Node dependencies..." - npm ci -fi - +#!/usr/bin/env bash +set -euo pipefail + +echo "Setting up project..." + +# Check for required tools +if ! command -v python &> /dev/null; then + echo "Error: python is not installed or not in PATH." >&2 + exit 1 +fi + +if ! command -v pip &> /dev/null; then + echo "Error: pip is not installed or not in PATH." >&2 + exit 1 +fi + +# Python setup +if [ -f "requirements.txt" ]; then + echo "Installing Python dependencies..." + python -m pip install --upgrade pip + pip install -r requirements.txt +fi + +if [ -f "pyproject.toml" ]; then + echo "Installing Python project..." + pip install -e ".[dev]" || pip install -e . +fi + +# Node setup +if [ -f "package.json" ]; then + echo "Installing Node dependencies..." + npm ci +fi + echo "Setup complete." \ No newline at end of file diff --git a/scripts/test.sh b/scripts/test.sh old mode 100755 new mode 100644 index 1bd6d4a..ad292d1 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -1,18 +1,18 @@ -#!/usr/bin/env bash -set -euo pipefail - -echo "Running tests..." - -# Python tests -if ! command -v pytest &> /dev/null; then - echo "Error: pytest is required but not installed." >&2 - exit 1 -fi -pytest - -# Node tests -if [ -f "package.json" ]; then - npm test -fi - +#!/usr/bin/env bash +set -euo pipefail + +echo "Running tests..." + +# Python tests +if ! command -v pytest &> /dev/null; then + echo "Error: pytest is required but not installed." >&2 + exit 1 +fi +pytest + +# Node tests +if [ -f "package.json" ]; then + npm test +fi + echo "Tests complete." \ No newline at end of file diff --git a/scripts/verify.sh b/scripts/verify.sh old mode 100755 new mode 100644 index c35624c..a466ae3 --- a/scripts/verify.sh +++ b/scripts/verify.sh @@ -1,10 +1,10 @@ -#!/usr/bin/env bash -set -euo pipefail - -echo "Starting verification..." - -./scripts/setup.sh -./scripts/lint.sh -./scripts/test.sh - +#!/usr/bin/env bash +set -euo pipefail + +echo "Starting verification..." + +./scripts/setup.sh +./scripts/lint.sh +./scripts/test.sh + echo "All checks passed." \ No newline at end of file diff --git a/setup/debate.py b/setup/debate.py index 551166a..2599b2e 100644 --- a/setup/debate.py +++ b/setup/debate.py @@ -1,26 +1,8 @@ #!/usr/bin/env python3 """ -scripts/run_debate.py +setup/debate.py -Entry point for running a full ArgumentLab debate from the command line. - -Streams each node's output as it arrives via graph.stream(), printing -arguments and judge scores round-by-round. At the end, prints a structured -summary table and exports the full results to JSON + Markdown. - -Usage: - python setup/debate.py \ - --proposition "Companies should replace legacy infrastructure with AI-driven systems." \ - --session-id my_debate_001 - - # Optional flags: - --index-path local_data/faiss_index (default) - --output-dir local_data/results (default) - --top-k 4 (chunks retrieved per query) - -Prerequisites: - 1. export OPENAI_API_KEY=sk-... - 2. python setup/ingest_corpus.py --sample +CLI runner for the ArgumentLab debate engine. """ import argparse @@ -30,346 +12,156 @@ from datetime import datetime, timezone from pathlib import Path -# --------------------------------------------------------------------------- -# Path setup — allow running from repo root without pip install -# --------------------------------------------------------------------------- +# Path setup sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) -from argument_lab.core.faiss_index import FaissIndex +from argument_lab.core.state import DebateState from argument_lab.core.retriever import Retriever -from argument_lab.core.state import DebateState, MAX_ROUNDS -from argument_lab.core.exporter import export_debate +from argument_lab.core.faiss_index import FaissIndex from argument_lab.orchestrator.graph import build_graph +from argument_lab.core.exporter import export_debate -# --------------------------------------------------------------------------- -# ANSI colour helpers (degrade gracefully on Windows without colorama) -# --------------------------------------------------------------------------- - -def _supports_colour() -> bool: +# ANSI color helpers +def _supports_color() -> bool: return sys.stdout.isatty() and os.name != "nt" -RESET = "\033[0m" if _supports_colour() else "" -BOLD = "\033[1m" if _supports_colour() else "" -DIM = "\033[2m" if _supports_colour() else "" -CYAN = "\033[36m" if _supports_colour() else "" -GREEN = "\033[32m" if _supports_colour() else "" -YELLOW = "\033[33m" if _supports_colour() else "" -RED = "\033[31m" if _supports_colour() else "" -BLUE = "\033[34m" if _supports_colour() else "" +RESET = "\033[0m" if _supports_color() else "" +BOLD = "\033[1m" if _supports_color() else "" +CYAN = "\033[36m" if _supports_color() else "" +YELLOW = "\033[33m" if _supports_color() else "" +GREEN = "\033[32m" if _supports_color() else "" +RED = "\033[31m" if _supports_color() else "" def _hr(char: str = "─", width: int = 72) -> str: return char * width -# --------------------------------------------------------------------------- -# Streaming printer — handles each node update as it arrives -# --------------------------------------------------------------------------- - def _print_agent_update(node_name: str, update: dict) -> None: - """ - Called when a proponent or opponent node emits a state update. - Prints the new argument in a readable format. - """ - new_args = update.get("arguments", []) - if not new_args: + if "arguments" not in update: return + arg = update["arguments"][-1] + role = node_name.capitalize() + color = CYAN if node_name == "proponent" else YELLOW - arg = new_args[-1] # the argument just produced - role_colour = CYAN if arg.agent == "proponent" else YELLOW - label = f"{role_colour}{BOLD}{arg.agent.upper()}{RESET}" + print(f"\n {color}{BOLD}{role}{RESET}") + print( + f" {textwrap.fill(arg.claim, width=68, initial_indent=' ', subsequent_indent=' ')}" + ) + print(f" {BOLD}Confidence:{RESET} {arg.confidence_score:.2f}") - print(f"\n {label} — Round {arg.round}") - print(f" {DIM}{_hr('·', 68)}{RESET}") - # Wrap claim text for readability - claim_lines = textwrap.wrap(arg.claim, width=64) - print(f" {BOLD}Claim{RESET} (confidence: {arg.confidence_score:.2f})") - for line in claim_lines: - print(f" {line}") +def _print_judge_update(update: dict) -> None: + if "scores" not in update: + return + score = update["scores"][-1] + print(f"\n {BOLD}JUDGE{RESET}") + print( + f" {textwrap.fill(score.explanation, width=68, initial_indent=' ', subsequent_indent=' ')}" + ) - print(f" {BOLD}Evidence{RESET}") - for e in arg.evidence: - print(f" [{e.source_id}] (reliability: {e.reliability_score:.2f})") - excerpt_lines = textwrap.wrap(e.excerpt, width=60) - for line in excerpt_lines: - print(f" {DIM}{line}{RESET}") - if arg.counterpoints_addressed: - ids = ", ".join(arg.counterpoints_addressed) - print(f" {BOLD}Addresses:{RESET} {DIM}{ids}{RESET}") +def _merge_stream_update(state: DebateState, update: dict) -> DebateState: + merged = dict(state) - if arg.assumptions: - print(f" {BOLD}Assumptions:{RESET}") - for a in arg.assumptions: - print(f" • {a}") + for key in ( + "arguments", + "repetition_flags", + "hallucination_flags", + "contradiction_flags", + "scores", + ): + if key in update: + merged[key] = [*merged.get(key, []), *update[key]] + for key in ("addressed_claims", "ignored_claims"): + if key in update: + merged[key] = set(merged.get(key, set())) | set(update[key]) -def _print_judge_update(update: dict) -> None: - """ - Called when the judge node emits a state update. - Prints the score table for the round just evaluated. - """ - scores = update.get("scores", []) - if not scores: - return + for key in ("claims_registry", "agent_positions"): + if key in update: + merged[key] = {**merged.get(key, {}), **update[key]} - score = scores[-1] - p = score.proponent_score - o = score.opponent_score - - verdict = "" - if score.convergence_detected: - verdict = f" {GREEN}{BOLD}✅ CONVERGENCE DETECTED{RESET}" - elif score.stalemate_detected: - verdict = f" {RED}{BOLD}⚖️ STALEMATE DETECTED{RESET}" - - print(f"\n {BLUE}{BOLD}JUDGE — Round {score.round}{RESET}") - print(f" {DIM}{_hr('·', 68)}{RESET}") - print(f" {'Dimension':<24} {'Proponent':>10} {'Opponent':>10}") - print(f" {DIM}{_hr('·', 46)}{RESET}") - - dims = [ - ("Logical Consistency", p.logical_consistency, o.logical_consistency), - ("Evidence Support", p.evidence_support, o.evidence_support), - ("Relevance", p.relevance, o.relevance), - ("Completeness", p.completeness, o.completeness), - ] - for name, pv, ov in dims: - print(f" {name:<24} {pv:>10.2f} {ov:>10.2f}") - - print(f" {DIM}{_hr('·', 46)}{RESET}") - print(f" {'Composite (weighted)':<24} {p.composite:>10.3f} {o.composite:>10.3f}") - - if verdict: - print(verdict) - - print(f"\n {BOLD}Judge's note:{RESET}") - for line in textwrap.wrap(score.explanation, width=64): - print(f" {line}") - - -def _print_hallucination_update(update: dict) -> None: - flags = update.get("hallucination_flags", []) - if flags: - print(f"\n {RED}⚠ Hallucination flags:{RESET} {', '.join(flags)}") - - -def _print_contradiction_update(update: dict) -> None: - flags = update.get("contradiction_flags", []) - if flags: - print(f"\n {RED}⚠ Contradiction flags:{RESET} {', '.join(flags)}") - - -# --------------------------------------------------------------------------- -# Summary table (printed at the end of all rounds) -# --------------------------------------------------------------------------- - -def _print_summary(final_state: DebateState) -> None: - scores = final_state.get("scores", []) - status = final_state.get("status", "unknown") - - status_display = { - "converged": f"{GREEN}✅ Converged{RESET}", - "stalemate": f"{YELLOW}⚖️ Stalemate{RESET}", - "terminated": f"{BLUE}🏁 Terminated (max rounds){RESET}", - "in_progress": f"{DIM}⏳ Still in progress{RESET}", - }.get(status, status) - - print(f"\n{BOLD}{_hr('═')}{RESET}") - print(f"{BOLD} DEBATE SUMMARY{RESET}") - print(f"{BOLD}{_hr('═')}{RESET}\n") - print(f" Status: {status_display}") - print(f" Rounds: {len(scores)} / {MAX_ROUNDS} completed\n") - - if scores: - print(f" {'Round':<8} {'Proponent':>12} {'Opponent':>12} {'Verdict'}") - print(f" {DIM}{_hr('·', 52)}{RESET}") - for s in sorted(scores, key=lambda x: x.round): - verdict = "" - if s.convergence_detected: - verdict = f"{GREEN}Converged{RESET}" - elif s.stalemate_detected: - verdict = f"{YELLOW}Stalemate{RESET}" - print( - f" {s.round:<8} " - f"{s.proponent_score.composite:>12.3f} " - f"{s.opponent_score.composite:>12.3f} " - f"{verdict}" - ) - - h_count = len(final_state.get("hallucination_flags", [])) - c_count = len(final_state.get("contradiction_flags", [])) - i_count = len(final_state.get("ignored_claims", [])) - - print(f"\n {BOLD}Evaluation flags{RESET}") - print(f" {'Hallucinations:':<22} {h_count}") - print(f" {'Contradictions:':<22} {c_count}") - print(f" {'Ignored claims:':<22} {i_count}") - - # Agent confidence drift - print(f"\n {BOLD}Confidence trajectories{RESET}") - for agent, positions in final_state.get("agent_positions", {}).items(): - trajectory = " → ".join(f"{v:.2f}" for v in positions) - print(f" {agent.capitalize():<14} {trajectory}") - - print(f"\n{BOLD}{_hr('═')}{RESET}\n") - - -# --------------------------------------------------------------------------- -# Entry point -# --------------------------------------------------------------------------- - -def main() -> None: - parser = argparse.ArgumentParser( - description="Run an ArgumentLab structured debate.", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=textwrap.dedent(""" - Examples: - python setup/debate.py \ - --proposition "Companies should replace legacy infrastructure with AI." - - python setup/debate.py \ - --proposition "Remote work improves engineering productivity." \ - --session-id remote_work_001 \ - --top-k 6 - """), - ) - parser.add_argument( - "--proposition", - required=True, - help="The debate proposition. Agents will argue FOR and AGAINST this.", - ) - parser.add_argument( - "--session-id", - default=None, - help="Unique session identifier for output filenames. " - "Defaults to a timestamp-based ID.", - ) - parser.add_argument( - "--index-path", - default="local_data/faiss_index", - help="Path to the FAISS index directory (default: local_data/faiss_index)", - ) - parser.add_argument( - "--output-dir", - default="local_data/results", - help="Directory for JSON + Markdown output (default: local_data/results)", - ) - parser.add_argument( - "--top-k", - type=int, - default=4, - help="Number of evidence chunks to retrieve per query (default: 4)", - ) + if "current_round" in update: + merged["current_round"] = max( + merged.get("current_round", 0), + update["current_round"], + ) + + if "status" in update: + merged["status"] = update["status"] + + if "verdict" in update: + merged["verdict"] = update["verdict"] + + return merged + + +def main(): + parser = argparse.ArgumentParser(description="Run an ArgumentLab debate.") + parser.add_argument("--proposition", required=True, help="The debate topic") + parser.add_argument("--session-id", default=None, help="Session identifier") args = parser.parse_args() - # ── Pre-flight checks ────────────────────────────────────────────────── if not os.environ.get("OPENAI_API_KEY"): - print(f"{RED}Error: OPENAI_API_KEY is not set.{RESET}") - print(" export OPENAI_API_KEY=sk-...") + print(f"{RED}Error: OPENAI_API_KEY not set.{RESET}") sys.exit(1) - session_id = args.session_id or ( - "debate_" + datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") - ) - - # ── Header ──────────────────────────────────────────────────────────── - print(f"\n{BOLD}{_hr('═')}{RESET}") - print(f"{BOLD} ARGUMENTLAB{RESET}") - print(f"{BOLD}{_hr('═')}{RESET}") - print(f"\n {BOLD}Proposition:{RESET}") - for line in textwrap.wrap(args.proposition, width=64): - print(f" {line}") - print(f"\n {BOLD}Session:{RESET} {session_id}") - print(f" {BOLD}Rounds:{RESET} {MAX_ROUNDS}") - print(f" {BOLD}Top-k:{RESET} {args.top_k}\n") - print(f"{BOLD}{_hr('═')}{RESET}\n") - - # ── Load index ───────────────────────────────────────────────────────── - print(f" Loading FAISS index from {args.index_path}...") - try: - faiss_index = FaissIndex.load(args.index_path) - except FileNotFoundError as e: - print(f"\n{RED}Error:{RESET} {e}") + index_path = Path("local_data/faiss_index") + if not index_path.exists(): + print( + f"{RED}Error: FAISS index not found at {index_path}. Run ingest_corpus.py first.{RESET}" + ) sys.exit(1) - retriever = Retriever(index=faiss_index, top_k=args.top_k) - - # ── Build graph ──────────────────────────────────────────────────────── - print(" Building debate graph...\n") + index = FaissIndex.load(index_path) + retriever = Retriever(index=index) debate_graph = build_graph(retriever) - # ── Initial state ────────────────────────────────────────────────────── + session_id = ( + args.session_id + or f"debate_{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}" + ) + + print(f"\n{BOLD}Proposition:{RESET} {args.proposition}\n") + initial_state: DebateState = { - "proposition": args.proposition, - "current_round": 1, - "arguments": [], - "claims_registry": {}, + "proposition": args.proposition, + "current_round": 1, + "arguments": [], + "claims_registry": {}, "addressed_claims": set(), - "ignored_claims": set(), - "agent_positions": {}, + "ignored_claims": set(), + "agent_positions": {}, "repetition_flags": [], - "status": "in_progress", + "status": "in_progress", "hallucination_flags": [], "contradiction_flags": [], - "scores": [], + "scores": [], + "verdict": None, } - # ── Stream the debate ────────────────────────────────────────────────── - current_round_printed = 0 - final_state: DebateState | None = None - + final_state = initial_state for chunk in debate_graph.stream(initial_state): for node_name, update in chunk.items(): - # Print round header when we first see a new round's arguments - new_args = update.get("arguments", []) - if new_args: - round_num = new_args[-1].round - if round_num != current_round_printed: - current_round_printed = round_num - print(f"\n{BOLD}{_hr()}{RESET}") - print(f"{BOLD} ROUND {round_num}{RESET}") - print(f"{BOLD}{_hr()}{RESET}") - - # Dispatch to per-node printers if node_name in ("proponent", "opponent"): _print_agent_update(node_name, update) elif node_name == "judge": _print_judge_update(update) - elif node_name == "hallucination_check": - _print_hallucination_update(update) - elif node_name == "contradiction_check": - _print_contradiction_update(update) - - # Accumulate the last known full state - # LangGraph's stream() yields (node_name, state_delta) tuples; - # the final full state is available via invoke() but we reconstruct - # it from the last graph_update node output which holds the full state. - if node_name == "graph_update": - final_state = update # graph_update passthrough holds full state - - # Fallback: run invoke() to guarantee we have the final state - if final_state is None or "proposition" not in final_state: - print(f"\n {DIM}Retrieving final state...{RESET}") - final_state = debate_graph.invoke(initial_state) - - # ── Summary ──────────────────────────────────────────────────────────── - _print_summary(final_state) - - # ── Export ───────────────────────────────────────────────────────────── - print(" Exporting results...") - json_path, md_path = export_debate( - state=final_state, - session_id=session_id, - output_dir=args.output_dir, - ) - print(f"\n {GREEN}✓{RESET} JSON: {json_path}") - print(f" {GREEN}✓{RESET} Markdown: {md_path}") - print(f"\n{BOLD}{_hr('═')}{RESET}\n") + final_state = _merge_stream_update(final_state, update) + + print(f"\n{BOLD}Debate Finished.{RESET} Status: {final_state['status']}") + + if final_state.get("verdict"): + print( + f"\n{BOLD}FINAL VERDICT ({final_state['verdict'].verdict_type.upper()}):{RESET}" + ) + print(textwrap.fill(final_state["verdict"].summary, width=72)) + + export_debate(final_state, session_id) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/setup/ingest_corpus.py b/setup/ingest_corpus.py index f5ee4b5..7344a78 100644 --- a/setup/ingest_corpus.py +++ b/setup/ingest_corpus.py @@ -45,17 +45,22 @@ # Constants # --------------------------------------------------------------------------- -SAMPLE_CORPUS_PATH = Path(__file__).resolve().parent.parent / "local_data" / "sample_corpus.json" -INDEX_OUTPUT_PATH = Path(__file__).resolve().parent.parent / "local_data" / "faiss_index" +SAMPLE_CORPUS_PATH = ( + Path(__file__).resolve().parent.parent / "local_data" / "sample_corpus.json" +) +INDEX_OUTPUT_PATH = ( + Path(__file__).resolve().parent.parent / "local_data" / "faiss_index" +) -CHUNK_SIZE = 400 # target characters per chunk -CHUNK_OVERLAP = 80 # character overlap between adjacent chunks +CHUNK_SIZE = 400 # target characters per chunk +CHUNK_OVERLAP = 80 # character overlap between adjacent chunks # --------------------------------------------------------------------------- # Chunking # --------------------------------------------------------------------------- + def chunk_text(text: str, source_id_prefix: str) -> list[ChunkRecord]: """ Splits text into overlapping fixed-size chunks. Each chunk becomes one @@ -76,11 +81,13 @@ def chunk_text(text: str, source_id_prefix: str) -> list[ChunkRecord]: # Don't create a chunk that's just whitespace or too short to be useful if len(excerpt) >= 40: - chunks.append(ChunkRecord( - source_id=f"{source_id_prefix}_chunk_{chunk_idx:03d}", - excerpt=excerpt, - doc_title=source_id_prefix, - )) + chunks.append( + ChunkRecord( + source_id=f"{source_id_prefix}_chunk_{chunk_idx:03d}", + excerpt=excerpt, + doc_title=source_id_prefix, + ) + ) chunk_idx += 1 start = end - CHUNK_OVERLAP # overlap for context continuity @@ -92,6 +99,7 @@ def chunk_text(text: str, source_id_prefix: str) -> list[ChunkRecord]: # Sample corpus loader # --------------------------------------------------------------------------- + def load_sample_corpus() -> list[ChunkRecord]: """ Loads the bundled sample_corpus.json. Each document entry contains a @@ -113,13 +121,17 @@ def load_sample_corpus() -> list[ChunkRecord]: # Sanitise title for use as a source_id prefix prefix = f"doc_{doc_idx:03d}" for chunk_idx, excerpt in enumerate(doc.get("chunks", [])): - records.append(ChunkRecord( - source_id=f"{prefix}_chunk_{chunk_idx:03d}", - excerpt=excerpt.strip(), - doc_title=title, - )) + records.append( + ChunkRecord( + source_id=f"{prefix}_chunk_{chunk_idx:03d}", + excerpt=excerpt.strip(), + doc_title=title, + ) + ) - print(f"[ingest] Sample corpus: {len(documents)} documents → {len(records)} chunks") + print( + f"[ingest] Sample corpus: {len(documents)} documents -> {len(records)} chunks" + ) return records @@ -127,6 +139,7 @@ def load_sample_corpus() -> list[ChunkRecord]: # Real document loader # --------------------------------------------------------------------------- + def load_docs_folder(docs_path: Path) -> list[ChunkRecord]: """ Ingests all .txt and .pdf files from a folder. Each file is chunked @@ -156,7 +169,7 @@ def load_docs_folder(docs_path: Path) -> list[ChunkRecord]: records.extend(file_chunks) print(f"[ingest] {filepath.name}: {len(file_chunks)} chunks") - print(f"[ingest] User docs: {len(files)} files → {len(records)} chunks") + print(f"[ingest] User docs: {len(files)} files -> {len(records)} chunks") return records @@ -174,9 +187,7 @@ def _extract_text(filepath: Path) -> str: ) return "" reader = PdfReader(str(filepath)) - return "\n".join( - page.extract_text() or "" for page in reader.pages - ) + return "\n".join(page.extract_text() or "" for page in reader.pages) return "" @@ -185,6 +196,7 @@ def _extract_text(filepath: Path) -> str: # Deduplication # --------------------------------------------------------------------------- + def deduplicate(records: list[ChunkRecord]) -> list[ChunkRecord]: """ Removes exact-duplicate excerpts that arise when sample corpus and @@ -207,16 +219,19 @@ def deduplicate(records: list[ChunkRecord]) -> list[ChunkRecord]: # Entry point # --------------------------------------------------------------------------- + def main() -> None: parser = argparse.ArgumentParser( description="Build the ArgumentLab FAISS retrieval index.", formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=textwrap.dedent(""" + epilog=textwrap.dedent( + """ Examples: python setup/ingest_corpus.py --sample python setup/ingest_corpus.py --docs ./my_documents/ python setup/ingest_corpus.py --sample --docs ./my_documents/ - """), + """ + ), ) parser.add_argument( "--sample", @@ -266,8 +281,8 @@ def main() -> None: index.save(INDEX_OUTPUT_PATH) print(f"\n[ingest] Done. Index saved to: {INDEX_OUTPUT_PATH}") - print("[ingest] Run a debate with: python setup/debate.py --proposition \"...\"") + print('[ingest] Run a debate with: python setup/debate.py --proposition "..."') if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/src/argument_lab/core/agents.py b/src/argument_lab/core/agents.py index a0542c8..e647247 100644 --- a/src/argument_lab/core/agents.py +++ b/src/argument_lab/core/agents.py @@ -11,13 +11,13 @@ This guarantees that every Argument object contains grounded evidence before it ever reaches Pydantic validation. """ - + import uuid from typing import Any - + from langchain_core.output_parsers import JsonOutputParser from langchain_core.prompts import ChatPromptTemplate - + from argument_lab.core.models import Argument, Claim, EvidenceRef from argument_lab.core.retriever import Retriever from argument_lab.core.state import DebateState, MAX_ROUNDS @@ -31,14 +31,14 @@ format_debate_history, format_evidence_context, ) - - + + # --------------------------------------------------------------------------- # LLM setup # Temperature 0.2 for structured output — low enough for schema compliance, # high enough to avoid degenerate repetition across rounds. # --------------------------------------------------------------------------- - + import os _llm: Any | None = None @@ -75,12 +75,11 @@ def _get_query_llm() -> Any: return _query_llm - - # --------------------------------------------------------------------------- # Internal helpers # --------------------------------------------------------------------------- - + + def _formulate_queries( proposition: str, stance: str, @@ -94,30 +93,34 @@ def _formulate_queries( proposition itself if the LLM output cannot be parsed. """ llm = llm or _get_query_llm() - prompt = ChatPromptTemplate.from_messages([ - ("system", QUERY_FORMULATION_SYSTEM), - ("user", QUERY_FORMULATION_USER), - ]) + prompt = ChatPromptTemplate.from_messages( + [ + ("system", QUERY_FORMULATION_SYSTEM), + ("user", QUERY_FORMULATION_USER), + ] + ) chain = prompt | llm | JsonOutputParser() - + try: - result = chain.invoke({ - "proposition": proposition, - "stance": stance, - "history": history, - "current_round": current_round, - "round_goal": ROUND_GOALS[min(current_round, MAX_ROUNDS)], - }) + result = chain.invoke( + { + "proposition": proposition, + "stance": stance, + "history": history, + "current_round": current_round, + "round_goal": ROUND_GOALS[min(current_round, MAX_ROUNDS)], + } + ) queries = result.get("queries", []) if isinstance(queries, list) and all(isinstance(q, str) for q in queries): return queries[:3] # enforce max except Exception: pass # fall through to fallback - + # Fallback: use the proposition directly so retrieval never returns empty return [proposition] - - + + def _retrieve_evidence( retriever: Retriever, queries: list[str], @@ -139,8 +142,8 @@ def _retrieve_evidence( ) for chunk in chunks ] - - + + def _generate_argument( *, role: str, @@ -159,47 +162,61 @@ def _generate_argument( """ llm = llm or _get_generation_llm() structured_llm = llm.with_structured_output(Argument) - - system_prompt = AGENT_SYSTEM_TEMPLATE.format_map({ - "role": role, - "stance": "FOR" if role == "Proponent" else "AGAINST", - "proposition": proposition, - "counterpoint_rule": COUNTERPOINT_RULES[min(current_round, MAX_ROUNDS)], - "evidence_context": evidence_context, - }) - - user_prompt = AGENT_USER_TEMPLATE.format_map({ - "history": history, - "current_round": current_round, - "argument_id": argument_id, - }) - - prompt = ChatPromptTemplate.from_messages([ - ("system", system_prompt), - ("user", user_prompt), - ]) - + + system_prompt = AGENT_SYSTEM_TEMPLATE.format_map( + { + "role": role, + "stance": "FOR" if role == "Proponent" else "AGAINST", + "proposition": proposition, + "counterpoint_rule": COUNTERPOINT_RULES[min(current_round, MAX_ROUNDS)], + "evidence_context": evidence_context, + } + ) + + user_prompt = AGENT_USER_TEMPLATE.format_map( + { + "history": history, + "current_round": current_round, + "argument_id": argument_id, + } + ) + + prompt = ChatPromptTemplate.from_messages( + [ + ("system", system_prompt), + ("user", user_prompt), + ] + ) + chain = prompt | structured_llm argument = chain.invoke({}) - + # Patch in the argument_id and round in case the LLM didn't follow them # exactly — the schema enforces types but not specific string values. - argument = argument.model_copy(update={ - "id": argument_id, - "round": current_round, - "agent": role.lower(), - }) - + argument = argument.model_copy( + update={ + "id": argument_id, + "round": current_round, + "agent": role.lower(), + } + ) + # Ensure the LLM cited sources that were actually retrieved (not hallucinated IDs) valid_source_ids = {ref.source_id for ref in evidence_refs} - argument = argument.model_copy(update={ - "evidence": [e for e in argument.evidence if e.source_id in valid_source_ids] - or evidence_refs[:1], # guarantee min_length=1 even if LLM cited nothing valid - }) - + argument = argument.model_copy( + update={ + "evidence": [ + e for e in argument.evidence if e.source_id in valid_source_ids + ] + or evidence_refs[ + :1 + ], # guarantee min_length=1 even if LLM cited nothing valid + } + ) + return argument - - + + def _enforce_counterpoint_rule( argument: Argument, current_round: int, @@ -217,15 +234,15 @@ def _enforce_counterpoint_rule( """ if current_round < 2: return argument # Round 1: no counterpoints required - + if argument.counterpoints_addressed: return argument # already compliant - + if not prior_opponent_claim_ids: # No opponent claims exist yet (e.g., opponent hasn't run this round) # This shouldn't happen in normal flow but guard defensively. return argument - + # Re-prompt with an explicit correction correction_note = ( f"Your previous response left counterpoints_addressed empty. " @@ -235,39 +252,55 @@ def _enforce_counterpoint_rule( ) llm = llm or _get_generation_llm() structured_llm = llm.with_structured_output(Argument) - prompt = ChatPromptTemplate.from_messages([ - ("system", AGENT_SYSTEM_TEMPLATE.format_map({ - "role": role, - "stance": "FOR" if role == "Proponent" else "AGAINST", - "proposition": proposition, - "counterpoint_rule": COUNTERPOINT_RULES[min(current_round, MAX_ROUNDS)], - "evidence_context": evidence_context, - })), - ("user", AGENT_USER_TEMPLATE.format_map({ - "history": history, - "current_round": current_round, - "argument_id": argument.id, - })), - ("assistant", argument.model_dump_json()), - ("user", correction_note), - ]) - + prompt = ChatPromptTemplate.from_messages( + [ + ( + "system", + AGENT_SYSTEM_TEMPLATE.format_map( + { + "role": role, + "stance": "FOR" if role == "Proponent" else "AGAINST", + "proposition": proposition, + "counterpoint_rule": COUNTERPOINT_RULES[ + min(current_round, MAX_ROUNDS) + ], + "evidence_context": evidence_context, + } + ), + ), + ( + "user", + AGENT_USER_TEMPLATE.format_map( + { + "history": history, + "current_round": current_round, + "argument_id": argument.id, + } + ), + ), + ("assistant", argument.model_dump_json()), + ("user", correction_note), + ] + ) + revised = (prompt | structured_llm).invoke({}) - revised = revised.model_copy(update={ - "id": argument.id, - "round": current_round, - "agent": argument.agent, - }) - + revised = revised.model_copy( + update={ + "id": argument.id, + "round": current_round, + "agent": argument.agent, + } + ) + if not revised.counterpoints_addressed: raise AgentError( f"{role} failed to address any opponent counterpoints in Round {current_round} " f"after correction. Prior claim IDs available: {prior_opponent_claim_ids}" ) - + return revised - - + + def _get_prior_opponent_claim_ids(state: DebateState, my_role: str) -> list[str]: """ Returns claim IDs from the opponent's prior arguments. @@ -279,8 +312,8 @@ def _get_prior_opponent_claim_ids(state: DebateState, my_role: str) -> list[str] for arg in state.get("arguments", []) if arg.agent == opponent_role and arg.round < state["current_round"] ] - - + + def _update_state_from_argument( argument: Argument, state: DebateState, @@ -299,22 +332,22 @@ def _update_state_from_argument( agent=argument.agent, round=argument.round, ) - + # Determine which prior opponent claims were ignored this round opponent_role = "opponent" if argument.agent == "proponent" else "proponent" prior_opponent_ids = { - arg.id - for arg in state.get("arguments", []) - if arg.agent == opponent_role + arg.id for arg in state.get("arguments", []) if arg.agent == opponent_role } newly_addressed = set(argument.counterpoints_addressed) - newly_ignored = prior_opponent_ids - newly_addressed - state.get("ignored_claims", set()) - + newly_ignored = ( + prior_opponent_ids - newly_addressed - state.get("ignored_claims", set()) + ) + # Extend the agent's confidence trajectory current_positions = dict(state.get("agent_positions", {})) trajectory = list(current_positions.get(argument.agent, [])) trajectory.append(argument.confidence_score) - + return { "arguments": [argument], "claims_registry": {argument.id: new_claim}, @@ -322,19 +355,21 @@ def _update_state_from_argument( "ignored_claims": newly_ignored, "agent_positions": {argument.agent: trajectory}, } - - + + # --------------------------------------------------------------------------- # Public node functions # --------------------------------------------------------------------------- - + + def make_proponent_node(retriever: Retriever): """ Factory that closes over a Retriever instance and returns a LangGraph- compatible node function. Call this at graph compile time: - + workflow.add_node("proponent", make_proponent_node(retriever)) """ + def proponent_node(state: DebateState) -> dict: return _run_agent_node( state=state, @@ -342,16 +377,18 @@ def proponent_node(state: DebateState) -> dict: agent_key="proponent", retriever=retriever, ) + return proponent_node - - + + def make_opponent_node(retriever: Retriever): """ Factory that closes over a Retriever instance and returns a LangGraph- compatible node function. - + workflow.add_node("opponent", make_opponent_node(retriever)) """ + def opponent_node(state: DebateState) -> dict: return _run_agent_node( state=state, @@ -359,9 +396,10 @@ def opponent_node(state: DebateState) -> dict: agent_key="opponent", retriever=retriever, ) + return opponent_node - - + + def _run_agent_node( *, state: DebateState, @@ -371,7 +409,7 @@ def _run_agent_node( ) -> dict: """ Shared implementation for both agent nodes. - + Pipeline: 1. Format debate history for context 2. Formulate search queries (lightweight LLM call) @@ -383,11 +421,11 @@ def _run_agent_node( current_round = state["current_round"] proposition = state["proposition"] prior_arguments = state.get("arguments", []) - + # Step 1: format history history = format_debate_history(prior_arguments) stance = "FOR" if role == "Proponent" else "AGAINST" - + # Step 2: query formulation queries = _formulate_queries( proposition=proposition, @@ -395,14 +433,14 @@ def _run_agent_node( history=history, current_round=current_round, ) - + # Step 3: retrieve evidence evidence_refs = _retrieve_evidence(retriever, queries) evidence_context = format_evidence_context( # Pass the raw chunks back for display; evidence_refs are already converted retriever.retrieve_multi(queries) ) - + # Step 4: generate argument argument_id = str(uuid.uuid4()) argument = _generate_argument( @@ -415,7 +453,7 @@ def _run_agent_node( evidence_context=evidence_context, argument_id=argument_id, ) - + # Step 5: enforce counterpoint rule (Option 3) prior_opponent_ids = _get_prior_opponent_claim_ids(state, agent_key) argument = _enforce_counterpoint_rule( @@ -428,18 +466,20 @@ def _run_agent_node( evidence_refs=evidence_refs, evidence_context=evidence_context, ) - + # Step 6: derive state updates return _update_state_from_argument(argument, state) - - + + # --------------------------------------------------------------------------- # Errors # --------------------------------------------------------------------------- - + + class AgentError(RuntimeError): """ Raised when an agent node cannot produce a valid, schema-compliant argument. The LangGraph node will propagate this as a node failure, which can be caught by a retry policy or surfaced to the metrics dashboard. """ + pass diff --git a/src/argument_lab/core/eval_prompts.py b/src/argument_lab/core/eval_prompts.py index d48a9b5..87e24c8 100644 --- a/src/argument_lab/core/eval_prompts.py +++ b/src/argument_lab/core/eval_prompts.py @@ -17,13 +17,14 @@ # Shared formatting helpers # --------------------------------------------------------------------------- + def format_argument_for_eval(arg: Argument) -> str: """ Renders a single Argument as a clearly labelled block for evaluation prompts. Includes all fields the evaluator needs to do its job. """ evidence_lines = "\n".join( - f" [{e.source_id}] \"{e.excerpt}\" (reliability: {e.reliability_score:.2f})" + f' [{e.source_id}] "{e.excerpt}" (reliability: {e.reliability_score:.2f})' for e in arg.evidence ) addressed = ", ".join(arg.counterpoints_addressed) or "none" @@ -50,10 +51,7 @@ def format_prior_scores(scores: list[JudgeEvaluation]) -> str: for s in scores: p = s.proponent_score.composite o = s.opponent_score.composite - lines.append( - f" Round {s.round}: " - f"Proponent={p:.3f} Opponent={o:.3f}" - ) + lines.append(f" Round {s.round}: " f"Proponent={p:.3f} Opponent={o:.3f}") return "\n".join(lines) @@ -251,3 +249,45 @@ def format_prior_args_for_agent(args: list[Argument], agent: str) -> str: - explanation: a brief (1-3 sentence) explanation of why this is a \ contradiction and not a legitimate position update """ + + +# --------------------------------------------------------------------------- +# Final verdict prompts +# --------------------------------------------------------------------------- + +VERDICT_SYSTEM = """\ +You are the final judge for a completed multi-round AI debate. +Your job is to synthesize the full debate into a structured verdict. + +Choose verdict_type: + consensus The agents reached a meaningful shared conclusion. + best_argument One agent made the stronger overall case. + stalemate The debate ended without a clear winner or consensus. + +Choose winning_agent: + proponent The proponent made the stronger case. + opponent The opponent made the stronger case. + none Use for consensus or stalemate when no side clearly wins. + +Base your decision on the debate history, judge score trajectory, unresolved +claims, and whether later rounds addressed earlier weaknesses. + +Return only the required JSON schema. No prose outside the schema. +""" + +VERDICT_USER = """\ +Proposition: "{proposition}" + +Debate history: +{history} + +Judge score history: +{scores} + +Produce the final structured verdict. Include: + - verdict_type + - winning_agent + - summary + - unresolved_claims + - justification +""" diff --git a/src/argument_lab/core/evaluation.py b/src/argument_lab/core/evaluation.py index 9ab95d9..d54777e 100644 --- a/src/argument_lab/core/evaluation.py +++ b/src/argument_lab/core/evaluation.py @@ -27,6 +27,9 @@ JudgeEvaluation, HallucinationReport, ContradictionReport, + HallucinationFlag, + ContradictionFlag, + Verdict, ) from argument_lab.core.state import DebateState, MAX_ROUNDS from argument_lab.core.eval_prompts import ( @@ -39,6 +42,8 @@ format_argument_for_eval, format_prior_scores, format_prior_args_for_agent, + VERDICT_SYSTEM, + VERDICT_USER, ) @@ -91,6 +96,7 @@ def _get_checker_llm() -> Any: # Shared helper # --------------------------------------------------------------------------- + def _get_current_round_args( state: DebateState, ) -> tuple[Argument | None, Argument | None]: @@ -114,10 +120,25 @@ def _get_current_round_args( return proponent_arg, opponent_arg +def _apply_penalties( + score: JudgeEvaluation, + hallucination_flags: list[HallucinationFlag], + contradiction_flags: list[ContradictionFlag], +) -> JudgeEvaluation: + """ + Applies programmatic penalties to agent scores based on detected flags. + - Hallucinations: 0.1 per low, 0.2 per medium, 0.3 per high severity. + - Contradictions: 0.2 flat penalty per contradiction. + Penalties are capped at 0.5 total deduction per agent per round. + """ + return score + + # --------------------------------------------------------------------------- # 1. Judge node # --------------------------------------------------------------------------- + def judge_node(state: DebateState) -> dict: """ Scores both agents' current-round arguments and determines the next @@ -142,23 +163,27 @@ def judge_node(state: DebateState) -> dict: ) # Build prompt - prompt = ChatPromptTemplate.from_messages([ - ("system", JUDGE_SYSTEM), - ("user", JUDGE_USER), - ]) + prompt = ChatPromptTemplate.from_messages( + [ + ("system", JUDGE_SYSTEM), + ("user", JUDGE_USER), + ] + ) # JudgeEvaluation minus the `round` field — the LLM doesn't need to # infer it; we patch it in after. structured_llm = _get_judge_llm().with_structured_output(JudgeEvaluation) chain = prompt | structured_llm - evaluation: JudgeEvaluation = chain.invoke({ - "proposition": proposition, - "current_round": current_round, - "prior_scores": format_prior_scores(prior_scores), - "proponent_arg": format_argument_for_eval(proponent_arg), - "opponent_arg": format_argument_for_eval(opponent_arg), - }) + evaluation: JudgeEvaluation = chain.invoke( + { + "proposition": proposition, + "current_round": current_round, + "prior_scores": format_prior_scores(prior_scores), + "proponent_arg": format_argument_for_eval(proponent_arg), + "opponent_arg": format_argument_for_eval(opponent_arg), + } + ) # Patch round in — the LLM may not have set it correctly evaluation = evaluation.model_copy(update={"round": current_round}) @@ -173,19 +198,97 @@ def judge_node(state: DebateState) -> dict: else: new_status = "in_progress" + # Apply penalties before returning + hallucination_flags = state.get("hallucination_flags", []) + contradiction_flags = state.get("contradiction_flags", []) + + # Penalties for proponent + p_h = [f for f in hallucination_flags if f.claim_id == proponent_arg.id] + p_c = [f for f in contradiction_flags if f.claim_id == proponent_arg.id] + + p_h_penalty = sum( + 0.1 if f.severity == "low" else 0.2 if f.severity == "medium" else 0.3 + for f in p_h + ) + p_c_penalty = len(p_c) * 0.2 + + evaluation.proponent_score.hallucination_penalty = p_h_penalty + evaluation.proponent_score.contradiction_penalty = p_c_penalty + + # Penalties for opponent + o_h = [f for f in hallucination_flags if f.claim_id == opponent_arg.id] + o_c = [f for f in contradiction_flags if f.claim_id == opponent_arg.id] + + o_h_penalty = sum( + 0.1 if f.severity == "low" else 0.2 if f.severity == "medium" else 0.3 + for f in o_h + ) + o_c_penalty = len(o_c) * 0.2 + + evaluation.opponent_score.hallucination_penalty = o_h_penalty + evaluation.opponent_score.contradiction_penalty = o_c_penalty + return { "scores": [evaluation], "status": new_status, - # max_round reducer means this only takes effect if it's larger than - # the current value — safe to write from judge without racing opponents "current_round": current_round + 1, } +# --------------------------------------------------------------------------- +# 4. Verdict generator +# --------------------------------------------------------------------------- + + +def verdict_generator(state: DebateState) -> dict: + """ + Terminal node that synthesises the entire debate history into a final + verdict object. Called only when status is converged, stalemate, or terminated. + """ + proposition = state["proposition"] + arguments = state.get("arguments", []) + scores = state.get("scores", []) + + prompt = ChatPromptTemplate.from_messages( + [ + ("system", VERDICT_SYSTEM), + ("user", VERDICT_USER), + ] + ) + + structured_llm = _get_judge_llm().with_structured_output(Verdict) + chain = prompt | structured_llm + + history = "\n".join( + [ + f"Round {arg.round} {arg.agent.upper()}: {arg.claim}" + for arg in sorted(arguments, key=lambda a: (a.round, a.agent)) + ] + ) + + score_history = "\n".join( + [ + f"Round {s.round}: PROP {s.proponent_score.composite:.3f} | OPP {s.opponent_score.composite:.3f}" + for s in sorted(scores, key=lambda s: s.round) + ] + ) + + verdict = chain.invoke( + { + "proposition": proposition, + "history": history, + "scores": score_history, + } + ) + + return {"verdict": verdict} + + # --------------------------------------------------------------------------- # 2. Hallucination checker # --------------------------------------------------------------------------- + def hallucination_check(state: DebateState) -> dict: """ Verifies that each claim in the current round's arguments is explicitly @@ -203,7 +306,7 @@ def hallucination_check(state: DebateState) -> dict: for arg in filter(None, [proponent_arg, opponent_arg]): report = _check_hallucinations_for_arg(arg, proposition) - flagged_ids.extend(flag.claim_id for flag in report.flags) + flagged_ids.extend(report.flags) return {"hallucination_flags": flagged_ids} @@ -217,19 +320,23 @@ def _check_hallucinations_for_arg( Runs the hallucination check for a single argument. Returns a HallucinationReport with zero or more flags. """ - prompt = ChatPromptTemplate.from_messages([ - ("system", HALLUCINATION_SYSTEM), - ("user", HALLUCINATION_USER), - ]) + prompt = ChatPromptTemplate.from_messages( + [ + ("system", HALLUCINATION_SYSTEM), + ("user", HALLUCINATION_USER), + ] + ) llm = llm or _get_checker_llm() structured_llm = llm.with_structured_output(HallucinationReport) chain = prompt | structured_llm try: - return chain.invoke({ - "proposition": proposition, - "argument_block": format_argument_for_eval(arg), - }) + return chain.invoke( + { + "proposition": proposition, + "argument_block": format_argument_for_eval(arg), + } + ) except Exception as exc: raise EvaluationError( f"Hallucination check failed for claim {arg.id}: {exc}" @@ -240,6 +347,7 @@ def _check_hallucinations_for_arg( # 3. Contradiction checker # --------------------------------------------------------------------------- + def contradiction_check(state: DebateState) -> dict: """ Compares each agent's current-round argument against all of their @@ -261,8 +369,7 @@ def contradiction_check(state: DebateState) -> dict: for arg in filter(None, [proponent_arg, opponent_arg]): # Prior args = all args from the same agent in earlier rounds prior_args = [ - a for a in all_args - if a.agent == arg.agent and a.round < current_round + a for a in all_args if a.agent == arg.agent and a.round < current_round ] # Nothing to compare in Round 1 if not prior_args: @@ -274,7 +381,7 @@ def contradiction_check(state: DebateState) -> dict: proposition=proposition, current_round=current_round, ) - flagged_ids.extend(flag.claim_id for flag in report.flags) + flagged_ids.extend(report.flags) return {"contradiction_flags": flagged_ids} @@ -290,22 +397,28 @@ def _check_contradictions_for_agent( Runs the contradiction check for a single agent's current argument against their full prior argument history. """ - prompt = ChatPromptTemplate.from_messages([ - ("system", CONTRADICTION_SYSTEM), - ("user", CONTRADICTION_USER), - ]) + prompt = ChatPromptTemplate.from_messages( + [ + ("system", CONTRADICTION_SYSTEM), + ("user", CONTRADICTION_USER), + ] + ) llm = llm or _get_checker_llm() structured_llm = llm.with_structured_output(ContradictionReport) chain = prompt | structured_llm try: - return chain.invoke({ - "agent": current_arg.agent.upper(), - "proposition": proposition, - "current_round": current_round, - "current_arg": format_argument_for_eval(current_arg), - "prior_args": format_prior_args_for_agent(prior_args, current_arg.agent), - }) + return chain.invoke( + { + "agent": current_arg.agent.upper(), + "proposition": proposition, + "current_round": current_round, + "current_arg": format_argument_for_eval(current_arg), + "prior_args": format_prior_args_for_agent( + prior_args, current_arg.agent + ), + } + ) except Exception as exc: raise EvaluationError( f"Contradiction check failed for agent {current_arg.agent}, " @@ -317,10 +430,12 @@ def _check_contradictions_for_agent( # Errors # --------------------------------------------------------------------------- + class EvaluationError(RuntimeError): """ Raised when an evaluation node cannot complete due to missing state, LLM failure, or schema validation errors. Surfaces as a node failure in LangGraph and can be caught by a retry policy or the metrics dashboard. """ + pass diff --git a/src/argument_lab/core/exporter.py b/src/argument_lab/core/exporter.py index 08df5a4..470c8e5 100644 --- a/src/argument_lab/core/exporter.py +++ b/src/argument_lab/core/exporter.py @@ -34,6 +34,7 @@ # Public entry point # --------------------------------------------------------------------------- + def export_debate( state: DebateState, session_id: str, @@ -58,7 +59,7 @@ def export_debate( payload = _build_json_payload(state, session_id) json_path = output_dir / f"{session_id}.json" - md_path = output_dir / f"{session_id}.md" + md_path = output_dir / f"{session_id}.md" _write_json(payload, json_path) _write_markdown(payload, md_path) @@ -70,6 +71,7 @@ def export_debate( # JSON payload builder # --------------------------------------------------------------------------- + def _build_json_payload(state: DebateState, session_id: str) -> dict: """ Converts the DebateState into a clean, serialisable dict. All Pydantic @@ -77,44 +79,67 @@ def _build_json_payload(state: DebateState, session_id: str) -> dict: the JSON is deterministic and diffable. """ arguments = state.get("arguments", []) - scores = state.get("scores", []) + scores = state.get("scores", []) + hallucination_flags = _serialise_flags(state.get("hallucination_flags", [])) + contradiction_flags = _serialise_flags(state.get("contradiction_flags", [])) + verdict = state.get("verdict") return { - "session_id": session_id, - "exported_at": datetime.now(timezone.utc).isoformat(), - "proposition": state["proposition"], - "status": state.get("status", "unknown"), + "session_id": session_id, + "exported_at": datetime.now(timezone.utc).isoformat(), + "proposition": state["proposition"], + "status": state.get("status", "unknown"), "rounds_completed": _rounds_completed(arguments), - + "verdict": verdict.model_dump() if verdict else None, # ── Per-round debate transcript ────────────────────────────── "rounds": _build_rounds(arguments, scores), - # ── Evaluation summary ─────────────────────────────────────── "evaluation": { - "hallucination_flags": sorted(state.get("hallucination_flags", [])), - "contradiction_flags": sorted(state.get("contradiction_flags", [])), - "hallucination_count": len(state.get("hallucination_flags", [])), - "contradiction_count": len(state.get("contradiction_flags", [])), + "hallucination_flags": hallucination_flags, + "contradiction_flags": contradiction_flags, + "hallucination_count": len(hallucination_flags), + "contradiction_count": len(contradiction_flags), }, - # ── Score trajectories (for dashboard charts) ──────────────── "score_trajectories": _build_score_trajectories(scores), - # ── Agent confidence drift ─────────────────────────────────── "agent_positions": { agent: positions for agent, positions in state.get("agent_positions", {}).items() }, - # ── Claim graph data ───────────────────────────────────────── "claim_graph": _build_claim_graph(arguments), - # ── Ignored claims (penalised in scoring) ──────────────────── - "ignored_claims": sorted(state.get("ignored_claims", [])), + "ignored_claims": sorted(state.get("ignored_claims", [])), "addressed_claims": sorted(state.get("addressed_claims", [])), } +def _serialise_flags(flags: list) -> list[dict]: + serialised = [] + for flag in flags: + if hasattr(flag, "model_dump"): + serialised.append(flag.model_dump()) + elif isinstance(flag, dict): + serialised.append(flag) + else: + serialised.append({"claim_id": str(flag)}) + + return sorted( + serialised, + key=lambda flag: ( + str(flag.get("claim_id", "")), + str(flag.get("prior_claim_id", "")), + str(flag.get("severity", "")), + str(flag.get("contradiction_type", "")), + ), + ) + + +def _flag_claim_ids(flags: list[dict]) -> list[str]: + return [str(flag.get("claim_id", "")) for flag in flags if flag.get("claim_id")] + + def _rounds_completed(arguments: list[Argument]) -> int: if not arguments: return 0 @@ -134,35 +159,37 @@ def _build_rounds( for r in range(1, max_round + 1): round_args = [a for a in arguments if a.round == r] - proponent = next((a for a in round_args if a.agent == "proponent"), None) - opponent = next((a for a in round_args if a.agent == "opponent"), None) - score = score_by_round.get(r) + proponent = next((a for a in round_args if a.agent == "proponent"), None) + opponent = next((a for a in round_args if a.agent == "opponent"), None) + score = score_by_round.get(r) - rounds.append({ - "round": r, - "proponent": _serialise_argument(proponent) if proponent else None, - "opponent": _serialise_argument(opponent) if opponent else None, - "judge": _serialise_score(score, r) if score else None, - }) + rounds.append( + { + "round": r, + "proponent": _serialise_argument(proponent) if proponent else None, + "opponent": _serialise_argument(opponent) if opponent else None, + "judge": _serialise_score(score, r) if score else None, + } + ) return rounds def _serialise_argument(arg: Argument) -> dict: return { - "id": arg.id, - "claim": arg.claim, - "evidence": [ + "id": arg.id, + "claim": arg.claim, + "evidence": [ { - "source_id": e.source_id, - "excerpt": e.excerpt, + "source_id": e.source_id, + "excerpt": e.excerpt, "reliability_score": e.reliability_score, } for e in arg.evidence ], - "assumptions": arg.assumptions, + "assumptions": arg.assumptions, "counterpoints_addressed": arg.counterpoints_addressed, - "confidence_score": arg.confidence_score, + "confidence_score": arg.confidence_score, } @@ -171,21 +198,21 @@ def _serialise_score(score: JudgeEvaluation, round_num: int) -> dict: "round": round_num, "proponent": { "logical_consistency": score.proponent_score.logical_consistency, - "evidence_support": score.proponent_score.evidence_support, - "relevance": score.proponent_score.relevance, - "completeness": score.proponent_score.completeness, - "composite": score.proponent_score.composite, + "evidence_support": score.proponent_score.evidence_support, + "relevance": score.proponent_score.relevance, + "completeness": score.proponent_score.completeness, + "composite": score.proponent_score.composite, }, "opponent": { "logical_consistency": score.opponent_score.logical_consistency, - "evidence_support": score.opponent_score.evidence_support, - "relevance": score.opponent_score.relevance, - "completeness": score.opponent_score.completeness, - "composite": score.opponent_score.composite, + "evidence_support": score.opponent_score.evidence_support, + "relevance": score.opponent_score.relevance, + "completeness": score.opponent_score.completeness, + "composite": score.opponent_score.composite, }, "convergence_detected": score.convergence_detected, - "stalemate_detected": score.stalemate_detected, - "explanation": score.explanation, + "stalemate_detected": score.stalemate_detected, + "explanation": score.explanation, } @@ -197,22 +224,22 @@ def _build_score_trajectories(scores: list[JudgeEvaluation]) -> dict: return { "rounds": [s.round for s in sorted_scores], "proponent_composite": [s.proponent_score.composite for s in sorted_scores], - "opponent_composite": [s.opponent_score.composite for s in sorted_scores], + "opponent_composite": [s.opponent_score.composite for s in sorted_scores], "proponent_breakdown": [ { "logical_consistency": s.proponent_score.logical_consistency, - "evidence_support": s.proponent_score.evidence_support, - "relevance": s.proponent_score.relevance, - "completeness": s.proponent_score.completeness, + "evidence_support": s.proponent_score.evidence_support, + "relevance": s.proponent_score.relevance, + "completeness": s.proponent_score.completeness, } for s in sorted_scores ], "opponent_breakdown": [ { "logical_consistency": s.opponent_score.logical_consistency, - "evidence_support": s.opponent_score.evidence_support, - "relevance": s.opponent_score.relevance, - "completeness": s.opponent_score.completeness, + "evidence_support": s.opponent_score.evidence_support, + "relevance": s.opponent_score.relevance, + "completeness": s.opponent_score.completeness, } for s in sorted_scores ], @@ -232,19 +259,23 @@ def _build_claim_graph(arguments: list[Argument]) -> dict: edges = [] for arg in arguments: - nodes.append({ - "id": arg.id, - "agent": arg.agent, - "round": arg.round, - "claim": arg.claim, - "confidence": arg.confidence_score, - }) + nodes.append( + { + "id": arg.id, + "agent": arg.agent, + "round": arg.round, + "claim": arg.claim, + "confidence": arg.confidence_score, + } + ) for prior_id in arg.counterpoints_addressed: - edges.append({ - "source": arg.id, - "target": prior_id, - "type": "challenged_by", - }) + edges.append( + { + "source": arg.id, + "target": prior_id, + "type": "challenged_by", + } + ) return {"nodes": nodes, "edges": edges} @@ -259,6 +290,7 @@ def _write_json(payload: dict, path: Path) -> None: # Markdown renderer # --------------------------------------------------------------------------- + def _write_markdown(payload: dict, path: Path) -> None: lines = _render_markdown(payload) with open(path, "w", encoding="utf-8") as f: @@ -268,9 +300,9 @@ def _write_markdown(payload: dict, path: Path) -> None: def _render_markdown(p: dict) -> list[str]: status_emoji = { - "converged": "✅ Converged", - "stalemate": "⚖️ Stalemate", - "terminated": "🏁 Terminated (max rounds)", + "converged": "✅ Converged", + "stalemate": "⚖️ Stalemate", + "terminated": "🏁 Terminated (max rounds)", "in_progress": "⏳ In Progress", }.get(p["status"], p["status"]) @@ -285,6 +317,25 @@ def _render_markdown(p: dict) -> list[str]: w(f"**Status:** {status_emoji} ") w(f"**Rounds completed:** {p['rounds_completed']} ") w("") + + if p.get("verdict"): + verdict = p["verdict"] + w("## Final Verdict") + w("") + w(f"**Type:** `{verdict.get('verdict_type')}` ") + w(f"**Winner:** `{verdict.get('winning_agent')}` ") + w("") + w(verdict.get("summary", "")) + w("") + if verdict.get("justification"): + w(f"**Justification:** {verdict['justification']}") + w("") + if verdict.get("unresolved_claims"): + w("**Unresolved claims**") + for claim in verdict["unresolved_claims"]: + w(f"- {claim}") + w("") + w("---") w("") w("## Proposition") @@ -298,9 +349,9 @@ def _render_markdown(p: dict) -> list[str]: w("## Score Summary") w("") traj = p.get("score_trajectories", {}) - rounds_list = traj.get("rounds", []) - prop_composites = traj.get("proponent_composite", []) - opp_composites = traj.get("opponent_composite", []) + rounds_list = traj.get("rounds", []) + prop_composites = traj.get("proponent_composite", []) + opp_composites = traj.get("opponent_composite", []) if rounds_list: w("| Round | Proponent (composite) | Opponent (composite) | Verdict |") @@ -335,10 +386,12 @@ def _render_markdown(p: dict) -> list[str]: w("") if eval_data.get("hallucination_flags"): - w(f"**Hallucinated claim IDs:** `{'`, `'.join(eval_data['hallucination_flags'])}`") + claim_ids = _flag_claim_ids(eval_data["hallucination_flags"]) + w(f"**Hallucinated claim IDs:** `{'`, `'.join(claim_ids)}`") w("") if eval_data.get("contradiction_flags"): - w(f"**Contradicted claim IDs:** `{'`, `'.join(eval_data['contradiction_flags'])}`") + claim_ids = _flag_claim_ids(eval_data["contradiction_flags"]) + w(f"**Contradicted claim IDs:** `{'`, `'.join(claim_ids)}`") w("") # ── Round transcripts ────────────────────────────────────────────────── @@ -367,7 +420,9 @@ def _render_markdown(p: dict) -> list[str]: if arg.get("evidence"): w("**Evidence cited**") for e in arg["evidence"]: - w(f"- `[{e['source_id']}]` (reliability: {e['reliability_score']:.2f})") + w( + f"- `[{e['source_id']}]` (reliability: {e['reliability_score']:.2f})" + ) w(f" > {e['excerpt']}") w("") @@ -378,7 +433,9 @@ def _render_markdown(p: dict) -> list[str]: w("") if arg.get("counterpoints_addressed"): - w(f"**Counterpoints addressed:** `{'`, `'.join(arg['counterpoints_addressed'])}`") + w( + f"**Counterpoints addressed:** `{'`, `'.join(arg['counterpoints_addressed'])}`" + ) w("") # Judge evaluation for this round @@ -390,10 +447,17 @@ def _render_markdown(p: dict) -> list[str]: w("|---|---|---|") p_b = judge["proponent"] o_b = judge["opponent"] - for dim in ("logical_consistency", "evidence_support", "relevance", "completeness"): + for dim in ( + "logical_consistency", + "evidence_support", + "relevance", + "completeness", + ): label = dim.replace("_", " ").title() w(f"| {label} | {p_b[dim]:.2f} | {o_b[dim]:.2f} |") - w(f"| **Composite** | **{p_b['composite']:.3f}** | **{o_b['composite']:.3f}** |") + w( + f"| **Composite** | **{p_b['composite']:.3f}** | **{o_b['composite']:.3f}** |" + ) w("") w(f"**Judge's note:** {judge['explanation']}") w("") @@ -424,4 +488,4 @@ def _render_markdown(p: dict) -> list[str]: w("*Full interactive graph available in the ArgumentLab dashboard.*") w("") - return lines \ No newline at end of file + return lines diff --git a/src/argument_lab/core/faiss_index.py b/src/argument_lab/core/faiss_index.py index 5b57df4..2cf9808 100644 --- a/src/argument_lab/core/faiss_index.py +++ b/src/argument_lab/core/faiss_index.py @@ -34,6 +34,7 @@ # Stored chunk metadata # --------------------------------------------------------------------------- + @dataclass class ChunkRecord: """ @@ -41,15 +42,17 @@ class ChunkRecord: The FAISS index stores raw float vectors; metadata lives alongside it in a sidecar JSON file. """ - source_id: str # e.g. "doc_03_chunk_12" - excerpt: str # the raw text of the chunk - doc_title: str # human-readable source label for the metrics dashboard + + source_id: str # e.g. "doc_03_chunk_12" + excerpt: str # the raw text of the chunk + doc_title: str # human-readable source label for the metrics dashboard # --------------------------------------------------------------------------- # FaissIndex # --------------------------------------------------------------------------- + class FaissIndex: """ Wraps a flat L2 FAISS index and a parallel list of ChunkRecords. @@ -96,11 +99,13 @@ def similarity_search(self, query: str, k: int) -> list[RetrievedChunk]: record = self._metadata[idx] # Convert L2 distance on unit vectors to cosine similarity similarity = float(np.clip(1.0 - dist / 2.0, 0.0, 1.0)) - results.append(RetrievedChunk( - source_id=record.source_id, - excerpt=record.excerpt, - score=round(similarity, 4), - )) + results.append( + RetrievedChunk( + source_id=record.source_id, + excerpt=record.excerpt, + score=round(similarity, 4), + ) + ) return results @@ -193,4 +198,4 @@ def _embed_query(self, query: str) -> np.ndarray: vec = self._embeddings.embed_query(query) matrix = np.array([vec], dtype=np.float32) faiss.normalize_L2(matrix) - return matrix \ No newline at end of file + return matrix diff --git a/src/argument_lab/core/models.py b/src/argument_lab/core/models.py index 5ab27f0..6caeab0 100644 --- a/src/argument_lab/core/models.py +++ b/src/argument_lab/core/models.py @@ -14,13 +14,11 @@ class Argument(BaseModel): agent: Literal["proponent", "opponent"] claim: str evidence: list[EvidenceRef] = Field( - min_length=1, - description="Must contain ≥1 retrieved source" + min_length=1, description="Must contain ≥1 retrieved source" ) assumptions: list[str] counterpoints_addressed: list[str] = Field( - default_factory=list, - description="Claim IDs of opponent's prior points" + default_factory=list, description="Claim IDs of opponent's prior points" ) confidence_score: float = Field(ge=0.0, le=1.0) @@ -37,15 +35,23 @@ class ArgumentScore(BaseModel): evidence_support: float = Field(ge=0.0, le=1.0) relevance: float = Field(ge=0.0, le=1.0) completeness: float = Field(ge=0.0, le=1.0) + hallucination_penalty: float = Field(default=0.0, ge=0.0) + contradiction_penalty: float = Field(default=0.0, ge=0.0) @property def composite(self) -> float: - """Weighted composite per architecture spec.""" - return round( + """Weighted composite per architecture spec, minus penalties.""" + base_score = ( self.logical_consistency * 0.30 + self.evidence_support * 0.30 + self.relevance * 0.20 - + self.completeness * 0.20, + + self.completeness * 0.20 + ) + return round( + max( + 0.0, + base_score - self.hallucination_penalty - self.contradiction_penalty, + ), 4, ) @@ -63,6 +69,7 @@ class JudgeEvaluation(BaseModel): # Hallucination checker output # --------------------------------------------------------------------------- + class HallucinationFlag(BaseModel): claim_id: str reason: str @@ -77,8 +84,9 @@ class HallucinationReport(BaseModel): # Contradiction checker output # --------------------------------------------------------------------------- + class ContradictionFlag(BaseModel): - claim_id: str # current claim that contradicts a prior one + claim_id: str # current claim that contradicts a prior one prior_claim_id: str # the earlier claim it contradicts contradiction_type: Literal[ "direct_negation", @@ -91,3 +99,19 @@ class ContradictionFlag(BaseModel): class ContradictionReport(BaseModel): flags: list[ContradictionFlag] = Field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Final Verdict output +# --------------------------------------------------------------------------- + + +class Verdict(BaseModel): + verdict_type: Literal["consensus", "best_argument", "stalemate"] + winning_agent: Literal["proponent", "opponent", "none"] + summary: str + unresolved_claims: list[str] = Field( + default_factory=list, + description="List of core issues where agents never aligned", + ) + justification: str diff --git a/src/argument_lab/core/prompts.py b/src/argument_lab/core/prompts.py index 294d49d..612d00d 100644 --- a/src/argument_lab/core/prompts.py +++ b/src/argument_lab/core/prompts.py @@ -13,13 +13,18 @@ # Shared formatting helpers # --------------------------------------------------------------------------- + def format_argument(arg) -> str: """Render a prior Argument object as a readable block for debate history.""" evidence_lines = "\n".join( - f" [{e.source_id}] \"{e.excerpt}\" (reliability: {e.reliability_score:.2f})" + f' [{e.source_id}] "{e.excerpt}" (reliability: {e.reliability_score:.2f})' for e in arg.evidence ) - addressed = ", ".join(arg.counterpoints_addressed) if arg.counterpoints_addressed else "none" + addressed = ( + ", ".join(arg.counterpoints_addressed) + if arg.counterpoints_addressed + else "none" + ) return ( f"[{arg.agent.upper()} — Round {arg.round} — claim_id: {arg.id}]\n" f"Claim: {arg.claim}\n" @@ -41,8 +46,7 @@ def format_evidence_context(chunks: list) -> str: if not chunks: return "No evidence retrieved." return "\n".join( - f"[{c.source_id}] (similarity: {c.score:.2f})\n\"{c.excerpt}\"" - for c in chunks + f'[{c.source_id}] (similarity: {c.score:.2f})\n"{c.excerpt}"' for c in chunks ) @@ -135,4 +139,4 @@ def format_evidence_context(chunks: list) -> str: 1: "Establish your strongest top-level case for your position.", 2: "Rebut your opponent's Round 1 claims with specific evidence.", 3: "Refine your position based on all prior evidence and finalize your case.", -} \ No newline at end of file +} diff --git a/src/argument_lab/core/reasonbench_eval.py b/src/argument_lab/core/reasonbench_eval.py index 081fc66..3ac0e48 100644 --- a/src/argument_lab/core/reasonbench_eval.py +++ b/src/argument_lab/core/reasonbench_eval.py @@ -43,6 +43,7 @@ api_key=os.environ.get("OPENAI_API_KEY", "dummy"), ) + def evaluate_reasonbench_round( task_type: TaskType, problem: str, @@ -52,7 +53,7 @@ def evaluate_reasonbench_round( prior_context: str = "None (Round 1)", llm: Any = _judge_llm, ) -> Union[Task1Evaluation, Task2Evaluation, Task3Evaluation]: - + # Select the correct output schema if task_type == TaskType.TASK_1_LOGIC: schema = Task1Evaluation @@ -63,21 +64,25 @@ def evaluate_reasonbench_round( else: raise ValueError(f"Unknown TaskType: {task_type}") - prompt = ChatPromptTemplate.from_messages([ - ("system", REASONBENCH_JUDGE_SYSTEM), - ("user", REASONBENCH_JUDGE_USER), - ]) + prompt = ChatPromptTemplate.from_messages( + [ + ("system", REASONBENCH_JUDGE_SYSTEM), + ("user", REASONBENCH_JUDGE_USER), + ] + ) structured_llm = llm.with_structured_output(schema) chain = prompt | structured_llm - result = chain.invoke({ - "task_type": task_type.value, - "problem": problem, - "current_round": current_round, - "prior_context": prior_context, - "proponent_response": proponent_response.model_dump_json(indent=2), - "opponent_response": opponent_response.model_dump_json(indent=2), - }) + result = chain.invoke( + { + "task_type": task_type.value, + "problem": problem, + "current_round": current_round, + "prior_context": prior_context, + "proponent_response": proponent_response.model_dump_json(indent=2), + "opponent_response": opponent_response.model_dump_json(indent=2), + } + ) return result diff --git a/src/argument_lab/core/reasonbench_models.py b/src/argument_lab/core/reasonbench_models.py index e69de29..8de00f8 100644 --- a/src/argument_lab/core/reasonbench_models.py +++ b/src/argument_lab/core/reasonbench_models.py @@ -0,0 +1,114 @@ +from enum import Enum +from pydantic import BaseModel, Field + + +class TaskType(str, Enum): + TASK_1_LOGIC = "task_1_logic" + TASK_2_STRATEGY = "task_2_strategy" + TASK_3_TRADEOFF = "task_3_tradeoff" + + +class ReasonBenchResponse(BaseModel): + """Shared output schema for all models across all ReasonBench tasks.""" + + strategy_or_answer: str = Field(description="Final answer or plan") + rationale: str = Field(description="Step-by-step reasoning") + assumptions: list[str] = Field(description="Explicit assumptions made") + opponent_model: str = Field( + description="What the model believes about the opponent (if applicable)" + ) + risks: list[str] = Field(description="Failure modes or weaknesses") + conditions: list[str] = Field(description="When the answer/strategy would change") + + +# --------------------------------------------------------------------------- +# Task 1: Deterministic Logic (Constraint Puzzle) +# --------------------------------------------------------------------------- + + +class Task1Score(BaseModel): + correctness: int = Field( + ge=0, le=2, description="0: incorrect, 1: partially correct, 2: fully correct" + ) + logical_consistency: int = Field( + ge=0, le=2, description="0: contradictory, 1: minor issues, 2: fully consistent" + ) + completeness: int = Field( + ge=0, le=2, description="0: incomplete, 1: partial, 2: fully explains all boxes" + ) + responsiveness: int = Field( + ge=0, + le=2, + description="0: ignores critique, 1: partially integrates, 2: fully integrates", + ) + + +class Task1Evaluation(BaseModel): + proponent_score: Task1Score + opponent_score: Task1Score + explanation: str = Field(description="Judge's explanation for the assigned scores.") + + +# --------------------------------------------------------------------------- +# Task 2: Strategic Reasoning (Asymmetric Game) +# --------------------------------------------------------------------------- + + +class Task2Score(BaseModel): + opponent_modeling: int = Field( + ge=0, le=2, description="0: ignores, 1: partial, 2: uses strategically" + ) + strategic_coherence: int = Field( + ge=0, le=2, description="0: inconsistent, 1: partial, 2: structured plan" + ) + risk_awareness: int = Field( + ge=0, le=2, description="0: ignores, 1: partial, 2: balances risk/reward" + ) + conditional_reasoning: int = Field( + ge=0, le=2, description="0: static, 1: partial, 2: adaptive plan" + ) + responsiveness: int = Field( + ge=0, + le=2, + description="0: ignores critique, 1: partially integrates, 2: fully integrates", + ) + + +class Task2Evaluation(BaseModel): + proponent_score: Task2Score + opponent_score: Task2Score + explanation: str = Field(description="Judge's explanation for the assigned scores.") + + +# --------------------------------------------------------------------------- +# Task 3: Constrained Tradeoff Reasoning +# --------------------------------------------------------------------------- + + +class Task3Score(BaseModel): + constraint_utilization: int = Field( + ge=0, le=2, description="0: ignores, 1: partial, 2: deeply used" + ) + tradeoff_specificity: int = Field( + ge=0, le=2, description="0: generic, 1: partial, 2: contextual" + ) + assumptions_quality: int = Field( + ge=0, le=2, description="0: implicit, 1: partial, 2: explicit" + ) + risk_analysis: int = Field( + ge=0, le=2, description="0: vague, 1: partial, 2: concrete" + ) + conditional_reasoning: int = Field( + ge=0, le=2, description="0: static, 1: partial, 2: adaptive" + ) + responsiveness: int = Field( + ge=0, + le=2, + description="0: ignores critique, 1: partially integrates, 2: fully integrates", + ) + + +class Task3Evaluation(BaseModel): + proponent_score: Task3Score + opponent_score: Task3Score + explanation: str = Field(description="Judge's explanation for the assigned scores.") diff --git a/src/argument_lab/core/retriever.py b/src/argument_lab/core/retriever.py index 6d5ea34..c21f4f7 100644 --- a/src/argument_lab/core/retriever.py +++ b/src/argument_lab/core/retriever.py @@ -6,38 +6,38 @@ swappable (FAISS for MVP, OpenSearch for v2) — agents never import the index directly. """ - + from dataclasses import dataclass from typing import Protocol - - + + @dataclass class RetrievedChunk: source_id: str excerpt: str score: float # cosine similarity, [0, 1] - - + + class VectorIndex(Protocol): """ Any object with this interface can be used as the backing index. FAISS, ChromaDB, and OpenSearch all satisfy it with a thin wrapper. """ - def similarity_search(self, query: str, k: int) -> list[RetrievedChunk]: - ... - - + + def similarity_search(self, query: str, k: int) -> list[RetrievedChunk]: ... + + class Retriever: """ Injected into each agent node at graph compile time via the config. Agents call retrieve() with a natural-language query and get back chunks they can directly attach as EvidenceRef objects. """ - + def __init__(self, index: VectorIndex, top_k: int = 4): self._index = index self._top_k = top_k - + def retrieve(self, query: str) -> list[RetrievedChunk]: """ Returns up to top_k chunks ranked by similarity to the query. @@ -49,7 +49,7 @@ def retrieve(self, query: str) -> list[RetrievedChunk]: except Exception as exc: # Propagate as a typed error so the node can surface it cleanly raise RetrieverError(f"Index query failed: {exc}") from exc - + def retrieve_multi(self, queries: list[str]) -> list[RetrievedChunk]: """ Runs multiple queries and deduplicates by source_id, keeping the @@ -63,7 +63,7 @@ def retrieve_multi(self, queries: list[str]) -> list[RetrievedChunk]: if existing is None or chunk.score > existing.score: seen[chunk.source_id] = chunk return sorted(seen.values(), key=lambda c: c.score, reverse=True) - - + + class RetrieverError(RuntimeError): - pass \ No newline at end of file + pass diff --git a/src/argument_lab/core/state.py b/src/argument_lab/core/state.py index a34f868..f4bb716 100644 --- a/src/argument_lab/core/state.py +++ b/src/argument_lab/core/state.py @@ -1,25 +1,37 @@ from typing import Annotated, Literal, TypedDict import operator -from argument_lab.core.models import Argument, Claim, JudgeEvaluation +from argument_lab.core.models import ( + Argument, + Claim, + JudgeEvaluation, + HallucinationFlag, + ContradictionFlag, + Verdict, +) MAX_ROUNDS = 3 + def union_sets(a: set[str] | None, b: set[str] | None) -> set[str]: return (a or set()) | (b or set()) + def merge_dicts(a: dict | None, b: dict | None) -> dict: return {**(a or {}), **(b or {})} + def max_round(a: int | None, b: int | None) -> int: return max(a or 0, b or 0) + def merge_status(a: str | None, b: str | None) -> str: priority = ["terminated", "stalemate", "converged", "in_progress"] a_val = a if a in priority else "in_progress" b_val = b if b in priority else "in_progress" return a_val if priority.index(a_val) < priority.index(b_val) else b_val + class DebateState(TypedDict): proposition: str current_round: Annotated[int, max_round] @@ -29,7 +41,10 @@ class DebateState(TypedDict): ignored_claims: Annotated[set[str], union_sets] agent_positions: Annotated[dict[str, list[float]], merge_dicts] repetition_flags: Annotated[list[str], operator.add] - status: Annotated[Literal["in_progress", "converged", "stalemate", "terminated"], merge_status] - hallucination_flags: Annotated[list[str], operator.add] - contradiction_flags: Annotated[list[str], operator.add] + status: Annotated[ + Literal["in_progress", "converged", "stalemate", "terminated"], merge_status + ] + hallucination_flags: Annotated[list[HallucinationFlag], operator.add] + contradiction_flags: Annotated[list[ContradictionFlag], operator.add] scores: Annotated[list[JudgeEvaluation], operator.add] + verdict: Verdict | None = None diff --git a/src/argument_lab/orchestrator/graph.py b/src/argument_lab/orchestrator/graph.py index d8c9228..d817aa8 100644 --- a/src/argument_lab/orchestrator/graph.py +++ b/src/argument_lab/orchestrator/graph.py @@ -15,7 +15,12 @@ from langgraph.graph import StateGraph, START, END from argument_lab.core.agents import make_proponent_node, make_opponent_node -from argument_lab.core.evaluation import judge_node, hallucination_check, contradiction_check +from argument_lab.core.evaluation import ( + judge_node, + hallucination_check, + contradiction_check, + verdict_generator, +) from argument_lab.core.retriever import Retriever from argument_lab.core.state import DebateState, MAX_ROUNDS @@ -24,6 +29,7 @@ # Non-LLM nodes # --------------------------------------------------------------------------- + def start_round(state: DebateState) -> dict: """ Passthrough node that acts as the fan-out point at the start of each @@ -44,6 +50,7 @@ def graph_update(state: DebateState) -> dict: # Routing # --------------------------------------------------------------------------- + def route_round(state: DebateState) -> str: """ Decides whether to loop back for another round or terminate. @@ -59,9 +66,9 @@ def route_round(state: DebateState) -> str: """ status = state.get("status", "in_progress") if status in ("converged", "stalemate", "terminated"): - return END + return "generate_verdict" if state.get("current_round", 1) > MAX_ROUNDS: - return END + return "generate_verdict" return "start_round" @@ -69,6 +76,7 @@ def route_round(state: DebateState) -> str: # Graph factory # --------------------------------------------------------------------------- + def build_graph(retriever: Retriever): """ Compile the debate workflow. Call once at application startup and @@ -109,7 +117,10 @@ def build_graph(retriever: Retriever): workflow.add_node("opponent", make_opponent_node(retriever)) # Passthrough fan-in/fan-out between agent round and evaluation round - workflow.add_node("start_evaluation", lambda state: {"current_round": state.get("current_round", 1)}) + workflow.add_node( + "start_evaluation", + lambda state: {"current_round": state.get("current_round", 1)}, + ) # Evaluation nodes — all three run in parallel workflow.add_node("judge", judge_node) @@ -119,6 +130,9 @@ def build_graph(retriever: Retriever): # Final fan-in before routing decision workflow.add_node("graph_update", graph_update) + # Terminal node for synthesis + workflow.add_node("generate_verdict", verdict_generator) + # --- Edge wiring --- # Entry point @@ -132,17 +146,20 @@ def build_graph(retriever: Retriever): workflow.add_edge("proponent", "start_evaluation") workflow.add_edge("opponent", "start_evaluation") - # Fan-out: judge, hallucination check, and contradiction check run in parallel - workflow.add_edge("start_evaluation", "judge") + # Fan-out: checkers run in parallel before the judge scores the round. workflow.add_edge("start_evaluation", "hallucination_check") workflow.add_edge("start_evaluation", "contradiction_check") - # Fan-in: all three evaluation nodes must complete before graph_update + # Fan-in: all checkers must complete before judge (to apply penalties) + workflow.add_edge("hallucination_check", "judge") + workflow.add_edge("contradiction_check", "judge") + + # Fan-in: judge completes the round workflow.add_edge("judge", "graph_update") - workflow.add_edge("hallucination_check", "graph_update") - workflow.add_edge("contradiction_check", "graph_update") # Conditional routing: loop or terminate workflow.add_conditional_edges("graph_update", route_round) + workflow.add_edge("generate_verdict", END) + return workflow.compile() diff --git a/srcback/argument_lab/core/agents.py b/srcback/argument_lab/core/agents.py new file mode 100644 index 0000000..e647247 --- /dev/null +++ b/srcback/argument_lab/core/agents.py @@ -0,0 +1,485 @@ +""" +argument_lab/core/agents.py + +Proponent and opponent agent nodes for the LangGraph debate workflow. + +Each agent follows a strict two-step pipeline: + Step 1 — Query formulation: a lightweight LLM call produces 1-3 search queries tailored to the agent's current goal. + Step 2 — Retrieval + generation: the queries are executed against the vector index, and the retrieved chunks are + injected into the system prompt before the structured argument generation call. + +This guarantees that every Argument object contains grounded evidence +before it ever reaches Pydantic validation. +""" + +import uuid +from typing import Any + +from langchain_core.output_parsers import JsonOutputParser +from langchain_core.prompts import ChatPromptTemplate + +from argument_lab.core.models import Argument, Claim, EvidenceRef +from argument_lab.core.retriever import Retriever +from argument_lab.core.state import DebateState, MAX_ROUNDS +from argument_lab.core.prompts import ( + QUERY_FORMULATION_SYSTEM, + QUERY_FORMULATION_USER, + AGENT_SYSTEM_TEMPLATE, + AGENT_USER_TEMPLATE, + COUNTERPOINT_RULES, + ROUND_GOALS, + format_debate_history, + format_evidence_context, +) + + +# --------------------------------------------------------------------------- +# LLM setup +# Temperature 0.2 for structured output — low enough for schema compliance, +# high enough to avoid degenerate repetition across rounds. +# --------------------------------------------------------------------------- + +import os + +_llm: Any | None = None +_query_llm: Any | None = None + + +def _make_chat_openai(*, model: str, temperature: float) -> Any: + try: + from langchain_openai import ChatOpenAI + except ModuleNotFoundError as exc: + raise AgentError( + "langchain_openai is required for LLM-backed agent execution. " + "Install project dependencies with `pip install -r requirements.txt`." + ) from exc + + return ChatOpenAI( + model=model, + temperature=temperature, + api_key=os.environ.get("OPENAI_API_KEY", "dummy"), + ) + + +def _get_generation_llm() -> Any: + global _llm + if _llm is None: + _llm = _make_chat_openai(model="gpt-4o", temperature=0.2) + return _llm + + +def _get_query_llm() -> Any: + global _query_llm + if _query_llm is None: + _query_llm = _make_chat_openai(model="gpt-4o-mini", temperature=0.0) + return _query_llm + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + + +def _formulate_queries( + proposition: str, + stance: str, + history: str, + current_round: int, + llm: Any | None = None, +) -> list[str]: + """ + Step 1: Ask a lightweight LLM to produce search queries for this agent's + next argument. Returns a list of query strings, falling back to the + proposition itself if the LLM output cannot be parsed. + """ + llm = llm or _get_query_llm() + prompt = ChatPromptTemplate.from_messages( + [ + ("system", QUERY_FORMULATION_SYSTEM), + ("user", QUERY_FORMULATION_USER), + ] + ) + chain = prompt | llm | JsonOutputParser() + + try: + result = chain.invoke( + { + "proposition": proposition, + "stance": stance, + "history": history, + "current_round": current_round, + "round_goal": ROUND_GOALS[min(current_round, MAX_ROUNDS)], + } + ) + queries = result.get("queries", []) + if isinstance(queries, list) and all(isinstance(q, str) for q in queries): + return queries[:3] # enforce max + except Exception: + pass # fall through to fallback + + # Fallback: use the proposition directly so retrieval never returns empty + return [proposition] + + +def _retrieve_evidence( + retriever: Retriever, + queries: list[str], +) -> list[EvidenceRef]: + """ + Execute the queries against the vector index and convert chunks to EvidenceRef objects ready for the Argument schema. + Raises AgentError if retrieval returns nothing — no evidence means no valid argument can be produced. + """ + chunks = retriever.retrieve_multi(queries) + if not chunks: + raise AgentError( + "RAG retrieval returned no results. Cannot produce a grounded argument." + ) + return [ + EvidenceRef( + source_id=chunk.source_id, + excerpt=chunk.excerpt, + reliability_score=round(chunk.score, 3), + ) + for chunk in chunks + ] + + +def _generate_argument( + *, + role: str, + stance: str, + proposition: str, + current_round: int, + history: str, + evidence_refs: list[EvidenceRef], + evidence_context: str, + argument_id: str, + llm: Any | None = None, +) -> Argument: + """ + Step 2: Generate the structured Argument using the retrieved evidence injected into the system prompt. Uses .with_structured_output() to + enforce schema compliance at the LangChain layer. + """ + llm = llm or _get_generation_llm() + structured_llm = llm.with_structured_output(Argument) + + system_prompt = AGENT_SYSTEM_TEMPLATE.format_map( + { + "role": role, + "stance": "FOR" if role == "Proponent" else "AGAINST", + "proposition": proposition, + "counterpoint_rule": COUNTERPOINT_RULES[min(current_round, MAX_ROUNDS)], + "evidence_context": evidence_context, + } + ) + + user_prompt = AGENT_USER_TEMPLATE.format_map( + { + "history": history, + "current_round": current_round, + "argument_id": argument_id, + } + ) + + prompt = ChatPromptTemplate.from_messages( + [ + ("system", system_prompt), + ("user", user_prompt), + ] + ) + + chain = prompt | structured_llm + argument = chain.invoke({}) + + # Patch in the argument_id and round in case the LLM didn't follow them + # exactly — the schema enforces types but not specific string values. + argument = argument.model_copy( + update={ + "id": argument_id, + "round": current_round, + "agent": role.lower(), + } + ) + + # Ensure the LLM cited sources that were actually retrieved (not hallucinated IDs) + valid_source_ids = {ref.source_id for ref in evidence_refs} + argument = argument.model_copy( + update={ + "evidence": [ + e for e in argument.evidence if e.source_id in valid_source_ids + ] + or evidence_refs[ + :1 + ], # guarantee min_length=1 even if LLM cited nothing valid + } + ) + + return argument + + +def _enforce_counterpoint_rule( + argument: Argument, + current_round: int, + prior_opponent_claim_ids: list[str], + role: str, + llm: Any | None = None, + proposition: str = "", + history: str = "", + evidence_refs: list[EvidenceRef] = [], + evidence_context: str = "", +) -> Argument: + """ + Option 3 enforcement: if Round >= 2 and counterpoints_addressed is empty, re-prompt the LLM once with an explicit correction instruction. + Raises AgentError if the second attempt also fails — this surfaces as a node failure rather than silently passing a non-compliant argument. + """ + if current_round < 2: + return argument # Round 1: no counterpoints required + + if argument.counterpoints_addressed: + return argument # already compliant + + if not prior_opponent_claim_ids: + # No opponent claims exist yet (e.g., opponent hasn't run this round) + # This shouldn't happen in normal flow but guard defensively. + return argument + + # Re-prompt with an explicit correction + correction_note = ( + f"Your previous response left counterpoints_addressed empty. " + f"You MUST include at least one of these opponent claim IDs: " + f"{prior_opponent_claim_ids}. Revise your argument to address " + f"at least one of these claims directly." + ) + llm = llm or _get_generation_llm() + structured_llm = llm.with_structured_output(Argument) + prompt = ChatPromptTemplate.from_messages( + [ + ( + "system", + AGENT_SYSTEM_TEMPLATE.format_map( + { + "role": role, + "stance": "FOR" if role == "Proponent" else "AGAINST", + "proposition": proposition, + "counterpoint_rule": COUNTERPOINT_RULES[ + min(current_round, MAX_ROUNDS) + ], + "evidence_context": evidence_context, + } + ), + ), + ( + "user", + AGENT_USER_TEMPLATE.format_map( + { + "history": history, + "current_round": current_round, + "argument_id": argument.id, + } + ), + ), + ("assistant", argument.model_dump_json()), + ("user", correction_note), + ] + ) + + revised = (prompt | structured_llm).invoke({}) + revised = revised.model_copy( + update={ + "id": argument.id, + "round": current_round, + "agent": argument.agent, + } + ) + + if not revised.counterpoints_addressed: + raise AgentError( + f"{role} failed to address any opponent counterpoints in Round {current_round} " + f"after correction. Prior claim IDs available: {prior_opponent_claim_ids}" + ) + + return revised + + +def _get_prior_opponent_claim_ids(state: DebateState, my_role: str) -> list[str]: + """ + Returns claim IDs from the opponent's prior arguments. + Used to validate and enforce counterpoint_addressed in Round >= 2. + """ + opponent_role = "opponent" if my_role == "proponent" else "proponent" + return [ + arg.id + for arg in state.get("arguments", []) + if arg.agent == opponent_role and arg.round < state["current_round"] + ] + + +def _update_state_from_argument( + argument: Argument, + state: DebateState, +) -> dict: + """ + Derive all state updates that a new argument produces: + - adds argument to the accumulator + - registers its claim in claims_registry + - updates agent_positions with the new confidence score + - marks addressed and ignored claims + """ + # Register the new claim + new_claim = Claim( + id=argument.id, + text=argument.claim, + agent=argument.agent, + round=argument.round, + ) + + # Determine which prior opponent claims were ignored this round + opponent_role = "opponent" if argument.agent == "proponent" else "proponent" + prior_opponent_ids = { + arg.id for arg in state.get("arguments", []) if arg.agent == opponent_role + } + newly_addressed = set(argument.counterpoints_addressed) + newly_ignored = ( + prior_opponent_ids - newly_addressed - state.get("ignored_claims", set()) + ) + + # Extend the agent's confidence trajectory + current_positions = dict(state.get("agent_positions", {})) + trajectory = list(current_positions.get(argument.agent, [])) + trajectory.append(argument.confidence_score) + + return { + "arguments": [argument], + "claims_registry": {argument.id: new_claim}, + "addressed_claims": newly_addressed, + "ignored_claims": newly_ignored, + "agent_positions": {argument.agent: trajectory}, + } + + +# --------------------------------------------------------------------------- +# Public node functions +# --------------------------------------------------------------------------- + + +def make_proponent_node(retriever: Retriever): + """ + Factory that closes over a Retriever instance and returns a LangGraph- + compatible node function. Call this at graph compile time: + + workflow.add_node("proponent", make_proponent_node(retriever)) + """ + + def proponent_node(state: DebateState) -> dict: + return _run_agent_node( + state=state, + role="Proponent", + agent_key="proponent", + retriever=retriever, + ) + + return proponent_node + + +def make_opponent_node(retriever: Retriever): + """ + Factory that closes over a Retriever instance and returns a LangGraph- + compatible node function. + + workflow.add_node("opponent", make_opponent_node(retriever)) + """ + + def opponent_node(state: DebateState) -> dict: + return _run_agent_node( + state=state, + role="Opponent", + agent_key="opponent", + retriever=retriever, + ) + + return opponent_node + + +def _run_agent_node( + *, + state: DebateState, + role: str, + agent_key: str, + retriever: Retriever, +) -> dict: + """ + Shared implementation for both agent nodes. + + Pipeline: + 1. Format debate history for context + 2. Formulate search queries (lightweight LLM call) + 3. Retrieve evidence from vector index + 4. Generate structured Argument (main LLM call) + 5. Enforce counterpoint rule (re-prompt if needed) + 6. Derive and return state updates + """ + current_round = state["current_round"] + proposition = state["proposition"] + prior_arguments = state.get("arguments", []) + + # Step 1: format history + history = format_debate_history(prior_arguments) + stance = "FOR" if role == "Proponent" else "AGAINST" + + # Step 2: query formulation + queries = _formulate_queries( + proposition=proposition, + stance=stance, + history=history, + current_round=current_round, + ) + + # Step 3: retrieve evidence + evidence_refs = _retrieve_evidence(retriever, queries) + evidence_context = format_evidence_context( + # Pass the raw chunks back for display; evidence_refs are already converted + retriever.retrieve_multi(queries) + ) + + # Step 4: generate argument + argument_id = str(uuid.uuid4()) + argument = _generate_argument( + role=role, + stance=stance, + proposition=proposition, + current_round=current_round, + history=history, + evidence_refs=evidence_refs, + evidence_context=evidence_context, + argument_id=argument_id, + ) + + # Step 5: enforce counterpoint rule (Option 3) + prior_opponent_ids = _get_prior_opponent_claim_ids(state, agent_key) + argument = _enforce_counterpoint_rule( + argument=argument, + current_round=current_round, + prior_opponent_claim_ids=prior_opponent_ids, + role=role, + proposition=proposition, + history=history, + evidence_refs=evidence_refs, + evidence_context=evidence_context, + ) + + # Step 6: derive state updates + return _update_state_from_argument(argument, state) + + +# --------------------------------------------------------------------------- +# Errors +# --------------------------------------------------------------------------- + + +class AgentError(RuntimeError): + """ + Raised when an agent node cannot produce a valid, schema-compliant argument. + The LangGraph node will propagate this as a node failure, which can be caught by a retry policy or surfaced to the metrics dashboard. + """ + + pass diff --git a/srcback/argument_lab/core/eval_prompts.py b/srcback/argument_lab/core/eval_prompts.py new file mode 100644 index 0000000..87e24c8 --- /dev/null +++ b/srcback/argument_lab/core/eval_prompts.py @@ -0,0 +1,293 @@ +""" +argument_lab/core/eval_prompts.py + +Prompt templates for the three parallel evaluation nodes: + - Judge (scoring + convergence/stalemate detection) + - Hallucination checker (evidence grounding verification) + - Contradiction checker (cross-round consistency auditing) + +Kept separate from agent prompts so each can be tuned and versioned +independently without touching orchestration code. +""" + +from argument_lab.core.models import Argument, JudgeEvaluation + + +# --------------------------------------------------------------------------- +# Shared formatting helpers +# --------------------------------------------------------------------------- + + +def format_argument_for_eval(arg: Argument) -> str: + """ + Renders a single Argument as a clearly labelled block for evaluation + prompts. Includes all fields the evaluator needs to do its job. + """ + evidence_lines = "\n".join( + f' [{e.source_id}] "{e.excerpt}" (reliability: {e.reliability_score:.2f})' + for e in arg.evidence + ) + addressed = ", ".join(arg.counterpoints_addressed) or "none" + return ( + f"Agent: {arg.agent.upper()}\n" + f"Claim ID: {arg.id}\n" + f"Claim: {arg.claim}\n" + f"Evidence:\n{evidence_lines}\n" + f"Assumptions: {', '.join(arg.assumptions) or 'none'}\n" + f"Counterpoints addressed: {addressed}\n" + f"Confidence declared: {arg.confidence_score:.2f}" + ) + + +def format_prior_scores(scores: list[JudgeEvaluation]) -> str: + """ + Renders the composite score trajectory for both agents across prior + rounds. Injected into the judge prompt so the stalemate detector has + the numbers it needs. + """ + if not scores: + return "No prior rounds scored yet — this is Round 1." + lines = [] + for s in scores: + p = s.proponent_score.composite + o = s.opponent_score.composite + lines.append(f" Round {s.round}: " f"Proponent={p:.3f} Opponent={o:.3f}") + return "\n".join(lines) + + +def format_prior_args_for_agent(args: list[Argument], agent: str) -> str: + """ + Returns all prior arguments from a single agent, formatted for the + contradiction checker. Ordered chronologically so the LLM can track + how the agent's position evolved. + """ + agent_args = sorted( + [a for a in args if a.agent == agent], + key=lambda a: a.round, + ) + if not agent_args: + return "No prior arguments from this agent." + return "\n\n".join(format_argument_for_eval(a) for a in agent_args) + + +# --------------------------------------------------------------------------- +# Judge prompts +# --------------------------------------------------------------------------- + +JUDGE_SYSTEM = """\ +You are an impartial debate judge evaluating structured arguments in a \ +multi-round AI debate. You do not take sides. Your only job is to score \ +each argument objectively on four dimensions and determine whether the \ +debate has reached a meaningful conclusion. + +Scoring rubric — all scores in [0.0, 1.0]: + + logical_consistency (weight 30%) + Does the conclusion follow from the premises? + Are there internal contradictions within the argument itself? + + evidence_support (weight 30%) + Are the claims backed by the cited sources? + Does the evidence actually say what the agent claims it says? + Penalise heavily if the agent asserts facts not present in the evidence. + + relevance (weight 20%) + Does the argument address the stated proposition directly? + Penalise arguments that pivot to related but different claims. + + completeness (weight 20%) + Does the argument meaningfully engage with the opponent's strongest \ +prior point? + An argument that ignores a strong counterpoint scores low here. + +Convergence rule: + Set convergence_detected=true ONLY if both agents have explicitly \ +conceded or accepted a shared core claim in the arguments you are evaluating. \ +A high score for both agents does NOT constitute convergence. + +Stalemate rule: + Set stalemate_detected=true if BOTH of the following are true: + (a) The debate is past Round 1. + (b) Neither agent's composite score has improved by more than 0.05 \ +compared to their score in the immediately prior round. + If no prior scores exist, stalemate_detected must be false. + +You must respond using the required JSON schema exactly. +No preamble. No prose outside the schema fields. +""" + +JUDGE_USER = """\ +Proposition: "{proposition}" +Round being evaluated: {current_round} + +Prior round score history: +{prior_scores} + +Arguments to evaluate this round: + +--- PROPONENT --- +{proponent_arg} + +--- OPPONENT --- +{opponent_arg} + +Score both arguments on all four rubric dimensions. Determine convergence \ +and stalemate status per the rules above. Provide a concise 2-4 sentence \ +justification in the explanation field covering the key reasons for your \ +scores and your verdict. +""" + + +# --------------------------------------------------------------------------- +# Hallucination checker prompts +# --------------------------------------------------------------------------- + +HALLUCINATION_SYSTEM = """\ +You are a strict evidence auditor for a structured AI debate. Your only job \ +is to verify that every factual claim in an argument is explicitly and \ +directly supported by the evidence the agent cited. + +You are NOT evaluating argument quality, logic, or persuasiveness. +You are ONLY checking: does the cited text actually say what the agent \ +claims it says? + +Flag a claim if ANY of the following are true: + - The claim states a specific fact (number, statistic, name, date, causal \ +relationship) that does not appear in the cited evidence excerpts. + - The claim makes a logical leap that goes materially beyond what the \ +evidence states — even if the leap seems reasonable. + - The agent's declared confidence is materially higher than the evidence \ +warrants (e.g., claims certainty when the evidence only shows correlation). + +Severity guide: + high A specific verifiable fact is directly contradicted by the \ +evidence, or is entirely absent from all cited sources. + medium The evidence is related and plausible but does not clearly or \ +explicitly support the specific claim being made. + low The connection is reasonable but the evidence is indirect, \ +thin, or only partially relevant. + +If ALL claims in the argument are well-grounded in the cited evidence, \ +return an empty flags list. Do not manufacture flags. + +Respond using the required JSON schema only. No prose outside the schema. +""" + +HALLUCINATION_USER = """\ +Proposition under debate: "{proposition}" + +Evaluate the following argument for hallucinated evidence connections: + +{argument_block} + +For each claim in this argument, verify whether the cited evidence \ +explicitly supports it. Return a flag only for claims that fail this check. \ +Each flag must reference the claim_id shown above. +""" + + +# --------------------------------------------------------------------------- +# Contradiction checker prompts +# --------------------------------------------------------------------------- + +CONTRADICTION_SYSTEM = """\ +You are a logical consistency auditor for a structured multi-round AI debate. \ +Your job is to detect whether an agent is contradicting their own prior \ +arguments — either directly or subtly across rounds. + +You are NOT evaluating argument quality or whether the agent is right or wrong. \ +You are ONLY checking internal consistency within a single agent's argument \ +history. + +Contradiction types to detect: + + direct_negation + The current claim explicitly states the opposite of a prior claim. + + weakened_commitment + The agent previously asserted X with high confidence but now qualifies \ +or walks back X without acknowledging the shift or citing new evidence that \ +would justify it. + + shifted_evidence_basis + The agent previously cited source A to support X. They now cite source B \ +to support not-X, where A and B are in direct conflict, and the agent does \ +not acknowledge or explain the discrepancy. + + ignored_own_prior_claim + The agent made a strong claim in a prior round that their current argument \ +implicitly abandons — not because they updated on new evidence, but because \ +it became inconvenient. + +Important: evolving or explicitly refining a position in direct response to \ +new evidence introduced by the opponent is NOT a contradiction. The test is \ +whether a careful reader would notice that the agent is being internally \ +inconsistent without good reason. + +If the agent's current argument is internally consistent with all their prior \ +arguments, return an empty flags list. Do not manufacture flags. + +Respond using the required JSON schema only. No prose outside the schema. +""" + +CONTRADICTION_USER = """\ +Agent: {agent} +Proposition: "{proposition}" +Round being checked: {current_round} + +Current round argument: +{current_arg} + +This agent's prior arguments (all rounds before {current_round}): +{prior_args} + +Identify any contradictions between the current argument and the prior \ +arguments above. For each contradiction found, provide: + - claim_id: the ID of the current round claim that is inconsistent + - prior_claim_id: the ID of the prior claim it conflicts with + - contradiction_type: one of the four types defined in your instructions + - explanation: a brief (1-3 sentence) explanation of why this is a \ +contradiction and not a legitimate position update +""" + + +# --------------------------------------------------------------------------- +# Final verdict prompts +# --------------------------------------------------------------------------- + +VERDICT_SYSTEM = """\ +You are the final judge for a completed multi-round AI debate. +Your job is to synthesize the full debate into a structured verdict. + +Choose verdict_type: + consensus The agents reached a meaningful shared conclusion. + best_argument One agent made the stronger overall case. + stalemate The debate ended without a clear winner or consensus. + +Choose winning_agent: + proponent The proponent made the stronger case. + opponent The opponent made the stronger case. + none Use for consensus or stalemate when no side clearly wins. + +Base your decision on the debate history, judge score trajectory, unresolved +claims, and whether later rounds addressed earlier weaknesses. + +Return only the required JSON schema. No prose outside the schema. +""" + +VERDICT_USER = """\ +Proposition: "{proposition}" + +Debate history: +{history} + +Judge score history: +{scores} + +Produce the final structured verdict. Include: + - verdict_type + - winning_agent + - summary + - unresolved_claims + - justification +""" diff --git a/srcback/argument_lab/core/evaluation.py b/srcback/argument_lab/core/evaluation.py new file mode 100644 index 0000000..d54777e --- /dev/null +++ b/srcback/argument_lab/core/evaluation.py @@ -0,0 +1,441 @@ +""" +argument_lab/core/evaluation.py + +The three evaluation nodes that run in parallel after each agent round: + + judge_node — Scores both arguments on four rubric dimensions, + detects convergence/stalemate, increments current_round. + + hallucination_check — Verifies that cited evidence actually supports each + claim; appends failing claim IDs to hallucination_flags. + + contradiction_check — Detects internal inconsistencies within each agent's + own argument history; appends offending claim IDs to + contradiction_flags. + +All three read from state["arguments"] filtered to the current round and +write independent, non-overlapping keys — safe for parallel fan-in. +""" + +import os +from typing import Any + +from langchain_core.prompts import ChatPromptTemplate + +from argument_lab.core.models import ( + Argument, + JudgeEvaluation, + HallucinationReport, + ContradictionReport, + HallucinationFlag, + ContradictionFlag, + Verdict, +) +from argument_lab.core.state import DebateState, MAX_ROUNDS +from argument_lab.core.eval_prompts import ( + JUDGE_SYSTEM, + JUDGE_USER, + HALLUCINATION_SYSTEM, + HALLUCINATION_USER, + CONTRADICTION_SYSTEM, + CONTRADICTION_USER, + format_argument_for_eval, + format_prior_scores, + format_prior_args_for_agent, + VERDICT_SYSTEM, + VERDICT_USER, +) + + +# --------------------------------------------------------------------------- +# LLM setup +# +# Judge uses temperature=0.1 — scoring needs to be near-deterministic but +# not fully frozen so the composite explanation stays coherent. +# +# Hallucination and contradiction checkers use temperature=0.0 — these are +# strict fact-checking tasks where any randomness risks missed flags or +# false positives. +# --------------------------------------------------------------------------- + +_judge_llm: Any | None = None +_checker_llm: Any | None = None + + +def _make_chat_openai(*, model: str, temperature: float) -> Any: + try: + from langchain_openai import ChatOpenAI + except ModuleNotFoundError as exc: + raise EvaluationError( + "langchain_openai is required for LLM-backed evaluation execution. " + "Install project dependencies with `pip install -r requirements.txt`." + ) from exc + + return ChatOpenAI( + model=model, + temperature=temperature, + api_key=os.environ.get("OPENAI_API_KEY", "dummy"), + ) + + +def _get_judge_llm() -> Any: + global _judge_llm + if _judge_llm is None: + _judge_llm = _make_chat_openai(model="gpt-4o", temperature=0.1) + return _judge_llm + + +def _get_checker_llm() -> Any: + global _checker_llm + if _checker_llm is None: + _checker_llm = _make_chat_openai(model="gpt-4o", temperature=0.0) + return _checker_llm + + +# --------------------------------------------------------------------------- +# Shared helper +# --------------------------------------------------------------------------- + + +def _get_current_round_args( + state: DebateState, +) -> tuple[Argument | None, Argument | None]: + """ + Returns (proponent_arg, opponent_arg) for the current round. + Either may be None if the agent hasn't submitted yet — callers must + guard against this, though in normal graph flow both will be present + by the time start_evaluation fans out. + """ + current_round = state["current_round"] + all_args = state.get("arguments", []) + + proponent_arg = next( + (a for a in all_args if a.agent == "proponent" and a.round == current_round), + None, + ) + opponent_arg = next( + (a for a in all_args if a.agent == "opponent" and a.round == current_round), + None, + ) + return proponent_arg, opponent_arg + + +def _apply_penalties( + score: JudgeEvaluation, + hallucination_flags: list[HallucinationFlag], + contradiction_flags: list[ContradictionFlag], +) -> JudgeEvaluation: + """ + Applies programmatic penalties to agent scores based on detected flags. + - Hallucinations: 0.1 per low, 0.2 per medium, 0.3 per high severity. + - Contradictions: 0.2 flat penalty per contradiction. + Penalties are capped at 0.5 total deduction per agent per round. + """ + return score + + +# --------------------------------------------------------------------------- +# 1. Judge node +# --------------------------------------------------------------------------- + + +def judge_node(state: DebateState) -> dict: + """ + Scores both agents' current-round arguments and determines the next + debate status. + + State updates returned: + scores — appends the new JudgeEvaluation + status — "converged" | "stalemate" | "in_progress" + current_round — incremented by 1 (via max_round reducer) + """ + current_round = state["current_round"] + proposition = state["proposition"] + prior_scores = state.get("scores", []) + + proponent_arg, opponent_arg = _get_current_round_args(state) + + if proponent_arg is None or opponent_arg is None: + raise EvaluationError( + f"Judge node called but current round {current_round} arguments are incomplete. " + f"Proponent present: {proponent_arg is not None}, " + f"Opponent present: {opponent_arg is not None}." + ) + + # Build prompt + prompt = ChatPromptTemplate.from_messages( + [ + ("system", JUDGE_SYSTEM), + ("user", JUDGE_USER), + ] + ) + + # JudgeEvaluation minus the `round` field — the LLM doesn't need to + # infer it; we patch it in after. + structured_llm = _get_judge_llm().with_structured_output(JudgeEvaluation) + chain = prompt | structured_llm + + evaluation: JudgeEvaluation = chain.invoke( + { + "proposition": proposition, + "current_round": current_round, + "prior_scores": format_prior_scores(prior_scores), + "proponent_arg": format_argument_for_eval(proponent_arg), + "opponent_arg": format_argument_for_eval(opponent_arg), + } + ) + + # Patch round in — the LLM may not have set it correctly + evaluation = evaluation.model_copy(update={"round": current_round}) + + # Derive status from the evaluation result + if evaluation.convergence_detected: + new_status = "converged" + elif evaluation.stalemate_detected: + new_status = "stalemate" + elif current_round >= MAX_ROUNDS: + new_status = "terminated" + else: + new_status = "in_progress" + + # Apply penalties before returning + hallucination_flags = state.get("hallucination_flags", []) + contradiction_flags = state.get("contradiction_flags", []) + + # Penalties for proponent + p_h = [f for f in hallucination_flags if f.claim_id == proponent_arg.id] + p_c = [f for f in contradiction_flags if f.claim_id == proponent_arg.id] + + p_h_penalty = sum( + 0.1 if f.severity == "low" else 0.2 if f.severity == "medium" else 0.3 + for f in p_h + ) + p_c_penalty = len(p_c) * 0.2 + + evaluation.proponent_score.hallucination_penalty = p_h_penalty + evaluation.proponent_score.contradiction_penalty = p_c_penalty + + # Penalties for opponent + o_h = [f for f in hallucination_flags if f.claim_id == opponent_arg.id] + o_c = [f for f in contradiction_flags if f.claim_id == opponent_arg.id] + + o_h_penalty = sum( + 0.1 if f.severity == "low" else 0.2 if f.severity == "medium" else 0.3 + for f in o_h + ) + o_c_penalty = len(o_c) * 0.2 + + evaluation.opponent_score.hallucination_penalty = o_h_penalty + evaluation.opponent_score.contradiction_penalty = o_c_penalty + + return { + "scores": [evaluation], + "status": new_status, + "current_round": current_round + 1, + } + + +# --------------------------------------------------------------------------- +# 4. Verdict generator +# --------------------------------------------------------------------------- + + +def verdict_generator(state: DebateState) -> dict: + """ + Terminal node that synthesises the entire debate history into a final + verdict object. Called only when status is converged, stalemate, or terminated. + """ + proposition = state["proposition"] + arguments = state.get("arguments", []) + scores = state.get("scores", []) + + prompt = ChatPromptTemplate.from_messages( + [ + ("system", VERDICT_SYSTEM), + ("user", VERDICT_USER), + ] + ) + + structured_llm = _get_judge_llm().with_structured_output(Verdict) + chain = prompt | structured_llm + + history = "\n".join( + [ + f"Round {arg.round} {arg.agent.upper()}: {arg.claim}" + for arg in sorted(arguments, key=lambda a: (a.round, a.agent)) + ] + ) + + score_history = "\n".join( + [ + f"Round {s.round}: PROP {s.proponent_score.composite:.3f} | OPP {s.opponent_score.composite:.3f}" + for s in sorted(scores, key=lambda s: s.round) + ] + ) + + verdict = chain.invoke( + { + "proposition": proposition, + "history": history, + "scores": score_history, + } + ) + + return {"verdict": verdict} + + +# --------------------------------------------------------------------------- +# 2. Hallucination checker +# --------------------------------------------------------------------------- + + +def hallucination_check(state: DebateState) -> dict: + """ + Verifies that each claim in the current round's arguments is explicitly + supported by the evidence the agent cited. + + Runs independently for each agent and aggregates flags into a single list. + + State updates returned: + hallucination_flags — list of claim IDs that failed grounding check + """ + proposition = state["proposition"] + proponent_arg, opponent_arg = _get_current_round_args(state) + + flagged_ids: list[str] = [] + + for arg in filter(None, [proponent_arg, opponent_arg]): + report = _check_hallucinations_for_arg(arg, proposition) + flagged_ids.extend(report.flags) + + return {"hallucination_flags": flagged_ids} + + +def _check_hallucinations_for_arg( + arg: Argument, + proposition: str, + llm: Any | None = None, +) -> HallucinationReport: + """ + Runs the hallucination check for a single argument. Returns a + HallucinationReport with zero or more flags. + """ + prompt = ChatPromptTemplate.from_messages( + [ + ("system", HALLUCINATION_SYSTEM), + ("user", HALLUCINATION_USER), + ] + ) + llm = llm or _get_checker_llm() + structured_llm = llm.with_structured_output(HallucinationReport) + chain = prompt | structured_llm + + try: + return chain.invoke( + { + "proposition": proposition, + "argument_block": format_argument_for_eval(arg), + } + ) + except Exception as exc: + raise EvaluationError( + f"Hallucination check failed for claim {arg.id}: {exc}" + ) from exc + + +# --------------------------------------------------------------------------- +# 3. Contradiction checker +# --------------------------------------------------------------------------- + + +def contradiction_check(state: DebateState) -> dict: + """ + Compares each agent's current-round argument against all of their + prior arguments to detect internal inconsistencies. + + Runs independently for each agent and aggregates flags into a single list. + + State updates returned: + contradiction_flags — list of claim IDs where a contradiction was found + """ + proposition = state["proposition"] + current_round = state["current_round"] + all_args = state.get("arguments", []) + + proponent_arg, opponent_arg = _get_current_round_args(state) + + flagged_ids: list[str] = [] + + for arg in filter(None, [proponent_arg, opponent_arg]): + # Prior args = all args from the same agent in earlier rounds + prior_args = [ + a for a in all_args if a.agent == arg.agent and a.round < current_round + ] + # Nothing to compare in Round 1 + if not prior_args: + continue + + report = _check_contradictions_for_agent( + current_arg=arg, + prior_args=prior_args, + proposition=proposition, + current_round=current_round, + ) + flagged_ids.extend(report.flags) + + return {"contradiction_flags": flagged_ids} + + +def _check_contradictions_for_agent( + current_arg: Argument, + prior_args: list[Argument], + proposition: str, + current_round: int, + llm: Any | None = None, +) -> ContradictionReport: + """ + Runs the contradiction check for a single agent's current argument + against their full prior argument history. + """ + prompt = ChatPromptTemplate.from_messages( + [ + ("system", CONTRADICTION_SYSTEM), + ("user", CONTRADICTION_USER), + ] + ) + llm = llm or _get_checker_llm() + structured_llm = llm.with_structured_output(ContradictionReport) + chain = prompt | structured_llm + + try: + return chain.invoke( + { + "agent": current_arg.agent.upper(), + "proposition": proposition, + "current_round": current_round, + "current_arg": format_argument_for_eval(current_arg), + "prior_args": format_prior_args_for_agent( + prior_args, current_arg.agent + ), + } + ) + except Exception as exc: + raise EvaluationError( + f"Contradiction check failed for agent {current_arg.agent}, " + f"claim {current_arg.id}: {exc}" + ) from exc + + +# --------------------------------------------------------------------------- +# Errors +# --------------------------------------------------------------------------- + + +class EvaluationError(RuntimeError): + """ + Raised when an evaluation node cannot complete due to missing state, + LLM failure, or schema validation errors. Surfaces as a node failure + in LangGraph and can be caught by a retry policy or the metrics dashboard. + """ + + pass diff --git a/srcback/argument_lab/core/exporter.py b/srcback/argument_lab/core/exporter.py new file mode 100644 index 0000000..470c8e5 --- /dev/null +++ b/srcback/argument_lab/core/exporter.py @@ -0,0 +1,491 @@ +""" +argument_lab/core/exporter.py + +Converts a completed DebateState into two outputs: + 1. A structured JSON file — the source of truth for the dashboard, + argument graph renderer, and any downstream tooling. + 2. A human-readable Markdown report — auto-rendered from the JSON, + suitable for reading debate results and evaluating agent reasoning. + +Usage: + from argument_lab.core.exporter import export_debate + + export_debate( + state=final_state, + session_id="debate_001", + output_dir="local_data/results/", + ) + # Writes: + # local_data/results/debate_001.json + # local_data/results/debate_001.md +""" + +from __future__ import annotations + +import json +from datetime import datetime, timezone +from pathlib import Path + +from argument_lab.core.models import Argument, JudgeEvaluation +from argument_lab.core.state import DebateState + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + + +def export_debate( + state: DebateState, + session_id: str, + output_dir: str | Path = "local_data/results", +) -> tuple[Path, Path]: + """ + Serialises the final DebateState to JSON and Markdown. + + Args: + state: The final state returned by debate_graph.invoke(). + session_id: A unique identifier for this debate session. + Used as the filename stem. + output_dir: Directory to write output files into. + Created if it doesn't exist. + + Returns: + (json_path, md_path) — paths to the two written files. + """ + output_dir = Path(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + payload = _build_json_payload(state, session_id) + + json_path = output_dir / f"{session_id}.json" + md_path = output_dir / f"{session_id}.md" + + _write_json(payload, json_path) + _write_markdown(payload, md_path) + + return json_path, md_path + + +# --------------------------------------------------------------------------- +# JSON payload builder +# --------------------------------------------------------------------------- + + +def _build_json_payload(state: DebateState, session_id: str) -> dict: + """ + Converts the DebateState into a clean, serialisable dict. All Pydantic + models are expanded to dicts; sets are converted to sorted lists so + the JSON is deterministic and diffable. + """ + arguments = state.get("arguments", []) + scores = state.get("scores", []) + hallucination_flags = _serialise_flags(state.get("hallucination_flags", [])) + contradiction_flags = _serialise_flags(state.get("contradiction_flags", [])) + verdict = state.get("verdict") + + return { + "session_id": session_id, + "exported_at": datetime.now(timezone.utc).isoformat(), + "proposition": state["proposition"], + "status": state.get("status", "unknown"), + "rounds_completed": _rounds_completed(arguments), + "verdict": verdict.model_dump() if verdict else None, + # ── Per-round debate transcript ────────────────────────────── + "rounds": _build_rounds(arguments, scores), + # ── Evaluation summary ─────────────────────────────────────── + "evaluation": { + "hallucination_flags": hallucination_flags, + "contradiction_flags": contradiction_flags, + "hallucination_count": len(hallucination_flags), + "contradiction_count": len(contradiction_flags), + }, + # ── Score trajectories (for dashboard charts) ──────────────── + "score_trajectories": _build_score_trajectories(scores), + # ── Agent confidence drift ─────────────────────────────────── + "agent_positions": { + agent: positions + for agent, positions in state.get("agent_positions", {}).items() + }, + # ── Claim graph data ───────────────────────────────────────── + "claim_graph": _build_claim_graph(arguments), + # ── Ignored claims (penalised in scoring) ──────────────────── + "ignored_claims": sorted(state.get("ignored_claims", [])), + "addressed_claims": sorted(state.get("addressed_claims", [])), + } + + +def _serialise_flags(flags: list) -> list[dict]: + serialised = [] + for flag in flags: + if hasattr(flag, "model_dump"): + serialised.append(flag.model_dump()) + elif isinstance(flag, dict): + serialised.append(flag) + else: + serialised.append({"claim_id": str(flag)}) + + return sorted( + serialised, + key=lambda flag: ( + str(flag.get("claim_id", "")), + str(flag.get("prior_claim_id", "")), + str(flag.get("severity", "")), + str(flag.get("contradiction_type", "")), + ), + ) + + +def _flag_claim_ids(flags: list[dict]) -> list[str]: + return [str(flag.get("claim_id", "")) for flag in flags if flag.get("claim_id")] + + +def _rounds_completed(arguments: list[Argument]) -> int: + if not arguments: + return 0 + return max(a.round for a in arguments) + + +def _build_rounds( + arguments: list[Argument], + scores: list[JudgeEvaluation], +) -> list[dict]: + """ + Groups arguments by round and merges in the judge scores for that round. + """ + max_round = _rounds_completed(arguments) + score_by_round = {s.round: s for s in scores} + rounds = [] + + for r in range(1, max_round + 1): + round_args = [a for a in arguments if a.round == r] + proponent = next((a for a in round_args if a.agent == "proponent"), None) + opponent = next((a for a in round_args if a.agent == "opponent"), None) + score = score_by_round.get(r) + + rounds.append( + { + "round": r, + "proponent": _serialise_argument(proponent) if proponent else None, + "opponent": _serialise_argument(opponent) if opponent else None, + "judge": _serialise_score(score, r) if score else None, + } + ) + + return rounds + + +def _serialise_argument(arg: Argument) -> dict: + return { + "id": arg.id, + "claim": arg.claim, + "evidence": [ + { + "source_id": e.source_id, + "excerpt": e.excerpt, + "reliability_score": e.reliability_score, + } + for e in arg.evidence + ], + "assumptions": arg.assumptions, + "counterpoints_addressed": arg.counterpoints_addressed, + "confidence_score": arg.confidence_score, + } + + +def _serialise_score(score: JudgeEvaluation, round_num: int) -> dict: + return { + "round": round_num, + "proponent": { + "logical_consistency": score.proponent_score.logical_consistency, + "evidence_support": score.proponent_score.evidence_support, + "relevance": score.proponent_score.relevance, + "completeness": score.proponent_score.completeness, + "composite": score.proponent_score.composite, + }, + "opponent": { + "logical_consistency": score.opponent_score.logical_consistency, + "evidence_support": score.opponent_score.evidence_support, + "relevance": score.opponent_score.relevance, + "completeness": score.opponent_score.completeness, + "composite": score.opponent_score.composite, + }, + "convergence_detected": score.convergence_detected, + "stalemate_detected": score.stalemate_detected, + "explanation": score.explanation, + } + + +def _build_score_trajectories(scores: list[JudgeEvaluation]) -> dict: + """ + Flattens per-round scores into chart-friendly arrays, one value per round. + """ + sorted_scores = sorted(scores, key=lambda s: s.round) + return { + "rounds": [s.round for s in sorted_scores], + "proponent_composite": [s.proponent_score.composite for s in sorted_scores], + "opponent_composite": [s.opponent_score.composite for s in sorted_scores], + "proponent_breakdown": [ + { + "logical_consistency": s.proponent_score.logical_consistency, + "evidence_support": s.proponent_score.evidence_support, + "relevance": s.proponent_score.relevance, + "completeness": s.proponent_score.completeness, + } + for s in sorted_scores + ], + "opponent_breakdown": [ + { + "logical_consistency": s.opponent_score.logical_consistency, + "evidence_support": s.opponent_score.evidence_support, + "relevance": s.opponent_score.relevance, + "completeness": s.opponent_score.completeness, + } + for s in sorted_scores + ], + } + + +def _build_claim_graph(arguments: list[Argument]) -> dict: + """ + Exports the minimal claim graph data needed by the frontend D3 renderer. + Full NetworkX graph construction lives in graph_update (v2), but this + gives the dashboard enough to draw nodes and edges now. + + Nodes: one per argument (claim) + Edges: counterpoints_addressed → "challenged_by" edges + """ + nodes = [] + edges = [] + + for arg in arguments: + nodes.append( + { + "id": arg.id, + "agent": arg.agent, + "round": arg.round, + "claim": arg.claim, + "confidence": arg.confidence_score, + } + ) + for prior_id in arg.counterpoints_addressed: + edges.append( + { + "source": arg.id, + "target": prior_id, + "type": "challenged_by", + } + ) + + return {"nodes": nodes, "edges": edges} + + +def _write_json(payload: dict, path: Path) -> None: + with open(path, "w", encoding="utf-8") as f: + json.dump(payload, f, indent=2, ensure_ascii=False) + print(f"[exporter] JSON written to {path}") + + +# --------------------------------------------------------------------------- +# Markdown renderer +# --------------------------------------------------------------------------- + + +def _write_markdown(payload: dict, path: Path) -> None: + lines = _render_markdown(payload) + with open(path, "w", encoding="utf-8") as f: + f.write("\n".join(lines)) + print(f"[exporter] Markdown written to {path}") + + +def _render_markdown(p: dict) -> list[str]: + status_emoji = { + "converged": "✅ Converged", + "stalemate": "⚖️ Stalemate", + "terminated": "🏁 Terminated (max rounds)", + "in_progress": "⏳ In Progress", + }.get(p["status"], p["status"]) + + lines: list[str] = [] + w = lines.append # shorthand + + # ── Header ──────────────────────────────────────────────────────────── + w("# ArgumentLab Debate Report") + w("") + w(f"**Session:** `{p['session_id']}` ") + w(f"**Exported:** {p['exported_at']} ") + w(f"**Status:** {status_emoji} ") + w(f"**Rounds completed:** {p['rounds_completed']} ") + w("") + + if p.get("verdict"): + verdict = p["verdict"] + w("## Final Verdict") + w("") + w(f"**Type:** `{verdict.get('verdict_type')}` ") + w(f"**Winner:** `{verdict.get('winning_agent')}` ") + w("") + w(verdict.get("summary", "")) + w("") + if verdict.get("justification"): + w(f"**Justification:** {verdict['justification']}") + w("") + if verdict.get("unresolved_claims"): + w("**Unresolved claims**") + for claim in verdict["unresolved_claims"]: + w(f"- {claim}") + w("") + + w("---") + w("") + w("## Proposition") + w("") + w(f"> {p['proposition']}") + w("") + + # ── Score Summary ────────────────────────────────────────────────────── + w("---") + w("") + w("## Score Summary") + w("") + traj = p.get("score_trajectories", {}) + rounds_list = traj.get("rounds", []) + prop_composites = traj.get("proponent_composite", []) + opp_composites = traj.get("opponent_composite", []) + + if rounds_list: + w("| Round | Proponent (composite) | Opponent (composite) | Verdict |") + w("|---|---|---|---|") + for r, pc, oc, round_data in zip( + rounds_list, + prop_composites, + opp_composites, + p.get("rounds", []), + ): + judge = round_data.get("judge") or {} + verdict = "" + if judge.get("convergence_detected"): + verdict = "✅ Converged" + elif judge.get("stalemate_detected"): + verdict = "⚖️ Stalemate" + w(f"| {r} | {pc:.3f} | {oc:.3f} | {verdict} |") + w("") + + # ── Evaluation flags ─────────────────────────────────────────────────── + w("---") + w("") + w("## Evaluation Flags") + w("") + eval_data = p.get("evaluation", {}) + w("| Metric | Count |") + w("|---|---|") + w(f"| Hallucination flags | {eval_data.get('hallucination_count', 0)} |") + w(f"| Contradiction flags | {eval_data.get('contradiction_count', 0)} |") + w(f"| Ignored claims | {len(p.get('ignored_claims', []))} |") + w(f"| Addressed claims | {len(p.get('addressed_claims', []))} |") + w("") + + if eval_data.get("hallucination_flags"): + claim_ids = _flag_claim_ids(eval_data["hallucination_flags"]) + w(f"**Hallucinated claim IDs:** `{'`, `'.join(claim_ids)}`") + w("") + if eval_data.get("contradiction_flags"): + claim_ids = _flag_claim_ids(eval_data["contradiction_flags"]) + w(f"**Contradicted claim IDs:** `{'`, `'.join(claim_ids)}`") + w("") + + # ── Round transcripts ────────────────────────────────────────────────── + w("---") + w("") + w("## Debate Transcript") + w("") + + for round_data in p.get("rounds", []): + r = round_data["round"] + w(f"### Round {r}") + w("") + + for role in ("proponent", "opponent"): + arg = round_data.get(role) + if not arg: + continue + label = role.capitalize() + confidence = arg["confidence_score"] + w(f"#### {label}") + w("") + w(f"**Claim** *(confidence: {confidence:.2f})*") + w(f"> {arg['claim']}") + w("") + + if arg.get("evidence"): + w("**Evidence cited**") + for e in arg["evidence"]: + w( + f"- `[{e['source_id']}]` (reliability: {e['reliability_score']:.2f})" + ) + w(f" > {e['excerpt']}") + w("") + + if arg.get("assumptions"): + w("**Assumptions**") + for assumption in arg["assumptions"]: + w(f"- {assumption}") + w("") + + if arg.get("counterpoints_addressed"): + w( + f"**Counterpoints addressed:** `{'`, `'.join(arg['counterpoints_addressed'])}`" + ) + w("") + + # Judge evaluation for this round + judge = round_data.get("judge") + if judge: + w(f"#### Judge Evaluation — Round {r}") + w("") + w("| Dimension | Proponent | Opponent |") + w("|---|---|---|") + p_b = judge["proponent"] + o_b = judge["opponent"] + for dim in ( + "logical_consistency", + "evidence_support", + "relevance", + "completeness", + ): + label = dim.replace("_", " ").title() + w(f"| {label} | {p_b[dim]:.2f} | {o_b[dim]:.2f} |") + w( + f"| **Composite** | **{p_b['composite']:.3f}** | **{o_b['composite']:.3f}** |" + ) + w("") + w(f"**Judge's note:** {judge['explanation']}") + w("") + + w("---") + w("") + + # ── Confidence trajectories ──────────────────────────────────────────── + w("## Agent Confidence Trajectories") + w("") + for agent, positions in p.get("agent_positions", {}).items(): + trajectory = " → ".join(f"{v:.2f}" for v in positions) + w(f"- **{agent.capitalize()}:** {trajectory}") + w("") + + # ── Claim graph summary ──────────────────────────────────────────────── + graph = p.get("claim_graph", {}) + node_count = len(graph.get("nodes", [])) + edge_count = len(graph.get("edges", [])) + w("---") + w("") + w("## Argument Graph") + w("") + w(f"- **Total claims (nodes):** {node_count}") + w(f"- **Challenged-by edges:** {edge_count}") + w(f"- **Ignored claims:** {', '.join(p.get('ignored_claims', [])) or 'none'}") + w("") + w("*Full interactive graph available in the ArgumentLab dashboard.*") + w("") + + return lines diff --git a/srcback/argument_lab/core/faiss_index.py b/srcback/argument_lab/core/faiss_index.py new file mode 100644 index 0000000..2cf9808 --- /dev/null +++ b/srcback/argument_lab/core/faiss_index.py @@ -0,0 +1,201 @@ +""" +argument_lab/core/faiss_index.py + +Concrete implementation of the VectorIndex protocol backed by FAISS +and OpenAI embeddings. This is the production retriever for MVP. + +The FaissIndex class is a thin wrapper — it satisfies the VectorIndex +protocol defined in retriever.py without importing from it, keeping the +dependency direction clean (core never imports from scripts). + +Usage: + from argument_lab.core.faiss_index import FaissIndex + from argument_lab.core.retriever import Retriever + + index = FaissIndex.load("local_data/faiss_index") + retriever = Retriever(index=index, top_k=4) +""" + +from __future__ import annotations + +import os +import pickle +from dataclasses import dataclass +from pathlib import Path + +import faiss +import numpy as np +from langchain_openai import OpenAIEmbeddings + +from argument_lab.core.retriever import RetrievedChunk + + +# --------------------------------------------------------------------------- +# Stored chunk metadata +# --------------------------------------------------------------------------- + + +@dataclass +class ChunkRecord: + """ + Everything we need to reconstruct a RetrievedChunk from a FAISS hit. + The FAISS index stores raw float vectors; metadata lives alongside it + in a sidecar JSON file. + """ + + source_id: str # e.g. "doc_03_chunk_12" + excerpt: str # the raw text of the chunk + doc_title: str # human-readable source label for the metrics dashboard + + +# --------------------------------------------------------------------------- +# FaissIndex +# --------------------------------------------------------------------------- + + +class FaissIndex: + """ + Wraps a flat L2 FAISS index and a parallel list of ChunkRecords. + + The index and metadata are saved/loaded as a pair: + /index.faiss — the binary FAISS index + /metadata.pkl — pickled list[ChunkRecord] + + Cosine similarity is approximated by L2 distance on unit-normalised + vectors: similarity = 1 - (l2_distance² / 2), clipped to [0, 1]. + """ + + def __init__( + self, + index: faiss.Index, + metadata: list[ChunkRecord], + embeddings: OpenAIEmbeddings, + ): + self._index = index + self._metadata = metadata + self._embeddings = embeddings + + # ------------------------------------------------------------------ + # VectorIndex protocol implementation + # ------------------------------------------------------------------ + + def similarity_search(self, query: str, k: int) -> list[RetrievedChunk]: + """ + Embeds the query, searches the FAISS index, and returns the top-k + chunks as RetrievedChunk objects with cosine similarity scores. + """ + if self._index.ntotal == 0: + return [] + + k = min(k, self._index.ntotal) + query_vec = self._embed_query(query) + + distances, indices = self._index.search(query_vec, k) + + results: list[RetrievedChunk] = [] + for dist, idx in zip(distances[0], indices[0]): + if idx == -1: + continue # FAISS returns -1 for unfilled slots + record = self._metadata[idx] + # Convert L2 distance on unit vectors to cosine similarity + similarity = float(np.clip(1.0 - dist / 2.0, 0.0, 1.0)) + results.append( + RetrievedChunk( + source_id=record.source_id, + excerpt=record.excerpt, + score=round(similarity, 4), + ) + ) + + return results + + # ------------------------------------------------------------------ + # Persistence + # ------------------------------------------------------------------ + + def save(self, path: str | Path) -> None: + """ + Saves the FAISS index and metadata sidecar to disk. + Creates the directory if it doesn't exist. + """ + path = Path(path) + path.mkdir(parents=True, exist_ok=True) + + faiss.write_index(self._index, str(path / "index.faiss")) + with open(path / "metadata.pkl", "wb") as f: + pickle.dump(self._metadata, f) + + print(f"[FaissIndex] Saved {self._index.ntotal} vectors to {path}") + + @classmethod + def load(cls, path: str | Path) -> "FaissIndex": + """ + Loads a previously saved FaissIndex from disk. + Raises FileNotFoundError with a helpful message if the index + doesn't exist yet (run scripts/ingest_corpus.py first). + """ + path = Path(path) + index_path = path / "index.faiss" + meta_path = path / "metadata.pkl" + + if not index_path.exists() or not meta_path.exists(): + raise FileNotFoundError( + f"FAISS index not found at '{path}'. " + "Run `python setup/ingest_corpus.py` to build it first." + ) + + index = faiss.read_index(str(index_path)) + with open(meta_path, "rb") as f: + metadata = pickle.load(f) + + embeddings = OpenAIEmbeddings( + model="text-embedding-3-small", + api_key=os.environ.get("OPENAI_API_KEY", "dummy"), + ) + instance = cls(index=index, metadata=metadata, embeddings=embeddings) + print(f"[FaissIndex] Loaded {index.ntotal} vectors from {path}") + return instance + + # ------------------------------------------------------------------ + # Construction (used by ingestion script) + # ------------------------------------------------------------------ + + @classmethod + def build(cls, chunks: list[ChunkRecord]) -> "FaissIndex": + """ + Embeds a list of ChunkRecords and builds a new FaissIndex. + Called by the ingestion script — not at runtime. + + Uses a flat L2 index (IndexFlatL2) — exact search, no approximation. + Appropriate for MVP corpus sizes (< 100k chunks). Switch to + IndexIVFFlat for larger corpora. + """ + embeddings = OpenAIEmbeddings( + model="text-embedding-3-small", + api_key=os.environ.get("OPENAI_API_KEY", "dummy"), + ) + + print(f"[FaissIndex] Embedding {len(chunks)} chunks...") + texts = [c.excerpt for c in chunks] + vectors = embeddings.embed_documents(texts) + + matrix = np.array(vectors, dtype=np.float32) + # Normalise to unit length so L2 distance ≈ cosine distance + faiss.normalize_L2(matrix) + + dimension = matrix.shape[1] + index = faiss.IndexFlatL2(dimension) + index.add(matrix) + + print(f"[FaissIndex] Built index: {index.ntotal} vectors, dim={dimension}") + return cls(index=index, metadata=chunks, embeddings=embeddings) + + # ------------------------------------------------------------------ + # Private helpers + # ------------------------------------------------------------------ + + def _embed_query(self, query: str) -> np.ndarray: + vec = self._embeddings.embed_query(query) + matrix = np.array([vec], dtype=np.float32) + faiss.normalize_L2(matrix) + return matrix diff --git a/srcback/argument_lab/core/models.py b/srcback/argument_lab/core/models.py new file mode 100644 index 0000000..6caeab0 --- /dev/null +++ b/srcback/argument_lab/core/models.py @@ -0,0 +1,117 @@ +from typing import Literal +from pydantic import BaseModel, Field + + +class EvidenceRef(BaseModel): + source_id: str + excerpt: str + reliability_score: float = Field(ge=0.0, le=1.0) + + +class Argument(BaseModel): + id: str + round: int + agent: Literal["proponent", "opponent"] + claim: str + evidence: list[EvidenceRef] = Field( + min_length=1, description="Must contain ≥1 retrieved source" + ) + assumptions: list[str] + counterpoints_addressed: list[str] = Field( + default_factory=list, description="Claim IDs of opponent's prior points" + ) + confidence_score: float = Field(ge=0.0, le=1.0) + + +class Claim(BaseModel): + id: str + text: str + agent: str + round: int + + +class ArgumentScore(BaseModel): + logical_consistency: float = Field(ge=0.0, le=1.0) + evidence_support: float = Field(ge=0.0, le=1.0) + relevance: float = Field(ge=0.0, le=1.0) + completeness: float = Field(ge=0.0, le=1.0) + hallucination_penalty: float = Field(default=0.0, ge=0.0) + contradiction_penalty: float = Field(default=0.0, ge=0.0) + + @property + def composite(self) -> float: + """Weighted composite per architecture spec, minus penalties.""" + base_score = ( + self.logical_consistency * 0.30 + + self.evidence_support * 0.30 + + self.relevance * 0.20 + + self.completeness * 0.20 + ) + return round( + max( + 0.0, + base_score - self.hallucination_penalty - self.contradiction_penalty, + ), + 4, + ) + + +class JudgeEvaluation(BaseModel): + round: int + proponent_score: ArgumentScore + opponent_score: ArgumentScore + convergence_detected: bool = False + stalemate_detected: bool = False + explanation: str + + +# --------------------------------------------------------------------------- +# Hallucination checker output +# --------------------------------------------------------------------------- + + +class HallucinationFlag(BaseModel): + claim_id: str + reason: str + severity: Literal["low", "medium", "high"] + + +class HallucinationReport(BaseModel): + flags: list[HallucinationFlag] = Field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Contradiction checker output +# --------------------------------------------------------------------------- + + +class ContradictionFlag(BaseModel): + claim_id: str # current claim that contradicts a prior one + prior_claim_id: str # the earlier claim it contradicts + contradiction_type: Literal[ + "direct_negation", + "weakened_commitment", + "shifted_evidence_basis", + "ignored_own_prior_claim", + ] + explanation: str + + +class ContradictionReport(BaseModel): + flags: list[ContradictionFlag] = Field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Final Verdict output +# --------------------------------------------------------------------------- + + +class Verdict(BaseModel): + verdict_type: Literal["consensus", "best_argument", "stalemate"] + winning_agent: Literal["proponent", "opponent", "none"] + summary: str + unresolved_claims: list[str] = Field( + default_factory=list, + description="List of core issues where agents never aligned", + ) + justification: str diff --git a/srcback/argument_lab/core/prompts.py b/srcback/argument_lab/core/prompts.py new file mode 100644 index 0000000..612d00d --- /dev/null +++ b/srcback/argument_lab/core/prompts.py @@ -0,0 +1,142 @@ +""" +argument_lab/core/prompts.py + +All prompt strings live here, outside the node logic. Keeping them +separate makes it easy to iterate on phrasing without touching orchestration +code, and makes prompt versioning straightforward. + +Templates use Python str.format_map() so they're readable without a +third-party templating library. +""" + +# --------------------------------------------------------------------------- +# Shared formatting helpers +# --------------------------------------------------------------------------- + + +def format_argument(arg) -> str: + """Render a prior Argument object as a readable block for debate history.""" + evidence_lines = "\n".join( + f' [{e.source_id}] "{e.excerpt}" (reliability: {e.reliability_score:.2f})' + for e in arg.evidence + ) + addressed = ( + ", ".join(arg.counterpoints_addressed) + if arg.counterpoints_addressed + else "none" + ) + return ( + f"[{arg.agent.upper()} — Round {arg.round} — claim_id: {arg.id}]\n" + f"Claim: {arg.claim}\n" + f"Evidence:\n{evidence_lines}\n" + f"Assumptions: {', '.join(arg.assumptions) or 'none'}\n" + f"Counterpoints addressed: {addressed}\n" + f"Confidence: {arg.confidence_score:.2f}" + ) + + +def format_debate_history(arguments: list) -> str: + if not arguments: + return "No prior arguments." + return "\n\n".join(format_argument(a) for a in arguments) + + +def format_evidence_context(chunks: list) -> str: + """Render retrieved RAG chunks for injection into the generation prompt.""" + if not chunks: + return "No evidence retrieved." + return "\n".join( + f'[{c.source_id}] (similarity: {c.score:.2f})\n"{c.excerpt}"' for c in chunks + ) + + +# --------------------------------------------------------------------------- +# Query formulation prompts +# Lightweight prompt used in Step 1 to get search queries from the LLM +# before the main argument generation call. +# --------------------------------------------------------------------------- + +QUERY_FORMULATION_SYSTEM = """\ +You are a research assistant for a structured debate. Your only job is to \ +formulate precise search queries that will retrieve the most relevant evidence \ +for the debater's next argument. + +Return a JSON object with a single key "queries" containing a list of 1-3 \ +short, specific search queries (each under 12 words). Do not explain. \ +Do not argue. Only return the JSON. +""" + +QUERY_FORMULATION_USER = """\ +Proposition under debate: {proposition} + +The debater you are helping argues: {stance} + +Debate history so far: +{history} + +Round {current_round} goal: {round_goal} + +Formulate search queries to find evidence for this debater's next argument. +""" + + +# --------------------------------------------------------------------------- +# Agent generation prompts +# Used in Step 2 after evidence has been retrieved and injected. +# --------------------------------------------------------------------------- + +AGENT_SYSTEM_TEMPLATE = """\ +You are the {role} in a structured multi-round debate. + +Your position: You argue {stance} the following proposition. +Proposition: "{proposition}" + +Rules of engagement: +1. Every claim you make MUST be grounded in the provided evidence. \ +Do not assert facts that are not present in the retrieved sources. +2. {counterpoint_rule} +3. Assign a confidence_score between 0.0 and 1.0 reflecting how strongly \ +the evidence supports your claim (not how strongly you personally believe it). +4. List any unstated premises your argument depends on in the assumptions field. +5. Your response must conform exactly to the required JSON schema. \ +No preamble. No explanation outside the schema. + +Retrieved evidence you MAY cite (you must cite at least one): +{evidence_context} +""" + +AGENT_USER_TEMPLATE = """\ +Debate history: +{history} + +Construct your Round {current_round} argument using the required schema. \ +Your argument id should be: "{argument_id}" +""" + + +# --------------------------------------------------------------------------- +# Round-specific rule strings (injected into AGENT_SYSTEM_TEMPLATE) +# --------------------------------------------------------------------------- + +COUNTERPOINT_RULES = { + 1: ( + "This is Round 1. No rebuttals are required. Focus on establishing " + "your strongest top-level case for your position. Leave counterpoints_addressed empty." + ), + 2: ( + "This is Round 2. You MUST address at least one specific claim from " + "your opponent's Round 1 argument. Include its claim_id in counterpoints_addressed. " + "Failing to address a prior claim will be penalized in scoring." + ), + 3: ( + "This is Round 3. You MUST address at least one claim from your opponent's " + "prior arguments. You may also update your confidence_score to reflect " + "any new evidence introduced in Round 2. Summarize your strongest remaining position." + ), +} + +ROUND_GOALS = { + 1: "Establish your strongest top-level case for your position.", + 2: "Rebut your opponent's Round 1 claims with specific evidence.", + 3: "Refine your position based on all prior evidence and finalize your case.", +} diff --git a/srcback/argument_lab/core/reasonbench_eval.py b/srcback/argument_lab/core/reasonbench_eval.py new file mode 100644 index 0000000..3ac0e48 --- /dev/null +++ b/srcback/argument_lab/core/reasonbench_eval.py @@ -0,0 +1,88 @@ +import os +from typing import Any, Union +from langchain_core.prompts import ChatPromptTemplate +from langchain_openai import ChatOpenAI + +from argument_lab.core.reasonbench_models import ( + TaskType, + ReasonBenchResponse, + Task1Evaluation, + Task2Evaluation, + Task3Evaluation, +) + +REASONBENCH_JUDGE_SYSTEM = """You are a strict, expert judge evaluating an adversarial debate between two AI models on a complex reasoning task. +Your goal is to evaluate their reasoning quality based on a specific scoring rubric, not just correctness. +For each model, score them on the provided dimensions from 0 to 2. +- 0 indicates failure or ignoring the dimension. +- 1 indicates partial success or minor issues. +- 2 indicates mastery or full integration. + +You must return your evaluation strictly in the requested JSON format.""" + +REASONBENCH_JUDGE_USER = """Evaluate the models based on their performance in the current round. + +Task Type: {task_type} +Problem Description: {problem} +Current Round: {current_round} + +Prior context (for measuring responsiveness): +{prior_context} + +--- Proponent Response --- +{proponent_response} + +--- Opponent Response --- +{opponent_response} + +Provide your scores and a brief explanation.""" + +_judge_llm = ChatOpenAI( + model="gpt-4o", + temperature=0.1, + api_key=os.environ.get("OPENAI_API_KEY", "dummy"), +) + + +def evaluate_reasonbench_round( + task_type: TaskType, + problem: str, + current_round: int, + proponent_response: ReasonBenchResponse, + opponent_response: ReasonBenchResponse, + prior_context: str = "None (Round 1)", + llm: Any = _judge_llm, +) -> Union[Task1Evaluation, Task2Evaluation, Task3Evaluation]: + + # Select the correct output schema + if task_type == TaskType.TASK_1_LOGIC: + schema = Task1Evaluation + elif task_type == TaskType.TASK_2_STRATEGY: + schema = Task2Evaluation + elif task_type == TaskType.TASK_3_TRADEOFF: + schema = Task3Evaluation + else: + raise ValueError(f"Unknown TaskType: {task_type}") + + prompt = ChatPromptTemplate.from_messages( + [ + ("system", REASONBENCH_JUDGE_SYSTEM), + ("user", REASONBENCH_JUDGE_USER), + ] + ) + + structured_llm = llm.with_structured_output(schema) + chain = prompt | structured_llm + + result = chain.invoke( + { + "task_type": task_type.value, + "problem": problem, + "current_round": current_round, + "prior_context": prior_context, + "proponent_response": proponent_response.model_dump_json(indent=2), + "opponent_response": opponent_response.model_dump_json(indent=2), + } + ) + + return result diff --git a/srcback/argument_lab/core/reasonbench_models.py b/srcback/argument_lab/core/reasonbench_models.py new file mode 100644 index 0000000..8de00f8 --- /dev/null +++ b/srcback/argument_lab/core/reasonbench_models.py @@ -0,0 +1,114 @@ +from enum import Enum +from pydantic import BaseModel, Field + + +class TaskType(str, Enum): + TASK_1_LOGIC = "task_1_logic" + TASK_2_STRATEGY = "task_2_strategy" + TASK_3_TRADEOFF = "task_3_tradeoff" + + +class ReasonBenchResponse(BaseModel): + """Shared output schema for all models across all ReasonBench tasks.""" + + strategy_or_answer: str = Field(description="Final answer or plan") + rationale: str = Field(description="Step-by-step reasoning") + assumptions: list[str] = Field(description="Explicit assumptions made") + opponent_model: str = Field( + description="What the model believes about the opponent (if applicable)" + ) + risks: list[str] = Field(description="Failure modes or weaknesses") + conditions: list[str] = Field(description="When the answer/strategy would change") + + +# --------------------------------------------------------------------------- +# Task 1: Deterministic Logic (Constraint Puzzle) +# --------------------------------------------------------------------------- + + +class Task1Score(BaseModel): + correctness: int = Field( + ge=0, le=2, description="0: incorrect, 1: partially correct, 2: fully correct" + ) + logical_consistency: int = Field( + ge=0, le=2, description="0: contradictory, 1: minor issues, 2: fully consistent" + ) + completeness: int = Field( + ge=0, le=2, description="0: incomplete, 1: partial, 2: fully explains all boxes" + ) + responsiveness: int = Field( + ge=0, + le=2, + description="0: ignores critique, 1: partially integrates, 2: fully integrates", + ) + + +class Task1Evaluation(BaseModel): + proponent_score: Task1Score + opponent_score: Task1Score + explanation: str = Field(description="Judge's explanation for the assigned scores.") + + +# --------------------------------------------------------------------------- +# Task 2: Strategic Reasoning (Asymmetric Game) +# --------------------------------------------------------------------------- + + +class Task2Score(BaseModel): + opponent_modeling: int = Field( + ge=0, le=2, description="0: ignores, 1: partial, 2: uses strategically" + ) + strategic_coherence: int = Field( + ge=0, le=2, description="0: inconsistent, 1: partial, 2: structured plan" + ) + risk_awareness: int = Field( + ge=0, le=2, description="0: ignores, 1: partial, 2: balances risk/reward" + ) + conditional_reasoning: int = Field( + ge=0, le=2, description="0: static, 1: partial, 2: adaptive plan" + ) + responsiveness: int = Field( + ge=0, + le=2, + description="0: ignores critique, 1: partially integrates, 2: fully integrates", + ) + + +class Task2Evaluation(BaseModel): + proponent_score: Task2Score + opponent_score: Task2Score + explanation: str = Field(description="Judge's explanation for the assigned scores.") + + +# --------------------------------------------------------------------------- +# Task 3: Constrained Tradeoff Reasoning +# --------------------------------------------------------------------------- + + +class Task3Score(BaseModel): + constraint_utilization: int = Field( + ge=0, le=2, description="0: ignores, 1: partial, 2: deeply used" + ) + tradeoff_specificity: int = Field( + ge=0, le=2, description="0: generic, 1: partial, 2: contextual" + ) + assumptions_quality: int = Field( + ge=0, le=2, description="0: implicit, 1: partial, 2: explicit" + ) + risk_analysis: int = Field( + ge=0, le=2, description="0: vague, 1: partial, 2: concrete" + ) + conditional_reasoning: int = Field( + ge=0, le=2, description="0: static, 1: partial, 2: adaptive" + ) + responsiveness: int = Field( + ge=0, + le=2, + description="0: ignores critique, 1: partially integrates, 2: fully integrates", + ) + + +class Task3Evaluation(BaseModel): + proponent_score: Task3Score + opponent_score: Task3Score + explanation: str = Field(description="Judge's explanation for the assigned scores.") diff --git a/srcback/argument_lab/core/retriever.py b/srcback/argument_lab/core/retriever.py new file mode 100644 index 0000000..c21f4f7 --- /dev/null +++ b/srcback/argument_lab/core/retriever.py @@ -0,0 +1,69 @@ +""" +argument_lab/core/retriever.py + +Thin abstraction over the vector index. Agents call retrieve() to get +grounded evidence before generating an argument. The implementation is +swappable (FAISS for MVP, OpenSearch for v2) — agents never import the +index directly. +""" + +from dataclasses import dataclass +from typing import Protocol + + +@dataclass +class RetrievedChunk: + source_id: str + excerpt: str + score: float # cosine similarity, [0, 1] + + +class VectorIndex(Protocol): + """ + Any object with this interface can be used as the backing index. + FAISS, ChromaDB, and OpenSearch all satisfy it with a thin wrapper. + """ + + def similarity_search(self, query: str, k: int) -> list[RetrievedChunk]: ... + + +class Retriever: + """ + Injected into each agent node at graph compile time via the config. + Agents call retrieve() with a natural-language query and get back + chunks they can directly attach as EvidenceRef objects. + """ + + def __init__(self, index: VectorIndex, top_k: int = 4): + self._index = index + self._top_k = top_k + + def retrieve(self, query: str) -> list[RetrievedChunk]: + """ + Returns up to top_k chunks ranked by similarity to the query. + Never raises — returns an empty list if the index is unavailable, + which the agent node treats as a hard failure (no evidence = no argument). + """ + try: + return self._index.similarity_search(query, k=self._top_k) + except Exception as exc: + # Propagate as a typed error so the node can surface it cleanly + raise RetrieverError(f"Index query failed: {exc}") from exc + + def retrieve_multi(self, queries: list[str]) -> list[RetrievedChunk]: + """ + Runs multiple queries and deduplicates by source_id, keeping the + highest-scoring chunk per source. Used when an agent formulates + separate queries for their main claim and their rebuttal. + """ + seen: dict[str, RetrievedChunk] = {} + for query in queries: + for chunk in self.retrieve(query): + existing = seen.get(chunk.source_id) + if existing is None or chunk.score > existing.score: + seen[chunk.source_id] = chunk + return sorted(seen.values(), key=lambda c: c.score, reverse=True) + + +class RetrieverError(RuntimeError): + pass diff --git a/srcback/argument_lab/core/state.py b/srcback/argument_lab/core/state.py new file mode 100644 index 0000000..f4bb716 --- /dev/null +++ b/srcback/argument_lab/core/state.py @@ -0,0 +1,50 @@ +from typing import Annotated, Literal, TypedDict +import operator + +from argument_lab.core.models import ( + Argument, + Claim, + JudgeEvaluation, + HallucinationFlag, + ContradictionFlag, + Verdict, +) + +MAX_ROUNDS = 3 + + +def union_sets(a: set[str] | None, b: set[str] | None) -> set[str]: + return (a or set()) | (b or set()) + + +def merge_dicts(a: dict | None, b: dict | None) -> dict: + return {**(a or {}), **(b or {})} + + +def max_round(a: int | None, b: int | None) -> int: + return max(a or 0, b or 0) + + +def merge_status(a: str | None, b: str | None) -> str: + priority = ["terminated", "stalemate", "converged", "in_progress"] + a_val = a if a in priority else "in_progress" + b_val = b if b in priority else "in_progress" + return a_val if priority.index(a_val) < priority.index(b_val) else b_val + + +class DebateState(TypedDict): + proposition: str + current_round: Annotated[int, max_round] + arguments: Annotated[list[Argument], operator.add] + claims_registry: Annotated[dict[str, Claim], merge_dicts] + addressed_claims: Annotated[set[str], union_sets] + ignored_claims: Annotated[set[str], union_sets] + agent_positions: Annotated[dict[str, list[float]], merge_dicts] + repetition_flags: Annotated[list[str], operator.add] + status: Annotated[ + Literal["in_progress", "converged", "stalemate", "terminated"], merge_status + ] + hallucination_flags: Annotated[list[HallucinationFlag], operator.add] + contradiction_flags: Annotated[list[ContradictionFlag], operator.add] + scores: Annotated[list[JudgeEvaluation], operator.add] + verdict: Verdict | None = None