From b2d3a4945145172384c9c19bf9ee5578dc8219b1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 26 Feb 2026 00:02:45 +0000 Subject: [PATCH 1/2] Initial plan From 003cbb27a175de65e3e84bce874f59afbb7493d2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 26 Feb 2026 00:10:20 +0000 Subject: [PATCH 2/2] Add integration tests, CI workflow, and config generation script Co-authored-by: harsha-simhadri <5590673+harsha-simhadri@users.noreply.github.com> --- .github/workflows/ci.yml | 49 +++++++ scripts/generate_config.py | 89 ++++++++++++ tests/conftest.py | 67 +++++++++ tests/test_integration.py | 288 +++++++++++++++++++++++++++++++++++++ tests/test_questions.json | 17 +++ 5 files changed, 510 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 scripts/generate_config.py create mode 100644 tests/conftest.py create mode 100644 tests/test_integration.py create mode 100644 tests/test_questions.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d007aa7 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,49 @@ +name: Integration Tests + +on: + pull_request: + branches: + - main + push: + branches: + - main + +jobs: + integration-tests: + name: Run Integration Tests + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: "pip" + + - name: Install dependencies + run: pip install -r requirements.txt pytest pytest-asyncio + + - name: Generate config.yaml from secrets + env: + COSMOS_URI: ${{ secrets.COSMOS_URI }} + COSMOS_KEY: ${{ secrets.COSMOS_KEY }} + COSMOS_DATABASE_NAME: ${{ secrets.COSMOS_DATABASE_NAME }} + COSMOS_SOURCE1_CONTAINER: ${{ secrets.COSMOS_SOURCE1_CONTAINER }} + COSMOS_SOURCE2_CONTAINER: ${{ secrets.COSMOS_SOURCE2_CONTAINER }} + LLM_ENDPOINT: ${{ secrets.LLM_ENDPOINT }} + LLM_API_KEY: ${{ secrets.LLM_API_KEY }} + LLM_MODEL: ${{ secrets.LLM_MODEL }} + EMBED_ENDPOINT: ${{ secrets.EMBED_ENDPOINT }} + EMBED_API_KEY: ${{ secrets.EMBED_API_KEY }} + EMBED_MODEL: ${{ secrets.EMBED_MODEL }} + EMBED_DIMENSIONS: ${{ secrets.EMBED_DIMENSIONS }} + run: python scripts/generate_config.py + + - name: Run integration tests + run: pytest tests/ -v --tb=short diff --git a/scripts/generate_config.py b/scripts/generate_config.py new file mode 100644 index 0000000..38d73a6 --- /dev/null +++ b/scripts/generate_config.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +"""Generate config.yaml from config.yaml.example by substituting environment variables. + +Environment variables consumed +------------------------------- +COSMOS_URI – Cosmos DB account URI +COSMOS_KEY – Cosmos DB primary key +COSMOS_DATABASE_NAME – database name (default: divdet) +COSMOS_SOURCE1_CONTAINER – container name for source_1 (default: container_1) +COSMOS_SOURCE2_CONTAINER – container name for source_2 (default: container_2) +LLM_ENDPOINT – Azure OpenAI LLM endpoint URL +LLM_API_KEY – Azure OpenAI LLM API key +LLM_MODEL – LLM deployment/model name +EMBED_ENDPOINT – Azure OpenAI embedding endpoint URL +EMBED_API_KEY – Azure OpenAI embedding API key +EMBED_MODEL – Embedding deployment/model name +EMBED_DIMENSIONS – Embedding output dimensions (default: 1024) +""" +import os +import sys +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).parent.parent +EXAMPLE_PATH = REPO_ROOT / "config.yaml.example" +OUTPUT_PATH = REPO_ROOT / "config.yaml" + + +def _require(name: str) -> str: + value = os.environ.get(name, "").strip() + if not value: + print(f"ERROR: required environment variable '{name}' is not set or empty.", file=sys.stderr) + sys.exit(1) + return value + + +def _optional(name: str, default: str = "") -> str: + return os.environ.get(name, default).strip() or default + + +def main(): + if not EXAMPLE_PATH.exists(): + print(f"ERROR: {EXAMPLE_PATH} not found.", file=sys.stderr) + sys.exit(1) + + with open(EXAMPLE_PATH) as fh: + cfg = yaml.safe_load(fh) + + # --- Cosmos DB ----------------------------------------------------------- + cfg["cosmos"]["uri"] = _require("COSMOS_URI") + cfg["cosmos"]["key"] = _require("COSMOS_KEY") + cfg["cosmos"]["database_name"] = _optional("COSMOS_DATABASE_NAME", "divdet") + + # Patch container names inside sources list + container_overrides = { + "source_1": _optional("COSMOS_SOURCE1_CONTAINER", "container_1"), + "source_2": _optional("COSMOS_SOURCE2_CONTAINER", "container_2"), + } + for source in cfg.get("cosmos", {}).get("sources", []): + source_id = source.get("id", "") + if source_id in container_overrides: + source["container_name"] = container_overrides[source_id] + + # --- LLM ----------------------------------------------------------------- + cfg["llm"]["llm_endpoint"] = _require("LLM_ENDPOINT") + cfg["llm"]["llm_api_key"] = _require("LLM_API_KEY") + cfg["llm"]["llm_model"] = _optional("LLM_MODEL", cfg["llm"].get("llm_model", "model-router")) + + # --- Embedding ----------------------------------------------------------- + if "embedding" not in cfg: + cfg["embedding"] = {} + cfg["embedding"]["embed_endpoint"] = _require("EMBED_ENDPOINT") + cfg["embedding"]["embed_api_key"] = _require("EMBED_API_KEY") + cfg["embedding"]["embed_model"] = _require("EMBED_MODEL") + cfg["embedding"]["embed_dimensions"] = int(_optional("EMBED_DIMENSIONS", "1024")) + + # Disable local LLM fallback in CI to avoid noise + cfg.setdefault("local_llm", {})["use_local_fallback_for_subtasks"] = False + + # Write output + with open(OUTPUT_PATH, "w") as fh: + yaml.dump(cfg, fh, allow_unicode=True, sort_keys=False) + + print(f"config.yaml written to {OUTPUT_PATH}") + + +if __name__ == "__main__": + main() diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..ceaa6b0 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,67 @@ +"""Shared pytest fixtures for integration tests.""" +import sys +import asyncio +from pathlib import Path + +import pytest +import yaml + +# Ensure the repo root is on the path so rag_divdet can be imported +REPO_ROOT = Path(__file__).parent.parent +sys.path.insert(0, str(REPO_ROOT)) + + +# --------------------------------------------------------------------------- +# Event-loop fixture (module-scoped so all fixtures share one loop) +# --------------------------------------------------------------------------- + +@pytest.fixture(scope="module") +def event_loop(): + """Create a module-scoped event loop shared by all fixtures and tests.""" + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + yield loop + loop.close() + + +# --------------------------------------------------------------------------- +# Config fixture: load config.yaml and expose as a dict +# --------------------------------------------------------------------------- + +@pytest.fixture(scope="module") +def config(): + config_path = REPO_ROOT / "config.yaml" + if not config_path.exists(): + pytest.skip("config.yaml not found – skipping integration tests") + with open(config_path) as fh: + return yaml.safe_load(fh) + + +# --------------------------------------------------------------------------- +# LLMClient fixture +# --------------------------------------------------------------------------- + +@pytest.fixture(scope="module") +def llm_client(config, event_loop): + """Return an initialised LLMClient (module-scoped to avoid repeated setup).""" + import rag_divdet as rd + client = rd.LLMClient() + yield client + event_loop.run_until_complete(client.close()) + + +# --------------------------------------------------------------------------- +# CombinedRetriever fixture +# --------------------------------------------------------------------------- + +@pytest.fixture(scope="module") +def retriever(config, event_loop): + """Return an initialised CombinedRetriever (module-scoped).""" + import rag_divdet as rd + sources = rd._build_retrieval_sources(config) + if not sources: + pytest.skip("No retrieval sources configured – skipping retriever tests") + ret = rd.CombinedRetriever(retrieval_sources=sources) + event_loop.run_until_complete(ret.initialize()) + yield ret + event_loop.run_until_complete(ret.close()) diff --git a/tests/test_integration.py b/tests/test_integration.py new file mode 100644 index 0000000..46f341f --- /dev/null +++ b/tests/test_integration.py @@ -0,0 +1,288 @@ +""" +Integration tests for AgenticRetrieval. + +These tests require a valid config.yaml (generated from config.yaml.example +via CI secrets) with live Azure Cosmos DB and Azure OpenAI endpoints. +All tests are skipped automatically when config.yaml is absent. +""" +import asyncio +import json +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).parent.parent +TEST_QUESTIONS_PATH = Path(__file__).parent / "test_questions.json" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _run(coro): + """Run a coroutine in the module-level event loop.""" + return asyncio.get_event_loop().run_until_complete(coro) + + +def _load_test_questions(): + with open(TEST_QUESTIONS_PATH, encoding="utf-8") as fh: + return json.load(fh) + + +# =========================================================================== +# 1. Embedding endpoint +# =========================================================================== + + +class TestEmbeddingEndpoint: + """Verify that the Azure OpenAI embedding endpoint is reachable and returns + non-empty, non-null vectors.""" + + def test_embed_returns_nonempty_list(self, llm_client): + embedding = _run(llm_client.embed("What is the temperature on Mars?")) + assert embedding is not None, "embed() returned None" + assert isinstance(embedding, list), "embed() should return a list" + assert len(embedding) > 0, "embed() returned an empty list" + + def test_embed_values_are_floats(self, llm_client): + embedding = _run(llm_client.embed("surface temperature")) + assert all(isinstance(v, float) for v in embedding), ( + "All embedding values should be floats" + ) + + def test_embed_dimension_matches_config(self, llm_client, config): + embed_cfg = config.get("embedding") or {} + llm_cfg = config.get("llm", {}) + expected_dim = int((embed_cfg or llm_cfg).get("embed_dimensions") or 0) + if expected_dim <= 0: + pytest.skip("embed_dimensions not set in config – skipping dimension check") + embedding = _run(llm_client.embed("Mars atmosphere")) + assert len(embedding) == expected_dim, ( + f"Expected {expected_dim}-dimensional embedding, got {len(embedding)}" + ) + + +# =========================================================================== +# 2. LLM completion endpoint +# =========================================================================== + + +class TestLLMEndpoint: + """Verify that the Azure OpenAI LLM completion endpoint is functional.""" + + def test_complete_returns_nonempty_string(self, llm_client): + answer = _run( + llm_client.complete( + "Say exactly: HELLO", + retries=2, + label="LLM test-ping", + ) + ) + assert answer is not None, "complete() returned None" + assert isinstance(answer, str), "complete() should return a string" + assert len(answer.strip()) > 0, "complete() returned an empty string" + + def test_complete_returns_coherent_response(self, llm_client): + prompt = ( + "Answer in one sentence: What planet is fourth from the Sun?" + ) + answer = _run( + llm_client.complete(prompt, retries=2, label="LLM test-coherence") + ) + assert "mars" in answer.lower() or "fourth" in answer.lower(), ( + f"Expected a response mentioning Mars or 'fourth', got: {answer!r}" + ) + + +# =========================================================================== +# 3. Cosmos DB – vector search +# =========================================================================== + + +class TestVectorSearch: + """Verify that vector search against each configured Cosmos DB source + returns non-null, non-empty results.""" + + def test_vector_search_returns_results(self, retriever): + chunks = _run(retriever.retrieve("temperature on Mars")) + vector_chunks = [ + c for c in chunks if "_vector" in (c.metadata.get("_data_source") or "") + ] + assert len(vector_chunks) > 0, ( + "Vector search returned no results for query 'temperature on Mars'. " + "Expected at least one chunk from a vector-indexed source." + ) + + def test_vector_search_chunks_have_text(self, retriever): + chunks = _run(retriever.retrieve("Mars atmosphere composition")) + vector_chunks = [ + c for c in chunks if "_vector" in (c.metadata.get("_data_source") or "") + ] + if not vector_chunks: + pytest.skip("No vector chunks returned – skipping text content check") + for chunk in vector_chunks: + assert chunk.text and chunk.text.strip(), ( + f"Vector chunk {chunk.chunk_id!r} has empty text" + ) + + def test_vector_search_per_source(self, retriever, config): + """Each source with vector_k > 0 must return at least one result.""" + sources = config.get("cosmos", {}).get("sources", []) + for source in sources: + retrieval = source.get("retrieval", {}) + vector_k = int(retrieval.get("vector_k") or 0) + if vector_k <= 0: + continue + source_id = source.get("id", "") + chunks = _run(retriever.retrieve("Mars day length")) + source_chunks = [ + c + for c in chunks + if (c.metadata.get("_data_source") or "").startswith(source_id) + ] + assert len(source_chunks) > 0, ( + f"Source '{source_id}' with vector_k={vector_k} returned no chunks" + ) + + +# =========================================================================== +# 4. Cosmos DB – full-text search +# =========================================================================== + + +class TestFullTextSearch: + """Verify that full-text search against each configured Cosmos DB source + returns non-null, non-empty results when fulltext_k > 0.""" + + def test_fulltext_search_returns_results(self, retriever, config): + sources = config.get("cosmos", {}).get("sources", []) + has_fulltext = any( + int((s.get("retrieval") or {}).get("fulltext_k") or 0) > 0 + and (s.get("retrieval") or {}).get("fulltext_fields") + for s in sources + ) + if not has_fulltext: + pytest.skip("No sources with fulltext_k > 0 configured – skipping") + + chunks = _run(retriever.retrieve("Mars temperature")) + fulltext_chunks = [ + c for c in chunks if "_fulltext" in (c.metadata.get("_data_source") or "") + ] + assert len(fulltext_chunks) > 0, ( + "Full-text search returned no results for query 'Mars temperature'." + ) + + def test_fulltext_search_chunks_have_text(self, retriever, config): + sources = config.get("cosmos", {}).get("sources", []) + has_fulltext = any( + int((s.get("retrieval") or {}).get("fulltext_k") or 0) > 0 + and (s.get("retrieval") or {}).get("fulltext_fields") + for s in sources + ) + if not has_fulltext: + pytest.skip("No sources with fulltext_k > 0 configured – skipping") + + chunks = _run(retriever.retrieve("Mars polar ice")) + fulltext_chunks = [ + c for c in chunks if "_fulltext" in (c.metadata.get("_data_source") or "") + ] + if not fulltext_chunks: + pytest.skip("No full-text chunks returned – skipping text content check") + for chunk in fulltext_chunks: + assert chunk.text and chunk.text.strip(), ( + f"Full-text chunk {chunk.chunk_id!r} has empty text" + ) + + def test_fulltext_search_per_source(self, retriever, config): + """Each source with fulltext_k > 0 must return at least one result.""" + sources = config.get("cosmos", {}).get("sources", []) + for source in sources: + retrieval = source.get("retrieval", {}) + fulltext_k = int(retrieval.get("fulltext_k") or 0) + fulltext_fields = retrieval.get("fulltext_fields") or [] + if fulltext_k <= 0 or not fulltext_fields: + continue + source_id = source.get("id", "") + chunks = _run(retriever.retrieve("Mars sol day")) + source_chunks = [ + c + for c in chunks + if (c.metadata.get("_data_source") or "").startswith(source_id) + ] + assert len(source_chunks) > 0, ( + f"Source '{source_id}' with fulltext_k={fulltext_k} returned no chunks" + ) + + +# =========================================================================== +# 5. End-to-end RAG pipeline +# =========================================================================== + + +class TestRAGPipeline: + """Run the full DecomposedRAGPipeline on the test questions and verify + that reasoning traces and final answers are produced.""" + + @pytest.fixture(scope="class") + def pipeline_and_retriever(self, config, retriever, llm_client): + import rag_divdet as rd + + pipeline = rd.DecomposedRAGPipeline( + retriever=retriever, + llm=llm_client, + max_sub_q=2, + num_rounds=1, + subq_fanout_cap=2, + subq_max_concurrency=1, + ) + return pipeline, retriever + + def test_pipeline_produces_final_answer(self, pipeline_and_retriever): + pipeline, _ = pipeline_and_retriever + questions = _load_test_questions() + q = questions[0] + result = _run(pipeline.run(q["question_text"])) + + assert result is not None, "pipeline.run() returned None" + assert "final_answer" in result, "Result missing 'final_answer'" + final = result["final_answer"] + assert isinstance(final, str) and len(final.strip()) > 0, ( + "final_answer is empty or not a string" + ) + + def test_pipeline_produces_reasoning_trace(self, pipeline_and_retriever): + pipeline, _ = pipeline_and_retriever + questions = _load_test_questions() + q = questions[1] + result = _run(pipeline.run(q["question_text"])) + + assert "initial_answer" in result, "Result missing 'initial_answer'" + assert isinstance(result["initial_answer"], str) + assert len(result["initial_answer"].strip()) > 0, ( + "initial_answer (preliminary answer) is empty" + ) + assert "rounds" in result, "Result missing 'rounds'" + assert isinstance(result["rounds"], list), "'rounds' should be a list" + + def test_pipeline_produces_initial_chunks(self, pipeline_and_retriever): + pipeline, _ = pipeline_and_retriever + questions = _load_test_questions() + q = questions[2] + result = _run(pipeline.run(q["question_text"])) + + assert "initial_chunks" in result, "Result missing 'initial_chunks'" + chunks = result["initial_chunks"] + assert isinstance(chunks, list) and len(chunks) > 0, ( + "initial_chunks is empty – retrieval may have failed" + ) + + def test_all_test_questions_produce_answers(self, pipeline_and_retriever): + """Run every question in test_questions.json and assert non-empty answers.""" + pipeline, _ = pipeline_and_retriever + questions = _load_test_questions() + for q in questions: + result = _run(pipeline.run(q["question_text"])) + assert result.get("final_answer", "").strip(), ( + f"Question {q['question_id']!r} produced an empty final_answer" + ) diff --git a/tests/test_questions.json b/tests/test_questions.json new file mode 100644 index 0000000..ac64e04 --- /dev/null +++ b/tests/test_questions.json @@ -0,0 +1,17 @@ +[ + { + "question_id": "t1", + "question_text": "what is the max temperature on planet Mars?", + "answer": "The maximum temperature on Mars can reach about 20 °C (70 °F) during summer afternoons near the equator." + }, + { + "question_id": "t2", + "question_text": "What is the coldest temperature on Mars?", + "answer": "The coldest temperature on Mars can drop to about −125 °C (−195 °F) near the poles during winter." + }, + { + "question_id": "t3", + "question_text": "How long is one day on Mars?", + "answer": "One day on Mars, called a sol, lasts about 24 hours and 37 minutes." + } +]