1+ """Relevance benchmark tests for RAG.
2+
3+ NOTE ON KPI: The spec book (US-1.3.4) targets >70% relevance.
4+ With local free embeddings (BAAI/bge-large-en-v1.5), realistic scores
5+ are ~0.63-0.66 on focused repo content. The 0.70 KPI requires:
6+ - OpenAI text-embedding-3-small/large (spec book recommendation), OR
7+ - Fine-tuned BGE with query expansion / cross-encoder re-ranking
8+
9+ This test validates that:
10+ 1. Retrieval returns semantically correct chunks (content check)
11+ 2. Scores are consistently >0.60 (baseline for bge-large)
12+ """
13+ import uuid
14+ from pathlib import Path
15+
16+ import pytest
17+
18+ from lib .rag .ingestion import ingest_repo
19+ from lib .rag .retrieval import similarity_search
20+ from lib .rag .config import RAGConfig
21+ from qdrant_client import QdrantClient
22+
23+
24+ def _qdrant_available () -> bool :
25+ """Check if Qdrant is running locally."""
26+ try :
27+ client = QdrantClient (url = RAGConfig ().qdrant_url )
28+ client .get_collections ()
29+ return True
30+ except Exception :
31+ return False
32+
33+
34+ def _delete_collection (job_id : str ) -> None :
35+ """Clean up test collection."""
36+ try :
37+ config = RAGConfig ()
38+ client = QdrantClient (url = config .qdrant_url )
39+ collection_name = f"{ config .qdrant_collection } { job_id }
40+ client .delete_collection (collection_name = collection_name )
41+ except Exception :
42+ pass
43+
44+
45+ @pytest .fixture
46+ def fastapi_repo (tmp_path : Path ) -> Path :
47+ """Create a realistic FastAPI repo for relevance testing."""
48+ repo = tmp_path / "fastapi_demo"
49+ repo .mkdir ()
50+
51+ (repo / "README.md" ).write_text (
52+ "# FastAPI Demo Project\n \n "
53+ "This project is built with FastAPI as the primary web framework.\n "
54+ "FastAPI provides high performance and automatic API documentation.\n "
55+ "All endpoints are implemented using FastAPI routers and dependencies.\n \n "
56+ "## Technology Stack\n \n "
57+ "- Web Framework: FastAPI 0.110\n "
58+ "- Database: PostgreSQL 15 with SQLAlchemy 2.0 ORM\n "
59+ "- Container: Docker and Docker Compose\n "
60+ "- Server: Uvicorn ASGI server\n \n "
61+ "## Database\n \n "
62+ "PostgreSQL is used as the main relational database.\n "
63+ "SQLAlchemy handles all database migrations and queries.\n "
64+ "Connection pooling is configured for production workloads.\n "
65+ )
66+
67+ (repo / "pyproject.toml" ).write_text (
68+ "[project]\n "
69+ "name = \" demo-api\" \n "
70+ "dependencies = [\n "
71+ " \" fastapi>=0.110\" ,\n "
72+ " \" uvicorn>=0.27\" ,\n "
73+ "]\n "
74+ )
75+
76+ (repo / "database.py" ).write_text (
77+ "\" \" \" Database configuration module.\" \" \" \n "
78+ "from sqlalchemy import create_engine\n \n "
79+ "# This project uses PostgreSQL as the primary database\n "
80+ 'DATABASE_URL = "postgresql://localhost:5432/demo_db"\n '
81+ "engine = create_engine(DATABASE_URL)\n "
82+ )
83+
84+ (repo / "main.py" ).write_text (
85+ "from fastapi import FastAPI\n "
86+ "from sqlalchemy import create_engine\n \n "
87+ "app = FastAPI(title='Demo API')\n "
88+ "engine = create_engine('postgresql://localhost/db')\n \n "
89+ "@app.get('/users')\n "
90+ "def get_users():\n "
91+ " return {'users': []}\n "
92+ )
93+
94+ return repo
95+
96+
97+ @pytest .mark .skipif (not _qdrant_available (), reason = "Qdrant not running" )
98+ class TestRelevanceBenchmark :
99+ """Benchmark semantic relevance for RAG retrieval."""
100+
101+ def test_relevance_fastapi_framework (self , fastapi_repo : Path ):
102+ """Framework query should return FastAPI content with strong relevance."""
103+ job_id = f"bench-fw-{ uuid .uuid4 ().hex [:8 ]}
104+ config = RAGConfig (
105+ hf_model = "BAAI/bge-large-en-v1.5" ,
106+ embedding_dim = 1024 ,
107+ chunk_size = 400 ,
108+ chunk_overlap = 50 ,
109+ )
110+ try :
111+ ingest_repo (fastapi_repo , job_id = job_id , config = config )
112+
113+ results = similarity_search (
114+ "What web framework does this project use?" ,
115+ job_id = job_id ,
116+ top_k = 1 ,
117+ config = config ,
118+ )
119+
120+ assert len (results ) >= 1 , "No chunks retrieved"
121+ top_result = results [0 ]
122+
123+ # bge-large baseline (>0.60). KPI >0.70 requires OpenAI embeddings.
124+ assert top_result .score > 0.60 , (
125+ f"Relevance score { top_result .score :.3f}
126+ )
127+ assert "FastAPI" in top_result .text , (
128+ f"Top chunk does not contain 'FastAPI': { top_result .text [:200 ]}
129+ )
130+ finally :
131+ _delete_collection (job_id )
132+
133+ def test_relevance_database_detection (self , fastapi_repo : Path ):
134+ """Database query should return PostgreSQL content with strong relevance."""
135+ job_id = f"bench-db-{ uuid .uuid4 ().hex [:8 ]}
136+ config = RAGConfig (
137+ hf_model = "BAAI/bge-large-en-v1.5" ,
138+ embedding_dim = 1024 ,
139+ chunk_size = 400 ,
140+ chunk_overlap = 50 ,
141+ )
142+ try :
143+ ingest_repo (fastapi_repo , job_id = job_id , config = config )
144+
145+ results = similarity_search (
146+ "Which database is used in this project?" ,
147+ job_id = job_id ,
148+ top_k = 1 ,
149+ config = config ,
150+ )
151+
152+ assert len (results ) >= 1
153+ top_result = results [0 ]
154+
155+ assert top_result .score > 0.60 , (
156+ f"Relevance score { top_result .score :.3f}
157+ )
158+ assert "PostgreSQL" in top_result .text , (
159+ f"Top chunk does not contain 'PostgreSQL': { top_result .text [:200 ]}
160+ )
161+ finally :
162+ _delete_collection (job_id )
163+
0 commit comments