diff --git a/compliance/TEST09/e2e-rag/README.md b/compliance/TEST09/e2e-rag-qna/README.md similarity index 94% rename from compliance/TEST09/e2e-rag/README.md rename to compliance/TEST09/e2e-rag-qna/README.md index 90dcf04a39..e1d2fcb0b0 100644 --- a/compliance/TEST09/e2e-rag/README.md +++ b/compliance/TEST09/e2e-rag-qna/README.md @@ -1,4 +1,4 @@ -# TEST09 Compliance for E2E-RAG Workload +# TEST09 Compliance for E2E-RAG-QnA Workload ## Overview @@ -65,7 +65,7 @@ Copy the audit.config to your working directory: ```bash cd inference/e2e-rag -cp ../compliance/TEST09/e2e-rag/audit.config ./ +cp ../compliance/TEST09/e2e-rag-qna/audit.config ./ ``` ### Part II: Run Performance Test @@ -94,8 +94,8 @@ python3 reference_mlperf.py \ ```bash python3 inference/e2e/third_party/mlperf-inference/compliance/TEST09/run_verification.py \ -c run_output_test09 \ - -o submission/compliance/e2e-rag/Offline \ - --audit-config ../compliance/TEST09/e2e-rag/audit.config + -o submission/compliance/e2e-rag-qna/Offline \ + --audit-config ../compliance/TEST09/e2e-rag-qna/audit.config ``` ### Expected Output diff --git a/compliance/TEST09/e2e-rag/audit.config b/compliance/TEST09/e2e-rag-qna/audit.config similarity index 100% rename from compliance/TEST09/e2e-rag/audit.config rename to compliance/TEST09/e2e-rag-qna/audit.config diff --git a/e2e-rag/.gitignore b/e2e-rag/.gitignore index 817a93c1f3..b52c4e4a5e 100644 --- a/e2e-rag/.gitignore +++ b/e2e-rag/.gitignore @@ -37,6 +37,16 @@ result_*.json temp_complete_kpi_*.json run_output_datasetup_accuracy/ run_output_datasetup/ +run_output_*/ +run_output_test09/ +accuracy_results.json +/smoke_manifest.json +/db_manifest_*.json +/db_manifest_*.json.gz +doc_html_smoke/ +/submission/ +/audit.config +/verify_output_len.txt colbert-ir_colbertv2.0/ frames-benchmark-dataset/ intfloat_e5-base-v2/ diff --git a/e2e-rag/accuracy_eval.py b/e2e-rag/accuracy_eval.py index 29ce199482..cec921683c 100644 --- a/e2e-rag/accuracy_eval.py +++ b/e2e-rag/accuracy_eval.py @@ -32,14 +32,14 @@ # OpenRouter configuration -DEFAULT_JUDGE_URL = "http://127.0.0.1:8123/v1/chat/completions" -DEFAULT_JUDGE_MODEL = "gpt-oss-20b" +DEFAULT_JUDGE_URL = "http://127.0.0.1:8125/v1/chat/completions" +DEFAULT_JUDGE_MODEL = "meta-llama/Llama-3.1-8B-Instruct" # Masked API key (set OPENROUTER_API_KEY environment variable to use OpenRouter) OPENROUTER_API_KEY = os.environ.get('OPENROUTER_API_KEY', 'sk-or-v1-****') -JUDGE_PROMPT = """You are an expert evaluator comparing LLM-generated answers to ground truth answers. +JUDGE_PROMPT = """You are grading whether an LLM answer is correct against a ground truth answer. QUESTION: {question} @@ -47,8 +47,14 @@ LLM ANSWER: {llm_answer} -Evaluate if the LLM answer is factually correct compared to the ground truth. -Consider semantic equivalence, not just exact string matching. +Grade in two steps. + +STEP 1 - If the LLM answer is empty, "Unknown", "I don't know", "cannot be determined", or otherwise does not commit to an answer, then it is WRONG: output correct=false immediately and do not go to step 2. + +STEP 2 - Otherwise compare it to the ground truth by meaning, not wording. correct=true only if it supplies every fact the ground truth requires and each clearly matches; if you are unsure or the match is only partial, output correct=false. Rules: +- If the ground truth is a list or has multiple parts, an answer missing any of them is correct=false. +- Every number, date, and name must match the ground truth; a different or differently-rounded value is correct=false, a different name is correct=false. +- Do NOT penalize harmless extras or omissions when the required facts match: a missing suffix like "Inc.", an added state/country, a full middle name, missing units when the number is right, or a briefer/longer phrasing. Return your evaluation in JSON format: {{ @@ -91,11 +97,19 @@ def call_judge(question: str, ground_truth: str, llm_answer: str, # Parse JSON response content = content.strip() - if content.startswith("```"): - content = content.split("```")[1] - if content.startswith("json"): - content = content[4:] - content = content.strip() + + # Extract JSON from markdown code blocks + if "```" in content: + json_block_match = re.search(r'```(?:json)?\s*\n?(.*?)\n?```', content, re.DOTALL) + if json_block_match: + content = json_block_match.group(1).strip() + + # Try to extract JSON object + json_match = re.search(r'\{.*\}', content, re.DOTALL) + if json_match: + content = json_match.group(0) + else: + return {"correct": False, "reasoning": "No JSON found in judge response"} judge_result = json.loads(content) return judge_result @@ -297,6 +311,15 @@ def main(): json.dump(metrics, f, indent=2) print(f"Detailed results saved to {args.output}") + # Write accuracy.txt into the loadgen log dir in MLPerf format. The + # submission checker parses the LLM judge answer accuracy (as a percentage) + # from the "Accuracy:" line. The hash= line and log truncation are added + # later by tools/submission/truncate_accuracy_log.py during submission prep. + accuracy_txt_path = os.path.join(args.log_dir, "accuracy.txt") + with open(accuracy_txt_path, 'w') as f: + f.write(f"Accuracy: {metrics['answer_accuracy'] * 100:.4f}\n") + print(f"Accuracy report saved to {accuracy_txt_path}") + if __name__ == "__main__": main() diff --git a/e2e-rag/config.template.sh b/e2e-rag/config.template.sh index 33b80080c1..6be88f0c34 100644 --- a/e2e-rag/config.template.sh +++ b/e2e-rag/config.template.sh @@ -36,22 +36,22 @@ INFERENCE_N_QUERIES=5 INFERENCE_NUM_WORKERS=1 # LLM endpoints (vLLM, OpenRouter, etc.) -INFERENCE_LLM_URL="http://127.0.0.1:8123/v1/chat/completions" -INFERENCE_MODEL="/model/gpt-oss-20b-mxfp4" -INFERENCE_QUERY_MODEL="/model/gpt-oss-120b-mxfp4" +INFERENCE_LLM_URL="http://127.0.0.1:8192/v1/chat/completions" +INFERENCE_MODEL="gpt-oss-20b-mxfp4" +INFERENCE_QUERY_MODEL="gpt-oss-120b-mxfp4" # Per-component endpoint splits. Each defaults to INFERENCE_LLM_URL / # INFERENCE_MODEL when empty. Set when components live on different servers # (e.g. small grader on one vLLM, large query/sufficiency on another). -# INFERENCE_GRADER_URL="http://127.0.0.1:8124/v1/chat/completions" -# INFERENCE_GRADER_MODEL="/model/gpt-oss-20b" +# INFERENCE_GRADER_URL="http://127.0.0.1:8192/v1/chat/completions" +# INFERENCE_GRADER_MODEL="gpt-oss-20b-mxfp4" # INFERENCE_QUERY_URL="http://127.0.0.1:8123/v1/chat/completions" # INFERENCE_SUFFICIENCY_URL="http://127.0.0.1:8123/v1/chat/completions" -# INFERENCE_SUFFICIENCY_MODEL="/model/gpt-oss-120b" +# INFERENCE_SUFFICIENCY_MODEL="gpt-oss-120b-mxfp4" # Judge (used by evaluate.py at the end of the run scripts) INFERENCE_JUDGE_URL="https://openrouter.ai/api/v1/chat/completions" -INFERENCE_JUDGE_MODEL="openai/gpt-oss-20b" +INFERENCE_JUDGE_MODEL="Llama3.1-8B-v1" # ── Oracle evaluation (run_oracle.sh) ───────────────────────────────────────── INFERENCE_ORACLE_BATCH_SIZE=4 diff --git a/e2e-rag/datasetup_accuracy_eval.py b/e2e-rag/datasetup_accuracy_eval.py index 7df71d03f0..d92dfa7972 100755 --- a/e2e-rag/datasetup_accuracy_eval.py +++ b/e2e-rag/datasetup_accuracy_eval.py @@ -268,7 +268,9 @@ def validate_database(database_path, retriever_model): return validation_results -def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None): +def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None, + manifest_path=None, cosine_threshold=0.9999, + top_k_depth=3): """ Evaluate accuracy of datasetup workload. @@ -277,6 +279,11 @@ def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None): output_dir: Directory containing SUT output files database_path: Path to the saved database file retriever_model: Path to retriever model (for validation) + manifest_path: Path to reference DB manifest for cross-system + verification. If None, the manifest check is skipped. + cosine_threshold: Minimum sample-embedding cosine similarity for the + manifest check. + top_k_depth: Probe-query top-K rank match depth for the manifest check. Returns: dict: Accuracy results @@ -448,9 +455,67 @@ def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None): print("="*80) print() + # Cross-system manifest verification (corpus fingerprint, sample-embedding + # cosine, probe-query top-K ranks) against a reference manifest. + manifest_results = None + if manifest_path: + print("="*80) + print("DB Manifest Verification") + print("="*80) + print(f"Manifest: {manifest_path}") + print() + + if not os.path.exists(manifest_path): + manifest_results = { + "passed": False, + "error": "manifest_not_found", + "manifest_path": manifest_path, + } + print(f" ✗ Manifest not found: {manifest_path}") + elif not os.path.exists(database_path): + manifest_results = { + "passed": False, + "error": "database_not_found", + "database_path": database_path, + } + print(f" ✗ Database not found: {database_path}") + else: + try: + from db_manifest import verify_manifest + manifest_results = verify_manifest( + database_path, + manifest_path, + retriever_model=retriever_model, + cosine_threshold=cosine_threshold, + top_k_depth=top_k_depth, + ) + if manifest_results["passed"]: + print(" ✓ Manifest verification PASSED") + else: + print(" ✗ Manifest verification FAILED:") + for failure in manifest_results["failures"]: + print(f" - {failure}") + except Exception as e: + manifest_results = {"passed": False, "error": str(e)} + print(f" ✗ Manifest verification error: {e}") + + accuracy_results["manifest"] = manifest_results + # Overall pass now also requires the manifest check to pass. + accuracy_results["passed"] = accuracy_results["passed"] and manifest_results["passed"] + + print() + print(f"Manifest: {'✅ PASSED' if manifest_results['passed'] else '❌ FAILED'}") + print("="*80) + print() + # Write accuracy.txt in MLPerf format accuracy_txt_path = os.path.join(log_dir, "accuracy.txt") with open(accuracy_txt_path, 'w') as f: + # Checker-compliant metric line first: the submission checker parses the + # RAG-DB accuracy (document indexing success rate as a percentage) from + # this "Accuracy:" line via the E2E_ACCURACY pattern. The hash= line and + # log truncation are added later by truncate_accuracy_log.py. + f.write(f"Accuracy: {actual_success_rate * 100:.4f}\n") f.write("="*80 + "\n") f.write("RAG-DB Accuracy Report\n") f.write("="*80 + "\n") @@ -488,6 +553,23 @@ def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None): f.write(f"Validation status: {'PASS' if validation_results['passed'] else 'FAIL'}\n") f.write("\n") + # Always emit the manifest section so its status (or that it was + # skipped) is visible in every accuracy report. + f.write("DB Manifest Verification:\n") + f.write("-"*80 + "\n") + if manifest_results is None: + f.write(" Manifest status: SKIPPED (no manifest provided)\n") + else: + if manifest_results.get("error"): + f.write(f" Error: {manifest_results['error']}\n") + metrics = manifest_results.get("metrics", {}) + for key, value in metrics.items(): + f.write(f" {key}: {value}\n") + for failure in manifest_results.get("failures", []): + f.write(f" MISMATCH: {failure}\n") + f.write(f" Manifest status: {'PASS' if manifest_results['passed'] else 'FAIL'}\n") + f.write("\n") + f.write("="*80 + "\n") f.write(f"Overall Result: {'PASS' if accuracy_results['passed'] else 'FAIL'}\n") f.write("="*80 + "\n") @@ -520,13 +602,36 @@ def main(): ) parser.add_argument( "--retriever_model", - default="/data/model/e5-base-v2", + default="intfloat_e5-base-v2/e5-base-v2", help="Path to retriever model (for validation)" ) + parser.add_argument( + "--manifest", + default=None, + help="Path to reference DB manifest (.json/.json.gz) for cross-system " + "verification. If omitted, the manifest check is skipped." + ) + parser.add_argument( + "--cosine_threshold", + type=float, + default=0.9999, + help="Minimum sample-embedding cosine similarity for the manifest check" + ) + parser.add_argument( + "--top_k_depth", + type=int, + default=3, + help="Probe-query top-K rank match depth for the manifest check" + ) args = parser.parse_args() - results = evaluate_accuracy(args.log_dir, args.output_dir, args.database, args.retriever_model) + results = evaluate_accuracy( + args.log_dir, args.output_dir, args.database, args.retriever_model, + manifest_path=args.manifest, + cosine_threshold=args.cosine_threshold, + top_k_depth=args.top_k_depth, + ) # Exit with appropriate code if results.get("passed", False): diff --git a/e2e-rag/db_manifest.py b/e2e-rag/db_manifest.py index 1cfe20f85f..7f3545edc9 100644 --- a/e2e-rag/db_manifest.py +++ b/e2e-rag/db_manifest.py @@ -164,14 +164,42 @@ def cmd_write(args): print(f"[manifest] wrote {args.output}") -def cmd_verify(args): - with _open_manifest(args.manifest, "rt") as f: +def verify_manifest(db_path: str, manifest_path: str, + retriever_model: str = None, + cosine_threshold: float = DEFAULT_COSINE_THRESHOLD, + top_k_depth: int = DEFAULT_TOP_K_DEPTH) -> Dict: + """Verify a vector DB against a reference manifest. + + Args: + db_path: Path to the local vector DB to check. + manifest_path: Path to the reference manifest (.json or .json.gz). + retriever_model: Retriever model to load the DB with. If None, falls + back to the manifest's stored ``retriever_model``. The manifest + value is often a system-specific absolute path, so callers on other + systems should pass their own local model path here. + cosine_threshold: Minimum sample-embedding cosine similarity. + top_k_depth: Probe-query top-K rank match depth. + + Returns: + dict with keys ``passed`` (bool), ``failures`` (list[str]), and + ``metrics`` (dict of observed values). Never raises on mismatch; the + CLI wrapper is responsible for translating a failure into an exit code. + """ + with _open_manifest(manifest_path, "rt") as f: manifest = json.load(f) - db = _load_db(args.db, manifest["retriever_model"]) + # Prefer an explicit retriever model; the manifest's value may be an + # absolute path that only exists on the system that wrote it. + model = retriever_model or manifest["retriever_model"] + db = _load_db(db_path, model) total_passages = len(db._vector_store.index_to_docstore_id) failures = [] + metrics = { + "total_passages": total_passages, + "embedding_dim": db._embedding_dimension, + "retriever_model": model, + } # Exact-match fields. if total_passages != manifest["total_passages"]: @@ -186,6 +214,7 @@ def cmd_verify(args): # Corpus fingerprint (sha256 of all passage texts in index order). local_corpus_sha = _sha256_docstore(db) + metrics["corpus_sha256_match"] = (local_corpus_sha == manifest["corpus_sha256"]) if local_corpus_sha != manifest["corpus_sha256"]: failures.append( f"corpus sha256 mismatch:\n" @@ -208,12 +237,14 @@ def cmd_verify(args): if cosines: worst_idx, worst_cos = min(cosines, key=lambda x: x[1]) mean_cos = sum(c for _, c in cosines) / len(cosines) + metrics["sample_cosine_mean"] = mean_cos + metrics["sample_cosine_min"] = worst_cos print(f"[verify] sample embeddings: mean cosine={mean_cos:.6f} " - f"min={worst_cos:.6f} (idx={worst_idx}) threshold={args.cosine_threshold}") - if worst_cos < args.cosine_threshold: + f"min={worst_cos:.6f} (idx={worst_idx}) threshold={cosine_threshold}") + if worst_cos < cosine_threshold: failures.append( f"sample embedding cosine below threshold: " - f"min={worst_cos:.6f} (idx={worst_idx}) < threshold={args.cosine_threshold}\n" + f"min={worst_cos:.6f} (idx={worst_idx}) < threshold={cosine_threshold}\n" f" mean={mean_cos:.6f}" ) @@ -224,24 +255,37 @@ def cmd_verify(args): rank_failures = [] for entry in local_top: - local_urls = entry["top_k_urls"][:args.top_k_depth] - ref_urls = ref_top.get(entry["index"], [])[:args.top_k_depth] + local_urls = entry["top_k_urls"][:top_k_depth] + ref_urls = ref_top.get(entry["index"], [])[:top_k_depth] if local_urls != ref_urls: rank_failures.append( - f" query idx {entry['index']}: top-{args.top_k_depth} differs\n" + f" query idx {entry['index']}: top-{top_k_depth} differs\n" f" local : {local_urls}\n" f" ref : {ref_urls}" ) + metrics["probe_queries_total"] = len(probe_queries) + metrics["probe_queries_matched"] = len(probe_queries) - len(rank_failures) print(f"[verify] probe queries: {len(probe_queries)} queries, " - f"top-{args.top_k_depth} {len(probe_queries) - len(rank_failures)}/" + f"top-{top_k_depth} {len(probe_queries) - len(rank_failures)}/" f"{len(probe_queries)} match") if rank_failures: failures.append("probe-query top-K rank mismatch:\n" + "\n".join(rank_failures)) - if failures: + return {"passed": not failures, "failures": failures, "metrics": metrics} + + +def cmd_verify(args): + result = verify_manifest( + args.db, + args.manifest, + retriever_model=args.retriever_model, + cosine_threshold=args.cosine_threshold, + top_k_depth=args.top_k_depth, + ) + if not result["passed"]: print("\n[verify] FAILED:") - for f in failures: + for f in result["failures"]: print(f" - {f}") sys.exit(1) print("\n[verify] OK") @@ -254,7 +298,7 @@ def main(): pw = sub.add_parser("write", help="Generate a reference manifest from a DB.") pw.add_argument("--db", required=True) - pw.add_argument("--retriever_model", default="intfloat/e5-base-v2") + pw.add_argument("--retriever_model", default="intfloat_e5-base-v2/e5-base-v2") pw.add_argument("--dataset", default="data/frames_dataset.tsv") pw.add_argument("--output", required=True) pw.set_defaults(func=cmd_write) @@ -262,6 +306,13 @@ def main(): pv = sub.add_parser("verify", help="Verify a DB against a reference manifest.") pv.add_argument("--db", required=True) pv.add_argument("--manifest", required=True) + pv.add_argument( + "--retriever_model", + default=None, + help="Retriever model to load the DB with. Defaults to the manifest's " + "stored value, which may be a system-specific absolute path; pass " + "your local model path to verify on a different system.", + ) pv.add_argument("--cosine-threshold", type=float, default=DEFAULT_COSINE_THRESHOLD) pv.add_argument("--top-k-depth", type=int, default=DEFAULT_TOP_K_DEPTH) pv.set_defaults(func=cmd_verify) diff --git a/e2e-rag/evaluate.py b/e2e-rag/evaluate.py index 9c94ce8890..36b9a1258a 100644 --- a/e2e-rag/evaluate.py +++ b/e2e-rag/evaluate.py @@ -40,8 +40,8 @@ import requests # LLM judge configuration (defaults to local vLLM) -DEFAULT_JUDGE_URL = "http://127.0.0.1:8123/v1/chat/completions" -DEFAULT_JUDGE_MODEL = "gpt-oss-20b" +DEFAULT_JUDGE_URL = "http://127.0.0.1:8192/v1/chat/completions" +DEFAULT_JUDGE_MODEL = "gpt-oss-20b-mxfp4" OPENROUTER_API_KEY = os.environ.get('OPENROUTER_API_KEY', '') diff --git a/e2e-rag/evaluation.py b/e2e-rag/evaluation.py index c08ff70b8b..fe45956773 100644 --- a/e2e-rag/evaluation.py +++ b/e2e-rag/evaluation.py @@ -150,28 +150,11 @@ def evaluate_retrieval_query(rag_db, query: str, expected_urls: List[str], f"Warning: No documents retrieved for query: {query[:50]}") else: reranking_start = time.perf_counter() - # Extract text content for reranking (rerank expects strings) - passages = [result.page_content for result in results] - scored_passages = rag_db.rerank(query, passages) - - # Reconstruct document objects with reranked order - # scored_passages is [(text, score), ...] ordered by score - reranked_results = [] - for text, score in scored_passages: - # Find the original document object for this text - for doc in results: - if doc.page_content == text: - reranked_results.append(doc) - break - - # Apply top_k_reranking limit AFTER reranking - # For adaptive strategies (top_p, relative, etc.), respect the number of documents - # selected by the strategy, only limit for fixed_k + # Rerank Documents by score; fixed_k takes top-k, adaptive keeps all. + reranked_results = rag_db.rerank_documents(query, results) if retrieval_strategy == "fixed_k": results = reranked_results[:top_k_reranking] else: - # For adaptive strategies, keep all documents selected by the - # strategy results = reranked_results reranking_time = time.perf_counter() - reranking_start diff --git a/e2e-rag/multi_shot_retrieval.py b/e2e-rag/multi_shot_retrieval.py index 3ee84c731d..6a3b224937 100644 --- a/e2e-rag/multi_shot_retrieval.py +++ b/e2e-rag/multi_shot_retrieval.py @@ -502,11 +502,19 @@ def evaluate_document_relevance(question: str, print(f" Warning: Relevance check returned empty, marking all as relevant") return {"relevance": [1] * len(new_documents)} - if llm_output.startswith("```"): - llm_output = llm_output.split("```")[1] - if llm_output.startswith("json"): - llm_output = llm_output[4:] - llm_output = llm_output.strip() + # Extract JSON from markdown code blocks + if "```" in llm_output: + json_block_match = re.search(r'```(?:json)?\s*\n?(.*?)\n?```', llm_output, re.DOTALL) + if json_block_match: + llm_output = json_block_match.group(1).strip() + + # Try to extract JSON object + json_match = re.search(r'\{.*\}', llm_output, re.DOTALL) + if json_match: + llm_output = json_match.group(0) + else: + print(f" Warning: No JSON in relevance response, marking all as relevant") + return {"relevance": [1] * len(new_documents)} relevance_result = json.loads(llm_output) relevance = relevance_result.get("relevance", []) @@ -610,16 +618,21 @@ def check_sufficiency(question: str, return {"sufficient": True, "reasoning": "Max iterations reached"} return {"sufficient": False, "reasoning": "LLM returned empty"} - if llm_output.startswith("```"): - llm_output = llm_output.split("```")[1] - if llm_output.startswith("json"): - llm_output = llm_output[4:] - llm_output = llm_output.strip() + # Extract JSON from markdown code blocks + if "```" in llm_output: + json_block_match = re.search(r'```(?:json)?\s*\n?(.*?)\n?```', llm_output, re.DOTALL) + if json_block_match: + llm_output = json_block_match.group(1).strip() # Try to extract JSON json_match = re.search(r'\{.*\}', llm_output, re.DOTALL) if json_match: llm_output = json_match.group(0) + else: + print(f" Warning: No JSON in sufficiency response") + if iteration >= max_iterations: + return {"sufficient": True, "reasoning": "Max iterations reached, no JSON"} + return {"sufficient": False, "reasoning": "No JSON in response"} result = json.loads(llm_output) sufficient = result.get("sufficient", False) @@ -815,16 +828,21 @@ def generate_search_queries(question: str, print(f" Warning: Query generation returned empty") return {"queries": [question], "feedback": "LLM returned empty"} - if llm_output.startswith("```"): - llm_output = llm_output.split("```")[1] - if llm_output.startswith("json"): - llm_output = llm_output[4:] - llm_output = llm_output.strip() + # Extract JSON from markdown code blocks + if "```" in llm_output: + # Find content between ```json and ``` or just between ``` markers + json_block_match = re.search(r'```(?:json)?\s*\n?(.*?)\n?```', llm_output, re.DOTALL) + if json_block_match: + llm_output = json_block_match.group(1).strip() # Try to extract JSON object even from mixed text/markdown responses json_match = re.search(r'\{.*\}', llm_output, re.DOTALL) if json_match: llm_output = json_match.group(0) + else: + # No JSON found in response + print(f" Warning: No JSON found in LLM response") + return {"queries": [question], "feedback": "No JSON in LLM response"} query_result = json.loads(llm_output) return { @@ -1001,14 +1019,27 @@ def query_rewriter(question: str, new_documents: List[tuple], } llm_output = llm_output.strip() - - # Parse JSON output - handle markdown code blocks - if llm_output.startswith("```"): - llm_output = llm_output.split("```")[1] - if llm_output.startswith("json"): - llm_output = llm_output[4:] - llm_output = llm_output.strip() - + + # Extract JSON from markdown code blocks + if "```" in llm_output: + json_block_match = re.search(r'```(?:json)?\s*\n?(.*?)\n?```', llm_output, re.DOTALL) + if json_block_match: + llm_output = json_block_match.group(1).strip() + + # Try to extract JSON object + json_match = re.search(r'\{.*\}', llm_output, re.DOTALL) + if json_match: + llm_output = json_match.group(0) + else: + print(f" Warning: No JSON in query rewriter response, using fallback") + return { + "relevance": [0] * len(new_documents), + "summaries": [""] * len(new_documents), + "queries": [question], + "feedback": "No JSON in LLM response", + "answer": "" + } + result_data = json.loads(llm_output) # Validate format @@ -1407,15 +1438,10 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], if verbose: print(f" Reranking {len(results)} docs for this subquery to top {target_docs_per_subquery}...") - # Extract contents for reranking - contents = [r.page_content for r in results] - scored_passages = rag_db.rerank(sub_query, contents) - - # Reorder results by reranking scores and take top-k - reranked_indices = [i for i, _ in sorted(enumerate(scored_passages), - key=lambda x: x[1][1], reverse=True)] - results = [results[idx] for idx in reranked_indices[:target_docs_per_subquery]] - + # Rerank Documents by score and take top-k. + results = rag_db.rerank_documents(sub_query, results, + top_k=target_docs_per_subquery) + if verbose: print(f" After reranking: keeping top {len(results)} docs") elif len(results) > target_docs_per_subquery: @@ -1424,12 +1450,33 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], # Add to new_docs for evaluation (avoid duplicates) for result in results: + # DEBUG: Log metadata structure for first result + if not all_retrieved_urls: # Log only once + print(f" [DEBUG] Sample result metadata keys: {list(result.metadata.keys())}") + print(f" [DEBUG] Sample result metadata: {result.metadata}") + + # Try to get URL from metadata - support both 'original_url' and 'source' fields + url = None if 'original_url' in result.metadata and result.metadata['original_url']: url = result.metadata['original_url'] - if url not in all_retrieved_urls: - all_retrieved_urls.add(url) - new_docs.append((url, result.page_content)) - iteration_results.append(result) + elif 'source' in result.metadata and result.metadata['source']: + # Convert source filename to Wikipedia URL + # e.g., "en.wikipedia.org_wiki_Kirk_Watson.html" -> "https://en.wikipedia.org/wiki/Kirk_Watson" + source = result.metadata['source'] + if source.startswith('en.wikipedia.org_wiki_'): + # Remove prefix and .html suffix + page_name = source.replace('en.wikipedia.org_wiki_', '').replace('.html', '') + url = f"https://en.wikipedia.org/wiki/{page_name}" + if not all_retrieved_urls: # Log conversion once + print(f" [DEBUG] Converted source to URL: {source} -> {url}") + + if url and url not in all_retrieved_urls: + all_retrieved_urls.add(url) + new_docs.append((url, result.page_content)) + iteration_results.append(result) + elif not url and not all_retrieved_urls: + # DEBUG: Log why URL was skipped + print(f" [DEBUG] Skipping result - no valid original_url or source in metadata") # Track how many NEW docs this query found docs_found_by_query = len(new_docs) - query_start_count @@ -1474,7 +1521,14 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], retrieved_urls.append(doc[0]) # url is first element elif len(doc) == 2: # Handle old format for backward compatibility retrieved_urls.append(doc[0]) # url is first element - + + # DEBUG: Log kept_docs structure + if not retrieved_urls and kept_docs: + print(f" WARNING: kept_docs has {len(kept_docs)} items but retrieved_urls is empty!") + print(f" First kept_doc structure: {type(kept_docs[0])}, len={len(kept_docs[0]) if hasattr(kept_docs[0], '__len__') else 'N/A'}") + if kept_docs: + print(f" First kept_doc sample: {str(kept_docs[0])[:200]}...") + # Limit to top_k_reranking (reranking already done per-subquery) retrieved_urls = retrieved_urls[:top_k_reranking] @@ -1482,7 +1536,12 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], from evaluation import calculate_retrieval_metrics expected_set = set(url for url in expected_urls if url and url.strip()) metrics = calculate_retrieval_metrics(list(expected_set), retrieved_urls) - + + # Add backward-compatible aliases for SUT (expects precision/recall/f1 not precision@N/recall@N/f1@N) + metrics['precision'] = metrics.get('precision@N', 0) + metrics['recall'] = metrics.get('recall@N', 0) + metrics['f1'] = metrics.get('f1@N', 0) + # Add iteration statistics query_llm_time = (llm_end_time - llm_start_time) if (llm_start_time is not None and llm_end_time is not None) else total_time metrics.update({ @@ -1494,6 +1553,7 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], 'sufficient': sufficient, 'avg_iteration_time': sum(iteration_times) / len(iteration_times) if iteration_times else 0, 'llm_answer': final_answer, + 'retrieved_urls': retrieved_urls, }) # Print final results diff --git a/e2e-rag/oracle_single_shot.py b/e2e-rag/oracle_single_shot.py index 1a39f7fd65..43bd0e29f1 100644 --- a/e2e-rag/oracle_single_shot.py +++ b/e2e-rag/oracle_single_shot.py @@ -46,9 +46,7 @@ DEFAULT_CHECKPOINT_FILE = "oracle_checkpoint.pkl" DEFAULT_SERVICE_URL = "http://localhost:8123/v1/chat/completions" -#DEFAULT_MODEL_NAME = "/mnt/weka/data/pytorch/llama3.3/Meta-Llama-3.3-70B-Instruct" -#DEFAULT_MODEL_NAME = "/mnt/weka/data/pytorch/llama3.1/Meta-Llama-3.1-405B-Instruct-v2" -DEFAULT_MODEL_NAME = "/model/gpt-oss-120b-mxfp4" +DEFAULT_MODEL_NAME = "gpt-oss-120b-mxfp4" DEFAULT_BATCH_SIZE = 1 DEFAULT_TIMEOUT = 2400 # For reasoning model, it should be large enough diff --git a/e2e-rag/params.py b/e2e-rag/params.py index b59e573937..16a62f8553 100644 --- a/e2e-rag/params.py +++ b/e2e-rag/params.py @@ -189,7 +189,7 @@ def suggest_value(self, trial): name="retriever_model", arg_names=["--retriever_model"], type=str, - default="intfloat/e5-base-v2", + default="intfloat_e5-base-v2/e5-base-v2", help="Model to use for embedding-based retrieval", category="common", applies_to=["vector"] @@ -198,7 +198,7 @@ def suggest_value(self, trial): name="reranker_model", arg_names=["--reranker_model"], type=str, - default="colbert-ir/colbertv2.0", + default="colbert-ir_colbertv2.0/colbertv2.0", help="Model to use for reranking (currently unused)", category="common", applies_to=["both"] @@ -320,8 +320,8 @@ def suggest_value(self, trial): name="llm_service_url", arg_names=["--llm_service_url"], type=str, - default="http://127.0.0.1:8123/v1/chat/completions", - help="URL for the LLM service endpoint", + default="http://127.0.0.1:8192/v1/chat/completions", + help="URL for the LLM service endpoint (20B model on port 8192)", category="general", applies_to=["both"] ), @@ -338,14 +338,14 @@ def suggest_value(self, trial): name="query_model", arg_names=["--query_model"], type=str, - default=None, - help="LLM model name/path for query generation (generate_search_queries). Defaults to --llm_model if not set. Example: /model/gpt-oss-120b", + default="gpt-oss-120b-mxfp4", + help="LLM model name/path for query generation (generate_search_queries). Defaults to --llm_model if not set. Example: gpt-oss-120b-mxfp4", category="general", applies_to=["both"] ), # Per-component endpoint overrides. Each defaults to --llm_service_url when # not set; same for the model. Lets you split components across separate - # vLLM servers (e.g. 20B grader on :8124, 120B query/sufficiency on :8123). + # vLLM servers (e.g. 20B grader on :8192, 120B query/sufficiency on :8123). ParamDef( name="grader_service_url", arg_names=["--grader-service-url"], @@ -368,8 +368,8 @@ def suggest_value(self, trial): name="query_service_url", arg_names=["--query-service-url"], type=str, - default=None, - help="LLM service URL for query generation (default: --llm_service_url).", + default="http://127.0.0.1:8123/v1/chat/completions", + help="LLM service URL for query generation (120B model on port 8123, default: --llm_service_url).", category="general", applies_to=["both"] ), diff --git a/e2e-rag/reference_mlperf.py b/e2e-rag/reference_mlperf.py index 83ebc0fcd0..6c98f12d29 100644 --- a/e2e-rag/reference_mlperf.py +++ b/e2e-rag/reference_mlperf.py @@ -204,7 +204,7 @@ def main(): # Load config files if os.path.exists(args.user_conf): - settings.FromConfig(args.user_conf, "rag-qna", args.scenario) + settings.FromConfig(args.user_conf, "e2e-rag-qna", args.scenario) print(f"Loaded user config from {args.user_conf}") else: print(f"Warning: User config not found: {args.user_conf}") diff --git a/e2e-rag/reference_mlperf_accuracy.sh b/e2e-rag/reference_mlperf_accuracy.sh index 8e1a50f19d..542a222cfb 100644 --- a/e2e-rag/reference_mlperf_accuracy.sh +++ b/e2e-rag/reference_mlperf_accuracy.sh @@ -14,7 +14,7 @@ # limitations under the License. # ============================================================================ -# Accuracy test script for E2E DocGrader workload with MLPerf Loadgen +# Accuracy test script for E2E-RAG-QnA workload with MLPerf Loadgen echo "Time Start: $(date +%s)" @@ -23,8 +23,8 @@ export WORKSPACE_DIR=${WORKSPACE_DIR:-"/workspace"} export DATA_DIR=${DATA_DIR:-"frames-benchmark-dataset"} export DATASET_PATH="${DATA_DIR}/frames_dataset.tsv" export DATABASE="${DATABASE:-vector_html_hnsw_len768_ov32_word.db}" -export RUN_LOGS=${WORKSPACE_DIR}/run_output -export OUTPUT_DIR=${WORKSPACE_DIR}/output +export RUN_LOGS=${WORKSPACE_DIR}/run_output_e2e-rag-qna/accuracy +export OUTPUT_DIR=${WORKSPACE_DIR}/output_e2e-rag-qna/accuracy export SCENARIO="${SCENARIO:-Offline}" # Threading configuration @@ -48,12 +48,17 @@ export RERANKER_MODEL=${RERANKER_MODEL:-colbert-ir_colbertv2.0/colbertv2.0} export OPENROUTER_API_KEY=${OPENROUTER_API_KEY:-sk-or-v1-****} # Default to local vLLM server -export LLM_SERVICE_URL=${LLM_SERVICE_URL:-http://127.0.0.1:8123/v1/chat/completions} -export LLM_MODEL=${LLM_MODEL:-gpt-oss-20b} -export QUERY_MODEL=${QUERY_MODEL:-gpt-oss-120b} +export LLM_SERVICE_URL=${LLM_SERVICE_URL:-http://127.0.0.1:8192/v1/chat/completions} +export LLM_MODEL=${LLM_MODEL:-gpt-oss-20b-mxfp4} +export QUERY_MODEL=${QUERY_MODEL:-gpt-oss-120b-mxfp4} + +# Query and sufficiency use 120B model on port 8123 +export QUERY_SERVICE_URL=${QUERY_SERVICE_URL:-http://127.0.0.1:8123/v1/chat/completions} +export SUFFICIENCY_SERVICE_URL=${SUFFICIENCY_SERVICE_URL:-http://127.0.0.1:8123/v1/chat/completions} +export SUFFICIENCY_MODEL=${SUFFICIENCY_MODEL:-gpt-oss-120b-mxfp4} # Judge LLM configuration (for accuracy evaluation) -export JUDGE_SERVICE_URL=${JUDGE_SERVICE_URL:-http://127.0.0.1:8125/v1/chat/completions} +export JUDGE_SERVICE_URL=${JUDGE_SERVICE_URL:-http://127.0.0.1:8193/v1/chat/completions} export JUDGE_MODEL=${JUDGE_MODEL:-meta-llama/Llama-3.1-8B-Instruct} echo " LLM Service URL: ${LLM_SERVICE_URL}" @@ -79,7 +84,7 @@ if [ -n "${PERF_COUNT}" ]; then fi # Update user.conf with threading configuration -sed -i "s/^e2e.Offline.max_async_queries = .*/e2e.Offline.max_async_queries = ${MAX_ASYNC_QUERIES}/" user.conf +sed -i "s/^e2e-rag-qna.Offline.max_async_queries = .*/e2e-rag-qna.Offline.max_async_queries = ${MAX_ASYNC_QUERIES}/" user.conf # Run loadgen accuracy test python3 reference_mlperf.py \ @@ -98,8 +103,14 @@ python3 reference_mlperf.py \ --llm_service_url ${LLM_SERVICE_URL} \ --llm_model ${LLM_MODEL} \ --query_model ${QUERY_MODEL} \ + --query-service-url ${QUERY_SERVICE_URL} \ + --sufficiency-service-url ${SUFFICIENCY_SERVICE_URL} \ + --sufficiency-model ${SUFFICIENCY_MODEL} \ --judge_service_url ${JUDGE_SERVICE_URL} \ --judge_model ${JUDGE_MODEL} \ --accuracy +EXIT_CODE=$? + echo "Time Stop: $(date +%s)" +exit ${EXIT_CODE} diff --git a/e2e-rag/reference_mlperf_datasetup.py b/e2e-rag/reference_mlperf_datasetup.py index 5ec27d1232..ce2feeac4b 100644 --- a/e2e-rag/reference_mlperf_datasetup.py +++ b/e2e-rag/reference_mlperf_datasetup.py @@ -210,7 +210,7 @@ def main(): # Load config files if os.path.exists(args.user_conf): - settings.FromConfig(args.user_conf, "rag-db", args.scenario) + settings.FromConfig(args.user_conf, "e2e-rag-db", args.scenario) print(f"Loaded user config from {args.user_conf}") else: print(f"Warning: User config not found: {args.user_conf}") diff --git a/e2e-rag/reference_mlperf_datasetup.sh b/e2e-rag/reference_mlperf_datasetup.sh index fba93c0a24..df82def111 100755 --- a/e2e-rag/reference_mlperf_datasetup.sh +++ b/e2e-rag/reference_mlperf_datasetup.sh @@ -22,8 +22,8 @@ echo "Time Start: $(date +%s)" export WORKSPACE_DIR=${WORKSPACE_DIR:-"/workspace"} export DOCUMENTS_DIR=${DOCUMENTS_DIR:-"doc_html"} export DATABASE="${DATABASE:-vector_html_hnsw_len768_ov32_word}" -export RUN_LOGS=${WORKSPACE_DIR}/run_output_datasetup -export OUTPUT_DIR=${WORKSPACE_DIR}/output_datasetup +export RUN_LOGS=${WORKSPACE_DIR}/run_output_e2e-rag-db/performance +export OUTPUT_DIR=${WORKSPACE_DIR}/output_e2e-rag-db/performance export SCENARIO="${SCENARIO:-Offline}" # Chunking configuration @@ -81,16 +81,16 @@ fi if [ -f "user.conf" ]; then # Update max_async_queries to match HTML count (send all at once) # Update min_query_count to match HTML count - if grep -q "e2e-datasetup.Offline.max_async_queries" user.conf; then - sed -i "s/^e2e-datasetup.Offline.max_async_queries = .*/e2e-datasetup.Offline.max_async_queries = ${HTML_COUNT}/" user.conf + if grep -q "e2e-rag-db.Offline.max_async_queries" user.conf; then + sed -i "s/^e2e-rag-db.Offline.max_async_queries = .*/e2e-rag-db.Offline.max_async_queries = ${HTML_COUNT}/" user.conf else - echo "e2e-datasetup.Offline.max_async_queries = ${HTML_COUNT}" >> user.conf + echo "e2e-rag-db.Offline.max_async_queries = ${HTML_COUNT}" >> user.conf fi - if grep -q "e2e-datasetup.Offline.min_query_count" user.conf; then - sed -i "s/^e2e-datasetup.Offline.min_query_count = .*/e2e-datasetup.Offline.min_query_count = ${HTML_COUNT}/" user.conf + if grep -q "e2e-rag-db.Offline.min_query_count" user.conf; then + sed -i "s/^e2e-rag-db.Offline.min_query_count = .*/e2e-rag-db.Offline.min_query_count = ${HTML_COUNT}/" user.conf else - echo "e2e-datasetup.Offline.min_query_count = ${HTML_COUNT}" >> user.conf + echo "e2e-rag-db.Offline.min_query_count = ${HTML_COUNT}" >> user.conf fi echo " Loadgen configured to dispatch all ${HTML_COUNT} files at once" diff --git a/e2e-rag/reference_mlperf_datasetup_accuracy.sh b/e2e-rag/reference_mlperf_datasetup_accuracy.sh index 1a6faaa598..85e262c88b 100755 --- a/e2e-rag/reference_mlperf_datasetup_accuracy.sh +++ b/e2e-rag/reference_mlperf_datasetup_accuracy.sh @@ -22,8 +22,8 @@ echo "Time Start: $(date +%s)" export WORKSPACE_DIR=${WORKSPACE_DIR:-"/workspace"} export DOCUMENTS_DIR=${DOCUMENTS_DIR:-"doc_html"} export DATABASE="${DATABASE:-vector_html_hnsw_len768_ov32_word}" -export RUN_LOGS=${WORKSPACE_DIR}/run_output_datasetup_accuracy -export OUTPUT_DIR=${WORKSPACE_DIR}/output_datasetup_accuracy +export RUN_LOGS=${WORKSPACE_DIR}/run_output_e2e-rag-db/accuracy +export OUTPUT_DIR=${WORKSPACE_DIR}/output_e2e-rag-db/accuracy export SCENARIO="${SCENARIO:-Offline}" # Chunking configuration @@ -42,6 +42,15 @@ export NUM_EMBEDDING_DEVICES=${NUM_EMBEDDING_DEVICES:-1} # Vector database configuration export VECTOR_INDEX_METHOD=${VECTOR_INDEX_METHOD:-"hnsw"} +# Reference DB manifest for cross-system verification (corpus fingerprint, +# sample-embedding cosine, probe-query top-K ranks). +# Set to "" or "none" to skip the manifest check. Note: ${VAR:-default} treats +# an empty value the same as unset, so an explicit "none" sentinel is the +# reliable way to skip from a parent script that exports MANIFEST="". +export MANIFEST=${MANIFEST-scripts/db_manifest_intel_xpu.json.gz} +export COSINE_THRESHOLD=${COSINE_THRESHOLD:-0.9999} +export TOP_K_DEPTH=${TOP_K_DEPTH:-3} + # Performance options export BENCHMARK=${BENCHMARK:-false} export MAX_WORKERS=${MAX_WORKERS:-4} @@ -81,16 +90,16 @@ fi if [ -f "user.conf" ]; then # Update max_async_queries to match HTML count (send all at once) # Update min_query_count to match HTML count - if grep -q "e2e-datasetup.Offline.max_async_queries" user.conf; then - sed -i "s/^e2e-datasetup.Offline.max_async_queries = .*/e2e-datasetup.Offline.max_async_queries = ${HTML_COUNT}/" user.conf + if grep -q "e2e-rag-db.Offline.max_async_queries" user.conf; then + sed -i "s/^e2e-rag-db.Offline.max_async_queries = .*/e2e-rag-db.Offline.max_async_queries = ${HTML_COUNT}/" user.conf else - echo "e2e-datasetup.Offline.max_async_queries = ${HTML_COUNT}" >> user.conf + echo "e2e-rag-db.Offline.max_async_queries = ${HTML_COUNT}" >> user.conf fi - if grep -q "e2e-datasetup.Offline.min_query_count" user.conf; then - sed -i "s/^e2e-datasetup.Offline.min_query_count = .*/e2e-datasetup.Offline.min_query_count = ${HTML_COUNT}/" user.conf + if grep -q "e2e-rag-db.Offline.min_query_count" user.conf; then + sed -i "s/^e2e-rag-db.Offline.min_query_count = .*/e2e-rag-db.Offline.min_query_count = ${HTML_COUNT}/" user.conf else - echo "e2e-datasetup.Offline.min_query_count = ${HTML_COUNT}" >> user.conf + echo "e2e-rag-db.Offline.min_query_count = ${HTML_COUNT}" >> user.conf fi echo " Loadgen configured to dispatch all ${HTML_COUNT} files at once (accuracy mode)" @@ -133,11 +142,18 @@ if [ ${EXIT_CODE} -eq 0 ]; then echo "Running Accuracy Evaluation" echo "============================================================" + # Build optional manifest argument (skip check if MANIFEST is empty or "none") + MANIFEST_ARG="" + if [ -n "${MANIFEST}" ] && [ "${MANIFEST,,}" != "none" ]; then + MANIFEST_ARG="--manifest ${MANIFEST} --cosine_threshold ${COSINE_THRESHOLD} --top_k_depth ${TOP_K_DEPTH}" + fi + python3 datasetup_accuracy_eval.py \ --log_dir ${RUN_LOGS} \ --output_dir ${OUTPUT_DIR} \ --database ${DATABASE}.db \ - --retriever_model ${RETRIEVER_MODEL} + --retriever_model ${RETRIEVER_MODEL} \ + ${MANIFEST_ARG} EVAL_EXIT_CODE=$? diff --git a/e2e-rag/reference_mlperf_perf.sh b/e2e-rag/reference_mlperf_perf.sh index eb38215279..7e784f502d 100644 --- a/e2e-rag/reference_mlperf_perf.sh +++ b/e2e-rag/reference_mlperf_perf.sh @@ -14,7 +14,7 @@ # limitations under the License. # ============================================================================ -# Performance test script for E2E DocGrader workload with MLPerf Loadgen +# Performance test script for E2E-RAG-QnA workload with MLPerf Loadgen echo "Time Start: $(date +%s)" @@ -23,8 +23,8 @@ export WORKSPACE_DIR=${WORKSPACE_DIR:-"/workspace"} export DATA_DIR=${DATA_DIR:-"frames-benchmark-dataset"} export DATASET_PATH="${DATA_DIR}/frames_dataset.tsv" export DATABASE="${DATABASE:-vector_html_hnsw_len768_ov32_word.db}" -export RUN_LOGS=${WORKSPACE_DIR}/run_output -export OUTPUT_DIR=${WORKSPACE_DIR}/output +export RUN_LOGS=${WORKSPACE_DIR}/run_output_e2e-rag-qna/performance +export OUTPUT_DIR=${WORKSPACE_DIR}/output_e2e-rag-qna/performance export SCENARIO="${SCENARIO:-Offline}" # Threading configuration @@ -51,13 +51,16 @@ export RERANKER_MODEL=${RERANKER_MODEL:-colbert-ir_colbertv2.0/colbertv2.0} export OPENROUTER_API_KEY=${OPENROUTER_API_KEY:-sk-or-v1-****} # Separate service endpoints for each LLM component -export LLM_SERVICE_URL=${LLM_SERVICE_URL:-http://127.0.0.1:8123/v1/chat/completions} -export LLM_MODEL=${LLM_MODEL:-gpt-oss-20b} +export LLM_SERVICE_URL=${LLM_SERVICE_URL:-http://127.0.0.1:8192/v1/chat/completions} +export LLM_MODEL=${LLM_MODEL:-gpt-oss-20b-mxfp4} -export QUERY_SERVICE_URL=${QUERY_SERVICE_URL:-http://127.0.0.1:8124/v1/chat/completions} -export QUERY_MODEL=${QUERY_MODEL:-gpt-oss-120b} +export QUERY_SERVICE_URL=${QUERY_SERVICE_URL:-http://127.0.0.1:8123/v1/chat/completions} +export QUERY_MODEL=${QUERY_MODEL:-gpt-oss-120b-mxfp4} -export JUDGE_SERVICE_URL=${JUDGE_SERVICE_URL:-http://127.0.0.1:8125/v1/chat/completions} +export SUFFICIENCY_SERVICE_URL=${SUFFICIENCY_SERVICE_URL:-http://127.0.0.1:8123/v1/chat/completions} +export SUFFICIENCY_MODEL=${SUFFICIENCY_MODEL:-gpt-oss-120b-mxfp4} + +export JUDGE_SERVICE_URL=${JUDGE_SERVICE_URL:-http://127.0.0.1:8193/v1/chat/completions} export JUDGE_MODEL=${JUDGE_MODEL:-meta-llama/Llama-3.1-8B-Instruct} echo "Configuration:" @@ -80,7 +83,7 @@ echo " JUDGE_SERVICE_URL: ${JUDGE_SERVICE_URL}" echo " JUDGE_MODEL: ${JUDGE_MODEL}" # Update user.conf with threading configuration -sed -i "s/^e2e.Offline.max_async_queries = .*/e2e.Offline.max_async_queries = ${MAX_ASYNC_QUERIES}/" user.conf +sed -i "s/^e2e-rag-qna.Offline.max_async_queries = .*/e2e-rag-qna.Offline.max_async_queries = ${MAX_ASYNC_QUERIES}/" user.conf # Build perf cache argument if file exists PERF_CACHE_ARG="" @@ -109,8 +112,13 @@ python3 reference_mlperf.py \ --llm_model ${LLM_MODEL} \ --query_service_url ${QUERY_SERVICE_URL} \ --query_model ${QUERY_MODEL} \ + --sufficiency-service-url ${SUFFICIENCY_SERVICE_URL} \ + --sufficiency-model ${SUFFICIENCY_MODEL} \ --judge_service_url ${JUDGE_SERVICE_URL} \ --judge_model ${JUDGE_MODEL} \ ${PERF_CACHE_ARG} +EXIT_CODE=$? + echo "Time Stop: $(date +%s)" +exit ${EXIT_CODE} diff --git a/e2e-rag/reranker_worker.py b/e2e-rag/reranker_worker.py index e3acfea045..04ce552044 100644 --- a/e2e-rag/reranker_worker.py +++ b/e2e-rag/reranker_worker.py @@ -21,7 +21,7 @@ - Its own CPU affinity + memory binding (per-NUMA-node placement). - Isolation from the main process's GIL / thread pool. -Public API kept: RerankerQueue.submit(query, passages) -> [(passage, score), ...] +Public API kept: RerankerQueue.submit(query, passages) -> [(passage, score), ...] in input order. """ import ctypes @@ -126,9 +126,8 @@ def _do_rerank(model, tokenizer, device: str, query: str, passages: List[str]) - sim = sim.masked_fill(~d_mask.unsqueeze(1).bool(), float('-inf')) scores = sim.max(dim=-1).values.sum(dim=-1) - scored_passages = list(zip(passages, scores.float().tolist())) - scored_passages.sort(key=lambda x: x[1], reverse=True) - return scored_passages + # Return (passage, score) in input order; caller sorts/selects. + return list(zip(passages, scores.float().tolist())) class RerankerQueue: diff --git a/e2e-rag/retrieve/ragdb.py b/e2e-rag/retrieve/ragdb.py index 19f65c4416..33bd75900a 100644 --- a/e2e-rag/retrieve/ragdb.py +++ b/e2e-rag/retrieve/ragdb.py @@ -241,38 +241,20 @@ def shutdown_reranker(self): self._reranker_queue = None def rerank(self, query: str, passages: List[str]): - """Rerank passages via the reranker queue (ColBERT MaxSim).""" + """Score passages via the reranker; returns (passage, score) in input order.""" if self._reranker_queue: return self._reranker_queue.submit(query, passages) return [(p, 0.0) for p in passages] - def lookup_with_rerank(self, query: str, k: int, rerank_k: int = None) -> List[Any]: - """Retrieve and rerank passages.""" - if rerank_k is None: - rerank_k = k - - # Get initial results - results = self.lookup(query, k=rerank_k) - - # If no reranker or fewer results than requested, return as-is - if self._reranker_queue is None or len(results) <= k: - return results[:k] - - # Extract passages for reranking - passages = [result.page_content for result in results] - - # Rerank - reranked_passages = self.rerank(query, passages) - - # Map back to original results and return top-k - reranked_results = [] - for passage, score in reranked_passages[:k]: - for result in results: - if result.page_content == passage: - reranked_results.append(result) - break - - return reranked_results + def rerank_documents(self, query: str, documents: List[Any], top_k: int = None) -> List[Any]: + """Rerank Documents by score (desc) and return top-k; sole place sorting happens.""" + if not documents: + return [] + passages = [d.page_content for d in documents] + scored = self.rerank(query, passages) # input order: scored[i] <-> documents[i] + order = sorted(range(len(documents)), key=lambda i: scored[i][1], reverse=True) + ranked = [documents[i] for i in order] + return ranked[:top_k] if top_k is not None else ranked @property def device(self) -> str: diff --git a/e2e-rag/run_compliance_test09.sh b/e2e-rag/run_compliance_test09.sh index 483d6590bc..4ee05e7209 100755 --- a/e2e-rag/run_compliance_test09.sh +++ b/e2e-rag/run_compliance_test09.sh @@ -14,7 +14,7 @@ # limitations under the License. # ============================================================================ -# TEST09 Compliance Test Runner for E2E DocGrader Workload +# TEST09 Compliance Test Runner for E2E-RAG-QnA Workload # Automates: setup -> run -> verify -> cleanup workflow set -e # Exit on error @@ -27,7 +27,11 @@ echo "" # Configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -COMPLIANCE_DIR="${SCRIPT_DIR}/../compliance/TEST09/e2e-rag" +# The TEST09 config comes from the main inference repo's compliance tree. +# Copy compliance/TEST09/e2e-rag-qna/ into this directory before running +# (e.g. when e2e-rag is mounted standalone into a container), or override +# COMPLIANCE_DIR to point at it. +COMPLIANCE_DIR="${COMPLIANCE_DIR:-${SCRIPT_DIR}/../compliance/TEST09/e2e-rag-qna}" AUDIT_CONFIG="${COMPLIANCE_DIR}/audit.config" WORKING_AUDIT_CONFIG="${SCRIPT_DIR}/audit.config" TEST09_VERIFICATION="${SCRIPT_DIR}/third_party/mlperf-inference/compliance/TEST09/run_verification.py" @@ -39,11 +43,12 @@ export DATASET_PATH="${DATA_DIR}/frames_dataset.tsv" export DATABASE="${DATABASE:-vector_html_hnsw_len768_ov32_word.db}" export RUN_LOGS=${WORKSPACE_DIR}/run_output_test09 export OUTPUT_DIR=${WORKSPACE_DIR}/output_test09 -export SUBMISSION_DIR=${WORKSPACE_DIR}/submission/compliance/e2e-rag/Offline +export SUBMISSION_DIR=${WORKSPACE_DIR}/submission/compliance/e2e-rag-qna/Offline export SCENARIO="${SCENARIO:-Offline}" -# Performance testing - full dataset for compliance -export PERF_COUNT=824 +# Performance testing - full dataset for compliance. +# Overridable (e.g. for a smoke run); a valid TEST09 submission needs 824. +export PERF_COUNT=${PERF_COUNT:-824} # Threading configuration export MAX_ASYNC_QUERIES=${MAX_ASYNC_QUERIES:-10} @@ -55,15 +60,17 @@ export MAX_SUB_QUERIES=${MAX_SUB_QUERIES:-3} export TOP_K_RETRIEVER=${TOP_K_RETRIEVER:-10} # Model paths -export RETRIEVER_MODEL=${RETRIEVER_MODEL:-/data/model/e5-base-v2} -export RERANKER_MODEL=${RERANKER_MODEL:-/data/model/colbertv2.0} +export RETRIEVER_MODEL=${RETRIEVER_MODEL:-intfloat_e5-base-v2/e5-base-v2} +export RERANKER_MODEL=${RERANKER_MODEL:-colbert-ir_colbertv2.0/colbertv2.0} # LLM service configuration -export LLM_SERVICE_URL=${LLM_SERVICE_URL:-http://127.0.0.1:8123/v1/chat/completions} -export LLM_MODEL=${LLM_MODEL:-gpt-oss-20b} -export QUERY_SERVICE_URL=${QUERY_SERVICE_URL:-http://127.0.0.1:8124/v1/chat/completions} -export QUERY_MODEL=${QUERY_MODEL:-gpt-oss-120b} -export JUDGE_SERVICE_URL=${JUDGE_SERVICE_URL:-http://127.0.0.1:8125/v1/chat/completions} +export LLM_SERVICE_URL=${LLM_SERVICE_URL:-http://127.0.0.1:8192/v1/chat/completions} +export LLM_MODEL=${LLM_MODEL:-gpt-oss-20b-mxfp4} +export QUERY_SERVICE_URL=${QUERY_SERVICE_URL:-http://127.0.0.1:8123/v1/chat/completions} +export QUERY_MODEL=${QUERY_MODEL:-gpt-oss-120b-mxfp4} +export SUFFICIENCY_SERVICE_URL=${SUFFICIENCY_SERVICE_URL:-http://127.0.0.1:8123/v1/chat/completions} +export SUFFICIENCY_MODEL=${SUFFICIENCY_MODEL:-gpt-oss-120b-mxfp4} +export JUDGE_SERVICE_URL=${JUDGE_SERVICE_URL:-http://127.0.0.1:8193/v1/chat/completions} export JUDGE_MODEL=${JUDGE_MODEL:-meta-llama/Llama-3.1-8B-Instruct} # Performance cache file (optional - for faster testing) @@ -109,7 +116,11 @@ mkdir -p "${SUBMISSION_DIR}" # Copy audit.config to working directory echo "Copying audit.config to working directory..." cp "${AUDIT_CONFIG}" "${WORKING_AUDIT_CONFIG}" -echo "✓ audit.config copied to ${WORKING_AUDIT_CONFIG}" +# Keep the audit.config's min_query_count in sync with PERF_COUNT. Otherwise +# loadgen honors the config's min_query_count (824) and loops back up to it even +# when PERF_COUNT is smaller (e.g. a smoke run). A real submission uses 824. +sed -i "s/^\*\.\*\.min_query_count = .*/*.*.min_query_count = ${PERF_COUNT}/" "${WORKING_AUDIT_CONFIG}" +echo "✓ audit.config copied to ${WORKING_AUDIT_CONFIG} (min_query_count=${PERF_COUNT})" echo "" # ============================================================================ @@ -129,11 +140,15 @@ if [ -n "${PERF_CACHE_FILE}" ] && [ -f "${PERF_CACHE_FILE}" ]; then fi # Run loadgen performance test -# Note: LoadGen automatically detects audit.config in the current directory +# reference_mlperf.py passes --audit_conf explicitly to StartTestWithLogSettings, +# so we must point it at the copied audit.config (named audit.config, whereas +# the default arg is audit.conf). Without this, loadgen never applies the TEST09 +# accuracy_log_sampling_target and mlperf_log_accuracy.json comes out empty. python3 reference_mlperf.py \ --dataset_path ${DATASET_PATH} \ --database ${DATABASE} \ --scenario ${SCENARIO} \ + --audit_conf ${WORKING_AUDIT_CONFIG} \ --log_dir ${RUN_LOGS} \ --output_dir ${OUTPUT_DIR} \ --perf_count ${PERF_COUNT} \ @@ -147,6 +162,8 @@ python3 reference_mlperf.py \ --llm_model ${LLM_MODEL} \ --query_service_url ${QUERY_SERVICE_URL} \ --query_model ${QUERY_MODEL} \ + --sufficiency-service-url ${SUFFICIENCY_SERVICE_URL} \ + --sufficiency-model ${SUFFICIENCY_MODEL} \ --judge_service_url ${JUDGE_SERVICE_URL} \ --judge_model ${JUDGE_MODEL} \ ${PERF_CACHE_ARG} diff --git a/e2e-rag/scripts/run_ingestion.sh b/e2e-rag/scripts/run_ingestion.sh index 7b3348bbcd..f4c21845fc 100644 --- a/e2e-rag/scripts/run_ingestion.sh +++ b/e2e-rag/scripts/run_ingestion.sh @@ -27,7 +27,7 @@ INGESTION_EMBEDDING_DEVICE="${INGESTION_EMBEDDING_DEVICE:-${INGESTION_DEVICE}}" INGESTION_NUM_EMBEDDING_DEVICES="${INGESTION_NUM_EMBEDDING_DEVICES:-4}" INGESTION_CHUNK_LEN="${INGESTION_CHUNK_LEN:-768}" INGESTION_CHUNK_OVERLAP="${INGESTION_CHUNK_OVERLAP:-32}" -INGESTION_RETRIEVER_MODEL="${INGESTION_RETRIEVER_MODEL:-/data/model/e5-base-v2}" +INGESTION_RETRIEVER_MODEL="${INGESTION_RETRIEVER_MODEL:-intfloat_e5-base-v2/e5-base-v2}" INGESTION_DOC_DIR="${INGESTION_DOC_DIR:-doc_html}" INGESTION_PASSAGES_JSON="${INGESTION_PASSAGES_JSON:-passages/doc_html_len${INGESTION_CHUNK_LEN}_ov${INGESTION_CHUNK_OVERLAP}_word.json}" INGESTION_DB="${INGESTION_DB:-vector_html_hnsw_len${INGESTION_CHUNK_LEN}_ov${INGESTION_CHUNK_OVERLAP}_word}" diff --git a/e2e-rag/scripts/run_multi_shot.sh b/e2e-rag/scripts/run_multi_shot.sh index 445bfd3bf9..105315e1db 100644 --- a/e2e-rag/scripts/run_multi_shot.sh +++ b/e2e-rag/scripts/run_multi_shot.sh @@ -14,7 +14,7 @@ # e.g.: INFERENCE_DEVICE=cpu bash scripts/run_multi_shot.sh 50 # # Prerequisites: -# - Local vLLM server running on port 8123 (default) +# - Local vLLM servers running: 20B on port 8192, 120B on port 8123 (default) # - OR set OPENROUTER_API_KEY environment variable to use OpenRouter # - scripts/run_ingestion.sh has been run (vector DB exists) # @@ -38,7 +38,7 @@ INFERENCE_DEVICE="${INFERENCE_DEVICE:-cpu}" INFERENCE_EMBEDDING_DEVICE="${INFERENCE_EMBEDDING_DEVICE:-${INFERENCE_DEVICE}}" INFERENCE_RERANKER_DEVICE="${INFERENCE_RERANKER_DEVICE:-${INFERENCE_DEVICE}}" INFERENCE_DB="${INFERENCE_DB:-vector_html_hnsw_len768_ov32_word}" -INFERENCE_RETRIEVER_MODEL="${INFERENCE_RETRIEVER_MODEL:-/data/model/e5-base-v2}" +INFERENCE_RETRIEVER_MODEL="${INFERENCE_RETRIEVER_MODEL:-intfloat_e5-base-v2/e5-base-v2}" INFERENCE_TOP_K_RETRIEVER="${INFERENCE_TOP_K_RETRIEVER:-15}" INFERENCE_MAX_ITERATIONS="${INFERENCE_MAX_ITERATIONS:-5}" INFERENCE_MAX_SUB_QUERIES="${INFERENCE_MAX_SUB_QUERIES:-3}" @@ -47,18 +47,18 @@ INFERENCE_REASONING="${INFERENCE_REASONING:-medium}" INFERENCE_MAX_RETRIES="${INFERENCE_MAX_RETRIES:-5}" INFERENCE_N_QUERIES="${INFERENCE_N_QUERIES:-5}" INFERENCE_NUM_WORKERS="${INFERENCE_NUM_WORKERS:-1}" -INFERENCE_LLM_URL="${INFERENCE_LLM_URL:-http://127.0.0.1:8123/v1/chat/completions}" -INFERENCE_MODEL="${INFERENCE_MODEL:-/model/gpt-oss-20b-mxfp4}" -INFERENCE_QUERY_MODEL="${INFERENCE_QUERY_MODEL:-/model/gpt-oss-120b-mxfp4}" +INFERENCE_LLM_URL="${INFERENCE_LLM_URL:-http://127.0.0.1:8192/v1/chat/completions}" +INFERENCE_MODEL="${INFERENCE_MODEL:-gpt-oss-20b-mxfp4}" +INFERENCE_QUERY_MODEL="${INFERENCE_QUERY_MODEL:-gpt-oss-120b-mxfp4}" # Per-component endpoint splits. Empty -> inherit INFERENCE_LLM_URL / INFERENCE_MODEL. INFERENCE_GRADER_URL="${INFERENCE_GRADER_URL:-}" INFERENCE_GRADER_MODEL="${INFERENCE_GRADER_MODEL:-}" INFERENCE_QUERY_URL="${INFERENCE_QUERY_URL:-}" INFERENCE_SUFFICIENCY_URL="${INFERENCE_SUFFICIENCY_URL:-}" -INFERENCE_SUFFICIENCY_MODEL="${INFERENCE_SUFFICIENCY_MODEL:-}" -INFERENCE_JUDGE_URL="${INFERENCE_JUDGE_URL:-http://127.0.0.1:8123/v1/chat/completions}" -INFERENCE_JUDGE_MODEL="${INFERENCE_JUDGE_MODEL:-gpt-oss-20b}" +INFERENCE_SUFFICIENCY_MODEL="${INFERENCE_SUFFICIENCY_MODEL:-gpt-oss-120b-mxfp4}" +INFERENCE_JUDGE_URL="${INFERENCE_JUDGE_URL:-http://127.0.0.1:8192/v1/chat/completions}" +INFERENCE_JUDGE_MODEL="${INFERENCE_JUDGE_MODEL:-gpt-oss-20b-mxfp4}" INFERENCE_PERF_TEST_MODE="${INFERENCE_PERF_TEST_MODE:-}" # Positional args override config. diff --git a/e2e-rag/scripts/run_oracle.sh b/e2e-rag/scripts/run_oracle.sh index 7539a6ac15..8e57ab687a 100644 --- a/e2e-rag/scripts/run_oracle.sh +++ b/e2e-rag/scripts/run_oracle.sh @@ -26,7 +26,7 @@ else fi INFERENCE_LLM_URL="${INFERENCE_LLM_URL:-http://127.0.0.1:8123/v1/chat/completions}" -INFERENCE_MODEL="${INFERENCE_MODEL:-/model/gpt-oss-20b-mxfp4}" +INFERENCE_MODEL="${INFERENCE_MODEL:-gpt-oss-120b-mxfp4}" INFERENCE_N_QUERIES="${INFERENCE_N_QUERIES:-5}" INFERENCE_ORACLE_BATCH_SIZE="${INFERENCE_ORACLE_BATCH_SIZE:-4}" INFERENCE_ORACLE_TIMEOUT="${INFERENCE_ORACLE_TIMEOUT:-2400}" diff --git a/e2e-rag/scripts/run_single_shot.sh b/e2e-rag/scripts/run_single_shot.sh index 148ce38b57..f0c4503f64 100644 --- a/e2e-rag/scripts/run_single_shot.sh +++ b/e2e-rag/scripts/run_single_shot.sh @@ -28,11 +28,11 @@ INFERENCE_DEVICE="${INFERENCE_DEVICE:-cpu}" INFERENCE_EMBEDDING_DEVICE="${INFERENCE_EMBEDDING_DEVICE:-${INFERENCE_DEVICE}}" INFERENCE_RERANKER_DEVICE="${INFERENCE_RERANKER_DEVICE:-${INFERENCE_DEVICE}}" INFERENCE_DB="${INFERENCE_DB:-vector_html_hnsw_len768_ov32_word}" -INFERENCE_RETRIEVER_MODEL="${INFERENCE_RETRIEVER_MODEL:-/data/model/e5-base-v2}" +INFERENCE_RETRIEVER_MODEL="${INFERENCE_RETRIEVER_MODEL:-intfloat_e5-base-v2/e5-base-v2}" INFERENCE_TOP_K_RETRIEVER="${INFERENCE_TOP_K_RETRIEVER:-15}" INFERENCE_N_QUERIES="${INFERENCE_N_QUERIES:-5}" -INFERENCE_LLM_URL="${INFERENCE_LLM_URL:-http://127.0.0.1:8123/v1/chat/completions}" -INFERENCE_MODEL="${INFERENCE_MODEL:-/model/gpt-oss-20b-mxfp4}" +INFERENCE_LLM_URL="${INFERENCE_LLM_URL:-http://127.0.0.1:8192/v1/chat/completions}" +INFERENCE_MODEL="${INFERENCE_MODEL:-gpt-oss-20b-mxfp4}" INFERENCE_JUDGE_URL="${INFERENCE_JUDGE_URL:-https://openrouter.ai/api/v1/chat/completions}" INFERENCE_JUDGE_MODEL="${INFERENCE_JUDGE_MODEL:-openai/gpt-oss-20b}" diff --git a/e2e-rag/scripts/start_vllm_server.sh b/e2e-rag/scripts/start_vllm_server.sh index 6a7871a555..1227129e53 100644 --- a/e2e-rag/scripts/start_vllm_server.sh +++ b/e2e-rag/scripts/start_vllm_server.sh @@ -1,5 +1,5 @@ python3 -m vllm.entrypoints.openai.api_server \ - --model /model/gpt-oss-20b-mxfp4 \ + --model gpt-oss-20b-mxfp4 \ --dtype bfloat16 \ --enforce-eager \ --host 0.0.0.0 \ @@ -10,6 +10,6 @@ python3 -m vllm.entrypoints.openai.api_server \ --disable-log-requests \ --max-model-len=131072 \ --block-size 64 \ - --port 8123 \ + --port 8192 \ -tp 4 \ --async_scheduling \ No newline at end of file diff --git a/e2e-rag/scripts/verify_db_manifest.sh b/e2e-rag/scripts/verify_db_manifest.sh index 495f61478a..1a825fc386 100644 --- a/e2e-rag/scripts/verify_db_manifest.sh +++ b/e2e-rag/scripts/verify_db_manifest.sh @@ -29,6 +29,7 @@ else fi INFERENCE_DB="${INFERENCE_DB:-vector_html_hnsw_len768_ov32_word}" +INFERENCE_RETRIEVER_MODEL="${INFERENCE_RETRIEVER_MODEL:-intfloat_e5-base-v2/e5-base-v2}" MANIFEST="$1" COSINE_THRESHOLD="${2:-0.9999}" @@ -36,6 +37,7 @@ TOP_K_DEPTH="${3:-3}" echo "=== Verifying DB against manifest ===" echo " DB: ${INFERENCE_DB}" +echo " Retriever: ${INFERENCE_RETRIEVER_MODEL}" echo " Manifest: ${MANIFEST}" echo " Cosine threshold: ${COSINE_THRESHOLD}" echo " Top-K depth: ${TOP_K_DEPTH}" @@ -44,5 +46,6 @@ echo "" python3 -u db_manifest.py verify \ --db "${INFERENCE_DB}" \ --manifest "${MANIFEST}" \ + --retriever_model "${INFERENCE_RETRIEVER_MODEL}" \ --cosine-threshold "${COSINE_THRESHOLD}" \ --top-k-depth "${TOP_K_DEPTH}" diff --git a/e2e-rag/scripts/write_db_manifest.sh b/e2e-rag/scripts/write_db_manifest.sh index 01b6580f8d..2b29767fa8 100644 --- a/e2e-rag/scripts/write_db_manifest.sh +++ b/e2e-rag/scripts/write_db_manifest.sh @@ -22,7 +22,7 @@ else fi INFERENCE_DB="${INFERENCE_DB:-vector_html_hnsw_len768_ov32_word}" -INFERENCE_RETRIEVER_MODEL="${INFERENCE_RETRIEVER_MODEL:-/data/model/e5-base-v2}" +INFERENCE_RETRIEVER_MODEL="${INFERENCE_RETRIEVER_MODEL:-intfloat_e5-base-v2/e5-base-v2}" OUTPUT="${1:-db_manifest_$(hostname -s).json.gz}" diff --git a/e2e-rag/user.conf b/e2e-rag/user.conf index e920ef67ad..c023795e57 100644 --- a/e2e-rag/user.conf +++ b/e2e-rag/user.conf @@ -11,14 +11,14 @@ # min_query_count takes priority - loadgen will run AT LEAST this many unique queries # min_duration is secondary - only matters if fewer queries would complete faster # max_async_queries controls concurrent query processing (default: 1) -rag-qna.Offline.target_qps = 0.11 -rag-qna.Offline.min_duration = 0 -rag-qna.Offline.min_query_count = 824 -rag-qna.Offline.max_async_queries = 10 +e2e-rag-qna.Offline.target_qps = 0.11 +e2e-rag-qna.Offline.min_duration = 0 +e2e-rag-qna.Offline.min_query_count = 824 +e2e-rag-qna.Offline.max_async_queries = 10 # RAG DB workload settings # Send all documents at once for parallel processing -rag-db.Offline.target_qps = 0 -rag-db.Offline.min_duration = 0 -rag-db.Offline.min_query_count = 2503 -rag-db.Offline.max_async_queries = 2503 +e2e-rag-db.Offline.target_qps = 0 +e2e-rag-db.Offline.min_duration = 0 +e2e-rag-db.Offline.min_query_count = 2515 +e2e-rag-db.Offline.max_async_queries = 2515