E2e rag robustness fixes

compliance/TEST09/e2e-rag/README.md → compliance/TEST09/e2e-rag-qna/README.md

@@ -1,4 +1,4 @@
-# TEST09 Compliance for E2E-RAG Workload
+# TEST09 Compliance for E2E-RAG-QnA Workload
 
 ## Overview
 
@@ -65,7 +65,7 @@ Copy the audit.config to your working directory:
 
 ```bash
 cd inference/e2e-rag
-cp ../compliance/TEST09/e2e-rag/audit.config ./
+cp ../compliance/TEST09/e2e-rag-qna/audit.config ./
 ```
 
 ### Part II: Run Performance Test
@@ -94,8 +94,8 @@ python3 reference_mlperf.py \
 ```bash
 python3 inference/e2e/third_party/mlperf-inference/compliance/TEST09/run_verification.py \
     -c run_output_test09 \
-    -o submission/compliance/e2e-rag/Offline \
-    --audit-config ../compliance/TEST09/e2e-rag/audit.config
+    -o submission/compliance/e2e-rag-qna/Offline \
+    --audit-config ../compliance/TEST09/e2e-rag-qna/audit.config
 ```
 
 ### Expected Output

e2e-rag/.gitignore

@@ -37,6 +37,16 @@ result_*.json
 temp_complete_kpi_*.json
 run_output_datasetup_accuracy/
 run_output_datasetup/
+run_output_*/
+run_output_test09/
+accuracy_results.json
+/smoke_manifest.json
+/db_manifest_*.json
+/db_manifest_*.json.gz
+doc_html_smoke/
+/submission/
+/audit.config
+/verify_output_len.txt
 colbert-ir_colbertv2.0/
 frames-benchmark-dataset/
 intfloat_e5-base-v2/

e2e-rag/accuracy_eval.py

@@ -32,23 +32,29 @@
 
 
 # OpenRouter configuration
-DEFAULT_JUDGE_URL = "http://127.0.0.1:8123/v1/chat/completions"
-DEFAULT_JUDGE_MODEL = "gpt-oss-20b"
+DEFAULT_JUDGE_URL = "http://127.0.0.1:8125/v1/chat/completions"
+DEFAULT_JUDGE_MODEL = "meta-llama/Llama-3.1-8B-Instruct"
 # Masked API key (set OPENROUTER_API_KEY environment variable to use OpenRouter)
 OPENROUTER_API_KEY = os.environ.get('OPENROUTER_API_KEY',
     'sk-or-v1-****')
 
 
-JUDGE_PROMPT = """You are an expert evaluator comparing LLM-generated answers to ground truth answers.
+JUDGE_PROMPT = """You are grading whether an LLM answer is correct against a ground truth answer.
 
 QUESTION: {question}
 
 GROUND TRUTH ANSWER: {ground_truth}
 
 LLM ANSWER: {llm_answer}
 
-Evaluate if the LLM answer is factually correct compared to the ground truth.
-Consider semantic equivalence, not just exact string matching.
+Grade in two steps.
+
+STEP 1 - If the LLM answer is empty, "Unknown", "I don't know", "cannot be determined", or otherwise does not commit to an answer, then it is WRONG: output correct=false immediately and do not go to step 2.
+
+STEP 2 - Otherwise compare it to the ground truth by meaning, not wording. correct=true only if it supplies every fact the ground truth requires and each clearly matches; if you are unsure or the match is only partial, output correct=false. Rules:
+- If the ground truth is a list or has multiple parts, an answer missing any of them is correct=false.
+- Every number, date, and name must match the ground truth; a different or differently-rounded value is correct=false, a different name is correct=false.
+- Do NOT penalize harmless extras or omissions when the required facts match: a missing suffix like "Inc.", an added state/country, a full middle name, missing units when the number is right, or a briefer/longer phrasing.
 
 Return your evaluation in JSON format:
 {{
@@ -91,11 +97,19 @@ def call_judge(question: str, ground_truth: str, llm_answer: str,
 
         # Parse JSON response
         content = content.strip()
-        if content.startswith("```"):
-            content = content.split("```")[1]
-            if content.startswith("json"):
-                content = content[4:]
-            content = content.strip()
+
+        # Extract JSON from markdown code blocks
+        if "```" in content:
+            json_block_match = re.search(r'```(?:json)?\s*\n?(.*?)\n?```', content, re.DOTALL)
+            if json_block_match:
+                content = json_block_match.group(1).strip()
+
+        # Try to extract JSON object
+        json_match = re.search(r'\{.*\}', content, re.DOTALL)
+        if json_match:
+            content = json_match.group(0)
+        else:
+            return {"correct": False, "reasoning": "No JSON found in judge response"}
 
         judge_result = json.loads(content)
         return judge_result
@@ -297,6 +311,15 @@ def main():
         json.dump(metrics, f, indent=2)
     print(f"Detailed results saved to {args.output}")
 
+    # Write accuracy.txt into the loadgen log dir in MLPerf format. The
+    # submission checker parses the LLM judge answer accuracy (as a percentage)
+    # from the "Accuracy:" line. The hash= line and log truncation are added
+    # later by tools/submission/truncate_accuracy_log.py during submission prep.
+    accuracy_txt_path = os.path.join(args.log_dir, "accuracy.txt")
+    with open(accuracy_txt_path, 'w') as f:
+        f.write(f"Accuracy: {metrics['answer_accuracy'] * 100:.4f}\n")
+    print(f"Accuracy report saved to {accuracy_txt_path}")
+
 
 if __name__ == "__main__":
     main()

e2e-rag/config.template.sh

@@ -36,22 +36,22 @@ INFERENCE_N_QUERIES=5
 INFERENCE_NUM_WORKERS=1
 
 # LLM endpoints (vLLM, OpenRouter, etc.)
-INFERENCE_LLM_URL="http://127.0.0.1:8123/v1/chat/completions"
-INFERENCE_MODEL="/model/gpt-oss-20b-mxfp4"
-INFERENCE_QUERY_MODEL="/model/gpt-oss-120b-mxfp4"
+INFERENCE_LLM_URL="http://127.0.0.1:8192/v1/chat/completions"
+INFERENCE_MODEL="gpt-oss-20b-mxfp4"
+INFERENCE_QUERY_MODEL="gpt-oss-120b-mxfp4"
 
 # Per-component endpoint splits. Each defaults to INFERENCE_LLM_URL /
 # INFERENCE_MODEL when empty. Set when components live on different servers
 # (e.g. small grader on one vLLM, large query/sufficiency on another).
-# INFERENCE_GRADER_URL="http://127.0.0.1:8124/v1/chat/completions"
-# INFERENCE_GRADER_MODEL="/model/gpt-oss-20b"
+# INFERENCE_GRADER_URL="http://127.0.0.1:8192/v1/chat/completions"
+# INFERENCE_GRADER_MODEL="gpt-oss-20b-mxfp4"
 # INFERENCE_QUERY_URL="http://127.0.0.1:8123/v1/chat/completions"
 # INFERENCE_SUFFICIENCY_URL="http://127.0.0.1:8123/v1/chat/completions"
-# INFERENCE_SUFFICIENCY_MODEL="/model/gpt-oss-120b"
+# INFERENCE_SUFFICIENCY_MODEL="gpt-oss-120b-mxfp4"
 
 # Judge (used by evaluate.py at the end of the run scripts)
 INFERENCE_JUDGE_URL="https://openrouter.ai/api/v1/chat/completions"
-INFERENCE_JUDGE_MODEL="openai/gpt-oss-20b"
+INFERENCE_JUDGE_MODEL="Llama3.1-8B-v1"
 
 # ── Oracle evaluation (run_oracle.sh) ─────────────────────────────────────────
 INFERENCE_ORACLE_BATCH_SIZE=4

e2e-rag/datasetup_accuracy_eval.py

@@ -268,7 +268,9 @@ def validate_database(database_path, retriever_model):
     return validation_results
 
 
-def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None):
+def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None,
+                      manifest_path=None, cosine_threshold=0.9999,
+                      top_k_depth=3):
     """
     Evaluate accuracy of datasetup workload.
 
@@ -277,6 +279,11 @@ def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None):
         output_dir: Directory containing SUT output files
         database_path: Path to the saved database file
         retriever_model: Path to retriever model (for validation)
+        manifest_path: Path to reference DB manifest for cross-system
+            verification. If None, the manifest check is skipped.
+        cosine_threshold: Minimum sample-embedding cosine similarity for the
+            manifest check.
+        top_k_depth: Probe-query top-K rank match depth for the manifest check.
 
     Returns:
         dict: Accuracy results
@@ -448,9 +455,67 @@ def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None):
         print("="*80)
         print()
 
+    # Cross-system manifest verification (corpus fingerprint, sample-embedding
+    # cosine, probe-query top-K ranks) against a reference manifest.
+    manifest_results = None
+    if manifest_path:
+        print("="*80)
+        print("DB Manifest Verification")
+        print("="*80)
+        print(f"Manifest: {manifest_path}")
+        print()
+
+        if not os.path.exists(manifest_path):
+            manifest_results = {
+                "passed": False,
+                "error": "manifest_not_found",
+                "manifest_path": manifest_path,
+            }
+            print(f"  ✗ Manifest not found: {manifest_path}")
+        elif not os.path.exists(database_path):
+            manifest_results = {
+                "passed": False,
+                "error": "database_not_found",
+                "database_path": database_path,
+            }
+            print(f"  ✗ Database not found: {database_path}")
+        else:
+            try:
+                from db_manifest import verify_manifest
+                manifest_results = verify_manifest(
+                    database_path,
+                    manifest_path,
+                    retriever_model=retriever_model,
+                    cosine_threshold=cosine_threshold,
+                    top_k_depth=top_k_depth,
+                )
+                if manifest_results["passed"]:
+                    print("  ✓ Manifest verification PASSED")
+                else:
+                    print("  ✗ Manifest verification FAILED:")
+                    for failure in manifest_results["failures"]:
+                        print(f"    - {failure}")
+            except Exception as e:
+                manifest_results = {"passed": False, "error": str(e)}
+                print(f"  ✗ Manifest verification error: {e}")
+
+        accuracy_results["manifest"] = manifest_results
+        # Overall pass now also requires the manifest check to pass.
+        accuracy_results["passed"] = accuracy_results["passed"] and manifest_results["passed"]
+
+        print()
+        print(f"Manifest: {'✅ PASSED' if manifest_results['passed'] else '❌ FAILED'}")
+        print("="*80)
+        print()
+
     # Write accuracy.txt in MLPerf format
     accuracy_txt_path = os.path.join(log_dir, "accuracy.txt")
     with open(accuracy_txt_path, 'w') as f:
+        # Checker-compliant metric line first: the submission checker parses the
+        # RAG-DB accuracy (document indexing success rate as a percentage) from
+        # this "Accuracy:" line via the E2E_ACCURACY pattern. The hash= line and
+        # log truncation are added later by truncate_accuracy_log.py.
+        f.write(f"Accuracy: {actual_success_rate * 100:.4f}\n")
         f.write("="*80 + "\n")
         f.write("RAG-DB Accuracy Report\n")
         f.write("="*80 + "\n")
@@ -488,6 +553,23 @@ def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None):
             f.write(f"Validation status: {'PASS' if validation_results['passed'] else 'FAIL'}\n")
             f.write("\n")
 
+        # Always emit the manifest section so its status (or that it was
+        # skipped) is visible in every accuracy report.
+        f.write("DB Manifest Verification:\n")
+        f.write("-"*80 + "\n")
+        if manifest_results is None:
+            f.write("  Manifest status: SKIPPED (no manifest provided)\n")
+        else:
+            if manifest_results.get("error"):
+                f.write(f"  Error: {manifest_results['error']}\n")
+            metrics = manifest_results.get("metrics", {})
+            for key, value in metrics.items():
+                f.write(f"  {key}: {value}\n")
+            for failure in manifest_results.get("failures", []):
+                f.write(f"  MISMATCH: {failure}\n")
+            f.write(f"  Manifest status: {'PASS' if manifest_results['passed'] else 'FAIL'}\n")
+        f.write("\n")
+
         f.write("="*80 + "\n")
         f.write(f"Overall Result: {'PASS' if accuracy_results['passed'] else 'FAIL'}\n")
         f.write("="*80 + "\n")
@@ -520,13 +602,36 @@ def main():
     )
     parser.add_argument(
         "--retriever_model",
-        default="/data/model/e5-base-v2",
+        default="intfloat_e5-base-v2/e5-base-v2",
         help="Path to retriever model (for validation)"
     )
+    parser.add_argument(
+        "--manifest",
+        default=None,
+        help="Path to reference DB manifest (.json/.json.gz) for cross-system "
+             "verification. If omitted, the manifest check is skipped."
+    )
+    parser.add_argument(
+        "--cosine_threshold",
+        type=float,
+        default=0.9999,
+        help="Minimum sample-embedding cosine similarity for the manifest check"
+    )
+    parser.add_argument(
+        "--top_k_depth",
+        type=int,
+        default=3,
+        help="Probe-query top-K rank match depth for the manifest check"
+    )
 
     args = parser.parse_args()
 
-    results = evaluate_accuracy(args.log_dir, args.output_dir, args.database, args.retriever_model)
+    results = evaluate_accuracy(
+        args.log_dir, args.output_dir, args.database, args.retriever_model,
+        manifest_path=args.manifest,
+        cosine_threshold=args.cosine_threshold,
+        top_k_depth=args.top_k_depth,
+    )
 
     # Exit with appropriate code
     if results.get("passed", False):