diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000000..ed72b559f9 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,8 @@ +[run] +# Coverage is scoped to the submission checker (the only code under test). +# The CI step also passes --cov=submission_checker; keeping it here means a +# bare `pytest --cov` / `coverage` run is scoped correctly too. +source = submission_checker +omit = + */site-packages/* + */dist-packages/* diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml new file mode 100644 index 0000000000..3a9bf5f882 --- /dev/null +++ b/.github/workflows/codecov.yml @@ -0,0 +1,74 @@ +name: Tests and Coverage + +on: + push: + branches: ["arav-codecov-impl"] + paths: + - "tools/submission/**" + - ".github/workflows/codecov.yml" + pull_request: + branches: ["arav-codecov-impl"] + paths: + - "tools/submission/**" + - ".github/workflows/codecov.yml" + +# Keep the pinned inputs in sync with tools/submission/tests/test_snapshot.py +env: + SUBMISSION_REPO: https://github.com/mlcommons/inference_results_v6.0 + PINNED_SHA: 4d3916ac9cf474b679cdfcf492d43a0559418ad1 + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install test dependencies + run: pip install pytest pytest-cov syrupy pyyaml + + - name: Fetch pinned submissions repo + run: | + set -euo pipefail + # Partial clone: fetch only the blobs we check out. The sparse-checkout + # excludes large artifacts the checker never reads (accuracy/trace + # dumps, dataset JSONs, model caches, vendored binaries). + git clone --filter=blob:none --no-checkout "$SUBMISSION_REPO" submissions + cd submissions + git sparse-checkout set --no-cone \ + '/*' \ + '!openimages-mlperf-v21.json' \ + '!mlperf_log_trace.json' \ + '!mlc-deps.png' \ + '!environment.pickle' \ + '!unet.cache' \ + '!libamdhip64.so.6' \ + '!val_map.txt' + git checkout "$PINNED_SHA" + + - name: Run tests with coverage + # The snapshot test runs the checker as a subprocess; pytest-cov + # instruments it automatically, so coverage of submission_checker is + # captured end-to-end. + env: + MLPERF_SUBMISSION_DIR: ${{ github.workspace }}/submissions + # Resolve `submission_checker` as an importable package so coverage + # can map the subprocess data to it (a path-based --cov would not + # match, because the checker subprocess runs from tools/submission). + PYTHONPATH: tools/submission + run: > + pytest tools/submission/tests/ + --cov=submission_checker + --cov-report=xml + --cov-report=term-missing + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v5 + with: + files: coverage.xml + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/test-submission-checker-snapshot.yml b/.github/workflows/test-submission-checker-snapshot.yml new file mode 100644 index 0000000000..4b60d07534 --- /dev/null +++ b/.github/workflows/test-submission-checker-snapshot.yml @@ -0,0 +1,64 @@ +# Snapshot (golden-file) test for the MLPerf inference submission checker. +# Runs the checker end-to-end against a pinned commit of a real submission +# repository and diffs its summary.csv against a committed golden snapshot. +# See tools/submission/tests/test_snapshot.py for details and update instructions. + +name: Snapshot test for MLPerf inference submission checker + +on: + pull_request: + branches: [ "master", "dev" ] + paths: + - tools/submission/** + - .github/workflows/test-submission-checker-snapshot.yml + - '!**.md' + +# Keep these in sync with tools/submission/tests/test_snapshot.py +env: + SUBMISSION_REPO: https://github.com/mlcommons/inference_results_v6.0 + PINNED_SHA: 4d3916ac9cf474b679cdfcf492d43a0559418ad1 + +jobs: + snapshot: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: [ "3.10", "3.12" ] + + steps: + - name: Checkout inference repo + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: python3 -m pip install pytest syrupy pyyaml + + - name: Fetch pinned submissions repo + run: | + set -euo pipefail + # Partial clone: fetch only the blobs we actually check out. Combined + # with the sparse-checkout excludes below, this skips large artifacts + # the checker never reads (accuracy/trace dumps, dataset JSONs, model + # caches, vendored binaries), trimming the ~2.2 GB tree to ~1.6 GB. + git clone --filter=blob:none --no-checkout "$SUBMISSION_REPO" submissions + cd submissions + git sparse-checkout set --no-cone \ + '/*' \ + '!openimages-mlperf-v21.json' \ + '!mlperf_log_trace.json' \ + '!mlc-deps.png' \ + '!environment.pickle' \ + '!unet.cache' \ + '!libamdhip64.so.6' \ + '!val_map.txt' + git checkout "$PINNED_SHA" + + - name: Run submission checker snapshot test + env: + MLPERF_SUBMISSION_DIR: ${{ github.workspace }}/submissions + run: python3 -m pytest tools/submission/tests/test_snapshot.py -q diff --git a/e2e-rag/QSL.py b/e2e-rag/QSL.py index 93e5f7933c..164c25c693 100644 --- a/e2e-rag/QSL.py +++ b/e2e-rag/QSL.py @@ -85,7 +85,8 @@ def __init__(self, dataset_path, perf_count=None, skip_qsl=False): print(f"Dataset loaded: {self.count} queries") if perf_count is not None: - print(f" (limited to first {perf_count} queries for performance testing)") + print( + f" (limited to first {perf_count} queries for performance testing)") def load_query_samples(self, sample_list): """ @@ -161,15 +162,12 @@ def __init__(self, dataset_path, perf_count=None): # limitations under the License. # ============================================================================= + """ Query Sample Library for RAG-QnA workload. Loads queries from frames_dataset.tsv and provides them to MLPerf Loadgen. """ -import os -import pandas as pd -import mlperf_loadgen as lg - class E2EQSL: """Query Sample Library for RAG-QnA multi-hop RAG benchmark.""" @@ -233,7 +231,8 @@ def __init__(self, dataset_path, perf_count=None, skip_qsl=False): print(f"Dataset loaded: {self.count} queries") if perf_count is not None: - print(f" (limited to first {perf_count} queries for performance testing)") + print( + f" (limited to first {perf_count} queries for performance testing)") def load_query_samples(self, sample_list): """ diff --git a/e2e-rag/accuracy_eval.py b/e2e-rag/accuracy_eval.py index 29ce199482..a4561c7449 100644 --- a/e2e-rag/accuracy_eval.py +++ b/e2e-rag/accuracy_eval.py @@ -34,9 +34,10 @@ # OpenRouter configuration DEFAULT_JUDGE_URL = "http://127.0.0.1:8123/v1/chat/completions" DEFAULT_JUDGE_MODEL = "gpt-oss-20b" -# Masked API key (set OPENROUTER_API_KEY environment variable to use OpenRouter) +# Masked API key (set OPENROUTER_API_KEY environment variable to use +# OpenRouter) OPENROUTER_API_KEY = os.environ.get('OPENROUTER_API_KEY', - 'sk-or-v1-****') + 'sk-or-v1-****') JUDGE_PROMPT = """You are an expert evaluator comparing LLM-generated answers to ground truth answers. @@ -83,7 +84,11 @@ def call_judge(question: str, ground_truth: str, llm_answer: str, } try: - response = requests.post(service_url, json=payload, headers=headers, timeout=60) + response = requests.post( + service_url, + json=payload, + headers=headers, + timeout=60) response.raise_for_status() result = response.json() @@ -105,7 +110,8 @@ def call_judge(question: str, ground_truth: str, llm_answer: str, return {"correct": False, "reasoning": f"Judge error: {e}"} -def calculate_retrieval_metrics(retrieved_urls: List[str], expected_urls: List[str]) -> Dict: +def calculate_retrieval_metrics( + retrieved_urls: List[str], expected_urls: List[str]) -> Dict: """Calculate precision, recall, F1 for retrieval.""" retrieved_set = set(retrieved_urls) @@ -118,7 +124,8 @@ def calculate_retrieval_metrics(retrieved_urls: List[str], expected_urls: List[s precision = len(correct) / len(retrieved_set) if retrieved_set else 0.0 recall = len(correct) / len(expected_set) if expected_set else 0.0 - f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0.0 + f1 = 2 * precision * recall / \ + (precision + recall) if (precision + recall) > 0 else 0.0 return { "precision": precision, @@ -128,8 +135,8 @@ def calculate_retrieval_metrics(retrieved_urls: List[str], expected_urls: List[s def evaluate_results(results: Dict, dataset_path: str, num_workers: int = 4, - judge_service_url: str = DEFAULT_JUDGE_URL, - judge_model: str = DEFAULT_JUDGE_MODEL) -> Dict: + judge_service_url: str = DEFAULT_JUDGE_URL, + judge_model: str = DEFAULT_JUDGE_MODEL) -> Dict: """ Evaluate loadgen results. @@ -190,12 +197,13 @@ def evaluate_single_query(query_id, result): expected_urls = gt_data['expected_urls'] # Calculate retrieval metrics - retrieval_metrics = calculate_retrieval_metrics(retrieved_urls, expected_urls) + retrieval_metrics = calculate_retrieval_metrics( + retrieved_urls, expected_urls) # Judge answer correctness judge_result = call_judge(query, ground_truth, llm_answer, - service_url=judge_service_url, - model_name=judge_model) + service_url=judge_service_url, + model_name=judge_model) answer_correct = judge_result.get('correct', False) return { @@ -229,7 +237,8 @@ def evaluate_single_query(query_id, result): total_queries += 1 if total_queries % 10 == 0: - print(f" Evaluated {total_queries}/{len(results)} queries...") + print( + f" Evaluated {total_queries}/{len(results)} queries...") except Exception as e: print(f"Error evaluating query: {e}") @@ -257,14 +266,37 @@ def evaluate_single_query(query_id, result): def main(): - parser = argparse.ArgumentParser(description="Evaluate RAG-QnA loadgen accuracy") - parser.add_argument('--log_dir', required=True, help='Loadgen log directory') - parser.add_argument('--results_file', required=True, help='SUT results JSON file') - parser.add_argument('--dataset_path', required=True, help='Path to frames_dataset.tsv') - parser.add_argument('--num_workers', type=int, default=4, help='Number of parallel judge workers') - parser.add_argument('--output', default='accuracy_results.json', help='Output file for detailed results') - parser.add_argument('--judge_service_url', default=DEFAULT_JUDGE_URL, help='Judge LLM service URL') - parser.add_argument('--judge_model', default=DEFAULT_JUDGE_MODEL, help='Judge LLM model name') + parser = argparse.ArgumentParser( + description="Evaluate RAG-QnA loadgen accuracy") + parser.add_argument( + '--log_dir', + required=True, + help='Loadgen log directory') + parser.add_argument( + '--results_file', + required=True, + help='SUT results JSON file') + parser.add_argument( + '--dataset_path', + required=True, + help='Path to frames_dataset.tsv') + parser.add_argument( + '--num_workers', + type=int, + default=4, + help='Number of parallel judge workers') + parser.add_argument( + '--output', + default='accuracy_results.json', + help='Output file for detailed results') + parser.add_argument( + '--judge_service_url', + default=DEFAULT_JUDGE_URL, + help='Judge LLM service URL') + parser.add_argument( + '--judge_model', + default=DEFAULT_JUDGE_MODEL, + help='Judge LLM model name') args = parser.parse_args() # Load results @@ -280,9 +312,9 @@ def main(): judge_model=args.judge_model) # Print summary - print("\n" + "="*80) + print("\n" + "=" * 80) print("ACCURACY EVALUATION RESULTS") - print("="*80) + print("=" * 80) print(f"Total Queries: {metrics['total_queries']}") print(f"\nRetrieval Metrics:") print(f" Precision@N: {metrics['retrieval_precision']:.3f}") @@ -290,7 +322,7 @@ def main(): print(f" F1@N: {metrics['retrieval_f1']:.3f}") print(f"\nAnswer Quality:") print(f" LLM Judge Accuracy: {metrics['answer_accuracy']:.3f}") - print("="*80 + "\n") + print("=" * 80 + "\n") # Save detailed results with open(args.output, 'w') as f: diff --git a/e2e-rag/datasetup_accuracy_eval.py b/e2e-rag/datasetup_accuracy_eval.py index 7df71d03f0..3f89041b46 100755 --- a/e2e-rag/datasetup_accuracy_eval.py +++ b/e2e-rag/datasetup_accuracy_eval.py @@ -68,7 +68,8 @@ def parse_accuracy_log(log_path): try: qsl_idx = entry['qsl_idx'] - # Handle data field - can be hex string or list of bytes + # Handle data field - can be hex string or list of + # bytes data_field = entry['data'] if isinstance(data_field, str): # Hex string - convert to bytes @@ -180,7 +181,8 @@ def validate_database(database_path, retriever_model): print(f" ✗ Vector count: Cannot access vector store") # Check 2: Docstore consistency - if hasattr(db, '_vector_store') and hasattr(db._vector_store, 'index_to_docstore_id'): + if hasattr(db, '_vector_store') and hasattr( + db._vector_store, 'index_to_docstore_id'): docstore_count = len(db._vector_store.index_to_docstore_id) check_passed = (docstore_count == vector_count) validation_results["checks"].append({ @@ -194,7 +196,8 @@ def validate_database(database_path, retriever_model): if check_passed: print(f" ✓ Docstore consistency: {docstore_count} documents") else: - print(f" ✗ Docstore consistency: {vector_count} vectors but {docstore_count} documents") + print( + f" ✗ Docstore consistency: {vector_count} vectors but {docstore_count} documents") else: validation_results["checks"].append({ "name": "docstore_consistency", @@ -206,7 +209,8 @@ def validate_database(database_path, retriever_model): print(f" ✗ Docstore consistency: Cannot access docstore") # Check 3: Index dimension - if hasattr(db, '_vector_store') and hasattr(db._vector_store.index, 'd'): + if hasattr(db, '_vector_store') and hasattr( + db._vector_store.index, 'd'): dimension = db._vector_store.index.d expected_dim = 768 # e5-base-v2 check_passed = (dimension == expected_dim) @@ -218,7 +222,8 @@ def validate_database(database_path, retriever_model): "expected": expected_dim }) # Don't fail on dimension mismatch, just warn - print(f" {'✓' if check_passed else '⚠'} Index dimension: {dimension} (expected {expected_dim})") + print( + f" {'✓' if check_passed else '⚠'} Index dimension: {dimension} (expected {expected_dim})") else: validation_results["checks"].append({ "name": "index_dimension", @@ -241,7 +246,8 @@ def validate_database(database_path, retriever_model): }) validation_results["passed"] &= check_passed if check_passed: - print(f" ✓ Sample retrieval: Retrieved {len(results)} results") + print( + f" ✓ Sample retrieval: Retrieved {len(results)} results") else: print(f" ✗ Sample retrieval: No results returned") except Exception as e: @@ -268,7 +274,8 @@ def validate_database(database_path, retriever_model): return validation_results -def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None): +def evaluate_accuracy(log_dir, output_dir, database_path, + retriever_model=None): """ Evaluate accuracy of datasetup workload. @@ -281,9 +288,9 @@ def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None): Returns: dict: Accuracy results """ - print("="*80) + print("=" * 80) print("RAG-DB Accuracy Evaluation") - print("="*80) + print("=" * 80) print(f"Started: {datetime.now().isoformat()}") print() @@ -320,18 +327,21 @@ def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None): # MD5 hash (32 hex characters = 32 bytes when encoded) try: md5_str = data.decode('utf-8') - if len(md5_str) == 32 and all(c in '0123456789abcdef' for c in md5_str): + if len(md5_str) == 32 and all( + c in '0123456789abcdef' for c in md5_str): md5_response = md5_str md5_qsl_idx = qsl_idx success_count += 1 # MD5 response counts as success else: - print(f"Warning: Invalid MD5 format at qsl_idx {qsl_idx}: {md5_str}") + print( + f"Warning: Invalid MD5 format at qsl_idx {qsl_idx}: {md5_str}") failure_count += 1 except UnicodeDecodeError: print(f"Warning: Cannot decode response at qsl_idx {qsl_idx}") failure_count += 1 else: - print(f"Warning: Unexpected response length {len(data)} at qsl_idx {qsl_idx}") + print( + f"Warning: Unexpected response length {len(data)} at qsl_idx {qsl_idx}") failure_count += 1 print(f"Response Summary:") @@ -372,13 +382,14 @@ def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None): # Compare MD5s md5_match = (md5_response == actual_md5) - print("="*80) + print("=" * 80) print("Accuracy Results:") - print("="*80) + print("=" * 80) print(f" Total files processed: {success_count + failure_count}") print(f" Successful: {success_count}") print(f" Failed: {failure_count}") - print(f" Success rate: {100.0 * success_count / (success_count + failure_count):.2f}%") + print( + f" Success rate: {100.0 * success_count / (success_count + failure_count):.2f}%") print() print(f" MD5 returned: {md5_response}") print(f" MD5 actual: {actual_md5}") @@ -390,12 +401,13 @@ def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None): # 1. At least 99% of files succeeded # 2. MD5 matches min_success_rate = 0.99 - actual_success_rate = success_count / (success_count + failure_count) if (success_count + failure_count) > 0 else 0 + actual_success_rate = success_count / \ + (success_count + failure_count) if (success_count + failure_count) > 0 else 0 passed = (actual_success_rate >= min_success_rate) and md5_match print(f"Overall: {'✅ PASSED' if passed else '❌ FAILED'}") - print("="*80) + print("=" * 80) print() # Save results @@ -423,9 +435,9 @@ def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None): # Run database validation if retriever model provided validation_results = None if retriever_model and os.path.exists(database_path): - print("="*80) + print("=" * 80) print("Database Validation") - print("="*80) + print("=" * 80) print() validation_results = validate_database(database_path, retriever_model) @@ -436,48 +448,54 @@ def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None): overall_passed = passed and validation_results["passed"] accuracy_results["passed"] = overall_passed - print("="*80) + print("=" * 80) print("Validation Summary:") - print("="*80) + print("=" * 80) for check in validation_results["checks"]: - status_symbol = "✓" if check["result"] == "PASS" else ("⚠" if check["result"] == "WARN" else "✗") - print(f" {status_symbol} {check['description']}: {check['result']}") + status_symbol = "✓" if check["result"] == "PASS" else ( + "⚠" if check["result"] == "WARN" else "✗") + print( + f" {status_symbol} {check['description']}: {check['result']}") print() - print(f"Validation: {'✅ PASSED' if validation_results['passed'] else '❌ FAILED'}") + print( + f"Validation: {'✅ PASSED' if validation_results['passed'] else '❌ FAILED'}") print(f"Overall: {'✅ PASSED' if overall_passed else '❌ FAILED'}") - print("="*80) + print("=" * 80) print() # Write accuracy.txt in MLPerf format accuracy_txt_path = os.path.join(log_dir, "accuracy.txt") with open(accuracy_txt_path, 'w') as f: - f.write("="*80 + "\n") + f.write("=" * 80 + "\n") f.write("RAG-DB Accuracy Report\n") - f.write("="*80 + "\n") + f.write("=" * 80 + "\n") f.write(f"Timestamp: {datetime.now().isoformat()}\n") f.write(f"Database: {database_path}\n") f.write("\n") f.write("File Processing Results:\n") - f.write("-"*80 + "\n") + f.write("-" * 80 + "\n") f.write(f"Total files: {accuracy_results['total_files']}\n") f.write(f"Successful: {accuracy_results['successful_files']}\n") f.write(f"Failed: {accuracy_results['failed_files']}\n") f.write(f"Success rate: {accuracy_results['success_rate']*100:.2f}%\n") - f.write(f"Required rate: {accuracy_results['min_success_rate_required']*100:.2f}%\n") - f.write(f"Status: {'PASS' if actual_success_rate >= min_success_rate else 'FAIL'}\n") + f.write( + f"Required rate: {accuracy_results['min_success_rate_required']*100:.2f}%\n") + f.write( + f"Status: {'PASS' if actual_success_rate >= min_success_rate else 'FAIL'}\n") f.write("\n") f.write("MD5 Verification:\n") - f.write("-"*80 + "\n") + f.write("-" * 80 + "\n") f.write(f"MD5 returned by SUT: {accuracy_results['md5_response']}\n") f.write(f"MD5 actual (computed): {accuracy_results['md5_actual']}\n") - f.write(f"MD5 match: {'PASS' if accuracy_results['md5_match'] else 'FAIL'}\n") + f.write( + f"MD5 match: {'PASS' if accuracy_results['md5_match'] else 'FAIL'}\n") f.write("\n") if validation_results: f.write("Database Validation:\n") - f.write("-"*80 + "\n") + f.write("-" * 80 + "\n") for check in validation_results["checks"]: f.write(f" {check['name']}: {check['result']}\n") f.write(f" - {check['description']}\n") @@ -485,12 +503,14 @@ def evaluate_accuracy(log_dir, output_dir, database_path, retriever_model=None): f.write(f" - Value: {check['value']}\n") if 'error' in check: f.write(f" - Error: {check['error']}\n") - f.write(f"Validation status: {'PASS' if validation_results['passed'] else 'FAIL'}\n") + f.write( + f"Validation status: {'PASS' if validation_results['passed'] else 'FAIL'}\n") f.write("\n") - f.write("="*80 + "\n") - f.write(f"Overall Result: {'PASS' if accuracy_results['passed'] else 'FAIL'}\n") - f.write("="*80 + "\n") + f.write("=" * 80 + "\n") + f.write( + f"Overall Result: {'PASS' if accuracy_results['passed'] else 'FAIL'}\n") + f.write("=" * 80 + "\n") print(f"Accuracy report saved to: {accuracy_txt_path}") print() @@ -526,7 +546,11 @@ def main(): args = parser.parse_args() - results = evaluate_accuracy(args.log_dir, args.output_dir, args.database, args.retriever_model) + results = evaluate_accuracy( + args.log_dir, + args.output_dir, + args.database, + args.retriever_model) # Exit with appropriate code if results.get("passed", False): diff --git a/e2e-rag/db_manifest.py b/e2e-rag/db_manifest.py index 1cfe20f85f..39f8cbbc0a 100644 --- a/e2e-rag/db_manifest.py +++ b/e2e-rag/db_manifest.py @@ -112,7 +112,8 @@ def _gather_top_k(db: VectorDB, queries: List[Dict], k: int) -> List[Dict]: urls = [] for doc in results: md = getattr(doc, "metadata", None) or {} - url = md.get("original_url") or md.get("source") or md.get("base_filename") or "" + url = md.get("original_url") or md.get( + "source") or md.get("base_filename") or "" urls.append(url) out.append({"index": q["index"], "top_k_urls": urls}) return out @@ -139,10 +140,12 @@ def cmd_write(args): db = _load_db(args.db, args.retriever_model) total_passages = len(db._vector_store.index_to_docstore_id) - print(f"[manifest] DB has {total_passages} passages, dim={db._embedding_dimension}") + print( + f"[manifest] DB has {total_passages} passages, dim={db._embedding_dimension}") corpus_sha = _sha256_docstore(db) - sample_block = _gather_sample_embeddings(db, total_passages, NUM_SAMPLE_EMBEDDINGS) + sample_block = _gather_sample_embeddings( + db, total_passages, NUM_SAMPLE_EMBEDDINGS) probe_queries = _load_probe_queries(args.dataset, NUM_PROBE_QUERIES) probe_block = _gather_top_k(db, probe_queries, PROBE_TOP_K) @@ -237,7 +240,9 @@ def cmd_verify(args): f"top-{args.top_k_depth} {len(probe_queries) - len(rank_failures)}/" f"{len(probe_queries)} match") if rank_failures: - failures.append("probe-query top-K rank mismatch:\n" + "\n".join(rank_failures)) + failures.append( + "probe-query top-K rank mismatch:\n" + + "\n".join(rank_failures)) if failures: print("\n[verify] FAILED:") @@ -252,17 +257,24 @@ def main(): formatter_class=argparse.RawDescriptionHelpFormatter) sub = parser.add_subparsers(dest="cmd", required=True) - pw = sub.add_parser("write", help="Generate a reference manifest from a DB.") + pw = sub.add_parser( + "write", + help="Generate a reference manifest from a DB.") pw.add_argument("--db", required=True) pw.add_argument("--retriever_model", default="intfloat/e5-base-v2") pw.add_argument("--dataset", default="data/frames_dataset.tsv") pw.add_argument("--output", required=True) pw.set_defaults(func=cmd_write) - pv = sub.add_parser("verify", help="Verify a DB against a reference manifest.") + pv = sub.add_parser( + "verify", + help="Verify a DB against a reference manifest.") pv.add_argument("--db", required=True) pv.add_argument("--manifest", required=True) - pv.add_argument("--cosine-threshold", type=float, default=DEFAULT_COSINE_THRESHOLD) + pv.add_argument( + "--cosine-threshold", + type=float, + default=DEFAULT_COSINE_THRESHOLD) pv.add_argument("--top-k-depth", type=int, default=DEFAULT_TOP_K_DEPTH) pv.set_defaults(func=cmd_verify) diff --git a/e2e-rag/download_docs.py b/e2e-rag/download_docs.py index 1778e97e21..f5d1ad46e8 100644 --- a/e2e-rag/download_docs.py +++ b/e2e-rag/download_docs.py @@ -78,56 +78,61 @@ def fix_malformed_url(url: str) -> str: return url - - - class BaseDownloader(ABC): """Base class for downloading web pages in different formats.""" - + def __init__(self, output_dir: str, processes: int = 10): self.output_dir = Path(output_dir) self.output_dir.mkdir(exist_ok=True) self.processes = processes self.url_mapping = {} - + @abstractmethod def get_file_extension(self) -> str: """Return the file extension for this downloader.""" pass - + def create_filename(self, url: str) -> str: """Generate a filename from URL.""" - filename = url.replace("https://", "").replace("/", "_").replace(":", "_") - + filename = url.replace( + "https://", + "").replace( + "/", + "_").replace( + ":", + "_") + # Truncate if too long max_length = 200 if len(filename) > max_length: filename = filename[:max_length] - + return filename + self.get_file_extension() - + @abstractmethod - def download_single_url(self, url: str, output_path: Path) -> Tuple[bool, str]: + def download_single_url( + self, url: str, output_path: Path) -> Tuple[bool, str]: """ Download a single URL to the specified path. - + Returns: Tuple of (success: bool, error_message: str) """ pass - - def process_url(self, args_tuple: Tuple[str, Path, int, int]) -> Tuple[bool, str, str, str]: + + def process_url( + self, args_tuple: Tuple[str, Path, int, int]) -> Tuple[bool, str, str, str]: """Process a single URL - designed for multiprocessing.""" url, output_dir, index, total = args_tuple - + # Create safe filename filename = self.create_filename(url) output_path = output_dir / filename - + # Skip if file already exists if output_path.exists(): return True, filename, "Skipping", url - + # Download the URL try: success, error_msg = self.download_single_url(url, output_path) @@ -141,76 +146,82 @@ def process_url(self, args_tuple: Tuple[str, Path, int, int]) -> Tuple[bool, str return False, filename, error_msg, url else: return False, filename, error_msg, url - + except Exception as e: return False, filename, f"Exception: {str(e)[:100]}", url - - def download_urls(self, urls: List[str], retry_failures: bool = True) -> Dict[str, Any]: + + def download_urls( + self, urls: List[str], retry_failures: bool = True) -> Dict[str, Any]: """Download multiple URLs with parallel processing and progress tracking.""" - + if not urls: print("No URLs found to process") return {"successful": 0, "failed": 0, "failed_urls": []} - - print(f"Processing {len(urls)} URLs with {self.processes} parallel processes...") - + + print( + f"Processing {len(urls)} URLs with {self.processes} parallel processes...") + # Create progress bar progress_bar = tqdm( total=len(urls), desc="Starting downloads...", unit="URL" ) - + # Process URLs in parallel with progress bar start_time = time.time() - + # Prepare arguments for multiprocessing process_args = [(url, self.output_dir, i + 1, len(urls)) - for i, url in enumerate(urls)] - + for i, url in enumerate(urls)] + # Process with progress bar updates results = [] failed_urls = [] # Track failed URLs for detailed reporting - + with Pool(processes=self.processes) as pool: for result in pool.imap(self.process_url, process_args): success, filename, status, url = result results.append((success, filename)) - + base_filename = get_base_filename(filename) self.url_mapping[base_filename] = url - + # Update progress bar with status if status == "Skipping": - progress_bar.set_description(f"Skipping: {filename[:30]}...") + progress_bar.set_description( + f"Skipping: {filename[:30]}...") elif status == "Success": - progress_bar.set_description(f"✓ Success: {filename[:30]}...") + progress_bar.set_description( + f"✓ Success: {filename[:30]}...") else: # This is a failure case - progress_bar.set_description(f"✗ {status}: {filename[:30]}...") + progress_bar.set_description( + f"✗ {status}: {filename[:30]}...") failed_urls.append((filename, status, url)) print(f"\n❌ FAILED: {filename}") print(f" URL: {url}") print(f" Error: {status}") - + progress_bar.update(1) - + progress_bar.close() - + # Save URL mapping to JSON file self.save_url_mapping() - + # Count results successful = sum(1 for success, _ in results if success) failed = len(results) - successful - + end_time = time.time() duration = end_time - start_time - - print(f"\nDownload complete! Successful: {successful}, Failed: {failed}") + + print( + f"\nDownload complete! Successful: {successful}, Failed: {failed}") print(f"Total time: {duration:.2f} seconds") print(f"Average time per URL: {duration/len(urls):.2f} seconds") - + # Print detailed failure report if there were failures if failed_urls: print(f"\n=== FAILED DOWNLOADS DETAILS ===") @@ -219,7 +230,7 @@ def download_urls(self, urls: List[str], retry_failures: bool = True) -> Dict[st print(f" URL: {url}") print(f" Error: {status}") print() - + # Retry failed URLs if requested if retry_failures: retry_result = self.retry_failed_urls(failed_urls) @@ -227,23 +238,24 @@ def download_urls(self, urls: List[str], retry_failures: bool = True) -> Dict[st failed = retry_result["still_failed"] else: print(f"\n✅ All downloads completed successfully!") - + return { "successful": successful, "failed": failed, "failed_urls": failed_urls, "duration": duration } - + def save_url_mapping(self): """Save URL mapping to JSON file.""" save_url_mapping(str(self.output_dir), self.url_mapping) - - def retry_failed_urls(self, failed_urls: List[Tuple[str, str, str]]) -> Dict[str, int]: + + def retry_failed_urls( + self, failed_urls: List[Tuple[str, str, str]]) -> Dict[str, int]: """Retry downloading failed URLs, but skip certain types of permanent failures.""" if not failed_urls: return {"successful": 0, "still_failed": 0} - + # Filter out failures that shouldn't be retried (permanent failures). # Note: We intentionally do NOT include "HTTP error 4" here because that # would also match retryable 429 (Too Many Requests) responses. @@ -252,62 +264,72 @@ def retry_failed_urls(self, failed_urls: List[Tuple[str, str, str]]) -> Dict[str "HTTP error 410", "HTTP error 451", "Invalid or empty content", ] - + retryable_urls = [] permanent_failures = [] - + for filename, status, url in failed_urls: - is_permanent = any(keyword in status for keyword in permanent_failure_keywords) + is_permanent = any( + keyword in status for keyword in permanent_failure_keywords) if is_permanent: permanent_failures.append((filename, status, url)) else: retryable_urls.append((filename, status, url)) - + if permanent_failures: - print(f"\nSkipping {len(permanent_failures)} permanent failures (404s, etc.)") - + print( + f"\nSkipping {len(permanent_failures)} permanent failures (404s, etc.)") + if not retryable_urls: print("No retryable URLs found.") return {"successful": 0, "still_failed": len(permanent_failures)} - + print(f"\n=== RETRYING FAILED DOWNLOADS ===") - print(f"Retrying {len(retryable_urls)} failed URLs (skipping {len(permanent_failures)} permanent failures)...") - + print( + f"Retrying {len(retryable_urls)} failed URLs (skipping {len(permanent_failures)} permanent failures)...") + # Prepare arguments for retry retry_args = [(url, self.output_dir, i + 1, len(retryable_urls)) - for i, (_, _, url) in enumerate(retryable_urls)] - + for i, (_, _, url) in enumerate(retryable_urls)] + # Create progress bar for retry - progress_bar = tqdm(total=len(retryable_urls), desc="Retrying...", unit="URL") - + progress_bar = tqdm( + total=len(retryable_urls), + desc="Retrying...", + unit="URL") + successful_retries = 0 still_failed = [] - + with Pool(processes=self.processes) as pool: for result in pool.imap(self.process_url, retry_args): success, filename, status, url = result - + if success: - progress_bar.set_description(f"✓ Retry Success: {filename[:30]}...") + progress_bar.set_description( + f"✓ Retry Success: {filename[:30]}...") successful_retries += 1 else: - progress_bar.set_description(f"✗ Retry Failed: {filename[:30]}...") + progress_bar.set_description( + f"✗ Retry Failed: {filename[:30]}...") still_failed.append((filename, status, url)) print(f"\n❌ RETRY FAILED: {filename}") print(f" URL: {url}") print(f" Error: {status}") - + progress_bar.update(1) - + progress_bar.close() - + # Combine still failed with permanent failures all_failed = still_failed + permanent_failures - - print(f"\nRetry complete! Successfully retried: {successful_retries}, Still failed: {len(all_failed)}") - print(f" - Retryable failures: {len(still_failed)}") - print(f" - Permanent failures (404s, etc.): {len(permanent_failures)}") - + + print( + f"\nRetry complete! Successfully retried: {successful_retries}, Still failed: {len(all_failed)}") + print(f" - Retryable failures: {len(still_failed)}") + print( + f" - Permanent failures (404s, etc.): {len(permanent_failures)}") + if all_failed: print(f"\n=== STILL FAILED AFTER RETRY ===") for i, (filename, status, url) in enumerate(all_failed, 1): @@ -315,18 +337,19 @@ def retry_failed_urls(self, failed_urls: List[Tuple[str, str, str]]) -> Dict[str print(f" URL: {url}") print(f" Error: {status}") print() - - return {"successful": successful_retries, "still_failed": len(all_failed)} + + return {"successful": successful_retries, + "still_failed": len(all_failed)} class PDFDownloader(BaseDownloader): """Download web pages as PDFs using wkhtmltopdf.""" - + def get_file_extension(self) -> str: return ".pdf" - - - def download_single_url(self, url: str, output_path: Path) -> Tuple[bool, str]: + + def download_single_url( + self, url: str, output_path: Path) -> Tuple[bool, str]: """Download a single URL as PDF using wkhtmltopdf with try-fix-retry approach.""" def attempt_pdf_download(target_url: str) -> Tuple[bool, str, bool]: """ @@ -339,7 +362,7 @@ def attempt_pdf_download(target_url: str) -> Tuple[bool, str, bool]: f'--load-error-handling ignore --load-media-error-handling ignore ' f'--javascript-delay 2000 "{target_url}" "{output_path}"' ) - + try: result = subprocess.run( command, @@ -348,7 +371,7 @@ def attempt_pdf_download(target_url: str) -> Tuple[bool, str, bool]: text=True, timeout=120 ) - + if result.returncode == 0: return True, "Success", False else: @@ -357,20 +380,20 @@ def attempt_pdf_download(target_url: str) -> Tuple[bool, str, bool]: stderr_text = result.stderr[:200] error_msg += f" - {stderr_text}" # Check for 404-like errors in stderr - is_404 = any(phrase in stderr_text.lower() for phrase in - ['404', 'not found', 'page not found', 'http error']) + is_404 = any(phrase in stderr_text.lower() for phrase in + ['404', 'not found', 'page not found', 'http error']) return False, error_msg, is_404 return False, error_msg, False - + except subprocess.TimeoutExpired: return False, "Timeout (120s)", False - + # Try original URL first success, error_msg, is_404 = attempt_pdf_download(url) - + if success: return True, "Success" - + # If it was a 404-like error, try to fix the URL and retry if is_404: fixed_url = fix_malformed_url(url) @@ -381,30 +404,32 @@ def attempt_pdf_download(target_url: str) -> Tuple[bool, str, bool]: return True, f"Success (fixed URL)" else: return False, f"Original: {error_msg}; Fixed attempt: {retry_error_msg}" - + # Return original error if no fix was attempted or fix failed return False, error_msg class HTMLDownloader(BaseDownloader): """Download web pages as HTML files using requests.""" - + # Per-process retry policy for transient errors (429, 5xx) MAX_RETRIES = 5 BASE_BACKOFF = 2.0 # seconds; exponential: BASE_BACKOFF * 2**attempt MAX_BACKOFF = 60.0 # cap on a single sleep - def __init__(self, output_dir: str, processes: int = 4, delay: float = 1.0, timeout: int = 30): + def __init__(self, output_dir: str, processes: int = 4, + delay: float = 1.0, timeout: int = 30): # HTML downloading can use parallel processes with rate limiting super().__init__(output_dir, processes=processes) self.delay = delay self.timeout = timeout - + if requests is None: - raise ImportError("requests package is required for HTML downloads. Install with: pip install requests") - + raise ImportError( + "requests package is required for HTML downloads. Install with: pip install requests") + self.session = requests.Session() - + # Wikipedia's User-Agent policy asks for a descriptive UA that identifies # the tool/operator and includes a contact URL or email. Generic browser # UAs are aggressively rate-limited. Operators may override via the @@ -420,12 +445,12 @@ def __init__(self, output_dir: str, processes: int = 4, delay: float = 1.0, time 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.9', }) - + def get_file_extension(self) -> str: return ".html" - - - def _compute_backoff(self, attempt: int, retry_after_header: Optional[str]) -> float: + + def _compute_backoff(self, attempt: int, + retry_after_header: Optional[str]) -> float: """Compute backoff delay honoring Retry-After if present, with jitter.""" if retry_after_header: try: @@ -437,7 +462,8 @@ def _compute_backoff(self, attempt: int, retry_after_header: Optional[str]) -> f cap = min(self.BASE_BACKOFF * (2 ** attempt), self.MAX_BACKOFF) return random.uniform(self.BASE_BACKOFF, cap) - def download_single_url(self, url: str, output_path: Path) -> Tuple[bool, str]: + def download_single_url( + self, url: str, output_path: Path) -> Tuple[bool, str]: """Download a single URL as HTML using requests with try-fix-retry approach.""" def attempt_download(target_url: str) -> Tuple[bool, str, bool]: """ @@ -447,7 +473,8 @@ def attempt_download(target_url: str) -> Tuple[bool, str, bool]: last_error = "Unknown error" for attempt in range(self.MAX_RETRIES): try: - response = self.session.get(target_url, timeout=self.timeout) + response = self.session.get( + target_url, timeout=self.timeout) # Retry on 429 (rate limit) and 5xx (server errors) if response.status_code == 429 or 500 <= response.status_code < 600: @@ -472,14 +499,16 @@ def attempt_download(target_url: str) -> Tuple[bool, str, bool]: # Check if we got redirected to a different article if response.url != target_url: - print(f"Redirected from {target_url} to {response.url}") + print( + f"Redirected from {target_url} to {response.url}") if 'charset' in response.headers.get('content-type', ''): encoding = response.encoding else: encoding = 'utf-8' - html_content = response.content.decode(encoding, errors='ignore') + html_content = response.content.decode( + encoding, errors='ignore') if 'wikipedia' not in html_content.lower() or len(html_content) < 1000: return False, f"Invalid or empty content (length: {len(html_content)})", False @@ -514,13 +543,13 @@ def attempt_download(target_url: str) -> Tuple[bool, str, bool]: return False, f"Exception: {str(e)[:100]}", False return False, last_error, False - + # Try original URL first success, error_msg, is_404 = attempt_download(url) - + if success: return True, "Success" - + # If it was a 404-like error, try to fix the URL and retry if is_404: fixed_url = fix_malformed_url(url) @@ -531,21 +560,15 @@ def attempt_download(target_url: str) -> Tuple[bool, str, bool]: return True, f"Success (fixed URL)" else: return False, f"Original: {error_msg}; Fixed attempt: {retry_error_msg}" - + # Return original error if no fix was attempted or fix failed return False, error_msg - + def download_urls(self, urls: List[str], retry_failures: bool = True): """Download URLs with parallel processing and rate limiting.""" return super().download_urls(urls, retry_failures) - - - - - - def download_frames_dataset(output_dir): """Download the FRAMES dataset from Hugging Face and save as TSV.""" print("Downloading FRAMES dataset from Hugging Face...") @@ -623,7 +646,8 @@ def extract_wikipedia_links(item): return links -def extract_urls_from_frames_dataset(tsv_path: str, max_urls: Optional[int] = None) -> List[str]: +def extract_urls_from_frames_dataset( + tsv_path: str, max_urls: Optional[int] = None) -> List[str]: """Extract unique Wikipedia URLs from FRAMES dataset TSV file.""" # Load dataset df = pd.read_csv(tsv_path, sep='\t') @@ -635,15 +659,12 @@ def extract_urls_from_frames_dataset(tsv_path: str, max_urls: Optional[int] = No urls.add(link) urls = list(urls) - + # Limit number if specified if max_urls and max_urls > 0: urls = urls[:max_urls] - - return urls - - + return urls def main(): @@ -651,15 +672,15 @@ def main(): description='Download Wikipedia pages as PDFs or HTML files from FRAMES dataset or other sources.\nBy default, URLs are validated before downloading to avoid 404 errors.', formatter_class=argparse.RawTextHelpFormatter ) - + # Format selection parser.add_argument( - '--format', - choices=['pdf', 'html'], + '--format', + choices=['pdf', 'html'], default='pdf', help='Output format: pdf or html (default: pdf)' ) - + # URL sources (mutually exclusive) url_group = parser.add_mutually_exclusive_group() url_group.add_argument( @@ -668,15 +689,15 @@ def main(): ) url_group.add_argument( - '--urls', - nargs='+', + '--urls', + nargs='+', help='List of URLs to download' ) url_group.add_argument( - '--url-file', + '--url-file', help='File containing URLs (one per line)' ) - + # Output options parser.add_argument( '--output-dir', @@ -687,7 +708,7 @@ def main(): default='frames-benchmark-dataset', help='Directory for dataset files (default: frames-benchmark-dataset)' ) - + # Processing options parser.add_argument( '--max-files', @@ -702,7 +723,7 @@ def main(): 'Wikipedia rate-limits aggressive HTML scrapers; ' 'consider --processes 4 for HTML.' ) - + # HTML-specific options parser.add_argument( '--delay', @@ -717,7 +738,7 @@ def main(): default=30, help='Timeout for HTML requests in seconds (default: 30)' ) - + # Dataset options parser.add_argument( '--download-dataset', @@ -733,7 +754,7 @@ def main(): # Get URLs from various sources urls = None - + if args.url_file: try: with open(args.url_file, 'r', encoding='utf-8') as f: @@ -742,14 +763,14 @@ def main(): except Exception as e: print(f"Error loading URLs from file: {e}") return - + elif args.urls: urls = args.urls - + elif args.tsv_path or args.download_dataset: # Handle FRAMES dataset tsv_path = args.tsv_path - + if tsv_path is None and args.download_dataset: print("=== DOWNLOADING FRAMES DATASET ===") tsv_path = download_frames_dataset(args.data_dir) @@ -759,10 +780,10 @@ def main(): elif tsv_path is None: print("❌ No TSV path provided. Use --tsv-path or --download-dataset") return - + urls = extract_urls_from_frames_dataset(tsv_path, args.max_files) print(f"Extracted {len(urls)} URLs from FRAMES dataset: {tsv_path}") - + else: # Default: download dataset print("=== DOWNLOADING FRAMES DATASET ===") @@ -770,10 +791,10 @@ def main(): if tsv_path is None: print("❌ Failed to download FRAMES dataset. Exiting.") return - + urls = extract_urls_from_frames_dataset(tsv_path, args.max_files) print(f"Extracted {len(urls)} URLs from FRAMES dataset: {tsv_path}") - + if not urls: print("No URLs found to download") return @@ -784,21 +805,22 @@ def main(): user = getpass.getuser() xdg_runtime_dir = f"/tmp/runtime-{user}" os.environ["XDG_RUNTIME_DIR"] = xdg_runtime_dir - + downloader = PDFDownloader(args.output_dir, args.processes) - + elif args.format == 'html': if requests is None: - print("❌ HTML format requires 'requests' package. Install with: pip install requests") + print( + "❌ HTML format requires 'requests' package. Install with: pip install requests") return - + downloader = HTMLDownloader( output_dir=args.output_dir, processes=args.processes, delay=args.delay, timeout=args.timeout ) - + else: print(f"❌ Unsupported format: {args.format}") return diff --git a/e2e-rag/evaluate.py b/e2e-rag/evaluate.py index 9c94ce8890..b963b98124 100644 --- a/e2e-rag/evaluate.py +++ b/e2e-rag/evaluate.py @@ -51,16 +51,18 @@ def load_results(path: Path): # Load pandas DataFrame checkpoint with open(path, 'rb') as f: df = pickle.load(f) - + # Convert DataFrame to dict: query -> llm_answer # Only include successfully completed queries successful = df[df['success'] == True] - return {row['query']: row['llm_answer'] for _, row in successful.iterrows()} + return {row['query']: row['llm_answer'] + for _, row in successful.iterrows()} else: # Legacy JSON format data = json.loads(path.read_text(encoding="utf-8")) results = data.get("results", []) - return {entry.get("prompt"): entry.get("llm_answer", "") for entry in results if entry.get("prompt")} + return {entry.get("prompt"): entry.get("llm_answer", "") + for entry in results if entry.get("prompt")} def _parse_score_value(value) -> int: @@ -122,7 +124,8 @@ def _extract_json_dict(content: str) -> Optional[dict]: return None -def call_judge(session: requests.Session, service_url: str, model: str, question: str, gold: str, pred: str): +def call_judge(session: requests.Session, service_url: str, + model: str, question: str, gold: str, pred: str): prompt = ( "You judge whether the model answer correctly answers the question based on semantic equivalence to the gold answer.\n\n" "GRADING RULES:\n" @@ -168,12 +171,17 @@ def call_judge(session: requests.Session, service_url: str, model: str, question "X-Title": "RAG-QnA Evaluation" } - response = session.post(service_url, json=payload, headers=headers, timeout=120) + response = session.post( + service_url, + json=payload, + headers=headers, + timeout=120) response.raise_for_status() data = response.json() # Defensive: handle missing or malformed 'choices' in response choices = data.get("choices") - if not choices or not isinstance(choices, list) or not choices[0] or "message" not in choices[0] or "content" not in choices[0]["message"]: + if not choices or not isinstance( + choices, list) or not choices[0] or "message" not in choices[0] or "content" not in choices[0]["message"]: print("[ERROR] Judge response missing 'choices' or 'content':", data) # Return score 0, explanation with raw response, and raw data return 0, f"Malformed judge response: {data}", str(data) @@ -207,11 +215,13 @@ def call_judge(session: requests.Session, service_url: str, model: str, question def _judge_row(idx, prompt, gold, pred, service_url, model): """Call judge for a single row, returning (idx, prompt, gold, pred, score, explanation, raw).""" session = requests.Session() - score, explanation, raw = call_judge(session, service_url, model, prompt, gold, pred) + score, explanation, raw = call_judge( + session, service_url, model, prompt, gold, pred) return idx, prompt, gold, pred, score, explanation, raw -def evaluate(results_path: Path, dataset_path: Path, service_url: str, model: str, batch_size: int = 16): +def evaluate(results_path: Path, dataset_path: Path, + service_url: str, model: str, batch_size: int = 16): # Check for OpenRouter API key if using OpenRouter if "openrouter.ai" in service_url and not OPENROUTER_API_KEY: print("ERROR: OPENROUTER_API_KEY environment variable not set") @@ -227,13 +237,15 @@ def evaluate(results_path: Path, dataset_path: Path, service_url: str, model: st print(f"CHECKPOINT STATISTICS") print("=" * 80) print(f"Total queries in checkpoint: {len(checkpoint_df)}") - print(f"Successful queries: {(checkpoint_df['success'] == True).sum()}") + print( + f"Successful queries: {(checkpoint_df['success'] == True).sum()}") print(f"Failed queries: {(checkpoint_df['success'] == False).sum()}") if 'num_docs' in checkpoint_df.columns: total_docs = checkpoint_df['num_docs'].sum() total_missing = checkpoint_df['num_missing_docs'].sum() print(f"Total documents referenced: {total_docs}") - print(f"Missing documents: {total_missing} ({100*total_missing/total_docs:.2f}%)") + print( + f"Missing documents: {total_missing} ({100*total_missing/total_docs:.2f}%)") print("=" * 80) print() @@ -256,7 +268,8 @@ def evaluate(results_path: Path, dataset_path: Path, service_url: str, model: st total = len(items) unknown = sum(1 for _, _, _, pred in items if pred.lower() == "unknown") - # Submit all judge calls in parallel (batch_size workers), print as they complete + # Submit all judge calls in parallel (batch_size workers), print as they + # complete score_sum = 0 with ThreadPoolExecutor(max_workers=batch_size) as executor: futures = { @@ -284,15 +297,38 @@ def evaluate(results_path: Path, dataset_path: Path, service_url: str, model: st def parse_args(): - parser = argparse.ArgumentParser(description="Evaluate single-shot results using an LLM judge.") - parser.add_argument("results", type=Path, help="Path to results (result_single_shot.json or oracle_checkpoint.pkl)") - parser.add_argument("--dataset", type=Path, default=Path("data/frames_dataset.tsv"), help="Evaluation dataset TSV") - parser.add_argument("--judge-url", default=DEFAULT_JUDGE_URL, help="Judge service endpoint") - parser.add_argument("--judge-model", default=DEFAULT_JUDGE_MODEL, help="Judge model identifier") - parser.add_argument("--batch-size", type=int, default=16, help="Number of concurrent judge requests (default: 16)") + parser = argparse.ArgumentParser( + description="Evaluate single-shot results using an LLM judge.") + parser.add_argument( + "results", + type=Path, + help="Path to results (result_single_shot.json or oracle_checkpoint.pkl)") + parser.add_argument( + "--dataset", + type=Path, + default=Path("data/frames_dataset.tsv"), + help="Evaluation dataset TSV") + parser.add_argument( + "--judge-url", + default=DEFAULT_JUDGE_URL, + help="Judge service endpoint") + parser.add_argument( + "--judge-model", + default=DEFAULT_JUDGE_MODEL, + help="Judge model identifier") + parser.add_argument( + "--batch-size", + type=int, + default=16, + help="Number of concurrent judge requests (default: 16)") return parser.parse_args() if __name__ == "__main__": args = parse_args() - evaluate(args.results, args.dataset, args.judge_url, args.judge_model, args.batch_size) + evaluate( + args.results, + args.dataset, + args.judge_url, + args.judge_model, + args.batch_size) diff --git a/e2e-rag/ingestion_monitor.py b/e2e-rag/ingestion_monitor.py index b3f1e69179..17aa004322 100644 --- a/e2e-rag/ingestion_monitor.py +++ b/e2e-rag/ingestion_monitor.py @@ -20,17 +20,17 @@ Usage: from ingestion_monitor import IngestionMonitor - + monitor = IngestionMonitor() - + # Track document processing with monitor.track_component("html_parsing"): process_html_files(files) - - # Track embedding generation + + # Track embedding generation with monitor.track_component("embedding_generation"): embeddings = generate_embeddings(texts) - + # Get performance report report = monitor.get_performance_report() """ @@ -43,6 +43,7 @@ from dataclasses import dataclass, asdict from pathlib import Path + @dataclass class ComponentMetrics: """Metrics for a single pipeline component.""" @@ -56,6 +57,7 @@ class ComponentMetrics: is_pipeline_input: bool = False # Mark if this is a pipeline input component is_pipeline_output: bool = False # Mark if this is a pipeline output component + @dataclass class IndexingTrendPoint: """Single data point for indexing performance trend.""" @@ -64,7 +66,8 @@ class IndexingTrendPoint: indexing_time: float # Time to add this batch (seconds) throughput_items_per_sec: float cumulative_time: float # Total time so far - + + @dataclass class IngestionReport: """Complete ingestion performance report.""" @@ -75,12 +78,14 @@ class IngestionReport: overall_throughput_mb_per_sec: float components: List[ComponentMetrics] bottleneck_component: str - indexing_trend: List[IndexingTrendPoint] = None # For scaling analysis = "none" + # For scaling analysis = "none" + indexing_trend: List[IndexingTrendPoint] = None bottleneck_component: str + class IngestionMonitor: """Real-time ingestion performance monitoring.""" - + def __init__(self): self.components: Dict[str, ComponentMetrics] = {} self.start_time = None # Will be set when ingestion starts @@ -88,17 +93,17 @@ def __init__(self): self.component_start_time = None self.indexing_trend: List[IndexingTrendPoint] = [] self.cumulative_indexing_time = 0.0 - + def start_ingestion(self): """Mark the start of ingestion. Should be called at the beginning of ingest().""" self.start_time = time.time() - + @contextmanager - def track_component(self, component_name: str, input_size_bytes: int = 0, - items_count: int = 0, text_only: bool = False, - is_pipeline_input: bool = False, is_pipeline_output: bool = False): + def track_component(self, component_name: str, input_size_bytes: int = 0, + items_count: int = 0, text_only: bool = False, + is_pipeline_input: bool = False, is_pipeline_output: bool = False): """Context manager to track performance of a pipeline component. - + Args: component_name: Name of the component being tracked input_size_bytes: Input data size in bytes @@ -108,53 +113,54 @@ def track_component(self, component_name: str, input_size_bytes: int = 0, is_pipeline_output: If True, mark as pipeline output component for aggregation """ start_time = time.time() - + class ComponentContext: def __init__(self): self.input_size_bytes = input_size_bytes self.items_count = items_count self.text_only = text_only - + def set_input_size(self, size_bytes: int): self.input_size_bytes = size_bytes - + def set_item_count(self, count: int): self.items_count = count - + def add_text_bytes(self, text_bytes: int): """Add text-only bytes for passage tracking.""" self.input_size_bytes += text_bytes - + context = ComponentContext() - + try: self.current_component = component_name self.component_start_time = start_time yield context - + finally: end_time = time.time() duration = end_time - start_time - - # Calculate throughput + + # Calculate throughput total_input = context.input_size_bytes - total_output = 0 # Output will be set separately + total_output = 0 # Output will be set separately total_items = context.items_count total_duration = duration - + # Check if component already exists (accumulate metrics) if component_name in self.components: existing = self.components[component_name] # Accumulate metrics total_duration = existing.duration + duration total_input = existing.input_size_bytes + context.input_size_bytes - total_output = existing.output_size_bytes + 0 + total_output = existing.output_size_bytes + 0 total_items = existing.items_processed + context.items_count - + is_pipeline_input = is_pipeline_input or existing.is_pipeline_input is_pipeline_output = is_pipeline_output or existing.is_pipeline_output - throughput_mb = (total_input / (1024 * 1024)) / total_duration if total_duration > 0 else 0 + throughput_mb = (total_input / (1024 * 1024)) / \ + total_duration if total_duration > 0 else 0 throughput_items = total_items / total_duration if total_duration > 0 else 0 self.components[component_name] = ComponentMetrics( @@ -168,18 +174,18 @@ def add_text_bytes(self, text_bytes: int): is_pipeline_input=is_pipeline_input, is_pipeline_output=is_pipeline_output ) - + def set_output_size(self, component_name: str, output_size_bytes: int): """Set the output size for a component after processing.""" if component_name in self.components: self.components[component_name].output_size_bytes = output_size_bytes - + def set_output_size_callback(self, component_name: str, callback_fn): """Set the output size for a component using a callback function. - + This is useful when the output size calculation is complex or requires accessing class-specific data (e.g., BM25 index files). - + Args: component_name: Name of the component callback_fn: Function that returns the output size in bytes @@ -189,40 +195,41 @@ def set_output_size_callback(self, component_name: str, callback_fn): output_size = callback_fn() self.components[component_name].output_size_bytes = output_size except Exception as e: - print(f"Warning: Failed to calculate output size for {component_name}: {e}") - + print( + f"Warning: Failed to calculate output size for {component_name}: {e}") + @contextmanager def track_ingestion(self): """Track overall ingestion performance.""" self.start_time = time.time() # Set start_time for get_performance_report() - + class IngestionContext: def __init__(self): self.item_count = 0 - + def set_item_count(self, count: int): self.item_count = count - + context = IngestionContext() - + try: yield context finally: pass # start_time is checked by get_performance_report() - - def track_incremental_indexing(self, db_size_before: int, batch_size: int, - indexing_time: float): + + def track_incremental_indexing(self, db_size_before: int, batch_size: int, + indexing_time: float): """Track indexing performance for incremental batches to analyze scaling trends. - + Args: db_size_before: Number of items in DB before adding this batch - batch_size: Number of items added in this batch + batch_size: Number of items added in this batch indexing_time: Time taken to index this batch (seconds) """ db_size_after = db_size_before + batch_size throughput = batch_size / indexing_time if indexing_time > 0 else 0 self.cumulative_indexing_time += indexing_time - + trend_point = IndexingTrendPoint( db_size=db_size_after, batch_size=batch_size, @@ -230,36 +237,44 @@ def track_incremental_indexing(self, db_size_before: int, batch_size: int, throughput_items_per_sec=throughput, cumulative_time=self.cumulative_indexing_time ) - + self.indexing_trend.append(trend_point) - + def get_performance_report(self) -> IngestionReport: """Generate comprehensive performance report.""" # Calculate duration from when start_ingestion() was called if self.start_time is None: - raise ValueError("start_ingestion() must be called before getting performance report") - + raise ValueError( + "start_ingestion() must be called before getting performance report") + total_duration = time.time() - self.start_time - + # Aggregate metrics based on pipeline input/output flags # If no flags set, fall back to first component for input - input_components = [c for c in self.components.values() if c.is_pipeline_input] - output_components = [c for c in self.components.values() if c.is_pipeline_output] - + input_components = [ + c for c in self.components.values() if c.is_pipeline_input] + output_components = [ + c for c in self.components.values() if c.is_pipeline_output] + total_input = sum(c.input_size_bytes for c in input_components) total_items = sum(c.items_processed for c in input_components) total_output = sum(c.output_size_bytes for c in output_components) - - overall_throughput = (total_input / (1024 * 1024)) / total_duration if total_duration > 0 else 0 - + + overall_throughput = (total_input / (1024 * 1024)) / \ + total_duration if total_duration > 0 else 0 + # Find bottleneck and calculate efficiency ratio bottleneck_name = "none" - + if self.components: - bottleneck = min(self.components.values(), key=lambda x: x.throughput_mb_per_sec) - fastest = max(self.components.values(), key=lambda x: x.throughput_mb_per_sec) + bottleneck = min( + self.components.values(), + key=lambda x: x.throughput_mb_per_sec) + fastest = max( + self.components.values(), + key=lambda x: x.throughput_mb_per_sec) bottleneck_name = bottleneck.name - + return IngestionReport( total_duration=total_duration, total_input_bytes=total_input, @@ -270,60 +285,72 @@ def get_performance_report(self) -> IngestionReport: bottleneck_component=bottleneck_name, indexing_trend=self.indexing_trend if self.indexing_trend else None ) - + def save_report(self, filename: str = "ingestion_performance.json"): """Save performance report to JSON file.""" report = self.get_performance_report() - + # Convert to serializable format report_dict = asdict(report) - + with open(filename, 'w') as f: json.dump(report_dict, f, indent=2) - + return report_dict - + def print_summary(self): """Print detailed performance summary with individual components.""" report = self.get_performance_report() - + print("🚀 INGESTION PERFORMANCE SUMMARY") print("=" * 60) print(f"📊 Overall Metrics:") print(f" Total duration: {report.total_duration:.2f}s") - print(f" Overall throughput: {report.overall_throughput_mb_per_sec:.2f} MB/s") + print( + f" Overall throughput: {report.overall_throughput_mb_per_sec:.2f} MB/s") print(f" Items processed: {report.total_items:,}") - + # DEBUG: Show detailed breakdown of input data aggregation - input_components = [c for c in report.components if c.is_pipeline_input] + input_components = [ + c for c in report.components if c.is_pipeline_input] print(f"\n🔍 DEBUG: Input Data Breakdown (is_pipeline_input=True):") print(f" {'Component':<30} {'Input Size (MB)':<20} {'Items':<15}") print(f" {'-'*65}") total_input_debug = 0 for comp in input_components: - input_mb = comp.input_size_bytes / (1024*1024) + input_mb = comp.input_size_bytes / (1024 * 1024) total_input_debug += comp.input_size_bytes - print(f" {comp.name:<30} {input_mb:>18.2f} MB {comp.items_processed:>12,}") + print( + f" {comp.name:<30} {input_mb:>18.2f} MB {comp.items_processed:>12,}") print(f" {'-'*65}") - print(f" {'TOTAL AGGREGATED INPUT':<30} {total_input_debug/(1024*1024):>18.2f} MB") - - # Show input data size from report (aggregated from marked input components or first component) - print(f"\n Input data size (from report): {report.total_input_bytes / (1024*1024):.2f} MB") - + print( + f" {'TOTAL AGGREGATED INPUT':<30} {total_input_debug/(1024*1024):>18.2f} MB") + + # Show input data size from report (aggregated from marked input + # components or first component) + print( + f"\n Input data size (from report): {report.total_input_bytes / (1024*1024):.2f} MB") + # Show output size and expansion ratio if output data exists if report.total_output_bytes > 0: - output_size_mb = report.total_output_bytes / (1024*1024) - expansion_ratio = report.total_output_bytes / report.total_input_bytes if report.total_input_bytes > 0 else 0 + output_size_mb = report.total_output_bytes / (1024 * 1024) + expansion_ratio = report.total_output_bytes / \ + report.total_input_bytes if report.total_input_bytes > 0 else 0 print(f" Output data size: {output_size_mb:.2f} MB") print(f" Output/Input ratio: {expansion_ratio:.1f}x") - + print(f" Bottleneck component: {report.bottleneck_component}") - + print(f"\n🔧 Component Performance Details:") - for component in sorted(report.components, key=lambda x: x.duration, reverse=True): - percentage = (component.duration / report.total_duration) * 100 if report.total_duration > 0 else 0 - mb_processed = component.input_size_bytes / (1024*1024) - avg_latency_ms = (component.duration * 1000 / component.items_processed) if component.items_processed > 0 else 0 + for component in sorted( + report.components, key=lambda x: x.duration, reverse=True): + percentage = (component.duration / report.total_duration) * \ + 100 if report.total_duration > 0 else 0 + mb_processed = component.input_size_bytes / (1024 * 1024) + avg_latency_ms = ( + component.duration * + 1000 / + component.items_processed) if component.items_processed > 0 else 0 pipeline_flags = [] if component.is_pipeline_input: pipeline_flags.append("INPUT") @@ -331,33 +358,38 @@ def print_summary(self): pipeline_flags.append("OUTPUT") flag_str = f" [{', '.join(pipeline_flags)}]" if pipeline_flags else "" print(f" 📈 {component.name}{flag_str}:") - print(f" ⏱️ Duration: {component.duration:.3f}s ({percentage:.1f}% of total)") - print(f" 🚀 Throughput: {component.throughput_mb_per_sec:.2f} MB/s") + print( + f" ⏱️ Duration: {component.duration:.3f}s ({percentage:.1f}% of total)") + print( + f" 🚀 Throughput: {component.throughput_mb_per_sec:.2f} MB/s") print(f" 📦 Items: {component.items_processed:,}") print(f" 💾 Data: {mb_processed:.2f} MB") print(f" ⚡ Avg latency: {avg_latency_ms:.2f}ms per item") print() - + # Print indexing trend analysis if available if report.indexing_trend and len(report.indexing_trend) > 1: print("📈 VECTOR DB INDEXING SCALING ANALYSIS") print("=" * 60) print("DB Size → Batch Time (Throughput)") - + for i, point in enumerate(report.indexing_trend): db_size_k = point.db_size // 1000 if point.db_size >= 1000 else point.db_size size_unit = "K" if point.db_size >= 1000 else "" - - print(f" {db_size_k:>4}{size_unit} docs → {point.indexing_time:>6.3f}s ({point.throughput_items_per_sec:>6.1f} docs/sec)") - + + print( + f" {db_size_k:>4}{size_unit} docs → {point.indexing_time:>6.3f}s ({point.throughput_items_per_sec:>6.1f} docs/sec)") + # Calculate scaling trend if len(report.indexing_trend) >= 3: first_point = report.indexing_trend[0] last_point = report.indexing_trend[-1] - - size_ratio = last_point.db_size / first_point.db_size if first_point.db_size > 0 else 0 - time_ratio = last_point.indexing_time / first_point.indexing_time if first_point.indexing_time > 0 else 0 - + + size_ratio = last_point.db_size / \ + first_point.db_size if first_point.db_size > 0 else 0 + time_ratio = last_point.indexing_time / \ + first_point.indexing_time if first_point.indexing_time > 0 else 0 + if size_ratio > 1: scaling_factor = time_ratio / size_ratio if scaling_factor > 1.5: @@ -366,9 +398,10 @@ def print_summary(self): trend_desc = "📊 Linear scaling (time proportional to size)" else: trend_desc = "📉 Sub-linear scaling (indexing gets more efficient)" - + print(f"\n💡 Trend Analysis:") - print(f" Size increased {size_ratio:.1f}x, time increased {time_ratio:.1f}x") + print( + f" Size increased {size_ratio:.1f}x, time increased {time_ratio:.1f}x") print(f" {trend_desc}") print() @@ -376,13 +409,14 @@ def print_summary(self): if __name__ == "__main__": # Example usage monitor = IngestionMonitor() - + # Simulate components - with monitor.track_component("html_parsing", 1024*1024, 100): # 1MB, 100 files + with monitor.track_component("html_parsing", 1024 * 1024, 100): # 1MB, 100 files time.sleep(0.1) - - with monitor.track_component("embedding_generation", 512*1024, 500): # 512KB, 500 chunks + + # 512KB, 500 chunks + with monitor.track_component("embedding_generation", 512 * 1024, 500): time.sleep(0.5) - + monitor.print_summary() monitor.save_report("example_performance.json") diff --git a/e2e-rag/llm_logger.py b/e2e-rag/llm_logger.py index 7207edf1e8..99f1d7e0b3 100644 --- a/e2e-rag/llm_logger.py +++ b/e2e-rag/llm_logger.py @@ -31,7 +31,8 @@ class LLMLogger: """Logger for tracking all LLM calls with full input/output and metrics.""" - def __init__(self, output_file: str = None, experiment_metadata: Dict[str, Any] = None): + def __init__(self, output_file: str = None, + experiment_metadata: Dict[str, Any] = None): self.session_id = str(uuid.uuid4()) self.queries = [] self.output_file = output_file @@ -159,7 +160,8 @@ def _append_query_to_file(self, query_data: Dict): data['queries'].append(query_data) # Update experiment summary - data['experiment_summary'] = self._calculate_experiment_summary(data['queries']) + data['experiment_summary'] = self._calculate_experiment_summary( + data['queries']) # Write back with open(self.output_file, 'w', encoding='utf-8') as f: @@ -187,7 +189,8 @@ def _calculate_experiment_summary(self, queries: List[Dict]) -> Dict: } # Add retrieval/answer metrics if available - queries_with_retrieval = [q for q in queries if "retrieval_results" in q] + queries_with_retrieval = [ + q for q in queries if "retrieval_results" in q] if queries_with_retrieval: experiment_summary["retrieval_metrics"] = { "average_precision": round(sum(q["retrieval_results"].get("precision", 0) for q in queries_with_retrieval) / len(queries_with_retrieval), 4), @@ -197,7 +200,9 @@ def _calculate_experiment_summary(self, queries: List[Dict]) -> Dict: queries_with_answers = [q for q in queries if "answer_results" in q] if queries_with_answers: - correct_count = sum(1 for q in queries_with_answers if q["answer_results"].get("judge_score", 0) >= 4) + correct_count = sum( + 1 for q in queries_with_answers if q["answer_results"].get( + "judge_score", 0) >= 4) experiment_summary["answer_metrics"] = { "average_judge_score": round(sum(q["answer_results"].get("judge_score", 0) for q in queries_with_answers) / len(queries_with_answers), 2), "queries_correct": correct_count, @@ -207,14 +212,17 @@ def _calculate_experiment_summary(self, queries: List[Dict]) -> Dict: return experiment_summary - def end_query(self, retrieval_results: Dict = None, answer_results: Dict = None, wall_time_s: float = None): + def end_query(self, retrieval_results: Dict = None, + answer_results: Dict = None, wall_time_s: float = None): """Finish logging current query, compute summary, and write to file""" if self.current_query: - self.current_query["timestamp_end"] = datetime.utcnow().isoformat() + "Z" + self.current_query["timestamp_end"] = datetime.utcnow( + ).isoformat() + "Z" # Calculate summary llm_calls = self.current_query["llm_calls"] - hop_counts = [c["hop_count"] for c in llm_calls if c["hop_count"] is not None] + hop_counts = [c["hop_count"] + for c in llm_calls if c["hop_count"] is not None] summary = { "total_llm_calls": len(llm_calls), @@ -244,7 +252,8 @@ def end_query(self, retrieval_results: Dict = None, answer_results: Dict = None, self.current_query = None - def save(self, output_file: str = None, experiment_metadata: Dict[str, Any] = None): + def save(self, output_file: str = None, + experiment_metadata: Dict[str, Any] = None): """Save all logs to JSON file (legacy method for backward compatibility). Note: If logger was initialized with output_file, logs are already written @@ -280,21 +289,28 @@ def save(self, output_file: str = None, experiment_metadata: Dict[str, Any] = No print(f"LLM logs saved to: {target_file}") print(f"Total queries: {len(self.queries)}") if experiment_summary: - print(f"Total LLM calls: {experiment_summary.get('total_llm_calls', 0)}") + print( + f"Total LLM calls: {experiment_summary.get('total_llm_calls', 0)}") print(f"Total tokens: {experiment_summary.get('total_tokens', 0):,} (input: {experiment_summary.get('total_input_tokens', 0):,}, output: {experiment_summary.get('total_output_tokens', 0):,})") # Per-query latency distribution (wall time: query to answer) per_query_latencies = sorted( - (q["summary"].get("total_wall_time_ms") or q["summary"].get("total_latency_ms", 0)) / 1000 + (q["summary"].get("total_wall_time_ms") + or q["summary"].get("total_latency_ms", 0)) / 1000 for q in self.queries if "summary" in q ) n = len(per_query_latencies) if n > 0: mean_lat = sum(per_query_latencies) / n - median_lat = per_query_latencies[n // 2] if n % 2 == 1 else (per_query_latencies[n // 2 - 1] + per_query_latencies[n // 2]) / 2 - p90_lat = per_query_latencies[int(n * 0.90)] if n >= 10 else per_query_latencies[-1] - p99_lat = per_query_latencies[int(n * 0.99)] if n >= 100 else per_query_latencies[-1] + median_lat = per_query_latencies[n // 2] if n % 2 == 1 else ( + per_query_latencies[n // 2 - 1] + per_query_latencies[n // 2]) / 2 + p90_lat = per_query_latencies[int( + n * 0.90)] if n >= 10 else per_query_latencies[-1] + p99_lat = per_query_latencies[int( + n * 0.99)] if n >= 100 else per_query_latencies[-1] total_latency_s = sum(per_query_latencies) - print(f"Per-query latency (query-to-answer): mean={mean_lat:.2f}s median={median_lat:.2f}s p90={p90_lat:.2f}s p99={p99_lat:.2f}s") - print(f"Throughput: {n / total_latency_s:.4f} queries/sec ({total_latency_s / n:.2f}s per query)") + print( + f"Per-query latency (query-to-answer): mean={mean_lat:.2f}s median={median_lat:.2f}s p90={p90_lat:.2f}s p99={p99_lat:.2f}s") + print( + f"Throughput: {n / total_latency_s:.4f} queries/sec ({total_latency_s / n:.2f}s per query)") print(f"{'='*80}\n") diff --git a/e2e-rag/measure_indexing_with_chunking.py b/e2e-rag/measure_indexing_with_chunking.py index b8069c24f0..1ef82c2a1a 100644 --- a/e2e-rag/measure_indexing_with_chunking.py +++ b/e2e-rag/measure_indexing_with_chunking.py @@ -188,14 +188,17 @@ def main(): # Validate required arguments if not args.documents and not args.ingest: - parser.error("Either --documents (for raw docs) or --ingest (for pre-chunked passages) is required") + parser.error( + "Either --documents (for raw docs) or --ingest (for pre-chunked passages) is required") # Set default database name if not provided if args.database is None: args.database = VectorDB.get_default_db_name() - db_file_path = args.database if args.database.endswith('.db') else f"{args.database}.db" - db_base_name = args.database.replace('.db', '') if args.database.endswith('.db') else args.database + db_file_path = args.database if args.database.endswith( + '.db') else f"{args.database}.db" + db_base_name = args.database.replace( + '.db', '') if args.database.endswith('.db') else args.database # Check if database already exists db_exists = Path(db_file_path).exists() @@ -249,8 +252,10 @@ def main(): if args.save_passages: passages_file = args.save_passages else: - # Generate filename based on source directory and chunking parameters - source_dir_name = os.path.basename(os.path.normpath(args.documents)) + # Generate filename based on source directory and chunking + # parameters + source_dir_name = os.path.basename( + os.path.normpath(args.documents)) passages_file = f"passages_{source_dir_name}_len{args.chunk_size}_ov{args.chunk_overlap}_{args.text_boundary}.json" print(f" Auto-generated passages filename: {passages_file}") @@ -358,7 +363,8 @@ def main(): # STEP 5: VALIDATE DATABASE (after save, not part of perf) # ============================================================ print("[5/5] Validating database...") - validation_results = validate_database(rag_db, expected_passages=num_passages if args.documents else None) + validation_results = validate_database( + rag_db, expected_passages=num_passages if args.documents else None) vector_count = validation_results["vector_count"] if validation_results["validation_passed"]: @@ -460,7 +466,7 @@ def main(): import shutil try: shutil.rmtree(temp_text_dir) - except: + except BaseException: pass return 0 diff --git a/e2e-rag/multi_shot_retrieval.py b/e2e-rag/multi_shot_retrieval.py index 3ee84c731d..905b83bfbb 100644 --- a/e2e-rag/multi_shot_retrieval.py +++ b/e2e-rag/multi_shot_retrieval.py @@ -28,6 +28,13 @@ Prompt → Query Rewriter (LLM) → k Sub-queries → Retrieval → Reranking → Evaluation """ +import requests +from llm_logger import LLMLogger +from params import add_all_args +from utils import (set_deterministic_seeds, filter_dataset_by_difficulty, + setup_llm_config, get_device_config) +from evaluation import evaluate_retrieval_query, run_evaluation +from retrieve import VectorDB import argparse import json import re @@ -46,13 +53,6 @@ # Get OpenRouter API key from environment OPENROUTER_API_KEY = os.environ.get('OPENROUTER_API_KEY', '') -from retrieve import VectorDB -from evaluation import evaluate_retrieval_query, run_evaluation -from utils import (set_deterministic_seeds, filter_dataset_by_difficulty, - setup_llm_config, get_device_config) -from params import add_all_args -from llm_logger import LLMLogger -import requests # Prompts @@ -171,15 +171,15 @@ **If there are NO NEW documents to evaluate, skip this task and go to TASK 3** -SUMMARY REQUIREMENTS: +SUMMARY REQUIREMENTS: - Extract and preserve specific details - only facts from the document TASK 2: CHECK IF SUFFICIENT AND CONNECT INFORMATION Review ALL KEPT documents and summaries. Actively connect facts across documents: -- Identify entities by matching names across summaries +- Identify entities by matching names across summaries - Chain relationships (A → B → C) -- Cross-reference dates/events +- Cross-reference dates/events - Build complete chains: Person → Family member → Attribute, or Event → Year → Cross-reference If you can construct a complete answer chain with specific names/facts from kept documents, provide final answer. @@ -204,7 +204,7 @@ **For People/Biography:** - Full article: Just the person's name "Harriet Lane" -- Family: "Person X family", "Person X parents" +- Family: "Person X family", "Person X parents" - Specific relative: "Person X" then extract family, don't search "Person X mother" repeatedly **For Events/Dates:** @@ -240,7 +240,7 @@ {{"relevance": [], "summaries": [], "queries": ["Nth position holder name", "specific event list"], "feedback": "Starting with direct entity/list searches."}} CRITICAL REQUIREMENTS: -- If NO NEW documents: return empty arrays: "relevance": [], "summaries": [] +- If NO NEW documents: return empty arrays: "relevance": [], "summaries": [] - If {len_new_docs} NEW documents: return exactly {len_new_docs} relevance scores and {len_new_docs} summaries - For relevant docs (relevance=1): summary MUST extract specific facts (names, dates, relationships, family details from infobox/text) - For irrelevant docs (relevance=0): summary MUST be empty string "" @@ -254,9 +254,6 @@ Respond only in JSON format""" - - - def get_chat_completions_headers(service_url: str): """Headers for an OpenAI-compatible /v1/chat/completions request. @@ -272,21 +269,22 @@ def get_chat_completions_headers(service_url: str): } return {} + def call_chat_completions(service_url: str, model_name: str, messages: List[Dict], - temperature: float = 1.0, max_tokens: int = 4096, - top_p: float = 1.0, - top_k: int = -1, - reasoning_effort: str = "medium", - frequency_penalty: float = 0.0, - presence_penalty: float = 0.0, - repetition_penalty: float = 1.0, - max_retries: int = 5, - logger: Optional[LLMLogger] = None, - component: str = "unknown", - hop_count: Optional[int] = None, - context: Dict[str, Any] = None, - perf_test_cache: Optional[Any] = None, - query_id: Optional[str] = None) -> str: + temperature: float = 1.0, max_tokens: int = 4096, + top_p: float = 1.0, + top_k: int = -1, + reasoning_effort: str = "medium", + frequency_penalty: float = 0.0, + presence_penalty: float = 0.0, + repetition_penalty: float = 1.0, + max_retries: int = 5, + logger: Optional[LLMLogger] = None, + component: str = "unknown", + hop_count: Optional[int] = None, + context: Dict[str, Any] = None, + perf_test_cache: Optional[Any] = None, + query_id: Optional[str] = None) -> str: """Call OpenRouter API with proper authentication and logging. Default sampling parameters: @@ -307,7 +305,8 @@ def call_chat_completions(service_url: str, model_name: str, messages: List[Dict if reasoning_effort != "medium": payload["reasoning_effort"] = reasoning_effort else: - payload["reasoning_effort"] = reasoning_effort # Always include for logging + # Always include for logging + payload["reasoning_effort"] = reasoning_effort # Add optional sampling parameters if frequency_penalty != 0.0: @@ -322,22 +321,32 @@ def call_chat_completions(service_url: str, model_name: str, messages: List[Dict # Performance test mode: check if we have cached response cached_response = None if perf_test_cache and query_id and component and hop_count is not None: - cached_response = perf_test_cache.get_response(query_id, component, hop_count) + cached_response = perf_test_cache.get_response( + query_id, component, hop_count) if cached_response: print(f" [PERF TEST MODE] Will attempt real LLM call for performance measurement, but return cached response for deterministic pipeline") - print(f" [PERF TEST MODE] CRITICAL: LLM call MUST succeed - test will STOP if LLM service is unavailable") + print( + f" [PERF TEST MODE] CRITICAL: LLM call MUST succeed - test will STOP if LLM service is unavailable") else: - print(f" [WARNING] No cached response for {component} hop {hop_count}, will use real LLM response") + print( + f" [WARNING] No cached response for {component} hop {hop_count}, will use real LLM response") for attempt in range(max_retries): start_time = time.time() try: - response = requests.post(service_url, json=payload, headers=headers, timeout=120) + response = requests.post( + service_url, + json=payload, + headers=headers, + timeout=120) # Retry on rate limit if response.status_code == 429: - retry_after = int(response.headers.get('Retry-After', 2 ** attempt)) - print(f" Rate limited (429). Retrying in {retry_after}s (attempt {attempt+1}/{max_retries})") + retry_after = int( + response.headers.get( + 'Retry-After', 2 ** attempt)) + print( + f" Rate limited (429). Retrying in {retry_after}s (attempt {attempt+1}/{max_retries})") time.sleep(retry_after) continue @@ -352,12 +361,14 @@ def call_chat_completions(service_url: str, model_name: str, messages: List[Dict if not llm_output: reasoning_content = message.get('reasoning_content') or '' if reasoning_content: - json_match = re.search(r'\{.*\}', reasoning_content, re.DOTALL) + json_match = re.search( + r'\{.*\}', reasoning_content, re.DOTALL) if json_match: llm_output = json_match.group(0) if not llm_output: - print(f" WARNING [{component}]: LLM returned empty content. Raw response: {json.dumps(result)[:500]}") + print( + f" WARNING [{component}]: LLM returned empty content. Raw response: {json.dumps(result)[:500]}") # Log this call if logger: @@ -368,14 +379,19 @@ def call_chat_completions(service_url: str, model_name: str, messages: List[Dict response=result, latency_ms=latency_ms, context=context or {}, - simulated_response=cached_response # None in normal mode, cached value in perf test mode + # None in normal mode, cached value in perf test mode + simulated_response=cached_response ) - # In perf test mode, return cached response instead of real LLM output + # In perf test mode, return cached response instead of real LLM + # output if cached_response: - print(f" [PERF TEST MODE] Returning simulated response (LLM generated: {len(llm_output)} chars, Simulated: {len(cached_response)} chars)") - print(f" [PERF TEST MODE] Real LLM output: {llm_output[:200]}{'...' if len(llm_output) > 200 else ''}") - print(f" [PERF TEST MODE] Cached output (used in pipeline): {cached_response[:200]}{'...' if len(cached_response) > 200 else ''}") + print( + f" [PERF TEST MODE] Returning simulated response (LLM generated: {len(llm_output)} chars, Simulated: {len(cached_response)} chars)") + print( + f" [PERF TEST MODE] Real LLM output: {llm_output[:200]}{'...' if len(llm_output) > 200 else ''}") + print( + f" [PERF TEST MODE] Cached output (used in pipeline): {cached_response[:200]}{'...' if len(cached_response) > 200 else ''}") return cached_response return llm_output @@ -384,54 +400,71 @@ def call_chat_completions(service_url: str, model_name: str, messages: List[Dict status = e.response.status_code if e.response is not None else None if status in (502, 503, 504) and attempt < max_retries - 1: wait = 2 ** attempt - print(f" Server error ({status}). Retrying in {wait}s (attempt {attempt+1}/{max_retries})") + print( + f" Server error ({status}). Retrying in {wait}s (attempt {attempt+1}/{max_retries})") time.sleep(wait) continue # In perf test mode, LLM calls MUST succeed for valid benchmarking if cached_response: print(f" ERROR [{component}]: HTTP {status}: {e}") - print(f" [PERF TEST MODE FATAL] LLM call failed - cannot proceed with cached response") - print(f" [PERF TEST MODE FATAL] Performance benchmarking requires all LLM calls to succeed for run-to-run equivalency") - print(f" [PERF TEST MODE FATAL] Please ensure LLM service is running on {service_url}") - raise RuntimeError(f"Perf test mode requires LLM service to be available. LLM call failed for {component}") from e + print( + f" [PERF TEST MODE FATAL] LLM call failed - cannot proceed with cached response") + print( + f" [PERF TEST MODE FATAL] Performance benchmarking requires all LLM calls to succeed for run-to-run equivalency") + print( + f" [PERF TEST MODE FATAL] Please ensure LLM service is running on {service_url}") + raise RuntimeError( + f"Perf test mode requires LLM service to be available. LLM call failed for {component}") from e print(f" ERROR [{component}]: HTTP {status}: {e}") raise except requests.exceptions.Timeout: if attempt < max_retries - 1: wait = 2 ** attempt - print(f" Timeout. Retrying in {wait}s (attempt {attempt+1}/{max_retries})") + print( + f" Timeout. Retrying in {wait}s (attempt {attempt+1}/{max_retries})") time.sleep(wait) continue # In perf test mode, LLM calls MUST succeed for valid benchmarking if cached_response: - print(f" ERROR [{component}]: Request timed out after {max_retries} attempts") - print(f" [PERF TEST MODE FATAL] LLM call timed out - cannot proceed with cached response") - print(f" [PERF TEST MODE FATAL] Performance benchmarking requires all LLM calls to succeed for run-to-run equivalency") - raise RuntimeError(f"Perf test mode requires LLM service to respond. LLM call timed out for {component}") - - print(f" ERROR [{component}]: Request timed out after {max_retries} attempts") + print( + f" ERROR [{component}]: Request timed out after {max_retries} attempts") + print( + f" [PERF TEST MODE FATAL] LLM call timed out - cannot proceed with cached response") + print( + f" [PERF TEST MODE FATAL] Performance benchmarking requires all LLM calls to succeed for run-to-run equivalency") + raise RuntimeError( + f"Perf test mode requires LLM service to respond. LLM call timed out for {component}") + + print( + f" ERROR [{component}]: Request timed out after {max_retries} attempts") raise except Exception as e: # In perf test mode, LLM calls MUST succeed for valid benchmarking if cached_response: print(f" ERROR [{component}]: {e}") - print(f" [PERF TEST MODE FATAL] LLM call failed with exception - cannot proceed with cached response") - print(f" [PERF TEST MODE FATAL] Performance benchmarking requires all LLM calls to succeed for run-to-run equivalency") - print(f" [PERF TEST MODE FATAL] Please ensure LLM service is running and accessible") - raise RuntimeError(f"Perf test mode requires LLM service to be available. LLM call failed for {component}") from e + print( + f" [PERF TEST MODE FATAL] LLM call failed with exception - cannot proceed with cached response") + print( + f" [PERF TEST MODE FATAL] Performance benchmarking requires all LLM calls to succeed for run-to-run equivalency") + print( + f" [PERF TEST MODE FATAL] Please ensure LLM service is running and accessible") + raise RuntimeError( + f"Perf test mode requires LLM service to be available. LLM call failed for {component}") from e print(f" ERROR [{component}]: {e}") raise print(f" ERROR [{component}]: Max retries ({max_retries}) exceeded") if cached_response: - print(f" [PERF TEST MODE] Real LLM call failed, returning cached response") + print( + f" [PERF TEST MODE] Real LLM call failed, returning cached response") return cached_response raise RuntimeError(f"LLM call failed after {max_retries} retries") + def evaluate_document_relevance(question: str, new_documents: List[tuple], kept_documents: List[tuple], @@ -512,15 +545,18 @@ def evaluate_document_relevance(question: str, relevance = relevance_result.get("relevance", []) if len(relevance) != len(new_documents): - print(f" Warning: Relevance mismatch. Expected {len(new_documents)}, got {len(relevance)}") + print( + f" Warning: Relevance mismatch. Expected {len(new_documents)}, got {len(relevance)}") return {"relevance": [1] * len(new_documents)} return {"relevance": relevance} except Exception as e: # In perf test mode, propagate fatal errors (don't fallback) - perf_test_cache = llm_config.get('perf_test_cache') if llm_config else None - if perf_test_cache and isinstance(e, RuntimeError) and "Perf test mode requires" in str(e): + perf_test_cache = llm_config.get( + 'perf_test_cache') if llm_config else None + if perf_test_cache and isinstance( + e, RuntimeError) and "Perf test mode requires" in str(e): # This is a perf test mode fatal error - must propagate it raise @@ -529,13 +565,13 @@ def evaluate_document_relevance(question: str, def check_sufficiency(question: str, - kept_documents: List[tuple], - iteration: int, - max_iterations: int, - llm_config: Optional[Dict[str, Any]] = None, - logger: Optional[LLMLogger] = None, - hop_count: int = 1, - query_id: Optional[str] = None) -> Dict[str, Any]: + kept_documents: List[tuple], + iteration: int, + max_iterations: int, + llm_config: Optional[Dict[str, Any]] = None, + logger: Optional[LLMLogger] = None, + hop_count: int = 1, + query_id: Optional[str] = None) -> Dict[str, Any]: """ Check if kept documents are sufficient to answer the question. Uses gpt-oss-120b model via OpenRouter. @@ -601,13 +637,15 @@ def check_sufficiency(question: str, query_id=query_id ) - print(f" [DEBUG] Sufficiency check raw output: {llm_output[:200]}...") + print( + f" [DEBUG] Sufficiency check raw output: {llm_output[:200]}...") if not llm_output: print(f" Warning: Sufficiency check returned empty") # On final iteration, force sufficient if iteration >= max_iterations: - return {"sufficient": True, "reasoning": "Max iterations reached"} + return {"sufficient": True, + "reasoning": "Max iterations reached"} return {"sufficient": False, "reasoning": "LLM returned empty"} if llm_output.startswith("```"): @@ -635,24 +673,27 @@ def check_sufficiency(question: str, except Exception as e: # In perf test mode, propagate fatal errors (don't fallback) - perf_test_cache = llm_config.get('perf_test_cache') if llm_config else None - if perf_test_cache and isinstance(e, RuntimeError) and "Perf test mode requires" in str(e): + perf_test_cache = llm_config.get( + 'perf_test_cache') if llm_config else None + if perf_test_cache and isinstance( + e, RuntimeError) and "Perf test mode requires" in str(e): # This is a perf test mode fatal error - must propagate it raise print(f" Error in sufficiency check: {e}") # On final iteration, force sufficient if iteration >= max_iterations: - return {"sufficient": True, "reasoning": f"Max iterations reached (error: {str(e)})"} + return {"sufficient": True, + "reasoning": f"Max iterations reached (error: {str(e)})"} return {"sufficient": False, "reasoning": f"Error: {str(e)}"} def generate_answer(question: str, - kept_documents: List[tuple], - llm_config: Optional[Dict[str, Any]] = None, - logger: Optional[LLMLogger] = None, - hop_count: Optional[int] = None, - query_id: Optional[str] = None) -> str: + kept_documents: List[tuple], + llm_config: Optional[Dict[str, Any]] = None, + logger: Optional[LLMLogger] = None, + hop_count: Optional[int] = None, + query_id: Optional[str] = None) -> str: """ Generate final answer from kept documents using gpt-oss-120b. @@ -726,9 +767,12 @@ def generate_answer(question: str, return llm_output.strip() except Exception as e: - # In perf test mode, propagate fatal errors (don't fallback to "Unknown") - perf_test_cache = llm_config.get('perf_test_cache') if llm_config else None - if perf_test_cache and isinstance(e, RuntimeError) and "Perf test mode requires" in str(e): + # In perf test mode, propagate fatal errors (don't fallback to + # "Unknown") + perf_test_cache = llm_config.get( + 'perf_test_cache') if llm_config else None + if perf_test_cache and isinstance( + e, RuntimeError) and "Perf test mode requires" in str(e): # This is a perf test mode fatal error - must propagate it raise @@ -737,15 +781,15 @@ def generate_answer(question: str, def generate_search_queries(question: str, - kept_documents: List[tuple], - max_queries: int = 3, - query_history: Optional[List[str]] = None, - query_results: Optional[List[int]] = None, - feedback_history: Optional[List[str]] = None, - llm_config: Optional[Dict[str, Any]] = None, - logger: Optional[LLMLogger] = None, - hop_count: int = 1, - query_id: Optional[str] = None) -> Dict[str, Any]: + kept_documents: List[tuple], + max_queries: int = 3, + query_history: Optional[List[str]] = None, + query_results: Optional[List[int]] = None, + feedback_history: Optional[List[str]] = None, + llm_config: Optional[Dict[str, Any]] = None, + logger: Optional[LLMLogger] = None, + hop_count: int = 1, + query_id: Optional[str] = None) -> Dict[str, Any]: """ Generate search queries using gpt-oss-120b via OpenRouter. """ @@ -771,7 +815,8 @@ def generate_search_queries(question: str, history_text = "No queries yet" # Format feedback - feedback_text = "\n".join(feedback_history) if feedback_history else "Iteration 1 - Initial search" + feedback_text = "\n".join( + feedback_history) if feedback_history else "Iteration 1 - Initial search" prompt = QUERY_GENERATION_PROMPT.format( question=question, @@ -833,9 +878,12 @@ def generate_search_queries(question: str, } except Exception as e: - # In perf test mode, propagate fatal errors (don't fallback to original question) - perf_test_cache = llm_config.get('perf_test_cache') if llm_config else None - if perf_test_cache and isinstance(e, RuntimeError) and "Perf test mode requires" in str(e): + # In perf test mode, propagate fatal errors (don't fallback to original + # question) + perf_test_cache = llm_config.get( + 'perf_test_cache') if llm_config else None + if perf_test_cache and isinstance( + e, RuntimeError) and "Perf test mode requires" in str(e): # This is a perf test mode fatal error - must propagate it raise @@ -854,7 +902,7 @@ def query_rewriter(question: str, new_documents: List[tuple], llm_config: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: """ Evaluates documents AND generates new queries in one LLM call. - + Args: question: The user's original question new_documents: List of NEW document texts to evaluate @@ -864,7 +912,7 @@ def query_rewriter(question: str, new_documents: List[tuple], query_history: List of previous search queries query_results: List of number of documents found for each query (parallel to query_history) previous_feedback: Feedback from previous iteration about what's missing - + Returns: Dict with: - 'relevance' (list of 0/1 for ONLY new_documents) @@ -879,18 +927,18 @@ def query_rewriter(question: str, new_documents: List[tuple], kept_context += f"\n[KEPT {i}] {doc[2]}\n" else: kept_context = "None" - - # Format NEW documents + + # Format NEW documents new_context = "" if new_documents: for i, doc in enumerate(new_documents, 1): new_context += f"\n[NEW {i}] {doc[1]}\n" else: new_context = "None" - + # Combine for context context = f"KEPT DOCUMENTS (already relevant):\n{kept_context}\n\nNEW DOCUMENTS (evaluate these):\n{new_context}" - + # Format query history with results - focus on failures for learning if query_history: failed_queries = [] @@ -901,31 +949,35 @@ def query_rewriter(question: str, new_documents: List[tuple], failed_queries.append(q) else: successful_queries.append(f"{q} ({num_docs} docs)") - + history_parts = [] if failed_queries: - history_parts.append(f"FAILED: {', '.join(failed_queries)}") # Last 3 failures + history_parts.append( + f"FAILED: {', '.join(failed_queries)}") # Last 3 failures if successful_queries: - history_parts.append(f"SUCCESS: {', '.join(successful_queries)}") # Last 2 successes - - history_text = "; ".join(history_parts) if history_parts else "No queries yet" + # Last 2 successes + history_parts.append(f"SUCCESS: {', '.join(successful_queries)}") + + history_text = "; ".join( + history_parts) if history_parts else "No queries yet" else: history_text = "No queries yet" - + # Build feedback history - show progression of what was tried and learned if feedback_history and len(feedback_history) > 0: unique_feedback = [] for fb in reversed(feedback_history): if fb and fb not in unique_feedback: unique_feedback.append(fb) - + if unique_feedback: - feedback_text = "PREVIOUS ATTEMPTS: " + " → ".join(reversed(unique_feedback)) + feedback_text = "PREVIOUS ATTEMPTS: " + \ + " → ".join(reversed(unique_feedback)) else: feedback_text = f"Iteration {len(query_history) + 1 if query_history else 1}" else: feedback_text = f"Iteration {len(query_history) + 1 if query_history else 1} - Initial search" - + print(f"Context: {context}") print(f"History: {history_text}") print(f"Feedback: {feedback_text}") @@ -938,12 +990,12 @@ def query_rewriter(question: str, new_documents: List[tuple], k=max_queries, len_new_docs=len(new_documents) ) - - system_message = f"""You are an expert at multi-hop reasoning and strategic search. - CRITICAL: Never repeat failed queries. - Always try completely different approaches when queries return 0 docs. + + system_message = f"""You are an expert at multi-hop reasoning and strategic search. + CRITICAL: Never repeat failed queries. + Always try completely different approaches when queries return 0 docs. Focus on atomic facts and progressive strategies.""" - + # Use LLM config if provided, otherwise use defaults if llm_config: model_name = llm_config["model_name"] @@ -973,10 +1025,10 @@ def query_rewriter(question: str, new_documents: List[tuple], response = requests.post(service_url, json=payload, timeout=300) response.raise_for_status() result = response.json() - + message = result['choices'][0]['message'] llm_output = message.get('content') - + # Fallback: use reasoning_content if content is empty (thinking models) reasoning_content = message.get('reasoning_content', '') if reasoning_content and not llm_output: @@ -985,12 +1037,15 @@ def query_rewriter(question: str, new_documents: List[tuple], json_match = re.search(r'\{.*\}', reasoning_content, re.DOTALL) if json_match: llm_output = json_match.group(0) - print(f" DEBUG: Extracted JSON from reasoning_content ({len(llm_output)} chars)") + print( + f" DEBUG: Extracted JSON from reasoning_content ({len(llm_output)} chars)") else: - print(f" DEBUG: No JSON found in reasoning_content snippet: {reasoning_content[:200]}") + print( + f" DEBUG: No JSON found in reasoning_content snippet: {reasoning_content[:200]}") if llm_output is None or not llm_output.strip(): - print(f" Warning: LLM returned empty content, using original query as fallback") + print( + f" Warning: LLM returned empty content, using original query as fallback") # Always fall back to original query - never return empty queries return { "relevance": [0] * len(new_documents), @@ -999,25 +1054,25 @@ def query_rewriter(question: str, new_documents: List[tuple], "feedback": "LLM returned empty response", "answer": "" } - + llm_output = llm_output.strip() - + # Parse JSON output - handle markdown code blocks if llm_output.startswith("```"): llm_output = llm_output.split("```")[1] if llm_output.startswith("json"): llm_output = llm_output[4:] llm_output = llm_output.strip() - + result_data = json.loads(llm_output) - - # Validate format + + # Validate format required_fields = ["relevance"] for field in required_fields: if field not in result_data: print(f"Warning: Missing required field '{field}' in response") result_data[field] = [0] * len(new_documents) - + # Ensure we have either "answer" OR "queries"+"feedback" if "answer" not in result_data: result_data["answer"] = "" @@ -1027,37 +1082,47 @@ def query_rewriter(question: str, new_documents: List[tuple], result_data["feedback"] = "" if "summaries" not in result_data: result_data["summaries"] = [""] * len(new_documents) - - # Ensure relevance array matches NEW document count - fix mismatches by padding/truncating + + # Ensure relevance array matches NEW document count - fix mismatches by + # padding/truncating if len(result_data["relevance"]) != len(new_documents): - print(f"Warning: Relevance array length mismatch. Expected {len(new_documents)}, got {len(result_data['relevance'])}. Auto-fixing.") - relevance = result_data["relevance"][:len(new_documents)] # Truncate if too long - while len(relevance) < len(new_documents): # Pad with 0s if too short + print( + f"Warning: Relevance array length mismatch. Expected {len(new_documents)}, got {len(result_data['relevance'])}. Auto-fixing.") + relevance = result_data["relevance"][:len( + new_documents)] # Truncate if too long + while len(relevance) < len( + new_documents): # Pad with 0s if too short relevance.append(0) result_data["relevance"] = relevance print(f"Fixed relevance array: {relevance}") - - # Ensure summaries array matches NEW document count - fix mismatches by padding/truncating + + # Ensure summaries array matches NEW document count - fix mismatches by + # padding/truncating if len(result_data["summaries"]) != len(new_documents): - print(f"Warning: Summaries array length mismatch. Expected {len(new_documents)}, got {len(result_data['summaries'])}. Auto-fixing.") - summaries = result_data["summaries"][:len(new_documents)] # Truncate if too long - while len(summaries) < len(new_documents): # Pad with empty strings if too short + print( + f"Warning: Summaries array length mismatch. Expected {len(new_documents)}, got {len(result_data['summaries'])}. Auto-fixing.") + summaries = result_data["summaries"][:len( + new_documents)] # Truncate if too long + while len(summaries) < len( + new_documents): # Pad with empty strings if too short summaries.append("") result_data["summaries"] = summaries print(f"Fixed summaries array length: {len(summaries)}") - + # Validate that relevant documents have non-empty summaries - for i, (rel, summary) in enumerate(zip(result_data["relevance"], result_data["summaries"])): + for i, (rel, summary) in enumerate( + zip(result_data["relevance"], result_data["summaries"])): if rel == 1 and not summary.strip(): - print(f"Warning: Document {i+1} marked relevant but has empty summary. This defeats the summarization purpose.") + print( + f"Warning: Document {i+1} marked relevant but has empty summary. This defeats the summarization purpose.") # Don't auto-fix here - let it be empty to debug the issue - + # Ensure queries is a list if not isinstance(result_data["queries"], list): result_data["queries"] = [] - + return result_data - + except requests.exceptions.RequestException as e: print(f"Error calling combined LLM: {e}") return { @@ -1106,14 +1171,14 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], **strategy_params) -> Dict[str, Any]: """ Multi-shot retrieval with iterative query refinement and document evaluation. - + Algorithm: 1. Generate initial search queries based on the original question 2. Retrieve documents for each query 3. Evaluate documents and check if sufficient to answer 4. If not sufficient: generate new queries based on what's missing, go to step 2 5. Repeat until sufficient or max_iterations reached - + Args: rag_db: RAG database instance original_query: Original user question @@ -1127,29 +1192,31 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], verbose: Print detailed information reasoning_effort: LLM reasoning level **strategy_params: Additional parameters for retrieval strategy - + Returns: Dictionary containing evaluation metrics and iteration statistics """ - + start_time = time.perf_counter() llm_start_time = None # set just before first generate_search_queries call llm_end_time = None # set just after generate_answer returns - + # Track iteration history query_history = [] query_results = [] # Track how many docs each query found - kept_docs = [] # List of (url, content, summary) tuples that were marked relevant - new_docs = [] # List of (url, content) tuples just retrieved this iteration + # List of (url, content, summary) tuples that were marked relevant + kept_docs = [] + # List of (url, content) tuples just retrieved this iteration + new_docs = [] all_retrieved_urls = set() iteration_times = [] previous_feedback = "" # Feedback from previous iteration feedback_history = [] # Track all feedback to show progression - + sufficient = False iteration = 0 final_answer = "" - + if verbose: print(f"\n{'='*80}") print(f"MULTI-SHOT RETRIEVAL") @@ -1158,29 +1225,33 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], print(f"Max iterations: {max_iterations}") print(f"Max sub-queries per iteration: {max_sub_queries}") print(f"{'='*80}\n") - + while not sufficient and iteration < max_iterations: iteration += 1 iteration_start = time.perf_counter() - + if verbose: print(f"\n{'─'*80}") print(f"ITERATION {iteration}/{max_iterations}") print(f"{'─'*80}") - - # Step 1: Use combined function to grade NEW docs AND generate new queries + + # Step 1: Use combined function to grade NEW docs AND generate new + # queries if verbose: print(f"\n Evaluating documents and generating queries...") - - # Aggressive summarization: use summaries after iteration 2 to improve information connection + + # Aggressive summarization: use summaries after iteration 2 to improve + # information connection total_content_length = sum(len(doc[1]) for doc in kept_docs) # Special handling for iteration 1: decompose original query first if iteration == 1 and not new_docs and not kept_docs: if verbose: - print(f" [ITERATION 1] Decomposing original query into sub-queries via generate_search_queries...") + print( + f" [ITERATION 1] Decomposing original query into sub-queries via generate_search_queries...") - # Use generate_search_queries for initial decomposition (uses query_model_name / gpt-oss-120b) + # Use generate_search_queries for initial decomposition (uses + # query_model_name / gpt-oss-120b) if llm_start_time is None: llm_start_time = time.perf_counter() query_result = generate_search_queries( @@ -1207,7 +1278,8 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], sufficient = False final_answer = "" - current_feedback = query_result.get("feedback", "Initial query decomposition") + current_feedback = query_result.get( + "feedback", "Initial query decomposition") relevance = [] summaries = [] reasoning_steps = "" @@ -1217,7 +1289,8 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], relevance = [] if new_docs: if verbose: - print(f" [CALL1] Evaluating {len(new_docs)} new documents with gpt-oss-20b...") + print( + f" [CALL1] Evaluating {len(new_docs)} new documents with gpt-oss-20b...") relevance_result = evaluate_document_relevance( question=original_query, @@ -1228,7 +1301,8 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], hop_count=iteration, query_id=query_id ) - relevance = relevance_result.get("relevance", [1] * len(new_docs)) + relevance = relevance_result.get( + "relevance", [1] * len(new_docs)) # Add relevant docs to kept_docs IMMEDIATELY for i, (url, content) in enumerate(new_docs): @@ -1236,14 +1310,16 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], kept_docs.append((url, content, content[:1000])) if verbose: - print(f" Marked {sum(relevance)} of {len(new_docs)} docs as relevant") + print( + f" Marked {sum(relevance)} of {len(new_docs)} docs as relevant") print(f" Relevance array: {relevance}") print(f" Total kept docs now: {len(kept_docs)}") # CALL 2: Check sufficiency - uses gpt-oss-120b if kept_docs: if verbose: - print(f" [CALL2] Checking sufficiency with gpt-oss-120b (iteration {iteration}/{max_iterations})...") + print( + f" [CALL2] Checking sufficiency with gpt-oss-120b (iteration {iteration}/{max_iterations})...") sufficiency_result = check_sufficiency( question=original_query, @@ -1267,11 +1343,13 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], sufficient = False sufficiency_reasoning = "No relevant documents kept yet" - # CALL 3a or 3b: Either generate answer (if sufficient) or generate queries (if not) + # CALL 3a or 3b: Either generate answer (if sufficient) or generate + # queries (if not) if sufficient: # CALL 3a: Generate final answer - uses gpt-oss-120b if verbose: - print(f" [CALL3a] Generating final answer with gpt-oss-120b...") + print( + f" [CALL3a] Generating final answer with gpt-oss-120b...") final_answer = generate_answer( question=original_query, @@ -1291,11 +1369,13 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], else: # CALL 3b: Generate search queries - uses gpt-oss-120b if verbose: - print(f" [CALL3b] Generating search queries with gpt-oss-120b...") + print( + f" [CALL3b] Generating search queries with gpt-oss-120b...") # Cap kept_docs sent to avoid context overflow MAX_DOCS_FOR_QUERY_GEN = 12 - docs_for_query_gen = kept_docs[-MAX_DOCS_FOR_QUERY_GEN:] if len(kept_docs) > MAX_DOCS_FOR_QUERY_GEN else kept_docs + docs_for_query_gen = kept_docs[-MAX_DOCS_FOR_QUERY_GEN:] if len( + kept_docs) > MAX_DOCS_FOR_QUERY_GEN else kept_docs query_result = generate_search_queries( question=original_query, @@ -1320,27 +1400,30 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], summaries = [] reasoning_steps = "" - # Add to feedback history if it's new and meaningful - if current_feedback and current_feedback.strip() and current_feedback != previous_feedback: + if current_feedback and current_feedback.strip( + ) and current_feedback != previous_feedback: feedback_history.append(current_feedback.strip()) previous_feedback = current_feedback # Only print status for iteration 1 after we've printed the queries # For iteration 2+, print status after CALL1/CALL2/CALL3 - # Skip printing here for iteration 1 (will print later after retrieval/grading) + # Skip printing here for iteration 1 (will print later after + # retrieval/grading) if verbose and iteration > 1: print(f" Sufficient: {'yes' if sufficient else 'no'}") print(f" Kept docs: {len(kept_docs)}") if new_docs: print(f" New docs evaluated: {len(new_docs)}") - print(f" Relevant new docs: {sum(relevance)}/{len(relevance)}") + print( + f" Relevant new docs: {sum(relevance)}/{len(relevance)}") print(f" Relevance array: {relevance}") # Show summary quality if summaries: non_empty_summaries = [s for s in summaries if s.strip()] - print(f" Generated summaries: {len(non_empty_summaries)}/{len(summaries)} non-empty") + print( + f" Generated summaries: {len(non_empty_summaries)}/{len(summaries)} non-empty") for i, summary in enumerate(summaries): if summary.strip() and relevance[i] == 1: print(f" Summary {i+1}: {summary}...") @@ -1349,10 +1432,11 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], if not sufficient: print(f" Feedback: {previous_feedback}") print(f" Generated {len(sub_queries)} new queries") - - # Clear new_docs for next iteration (already added to kept_docs in CALL1 block above) + + # Clear new_docs for next iteration (already added to kept_docs in + # CALL1 block above) new_docs = [] - + # If sufficient, we're done if sufficient: if verbose: @@ -1361,21 +1445,22 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], print(f" Answer: {final_answer[:200]}...") iteration_times.append(time.perf_counter() - iteration_start) break - - # If no queries generated, fall back to original query rather than stopping + + # If no queries generated, fall back to original query rather than + # stopping if not sub_queries: if verbose: print(f"\n ⚠ No new queries generated, falling back to original query") sub_queries = [original_query] - + if verbose: print(f"\n New queries:") for i, q in enumerate(sub_queries, 1): print(f" {i}. {q}") - + # Step 2: Retrieve for each sub-query and track results num_sub_queries = len(sub_queries) - #docs_per_subquery = max(1, top_k_retriever // num_sub_queries) + # docs_per_subquery = max(1, top_k_retriever // num_sub_queries) docs_per_subquery = max(1, top_k_retriever) iteration_results = [] @@ -1383,45 +1468,52 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], # Calculate target docs per subquery after reranking target_docs_per_subquery = max(3, top_k_retriever // num_sub_queries) - + for i, sub_query in enumerate(sub_queries, 1): if verbose: print(f"\n Retrieving for query {i}: {sub_query[:60]}...") - + query_start_count = len(new_docs) # Track docs before this query - + # Retrieve if retrieval_strategy == "fixed_k": results = rag_db.lookup(sub_query, k=docs_per_subquery) else: from retrieve.filter import filter original_max_results = strategy_params.get("max_results", 20) - #adjusted_max_results = max(1, original_max_results // num_sub_queries) + # adjusted_max_results = max(1, original_max_results // num_sub_queries) adjusted_max_results = max(1, original_max_results) strategy_params_copy = strategy_params.copy() strategy_params_copy["max_results"] = adjusted_max_results - results = filter(rag_db, sub_query, method=retrieval_strategy, **strategy_params_copy) - + results = filter( + rag_db, + sub_query, + method=retrieval_strategy, + **strategy_params_copy) + # Apply per-subquery reranking if enabled if not no_rerank and len(results) > target_docs_per_subquery: if verbose: - print(f" Reranking {len(results)} docs for this subquery to top {target_docs_per_subquery}...") - + print( + f" Reranking {len(results)} docs for this subquery to top {target_docs_per_subquery}...") + # Extract contents for reranking contents = [r.page_content for r in results] scored_passages = rag_db.rerank(sub_query, contents) - + # Reorder results by reranking scores and take top-k - reranked_indices = [i for i, _ in sorted(enumerate(scored_passages), + reranked_indices = [i for i, _ in sorted(enumerate(scored_passages), key=lambda x: x[1][1], reverse=True)] - results = [results[idx] for idx in reranked_indices[:target_docs_per_subquery]] - + results = [results[idx] + for idx in reranked_indices[:target_docs_per_subquery]] + if verbose: - print(f" After reranking: keeping top {len(results)} docs") + print( + f" After reranking: keeping top {len(results)} docs") elif len(results) > target_docs_per_subquery: # No reranking, just limit to target results = results[:target_docs_per_subquery] - + # Add to new_docs for evaluation (avoid duplicates) for result in results: if 'original_url' in result.metadata and result.metadata['original_url']: @@ -1430,27 +1522,30 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], all_retrieved_urls.add(url) new_docs.append((url, result.page_content)) iteration_results.append(result) - + # Track how many NEW docs this query found docs_found_by_query = len(new_docs) - query_start_count per_query_counts.append(docs_found_by_query) - + if verbose: - print(f" Retrieved {len(results)} docs, {docs_found_by_query} new unique docs from this query") + print( + f" Retrieved {len(results)} docs, {docs_found_by_query} new unique docs from this query") for j, result in enumerate(results, 1): url = result.metadata.get('original_url', 'N/A') passage = result.page_content[:300].replace('\n', ' ') print(f" [{j}] {url}\n {passage}...") - + # Add queries and their results to history for sub_query, count in zip(sub_queries, per_query_counts): query_history.append(sub_query) query_results.append(count) - + if verbose: - print(f" Total kept docs: {len(kept_docs)}, new docs to evaluate: {len(new_docs)}") + print( + f" Total kept docs: {len(kept_docs)}, new docs to evaluate: {len(new_docs)}") - # For iteration 1, print status summary now (after retrieval, before next iteration's grading) + # For iteration 1, print status summary now (after retrieval, before + # next iteration's grading) if verbose and iteration == 1: print(f"\n Iteration 1 Summary:") print(f" Generated {len(sub_queries)} new queries") @@ -1458,15 +1553,15 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], iteration_time = time.perf_counter() - iteration_start iteration_times.append(iteration_time) - + if iteration >= max_iterations: if verbose: print(f"\n ⚠ Maximum iterations reached") break - + # Final processing total_time = time.perf_counter() - start_time - + # Extract URLs from kept_docs retrieved_urls = [] for doc in kept_docs: @@ -1474,17 +1569,20 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], retrieved_urls.append(doc[0]) # url is first element elif len(doc) == 2: # Handle old format for backward compatibility retrieved_urls.append(doc[0]) # url is first element - + # Limit to top_k_reranking (reranking already done per-subquery) retrieved_urls = retrieved_urls[:top_k_reranking] - + # Calculate metrics from evaluation import calculate_retrieval_metrics expected_set = set(url for url in expected_urls if url and url.strip()) metrics = calculate_retrieval_metrics(list(expected_set), retrieved_urls) - + # Add iteration statistics - query_llm_time = (llm_end_time - llm_start_time) if (llm_start_time is not None and llm_end_time is not None) else total_time + query_llm_time = ( + llm_end_time - + llm_start_time) if ( + llm_start_time is not None and llm_end_time is not None) else total_time metrics.update({ 'total_time': total_time, 'query_llm_time': query_llm_time, @@ -1495,7 +1593,7 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], 'avg_iteration_time': sum(iteration_times) / len(iteration_times) if iteration_times else 0, 'llm_answer': final_answer, }) - + # Print final results if verbose: print(f"\n{'='*80}") @@ -1509,8 +1607,10 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], print(f"LLM Answer: {final_answer}") if expected_answer: print(f"Expected Answer: {expected_answer}") - print(f"Expected ({len(expected_set)}): {sorted(list(expected_set)[:3])}{'...' if len(expected_set) > 3 else ''}") - print(f"Retrieved ({len(retrieved_urls)} unique docs): {retrieved_urls[:3]}{'...' if len(retrieved_urls) > 3 else ''}") + print( + f"Expected ({len(expected_set)}): {sorted(list(expected_set)[:3])}{'...' if len(expected_set) > 3 else ''}") + print( + f"Retrieved ({len(retrieved_urls)} unique docs): {retrieved_urls[:3]}{'...' if len(retrieved_urls) > 3 else ''}") matches = len(expected_set.intersection(set(retrieved_urls))) print(f"Matches: {matches}") print(f"\nMetrics:") @@ -1519,10 +1619,11 @@ def multi_shot_retrieval(rag_db, original_query: str, expected_urls: List[str], print(f" F1@N: {metrics.get('f1@N', 0.0):.3f}") print(f" MAP: {metrics.get('average_precision', 0.0):.3f}") print(f"\nTiming:") - print(f" Avg per iteration: {metrics['avg_iteration_time']*1000:.1f}ms") + print( + f" Avg per iteration: {metrics['avg_iteration_time']*1000:.1f}ms") print(f" Total: {total_time*1000:.1f}ms") print(f"{'='*80}\n") - + return metrics @@ -1543,7 +1644,7 @@ def run_multi_shot_evaluation(rag_db, dataset_path: str, **strategy_params) -> Dict[str, float]: """ Run multi-shot evaluation on a dataset. - + Args: rag_db: RAG database instance dataset_path: Path to dataset TSV file @@ -1558,21 +1659,21 @@ def run_multi_shot_evaluation(rag_db, dataset_path: str, difficulty: Minimum number of answer links required (0 = no filtering) max_iterations: Maximum iterations for iterative retrieval (default: 10) **strategy_params: Additional parameters for retrieval strategy - + Returns: Dictionary of averaged metrics """ - + df = pd.read_csv(dataset_path, sep='\t') - + # Filter by difficulty if specified df = filter_dataset_by_difficulty(df, difficulty) - + if isinstance(max_queries, int) and max_queries > 0: df = df.head(max_queries) else: max_queries = len(df) - + print(f"\n{'='*80}") print(f"MULTI-SHOT EVALUATION") print(f"{'='*80}") @@ -1586,7 +1687,7 @@ def run_multi_shot_evaluation(rag_db, dataset_path: str, if difficulty > 0: print(f"Difficulty filter: >= {difficulty} answer links") print(f"{'='*80}\n") - + eval_wall_start = time.perf_counter() total_metrics = {} valid_queries = 0 @@ -1600,9 +1701,12 @@ def run_multi_shot_evaluation(rag_db, dataset_path: str, for col in df.columns: if col.startswith('wikipedia_link_') and pd.notna(row[col]): expected_urls.append(row[col].strip()) - expected_answer = row.get('Answer', '').strip() if 'Answer' in row and pd.notna(row.get('Answer')) else "" + expected_answer = row.get( + 'Answer', '').strip() if 'Answer' in row and pd.notna( + row.get('Answer')) else "" if expected_urls: - work_items.append((idx, row['Prompt'], expected_urls, expected_answer)) + work_items.append( + (idx, row['Prompt'], expected_urls, expected_answer)) def process_single_query(item): idx, prompt, expected_urls, expected_answer = item @@ -1630,7 +1734,8 @@ def process_single_query(item): ) query_wall_time = metrics.get('total_time', 0.0) - print(f" [QUERY TIMING] Query {idx+1}/{max_queries}: {query_wall_time:.2f}s") + print( + f" [QUERY TIMING] Query {idx+1}/{max_queries}: {query_wall_time:.2f}s") if logger: logger.end_query( @@ -1672,7 +1777,10 @@ def process_single_query(item): # Parallel execution with thread pool print(f"\n Using {num_workers} parallel workers") with ThreadPoolExecutor(max_workers=num_workers) as executor: - futures = {executor.submit(process_single_query, item): item for item in work_items} + futures = { + executor.submit( + process_single_query, + item): item for item in work_items} for future in as_completed(futures): try: idx, metrics, result = future.result() @@ -1689,45 +1797,60 @@ def process_single_query(item): print(f" Error processing query: {e}") import traceback traceback.print_exc() - + if valid_queries > 0: # Calculate averages - avg_metrics = {name: total / valid_queries for name, total in total_metrics.items()} - + avg_metrics = { + name: total / + valid_queries for name, + total in total_metrics.items()} + # Print summary print(f"\n{'='*80}") print(f"MULTI-SHOT EVALUATION SUMMARY ({valid_queries} queries)") print(f"{'='*80}") print(f"\nPRECISION METRICS:") - print(f" Precision@N: {avg_metrics.get('precision@N', 0.0):.3f}") + print( + f" Precision@N: {avg_metrics.get('precision@N', 0.0):.3f}") print(f"\nRECALL METRICS:") - print(f" Recall@N: {avg_metrics.get('recall@N', 0.0):.3f}") + print( + f" Recall@N: {avg_metrics.get('recall@N', 0.0):.3f}") print(f"\nF1 METRICS:") - print(f" F1@N: {avg_metrics.get('f1@N', 0.0):.3f}") + print( + f" F1@N: {avg_metrics.get('f1@N', 0.0):.3f}") print(f"\nRANKING METRICS:") - print(f" Mean Average Precision: {avg_metrics.get('average_precision', 0.0):.3f}") + print( + f" Mean Average Precision: {avg_metrics.get('average_precision', 0.0):.3f}") print(f"\nRETRIEVAL STATISTICS:") - print(f" Avg Sub-queries: {avg_metrics.get('num_sub_queries', 0.0):.1f}") - print(f" Avg Passages Retrieved: {avg_metrics.get('retrieved_passages_count', 0.0):.1f}") - print(f" Avg Unique Docs (N): {avg_metrics.get('retrieved_docs_count', 0.0):.1f}") + print( + f" Avg Sub-queries: {avg_metrics.get('num_sub_queries', 0.0):.1f}") + print( + f" Avg Passages Retrieved: {avg_metrics.get('retrieved_passages_count', 0.0):.1f}") + print( + f" Avg Unique Docs (N): {avg_metrics.get('retrieved_docs_count', 0.0):.1f}") print(f"\nTIMING:") - print(f" Avg Decomposition Time: {avg_metrics.get('decomposition_time', 0.0)*1000:.1f}ms") - print(f" Avg Retrieval Time: {avg_metrics.get('retrieval_time', 0.0)*1000:.1f}ms") + print( + f" Avg Decomposition Time: {avg_metrics.get('decomposition_time', 0.0)*1000:.1f}ms") + print( + f" Avg Retrieval Time: {avg_metrics.get('retrieval_time', 0.0)*1000:.1f}ms") if avg_metrics.get('reranking_time', 0.0) > 0: - print(f" Avg Reranking Time: {avg_metrics.get('reranking_time', 0.0)*1000:.1f}ms") - print(f" Avg Total Time: {avg_metrics.get('total_time', 0.0)*1000:.1f}ms") - print(f" Avg Query LLM Time: {avg_metrics.get('query_llm_time', 0.0)*1000:.1f}ms") + print( + f" Avg Reranking Time: {avg_metrics.get('reranking_time', 0.0)*1000:.1f}ms") + print( + f" Avg Total Time: {avg_metrics.get('total_time', 0.0)*1000:.1f}ms") + print( + f" Avg Query LLM Time: {avg_metrics.get('query_llm_time', 0.0)*1000:.1f}ms") eval_wall_time = time.perf_counter() - eval_wall_start qps = valid_queries / eval_wall_time if eval_wall_time > 0 else 0.0 print(f" Total Wall Time: {eval_wall_time:.1f}s") print(f" Throughput: {qps:.3f} queries/sec") print(f"{'='*80}\n") - + # Print detailed analysis if requested if detailed_analysis and all_query_metrics: from evaluation import _print_detailed_analysis _print_detailed_analysis(df, all_query_metrics, valid_queries) - + avg_metrics['_per_query_results'] = all_results return avg_metrics else: @@ -1738,26 +1861,26 @@ def process_single_query(item): if __name__ == "__main__": args = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description="Multi-shot retrieval with query decomposition") - + # Add all standard parameters add_all_args(args) - + # Add multi-shot specific parameters args.add_argument('--max-sub-queries', type=int, default=3, - help='Maximum number of sub-queries to generate (default: 3)') + help='Maximum number of sub-queries to generate (default: 3)') args.add_argument('--reasoning', type=str, default='medium', - choices=['low', 'medium', 'high'], - help='LLM reasoning level for query decomposition (default: medium)') + choices=['low', 'medium', 'high'], + help='LLM reasoning level for query decomposition (default: medium)') args.add_argument('--max-iterations', type=int, default=10, - help='Maximum number of retrieval iterations (default: 10)') + help='Maximum number of retrieval iterations (default: 10)') args.add_argument('--num-workers', type=int, default=1, - help='Number of parallel query workers (default: 1, sequential)') + help='Number of parallel query workers (default: 1, sequential)') args.add_argument('--temperature', type=float, default=1.0, - help='LLM sampling temperature (default: 1.0)') + help='LLM sampling temperature (default: 1.0)') args.add_argument('--max-retries', type=int, default=5, - help='Max retries for LLM calls on rate limit/server errors (default: 5)') + help='Max retries for LLM calls on rate limit/server errors (default: 5)') args.add_argument('--output-dir', type=str, default='.', - help='Directory for output files (default: current directory)') + help='Directory for output files (default: current directory)') # Special handling for --eval argument for action in args._actions: @@ -1765,12 +1888,12 @@ def process_single_query(item): action.type = lambda x: int(x) if x.isdigit() else True action.const = True break - + args = args.parse_args() - + # Set deterministic seeds set_deterministic_seeds(args.seed) - + # Setup LLM configuration with auto-detection llm_config = setup_llm_config(args) llm_config['temperature'] = args.temperature @@ -1797,13 +1920,15 @@ def process_single_query(item): # Setup device-specific environment device_config = get_device_config() print(f"Device Config: {device_config}") - + # Initialize database if args.database is None: args.database = VectorDB.get_default_db_name() - db_file_path = args.database if args.database.endswith('.db') else f"{args.database}.db" - db_base_name = args.database.replace('.db', '') if args.database.endswith('.db') else args.database + db_file_path = args.database if args.database.endswith( + '.db') else f"{args.database}.db" + db_base_name = args.database.replace( + '.db', '') if args.database.endswith('.db') else args.database rag_db = VectorDB( retriever_model=args.retriever_model, @@ -1816,14 +1941,15 @@ def process_single_query(item): reranker_device=args.reranker_device, benchmark=args.benchmark ) - + # Load database if os.path.exists(db_file_path): print(f"Loading existing database from {db_file_path}") rag_db.from_serialized(db_file_path) else: - raise ValueError(f"Database not found: {db_file_path}. Please create it first using single_shot_retrieval.py") - + raise ValueError( + f"Database not found: {db_file_path}. Please create it first using single_shot_retrieval.py") + # Build strategy parameters strategy_params = {"max_results": args.max_results} if args.retrieval_strategy == "top_p": @@ -1834,7 +1960,9 @@ def process_single_query(item): # Initialize LLM logger with incremental writing experiment_start_time = datetime.now() os.makedirs(args.output_dir, exist_ok=True) - log_filename = os.path.join(args.output_dir, f"llm_logs_multi_shot_{experiment_start_time.strftime('%Y%m%d_%H%M%S')}.json") + log_filename = os.path.join( + args.output_dir, + f"llm_logs_multi_shot_{experiment_start_time.strftime('%Y%m%d_%H%M%S')}.json") # Determine chunk size from database name chunk_size = 768 # default @@ -1848,7 +1976,8 @@ def process_single_query(item): llm_logger = LLMLogger( output_file=log_filename, experiment_metadata={ - "experiment_name": f"multi_shot_{db_base_name}_n{{queries}}", # Will be updated + # Will be updated + "experiment_name": f"multi_shot_{db_base_name}_n{{queries}}", "timestamp_start": experiment_start_time.isoformat(), "timestamp_end": "in_progress", "retrieval_mode": "multi_shot", @@ -1867,7 +1996,6 @@ def process_single_query(item): ) print(f"LLM logs will be written incrementally to: {log_filename}") - # Setup threading infrastructure if parallel workers requested if args.num_workers > 1: print(f"Enabling parallel execution with {args.num_workers} workers") @@ -1875,7 +2003,9 @@ def process_single_query(item): # Run evaluation or single query if args.eval: - max_queries = args.eval if isinstance(args.eval, int) and not isinstance(args.eval, bool) and args.eval > 0 else None + max_queries = args.eval if isinstance( + args.eval, int) and not isinstance( + args.eval, bool) and args.eval > 0 else None metrics = run_multi_shot_evaluation( rag_db, args.dataset, @@ -1894,7 +2024,7 @@ def process_single_query(item): num_workers=args.num_workers, **strategy_params ) - + # Save results per_query_results = metrics.pop('_per_query_results', []) results_data = { @@ -1904,7 +2034,7 @@ def process_single_query(item): "metrics": metrics, "results": per_query_results, } - + result_path = os.path.join(args.output_dir, "result_multi_shot.json") with open(result_path, "w") as f: json.dump(results_data, f, indent=2) @@ -1981,5 +2111,6 @@ def process_single_query(item): rq = rag_db._reranker_queue if rq is not None: avg_ms = rq.total_latency_ms / rq.total_requests if rq.total_requests else 0 - print(f"Reranker stats: {rq.total_requests} requests, {rq.total_documents} docs, {rq.total_latency_ms:.0f}ms total, {avg_ms:.1f}ms/request avg") + print( + f"Reranker stats: {rq.total_requests} requests, {rq.total_documents} docs, {rq.total_latency_ms:.0f}ms total, {avg_ms:.1f}ms/request avg") rag_db.shutdown_reranker() diff --git a/e2e-rag/oracle_single_shot.py b/e2e-rag/oracle_single_shot.py index 1a39f7fd65..1496daf1c5 100644 --- a/e2e-rag/oracle_single_shot.py +++ b/e2e-rag/oracle_single_shot.py @@ -46,14 +46,16 @@ DEFAULT_CHECKPOINT_FILE = "oracle_checkpoint.pkl" DEFAULT_SERVICE_URL = "http://localhost:8123/v1/chat/completions" -#DEFAULT_MODEL_NAME = "/mnt/weka/data/pytorch/llama3.3/Meta-Llama-3.3-70B-Instruct" -#DEFAULT_MODEL_NAME = "/mnt/weka/data/pytorch/llama3.1/Meta-Llama-3.1-405B-Instruct-v2" +# DEFAULT_MODEL_NAME = "/mnt/weka/data/pytorch/llama3.3/Meta-Llama-3.3-70B-Instruct" +# DEFAULT_MODEL_NAME = "/mnt/weka/data/pytorch/llama3.1/Meta-Llama-3.1-405B-Instruct-v2" DEFAULT_MODEL_NAME = "/model/gpt-oss-120b-mxfp4" DEFAULT_BATCH_SIZE = 1 DEFAULT_TIMEOUT = 2400 # For reasoning model, it should be large enough -DEFAULT_MAX_TOKENS = 10*1024 -MAX_TOTAL_CHARS = 400000 # total char budget split across all docs per query (~100K tokens @ 4 chars/token) +DEFAULT_MAX_TOKENS = 10 * 1024 +# total char budget split across all docs per query (~100K tokens @ 4 +# chars/token) +MAX_TOTAL_CHARS = 400000 # Global cache for URL to filename mapping _url_to_file_cache: Optional[Dict[str, Path]] = None @@ -145,27 +147,28 @@ def build_url_to_file_cache(wiki_dir: Path) -> Dict[str, Path]: Uses JSON metadata files to get accurate URL mapping. """ cache = {} - + print(f"Building URL cache from {wiki_dir}...") json_files = list(wiki_dir.glob("*.json")) - + for json_file in json_files: try: with open(json_file, 'r') as f: data = json.load(f) - + # Get URL without fragment url = data.get('url', '') source_url = data.get('source_url', '') - + # Remove fragment from source_url if present if '#' in source_url: source_url = source_url.split('#')[0] - + # Get corresponding .txt file txt_file = json_file.with_suffix('.txt') if txt_file.exists(): - # Map both url and source_url (without fragment) to the file + # Map both url and source_url (without fragment) to the + # file if url: cache[url] = txt_file if source_url and source_url != url: @@ -173,7 +176,7 @@ def build_url_to_file_cache(wiki_dir: Path) -> Dict[str, Path]: except Exception as e: # Skip files with errors continue - + print(f"Cached {len(cache)} URL mappings from {len(json_files)} JSON files") return cache @@ -184,27 +187,28 @@ def find_wiki_article(url: str, wiki_dir: Path) -> Optional[Path]: Uses JSON metadata for accurate matching. """ global _url_to_file_cache - + if not url or "wikipedia.org/wiki/" not in url: return None - + # Build cache on first call if _url_to_file_cache is None: _url_to_file_cache = build_url_to_file_cache(wiki_dir) - + # Remove fragment from URL if present url_no_fragment = url.split('#')[0] - + # Look up in cache if url in _url_to_file_cache: return _url_to_file_cache[url] elif url_no_fragment in _url_to_file_cache: return _url_to_file_cache[url_no_fragment] - + return None -def load_wiki_articles(wiki_urls: List[str], wiki_dir: Path, max_chars: int = MAX_TOTAL_CHARS) -> Tuple[List[str], List[str], List[int]]: +def load_wiki_articles(wiki_urls: List[str], wiki_dir: Path, + max_chars: int = MAX_TOTAL_CHARS) -> Tuple[List[str], List[str], List[int]]: """ Load Wikipedia articles from wiki_articles folder. max_chars is the TOTAL character budget shared across all docs for this query. @@ -225,7 +229,8 @@ def load_wiki_articles(wiki_urls: List[str], wiki_dir: Path, max_chars: int = MA if file_path and file_path.exists(): try: - content = file_path.read_text(encoding="utf-8", errors="ignore") + content = file_path.read_text( + encoding="utf-8", errors="ignore") truncated = content[:per_doc_limit] documents.append(truncated) file_paths.append(str(file_path)) @@ -239,14 +244,15 @@ def load_wiki_articles(wiki_urls: List[str], wiki_dir: Path, max_chars: int = MA documents.append("") file_paths.append("") doc_lengths.append(0) - + return documents, file_paths, doc_lengths -def generate_llm_answer(query: str, documents: List[str], urls: List[str], llm_config: Dict) -> str: +def generate_llm_answer( + query: str, documents: List[str], urls: List[str], llm_config: Dict) -> str: """ Generate LLM answer using the provided documents as context. - + This function is adapted from single_shot_retrieval.py _generate_llm_answer """ context_parts = [] @@ -255,15 +261,16 @@ def generate_llm_answer(query: str, documents: List[str], urls: List[str], llm_c source = url or "Unknown source" snippet = doc.strip() context_parts.append(f"[{idx}] Source: {source}\n{snippet}") - - evidence_block = "\n\n".join(context_parts) if context_parts else "No supporting documents were retrieved." - + + evidence_block = "\n\n".join( + context_parts) if context_parts else "No supporting documents were retrieved." + user_prompt = ( "Answer the question using only the provided evidence." " Respond with a single word or short phrase, or 'Unknown' if the evidence is insufficient.\n\n" f"Question:\n{query}\n\nEvidence:\n{evidence_block}" ) - + payload = { "model": llm_config["model_name"], "messages": [ @@ -283,11 +290,14 @@ def generate_llm_answer(query: str, documents: List[str], urls: List[str], llm_c if llm_config.get("reasoning_effort"): payload["reasoning_effort"] = llm_config["reasoning_effort"] - response = requests.post(llm_config["service_url"], json=payload, timeout=llm_config["timeout"]) + response = requests.post( + llm_config["service_url"], + json=payload, + timeout=llm_config["timeout"]) response.raise_for_status() data = response.json() - #from pprint import pprint - #pprint(data, indent=4) + # from pprint import pprint + # pprint(data, indent=4) return data["choices"][0]["message"]["content"].strip() @@ -315,7 +325,7 @@ def parse_wiki_links(wiki_links_str: str) -> List[str]: try: # Use ast.literal_eval to safely parse the string as a Python literal return ast.literal_eval(wiki_links_str) - except: + except BaseException: return [] @@ -344,7 +354,8 @@ def process_single( } try: - documents, file_paths, doc_lengths = load_wiki_articles(wiki_urls, wiki_dir) + documents, file_paths, doc_lengths = load_wiki_articles( + wiki_urls, wiki_dir) result["wiki_file_paths"] = str(file_paths) result["doc_lengths"] = str(doc_lengths) result["total_doc_length"] = sum(doc_lengths) @@ -354,9 +365,11 @@ def process_single( result["num_missing_docs"] = missing_count if missing_count > 0: - print(f" Query {idx}: {missing_count}/{len(wiki_urls)} documents missing") + print( + f" Query {idx}: {missing_count}/{len(wiki_urls)} documents missing") - llm_answer = generate_llm_answer(query, documents, wiki_urls, llm_config) + llm_answer = generate_llm_answer( + query, documents, wiki_urls, llm_config) result["llm_answer"] = llm_answer result["success"] = True @@ -386,7 +399,14 @@ def process_batch( futures_map = {} with ThreadPoolExecutor(max_workers=len(batch_data)) as executor: for idx, query, ground_truth, wiki_urls in batch_data: - future = executor.submit(process_single, idx, query, ground_truth, wiki_urls, wiki_dir, llm_config) + future = executor.submit( + process_single, + idx, + query, + ground_truth, + wiki_urls, + wiki_dir, + llm_config) futures_map[future] = idx results_map = {} @@ -400,38 +420,40 @@ def process_batch( def main(): args = parse_args() - + # Setup paths dataset_path = Path(args.dataset) wiki_dir = Path(args.wiki_articles_dir) checkpoint_file = Path(args.checkpoint_file) - + # Validate paths if not dataset_path.exists(): raise FileNotFoundError(f"Dataset not found: {dataset_path}") if not wiki_dir.exists(): - raise FileNotFoundError(f"Wiki articles directory not found: {wiki_dir}") + raise FileNotFoundError( + f"Wiki articles directory not found: {wiki_dir}") # Pre-build URL cache once before threads start global _url_to_file_cache _url_to_file_cache = build_url_to_file_cache(wiki_dir) - + # Load dataset print(f"Loading dataset from {dataset_path}...") df = pd.read_csv(dataset_path, sep="\t") - + # Apply max_queries limit if specified if args.max_queries: df = df.head(args.max_queries) - + print(f"Total queries in dataset: {len(df)}") - + # Load checkpoint checkpoint_df = load_checkpoint(checkpoint_file) - processed_indices = set(checkpoint_df["index"].tolist()) if not checkpoint_df.empty else set() - + processed_indices = set( + checkpoint_df["index"].tolist()) if not checkpoint_df.empty else set() + print(f"Already processed: {len(processed_indices)} queries") - + # LLM configuration llm_config = { "service_url": args.service_url, @@ -441,16 +463,16 @@ def main(): "enable_thinking": args.enable_thinking, "reasoning_effort": args.reasoning_effort, } - + # Determine which queries to process if args.retry_failed: # Retry only failed queries if not checkpoint_df.empty: failed_df = checkpoint_df[checkpoint_df["success"] == False] - queries_to_process = [(int(row["index"]), df.iloc[int(row["index"])]["Prompt"], + queries_to_process = [(int(row["index"]), df.iloc[int(row["index"])]["Prompt"], df.iloc[int(row["index"])]["Answer"], parse_wiki_links(df.iloc[int(row["index"])]["wiki_links"])) - for _, row in failed_df.iterrows()] + for _, row in failed_df.iterrows()] print(f"Retrying {len(queries_to_process)} failed queries...") else: queries_to_process = [] @@ -460,85 +482,93 @@ def main(): for idx, row in df.iterrows(): if idx not in processed_indices: wiki_urls = parse_wiki_links(row["wiki_links"]) - queries_to_process.append((idx, row["Prompt"], row["Answer"], wiki_urls)) - + queries_to_process.append( + (idx, row["Prompt"], row["Answer"], wiki_urls)) + print(f"Processing {len(queries_to_process)} new queries...") - + if not queries_to_process: # If no new queries and all done, check for failed ones if not args.retry_failed: - failed_count = len(checkpoint_df[checkpoint_df["success"] == False]) if not checkpoint_df.empty else 0 + failed_count = len( + checkpoint_df[checkpoint_df["success"] == False]) if not checkpoint_df.empty else 0 if failed_count > 0: - print(f"\nAll new queries processed. {failed_count} queries failed.") + print( + f"\nAll new queries processed. {failed_count} queries failed.") print("Run with --retry-failed to retry failed queries.") else: print("\nAll queries successfully processed!") else: print("No failed queries to retry!") return - + # Process in batches batch_size = args.batch_size total_batches = (len(queries_to_process) + batch_size - 1) // batch_size - + print(f"Batch size: {batch_size}") print(f"Total batches: {total_batches}") print(f"Service URL: {llm_config['service_url']}") print(f"Model: {llm_config['model_name']}\n") - + for batch_idx in range(total_batches): start_idx = batch_idx * batch_size end_idx = min(start_idx + batch_size, len(queries_to_process)) batch = queries_to_process[start_idx:end_idx] - + print(f"Processing batch {batch_idx + 1}/{total_batches} " f"(queries {start_idx + 1}-{end_idx})...") - + # Process batch batch_results = process_batch(batch, wiki_dir, llm_config) - + # Convert batch results to DataFrame batch_df = pd.DataFrame(batch_results) - + # Update checkpoint if args.retry_failed: # For retry, update existing results # Remove old entries for these indices indices_to_update = batch_df["index"].tolist() - checkpoint_df = checkpoint_df[~checkpoint_df["index"].isin(indices_to_update)] + checkpoint_df = checkpoint_df[~checkpoint_df["index"].isin( + indices_to_update)] # Append new results - checkpoint_df = pd.concat([checkpoint_df, batch_df], ignore_index=True) + checkpoint_df = pd.concat( + [checkpoint_df, batch_df], ignore_index=True) else: # For new queries, append results - checkpoint_df = pd.concat([checkpoint_df, batch_df], ignore_index=True) - + checkpoint_df = pd.concat( + [checkpoint_df, batch_df], ignore_index=True) + # Sort by index for consistency - checkpoint_df = checkpoint_df.sort_values("index").reset_index(drop=True) - + checkpoint_df = checkpoint_df.sort_values( + "index").reset_index(drop=True) + # Save checkpoint after each batch save_checkpoint(checkpoint_file, checkpoint_df) print(f" Checkpoint saved to {checkpoint_file}") - + # Show batch statistics success_count = batch_df["success"].sum() print(f" Batch success rate: {success_count}/{len(batch_results)}\n") - + # Final statistics - print("\n" + "="*60) + print("\n" + "=" * 60) print("Processing complete!") - print("="*60) - + print("=" * 60) + total_processed = len(checkpoint_df) total_success = checkpoint_df["success"].sum() total_failed = total_processed - total_success - + print(f"Total queries processed: {total_processed}") print(f"Successful: {total_success}") print(f"Failed: {total_failed}") - + if total_failed > 0: - print(f"\nRun with --retry-failed to retry {total_failed} failed queries.") - + print( + f"\nRun with --retry-failed to retry {total_failed} failed queries.") + print(f"\nResults saved to: {checkpoint_file}") diff --git a/e2e-rag/params.py b/e2e-rag/params.py index b59e573937..7333feac5e 100644 --- a/e2e-rag/params.py +++ b/e2e-rag/params.py @@ -22,7 +22,7 @@ Usage: from params import add_all_args, add_common_args, add_retrieval_args - + parser = argparse.ArgumentParser() add_all_args(parser) # Add all parameters # OR @@ -37,9 +37,12 @@ # ============================================================================ # Parameter Definitions # ============================================================================ + + class ParamDef: """Parameter definition with metadata.""" - def __init__(self, + + def __init__(self, name: str, arg_names: List[str], type: type, @@ -76,35 +79,36 @@ def __init__(self, self.category = category self.applies_to = applies_to or ["both"] self.optuna_suggest = optuna_suggest - + def add_to_parser(self, parser: argparse.ArgumentParser): """Add this parameter to an argument parser.""" kwargs = { 'help': self.help, 'default': self.default, } - + if self.action: kwargs['action'] = self.action else: kwargs['type'] = self.type - + if self.choices: kwargs['choices'] = self.choices - + if self.nargs: kwargs['nargs'] = self.nargs - + parser.add_argument(*self.arg_names, **kwargs) - + def suggest_value(self, trial): """Suggest a value for Optuna trial.""" if not self.optuna_suggest: - raise ValueError(f"No optuna_suggest config for parameter {self.name}") - + raise ValueError( + f"No optuna_suggest config for parameter {self.name}") + config = self.optuna_suggest suggest_type = config['type'] - + if suggest_type == 'float': return trial.suggest_float( self.name, @@ -477,7 +481,12 @@ def suggest_value(self, trial): choices=["fixed_k", "top_p", "relative"], category="strategy", applies_to=["both"], - optuna_suggest={'type': 'categorical', 'choices': ["fixed_k", "top_p", "relative"]} + optuna_suggest={ + 'type': 'categorical', + 'choices': [ + "fixed_k", + "top_p", + "relative"]} ), ParamDef( name="top_k_retriever", @@ -557,8 +566,10 @@ def suggest_value(self, trial): } # Method-specific parameters -BM25_METHOD_PARAMS = [p for p in ALL_PARAMS if "bm25" in p.applies_to or "both" in p.applies_to] -VECTOR_METHOD_PARAMS = [p for p in ALL_PARAMS if "vector" in p.applies_to or "both" in p.applies_to] +BM25_METHOD_PARAMS = [ + p for p in ALL_PARAMS if "bm25" in p.applies_to or "both" in p.applies_to] +VECTOR_METHOD_PARAMS = [ + p for p in ALL_PARAMS if "vector" in p.applies_to or "both" in p.applies_to] # Optimizable parameters (those with optuna_suggest defined) OPTIMIZABLE_PARAMS = [p for p in ALL_PARAMS if p.optuna_suggest is not None] @@ -568,24 +579,26 @@ def suggest_value(self, trial): # Helper Functions # ============================================================================ + def add_common_args(parser: argparse.ArgumentParser): """ Add common script parameters (ingest, database, query, etc.) - + Args: parser: ArgumentParser to add arguments to """ for param in COMMON_PARAMS: param.add_to_parser(parser) -def add_retrieval_args(parser: argparse.ArgumentParser, + +def add_retrieval_args(parser: argparse.ArgumentParser, method: Optional[str] = None, categories: Optional[List[str]] = None): """ Add retrieval parameters to an argument parser. Includes: General, BM25, Vector, Strategy, and Reranking parameters. Does NOT include common script parameters (use add_common_args for those). - + Args: parser: ArgumentParser to add arguments to method: Filter by method ('bm25', 'vector', or None for all) @@ -593,24 +606,26 @@ def add_retrieval_args(parser: argparse.ArgumentParser, """ # Get non-common params params_to_add = [p for p in ALL_PARAMS if p.category != "common"] - + # Filter by method if method: - params_to_add = [p for p in params_to_add - if method in p.applies_to or "both" in p.applies_to] - + params_to_add = [p for p in params_to_add + if method in p.applies_to or "both" in p.applies_to] + # Filter by category if categories: params_to_add = [p for p in params_to_add if p.category in categories] - + # Add to parser for param in params_to_add: param.add_to_parser(parser) -def add_all_args(parser: argparse.ArgumentParser, method: Optional[str] = None): + +def add_all_args(parser: argparse.ArgumentParser, + method: Optional[str] = None): """ Add all parameters (common + retrieval) to an argument parser. - + Args: parser: ArgumentParser to add arguments to method: Filter by method ('bm25', 'vector', or None for all) @@ -618,48 +633,51 @@ def add_all_args(parser: argparse.ArgumentParser, method: Optional[str] = None): add_common_args(parser) add_retrieval_args(parser, method=method) + def get_optimizable_params(method: Optional[str] = None) -> List[ParamDef]: """ Get list of parameters that can be optimized with Optuna. - + Args: method: Filter by method ('bm25', 'vector', or None for all) - + Returns: List of ParamDef objects """ params = OPTIMIZABLE_PARAMS - + if method: - params = [p for p in params - if method in p.applies_to or "both" in p.applies_to] - + params = [p for p in params + if method in p.applies_to or "both" in p.applies_to] + return params + def suggest_param(trial, param_name: str) -> Any: """ Suggest a parameter value for Optuna trial. - + Args: trial: Optuna trial object param_name: Parameter name - + Returns: Suggested value """ if param_name not in PARAM_BY_NAME: raise ValueError(f"Unknown parameter: {param_name}") - + param = PARAM_BY_NAME[param_name] return param.suggest_value(trial) + def get_default_params(method: str) -> Dict[str, Any]: """ Get default parameter values for a method. - + Args: method: 'bm25' or 'vector' - + Returns: Dictionary of parameter name -> default value """ @@ -669,34 +687,36 @@ def get_default_params(method: str) -> Dict[str, Any]: params = VECTOR_METHOD_PARAMS else: params = ALL_PARAMS - + return {p.name: p.default for p in params} -def format_params_for_cli(params: Dict[str, Any], skip_defaults: bool = True) -> List[str]: + +def format_params_for_cli( + params: Dict[str, Any], skip_defaults: bool = True) -> List[str]: """ Format parameter dictionary as CLI arguments. - + Args: params: Dictionary of parameter name -> value skip_defaults: If True, skip parameters that match their default values - + Returns: List of CLI argument strings """ args = [] - + for name, value in params.items(): if name not in PARAM_BY_NAME: continue - + param_def = PARAM_BY_NAME[name] - + # Skip if value matches default (when skip_defaults=True) if skip_defaults and value == param_def.default: continue - + cli_arg = param_def.arg_names[0] - + if param_def.action == "store_true": if value: args.append(cli_arg) @@ -705,40 +725,42 @@ def format_params_for_cli(params: Dict[str, Any], skip_defaults: bool = True) -> args.append(cli_arg) elif value is not None: args.extend([cli_arg, str(value)]) - + return args + def print_param_info(method: Optional[str] = None): """Print parameter information grouped by category.""" - + if method == "bm25": params = BM25_METHOD_PARAMS elif method == "vector": params = VECTOR_METHOD_PARAMS else: params = ALL_PARAMS - + # Group by category by_category = {} for param in params: if param.category not in by_category: by_category[param.category] = [] by_category[param.category].append(param) - + # Print for category in ["common", "general", "vector", "strategy", "reranking"]: if category not in by_category: continue - + print(f"\n{category.upper()} Parameters:") print("=" * 60) - + for param in by_category[category]: opt_marker = " [optimizable]" if param.optuna_suggest else "" print(f" {param.name}{opt_marker}") print(f" CLI: {', '.join(param.arg_names)}") print(f" Default: {param.default}") - print(f" Help: {param.help[:80]}..." if len(param.help) > 80 else f" Help: {param.help}") + print(f" Help: {param.help[:80]}..." if len( + param.help) > 80 else f" Help: {param.help}") if param.choices: print(f" Choices: {param.choices}") @@ -746,9 +768,10 @@ def print_param_info(method: Optional[str] = None): # Main (for testing/documentation) # ============================================================================ + if __name__ == "__main__": import sys - + if len(sys.argv) > 1 and sys.argv[1] == "list": method = sys.argv[2] if len(sys.argv) > 2 else None print_param_info(method) diff --git a/e2e-rag/perf_test_cache.py b/e2e-rag/perf_test_cache.py index 09be338e8f..0a29451071 100644 --- a/e2e-rag/perf_test_cache.py +++ b/e2e-rag/perf_test_cache.py @@ -89,7 +89,8 @@ def _build_index(self): print(f" Built cache index with {len(self.cache)} LLM responses") - def get_response(self, query_id: str, component: str, hop_count: int) -> Optional[str]: + def get_response(self, query_id: str, component: str, + hop_count: int) -> Optional[str]: """ Retrieve cached LLM response. @@ -104,7 +105,8 @@ def get_response(self, query_id: str, component: str, hop_count: int) -> Optiona key = (str(query_id), component, hop_count) return self.cache.get(key) - def has_response(self, query_id: str, component: str, hop_count: int) -> bool: + def has_response(self, query_id: str, component: str, + hop_count: int) -> bool: """ Check if cached response exists. diff --git a/e2e-rag/read_docs.py b/e2e-rag/read_docs.py index 12ba80272f..9859e6dd1a 100644 --- a/e2e-rag/read_docs.py +++ b/e2e-rag/read_docs.py @@ -47,12 +47,12 @@ class BaseDocumentExtractor(ABC): """Base class for document text extractors.""" - + @abstractmethod def extract_text(self, file_path: str) -> Optional[str]: """Extract text from a document file.""" pass - + @abstractmethod def get_supported_extensions(self) -> List[str]: """Return list of supported file extensions.""" @@ -61,7 +61,7 @@ def get_supported_extensions(self) -> List[str]: class PDFExtractor(BaseDocumentExtractor): """Extract text from PDF files using PyMuPDF (fitz).""" - + def extract_text(self, file_path: str) -> Optional[str]: """Extract text from a single PDF file.""" try: @@ -74,81 +74,83 @@ def extract_text(self, file_path: str) -> Optional[str]: doc.close() return "\n".join(extracted_text) - + except Exception as e: print(f"Error processing PDF {file_path}: {e}") return None - + def get_supported_extensions(self) -> List[str]: return ['.pdf'] class HTMLExtractor(BaseDocumentExtractor): """Extract text from HTML files using BeautifulSoup with focus on retrieval quality.""" - - def __init__(self, preserve_tables: bool = True, preserve_lists: bool = True, + + def __init__(self, preserve_tables: bool = True, preserve_lists: bool = True, text_boundary: str = "sentence"): """ Initialize HTML extractor with configurable options. - + Args: preserve_tables: Whether to preserve table structure - preserve_lists: Whether to preserve list structure + preserve_lists: Whether to preserve list structure text_boundary: Text boundary optimization - "sentence" (default), "word", or "none" """ if BeautifulSoup is None: - raise ImportError("BeautifulSoup is required for HTML processing. Install with: pip install beautifulsoup4") - + raise ImportError( + "BeautifulSoup is required for HTML processing. Install with: pip install beautifulsoup4") + self.preserve_tables = preserve_tables self.preserve_lists = preserve_lists self.text_boundary = text_boundary - + if text_boundary not in ["sentence", "word", "none"]: - raise ValueError("text_boundary must be 'sentence', 'word', or 'none'") - + raise ValueError( + "text_boundary must be 'sentence', 'word', or 'none'") + def extract_text(self, file_path: str) -> Optional[str]: """Extract text from a single HTML file.""" try: with open(file_path, 'r', encoding='utf-8') as f: html_content = f.read() - + return self.extract_text_from_html(html_content) - + except Exception as e: print(f"Error processing HTML {file_path}: {e}") return None - + def extract_text_from_html(self, html_content: str) -> str: """Extract clean text optimized for retrieval systems.""" # Use lxml parser for speed (fallback to html.parser if not available) try: soup = BeautifulSoup(html_content, 'lxml') - except: + except BaseException: soup = BeautifulSoup(html_content, 'html.parser') - + # Remove noise elements completely for element in soup(['script', 'style', 'nav', 'header', 'footer']): element.decompose() - + # Remove Wikipedia-specific metadata and navigation self._remove_wikipedia_metadata(soup) - + # Extract main content using priority order main_content = self._find_main_content(soup) - + # Get plain text with sentence separation if main_content: text = main_content.get_text(separator=' ', strip=True) else: text = soup.get_text(separator=' ', strip=True) - + # Clean and normalize the text return self._clean_text(text) - + def _extract_from_element(self, element) -> List[str]: """Extract text from an HTML element, preserving structure.""" text_parts = [] - + if isinstance(element, NavigableString): text = str(element).strip() if text: @@ -186,9 +188,9 @@ def _extract_from_element(self, element) -> List[str]: text = element.get_text().strip() if text: text_parts.append(text) - + return text_parts - + def _extract_table_text(self, table) -> str: """Extract text from a table element.""" rows = [] @@ -200,7 +202,7 @@ def _extract_table_text(self, table) -> str: if cells: rows.append(' | '.join(cells)) return '\n'.join(rows) - + def _extract_list_text(self, list_elem) -> str: """Extract text from a list element.""" items = [] @@ -210,75 +212,79 @@ def _extract_list_text(self, list_elem) -> str: prefix = '- ' if list_elem.name == 'ul' else f"{len(items) + 1}. " items.append(f"{prefix}{item_text}") return '\n'.join(items) - + def _clean_text(self, text: str) -> str: """Clean and normalize extracted text with configurable boundary optimization.""" # Basic normalization import unicodedata text = unicodedata.normalize('NFKC', text) - + # Replace various whitespace characters with standard space text = re.sub(r'[\u00A0\u2000-\u200B\u2028\u2029]', ' ', text) - + # Clean up whitespace text = text.strip().replace('\r', '\n') text = re.sub(r' +', ' ', text) # Multiple spaces -> single space - text = re.sub(r'\n+', '\n', text) # Multiple newlines -> single newline - + # Multiple newlines -> single newline + text = re.sub(r'\n+', '\n', text) + # Remove empty lines and extra spacing lines = [line.strip() for line in text.split('\n') if line.strip()] text = '\n'.join(lines) - + # Apply boundary optimization based on setting if self.text_boundary == "sentence": text = self._optimize_sentence_boundaries(text) elif self.text_boundary == "word": text = self._optimize_word_boundaries(text) # "none" - no boundary optimization - + return text - + def _optimize_sentence_boundaries(self, text: str) -> str: """Optimize text for sentence-level splitting and retrieval.""" # Add space after sentence endings if missing text = re.sub(r'([.!?])([A-Z])', r'\1 \2', text) - + # Handle common abbreviations that shouldn't split sentences # (e.g., "Mr.", "Dr.", "etc.", "U.S.") abbrev_pattern = r'\b(Mr|Mrs|Dr|Prof|etc|vs|Inc|Ltd|Corp|U\.S|U\.K|E\.g|I\.e)\.(\s+)([a-z])' text = re.sub(abbrev_pattern, r'\1.\2\3', text, flags=re.IGNORECASE) - + return text - + def _optimize_word_boundaries(self, text: str) -> str: """Optimize text for word-level processing and retrieval.""" # Ensure proper spacing around punctuation for better tokenization text = re.sub(r'([.!?,:;])([A-Za-z])', r'\1 \2', text) - + # Handle hyphenated words - keep them as single tokens text = re.sub(r'(\w+)-\s+(\w+)', r'\1-\2', text) - + # Normalize quotation marks and other punctuation - text = text.replace('"', '"').replace('"', '"') # Smart quotes to regular quotes - text = text.replace(''', "'").replace(''', "'") # Smart apostrophes to regular apostrophes - + text = text.replace( + '"', '"').replace( + '"', '"') # Smart quotes to regular quotes + text = text.replace(''', "'").replace(''', + "'") # Smart apostrophes to regular apostrophes + # Ensure consistent spacing text = re.sub(r'\s+', ' ', text) - + return text - + def _remove_wikipedia_metadata(self, soup): """Remove Wikipedia-specific metadata and navigation elements.""" # Wikipedia-specific noise removal selectors_to_remove = [ # Navigation and interface elements - '#mw-navigation', '.navbox', '.navigation-box', + '#mw-navigation', '.navbox', '.navigation-box', '.ambox', '.tmbox', # Edit links and metadata '.mw-editsection', '.edit-section', '.editlink', # References and citations (keep text but remove citation numbers) 'sup.reference', '.reference', '.citation', - # Disambiguation and hatnotes + # Disambiguation and hatnotes '.hatnote', '.dablink', '.rellink', # Categories and external links boxes '#catlinks', '.catlinks', '.external-links', @@ -287,11 +293,11 @@ def _remove_wikipedia_metadata(self, soup): # Image captions and metadata (keep main text) '.thumbcaption .metadata', '.image-metadata' ] - + for selector in selectors_to_remove: for element in soup.select(selector): element.decompose() - + def _find_main_content(self, soup): """Find the main content area with fallback strategy.""" # Priority order for content detection @@ -304,31 +310,31 @@ def _find_main_content(self, soup): '#content', # Generic content ID 'body' # Last resort ] - + for selector in content_selectors: content = soup.select_one(selector) if content: return content - + # Final fallback return soup - + def get_supported_extensions(self) -> List[str]: return ['.html', '.htm'] class DocumentProcessor: """Unified document processor that handles both PDF and HTML files.""" - - def __init__(self, preserve_tables: bool = True, preserve_lists: bool = True, + + def __init__(self, preserve_tables: bool = True, preserve_lists: bool = True, text_boundary: str = "sentence", benchmark: bool = False, processes: int = 4): """ Initialize document processor. - + Args: preserve_tables: Whether to preserve table structure (HTML only) - preserve_lists: Whether to preserve list structure (HTML only) + preserve_lists: Whether to preserve list structure (HTML only) text_boundary: Text boundary optimization - "sentence" (default), "word", or "none" benchmark: Enable performance monitoring processes: Number of parallel processes for document processing @@ -337,62 +343,64 @@ def __init__(self, preserve_tables: bool = True, preserve_lists: bool = True, self.extractors = { '.pdf': PDFExtractor(), } - + # Only add HTML extractor if BeautifulSoup is available if BeautifulSoup is not None: self.extractors.update({ '.html': HTMLExtractor(preserve_tables, preserve_lists, text_boundary), '.htm': HTMLExtractor(preserve_tables, preserve_lists, text_boundary), }) - + self.url_mapping = {} self.benchmark = benchmark self.monitor = None - + # Store config for worker processes self.preserve_tables = preserve_tables self.preserve_lists = preserve_lists self.text_boundary = text_boundary - + # Initialize monitoring if benchmark mode enabled if self.benchmark: from ingestion_monitor import IngestionMonitor self.monitor = IngestionMonitor() - + def get_supported_extensions(self) -> List[str]: """Get all supported file extensions.""" extensions = [] for extractor in self.extractors.values(): extensions.extend(extractor.get_supported_extensions()) return list(set(extensions)) - + @staticmethod - def process_single_file(args_tuple: Tuple) -> Optional[Tuple[str, str, List[str], str]]: + def process_single_file( + args_tuple: Tuple) -> Optional[Tuple[str, str, List[str], str]]: """ Process a single document file (worker function for multiprocessing). - + Args: - args_tuple: (doc_file_path, output_dir, url_mapping, preserve_tables, + args_tuple: (doc_file_path, output_dir, url_mapping, preserve_tables, preserve_lists, text_boundary, fixed_length, fixed_overlap, max_passage_length, passage_overlap) - + Returns: Tuple of (output_filename, text, passages, original_url) or None if processing failed """ - (doc_file_path, output_dir, url_mapping, preserve_tables, preserve_lists, + (doc_file_path, output_dir, url_mapping, preserve_tables, preserve_lists, text_boundary, fixed_length, fixed_overlap, max_passage_length, passage_overlap) = args_tuple - + doc_file = Path(doc_file_path) file_extension = doc_file.suffix.lower() - + # Create appropriate extractor if file_extension == '.pdf': extractor = PDFExtractor() elif file_extension in ['.html', '.htm'] and BeautifulSoup is not None: - extractor = HTMLExtractor(preserve_tables, preserve_lists, text_boundary) + extractor = HTMLExtractor( + preserve_tables, preserve_lists, text_boundary) else: return None - + # Extract text try: text = extractor.extract_text(str(doc_file)) @@ -401,33 +409,35 @@ def process_single_file(args_tuple: Tuple) -> Optional[Tuple[str, str, List[str] except Exception as e: print(f"Error extracting text from {doc_file}: {e}") return None - + # Split text into passages try: if fixed_length: - passages = split_into_fixed_passages(text, fixed_length, fixed_overlap or 32) + passages = split_into_fixed_passages( + text, fixed_length, fixed_overlap or 32) else: - passages = split_into_passages(text, max_passage_length, passage_overlap) + passages = split_into_passages( + text, max_passage_length, passage_overlap) except Exception as e: print(f"Error splitting text for {doc_file}: {e}") return None - + # Get original URL base_filename = get_base_filename(doc_file.name) original_url = url_mapping.get(base_filename, "") - + # Generate output filename output_filename = doc_file.stem + ".txt" - + return (output_filename, text, passages, original_url, doc_file.name) - + def process_documents(self, input_dir: str, output_dir: str, json_file: Optional[str] = None, - max_passage_length: int = 512, passage_overlap: int = 50, - fixed_length: Optional[int] = None, fixed_overlap: Optional[int] = None, - max_files: Optional[int] = None): + max_passage_length: int = 512, passage_overlap: int = 50, + fixed_length: Optional[int] = None, fixed_overlap: Optional[int] = None, + max_files: Optional[int] = None): """ Process documents in a directory, extracting text and splitting into passages. - + Args: input_dir: Directory containing document files output_dir: Directory to save extracted text files @@ -447,22 +457,23 @@ def process_documents(self, input_dir: str, output_dir: str, json_file: Optional # Find all supported document files supported_extensions = self.get_supported_extensions() document_files = [] - + for ext in supported_extensions: pattern = f"*{ext}" document_files.extend(input_path.glob(pattern)) - + # Sort for consistent processing order document_files = sorted(document_files) - + if not document_files: return - + if max_files: document_files = document_files[:max_files] - - print(f"Processing {len(document_files)} documents with {self.processes} parallel processes...") - + + print( + f"Processing {len(document_files)} documents with {self.processes} parallel processes...") + all_passages = [] passage_id = 0 @@ -472,22 +483,23 @@ def process_documents(self, input_dir: str, output_dir: str, json_file: Optional # Prepare arguments for multiprocessing process_args = [ - (str(doc_file), str(output_path), self.url_mapping, + (str(doc_file), str(output_path), self.url_mapping, self.preserve_tables, self.preserve_lists, self.text_boundary, fixed_length, fixed_overlap, max_passage_length, passage_overlap) for doc_file in document_files ] - + # Process documents in parallel with progress bar with Pool(processes=self.processes) as pool: with tqdm(total=len(document_files), desc="Processing documents") as pbar: - for result in pool.imap(self.process_single_file, process_args): + for result in pool.imap( + self.process_single_file, process_args): if result is None: pbar.update(1) continue - + output_filename, text, passages, original_url, doc_filename = result - + # Save text file output_file_path = output_path / output_filename try: @@ -497,19 +509,19 @@ def process_documents(self, input_dir: str, output_dir: str, json_file: Optional print(f"Error writing {output_file_path}: {e}") pbar.update(1) continue - + # Add passages to collection if JSON output requested if json_file: for passage in passages: passage_metadata = create_passage_metadata( doc_filename, passage_id, original_url=original_url) - + all_passages.append({ **passage_metadata, 'passage': passage, }) passage_id += 1 - + pbar.update(1) # Finalize monitoring and report @@ -538,65 +550,72 @@ def process_documents(self, input_dir: str, output_dir: str, json_file: Optional # Add component-level timing if monitoring was enabled if self.benchmark and self.monitor: for component_name, metrics in self.monitor.components.items(): - result[f'{component_name}_time_seconds'] = round(metrics.duration, 2) - result[f'{component_name}_throughput_mb_per_sec'] = round(metrics.throughput_mb_per_sec, 2) + result[f'{component_name}_time_seconds'] = round( + metrics.duration, 2) + result[f'{component_name}_throughput_mb_per_sec'] = round( + metrics.throughput_mb_per_sec, 2) return result - - def _process_document(self, doc_file: Path, file_extension: str) -> Optional[str]: + + def _process_document(self, doc_file: Path, + file_extension: str) -> Optional[str]: """Process a single document with optional monitoring.""" extractor = self.extractors[file_extension] - + if self.benchmark and self.monitor: - component_name = "html_parsing" if file_extension in ['.html', '.htm'] else "pdf_parsing" + component_name = "html_parsing" if file_extension in [ + '.html', '.htm'] else "pdf_parsing" file_size = doc_file.stat().st_size - with self.monitor.track_component(component_name, input_size_bytes=file_size, - items_count=1, is_pipeline_input=True): + with self.monitor.track_component(component_name, input_size_bytes=file_size, + items_count=1, is_pipeline_input=True): return extractor.extract_text(str(doc_file)) else: return extractor.extract_text(str(doc_file)) - + def _process_text_chunking(self, text: str, fixed_length: Optional[int], fixed_overlap: Optional[int], - max_passage_length: int, passage_overlap: int) -> List[str]: + max_passage_length: int, passage_overlap: int) -> List[str]: """Process text chunking with optional monitoring.""" def chunk_func(): if fixed_length: - return split_into_fixed_passages(text, fixed_length, fixed_overlap or 32) + return split_into_fixed_passages( + text, fixed_length, fixed_overlap or 32) else: - return split_into_passages(text, max_passage_length, passage_overlap) - + return split_into_passages( + text, max_passage_length, passage_overlap) + if self.benchmark and self.monitor: text_size = len(text.encode('utf-8')) - with self.monitor.track_component("text_chunking", input_size_bytes=text_size, - items_count=1, text_only=True) as ctx: + with self.monitor.track_component("text_chunking", input_size_bytes=text_size, + items_count=1, text_only=True) as ctx: passages = chunk_func() ctx.add_text_bytes(text_size) return passages else: return chunk_func() - - def _add_passages_to_collection(self, doc_file: Path, passages: List[str], - all_passages: List[Dict], passage_id: int) -> int: + + def _add_passages_to_collection(self, doc_file: Path, passages: List[str], + all_passages: List[Dict], passage_id: int) -> int: """Add passages to collection and return updated passage_id.""" base_filename = get_base_filename(doc_file.name) original_url = self.url_mapping.get(base_filename, "") - + for passage in passages: passage_metadata = create_passage_metadata( doc_file.name, passage_id, original_url=original_url) - + all_passages.append({ **passage_metadata, 'passage': passage, }) passage_id += 1 - + return passage_id - + def _report_processing_performance(self): """Report processing performance if monitoring is enabled.""" # Suppress the detailed benchmark summary - metrics are tracked internally - # and will be reported by the calling script (measure_indexing_with_chunking.py) + # and will be reported by the calling script + # (measure_indexing_with_chunking.py) pass @@ -605,31 +624,33 @@ def main(): description="Extract text from PDF and HTML files and split into passages", formatter_class=argparse.RawTextHelpFormatter ) - - parser.add_argument("input_dir", help="Directory containing document files (PDF/HTML)") + + parser.add_argument( + "input_dir", + help="Directory containing document files (PDF/HTML)") parser.add_argument("output_dir", help="Directory to save text files") parser.add_argument("--json", help="JSON file to save passage data") - parser.add_argument("--max-length", type=int, default=512, - help="Maximum passage length in characters (default: 512)") + parser.add_argument("--max-length", type=int, default=512, + help="Maximum passage length in characters (default: 512)") parser.add_argument("--overlap", type=int, default=50, - help="Overlap between passages in characters (default: 50)") + help="Overlap between passages in characters (default: 50)") parser.add_argument("--fixed-length", type=int, - help="Use fixed-length passages instead of variable-length") + help="Use fixed-length passages instead of variable-length") parser.add_argument("--fixed-overlap", type=int, default=32, - help="Overlap for fixed-length passages (default: 32)") + help="Overlap for fixed-length passages (default: 32)") parser.add_argument("--max-files", type=int, - help="Maximum number of files to process (for testing)") + help="Maximum number of files to process (for testing)") parser.add_argument("--no-tables", action="store_true", - help="Don't preserve table structure (HTML only)") - parser.add_argument("--no-lists", action="store_true", - help="Don't preserve list structure (HTML only)") - parser.add_argument("--text-boundary", choices=["sentence", "word", "none"], - default="sentence", - help="Text boundary optimization: 'sentence' (default), 'word', or 'none'") + help="Don't preserve table structure (HTML only)") + parser.add_argument("--no-lists", action="store_true", + help="Don't preserve list structure (HTML only)") + parser.add_argument("--text-boundary", choices=["sentence", "word", "none"], + default="sentence", + help="Text boundary optimization: 'sentence' (default), 'word', or 'none'") parser.add_argument("--processes", type=int, default=4, - help="Number of parallel processes for document processing (default: 4)") + help="Number of parallel processes for document processing (default: 4)") parser.add_argument("--benchmark", action="store_true", - help="Enable performance monitoring and detailed component analysis") + help="Enable performance monitoring and detailed component analysis") args = parser.parse_args() @@ -640,7 +661,7 @@ def main(): benchmark=args.benchmark, processes=args.processes ) - + processor.process_documents( input_dir=args.input_dir, output_dir=args.output_dir, diff --git a/e2e-rag/reference_SUT.py b/e2e-rag/reference_SUT.py index d1e3c5c723..9bccb2e6bd 100644 --- a/e2e-rag/reference_SUT.py +++ b/e2e-rag/reference_SUT.py @@ -112,8 +112,10 @@ def __init__( # Performance test mode if hasattr(args, 'perf_test_mode') and args.perf_test_mode: from perf_test_cache import PerfTestCache - log.info(f"Loading performance test cache from {args.perf_test_mode}") - self.llm_config['perf_test_cache'] = PerfTestCache(args.perf_test_mode) + log.info( + f"Loading performance test cache from {args.perf_test_mode}") + self.llm_config['perf_test_cache'] = PerfTestCache( + args.perf_test_mode) else: self.llm_config['perf_test_cache'] = None @@ -130,11 +132,14 @@ def __init__( # Initialize database log.info("Initializing RAG database...") self.rag_db = VectorDB( - retriever_model=args.retriever_model if hasattr(args, 'retriever_model') else 'BAAI/bge-base-en-v1.5', - reranker_model=args.reranker_model if hasattr(args, 'reranker_model') else 'BAAI/bge-reranker-base', + retriever_model=args.retriever_model if hasattr( + args, 'retriever_model') else 'BAAI/bge-base-en-v1.5', + reranker_model=args.reranker_model if hasattr( + args, 'reranker_model') else 'BAAI/bge-reranker-base', device=device, database=db_path.replace('.db', ''), - num_embedding_devices=args.num_embedding_devices if hasattr(args, 'num_embedding_devices') else 1, + num_embedding_devices=args.num_embedding_devices if hasattr( + args, 'num_embedding_devices') else 1, benchmark=args.benchmark if hasattr(args, 'benchmark') else False ) diff --git a/e2e-rag/reference_SUT_datasetup.py b/e2e-rag/reference_SUT_datasetup.py index d23d1d7802..62e81f0f8d 100644 --- a/e2e-rag/reference_SUT_datasetup.py +++ b/e2e-rag/reference_SUT_datasetup.py @@ -117,7 +117,8 @@ def __init__( # Initialize HTML extractor if not HAVE_HTML: - raise RuntimeError("BeautifulSoup required for HTML processing. Install with: pip install beautifulsoup4") + raise RuntimeError( + "BeautifulSoup required for HTML processing. Install with: pip install beautifulsoup4") log.info("Initializing HTML extractor...") self.html_extractor = HTMLExtractor( @@ -206,7 +207,8 @@ def _process_document(self, query_sample): file_path = document_info['file_path'] file_name = document_info['file_name'] - log.info(f"Processing document {sample_id} (QID: {query_id}): {file_name}") + log.info( + f"Processing document {sample_id} (QID: {query_id}): {file_name}") start_time = time.time() success = 0 # 0 = failure, 1 = success @@ -218,7 +220,8 @@ def _process_document(self, query_sample): if not text or len(text.strip()) == 0: # Empty file - create minimal passage with file name - log.warning(f"No text extracted from {file_name}, creating minimal passage") + log.warning( + f"No text extracted from {file_name}, creating minimal passage") text = f"Document: {file_name}" # Continue with text (even if minimal) @@ -232,22 +235,28 @@ def _process_document(self, query_sample): if not passages: # Even after splitting, no passages - this shouldn't happen now # but create one minimal passage as fallback - log.warning(f"No passages created from {file_name}, using text as-is") + log.warning( + f"No passages created from {file_name}, using text as-is") passages = [text] # Step 3: Generate embeddings (parallel, outside lock) - passage_metadata = [{'source': file_name, 'passage_id': i} for i in range(len(passages))] + passage_metadata = [{'source': file_name, 'passage_id': i} + for i in range(len(passages))] - # Generate embeddings WITHOUT holding db_lock (allows parallel embedding generation) - log.info(f"Generating embeddings for {len(passages)} passages from {file_name}") + # Generate embeddings WITHOUT holding db_lock (allows parallel + # embedding generation) + log.info( + f"Generating embeddings for {len(passages)} passages from {file_name}") if self.rag_db._num_embedding_devices > 1: embeddings = self.rag_db._embed_documents_parallel(passages) else: - embeddings = self.rag_db._embedding_model.embed_documents(passages) + embeddings = self.rag_db._embedding_model.embed_documents( + passages) log.info(f"Embeddings generated for {file_name}, adding to index") - # Step 4: Add to index (thread-safe, holds lock only for index update) + # Step 4: Add to index (thread-safe, holds lock only for index + # update) with self.db_lock: # Add embeddings and documents to vector store ids = self.rag_db._vector_store.add_embeddings( @@ -260,7 +269,8 @@ def _process_document(self, query_sample): self.total_passages_indexed += len(passages) success = 1 - log.info(f"Successfully indexed {len(passages)} passages from {file_name}") + log.info( + f"Successfully indexed {len(passages)} passages from {file_name}") # Check if this is the last file to complete with self.completion_lock: @@ -268,7 +278,8 @@ def _process_document(self, query_sample): is_last_file = (self.completed_count == len(self.qsl)) if is_last_file: - log.info(f"Last file completed! Saving database and computing MD5...") + log.info( + f"Last file completed! Saving database and computing MD5...") # Save database db_path = f"{self.database}.db" save_start = time.time() @@ -287,7 +298,8 @@ def _process_document(self, query_sample): self.db_md5 = md5_hash.hexdigest() md5_end = time.time() md5_duration = md5_end - md5_start - log.info(f"Database MD5: {self.db_md5} (computed in {md5_duration:.2f}s)") + log.info( + f"Database MD5: {self.db_md5} (computed in {md5_duration:.2f}s)") self.db_saved = True @@ -317,7 +329,8 @@ def _process_document(self, query_sample): } # Create response for loadgen - # For the last file, include MD5 hash; otherwise just success/failure byte + # For the last file, include MD5 hash; otherwise just success/failure + # byte if success and self.db_saved: # Last file: return MD5 hash as response response_bytes = self.db_md5.encode('utf-8') @@ -339,7 +352,8 @@ def _process_document(self, query_sample): ) lg.QuerySamplesComplete([response]) - log.info(f"Completed document {sample_id} (QID: {query_id}): {file_name}") + log.info( + f"Completed document {sample_id} (QID: {query_id}): {file_name}") def flush_queries(self): """ @@ -349,10 +363,12 @@ def flush_queries(self): log.info("Flushing queries...") if self.db_saved: - log.info(f"Database already saved by last file. MD5: {self.db_md5}") + log.info( + f"Database already saved by last file. MD5: {self.db_md5}") else: # Fallback: save database if somehow not done yet - log.warning("Database not saved by last file - saving now as fallback") + log.warning( + "Database not saved by last file - saving now as fallback") db_path = f"{self.database}.db" save_start = time.time() self.rag_db.serialize(db_path) @@ -379,13 +395,15 @@ def finalize(self): failed_count = len(self.failed_documents) success_count = total_docs - failed_count - avg_time_per_doc = sum(self.processing_times) / total_docs if total_docs > 0 else 0 - throughput_passages = self.total_passages_indexed / total_time if total_time > 0 else 0 + avg_time_per_doc = sum( + self.processing_times) / total_docs if total_docs > 0 else 0 + throughput_passages = self.total_passages_indexed / \ + total_time if total_time > 0 else 0 throughput_docs = total_docs / total_time if total_time > 0 else 0 - log.info("="*80) + log.info("=" * 80) log.info("Datasetup Complete") - log.info("="*80) + log.info("=" * 80) log.info(f"Total documents processed: {total_docs}") log.info(f"Successful: {success_count}") log.info(f"Failed: {failed_count}") @@ -394,10 +412,11 @@ def finalize(self): log.info(f"Throughput: {throughput_passages:.2f} passages/sec") log.info(f"Throughput: {throughput_docs:.2f} docs/sec") log.info(f"Average time per document: {avg_time_per_doc:.2f}s") - log.info("="*80) + log.info("=" * 80) # Cleanup reranker queue if it exists - if hasattr(self.rag_db, '_reranker_queue') and self.rag_db._reranker_queue is not None: + if hasattr( + self.rag_db, '_reranker_queue') and self.rag_db._reranker_queue is not None: log.info("Shutting down reranker queue...") self.rag_db._reranker_queue.stop() @@ -419,12 +438,14 @@ def save_results(self, output_path): failed_count = len(self.failed_documents) success_count = total_docs - failed_count - throughput_passages = self.total_passages_indexed / total_time if total_time > 0 else 0 + throughput_passages = self.total_passages_indexed / \ + total_time if total_time > 0 else 0 throughput_docs = total_docs / total_time if total_time > 0 else 0 # Get vector count from database vector_count = 0 - if hasattr(self.rag_db, '_vector_store') and hasattr(self.rag_db._vector_store, 'index'): + if hasattr(self.rag_db, '_vector_store') and hasattr( + self.rag_db._vector_store, 'index'): vector_count = self.rag_db._vector_store.index.ntotal output = { diff --git a/e2e-rag/reference_mlperf.py b/e2e-rag/reference_mlperf.py index 83ebc0fcd0..2b306c5742 100644 --- a/e2e-rag/reference_mlperf.py +++ b/e2e-rag/reference_mlperf.py @@ -90,7 +90,8 @@ def get_args(): help="Number of queries for performance testing (None = all)" ) - # Multi-shot specific parameters (these are unique to multi_shot_retrieval.py) + # Multi-shot specific parameters (these are unique to + # multi_shot_retrieval.py) parser.add_argument( '--max-sub-queries', type=int, @@ -171,9 +172,9 @@ def main(): os.makedirs(args.output_dir, exist_ok=True) # Initialize SUT - print("\n" + "="*80) + print("\n" + "=" * 80) print("Initializing RAG-QnA SUT...") - print("="*80) + print("=" * 80) sut = E2ESUT( dataset_path=args.dataset_path, @@ -194,9 +195,9 @@ def main(): args=args, # Pass full args for additional params ) - print("\n" + "="*80) + print("\n" + "=" * 80) print("SUT initialization complete") - print("="*80 + "\n") + print("=" * 80 + "\n") # Configure loadgen settings settings = lg.TestSettings() @@ -227,9 +228,9 @@ def main(): log_settings.log_output = log_output_settings # Run loadgen test - print("\n" + "="*80) + print("\n" + "=" * 80) print("Running MLPerf Loadgen test...") - print("="*80 + "\n") + print("=" * 80 + "\n") lg.StartTestWithLogSettings( sut.sut, @@ -239,9 +240,9 @@ def main(): args.audit_conf ) - print("\n" + "="*80) + print("\n" + "=" * 80) print("Loadgen test complete") - print("="*80 + "\n") + print("=" * 80 + "\n") # Finalize SUT (save logs, cleanup) sut.finalize() @@ -253,9 +254,9 @@ def main(): # Run accuracy evaluation if in accuracy mode if args.accuracy: - print("\n" + "="*80) + print("\n" + "=" * 80) print("Running accuracy evaluation...") - print("="*80 + "\n") + print("=" * 80 + "\n") cmd = [ "python3", @@ -269,9 +270,9 @@ def main(): print(f"Command: {' '.join(cmd)}") subprocess.check_call(cmd) - print("\n" + "="*80) + print("\n" + "=" * 80) print("Done!") - print("="*80) + print("=" * 80) if __name__ == "__main__": diff --git a/e2e-rag/reference_mlperf_datasetup.py b/e2e-rag/reference_mlperf_datasetup.py index 5ec27d1232..647cd8c97b 100644 --- a/e2e-rag/reference_mlperf_datasetup.py +++ b/e2e-rag/reference_mlperf_datasetup.py @@ -171,9 +171,9 @@ def main(): os.makedirs(args.output_dir, exist_ok=True) # Initialize SUT - print("\n" + "="*80) + print("\n" + "=" * 80) print("Initializing RAG-DB SUT...") - print("="*80) + print("=" * 80) sut = DatasetupSUT( documents_dir=args.documents_dir, @@ -192,9 +192,9 @@ def main(): args=args, ) - print("\n" + "="*80) + print("\n" + "=" * 80) print("SUT initialization complete") - print("="*80 + "\n") + print("=" * 80 + "\n") # Configure loadgen settings settings = lg.TestSettings() @@ -225,9 +225,9 @@ def main(): log_settings.log_output = log_output_settings # Run loadgen test - print("\n" + "="*80) + print("\n" + "=" * 80) print("Running MLPerf Loadgen test...") - print("="*80 + "\n") + print("=" * 80 + "\n") lg.StartTestWithLogSettings( sut.sut, @@ -237,9 +237,9 @@ def main(): args.audit_conf ) - print("\n" + "="*80) + print("\n" + "=" * 80) print("Loadgen test complete") - print("="*80 + "\n") + print("=" * 80 + "\n") # Finalize SUT (batch index, save database, cleanup) sut.finalize() @@ -249,9 +249,9 @@ def main(): sut.save_results(results_path) print(f"Results saved to {results_path}") - print("\n" + "="*80) + print("\n" + "=" * 80) print("Done!") - print("="*80) + print("=" * 80) if __name__ == "__main__": diff --git a/e2e-rag/reranker_worker.py b/e2e-rag/reranker_worker.py index e3acfea045..fc0b568a82 100644 --- a/e2e-rag/reranker_worker.py +++ b/e2e-rag/reranker_worker.py @@ -100,7 +100,8 @@ def _reranker_worker_main( response_q.put((request_id, None, repr(e))) -def _do_rerank(model, tokenizer, device: str, query: str, passages: List[str]) -> List[Tuple[str, float]]: +def _do_rerank(model, tokenizer, device: str, query: str, + passages: List[str]) -> List[Tuple[str, float]]: """ColBERT late-interaction reranking with MaxSim scoring.""" import torch @@ -180,10 +181,12 @@ def start(self): self._process.start() if not self._ready_event.wait(timeout=300): - raise RuntimeError("reranker child failed to become ready within 300s") + raise RuntimeError( + "reranker child failed to become ready within 300s") self._dispatcher_running = True - self._dispatcher_thread = threading.Thread(target=self._dispatcher_loop, daemon=True) + self._dispatcher_thread = threading.Thread( + target=self._dispatcher_loop, daemon=True) self._dispatcher_thread.start() def stop(self): @@ -215,12 +218,14 @@ def _dispatcher_loop(self): continue event, container = slot if err is not None: - container["error"] = RuntimeError(f"reranker child error: {err}") + container["error"] = RuntimeError( + f"reranker child error: {err}") else: container["result"] = result event.set() - def submit(self, query: str, passages: List[str]) -> List[Tuple[str, float]]: + def submit(self, query: str, + passages: List[str]) -> List[Tuple[str, float]]: if self._process is None or not self._process.is_alive(): raise RuntimeError("reranker process is not running") diff --git a/e2e-rag/retrieve/__init__.py b/e2e-rag/retrieve/__init__.py index b14c776219..2e5187bee9 100644 --- a/e2e-rag/retrieve/__init__.py +++ b/e2e-rag/retrieve/__init__.py @@ -22,4 +22,4 @@ from .vectordb import VectorDB from .filter import filter, get_score_statistics -__all__ = ['RagDB', 'VectorDB', 'filter', 'get_score_statistics'] \ No newline at end of file +__all__ = ['RagDB', 'VectorDB', 'filter', 'get_score_statistics'] diff --git a/e2e-rag/retrieve/filter.py b/e2e-rag/retrieve/filter.py index 710977dac5..aa7e07b4d9 100644 --- a/e2e-rag/retrieve/filter.py +++ b/e2e-rag/retrieve/filter.py @@ -24,7 +24,7 @@ Implements various thresholding approaches: - Top-p (nucleus sampling) - popular in NLP - Score threshold - absolute quality bar -- Relative threshold - adaptive to query difficulty +- Relative threshold - adaptive to query difficulty - Elbow method - natural breakpoints - Percentile-based - statistical cutoffs @@ -40,14 +40,14 @@ def softmax(scores: List[float], temperature: float = 1.0) -> List[float]: """Convert scores to probabilities using softmax with temperature scaling. - + Args: scores: List of scores to convert temperature: Temperature parameter (lower = sharper distribution) - temperature = 1.0: standard softmax - temperature < 1.0: sharper (more weight on top scores) - temperature > 1.0: smoother (more uniform) - + Returns: List of probabilities that sum to 1.0 """ @@ -58,95 +58,98 @@ def softmax(scores: List[float], temperature: float = 1.0) -> List[float]: return [exp_s / sum_exp for exp_s in exp_scores] -def top_p_filter(results_with_scores: List[Tuple[Any, float]], p: float = 0.9) -> List[Any]: +def top_p_filter( + results_with_scores: List[Tuple[Any, float]], p: float = 0.9) -> List[Any]: """ Top-p (nucleus) sampling: Take results until cumulative probability >= p - + Args: results_with_scores: List of (result, score) tuples, sorted by score DESC p: Cumulative probability threshold (0.8-0.95 typical) - + Returns: Filtered results list """ if not results_with_scores: return [] - + scores = [score for _, score in results_with_scores] - - #print(f"\n[DEBUG top_p_filter] p={p}, num_candidates={len(scores)}") - #print(f"[DEBUG] Score range: [{min(scores):.4f}, {max(scores):.4f}]") - #print(f"[DEBUG] Score mean: {sum(scores)/len(scores):.4f}") - #print(f"[DEBUG] First 10 scores: {[f'{s:.4f}' for s in scores[:10]]}") - + + # print(f"\n[DEBUG top_p_filter] p={p}, num_candidates={len(scores)}") + # print(f"[DEBUG] Score range: [{min(scores):.4f}, {max(scores):.4f}]") + # print(f"[DEBUG] Score mean: {sum(scores)/len(scores):.4f}") + # print(f"[DEBUG] First 10 scores: {[f'{s:.4f}' for s in scores[:10]]}") + # Use temperature scaling to sharpen the distribution # Lower temperature = more discriminative (top docs get higher probability) # For vector embeddings with compressed L2 distances, use very low temperature # Temperature = 0.01 to 0.05 for L2 distances in range [0.27-0.42] temperature = 1 - #temperature = 0.02 + # temperature = 0.02 probs = softmax(scores, temperature=temperature) - - #print(f"[DEBUG] Temperature: {temperature}") - #print(f"[DEBUG] Probability range: [{min(probs):.6f}, {max(probs):.6f}]") - #print(f"[DEBUG] First 10 probs: {[f'{p:.6f}' for p in probs[:10]]}") - #print(f"[DEBUG] Prob sum: {sum(probs):.6f}") - + + # print(f"[DEBUG] Temperature: {temperature}") + # print(f"[DEBUG] Probability range: [{min(probs):.6f}, {max(probs):.6f}]") + # print(f"[DEBUG] First 10 probs: {[f'{p:.6f}' for p in probs[:10]]}") + # print(f"[DEBUG] Prob sum: {sum(probs):.6f}") + cumulative_prob = 0.0 selected_results = [] - - for i, ((result, score), prob) in enumerate(zip(results_with_scores, probs)): + + for i, ((result, score), prob) in enumerate( + zip(results_with_scores, probs)): cumulative_prob += prob selected_results.append(result) - + if cumulative_prob >= p: - print(f"[DEBUG] Selected {i+1} documents (cumulative_prob={cumulative_prob:.4f} >= p={p})") + print( + f"[DEBUG] Selected {i+1} documents (cumulative_prob={cumulative_prob:.4f} >= p={p})") break - + return selected_results -def score_threshold_filter(results_with_scores: List[Tuple[Any, float]], - threshold: float, higher_better: bool = True) -> List[Any]: +def score_threshold_filter(results_with_scores: List[Tuple[Any, float]], + threshold: float, higher_better: bool = True) -> List[Any]: """ Absolute score threshold filtering. - + Args: results_with_scores: List of (result, score) tuples threshold: Absolute score cutoff higher_better: If True, keep scores >= threshold, else <= threshold - + Returns: Filtered results list """ selected_results = [] - + for result, score in results_with_scores: if higher_better and score >= threshold: selected_results.append(result) elif not higher_better and score <= threshold: selected_results.append(result) - + return selected_results -def relative_threshold_filter(results_with_scores: List[Tuple[Any, float]], - ratio: float = 0.8) -> List[Any]: +def relative_threshold_filter(results_with_scores: List[Tuple[Any, float]], + ratio: float = 0.8) -> List[Any]: """ Relative threshold: Keep top ratio fraction of results based on score range. - + For both positive and negative scores: - Calculates score range between best and worst - Keeps only results within top ratio% of that range - + Args: results_with_scores: List of (result, score) tuples (sorted desc, best first) ratio: Fraction of score range to keep (0.7-0.9 typical) e.g., 0.9 means keep top 90% of score range - + Returns: Filtered results list - + Example with negative scores: Scores: [-0.42, -0.43, -0.44, ..., -0.49] best=-0.42, worst=-0.49, range=0.07 @@ -156,85 +159,88 @@ def relative_threshold_filter(results_with_scores: List[Tuple[Any, float]], """ if not results_with_scores: return [] - + # Get best and worst scores best_score = results_with_scores[0][1] worst_score = results_with_scores[-1][1] - + # Calculate the score range score_range = best_score - worst_score # Always positive since sorted desc - + # Calculate threshold: start from best, move down by (1-ratio) of range cutoff_distance = score_range * (1 - ratio) threshold = best_score - cutoff_distance - - return score_threshold_filter(results_with_scores, threshold, higher_better=True) + return score_threshold_filter( + results_with_scores, threshold, higher_better=True) -def elbow_method_filter(results_with_scores: List[Tuple[Any, float]]) -> List[Any]: + +def elbow_method_filter( + results_with_scores: List[Tuple[Any, float]]) -> List[Any]: """ Elbow method: Find largest score gap and cut there. - + Args: results_with_scores: List of (result, score) tuples, sorted by score DESC - + Returns: Filtered results list """ if len(results_with_scores) <= 1: return [result for result, _ in results_with_scores] - + scores = [score for _, score in results_with_scores] - + # Calculate gaps between consecutive scores gaps = [] for i in range(len(scores) - 1): gap = scores[i] - scores[i + 1] # Assuming DESC order gaps.append(gap) - + # Find largest gap if not gaps: return [result for result, _ in results_with_scores] - + max_gap_idx = gaps.index(max(gaps)) cutoff_point = max_gap_idx + 1 # Include the score before the gap - + return [result for result, _ in results_with_scores[:cutoff_point]] -def percentile_filter(results_with_scores: List[Tuple[Any, float]], - percentile: float = 90.0) -> List[Any]: +def percentile_filter(results_with_scores: List[Tuple[Any, float]], + percentile: float = 90.0) -> List[Any]: """ Percentile-based filtering: Keep top X percentile of scores. - + Args: results_with_scores: List of (result, score) tuples percentile: Percentile threshold (80-95 typical) - + Returns: Filtered results list """ if not results_with_scores: return [] - + scores = [score for _, score in results_with_scores] threshold = np.percentile(scores, percentile) - - return score_threshold_filter(results_with_scores, threshold, higher_better=True) + + return score_threshold_filter( + results_with_scores, threshold, higher_better=True) -def filter(rag_db, query: str, method: str = "top_p", +def filter(rag_db, query: str, method: str = "top_p", max_results: int = 100, **kwargs) -> List[Any]: """ Perform adaptive retrieval using score-based filtering. - + Args: rag_db: RAG database instance (VectorDB) query: Search query method: Filtering method ("top_p", "score_threshold", "relative", "elbow", "percentile") max_results: Maximum results to retrieve initially **kwargs: Method-specific parameters - + Returns: Filtered results list """ @@ -242,32 +248,33 @@ def filter(rag_db, query: str, method: str = "top_p", if hasattr(rag_db, 'lookup_with_scores'): results_with_scores = rag_db.lookup_with_scores(query, k=max_results) else: - raise ValueError(f"Database {type(rag_db)} doesn't support score-based retrieval") - + raise ValueError( + f"Database {type(rag_db)} doesn't support score-based retrieval") + # Results already have proper similarity scores (higher is better) from lookup_with_scores # Sort by score (descending - higher is better) results_with_scores.sort(key=lambda x: x[1], reverse=True) - + # Apply filtering method if method == "top_p": p = kwargs.get("p", 0.9) return top_p_filter(results_with_scores, p) - + elif method == "score_threshold": threshold = kwargs.get("threshold", 5.0) return score_threshold_filter(results_with_scores, threshold) - + elif method == "relative": ratio = kwargs.get("ratio", 0.8) return relative_threshold_filter(results_with_scores, ratio) - + elif method == "elbow": return elbow_method_filter(results_with_scores) - + elif method == "percentile": percentile = kwargs.get("percentile", 90.0) return percentile_filter(results_with_scores, percentile) - + else: raise ValueError(f"Unknown filtering method: {method}") @@ -276,10 +283,10 @@ def get_score_statistics(rag_db, query: str, k: int = 100) -> Dict[str, float]: """Get score distribution statistics for threshold calibration.""" results_with_scores = rag_db.lookup_with_scores(query, k=k) scores = [score for _, score in results_with_scores] - + if not scores: return {} - + return { "min": min(scores), "max": max(scores), @@ -294,4 +301,4 @@ def get_score_statistics(rag_db, query: str, k: int = 100) -> Dict[str, float]: # Backward compatibility alias -adaptive_retrieval = filter \ No newline at end of file +adaptive_retrieval = filter diff --git a/e2e-rag/retrieve/ragdb.py b/e2e-rag/retrieve/ragdb.py index 19f65c4416..e4c7e1ba79 100644 --- a/e2e-rag/retrieve/ragdb.py +++ b/e2e-rag/retrieve/ragdb.py @@ -18,15 +18,17 @@ import os from typing import List, Dict, Any + class RagDB(abc.ABC): """Base class for retrieval-augmented generation databases.""" - + def __init__(self, reranker_model: str = None, device: str = "auto", benchmark: bool = False, reranker_device: str = None): self._reranker_model_name = reranker_model self._device = self._determine_device(device) # Reranker device defaults to inheriting from --device. - self._reranker_device = self._determine_device(reranker_device) if reranker_device else self._device + self._reranker_device = self._determine_device( + reranker_device) if reranker_device else self._device self._reranker_queue = None self._benchmark = benchmark self._monitor = None @@ -39,12 +41,12 @@ def __init__(self, reranker_model: str = None, device: str = "auto", # Initialize out-of-process reranker if specified if self._reranker_model_name: self._init_reranker() - + def _determine_device(self, device: str) -> str: """Determine the best device to use. Delegates to utils.detect_device() for auto detection so device-selection - logic lives in one place. ROCm maps to "cuda" + logic lives in one place. ROCm maps to "cuda" """ if device == "rocm": return "cuda" @@ -52,14 +54,14 @@ def _determine_device(self, device: str) -> str: from utils import detect_device return detect_device() return device - + @staticmethod def get_data_dir(db_name: str) -> str: """Get data directory based on database name.""" from pathlib import Path base_name = Path(db_name).stem # Remove .db extension if present return f"{base_name}_data" - + @staticmethod def get_db_path(db_name: str) -> str: """Get database file path based on database name.""" @@ -90,11 +92,11 @@ def _init_reranker(self): omp_threads=int(omp_threads) if omp_threads else None, ) self._reranker_queue.start() - - def _track_component(self, name: str, total_chars: int, item_count: int, func, - is_pipeline_input: bool = False, is_pipeline_output: bool = False): + + def _track_component(self, name: str, total_chars: int, item_count: int, func, + is_pipeline_input: bool = False, is_pipeline_output: bool = False): """Execute function with optional component tracking. - + Args: name: Component name total_chars: Input size in bytes @@ -104,26 +106,27 @@ def _track_component(self, name: str, total_chars: int, item_count: int, func, is_pipeline_output: Mark as pipeline output for aggregation """ if self._benchmark and self._monitor: - with self._monitor.track_component(name, input_size_bytes=total_chars, - items_count=item_count, text_only=True, - is_pipeline_input=is_pipeline_input, - is_pipeline_output=is_pipeline_output) as ctx: + with self._monitor.track_component(name, input_size_bytes=total_chars, + items_count=item_count, text_only=True, + is_pipeline_input=is_pipeline_input, + is_pipeline_output=is_pipeline_output) as ctx: result = func() ctx.add_text_bytes(total_chars) return result else: return func() - + def _start_ingestion_timer(self): """Start the ingestion timer. Works for both benchmark and non-benchmark modes.""" import time if self._benchmark and self._monitor: self._monitor.start_ingestion() return time.perf_counter() - - def _report_performance(self, ingestion_start_time: float, item_count: int, total_chars: int, db_type: str): + + def _report_performance(self, ingestion_start_time: float, + item_count: int, total_chars: int, db_type: str): """Report performance metrics with optional detailed breakdown. - + Args: ingestion_start_time: Start time from _start_ingestion_timer() (used only in non-benchmark mode) item_count: Number of items processed @@ -131,7 +134,7 @@ def _report_performance(self, ingestion_start_time: float, item_count: int, tota db_type: Database type string for display """ import time - + if self._benchmark and self._monitor: with self._monitor.track_ingestion() as ingestion_ctx: ingestion_ctx.set_item_count(item_count) @@ -142,9 +145,11 @@ def _report_performance(self, ingestion_start_time: float, item_count: int, tota duration = end_time - ingestion_start_time docs_per_sec = item_count / duration if duration > 0 else 0 chars_per_sec = total_chars / duration if duration > 0 else 0 - print(f"{db_type} ingestion: {item_count} docs, {total_chars:,} chars in {duration:.2f}s") - print(f" Performance: {docs_per_sec:.1f} docs/sec, {chars_per_sec/1024:.1f} KB/sec") - + print( + f"{db_type} ingestion: {item_count} docs, {total_chars:,} chars in {duration:.2f}s") + print( + f" Performance: {docs_per_sec:.1f} docs/sec, {chars_per_sec/1024:.1f} KB/sec") + def enable_threading(self): """Enable thread-safe access. Override in subclasses that need locks.""" pass @@ -153,26 +158,27 @@ def enable_threading(self): def ingest(self, passages: List[str], metadatas: List[Dict[str, Any]]): """Ingest passages and their metadata into the database.""" pass - + @abc.abstractmethod def lookup(self, query: str, k: int) -> List[Any]: """Retrieve top-k relevant passages for a query.""" pass - + @abc.abstractmethod def serialize(self, path: str): """Serialize the database to disk.""" pass - + @abc.abstractmethod def from_serialized(self, path: str): """Load the database from disk.""" pass - + def ingest_from_folder(self, folder_path: str, **kwargs): """Ingest data from a folder. Default implementation raises NotImplementedError.""" - raise NotImplementedError(f"Folder ingestion not supported for {self.__class__.__name__}") - + raise NotImplementedError( + f"Folder ingestion not supported for {self.__class__.__name__}") + def ingest_from_file(self, file_path: str, **kwargs): """Ingest data from a JSON file. Default implementation for JSON files. @@ -204,35 +210,39 @@ def ingest_from_file(self, file_path: str, **kwargs): # Use child for embedding doc_list.append(entry['child_passage']) # Store all metadata including parent - metadata = {k: v for k, v in entry.items() if k != 'child_passage'} + metadata = {k: v for k, v in entry.items() if k != + 'child_passage'} passage_metadata.append(metadata) else: # Flat format for entry in passage_data: doc_list.append(entry['passage']) - passage_metadata.append({k: v for k, v in entry.items() if k != 'passage'}) + passage_metadata.append( + {k: v for k, v in entry.items() if k != 'passage'}) print(f"Ingesting {len(doc_list)} passages from JSON file {file_path}") - return self.ingest(doc_list, passage_metadata, passages_path=file_path, **kwargs) + return self.ingest(doc_list, passage_metadata, + passages_path=file_path, **kwargs) def ingest_from_path(self, source_path: str, **kwargs): """Handle both file and folder ingestion. - + Default implementation that delegates to appropriate methods: - Folders: calls ingest_from_folder() (may raise NotImplementedError if not overridden) - Files: calls ingest_from_file() (default JSON implementation) """ from pathlib import Path - + source_path = Path(source_path) - + if source_path.is_dir(): print(f"Ingesting documents from folder {source_path}") return self.ingest_from_folder(source_path, **kwargs) elif source_path.is_file(): return self.ingest_from_file(source_path, **kwargs) else: - raise ValueError(f"Source path {source_path} is neither a file nor a directory") + raise ValueError( + f"Source path {source_path} is neither a file nor a directory") def shutdown_reranker(self): """Tear down the reranker child process. Safe to call multiple times.""" @@ -246,7 +256,8 @@ def rerank(self, query: str, passages: List[str]): return self._reranker_queue.submit(query, passages) return [(p, 0.0) for p in passages] - def lookup_with_rerank(self, query: str, k: int, rerank_k: int = None) -> List[Any]: + def lookup_with_rerank(self, query: str, k: int, + rerank_k: int = None) -> List[Any]: """Retrieve and rerank passages.""" if rerank_k is None: rerank_k = k @@ -257,13 +268,13 @@ def lookup_with_rerank(self, query: str, k: int, rerank_k: int = None) -> List[A # If no reranker or fewer results than requested, return as-is if self._reranker_queue is None or len(results) <= k: return results[:k] - + # Extract passages for reranking passages = [result.page_content for result in results] - + # Rerank reranked_passages = self.rerank(query, passages) - + # Map back to original results and return top-k reranked_results = [] for passage, score in reranked_passages[:k]: @@ -271,7 +282,7 @@ def lookup_with_rerank(self, query: str, k: int, rerank_k: int = None) -> List[A if result.page_content == passage: reranked_results.append(result) break - + return reranked_results @property diff --git a/e2e-rag/retrieve/vectordb.py b/e2e-rag/retrieve/vectordb.py index 73499e93d5..d0c392568a 100644 --- a/e2e-rag/retrieve/vectordb.py +++ b/e2e-rag/retrieve/vectordb.py @@ -44,7 +44,11 @@ def _alias_missing_faiss_swigfaiss_modules() -> None: serialized databases loadable across heterogeneous installs without rebuilding them. """ - candidates = ("swigfaiss_avx512_spr", "swigfaiss_avx512", "swigfaiss_avx2", "swigfaiss") + candidates = ( + "swigfaiss_avx512_spr", + "swigfaiss_avx512", + "swigfaiss_avx2", + "swigfaiss") available = None for name in candidates: full = f"faiss.{name}" @@ -66,7 +70,10 @@ def _alias_missing_faiss_swigfaiss_modules() -> None: except ImportError: sys.modules[full] = available -# Worker function for parallel embedding generation (must be at module level for multiprocessing) +# Worker function for parallel embedding generation (must be at module +# level for multiprocessing) + + def _parallel_embed_worker(device_id, input_chunk_indices, input_chunks, result_queue, model_name, encode_kwargs, base_device, numa_plan=None): """Worker function to generate embeddings on a specific device. @@ -120,9 +127,11 @@ def _parallel_embed_worker(device_id, input_chunk_indices, input_chunks, result_ encode_kwargs=encode_kwargs ) - print(f"✓ Device {device}: Loaded model, processing {len(input_chunks)} input chunk(s)") + print( + f"✓ Device {device}: Loaded model, processing {len(input_chunks)} input chunk(s)") - for input_chunk_idx, input_chunk in zip(input_chunk_indices, input_chunks): + for input_chunk_idx, input_chunk in zip( + input_chunk_indices, input_chunks): embeddings = embedder.embed_documents(input_chunk) result_queue.put((input_chunk_idx, embeddings)) @@ -133,24 +142,25 @@ def _parallel_embed_worker(device_id, input_chunk_indices, input_chunks, result_ for input_chunk_idx in input_chunk_indices: result_queue.put((input_chunk_idx, None)) + class VectorDB(RagDB): @classmethod def get_default_db_name(cls) -> str: """Get the default database filename for VectorDB.""" return "vector.db" - + def __init__(self, - retriever_model: str = None, - reranker_model: str = None, - device: str = "auto", - load_embeddings: bool = True, - num_embedding_devices: int = 1, - benchmark: bool = False, - hierarchical: bool = False, - embedding_device: str = None, - reranker_device: str = None, - **kwargs - ): + retriever_model: str = None, + reranker_model: str = None, + device: str = "auto", + load_embeddings: bool = True, + num_embedding_devices: int = 1, + benchmark: bool = False, + hierarchical: bool = False, + embedding_device: str = None, + reranker_device: str = None, + **kwargs + ): super().__init__(reranker_model, device, benchmark, reranker_device=reranker_device) self._retriever_model_name = retriever_model self._reranker_model_name = reranker_model @@ -159,7 +169,8 @@ def __init__(self, self._hierarchical = hierarchical self._embedding_lock = None # Embedding device defaults to inheriting from --device. - self._embedding_device = self._determine_device(embedding_device) if embedding_device else self._device + self._embedding_device = self._determine_device( + embedding_device) if embedding_device else self._device # For hierarchical mode: map child_index -> parent_passage self._parent_map = {} @@ -175,7 +186,8 @@ def __init__(self, # For single-device embedding, allocate one GPU now so the reranker # (allocated later) can't pick the same one. Multi-device path # allocates inside _embed_documents_parallel. - if num_embedding_devices == 1 and self._embedding_device in ("cuda", "xpu"): + if num_embedding_devices == 1 and self._embedding_device in ( + "cuda", "xpu"): from utils import resolve_gpu_device self._embedding_device = resolve_gpu_device( self._embedding_device, name="embedding", @@ -183,27 +195,32 @@ def __init__(self, ) # Initialize embedding model with device configuration - model_kwargs = {'device': self._embedding_device, 'local_files_only': True} + model_kwargs = { + 'device': self._embedding_device, + 'local_files_only': True} encode_kwargs = {'normalize_embeddings': True} - + self._embedding_model = HuggingFaceEmbeddings( model_name=self._retriever_model_name, model_kwargs=model_kwargs, encode_kwargs=encode_kwargs ) - self._embedding_dimension = len(self._embedding_model.embed_query("hello world")) - + self._embedding_dimension = len( + self._embedding_model.embed_query("hello world")) + # Check the dtype of the embedding without using numpy test_embedding_raw = self._embedding_model.embed_query("test") - + # Calculate dtype and itemsize from Python native list - if isinstance(test_embedding_raw, list) and len(test_embedding_raw) > 0: + if isinstance(test_embedding_raw, list) and len( + test_embedding_raw) > 0: test_element = test_embedding_raw[0] embedding_dtype = type(test_element) embedding_itemsize = test_element.__sizeof__() # Size in bytes of one element self._embedding_bytes_per_element = embedding_itemsize else: - raise ValueError("Embedding query did not return a valid list of floats.") + raise ValueError( + "Embedding query did not return a valid list of floats.") if self._benchmark: print(f" Embedding element type: {embedding_dtype}") @@ -220,12 +237,12 @@ def __init__(self, embedding_function=self._embedding_model, index=self._index, docstore=self._docstore, - index_to_docstore_id={}, # This will be populated as documents are added + index_to_docstore_id={}, # This will be populated as documents are added ) - + # Keep track of ingested documents self._doc_list = [] - + def _create_vector_index(self, dimension: int): """Create a FAISS HNSW vector index. @@ -241,12 +258,14 @@ def _create_vector_index(self, dimension: int): FAISS HNSW index """ # M: number of connections per layer (higher = better recall, more memory) - # efConstruction: quality of index construction (higher = better quality, slower build) + # efConstruction: quality of index construction (higher = better + # quality, slower build) M = 32 # Default: 32, good balance index = faiss.IndexHNSWFlat(dimension, M) index.hnsw.efConstruction = 200 # Default: 40 index.hnsw.efSearch = 100 # Search-time parameter, can be adjusted later return index + def _get_embeddings_cache_path(self, passages_path: str) -> str: """Get the cache path for embeddings based on passages file path.""" from pathlib import Path @@ -254,15 +273,16 @@ def _get_embeddings_cache_path(self, passages_path: str) -> str: # Replace extension with .emb.pkl cache_path = passages_path.with_suffix('.emb.pkl') return str(cache_path) - + def _save_embeddings_cache(self, embeddings: list, passages_path: str): """Save embeddings to a pickle file for reuse.""" - import os, pickle + import os + import pickle from pathlib import Path - + cache_path = self._get_embeddings_cache_path(passages_path) Path(cache_path).parent.mkdir(parents=True, exist_ok=True) - + if os.path.exists(cache_path): print(f"Embeddings cache exists: {cache_path}") return @@ -270,16 +290,16 @@ def _save_embeddings_cache(self, embeddings: list, passages_path: str): with open(cache_path, 'wb') as f: pickle.dump(embeddings, f) print(f"💾 Saved embeddings cache to {cache_path}") - + def _load_embeddings_cache(self, passages_path: str) -> list: """Load embeddings from cache if available.""" import pickle from pathlib import Path - + cache_path = self._get_embeddings_cache_path(passages_path) if not Path(cache_path).exists(): return None - + try: with open(cache_path, 'rb') as f: embeddings = pickle.load(f) @@ -288,7 +308,7 @@ def _load_embeddings_cache(self, passages_path: str) -> list: except Exception as e: print(f"⚠️ Failed to load embeddings cache: {e}") return None - + def _build_numa_plans(self, num_workers: int): """Parse INFERENCE_EMBEDDING_NUMA_NODES into per-worker (node, cpu_set). @@ -319,22 +339,23 @@ def _build_numa_plans(self, num_workers: int): def _embed_documents_parallel(self, passages: List[str]) -> list: """Generate embeddings using multiple devices in parallel. - + Uses the device type from --device option and spawns multiple workers. - + Args: passages: List of text passages to embed - + Returns: List of embeddings (one per passage) """ import torch import multiprocessing as mp - + # Use the embedding device (may differ from the global --device). # Strip any GPU index already allocated for the single-device path so # the parallel allocator gets a clean device-type string. - base_device = self._embedding_device.split(":")[0] # e.g., 'xpu', 'cuda', 'cpu', 'hpu' + base_device = self._embedding_device.split( + ":")[0] # e.g., 'xpu', 'cuda', 'cpu', 'hpu' num_workers = min(self._num_embedding_devices, len(passages)) @@ -350,7 +371,8 @@ def _embed_documents_parallel(self, passages: List[str]) -> list: override_env="INFERENCE_EMBEDDING_GPU_DEVICES", ) else: - # CPU / HPU: workers use the same device string; index field is ignored. + # CPU / HPU: workers use the same device string; index field is + # ignored. device_indices = list(range(num_workers)) # Set spawn method for device compatibility (required for XPU/CUDA) @@ -360,16 +382,19 @@ def _embed_documents_parallel(self, passages: List[str]) -> list: # Already set, ignore pass - print(f"🚀 Parallel embedding on {num_workers} {base_device.upper()} device(s)...") + print( + f"🚀 Parallel embedding on {num_workers} {base_device.upper()} device(s)...") # Optional per-worker NUMA pinning (CPU + memory + OMP). numa_plans = self._build_numa_plans(num_workers) # Split passages into per-worker input chunks. input_chunk_size = (len(passages) + num_workers - 1) // num_workers - input_chunks = [passages[i:i + input_chunk_size] for i in range(0, len(passages), input_chunk_size)] + input_chunks = [passages[i:i + input_chunk_size] + for i in range(0, len(passages), input_chunk_size)] - print(f" Split {len(passages)} passages into {len(input_chunks)} input chunk(s) (~{input_chunk_size} passages/device)") + print( + f" Split {len(passages)} passages into {len(input_chunks)} input chunk(s) (~{input_chunk_size} passages/device)") result_queue = mp.Queue() processes = [] @@ -379,11 +404,12 @@ def _embed_documents_parallel(self, passages: List[str]) -> list: # input_chunk_idx is the position in `input_chunks`; device_id is the GPU # index from the allocator. They differ when the allocator returns # non-contiguous indices. - for input_chunk_idx, device_id in enumerate(device_indices[:len(input_chunks)]): + for input_chunk_idx, device_id in enumerate( + device_indices[:len(input_chunks)]): numa_plan = numa_plans[input_chunk_idx] if numa_plans else None p = mp.Process(target=_parallel_embed_worker, - args=(device_id, [input_chunk_idx], [input_chunks[input_chunk_idx]], result_queue, - self._retriever_model_name, encode_kwargs, base_device, numa_plan)) + args=(device_id, [input_chunk_idx], [input_chunks[input_chunk_idx]], result_queue, + self._retriever_model_name, encode_kwargs, base_device, numa_plan)) p.start() processes.append(p) @@ -401,36 +427,37 @@ def _embed_documents_parallel(self, passages: List[str]) -> list: for i in range(len(input_chunks)): if i in results: all_embeddings.extend(results[i]) - - print(f"✓ Generated {len(all_embeddings)} embeddings across {num_workers} devices") - + + print( + f"✓ Generated {len(all_embeddings)} embeddings across {num_workers} devices") + return all_embeddings - + def _calculate_index_output_size(self): """Calculate the size of VectorDB output data (db file - metadata). - + Returns the total size in bytes of the serialized database file, excluding configuration metadata overhead. - + The .db file contains: - FAISS index (vectors) - Passages (docstore) - Metadata (small overhead) - + We estimate metadata size and subtract it from total file size. """ from pathlib import Path - + if not hasattr(self, '_serialize_path') or not self._serialize_path: return 0 - + db_path = Path(self._serialize_path) if not db_path.exists(): return 0 - + total_file_size = db_path.stat().st_size return total_file_size - + def ingest(self, passages: List[str], metadatas: List[dict], **kwargs): """Ingest passages with performance monitoring. @@ -457,7 +484,7 @@ def ingest(self, passages: List[str], metadatas: List[dict], **kwargs): print(f" Stored {len(self._parent_map)} parent mappings") total_chars = sum(len(passage) for passage in passages) - + # Handle embeddings: try to load from cache or generate new ones embeddings = None @@ -468,44 +495,58 @@ def ingest(self, passages: List[str], metadatas: List[dict], **kwargs): if embeddings is None: if self._num_embedding_devices > 1: # Use parallel embedding generation across multiple devices - embeddings = self._track_component("embedding_generation", total_chars, len(passages), - lambda: self._embed_documents_parallel(passages), - is_pipeline_input=True) + embeddings = self._track_component("embedding_generation", total_chars, len(passages), + lambda: self._embed_documents_parallel( + passages), + is_pipeline_input=True) else: # Single device embedding generation - embeddings = self._track_component("embedding_generation", total_chars, len(passages), - lambda: self._embedding_model.embed_documents(passages), - is_pipeline_input=True) - - - # Determine batch size: single batch for small datasets, multiple batches for scaling analysis - track_incremental = self._benchmark and self._monitor and len(passages) >= 500 + embeddings = self._track_component("embedding_generation", total_chars, len(passages), + lambda: self._embedding_model.embed_documents( + passages), + is_pipeline_input=True) + + # Determine batch size: single batch for small datasets, multiple + # batches for scaling analysis + track_incremental = self._benchmark and self._monitor and len( + passages) >= 500 if track_incremental: - batch_size = max(1000, len(passages) // 10) # 10 batches, minimum 1000 docs per batch - print(f"🔬 Incremental indexing analysis: {len(passages)} docs in batches of {batch_size}") + # 10 batches, minimum 1000 docs per batch + batch_size = max(1000, len(passages) // 10) + print( + f"🔬 Incremental indexing analysis: {len(passages)} docs in batches of {batch_size}") else: batch_size = len(passages) # Single batch - + # Track total indexing time for component metrics import time indexing_component_start = time.perf_counter() - + # Process in batches for i in range(0, len(passages), batch_size): batch_end = min(i + batch_size, len(passages)) - self._ingest_single_batch(passages, metadatas, embeddings, i, batch_end, track_incremental) - + self._ingest_single_batch( + passages, + metadatas, + embeddings, + i, + batch_end, + track_incremental) + indexing_component_end = time.perf_counter() indexing_component_duration = indexing_component_end - indexing_component_start - + # Create component metrics for the entire indexing operation if not track_incremental: - # For single batch, component was tracked inside _ingest_single_batch + # For single batch, component was tracked inside + # _ingest_single_batch pass elif self._monitor: - # For incremental, create component metrics here for the entire operation - embedding_bytes = len(passages) * self._embedding_dimension * self._embedding_bytes_per_element - + # For incremental, create component metrics here for the entire + # operation + embedding_bytes = len( + passages) * self._embedding_dimension * self._embedding_bytes_per_element + from ingestion_monitor import ComponentMetrics self._monitor.components["faiss_indexing"] = ComponentMetrics( name="faiss_indexing", @@ -513,25 +554,27 @@ def ingest(self, passages: List[str], metadatas: List[dict], **kwargs): input_size_bytes=embedding_bytes, output_size_bytes=embedding_bytes, # Vectors stored in FAISS index items_processed=len(passages), - throughput_mb_per_sec=(embedding_bytes / (1024 * 1024)) / indexing_component_duration if indexing_component_duration > 0 else 0, - throughput_items_per_sec=len(passages) / indexing_component_duration if indexing_component_duration > 0 else 0, + throughput_mb_per_sec=(embedding_bytes / (1024 * 1024)) / + indexing_component_duration if indexing_component_duration > 0 else 0, + throughput_items_per_sec=len( + passages) / indexing_component_duration if indexing_component_duration > 0 else 0, is_pipeline_input=False, is_pipeline_output=True ) - + # Store ingestion metrics for later reporting self._ingestion_start = ingestion_start self._ingestion_item_count = len(passages) self._ingestion_total_chars = total_chars - - # Save embeddings to cache + + # Save embeddings to cache if self._load_embeddings and passages_path: self._save_embeddings_cache(embeddings, passages_path) def _ingest_single_batch(self, passages: List[str], metadatas: List[dict], embeddings: list, - batch_start: int, batch_end: int, track_incremental: bool): + batch_start: int, batch_end: int, track_incremental: bool): """Ingest a batch of passages. Can be used for single or incremental indexing. - + Args: passages: All passages metadatas: All metadata @@ -541,40 +584,42 @@ def _ingest_single_batch(self, passages: List[str], metadatas: List[dict], embed track_incremental: Whether to track this batch for incremental analysis """ import time - + # Extract batch data batch_passages = passages[batch_start:batch_end] - batch_metadatas = metadatas[batch_start:batch_end] if metadatas else [{}] * (batch_end - batch_start) + batch_metadatas = metadatas[batch_start:batch_end] if metadatas else [ + {}] * (batch_end - batch_start) batch_embeddings = embeddings[batch_start:batch_end] - + # Track DB size before adding (for incremental tracking) db_size_before = len(self._doc_list) if track_incremental else 0 - + # Calculate embedding size for this batch - batch_embedding_bytes = len(batch_passages) * self._embedding_dimension * self._embedding_bytes_per_element - + batch_embedding_bytes = len( + batch_passages) * self._embedding_dimension * self._embedding_bytes_per_element + # Time and execute indexing operation indexing_start = time.perf_counter() - + if track_incremental: # For incremental: just add embeddings without component tracking self._vector_store.add_embeddings( - list(zip(batch_passages, batch_embeddings)), + list(zip(batch_passages, batch_embeddings)), batch_metadatas ) else: # For single batch: use component tracking self._track_component("faiss_indexing", batch_embedding_bytes, len(batch_passages), - lambda: self._vector_store.add_embeddings( - list(zip(batch_passages, batch_embeddings)), batch_metadatas), - is_pipeline_output=True) - + lambda: self._vector_store.add_embeddings( + list(zip(batch_passages, batch_embeddings)), batch_metadatas), + is_pipeline_output=True) + indexing_end = time.perf_counter() indexing_time = indexing_end - indexing_start - + # Update document list self._doc_list.extend(batch_passages) - + # Track for incremental analysis if requested if track_incremental and self._monitor: self._monitor.track_incremental_indexing( @@ -582,7 +627,7 @@ def _ingest_single_batch(self, passages: List[str], metadatas: List[dict], embed batch_size=len(batch_passages), indexing_time=indexing_time ) - + def enable_threading(self): """Enable thread-safe access to the embedding model.""" import threading @@ -598,9 +643,11 @@ def lookup(self, query: str, k: int): embedding = self.embed_query(query) else: embedding = self.embed_query(query) - results = self._vector_store.similarity_search_by_vector(embedding, k=k) + results = self._vector_store.similarity_search_by_vector( + embedding, k=k) - # In hierarchical mode: replace child passages with parents, deduplicate + # In hierarchical mode: replace child passages with parents, + # deduplicate if self._hierarchical and self._parent_map: parent_docs = [] seen_parents = set() @@ -631,7 +678,7 @@ def lookup(self, query: str, k: int): return parent_docs return results - + def lookup_with_scores(self, query: str, k: int): """ Lookup documents with similarity scores. @@ -645,13 +692,16 @@ def lookup_with_scores(self, query: str, k: int): embedding = self.embed_query(query) else: embedding = self.embed_query(query) - results_with_scores = self._vector_store.similarity_search_with_score_by_vector(embedding, k=k) + results_with_scores = self._vector_store.similarity_search_with_score_by_vector( + embedding, k=k) # FAISS returns (document, distance) where distance is L2 distance (lower is better) # Convert to similarity score (higher is better) by negating - results_with_similarity = [(doc, -distance) for doc, distance in results_with_scores] + results_with_similarity = [(doc, -distance) + for doc, distance in results_with_scores] - # In hierarchical mode: replace child passages with parents, deduplicate + # In hierarchical mode: replace child passages with parents, + # deduplicate if self._hierarchical and self._parent_map: parent_results = [] seen_parents = set() @@ -683,8 +733,6 @@ def lookup_with_scores(self, query: str, k: int): return results_with_similarity - - def serialize(self, path: str): # Store path for output size calculation self._serialize_path = path @@ -700,21 +748,27 @@ def serialize(self, path: str): parent_map_path = Path(path).with_suffix('.parent_map.pkl') with open(parent_map_path, 'wb') as f: pickle.dump(self._parent_map, f) - print(f"💾 Saved parent map ({len(self._parent_map)} entries) to {parent_map_path}") + print( + f"💾 Saved parent map ({len(self._parent_map)} entries) to {parent_map_path}") # Update output size after serialization (now file exists) if self._benchmark and self._monitor: - self._monitor.set_output_size_callback("faiss_indexing", self._calculate_index_output_size) + self._monitor.set_output_size_callback( + "faiss_indexing", self._calculate_index_output_size) # Report performance after serialization if benchmarking - if self._benchmark and self._monitor and hasattr(self, '_ingestion_start'): + if self._benchmark and self._monitor and hasattr( + self, '_ingestion_start'): # Determine db_type based on whether incremental was used - db_type = "VectorDB (Incremental)" if hasattr(self._monitor, 'indexing_trend') and len(self._monitor.indexing_trend) > 0 else "VectorDB" + db_type = "VectorDB (Incremental)" if hasattr( + self._monitor, 'indexing_trend') and len( + self._monitor.indexing_trend) > 0 else "VectorDB" self._report_performance(self._ingestion_start, self._ingestion_item_count, - self._ingestion_total_chars, db_type) + self._ingestion_total_chars, db_type) def from_serialized(self, path: str): - assert len(self._vector_store.index_to_docstore_id) == 0, "Vector store already has documents" + assert len( + self._vector_store.index_to_docstore_id) == 0, "Vector store already has documents" # Pickled FAISS indexes reference a specific SWIG submodule (e.g. # `faiss.swigfaiss_avx512`). Alias any missing submodules so DBs built # on one host load on hosts with a different faiss-cpu build. @@ -722,8 +776,8 @@ def from_serialized(self, path: str): with open(path, "rb") as f: data = f.read() self._vector_store = FAISS.deserialize_from_bytes(embeddings=self._embedding_model, - serialized=data, - allow_dangerous_deserialization=True) # <--- USE WITH CAUTION - Only deserialize files you trust + serialized=data, + allow_dangerous_deserialization=True) # <--- USE WITH CAUTION - Only deserialize files you trust # Load parent map if hierarchical mode if self._hierarchical: @@ -733,6 +787,8 @@ def from_serialized(self, path: str): if parent_map_path.exists(): with open(parent_map_path, 'rb') as f: self._parent_map = pickle.load(f) - print(f"✓ Loaded parent map ({len(self._parent_map)} entries) from {parent_map_path}") + print( + f"✓ Loaded parent map ({len(self._parent_map)} entries) from {parent_map_path}") else: - print(f"⚠️ Warning: Hierarchical mode enabled but no parent map found at {parent_map_path}") + print( + f"⚠️ Warning: Hierarchical mode enabled but no parent map found at {parent_map_path}") diff --git a/e2e-rag/single_shot_retrieval.py b/e2e-rag/single_shot_retrieval.py index 81e0d69c4c..aa1afe54ad 100644 --- a/e2e-rag/single_shot_retrieval.py +++ b/e2e-rag/single_shot_retrieval.py @@ -26,7 +26,8 @@ from utils import set_deterministic_seeds, setup_llm_config from params import add_all_args -# Taken below from frames: https://huggingface.co/datasets/google/frames-benchmark +# Taken below from frames: +# https://huggingface.co/datasets/google/frames-benchmark DEFAULT_QUERY = "Who won the French Open Mens Singles tournament the year that New York City FC won their first MLS Cup title?" MAX_PASSAGE_PREVIEW = 4096 FULL_DOC_MAX_CHARS = 39000 @@ -58,7 +59,8 @@ def _read_text(path: str) -> str: return path_obj.read_text(encoding="utf-8", errors="ignore") -def _load_document_text(metadata, base_dir=None, default_base_dir="doc_html", max_chars=FULL_DOC_MAX_CHARS): +def _load_document_text(metadata, base_dir=None, + default_base_dir="doc_html", max_chars=FULL_DOC_MAX_CHARS): target_dir = base_dir or default_base_dir base_filename = metadata.get("base_filename") if not base_filename: @@ -79,7 +81,8 @@ def _load_document_text(metadata, base_dir=None, default_base_dir="doc_html", ma return "", None -def _convert_results_to_entries(results, limit=5, full_doc=False, base_dir=None, default_base_dir="doc_html"): +def _convert_results_to_entries( + results, limit=5, full_doc=False, base_dir=None, default_base_dir="doc_html"): entries = [] seen_ids = set() count = 0 @@ -96,9 +99,15 @@ def _convert_results_to_entries(results, limit=5, full_doc=False, base_dir=None, default_base_dir=default_base_dir ) if not content: - content = getattr(doc, "page_content", metadata.get("content", ""))[:MAX_PASSAGE_PREVIEW] + content = getattr( + doc, "page_content", metadata.get( + "content", ""))[ + :MAX_PASSAGE_PREVIEW] else: - content = getattr(doc, "page_content", metadata.get("content", ""))[:MAX_PASSAGE_PREVIEW] + content = getattr( + doc, "page_content", metadata.get( + "content", ""))[ + :MAX_PASSAGE_PREVIEW] source_path = None entry = {"url": url, "content": content} if source_path: @@ -130,7 +139,8 @@ def _generate_llm_answer(query, doc_entries, llm_config): source = doc.get("url") or "Unknown source" snippet = doc.get("content", "").strip() context_parts.append(f"[{idx}] Source: {source}\n{snippet}") - evidence_block = "\n\n".join(context_parts) if context_parts else "No supporting documents were retrieved." + evidence_block = "\n\n".join( + context_parts) if context_parts else "No supporting documents were retrieved." user_prompt = ( "Answer the question using only the provided evidence." " Respond with a single word or short phrase, or 'Unknown' if the evidence is insufficient.\n\n" @@ -153,7 +163,10 @@ def _generate_llm_answer(query, doc_entries, llm_config): "temperature": 0.0, "max_tokens": max_tokens } - response = requests.post(llm_config["service_url"], json=payload, timeout=60) + response = requests.post( + llm_config["service_url"], + json=payload, + timeout=60) response.raise_for_status() data = response.json() message = data["choices"][0]["message"] @@ -164,14 +177,15 @@ def _generate_llm_answer(query, doc_entries, llm_config): return content.strip() if content.strip() else "Unknown" - if __name__ == "__main__": - args = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) - + args = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter) + # Add all parameters from centralized definitions - # This includes: Common, General, BM25, Vector, Strategy, and Reranking parameters + # This includes: Common, General, BM25, Vector, Strategy, and Reranking + # parameters add_all_args(args) - + # Special handling for --eval argument (needs custom type) # Override the default eval argument with custom type for action in args._actions: @@ -190,17 +204,19 @@ def _generate_llm_answer(query, doc_entries, llm_config): args.database = VectorDB.get_default_db_name() # Normalize database path: ensure .db extension for file operations - db_file_path = args.database if args.database.endswith('.db') else f"{args.database}.db" - db_base_name = args.database.replace('.db', '') if args.database.endswith('.db') else args.database + db_file_path = args.database if args.database.endswith( + '.db') else f"{args.database}.db" + db_base_name = args.database.replace( + '.db', '') if args.database.endswith('.db') else args.database # Create VectorDB instance (pass base name without .db) rag_db = VectorDB(retriever_model=args.retriever_model, reranker_model=args.reranker_model, device=args.device, - database=db_base_name, - load_embeddings=args.load_embeddings, num_embedding_devices=args.num_embedding_devices, - hierarchical=args.hierarchical, - embedding_device=args.embedding_device, - reranker_device=args.reranker_device, - benchmark=args.benchmark) + database=db_base_name, + load_embeddings=args.load_embeddings, num_embedding_devices=args.num_embedding_devices, + hierarchical=args.hierarchical, + embedding_device=args.embedding_device, + reranker_device=args.reranker_device, + benchmark=args.benchmark) if os.path.exists(db_file_path): # Load existing database @@ -208,18 +224,21 @@ def _generate_llm_answer(query, doc_entries, llm_config): rag_db.from_serialized(db_file_path) else: if not args.ingest: - raise ValueError("Either --database (existing) or --ingest (to create new) must be provided") - + raise ValueError( + "Either --database (existing) or --ingest (to create new) must be provided") + # Ingest from file or folder tic = time.time() rag_db.ingest_from_path(args.ingest) - + # Get number of passages for timing calculation - num_passages = len(rag_db._doc_list) # This should be available after ingestion + # This should be available after ingestion + num_passages = len(rag_db._doc_list) toc = time.time() - ingestion_speed = num_passages/(toc-tic) - print(f"Ingestion of {num_passages} passages took {toc - tic:.2f} seconds. {ingestion_speed:.2f} docs/sec") - + ingestion_speed = num_passages / (toc - tic) + print( + f"Ingestion of {num_passages} passages took {toc - tic:.2f} seconds. {ingestion_speed:.2f} docs/sec") + # Save the database (unless --no-save is specified) if not args.no_save: print(f"Saving database to {db_file_path}") @@ -229,16 +248,20 @@ def _generate_llm_answer(query, doc_entries, llm_config): # Run evaluation or single query lookup if args.eval: - max_queries = args.eval if isinstance(args.eval, int) and not isinstance(args.eval, bool) and args.eval > 0 else None - - # Build strategy_params with correct parameter names for filter function + max_queries = args.eval if isinstance( + args.eval, int) and not isinstance( + args.eval, bool) and args.eval > 0 else None + + # Build strategy_params with correct parameter names for filter + # function strategy_params = {"max_results": args.max_results} if args.retrieval_strategy == "top_p": strategy_params["p"] = args.top_p elif args.retrieval_strategy == "relative": strategy_params["ratio"] = args.relative_ratio - + answer_records = [] + def handle_result(prompt, retrieved_docs, metrics): urls = _extract_unique_urls(retrieved_docs) answer_text = None @@ -249,7 +272,8 @@ def handle_result(prompt, retrieved_docs, metrics): full_doc=args.full_doc_context, base_dir=doc_base_dir ) - answer_text = _generate_llm_answer(prompt, doc_entries, llm_config) + answer_text = _generate_llm_answer( + prompt, doc_entries, llm_config) print(f"LLM Answer: {answer_text}") if args.save_results: record = { @@ -270,19 +294,21 @@ def handle_result(prompt, retrieved_docs, metrics): retrieval_strategy=args.retrieval_strategy, detailed_analysis=True, difficulty=args.difficulty, - result_handler=handle_result if (args.generate_answer or args.save_results) else None, + result_handler=handle_result if ( + args.generate_answer or args.save_results) else None, **strategy_params ) - + # Save results for optimization results_data = { - "accuracy": metrics.get('legacy_score', 0.0), # Backward compatibility + # Backward compatibility + "accuracy": metrics.get('legacy_score', 0.0), "metrics": metrics } - + with open("results.json", "w") as f: json.dump(results_data, f, indent=2) - + if args.save_results: with open("result_single_shot.json", "w") as f: json.dump({ @@ -292,13 +318,13 @@ def handle_result(prompt, retrieved_docs, metrics): exit(0) # Exit after evaluation else: # Single query lookup - reuse evaluation code for consistency - + strategy_params = {} if args.retrieval_strategy == "top_p": strategy_params["p"] = args.top_p elif args.retrieval_strategy == "relative": strategy_params["ratio"] = args.relative_ratio - + # Time the retrieval tic = time.time() need_results = args.generate_answer or args.save_results @@ -329,7 +355,8 @@ def handle_result(prompt, retrieved_docs, metrics): full_doc=args.full_doc_context, base_dir=doc_base_dir ) - answer_value = _generate_llm_answer(args.query, doc_entries, llm_config) + answer_value = _generate_llm_answer( + args.query, doc_entries, llm_config) print(f"LLM Answer: {answer_value}") if args.save_results: @@ -345,5 +372,5 @@ def handle_result(prompt, retrieved_docs, metrics): "results": [record] }, f, indent=2) toc = time.time() - + print(f"\nLookup took {toc - tic:.3f} seconds") diff --git a/e2e-rag/text_splitter.py b/e2e-rag/text_splitter.py index 14c09a6752..a83f89718d 100644 --- a/e2e-rag/text_splitter.py +++ b/e2e-rag/text_splitter.py @@ -30,37 +30,39 @@ def clean_text(text: str) -> str: """Clean and normalize text.""" # Normalize whitespace text = re.sub(r'\s+', ' ', text.strip()) - + # Remove excessive newlines but preserve paragraph structure text = re.sub(r'\n\s*\n\s*\n+', '\n\n', text) - + return text -def find_sentence_boundary(text: str, start: int, end: int, search_window: int = 100) -> int: +def find_sentence_boundary(text: str, start: int, end: int, + search_window: int = 100) -> int: """ Find the best sentence boundary within the search window. - + Args: text: The text to search in start: Start position of the passage end: Desired end position search_window: Number of characters to look back for sentence boundary - + Returns: Best boundary position """ if end >= len(text): return len(text) - + # Look for sentence endings within the search window search_start = max(start, end - search_window) sentence_endings = ['.', '!', '?', '\n'] - + best_break = end for i in range(end - 1, search_start - 1, -1): if text[i] in sentence_endings: - # Check if it's followed by whitespace and uppercase letter (proper sentence end) + # Check if it's followed by whitespace and uppercase letter (proper + # sentence end) if i + 1 < len(text) and text[i + 1].isspace(): # Look for the next non-whitespace character j = i + 1 @@ -69,11 +71,12 @@ def find_sentence_boundary(text: str, start: int, end: int, search_window: int = if j < len(text) and (text[j].isupper() or text[j].isdigit()): best_break = i + 1 break - + return best_break -def split_into_passages(text: str, max_length: int = 512, overlap: int = 50) -> List[str]: +def split_into_passages(text: str, max_length: int = 512, + overlap: int = 50) -> List[str]: """ Split text into passages suitable for retrieval systems like ColBERT. @@ -87,7 +90,7 @@ def split_into_passages(text: str, max_length: int = 512, overlap: int = 50) -> """ # Clean up the text text = clean_text(text) - + if len(text) <= max_length: return [text] if text else [] @@ -97,7 +100,8 @@ def split_into_passages(text: str, max_length: int = 512, overlap: int = 50) -> while start < len(text): end = start + max_length - # If we're not at the end of the text, try to break at a sentence boundary + # If we're not at the end of the text, try to break at a sentence + # boundary if end < len(text): end = find_sentence_boundary(text, start, end) @@ -113,7 +117,8 @@ def split_into_passages(text: str, max_length: int = 512, overlap: int = 50) -> return passages -def split_into_fixed_passages(text: str, fixed_length: int = 256, overlap: int = 32) -> List[str]: +def split_into_fixed_passages( + text: str, fixed_length: int = 256, overlap: int = 32) -> List[str]: """ Split text into fixed-length passages with exact character counts. Useful for consistent passage lengths across datasets. @@ -128,7 +133,7 @@ def split_into_fixed_passages(text: str, fixed_length: int = 256, overlap: int = """ # Clean up the text text = clean_text(text) - + if len(text) <= fixed_length: return [text] if text else [] @@ -138,7 +143,7 @@ def split_into_fixed_passages(text: str, fixed_length: int = 256, overlap: int = while start < len(text): end = min(start + fixed_length, len(text)) passage = text[start:end].strip() - + if passage: passages.append(passage) @@ -150,15 +155,16 @@ def split_into_fixed_passages(text: str, fixed_length: int = 256, overlap: int = return passages -def create_passage_metadata(filename: str, passage_index: int, original_url: Optional[str] = None) -> dict: +def create_passage_metadata( + filename: str, passage_index: int, original_url: Optional[str] = None) -> dict: """ Create standardized metadata for a passage. - + Args: filename: Source filename (PDF or HTML) passage_index: Index of this passage within the document original_url: Original URL if available - + Returns: Dictionary containing passage metadata """ @@ -166,19 +172,20 @@ def create_passage_metadata(filename: str, passage_index: int, original_url: Opt base_filename = filename if '.' in filename: base_filename = '.'.join(filename.split('.')[:-1]) - + metadata = { 'index': passage_index, 'base_filename': base_filename } - + if original_url: metadata['original_url'] = original_url - + return metadata -def estimate_passage_count(text: str, max_length: int = 512, overlap: int = 50) -> int: +def estimate_passage_count( + text: str, max_length: int = 512, overlap: int = 50) -> int: """ Estimate the number of passages that will be created from text. Useful for progress tracking without actually splitting. @@ -227,7 +234,8 @@ def split_into_hierarchical_passages( if len(text) <= parent_length: # Single parent case - still create child chunks parent_text = text - children = split_into_passages(parent_text, child_length, child_overlap) + children = split_into_passages( + parent_text, child_length, child_overlap) results = [] for child_idx, child_text in enumerate(children): @@ -246,7 +254,8 @@ def split_into_hierarchical_passages( all_results = [] for parent_id, parent_text in enumerate(parent_chunks): # Split parent into children - children = split_into_passages(parent_text, child_length, child_overlap) + children = split_into_passages( + parent_text, child_length, child_overlap) # Create hierarchical entries for child_idx, child_text in enumerate(children): @@ -257,4 +266,4 @@ def split_into_hierarchical_passages( 'child_index': child_idx }) - return all_results \ No newline at end of file + return all_results diff --git a/e2e-rag/utils.py b/e2e-rag/utils.py index 6867c2aad3..50b9bddb05 100644 --- a/e2e-rag/utils.py +++ b/e2e-rag/utils.py @@ -27,7 +27,6 @@ from typing import Dict, Optional, Union, Any - def load_url_mapping(directory: str) -> Dict[str, str]: """Load URL mapping from url_mapping.json in specified directory.""" mapping_path = Path(directory) / "url_mapping.json" @@ -70,27 +69,28 @@ def set_deterministic_seeds(seed: int = 42) -> None: def filter_dataset_by_difficulty(df, difficulty: int = 0): """ Filter dataset by minimum number of answer links (difficulty level). - + Args: df: pandas DataFrame with dataset difficulty: Minimum number of answer links required (0 = no filtering) - + Returns: Filtered DataFrame with queries having >= difficulty answer links """ if difficulty <= 0: return df - + # Count answer links for each row link_counts = df.apply( - lambda row: sum(1 for col in df.columns - if col.startswith('wikipedia_link_') and row.notna()[col]), + lambda row: sum(1 for col in df.columns + if col.startswith('wikipedia_link_') and row.notna()[col]), axis=1 ) - + filtered_df = df[link_counts >= difficulty].reset_index(drop=True) - print(f"Filtered dataset by difficulty >= {difficulty}: {len(filtered_df)} queries remaining (from {len(df)} total)") - + print( + f"Filtered dataset by difficulty >= {difficulty}: {len(filtered_df)} queries remaining (from {len(df)} total)") + return filtered_df @@ -156,7 +156,8 @@ def set_mempolicy_membind(node: int) -> None: libnuma = ctypes.CDLL("libnuma.so.1", use_errno=True) rc = libnuma.set_mempolicy(MPOL_BIND, ctypes.byref(nodemask), maxnode) except (OSError, AttributeError): - # set_mempolicy is syscall 238 on x86_64; 237 on aarch64 (rare in this codebase). + # set_mempolicy is syscall 238 on x86_64; 237 on aarch64 (rare in this + # codebase). SYS_SET_MEMPOLICY_X86_64 = 238 libc = ctypes.CDLL("libc.so.6", use_errno=True) rc = libc.syscall(SYS_SET_MEMPOLICY_X86_64, MPOL_BIND, @@ -237,7 +238,8 @@ def pin_worker_to_node(node: int, cpu_set: list) -> None: if "OMP_NUM_THREADS" not in os.environ: os.environ["OMP_NUM_THREADS"] = str(len(cpu_set)) - print(f" [worker] node={node} cores={cpu_set[0]}..{cpu_set[-1]} ({len(cpu_set)}) OMP_NUM_THREADS={os.environ['OMP_NUM_THREADS']}") + print( + f" [worker] node={node} cores={cpu_set[0]}..{cpu_set[-1]} ({len(cpu_set)}) OMP_NUM_THREADS={os.environ['OMP_NUM_THREADS']}") def apply_numa_pinning() -> None: @@ -275,7 +277,8 @@ def apply_numa_pinning() -> None: print(f" sched_setaffinity failed: {e}; skipping pinning") return - print(f" Pinned CPU affinity to {len(cores)} cores: {cores[0]}..{cores[-1]}") + print( + f" Pinned CPU affinity to {len(cores)} cores: {cores[0]}..{cores[-1]}") def apply_cpu_threading_env() -> None: @@ -394,17 +397,20 @@ def _parse_override(self, override_env: str) -> list: ) return indices - def allocate(self, count: int = 1, name: str = "", override_env: str = "") -> list: + def allocate(self, count: int = 1, name: str = "", + override_env: str = "") -> list: if override_env: requested = self._parse_override(override_env) if requested: avail = [i for i in requested if i not in self._taken] source = f"{override_env}={','.join(map(str, requested))}" else: - avail = [i for i in self._all_indices if i not in self._taken and self._is_empty(i)] + avail = [ + i for i in self._all_indices if i not in self._taken and self._is_empty(i)] source = "auto" else: - avail = [i for i in self._all_indices if i not in self._taken and self._is_empty(i)] + avail = [ + i for i in self._all_indices if i not in self._taken and self._is_empty(i)] source = "auto" if len(avail) < count: @@ -417,7 +423,8 @@ def allocate(self, count: int = 1, name: str = "", override_env: str = "") -> li chosen = avail[:count] self._taken.update(chosen) label = name or self.device_type - print(f" Allocated {self.device_type}:{chosen} for {label} (via {source})") + print( + f" Allocated {self.device_type}:{chosen} for {label} (via {source})") return chosen @@ -428,14 +435,16 @@ def get_device_allocator(device_type: str) -> DeviceAllocator: return _DEVICE_ALLOCATORS[device_type] -def resolve_gpu_device(device: str, name: str = "", override_env: str = "") -> str: +def resolve_gpu_device(device: str, name: str = "", + override_env: str = "") -> str: """Map a bare device type ('cuda' / 'xpu') to a specific 'cuda:N' string. Returns `device` unchanged for cpu/hpu/auto/already-indexed strings. Errors if no empty GPU is available (use override_env to override). """ if device in ("cuda", "xpu"): - idx = get_device_allocator(device).allocate(count=1, name=name, override_env=override_env)[0] + idx = get_device_allocator(device).allocate( + count=1, name=name, override_env=override_env)[0] return f"{device}:{idx}" return device @@ -447,7 +456,8 @@ def detect_device() -> str: if torch.cuda.is_available(): if getattr(torch.version, "hip", None): - print(f"Using AMD ROCm GPU (torch.version.hip={torch.version.hip})") + print( + f"Using AMD ROCm GPU (torch.version.hip={torch.version.hip})") else: print("Using NVIDIA CUDA GPU") return "cuda" @@ -473,12 +483,14 @@ def get_model_info_from_service(service_url: str) -> Optional[Dict]: """Get model information from LLM service.""" try: # Try OpenAI-compatible API first - models_response = requests.get(f"{service_url.rstrip('/v1/chat/completions').rstrip('/v1')}/v1/models", timeout=10) + models_response = requests.get( + f"{service_url.rstrip('/v1/chat/completions').rstrip('/v1')}/v1/models", + timeout=10) if models_response.status_code == 200: models_data = models_response.json() if "data" in models_data and len(models_data["data"]) > 0: return models_data["data"][0] - + # Try alternative endpoints base_url = service_url.rstrip('/v1/chat/completions').rstrip('/v1') for endpoint in ["/models", "/info", "/v1/model"]: @@ -486,19 +498,19 @@ def get_model_info_from_service(service_url: str) -> Optional[Dict]: response = requests.get(f"{base_url}{endpoint}", timeout=5) if response.status_code == 200: return response.json() - except: + except BaseException: continue - + except Exception as e: print(f"Warning: Could not auto-detect model from {service_url}: {e}") - + return None def get_model_name_from_service(service_url: str) -> str: """Auto-detect model name from LLM service.""" model_info = get_model_info_from_service(service_url) - + if model_info: # Try different possible fields for model name for field in ["id", "model", "name", "model_name"]: @@ -512,18 +524,20 @@ def get_model_name_from_service(service_url: str) -> str: def get_max_tokens_from_service(service_url: str) -> int: """Auto-detect max tokens from LLM service.""" model_info = get_model_info_from_service(service_url) - + if model_info: # Try different possible fields for max tokens - for field in ["max_tokens", "max_length", "context_length", "max_context_length"]: + for field in ["max_tokens", "max_length", + "context_length", "max_context_length"]: if field in model_info and isinstance(model_info[field], int): return model_info[field] - + # Default fallback based on common models return 10240 -def resolve_config_value(value: Union[str, int], auto_func, *args) -> Union[str, int]: +def resolve_config_value( + value: Union[str, int], auto_func, *args) -> Union[str, int]: """Resolve configuration value that might be 'auto'.""" if value == "auto": return auto_func(*args) @@ -537,21 +551,22 @@ def get_device_config(): "device_count": 1, "device_memory": None } - + if torch is None: return config - + if config["device_type"] == "hpu": config["device_count"] = torch.hpu.device_count() - + elif config["device_type"] == "cuda": config["device_count"] = torch.cuda.device_count() if torch.cuda.is_available(): - config["device_memory"] = torch.cuda.get_device_properties(0).total_memory - + config["device_memory"] = torch.cuda.get_device_properties( + 0).total_memory + elif config["device_type"] == "xpu": config["device_count"] = torch.xpu.device_count() - + return config @@ -589,7 +604,8 @@ def setup_llm_config(args): grader_service_url = getattr(args, 'grader_service_url', None) or base_url grader_model_name = getattr(args, 'grader_model', None) or model_name query_service_url = getattr(args, 'query_service_url', None) or base_url - sufficiency_service_url = getattr(args, 'sufficiency_service_url', None) or base_url + sufficiency_service_url = getattr( + args, 'sufficiency_service_url', None) or base_url return { "service_url": base_url, diff --git a/loadgen/issue_query_controller.cc b/loadgen/issue_query_controller.cc index 4c5ca66f0c..c1abea9d14 100644 --- a/loadgen/issue_query_controller.cc +++ b/loadgen/issue_query_controller.cc @@ -459,8 +459,8 @@ void IssueQueryController::IssueQueriesInternal(size_t query_stride, #if USE_NEW_LOGGING_FORMAT std::stringstream ss; ss << "IssueQueryThread " << thread_idx - << " Ending early: Too many outstanding queries." - << " issued " << queries_issued_total << " outstanding " + << " Ending early: Too many outstanding queries." << " issued " + << queries_issued_total << " outstanding " << queries_outstanding; MLPERF_LOG_ERROR(detail, "error_runtime", ss.str()); #else @@ -499,8 +499,8 @@ void IssueQueryController::IssueQueriesInternal(size_t query_stride, #if USE_NEW_LOGGING_FORMAT std::stringstream ss; ss << "IssueQueryThread " << thread_idx - << " Ending early: Max query count reached." - << " query_count " << queries_issued; + << " Ending early: Max query count reached." << " query_count " + << queries_issued; MLPERF_LOG_ERROR(detail, "error_runtime", ss.str()); #else detail.Error("IssueQueryThread ", std::to_string(thread_idx), @@ -519,8 +519,8 @@ void IssueQueryController::IssueQueriesInternal(size_t query_stride, #if USE_NEW_LOGGING_FORMAT std::stringstream ss; ss << "IssueQueryThread " << thread_idx - << " Ending early: Max test duration reached." - << " duration_ns " << duration.count(); + << " Ending early: Max test duration reached." << " duration_ns " + << duration.count(); MLPERF_LOG_ERROR(detail, "error_runtime", ss.str()); #else detail.Error("IssueQueryThread ", std::to_string(thread_idx), diff --git a/loadgen/logging.cc b/loadgen/logging.cc index d7e83e54b9..807c1954a8 100644 --- a/loadgen/logging.cc +++ b/loadgen/logging.cc @@ -812,8 +812,7 @@ void Logger::CollectTlsLoggerStats(TlsLogger* tls_logger) { if (max_entry_vector_size > kTlsLogReservedEntryCount) { #if USE_NEW_LOGGING_FORMAT std::stringstream msg; - msg << "Logging allocation detected:" - << " tid: " << tls_logger->Tid() + msg << "Logging allocation detected:" << " tid: " << tls_logger->Tid() << " reserved_entries: " << kTlsLogReservedEntryCount << " max_entries: " << max_entry_vector_size; MLPERF_LOG_WARNING((*this), "warning_generic_message", msg.str()); diff --git a/tools/submission/submission_checker/checks/power/__init__.py b/tools/submission/submission_checker/checks/power/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/submission/tests/__snapshots__/test_snapshot/test_summary_csv_matches_snapshot.csv b/tools/submission/tests/__snapshots__/test_snapshot/test_summary_csv_matches_snapshot.csv new file mode 100644 index 0000000000..3092775844 --- /dev/null +++ b/tools/submission/tests/__snapshots__/test_snapshot/test_summary_csv_matches_snapshot.csv @@ -0,0 +1,521 @@ +"Organization","Availability","Division","SystemType","SystemName","Platform","Model","MlperfModel","Scenario","Result","Accuracy","number_of_nodes","host_processor_model_name","host_processors_per_node","host_processor_core_count","accelerator_model_name","accelerators_per_node","total_accelerators","Location","framework","operating_system","notes","compliance","errors","version","inferred","has_power","Units","weight_data_types","design_power_watts","PrivateID" +"AMD","available","closed","datacenter","AMD MI355X Custom Cluster","87xMI355X_22xEPYC_9575F","llama2-70b-99","llama2-70b-99","Interactive","785522.0","ROUGE1: 44.5439 ROUGE2: 22.1342 ROUGEL: 28.9124 TOKENS_PER_SAMPLE: 283.0","11","AMD EPYC 9575F","8","64","AMD Instinct MI355X 288GB HBM3e (x87)","8","88","closed/AMD/results/87xMI355X_22xEPYC_9575F/llama2-70b-99/Interactive/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0","Ubuntu 22.04.5 LTS","","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"AMD","available","closed","datacenter","AMD MI355X Custom Cluster","87xMI355X_22xEPYC_9575F","llama2-70b-99","llama2-70b-99","Offline","1042110.0","ROUGE1: 44.426 ROUGE2: 22.0612 ROUGEL: 28.834 TOKENS_PER_SAMPLE: 284.3","11","AMD EPYC 9575F","8","64","AMD Instinct MI355X 288GB HBM3e (x87)","8","88","closed/AMD/results/87xMI355X_22xEPYC_9575F/llama2-70b-99/Offline/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0","Ubuntu 22.04.5 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"AMD","available","closed","datacenter","AMD MI355X Custom Cluster","87xMI355X_22xEPYC_9575F","llama2-70b-99","llama2-70b-99","Server","1016380.0","ROUGE1: 44.5433 ROUGE2: 22.1372 ROUGEL: 28.9134 TOKENS_PER_SAMPLE: 282.9","11","AMD EPYC 9575F","8","64","AMD Instinct MI355X 288GB HBM3e (x87)","8","88","closed/AMD/results/87xMI355X_22xEPYC_9575F/llama2-70b-99/Server/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0","Ubuntu 22.04.5 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"AMD","available","closed","datacenter","AMD MI355X Custom Cluster","87xMI355X_22xEPYC_9575F","llama2-70b-99.9","llama2-70b-99.9","Interactive","785522.0","ROUGE1: 44.5439 ROUGE2: 22.1342 ROUGEL: 28.9124 TOKENS_PER_SAMPLE: 283.0","11","AMD EPYC 9575F","8","64","AMD Instinct MI355X 288GB HBM3e (x87)","8","88","closed/AMD/results/87xMI355X_22xEPYC_9575F/llama2-70b-99.9/Interactive/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0","Ubuntu 22.04.5 LTS","","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"AMD","available","closed","datacenter","AMD MI355X Custom Cluster","87xMI355X_22xEPYC_9575F","llama2-70b-99.9","llama2-70b-99.9","Offline","1042110.0","ROUGE1: 44.426 ROUGE2: 22.0612 ROUGEL: 28.834 TOKENS_PER_SAMPLE: 284.3","11","AMD EPYC 9575F","8","64","AMD Instinct MI355X 288GB HBM3e (x87)","8","88","closed/AMD/results/87xMI355X_22xEPYC_9575F/llama2-70b-99.9/Offline/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0","Ubuntu 22.04.5 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"AMD","available","closed","datacenter","AMD MI355X Custom Cluster","87xMI355X_22xEPYC_9575F","llama2-70b-99.9","llama2-70b-99.9","Server","1016380.0","ROUGE1: 44.5433 ROUGE2: 22.1372 ROUGEL: 28.9134 TOKENS_PER_SAMPLE: 282.9","11","AMD EPYC 9575F","8","64","AMD Instinct MI355X 288GB HBM3e (x87)","8","88","closed/AMD/results/87xMI355X_22xEPYC_9575F/llama2-70b-99.9/Server/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0","Ubuntu 22.04.5 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"AMD","available","closed","datacenter","AMD MI355X Custom Cluster","94xMI355X_24xEPYC_9575F","gpt-oss-120b","gpt-oss-120b","Offline","1031070.0","exact_match: 83.889","12","AMD EPYC 9575F","8","64","AMD Instinct MI355X 288GB HBM3e (x94)","8","96","closed/AMD/results/94xMI355X_24xEPYC_9575F/gpt-oss-120b/Offline/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0","Ubuntu 22.04.5 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"AMD","available","closed","datacenter","AMD MI355X Custom Cluster","94xMI355X_24xEPYC_9575F","gpt-oss-120b","gpt-oss-120b","Server","900054.0","exact_match: 82.888","12","AMD EPYC 9575F","8","64","AMD Instinct MI355X 288GB HBM3e (x94)","8","96","closed/AMD/results/94xMI355X_24xEPYC_9575F/gpt-oss-120b/Server/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0","Ubuntu 22.04.5 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"AMD","available","closed","datacenter","smci355-ccs-aus-m09-17 (AS -4126GS-NMR-LCC)","8xMI355X_2xEPYC_9575F","gpt-oss-120b","gpt-oss-120b","Offline","95004.0","exact_match: 83.396","1","AMD EPYC 9575F","8","64","AMD Instinct MI355X 288GB HBM3e","8","8","closed/AMD/results/8xMI355X_2xEPYC_9575F/gpt-oss-120b/Offline/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0","Ubuntu 22.04.5 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"AMD","available","closed","datacenter","smci355-ccs-aus-m09-17 (AS -4126GS-NMR-LCC)","8xMI355X_2xEPYC_9575F","gpt-oss-120b","gpt-oss-120b","Server","82136.1","exact_match: 83.813","1","AMD EPYC 9575F","8","64","AMD Instinct MI355X 288GB HBM3e","8","8","closed/AMD/results/8xMI355X_2xEPYC_9575F/gpt-oss-120b/Server/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0","Ubuntu 22.04.5 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"AMD","available","closed","datacenter","smci355-ccs-aus-m09-17 (AS -4126GS-NMR-LCC)","8xMI355X_2xEPYC_9575F","llama2-70b-99","llama2-70b-99","Interactive","73608.2","ROUGE1: 44.5411 ROUGE2: 22.1345 ROUGEL: 28.9133 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9575F","8","64","AMD Instinct MI355X 288GB HBM3e","8","8","closed/AMD/results/8xMI355X_2xEPYC_9575F/llama2-70b-99/Interactive/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0","Ubuntu 22.04.5 LTS","","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"AMD","available","closed","datacenter","smci355-ccs-aus-m09-17 (AS -4126GS-NMR-LCC)","8xMI355X_2xEPYC_9575F","llama2-70b-99","llama2-70b-99","Offline","103480.0","ROUGE1: 44.4495 ROUGE2: 22.0868 ROUGEL: 28.8592 TOKENS_PER_SAMPLE: 284.2","1","AMD EPYC 9575F","8","64","AMD Instinct MI355X 288GB HBM3e","8","8","closed/AMD/results/8xMI355X_2xEPYC_9575F/llama2-70b-99/Offline/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0","Ubuntu 22.04.5 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"AMD","available","closed","datacenter","smci355-ccs-aus-m09-17 (AS -4126GS-NMR-LCC)","8xMI355X_2xEPYC_9575F","llama2-70b-99","llama2-70b-99","Server","100282.0","ROUGE1: 44.5371 ROUGE2: 22.1319 ROUGEL: 28.9082 TOKENS_PER_SAMPLE: 283.0","1","AMD EPYC 9575F","8","64","AMD Instinct MI355X 288GB HBM3e","8","8","closed/AMD/results/8xMI355X_2xEPYC_9575F/llama2-70b-99/Server/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0","Ubuntu 22.04.5 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"AMD","available","closed","datacenter","smci355-ccs-aus-m09-17 (AS -4126GS-NMR-LCC)","8xMI355X_2xEPYC_9575F","llama2-70b-99.9","llama2-70b-99.9","Interactive","73608.2","ROUGE1: 44.5411 ROUGE2: 22.1345 ROUGEL: 28.9133 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9575F","8","64","AMD Instinct MI355X 288GB HBM3e","8","8","closed/AMD/results/8xMI355X_2xEPYC_9575F/llama2-70b-99.9/Interactive/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0","Ubuntu 22.04.5 LTS","","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"AMD","available","closed","datacenter","smci355-ccs-aus-m09-17 (AS -4126GS-NMR-LCC)","8xMI355X_2xEPYC_9575F","llama2-70b-99.9","llama2-70b-99.9","Offline","103480.0","ROUGE1: 44.4495 ROUGE2: 22.0868 ROUGEL: 28.8592 TOKENS_PER_SAMPLE: 284.2","1","AMD EPYC 9575F","8","64","AMD Instinct MI355X 288GB HBM3e","8","8","closed/AMD/results/8xMI355X_2xEPYC_9575F/llama2-70b-99.9/Offline/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0","Ubuntu 22.04.5 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"AMD","available","closed","datacenter","smci355-ccs-aus-m09-17 (AS -4126GS-NMR-LCC)","8xMI355X_2xEPYC_9575F","llama2-70b-99.9","llama2-70b-99.9","Server","100282.0","ROUGE1: 44.5371 ROUGE2: 22.1319 ROUGEL: 28.9082 TOKENS_PER_SAMPLE: 283.0","1","AMD EPYC 9575F","8","64","AMD Instinct MI355X 288GB HBM3e","8","8","closed/AMD/results/8xMI355X_2xEPYC_9575F/llama2-70b-99.9/Server/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0","Ubuntu 22.04.5 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"AMD","available","open","datacenter","SuperMicro AS-4126GS-NMR-LCC","8xMI355X_2xEPYC_9597F","wan-2.2-t2v-a14b","wan-2.2-t2v-a14b","SingleStream","27.384519297","vbench_score: 69.98","1","AMD EPYC 9575F","2","64","AMD Instinct MI355X 288GB HBM3E","8","8","open/AMD/results/8xMI355X_2xEPYC_9597F/wan-2.2-t2v-a14b/SingleStream/performance/run_1","xDiT, xfuser 0.4.5+git6ffc551, Pytorch 2.10.0+gitd9556b0, ROCm 7.11.0","Ubuntu 22.04.5 LTS","","open","0","v6.0","0","False","Latency (s)","fp8,bf16","","" +"AMD","available","open","datacenter","smci355-ccs-aus-m09-17 (AS -4126GS-NMR-LCC)","8xMI355X_2xEPYC_9575F","llama3.1-405b","llama3.1-405b","Interactive","793.994","ROUGEL: 21.51398069082574 exact_match: 86.51147584381204 TOKENS_PER_SAMPLE: 756.7","1","AMD EPYC 9575F","8","64","AMD Instinct MI355X 288GB HBM3e","8","8","open/AMD/results/8xMI355X_2xEPYC_9575F/llama3.1-405b/Interactive/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0, PyTorch 2.9.0a0+git1c57644, ROCm 7.1.0","Ubuntu 22.04.5 LTS","","open","0","v6.0","1","False","Tokens/s","fp4","","" +"AMD","available","open","datacenter","smci355-ccs-aus-m09-17 (AS -4126GS-NMR-LCC)","8xMI355X_2xEPYC_9575F","llama3.1-405b","llama3.1-405b","Offline","2011.17","ROUGEL: 21.545066368325006 exact_match: 82.66624751819988 TOKENS_PER_SAMPLE: 758.9","1","AMD EPYC 9575F","8","64","AMD Instinct MI355X 288GB HBM3e","8","8","open/AMD/results/8xMI355X_2xEPYC_9575F/llama3.1-405b/Offline/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0, PyTorch 2.9.0a0+git1c57644, ROCm 7.1.0","Ubuntu 22.04.5 LTS","","open","0","v6.0","0","False","Tokens/s","fp4","","" +"ASUSTeK","available","closed","datacenter","ESC8000A_E13P_RTX_PRO_6000_PCIE_96GBx8","ESC8000A_E13P_RTX_PRO_6000_PCIE_96GBx8_TRT","llama2-70b-99","llama2-70b-99","Interactive","6238.12","ROUGE1: 44.7596 ROUGE2: 22.3266 ROUGEL: 29.1163 TOKENS_PER_SAMPLE: 272.8","1","AMD EPYC 9475F","2","48","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/ASUSTeK/results/ESC8000A_E13P_RTX_PRO_6000_PCIE_96GBx8_TRT/llama2-70b-99/Interactive/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04","Data bandwidth for GPU-PCIe: 504 GB/s; PCIe-NIC: 126 GB/s","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"ASUSTeK","available","closed","datacenter","ESC8000A_E13P_RTX_PRO_6000_PCIE_96GBx8","ESC8000A_E13P_RTX_PRO_6000_PCIE_96GBx8_TRT","llama2-70b-99","llama2-70b-99","Offline","27812.9","ROUGE1: 44.7099 ROUGE2: 22.3189 ROUGEL: 29.083 TOKENS_PER_SAMPLE: 274.6","1","AMD EPYC 9475F","2","48","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/ASUSTeK/results/ESC8000A_E13P_RTX_PRO_6000_PCIE_96GBx8_TRT/llama2-70b-99/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04","Data bandwidth for GPU-PCIe: 504 GB/s; PCIe-NIC: 126 GB/s","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"ASUSTeK","available","closed","datacenter","ESC8000A_E13P_RTX_PRO_6000_PCIE_96GBx8","ESC8000A_E13P_RTX_PRO_6000_PCIE_96GBx8_TRT","llama2-70b-99","llama2-70b-99","Server","28038.5","ROUGE1: 44.765 ROUGE2: 22.33 ROUGEL: 29.1337 TOKENS_PER_SAMPLE: 274.9","1","AMD EPYC 9475F","2","48","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/ASUSTeK/results/ESC8000A_E13P_RTX_PRO_6000_PCIE_96GBx8_TRT/llama2-70b-99/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04","Data bandwidth for GPU-PCIe: 504 GB/s; PCIe-NIC: 126 GB/s","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"ASUSTeK","available","closed","datacenter","ESC8000A_E13P_RTX_PRO_6000_PCIE_96GBx8","ESC8000A_E13P_RTX_PRO_6000_PCIE_96GBx8_TRT","llama3.1-8b","llama3.1-8b","Interactive","34241.0","ROUGE1: 38.77 ROUGE2: 16.0196 ROUGEL: 24.5357 ROUGELSUM: 35.6853 GEN_LEN: 8178793","1","AMD EPYC 9475F","2","48","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/ASUSTeK/results/ESC8000A_E13P_RTX_PRO_6000_PCIE_96GBx8_TRT/llama3.1-8b/Interactive/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04","Data bandwidth for GPU-PCIe: 504 GB/s; PCIe-NIC: 126 GB/s","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"ASUSTeK","available","closed","datacenter","ESC8000A_E13P_RTX_PRO_6000_PCIE_96GBx8","ESC8000A_E13P_RTX_PRO_6000_PCIE_96GBx8_TRT","llama3.1-8b","llama3.1-8b","Offline","48791.0","ROUGE1: 38.7666 ROUGE2: 15.9806 ROUGEL: 24.5188 ROUGELSUM: 35.6677 GEN_LEN: 8148259","1","AMD EPYC 9475F","2","48","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/ASUSTeK/results/ESC8000A_E13P_RTX_PRO_6000_PCIE_96GBx8_TRT/llama3.1-8b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04","Data bandwidth for GPU-PCIe: 504 GB/s; PCIe-NIC: 126 GB/s","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"ASUSTeK","available","closed","datacenter","ESC8000A_E13P_RTX_PRO_6000_PCIE_96GBx8","ESC8000A_E13P_RTX_PRO_6000_PCIE_96GBx8_TRT","llama3.1-8b","llama3.1-8b","Server","48531.3","ROUGE1: 38.7519 ROUGE2: 16.0012 ROUGEL: 24.5188 ROUGELSUM: 35.6782 GEN_LEN: 8178012","1","AMD EPYC 9475F","2","48","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/ASUSTeK/results/ESC8000A_E13P_RTX_PRO_6000_PCIE_96GBx8_TRT/llama3.1-8b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04","Data bandwidth for GPU-PCIe: 504 GB/s; PCIe-NIC: 126 GB/s","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"ASUSTeK","available","closed","datacenter","ESC8000A_E13P_RTX_PRO_6000_PCIE_96GBx8","ESC8000A_E13P_RTX_PRO_6000_PCIE_96GBx8_TRT","whisper","whisper","Offline","18338.3","ACCURACY: 97.83967947314827","1","AMD EPYC 9475F","2","48","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/ASUSTeK/results/ESC8000A_E13P_RTX_PRO_6000_PCIE_96GBx8_TRT/whisper/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04","Data bandwidth for GPU-PCIe: 504 GB/s; PCIe-NIC: 126 GB/s","closed","0","v6.0","0","False","Samples/s","fp16","","" +"ASUSTeK","available","closed","datacenter","ESC8000_E12P_RTX_PRO_6000_PCIE_96GBx8","ESC8000_E12P_RTX_PRO_6000_PCIE_96GBx8_TRT","llama2-70b-99","llama2-70b-99","Interactive","6262.63","ROUGE1: 44.8215 ROUGE2: 22.3657 ROUGEL: 29.1382 TOKENS_PER_SAMPLE: 272.1","1","Intel(R) Xeon(R) 6787P","2","86","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/ASUSTeK/results/ESC8000_E12P_RTX_PRO_6000_PCIE_96GBx8_TRT/llama2-70b-99/Interactive/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04","Data bandwidth for GPU-PCIe: 504 GB/s; PCIe-NIC: 126 GB/s","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"ASUSTeK","available","closed","datacenter","ESC8000_E12P_RTX_PRO_6000_PCIE_96GBx8","ESC8000_E12P_RTX_PRO_6000_PCIE_96GBx8_TRT","llama2-70b-99","llama2-70b-99","Offline","27730.1","ROUGE1: 44.7344 ROUGE2: 22.3064 ROUGEL: 29.0956 TOKENS_PER_SAMPLE: 274.2","1","Intel(R) Xeon(R) 6787P","2","86","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/ASUSTeK/results/ESC8000_E12P_RTX_PRO_6000_PCIE_96GBx8_TRT/llama2-70b-99/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04","Data bandwidth for GPU-PCIe: 504 GB/s; PCIe-NIC: 126 GB/s","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"ASUSTeK","available","closed","datacenter","ESC8000_E12P_RTX_PRO_6000_PCIE_96GBx8","ESC8000_E12P_RTX_PRO_6000_PCIE_96GBx8_TRT","llama2-70b-99","llama2-70b-99","Server","27113.8","ROUGE1: 44.8535 ROUGE2: 22.4326 ROUGEL: 29.2034 TOKENS_PER_SAMPLE: 272.9","1","Intel(R) Xeon(R) 6787P","2","86","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/ASUSTeK/results/ESC8000_E12P_RTX_PRO_6000_PCIE_96GBx8_TRT/llama2-70b-99/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04","Data bandwidth for GPU-PCIe: 504 GB/s; PCIe-NIC: 126 GB/s","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"ASUSTeK","available","closed","datacenter","ESC8000_E12P_RTX_PRO_6000_PCIE_96GBx8","ESC8000_E12P_RTX_PRO_6000_PCIE_96GBx8_TRT","llama3.1-8b","llama3.1-8b","Offline","48613.8","ROUGE1: 38.7575 ROUGE2: 16.0094 ROUGEL: 24.5303 ROUGELSUM: 35.6835 GEN_LEN: 8178438","1","Intel(R) Xeon(R) 6787P","2","86","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/ASUSTeK/results/ESC8000_E12P_RTX_PRO_6000_PCIE_96GBx8_TRT/llama3.1-8b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04","Data bandwidth for GPU-PCIe: 504 GB/s; PCIe-NIC: 126 GB/s","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"ASUSTeK","available","closed","datacenter","ESC8000_E12P_RTX_PRO_6000_PCIE_96GBx8","ESC8000_E12P_RTX_PRO_6000_PCIE_96GBx8_TRT","llama3.1-8b","llama3.1-8b","Server","47581.0","ROUGE1: 38.75 ROUGE2: 15.9983 ROUGEL: 24.5119 ROUGELSUM: 35.6724 GEN_LEN: 8178586","1","Intel(R) Xeon(R) 6787P","2","86","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/ASUSTeK/results/ESC8000_E12P_RTX_PRO_6000_PCIE_96GBx8_TRT/llama3.1-8b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04","Data bandwidth for GPU-PCIe: 504 GB/s; PCIe-NIC: 126 GB/s","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"ASUSTeK","available","closed","datacenter","XA NB3I-E12","XA_NB3I-E12_B300x8_TRT","gpt-oss-120b","gpt-oss-120b","Interactive","26005.8","exact_match: 82.996","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/ASUSTeK/results/XA_NB3I-E12_B300x8_TRT/gpt-oss-120b/Interactive/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04","","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"ASUSTeK","available","closed","datacenter","XA NB3I-E12","XA_NB3I-E12_B300x8_TRT","gpt-oss-120b","gpt-oss-120b","Offline","110077.0","exact_match: 83.151","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/ASUSTeK/results/XA_NB3I-E12_B300x8_TRT/gpt-oss-120b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"ASUSTeK","available","closed","datacenter","XA NB3I-E12","XA_NB3I-E12_B300x8_TRT","gpt-oss-120b","gpt-oss-120b","Server","100656.0","exact_match: 83.515","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/ASUSTeK/results/XA_NB3I-E12_B300x8_TRT/gpt-oss-120b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"ASUSTeK","available","closed","datacenter","XA NB3I-E12","XA_NB3I-E12_B300x8_TRT","llama2-70b-99","llama2-70b-99","Offline","109625.0","ROUGE1: 44.799 ROUGE2: 22.3898 ROUGEL: 29.1767 TOKENS_PER_SAMPLE: 272.8","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/ASUSTeK/results/XA_NB3I-E12_B300x8_TRT/llama2-70b-99/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"ASUSTeK","available","closed","datacenter","XA NB3I-E12","XA_NB3I-E12_B300x8_TRT","llama2-70b-99","llama2-70b-99","Server","102186.0","ROUGE1: 44.8309 ROUGE2: 22.3869 ROUGEL: 29.1766 TOKENS_PER_SAMPLE: 272.2","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/ASUSTeK/results/XA_NB3I-E12_B300x8_TRT/llama2-70b-99/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"ASUSTeK","available","closed","datacenter","XA NB3I-E12","XA_NB3I-E12_B300x8_TRT","llama3.1-8b","llama3.1-8b","Offline","166745.0","ROUGE1: 38.4844 ROUGE2: 15.929 ROUGEL: 24.3386 ROUGELSUM: 35.6648 GEN_LEN: 8186789","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/ASUSTeK/results/XA_NB3I-E12_B300x8_TRT/llama3.1-8b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"ASUSTeK","available","closed","datacenter","XA NB3I-E12","XA_NB3I-E12_B300x8_TRT","llama3.1-8b","llama3.1-8b","Server","148067.0","ROUGE1: 38.4837 ROUGE2: 15.9227 ROUGEL: 24.3333 ROUGELSUM: 35.6577 GEN_LEN: 8185499","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/ASUSTeK/results/XA_NB3I-E12_B300x8_TRT/llama3.1-8b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Cisco","available","closed","datacenter","C240M8-1-node-2S-GNR_86C","C240M8_1_node_2S_GNR_86C","llama3_1-8b","llama3.1-8b","Offline","806.105","ROUGE1: 38.8227 ROUGE2: 16.0784 ROUGEL: 24.5847 ROUGELSUM: 35.8974 GEN_LEN: 8148551","1","Intel(R) Xeon(R) 6787P","2","86","N/A","0","0","closed/Cisco/results/C240M8_1_node_2S_GNR_86C/llama3_1-8b/Offline/performance/run_1","PyTorch","Ubuntu 24.04.3","","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Cisco","available","closed","datacenter","C240M8-1-node-2S-GNR_86C","C240M8_1_node_2S_GNR_86C","llama3_1-8b","llama3.1-8b","Server","258.538","ROUGE1: 38.8579 ROUGE2: 16.0896 ROUGEL: 24.6038 ROUGELSUM: 35.9322 GEN_LEN: 8145938","1","Intel(R) Xeon(R) 6787P","2","86","N/A","0","0","closed/Cisco/results/C240M8_1_node_2S_GNR_86C/llama3_1-8b/Server/performance/run_1","PyTorch","Ubuntu 24.04.3","","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Cisco","available","closed","datacenter","C240M8-1-node-2S-GNR_86C","C240M8_1_node_2S_GNR_86C","rgat","rgat","Offline","14283.5","acc: 71.803","1","Intel(R) Xeon(R) 6787P","2","86","N/A","0","0","closed/Cisco/results/C240M8_1_node_2S_GNR_86C/rgat/Offline/performance/run_1","PyTorch","Ubuntu 24.04.3","","closed","0","v6.0","0","False","Samples/s","INT8","","" +"Cisco","available","closed","datacenter","C240M8-1-node-2S-GNR_86C","C240M8_1_node_2S_GNR_86C","whisper","whisper","Offline","1422.99","ACCURACY: 97.88576267638467","1","Intel(R) Xeon(R) 6787P","2","86","N/A","0","0","closed/Cisco/results/C240M8_1_node_2S_GNR_86C/whisper/Offline/performance/run_1","PyTorch","Ubuntu 24.04.3","","closed","0","v6.0","0","False","Samples/s","INT8","","" +"Cisco","available","closed","datacenter","Cisco UCS 16xMI350X","Cisco_UCS_16xMI350x","llama2-70b-99","llama2-70b-99","Interactive","103633.0","ROUGE1: 44.6085 ROUGE2: 22.1816 ROUGEL: 28.9644 TOKENS_PER_SAMPLE: 281.7","2","AMD EPYC 9575F","2","64","AMD Instinct MI350X 288GB HBM3e","8","16","closed/Cisco/results/Cisco_UCS_16xMI350x/llama2-70b-99/Interactive/performance/run_1","vLLM","Ubuntu 24.04","","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"Cisco","available","closed","datacenter","Cisco UCS 16xMI350X","Cisco_UCS_16xMI350x","llama2-70b-99","llama2-70b-99","Offline","159932.0","ROUGE1: 44.4637 ROUGE2: 22.1149 ROUGEL: 28.8866 TOKENS_PER_SAMPLE: 283.3","2","AMD EPYC 9575F","2","64","AMD Instinct MI350X 288GB HBM3e","8","16","closed/Cisco/results/Cisco_UCS_16xMI350x/llama2-70b-99/Offline/performance/run_1","vLLM","Ubuntu 24.04","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Cisco","available","closed","datacenter","Cisco UCS 16xMI350X","Cisco_UCS_16xMI350x","llama2-70b-99","llama2-70b-99","Server","153022.0","ROUGE1: 44.6075 ROUGE2: 22.1813 ROUGEL: 28.9635 TOKENS_PER_SAMPLE: 281.8","2","AMD EPYC 9575F","2","64","AMD Instinct MI350X 288GB HBM3e","8","16","closed/Cisco/results/Cisco_UCS_16xMI350x/llama2-70b-99/Server/performance/run_1","vLLM","Ubuntu 24.04","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Cisco","available","closed","datacenter","Cisco UCS 8xMI350X","Cisco_UCS_8xMI350x","llama2-70b-99","llama2-70b-99","Interactive","51616.6","ROUGE1: 44.6101 ROUGE2: 22.1826 ROUGEL: 28.9649 TOKENS_PER_SAMPLE: 281.8","1","AMD EPYC 9575F","2","64","AMD Instinct MI350X 288GB HBM3e","8","8","closed/Cisco/results/Cisco_UCS_8xMI350x/llama2-70b-99/Interactive/performance/run_1","vLLM","Ubuntu 24.04","","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"Cisco","available","closed","datacenter","Cisco UCS 8xMI350X","Cisco_UCS_8xMI350x","llama2-70b-99","llama2-70b-99","Offline","78936.7","ROUGE1: 44.467 ROUGE2: 22.1014 ROUGEL: 28.8752 TOKENS_PER_SAMPLE: 283.8","1","AMD EPYC 9575F","2","64","AMD Instinct MI350X 288GB HBM3e","8","8","closed/Cisco/results/Cisco_UCS_8xMI350x/llama2-70b-99/Offline/performance/run_1","vLLM","Ubuntu 24.04","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Cisco","available","closed","datacenter","Cisco UCS 8xMI350X","Cisco_UCS_8xMI350x","llama2-70b-99","llama2-70b-99","Server","76617.4","ROUGE1: 44.6074 ROUGE2: 22.1814 ROUGEL: 28.9634 TOKENS_PER_SAMPLE: 281.8","1","AMD EPYC 9575F","2","64","AMD Instinct MI350X 288GB HBM3e","8","8","closed/Cisco/results/Cisco_UCS_8xMI350x/llama2-70b-99/Server/performance/run_1","vLLM","Ubuntu 24.04","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Cisco","available","closed","datacenter","Cisco UCS C845A M8 (8x H200-NVL-141GB, TensorRT)","C845A_M8_H200NVL_141GBx8_TRT","whisper","whisper","Offline","30195.2","ACCURACY: 97.82730759347906","1","AMD EPYC 9575F","2","64","NVIDIA H200-NVL-141GB","8","8","closed/Cisco/results/C845A_M8_H200NVL_141GBx8_TRT/whisper/Offline/performance/run_1","TensorRT 10.11, CUDA 12.9","Ubuntu 24.04","H200-NVL TGP 600W","closed","0","v6.0","0","False","Samples/s","fp16","","" +"Cisco","available","closed","datacenter","Cisco UCS C845A M8 (8x RTXPro6000-PCIe-96GB, TensorRT)","C845A_M8_RTXPro6000_96GBx8_TRT","whisper","whisper","Offline","18265.2","ACCURACY: 97.81874244601579","1","AMD EPYC 9575F","2","64","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/Cisco/results/C845A_M8_RTXPro6000_96GBx8_TRT/whisper/Offline/performance/run_1","TensorRT 10.11, CUDA 12.9","Ubuntu 24.04","NVIDIA RTX PRO 6000 Blackwell Server Edition TGP 600W","closed","0","v6.0","0","False","Samples/s","fp16","","" +"Cisco","available","closed","datacenter","Cisco UCS C880A M8 (8x NVIDIA B300-SXM-270GB, TensorRT)","C880A_M8_B300_288GBx8_TRT","deepseek-r1","deepseek-r1","Offline","69251.4","exact_match: 81.22151321786691 TOKENS_PER_SAMPLE: 3770.239744758432","1","Intel(R) Xeon(R) 6767P","2","128","NVIDIA B300-SXM-270GB","8","8","closed/Cisco/results/C880A_M8_B300_288GBx8_TRT/deepseek-r1/Offline/performance/run_1","TensorRT 10.13, CUDA 13.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Cisco","available","closed","datacenter","Cisco UCS C880A M8 (8x NVIDIA B300-SXM-270GB, TensorRT)","C880A_M8_B300_288GBx8_TRT","deepseek-r1","deepseek-r1","Server","58553.3","exact_match: 81.42661804922516 TOKENS_PER_SAMPLE: 3726.4544211485872","1","Intel(R) Xeon(R) 6767P","2","128","NVIDIA B300-SXM-270GB","8","8","closed/Cisco/results/C880A_M8_B300_288GBx8_TRT/deepseek-r1/Server/performance/run_1","TensorRT 10.13, CUDA 13.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Cisco","available","closed","datacenter","Cisco UCS C880A M8 (8x NVIDIA B300-SXM-270GB, TensorRT)","C880A_M8_B300_288GBx8_TRT","gpt-oss-120b","gpt-oss-120b","Offline","111496.0","exact_match: 82.846","1","Intel(R) Xeon(R) 6767P","2","128","NVIDIA B300-SXM-270GB","8","8","closed/Cisco/results/C880A_M8_B300_288GBx8_TRT/gpt-oss-120b/Offline/performance/run_1","TensorRT 10.13, CUDA 13.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Cisco","available","closed","datacenter","Cisco UCS C880A M8 (8x NVIDIA B300-SXM-270GB, TensorRT)","C880A_M8_B300_288GBx8_TRT","gpt-oss-120b","gpt-oss-120b","Server","110655.0","exact_match: 83.393","1","Intel(R) Xeon(R) 6767P","2","128","NVIDIA B300-SXM-270GB","8","8","closed/Cisco/results/C880A_M8_B300_288GBx8_TRT/gpt-oss-120b/Server/performance/run_1","TensorRT 10.13, CUDA 13.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Cisco","available","closed","datacenter","Cisco UCS C880A M8 (8x NVIDIA B300-SXM-270GB, TensorRT)","C880A_M8_B300_288GBx8_TRT","llama2-70b-99","llama2-70b-99","Offline","115730.0","ROUGE1: 44.7978 ROUGE2: 22.3733 ROUGEL: 29.1557 TOKENS_PER_SAMPLE: 273.1","1","Intel(R) Xeon(R) 6767P","2","128","NVIDIA B300-SXM-270GB","8","8","closed/Cisco/results/C880A_M8_B300_288GBx8_TRT/llama2-70b-99/Offline/performance/run_1","TensorRT 10.13, CUDA 13.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Cisco","available","closed","datacenter","Cisco UCS C880A M8 (8x NVIDIA B300-SXM-270GB, TensorRT)","C880A_M8_B300_288GBx8_TRT","llama2-70b-99","llama2-70b-99","Server","114504.0","ROUGE1: 44.8431 ROUGE2: 22.4093 ROUGEL: 29.1757 TOKENS_PER_SAMPLE: 272.7","1","Intel(R) Xeon(R) 6767P","2","128","NVIDIA B300-SXM-270GB","8","8","closed/Cisco/results/C880A_M8_B300_288GBx8_TRT/llama2-70b-99/Server/performance/run_1","TensorRT 10.13, CUDA 13.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Cisco","available","closed","datacenter","Cisco UCS C880A M8 (8x NVIDIA B300-SXM-270GB, TensorRT)","C880A_M8_B300_288GBx8_TRT","llama3.1-405b","llama3.1-405b","Offline","1971.11","ROUGEL: 21.82114944226222 exact_match: 89.9662276637988 TOKENS_PER_SAMPLE: 637.3","1","Intel(R) Xeon(R) 6767P","2","128","NVIDIA B300-SXM-270GB","8","8","closed/Cisco/results/C880A_M8_B300_288GBx8_TRT/llama3.1-405b/Offline/performance/run_1","TensorRT 10.13, CUDA 13.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Cisco","available","closed","datacenter","Cisco UCS C880A M8 (8x NVIDIA B300-SXM-270GB, TensorRT)","C880A_M8_B300_288GBx8_TRT","llama3.1-405b","llama3.1-405b","Server","1445.7","ROUGEL: 21.83555157124159 exact_match: 89.91636002647253 TOKENS_PER_SAMPLE: 659.2","1","Intel(R) Xeon(R) 6767P","2","128","NVIDIA B300-SXM-270GB","8","8","closed/Cisco/results/C880A_M8_B300_288GBx8_TRT/llama3.1-405b/Server/performance/run_1","TensorRT 10.13, CUDA 13.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Cisco","available","closed","datacenter","Cisco UCS C880A M8 (8x NVIDIA B300-SXM-270GB, TensorRT)","C880A_M8_B300_288GBx8_TRT","wan-2.2-t2v-a14b","wan-2.2-t2v-a14b","Offline","0.0573123","vbench_score: 70.1617","1","Intel(R) Xeon(R) 6767P","2","128","NVIDIA B300-SXM-270GB","8","8","closed/Cisco/results/C880A_M8_B300_288GBx8_TRT/wan-2.2-t2v-a14b/Offline/performance/run_1","TensorRT 10.13, CUDA 13.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Samples/s","fp32","","" +"Cisco","available","closed","datacenter","Cisco UCS C880A M8 (8x NVIDIA B300-SXM-270GB, TensorRT)","C880A_M8_B300_288GBx8_TRT","wan-2.2-t2v-a14b","wan-2.2-t2v-a14b","SingleStream","21.425919695","vbench_score: 70.3040","1","Intel(R) Xeon(R) 6767P","2","128","NVIDIA B300-SXM-270GB","8","8","closed/Cisco/results/C880A_M8_B300_288GBx8_TRT/wan-2.2-t2v-a14b/SingleStream/performance/run_1","TensorRT 10.13, CUDA 13.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Latency (s)","fp32","","" +"Cisco","available","closed","datacenter","Cisco UCS X210 M8 - X580P PCIe Node (4x H200-NVL-141GB, TensorRT)","X210M8_X580P_H200NVL_141GBx4_TRT","llama3.1-8b","llama3.1-8b","Offline","24216.6","ROUGE1: 38.7476 ROUGE2: 15.8938 ROUGEL: 24.4606 ROUGELSUM: 35.7751 GEN_LEN: 8209954","1","Intel(R) Xeon(R) 6747P","2","48","NVIDIA H200-NVL-141GB","4","4","closed/Cisco/results/X210M8_X580P_H200NVL_141GBx4_TRT/llama3.1-8b/Offline/performance/run_1","TensorRT 10.11, CUDA 12.9","Ubuntu 24.04","H200-NVL TGP 600W","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"Cisco","available","closed","datacenter","Cisco UCS X210 M8 - X580P PCIe Node (4x H200-NVL-141GB, TensorRT)","X210M8_X580P_H200NVL_141GBx4_TRT","llama3.1-8b","llama3.1-8b","Server","25731.6","ROUGE1: 38.7502 ROUGE2: 15.8913 ROUGEL: 24.457 ROUGELSUM: 35.7785 GEN_LEN: 8209110","1","Intel(R) Xeon(R) 6747P","2","48","NVIDIA H200-NVL-141GB","4","4","closed/Cisco/results/X210M8_X580P_H200NVL_141GBx4_TRT/llama3.1-8b/Server/performance/run_1","TensorRT 10.11, CUDA 12.9","Ubuntu 24.04","H200-NVL TGP 600W","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"Cisco","available","closed","datacenter","Cisco UCS X210 M8 - X580P PCIe Node (4x H200-NVL-141GB, TensorRT)","X210M8_X580P_H200NVL_141GBx4_TRT","whisper","whisper","Offline","15338.6","ACCURACY: 97.82350086127316","1","Intel(R) Xeon(R) 6747P","2","48","NVIDIA H200-NVL-141GB","4","4","closed/Cisco/results/X210M8_X580P_H200NVL_141GBx4_TRT/whisper/Offline/performance/run_1","TensorRT 10.11, CUDA 12.9","Ubuntu 24.04","H200-NVL TGP 600W","closed","0","v6.0","0","False","Samples/s","fp16","","" +"Cisco","available","closed","datacenter","Cisco UCS X215c M8 - X580P PCIe Node (4x RTXPro6000-PCIe-96GB, TensorRT)","X215M8_X580P_RTXPro6000_96GBx4_TRT","llama2-70b-99","llama2-70b-99","Offline","13529.8","ROUGE1: 44.7674 ROUGE2: 22.3465 ROUGEL: 29.1537 TOKENS_PER_SAMPLE: 273.2","1","AMD EPYC 9845","2","160","NVIDIA RTX PRO 6000 Blackwell Server Edition","4","4","closed/Cisco/results/X215M8_X580P_RTXPro6000_96GBx4_TRT/llama2-70b-99/Offline/performance/run_1","TensorRT 10.11, CUDA 12.9","Ubuntu 24.04","NVIDIA RTX PRO 6000 Blackwell Server Edition TGP 600W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Cisco","available","closed","datacenter","Cisco UCS X215c M8 - X580P PCIe Node (4x RTXPro6000-PCIe-96GB, TensorRT)","X215M8_X580P_RTXPro6000_96GBx4_TRT","llama2-70b-99","llama2-70b-99","Server","13029.4","ROUGE1: 44.7585 ROUGE2: 22.3383 ROUGEL: 29.1509 TOKENS_PER_SAMPLE: 273.7","1","AMD EPYC 9845","2","160","NVIDIA RTX PRO 6000 Blackwell Server Edition","4","4","closed/Cisco/results/X215M8_X580P_RTXPro6000_96GBx4_TRT/llama2-70b-99/Server/performance/run_1","TensorRT 10.11, CUDA 12.9","Ubuntu 24.04","NVIDIA RTX PRO 6000 Blackwell Server Edition TGP 600W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Cisco","available","closed","datacenter","Cisco UCS X215c M8 - X580P PCIe Node (4x RTXPro6000-PCIe-96GB, TensorRT)","X215M8_X580P_RTXPro6000_96GBx4_TRT","llama3.1-8b","llama3.1-8b","Offline","24013.5","ROUGE1: 38.5868 ROUGE2: 15.859 ROUGEL: 24.3111 ROUGELSUM: 35.538 GEN_LEN: 8181337","1","AMD EPYC 9845","2","160","NVIDIA RTX PRO 6000 Blackwell Server Edition","4","4","closed/Cisco/results/X215M8_X580P_RTXPro6000_96GBx4_TRT/llama3.1-8b/Offline/performance/run_1","TensorRT 10.11, CUDA 12.9","Ubuntu 24.04","NVIDIA RTX PRO 6000 Blackwell Server Edition TGP 600W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Cisco","available","closed","datacenter","Cisco UCS X215c M8 - X580P PCIe Node (4x RTXPro6000-PCIe-96GB, TensorRT)","X215M8_X580P_RTXPro6000_96GBx4_TRT","llama3.1-8b","llama3.1-8b","Server","23423.9","ROUGE1: 38.5862 ROUGE2: 15.8584 ROUGEL: 24.3129 ROUGELSUM: 35.5376 GEN_LEN: 8181285","1","AMD EPYC 9845","2","160","NVIDIA RTX PRO 6000 Blackwell Server Edition","4","4","closed/Cisco/results/X215M8_X580P_RTXPro6000_96GBx4_TRT/llama3.1-8b/Server/performance/run_1","TensorRT 10.11, CUDA 12.9","Ubuntu 24.04","NVIDIA RTX PRO 6000 Blackwell Server Edition TGP 600W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Cisco","available","closed","datacenter","Cisco UCS X215c M8 - X580P PCIe Node (4x RTXPro6000-PCIe-96GB, TensorRT)","X215M8_X580P_RTXPro6000_96GBx4_TRT","whisper","whisper","Offline","9099.08","ACCURACY: 97.81683907991282","1","AMD EPYC 9845","2","160","NVIDIA RTX PRO 6000 Blackwell Server Edition","4","4","closed/Cisco/results/X215M8_X580P_RTXPro6000_96GBx4_TRT/whisper/Offline/performance/run_1","TensorRT 10.11, CUDA 12.9","Ubuntu 24.04","NVIDIA RTX PRO 6000 Blackwell Server Edition TGP 600W","closed","0","v6.0","0","False","Samples/s","fp16","","" +"CoreWeave","available","closed","datacenter","CoreWeave GB200 NVL72 (64x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x64_TRT","deepseek-r1","deepseek-r1","Offline","468665.0","exact_match: 81.54056517775751 TOKENS_PER_SAMPLE: 3759.982224247949","16","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","64","closed/CoreWeave/results/GB200-NVL72_GB200-186GB_aarch64x64_TRT/deepseek-r1/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","GB200 NVL72","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"CoreWeave","available","closed","datacenter","CoreWeave GB200 NVL72 (64x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x64_TRT","deepseek-r1","deepseek-r1","Server","311531.0","exact_match: 81.26709206927985 TOKENS_PER_SAMPLE: 3818.1679580674568","16","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","64","closed/CoreWeave/results/GB200-NVL72_GB200-186GB_aarch64x64_TRT/deepseek-r1/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","GB200 NVL72","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"CoreWeave","available","closed","datacenter","CoreWeave GB200 NVL72 (64x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x64_TRT","gpt-oss-120b","gpt-oss-120b","Offline","774539.0","exact_match: 83.760","16","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","64","closed/CoreWeave/results/GB200-NVL72_GB200-186GB_aarch64x64_TRT/gpt-oss-120b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","GB200 NVL72","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"CoreWeave","available","closed","datacenter","CoreWeave GB200 NVL72 (64x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x64_TRT","gpt-oss-120b","gpt-oss-120b","Server","743550.0","exact_match: 83.085","16","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","64","closed/CoreWeave/results/GB200-NVL72_GB200-186GB_aarch64x64_TRT/gpt-oss-120b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","GB200 NVL72","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"CoreWeave","available","closed","datacenter","CoreWeave GB300 NVL72 (64x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x64_TRT","gpt-oss-120b","gpt-oss-120b","Offline","910468.0","exact_match: 83.871","16","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","64","closed/CoreWeave/results/GB300-NVL72_GB300-288GB_aarch64x64_TRT/gpt-oss-120b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","GB300 NVL72","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"CoreWeave","available","closed","datacenter","CoreWeave GB300 NVL72 (64x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x64_TRT","gpt-oss-120b","gpt-oss-120b","Server","956523.0","exact_match: 83.043","16","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","64","closed/CoreWeave/results/GB300-NVL72_GB300-288GB_aarch64x64_TRT/gpt-oss-120b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","GB300 NVL72","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"CoreWeave","available","closed","datacenter","CoreWeave GB300 NVL72 (8x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x8_TRT","deepseek-r1","deepseek-r1","Offline","78564.0","exact_match: 80.94804010938924 TOKENS_PER_SAMPLE: 3815.0455788514128","2","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","8","closed/CoreWeave/results/GB300-NVL72_GB300-288GB_aarch64x8_TRT/deepseek-r1/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","GB300 NVL72","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"CoreWeave","available","closed","datacenter","CoreWeave GB300 NVL72 (8x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x8_TRT","deepseek-r1","deepseek-r1","Server","44589.7","exact_match: 81.17593436645396 TOKENS_PER_SAMPLE: 3780.3126709206927","2","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","8","closed/CoreWeave/results/GB300-NVL72_GB300-288GB_aarch64x8_TRT/deepseek-r1/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","GB300 NVL72","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"CoreWeave","available","open","datacenter","CoreWeave GB300 NVL72 (64x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x64_TRT","deepseek-r1","deepseek-r1","Offline","627653.0","exact_match: 81.42661804922516 TOKENS_PER_SAMPLE: 3784.834092980857","16","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","64","open/CoreWeave/results/GB300-NVL72_GB300-288GB_aarch64x64_TRT/deepseek-r1/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","GB300 NVL72","open","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","1-node-2S-GNR_86C","1-node-2S-GNR_86C","llama3_1-8b","llama3.1-8b","Offline","807.675","ROUGE1: 38.8253 ROUGE2: 16.0656 ROUGEL: 24.58 ROUGELSUM: 35.9055 GEN_LEN: 8148260","1","Intel(R) Xeon(R) 6787P","2","86","N/A","0","0","closed/Dell/results/1-node-2S-GNR_86C/llama3_1-8b/Offline/performance/run_1","PyTorch","Ubuntu 24.04 LTS","Dell PowerEdge R770. INT8","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Dell","available","closed","datacenter","1-node-2S-GNR_86C","1-node-2S-GNR_86C","llama3_1-8b","llama3.1-8b","Server","245.63","ROUGE1: 38.8223 ROUGE2: 16.0515 ROUGEL: 24.5808 ROUGELSUM: 35.9033 GEN_LEN: 8145003","1","Intel(R) Xeon(R) 6787P","2","86","N/A","0","0","closed/Dell/results/1-node-2S-GNR_86C/llama3_1-8b/Server/performance/run_1","PyTorch","Ubuntu 24.04 LTS","Dell PowerEdge R770. INT8","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Dell","available","closed","datacenter","1-node-2S-GNR_86C","1-node-2S-GNR_86C","rgat","rgat","Offline","14283.7","acc: 72.540","1","Intel(R) Xeon(R) 6787P","2","86","N/A","0","0","closed/Dell/results/1-node-2S-GNR_86C/rgat/Offline/performance/run_1","PyTorch","Ubuntu 24.04 LTS","Dell PowerEdge R770. INT8","closed","0","v6.0","0","False","Samples/s","INT8","","" +"Dell","available","closed","datacenter","1-node-2S-GNR_86C","1-node-2S-GNR_86C","whisper","whisper","Offline","1414.26","ACCURACY: 97.88576267638467","1","Intel(R) Xeon(R) 6787P","2","86","N/A","0","0","closed/Dell/results/1-node-2S-GNR_86C/whisper/Offline/performance/run_1","PyTorch","Ubuntu 24.04 LTS","Dell PowerEdge R770. INT8","closed","0","v6.0","0","False","Samples/s","INT8","","" +"Dell","available","closed","datacenter","PowerEdge XE7740 (8x H200-NVL-141GB, TensorRT)","XE7740_H200_NVL_141GBx8_TRT","llama2-70b-99","llama2-70b-99","Interactive","16343.8","ROUGE1: 44.5695 ROUGE2: 22.1191 ROUGEL: 28.7481 TOKENS_PER_SAMPLE: 291.3","1","Intel(R) Xeon(R) 6747P","2","48","NVIDIA H200-NVL-141GB","8","8","closed/Dell/results/XE7740_H200_NVL_141GBx8_TRT/llama2-70b-99/Interactive/performance/run_1","TensorRT 10.13, CUDA 13.0","Ubuntu 24.04","H200 TGP 600W","closed","0","v6.0","1","False","Tokens/s","fp8","","" +"Dell","available","closed","datacenter","PowerEdge XE7740 (8x H200-NVL-141GB, TensorRT)","XE7740_H200_NVL_141GBx8_TRT","llama2-70b-99","llama2-70b-99","Offline","32004.0","ROUGE1: 44.5303 ROUGE2: 22.094 ROUGEL: 28.7029 TOKENS_PER_SAMPLE: 293.4","1","Intel(R) Xeon(R) 6747P","2","48","NVIDIA H200-NVL-141GB","8","8","closed/Dell/results/XE7740_H200_NVL_141GBx8_TRT/llama2-70b-99/Offline/performance/run_1","TensorRT 10.13, CUDA 13.0","Ubuntu 24.04","H200 TGP 600W","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"Dell","available","closed","datacenter","PowerEdge XE7740 (8x H200-NVL-141GB, TensorRT)","XE7740_H200_NVL_141GBx8_TRT","llama2-70b-99","llama2-70b-99","Server","29084.9","ROUGE1: 44.5359 ROUGE2: 22.1023 ROUGEL: 28.7072 TOKENS_PER_SAMPLE: 294.2","1","Intel(R) Xeon(R) 6747P","2","48","NVIDIA H200-NVL-141GB","8","8","closed/Dell/results/XE7740_H200_NVL_141GBx8_TRT/llama2-70b-99/Server/performance/run_1","TensorRT 10.13, CUDA 13.0","Ubuntu 24.04","H200 TGP 600W","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"Dell","available","closed","datacenter","PowerEdge XE7740 (8x H200-NVL-141GB, TensorRT)","XE7740_H200_NVL_141GBx8_TRT","llama2-70b-99.9","llama2-70b-99.9","Interactive","16343.8","ROUGE1: 44.5695 ROUGE2: 22.1191 ROUGEL: 28.7481 TOKENS_PER_SAMPLE: 291.3","1","Intel(R) Xeon(R) 6747P","2","48","NVIDIA H200-NVL-141GB","8","8","closed/Dell/results/XE7740_H200_NVL_141GBx8_TRT/llama2-70b-99.9/Interactive/performance/run_1","TensorRT 10.13, CUDA 13.0","Ubuntu 24.04","H200 TGP 600W","closed","0","v6.0","1","False","Tokens/s","fp8","","" +"Dell","available","closed","datacenter","PowerEdge XE7740 (8x H200-NVL-141GB, TensorRT)","XE7740_H200_NVL_141GBx8_TRT","llama2-70b-99.9","llama2-70b-99.9","Offline","32004.0","ROUGE1: 44.5303 ROUGE2: 22.094 ROUGEL: 28.7029 TOKENS_PER_SAMPLE: 293.4","1","Intel(R) Xeon(R) 6747P","2","48","NVIDIA H200-NVL-141GB","8","8","closed/Dell/results/XE7740_H200_NVL_141GBx8_TRT/llama2-70b-99.9/Offline/performance/run_1","TensorRT 10.13, CUDA 13.0","Ubuntu 24.04","H200 TGP 600W","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"Dell","available","closed","datacenter","PowerEdge XE7740 (8x H200-NVL-141GB, TensorRT)","XE7740_H200_NVL_141GBx8_TRT","llama2-70b-99.9","llama2-70b-99.9","Server","29084.9","ROUGE1: 44.5359 ROUGE2: 22.1023 ROUGEL: 28.7072 TOKENS_PER_SAMPLE: 294.2","1","Intel(R) Xeon(R) 6747P","2","48","NVIDIA H200-NVL-141GB","8","8","closed/Dell/results/XE7740_H200_NVL_141GBx8_TRT/llama2-70b-99.9/Server/performance/run_1","TensorRT 10.13, CUDA 13.0","Ubuntu 24.04","H200 TGP 600W","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"Dell","available","closed","datacenter","PowerEdge XE7740 (8x H200-NVL-141GB, TensorRT)","XE7740_H200_NVL_141GBx8_TRT","llama3.1-8b","llama3.1-8b","Offline","53193.6","ROUGE1: 38.7788 ROUGE2: 16.0073 ROUGEL: 24.5878 ROUGELSUM: 35.8159 GEN_LEN: 8207624","1","Intel(R) Xeon(R) 6747P","2","48","NVIDIA H200-NVL-141GB","8","8","closed/Dell/results/XE7740_H200_NVL_141GBx8_TRT/llama3.1-8b/Offline/performance/run_1","TensorRT 10.13, CUDA 13.0","Ubuntu 24.04","H200 TGP 600W","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"Dell","available","closed","datacenter","PowerEdge XE7740 (8x H200-NVL-141GB, TensorRT)","XE7740_H200_NVL_141GBx8_TRT","llama3.1-8b","llama3.1-8b","Server","45075.8","ROUGE1: 38.8383 ROUGE2: 16.0476 ROUGEL: 24.5941 ROUGELSUM: 35.8728 GEN_LEN: 8204099","1","Intel(R) Xeon(R) 6747P","2","48","NVIDIA H200-NVL-141GB","8","8","closed/Dell/results/XE7740_H200_NVL_141GBx8_TRT/llama3.1-8b/Server/performance/run_1","TensorRT 10.13, CUDA 13.0","Ubuntu 24.04","H200 TGP 600W","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"Dell","available","closed","datacenter","PowerEdge XE7740 (8x H200-NVL-141GB, TensorRT)","XE7740_H200_NVL_141GBx8_TRT","whisper","whisper","Offline","31969.2","ACCURACY: 97.8625198663837","1","Intel(R) Xeon(R) 6747P","2","48","NVIDIA H200-NVL-141GB","8","8","closed/Dell/results/XE7740_H200_NVL_141GBx8_TRT/whisper/Offline/performance/run_1","TensorRT 10.13, CUDA 13.0","Ubuntu 24.04","H200 TGP 600W","closed","0","v6.0","0","False","Samples/s","fp16","","" +"Dell","available","closed","datacenter","PowerEdge XE7740 (8x RTXPro6000-PCIe-96GB, TensorRT)","XE7740_RTXPro6000_PCIe_96GBx8_TRT","llama2-70b-99","llama2-70b-99","Offline","27034.8","ROUGE1: 44.8818 ROUGE2: 22.4053 ROUGEL: 29.202 TOKENS_PER_SAMPLE: 272.0","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/Dell/results/XE7740_RTXPro6000_PCIe_96GBx8_TRT/llama2-70b-99/Offline/performance/run_1","TensorRT 10.14, CUDA 13.0","Ubuntu 24.04","NVIDIA RTX PRO 6000 Blackwell Server Edition TGP 600W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE7740 (8x RTXPro6000-PCIe-96GB, TensorRT)","XE7740_RTXPro6000_PCIe_96GBx8_TRT","llama2-70b-99","llama2-70b-99","Server","25524.1","ROUGE1: 44.8742 ROUGE2: 22.4134 ROUGEL: 29.2014 TOKENS_PER_SAMPLE: 272.8","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/Dell/results/XE7740_RTXPro6000_PCIe_96GBx8_TRT/llama2-70b-99/Server/performance/run_1","TensorRT 10.14, CUDA 13.0","Ubuntu 24.04","NVIDIA RTX PRO 6000 Blackwell Server Edition TGP 600W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE7740 (8x RTXPro6000-PCIe-96GB, TensorRT)","XE7740_RTXPro6000_PCIe_96GBx8_TRT","llama2-70b-99.9","llama2-70b-99.9","Offline","27034.8","ROUGE1: 44.8818 ROUGE2: 22.4053 ROUGEL: 29.202 TOKENS_PER_SAMPLE: 272.0","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/Dell/results/XE7740_RTXPro6000_PCIe_96GBx8_TRT/llama2-70b-99.9/Offline/performance/run_1","TensorRT 10.14, CUDA 13.0","Ubuntu 24.04","NVIDIA RTX PRO 6000 Blackwell Server Edition TGP 600W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE7740 (8x RTXPro6000-PCIe-96GB, TensorRT)","XE7740_RTXPro6000_PCIe_96GBx8_TRT","llama2-70b-99.9","llama2-70b-99.9","Server","25524.1","ROUGE1: 44.8742 ROUGE2: 22.4134 ROUGEL: 29.2014 TOKENS_PER_SAMPLE: 272.8","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/Dell/results/XE7740_RTXPro6000_PCIe_96GBx8_TRT/llama2-70b-99.9/Server/performance/run_1","TensorRT 10.14, CUDA 13.0","Ubuntu 24.04","NVIDIA RTX PRO 6000 Blackwell Server Edition TGP 600W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE7740 (8x RTXPro6000-PCIe-96GB, TensorRT)","XE7740_RTXPro6000_PCIe_96GBx8_TRT","whisper","whisper","Offline","18231.5","ACCURACY: 97.85205135281745","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/Dell/results/XE7740_RTXPro6000_PCIe_96GBx8_TRT/whisper/Offline/performance/run_1","TensorRT 10.14, CUDA 13.0","Ubuntu 24.04","NVIDIA RTX PRO 6000 Blackwell Server Edition TGP 600W","closed","0","v6.0","0","False","Samples/s","fp16","","" +"Dell","available","closed","datacenter","PowerEdge XE7745 (8x RTXPro6000-PCIe-96GB, TensorRT)","XE7745_RTXPro6000_PCIe_96GBx8_TRT","llama2-70b-99","llama2-70b-99","Offline","26829.8","ROUGE1: 44.8777 ROUGE2: 22.408 ROUGEL: 29.1982 TOKENS_PER_SAMPLE: 272.0","1","AMD EPYC 9575F","2","64","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/Dell/results/XE7745_RTXPro6000_PCIe_96GBx8_TRT/llama2-70b-99/Offline/performance/run_1","TensorRT 10.14, CUDA 13.0","Ubuntu 24.04","NVIDIA RTX PRO 6000 Blackwell Server Edition TGP 600W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE7745 (8x RTXPro6000-PCIe-96GB, TensorRT)","XE7745_RTXPro6000_PCIe_96GBx8_TRT","llama2-70b-99","llama2-70b-99","Server","25536.1","ROUGE1: 44.8697 ROUGE2: 22.4115 ROUGEL: 29.1961 TOKENS_PER_SAMPLE: 272.9","1","AMD EPYC 9575F","2","64","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/Dell/results/XE7745_RTXPro6000_PCIe_96GBx8_TRT/llama2-70b-99/Server/performance/run_1","TensorRT 10.14, CUDA 13.0","Ubuntu 24.04","NVIDIA RTX PRO 6000 Blackwell Server Edition TGP 600W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE7745 (8x RTXPro6000-PCIe-96GB, TensorRT)","XE7745_RTXPro6000_PCIe_96GBx8_TRT","llama2-70b-99.9","llama2-70b-99.9","Offline","26829.8","ROUGE1: 44.8777 ROUGE2: 22.408 ROUGEL: 29.1982 TOKENS_PER_SAMPLE: 272.0","1","AMD EPYC 9575F","2","64","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/Dell/results/XE7745_RTXPro6000_PCIe_96GBx8_TRT/llama2-70b-99.9/Offline/performance/run_1","TensorRT 10.14, CUDA 13.0","Ubuntu 24.04","NVIDIA RTX PRO 6000 Blackwell Server Edition TGP 600W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE7745 (8x RTXPro6000-PCIe-96GB, TensorRT)","XE7745_RTXPro6000_PCIe_96GBx8_TRT","llama2-70b-99.9","llama2-70b-99.9","Server","25536.1","ROUGE1: 44.8697 ROUGE2: 22.4115 ROUGEL: 29.1961 TOKENS_PER_SAMPLE: 272.9","1","AMD EPYC 9575F","2","64","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/Dell/results/XE7745_RTXPro6000_PCIe_96GBx8_TRT/llama2-70b-99.9/Server/performance/run_1","TensorRT 10.14, CUDA 13.0","Ubuntu 24.04","NVIDIA RTX PRO 6000 Blackwell Server Edition TGP 600W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE7745 (8x RTXPro6000-PCIe-96GB, TensorRT)","XE7745_RTXPro6000_PCIe_96GBx8_TRT","llama3.1-8b","llama3.1-8b","Offline","49036.1","ROUGE1: 38.7694 ROUGE2: 16.0138 ROUGEL: 24.531 ROUGELSUM: 35.6934 GEN_LEN: 8179110","1","AMD EPYC 9575F","2","64","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/Dell/results/XE7745_RTXPro6000_PCIe_96GBx8_TRT/llama3.1-8b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.0","Ubuntu 24.04","NVIDIA RTX PRO 6000 Blackwell Server Edition TGP 600W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE7745 (8x RTXPro6000-PCIe-96GB, TensorRT)","XE7745_RTXPro6000_PCIe_96GBx8_TRT","llama3.1-8b","llama3.1-8b","Server","45008.6","ROUGE1: 38.7655 ROUGE2: 16.0173 ROUGEL: 24.5235 ROUGELSUM: 35.6909 GEN_LEN: 8178701","1","AMD EPYC 9575F","2","64","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/Dell/results/XE7745_RTXPro6000_PCIe_96GBx8_TRT/llama3.1-8b/Server/performance/run_1","TensorRT 10.14, CUDA 13.0","Ubuntu 24.04","NVIDIA RTX PRO 6000 Blackwell Server Edition TGP 600W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE7745 (8x RTXPro6000-PCIe-96GB, TensorRT)","XE7745_RTXPro6000_PCIe_96GBx8_TRT","whisper","whisper","Offline","18433.5","ACCURACY: 97.86156818333222","1","AMD EPYC 9575F","2","64","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/Dell/results/XE7745_RTXPro6000_PCIe_96GBx8_TRT/whisper/Offline/performance/run_1","TensorRT 10.14, CUDA 13.0","Ubuntu 24.04","NVIDIA RTX PRO 6000 Blackwell Server Edition TGP 600W","closed","0","v6.0","0","False","Samples/s","fp16","","" +"Dell","available","closed","datacenter","PowerEdge XE9680 (8x MI300X)","8xMI300X_2xPlatinum_8470","llama2-70b-99","llama2-70b-99","Offline","27072.5","ROUGE1: 44.4421 ROUGE2: 22.0546 ROUGEL: 28.6036 TOKENS_PER_SAMPLE: 301.2","1","Intel(R) Xeon(R) Platinum 8470","2","52","AMD Instinct MI300X 192GB HBM3","8","8","closed/Dell/results/8xMI300X_2xPlatinum_8470/llama2-70b-99/Offline/performance/run_1","vLLM 0.9.0.2.dev108+g71faa1880.d20260213, Pytorch 2.7.0+gitf717b2a, ROCm 6.4.1","Ubuntu 22.04.5 LTS","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"Dell","available","closed","datacenter","PowerEdge XE9680 (8x MI300X)","8xMI300X_2xPlatinum_8470","llama2-70b-99","llama2-70b-99","Server","24467.7","ROUGE1: 44.4296 ROUGE2: 22.0707 ROUGEL: 28.6339 TOKENS_PER_SAMPLE: 301.2","1","Intel(R) Xeon(R) Platinum 8470","2","52","AMD Instinct MI300X 192GB HBM3","8","8","closed/Dell/results/8xMI300X_2xPlatinum_8470/llama2-70b-99/Server/performance/run_1","vLLM 0.9.0.2.dev108+g71faa1880.d20260213, Pytorch 2.7.0+gitf717b2a, ROCm 6.4.1","Ubuntu 22.04.5 LTS","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"Dell","available","closed","datacenter","PowerEdge XE9680 (8x MI300X)","8xMI300X_2xPlatinum_8470","llama2-70b-99.9","llama2-70b-99.9","Offline","27072.5","ROUGE1: 44.4421 ROUGE2: 22.0546 ROUGEL: 28.6036 TOKENS_PER_SAMPLE: 301.2","1","Intel(R) Xeon(R) Platinum 8470","2","52","AMD Instinct MI300X 192GB HBM3","8","8","closed/Dell/results/8xMI300X_2xPlatinum_8470/llama2-70b-99.9/Offline/performance/run_1","vLLM 0.9.0.2.dev108+g71faa1880.d20260213, Pytorch 2.7.0+gitf717b2a, ROCm 6.4.1","Ubuntu 22.04.5 LTS","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"Dell","available","closed","datacenter","PowerEdge XE9680 (8x MI300X)","8xMI300X_2xPlatinum_8470","llama2-70b-99.9","llama2-70b-99.9","Server","24467.7","ROUGE1: 44.4296 ROUGE2: 22.0707 ROUGEL: 28.6339 TOKENS_PER_SAMPLE: 301.2","1","Intel(R) Xeon(R) Platinum 8470","2","52","AMD Instinct MI300X 192GB HBM3","8","8","closed/Dell/results/8xMI300X_2xPlatinum_8470/llama2-70b-99.9/Server/performance/run_1","vLLM 0.9.0.2.dev108+g71faa1880.d20260213, Pytorch 2.7.0+gitf717b2a, ROCm 6.4.1","Ubuntu 22.04.5 LTS","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"Dell","available","closed","datacenter","PowerEdge XE9780 (8x B200-SXM-180GB, TensorRT)","XE9780_x86_B200-SXM-180GBx8_TRT","deepseek-r1","deepseek-r1","Offline","58235.4","exact_match: 81.1531449407475 TOKENS_PER_SAMPLE: 3727.768231540565","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B200-SXM-180GB","8","8","closed/Dell/results/XE9780_x86_B200-SXM-180GBx8_TRT/deepseek-r1/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B200 TGP 1000W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE9780 (8x B200-SXM-180GB, TensorRT)","XE9780_x86_B200-SXM-180GBx8_TRT","deepseek-r1","deepseek-r1","Server","40949.5","exact_match: 81.0619872379216 TOKENS_PER_SAMPLE: 3741.5695077484047","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B200-SXM-180GB","8","8","closed/Dell/results/XE9780_x86_B200-SXM-180GBx8_TRT/deepseek-r1/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B200 TGP 1000W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE9780 (8x B300-SXM-270GB, TensorRT)","XE9780_x86_B300-SXM-270GBx8_TRT","deepseek-r1","deepseek-r1","Offline","69021.2","exact_match: 81.13035551504102 TOKENS_PER_SAMPLE: 3753.5875113947127","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/Dell/results/XE9780_x86_B300-SXM-270GBx8_TRT/deepseek-r1/Offline/performance/run_1","TensorRT 10.14, CUDA 13.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE9780 (8x B300-SXM-270GB, TensorRT)","XE9780_x86_B300-SXM-270GBx8_TRT","deepseek-r1","deepseek-r1","Server","46513.1","exact_match: 81.13035551504102 TOKENS_PER_SAMPLE: 3712.944165907019","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/Dell/results/XE9780_x86_B300-SXM-270GBx8_TRT/deepseek-r1/Server/performance/run_1","TensorRT 10.14, CUDA 13.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE9780L (8x B300-SXM-270GB, TensorRT)","XE9780L-B300-SXM-270GBx8_TRT","deepseek-r1","deepseek-r1","Offline","66287.9","exact_match: 81.63172288058341 TOKENS_PER_SAMPLE: 3764.0123062898815","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA B300-SXM-270GB","8","8","closed/Dell/results/XE9780L-B300-SXM-270GBx8_TRT/deepseek-r1/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE9780L (8x B300-SXM-270GB, TensorRT)","XE9780L-B300-SXM-270GBx8_TRT","deepseek-r1","deepseek-r1","Server","46510.6","exact_match: 81.35824977210574 TOKENS_PER_SAMPLE: 3716.6032360984505","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA B300-SXM-270GB","8","8","closed/Dell/results/XE9780L-B300-SXM-270GBx8_TRT/deepseek-r1/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE9780L (8x B300-SXM-270GB, TensorRT)","XE9780L-B300-SXM-270GBx8_TRT","llama2-70b-99","llama2-70b-99","Interactive","70724.3","ROUGE1: 44.8183 ROUGE2: 22.3646 ROUGEL: 29.1643 TOKENS_PER_SAMPLE: 274.0","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA B300-SXM-270GB","8","8","closed/Dell/results/XE9780L-B300-SXM-270GBx8_TRT/llama2-70b-99/Interactive/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE9780L (8x B300-SXM-270GB, TensorRT)","XE9780L-B300-SXM-270GBx8_TRT","llama2-70b-99","llama2-70b-99","Offline","108669.0","ROUGE1: 44.7649 ROUGE2: 22.3597 ROUGEL: 29.1343 TOKENS_PER_SAMPLE: 272.1","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA B300-SXM-270GB","8","8","closed/Dell/results/XE9780L-B300-SXM-270GBx8_TRT/llama2-70b-99/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE9780L (8x B300-SXM-270GB, TensorRT)","XE9780L-B300-SXM-270GBx8_TRT","llama2-70b-99","llama2-70b-99","Server","108392.0","ROUGE1: 44.7575 ROUGE2: 22.354 ROUGEL: 29.1306 TOKENS_PER_SAMPLE: 273.0","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA B300-SXM-270GB","8","8","closed/Dell/results/XE9780L-B300-SXM-270GBx8_TRT/llama2-70b-99/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE9780L (8x B300-SXM-270GB, TensorRT)","XE9780L-B300-SXM-270GBx8_TRT","llama2-70b-99.9","llama2-70b-99.9","Interactive","70724.3","ROUGE1: 44.8183 ROUGE2: 22.3646 ROUGEL: 29.1643 TOKENS_PER_SAMPLE: 274.0","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA B300-SXM-270GB","8","8","closed/Dell/results/XE9780L-B300-SXM-270GBx8_TRT/llama2-70b-99.9/Interactive/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE9780L (8x B300-SXM-270GB, TensorRT)","XE9780L-B300-SXM-270GBx8_TRT","llama2-70b-99.9","llama2-70b-99.9","Offline","108669.0","ROUGE1: 44.7649 ROUGE2: 22.3597 ROUGEL: 29.1343 TOKENS_PER_SAMPLE: 272.1","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA B300-SXM-270GB","8","8","closed/Dell/results/XE9780L-B300-SXM-270GBx8_TRT/llama2-70b-99.9/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE9780L (8x B300-SXM-270GB, TensorRT)","XE9780L-B300-SXM-270GBx8_TRT","llama2-70b-99.9","llama2-70b-99.9","Server","108392.0","ROUGE1: 44.7575 ROUGE2: 22.354 ROUGEL: 29.1306 TOKENS_PER_SAMPLE: 273.0","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA B300-SXM-270GB","8","8","closed/Dell/results/XE9780L-B300-SXM-270GBx8_TRT/llama2-70b-99.9/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE9780L (8x B300-SXM-270GB, TensorRT)","XE9780L_x86_B300-SXM-270GBx8_TRT","whisper","whisper","Offline","45658.8","ACCURACY: 97.84063115619975","1","Intel(R) Xeon(R) 6979P","2","86","NVIDIA B300-SXM-270GB","8","8","closed/Dell/results/XE9780L_x86_B300-SXM-270GBx8_TRT/whisper/Offline/performance/run_1","TensorRT 10.14, CUDA 13.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Samples/s","fp16","","" +"Dell","available","closed","datacenter","PowerEdge XE9785L (8x MI355X)","8xMI355X_2xEPYC_9655","gpt-oss-120b","gpt-oss-120b","Offline","91395.5","exact_match: 83.251","1","AMD EPYC 9655","2","96","AMD Instinct MI355X 288GB HBM3e","8","8","closed/Dell/results/8xMI355X_2xEPYC_9655/gpt-oss-120b/Offline/performance/run_1","vLLM 0.14.0rc3.dev30+gb026cf14e.d20260212.rocm700, Pytorch 2.9.1+git8907517, ROCm 7.0.0.70000-38~22.04 ","Ubuntu 24.04.3 LTS","MI355X TGP 1400W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE9785L (8x MI355X)","8xMI355X_2xEPYC_9655","gpt-oss-120b","gpt-oss-120b","Server","78603.5","exact_match: 83.122","1","AMD EPYC 9655","2","96","AMD Instinct MI355X 288GB HBM3e","8","8","closed/Dell/results/8xMI355X_2xEPYC_9655/gpt-oss-120b/Server/performance/run_1","vLLM 0.14.0rc3.dev30+gb026cf14e.d20260212.rocm700, Pytorch 2.9.1+git8907517, ROCm 7.0.0.70000-38~22.04 ","Ubuntu 24.04.3 LTS","MI355X TGP 1400W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE9785L (8x MI355X)","8xMI355X_2xEPYC_9655","llama2-70b-99","llama2-70b-99","Interactive","70610.8","ROUGE1: 44.5403 ROUGE2: 22.1342 ROUGEL: 28.9108 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9655","2","96","AMD Instinct MI355X 288GB HBM3e","8","8","closed/Dell/results/8xMI355X_2xEPYC_9655/llama2-70b-99/Interactive/performance/run_1","vLLM 0.14.0rc3.dev30+gb026cf14e.d20260212.rocm700, Pytorch 2.9.1+git8907517, ROCm 7.0.0.70000-38~22.04 ","Ubuntu 24.04.3 LTS","MI355X TGP 1400W","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE9785L (8x MI355X)","8xMI355X_2xEPYC_9655","llama2-70b-99","llama2-70b-99","Offline","100008.0","ROUGE1: 44.458 ROUGE2: 22.0859 ROUGEL: 28.8585 TOKENS_PER_SAMPLE: 284.2","1","AMD EPYC 9655","2","96","AMD Instinct MI355X 288GB HBM3e","8","8","closed/Dell/results/8xMI355X_2xEPYC_9655/llama2-70b-99/Offline/performance/run_1","vLLM 0.14.0rc3.dev30+gb026cf14e.d20260212.rocm700, Pytorch 2.9.1+git8907517, ROCm 7.0.0.70000-38~22.04 ","Ubuntu 24.04.3 LTS","MI355X TGP 1400W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE9785L (8x MI355X)","8xMI355X_2xEPYC_9655","llama2-70b-99","llama2-70b-99","Server","96734.4","ROUGE1: 44.5361 ROUGE2: 22.132 ROUGEL: 28.9076 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9655","2","96","AMD Instinct MI355X 288GB HBM3e","8","8","closed/Dell/results/8xMI355X_2xEPYC_9655/llama2-70b-99/Server/performance/run_1","vLLM 0.14.0rc3.dev30+gb026cf14e.d20260212.rocm700, Pytorch 2.9.1+git8907517, ROCm 7.0.0.70000-38~22.04 ","Ubuntu 24.04.3 LTS","MI355X TGP 1400W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE9785L (8x MI355X)","8xMI355X_2xEPYC_9655","llama2-70b-99.9","llama2-70b-99.9","Interactive","70610.8","ROUGE1: 44.5403 ROUGE2: 22.1342 ROUGEL: 28.9108 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9655","2","96","AMD Instinct MI355X 288GB HBM3e","8","8","closed/Dell/results/8xMI355X_2xEPYC_9655/llama2-70b-99.9/Interactive/performance/run_1","vLLM 0.14.0rc3.dev30+gb026cf14e.d20260212.rocm700, Pytorch 2.9.1+git8907517, ROCm 7.0.0.70000-38~22.04 ","Ubuntu 24.04.3 LTS","MI355X TGP 1400W","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE9785L (8x MI355X)","8xMI355X_2xEPYC_9655","llama2-70b-99.9","llama2-70b-99.9","Offline","100008.0","ROUGE1: 44.458 ROUGE2: 22.0859 ROUGEL: 28.8585 TOKENS_PER_SAMPLE: 284.2","1","AMD EPYC 9655","2","96","AMD Instinct MI355X 288GB HBM3e","8","8","closed/Dell/results/8xMI355X_2xEPYC_9655/llama2-70b-99.9/Offline/performance/run_1","vLLM 0.14.0rc3.dev30+gb026cf14e.d20260212.rocm700, Pytorch 2.9.1+git8907517, ROCm 7.0.0.70000-38~22.04 ","Ubuntu 24.04.3 LTS","MI355X TGP 1400W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","datacenter","PowerEdge XE9785L (8x MI355X)","8xMI355X_2xEPYC_9655","llama2-70b-99.9","llama2-70b-99.9","Server","96734.4","ROUGE1: 44.5361 ROUGE2: 22.132 ROUGEL: 28.9076 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9655","2","96","AMD Instinct MI355X 288GB HBM3e","8","8","closed/Dell/results/8xMI355X_2xEPYC_9655/llama2-70b-99.9/Server/performance/run_1","vLLM 0.14.0rc3.dev30+gb026cf14e.d20260212.rocm700, Pytorch 2.9.1+git8907517, ROCm 7.0.0.70000-38~22.04 ","Ubuntu 24.04.3 LTS","MI355X TGP 1400W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Dell","available","closed","edge","Dell Pro Max (1x GB10)","Dell_Pro_Max_GB10x1","yolo-95","yolo-95","multistream","4891.380208","mAP: 53.108","1","Cortex-X925","20","1","NVIDIA GB10","1","1","closed/Dell/results/Dell_Pro_Max_GB10x1/yolo-95/multistream/performance/run_1","CUDA 13.0","Ubuntu 24.04.3 LTS","Dell Pro Max with GB10. CPU only submission","closed","0","v6.0","1","False","Latency (ms)","fp4","","" +"Dell","available","closed","edge","Dell Pro Max (1x GB10)","Dell_Pro_Max_GB10x1","yolo-95","yolo-95","offline","1.74887","mAP: 53.108","1","Cortex-X925","20","1","NVIDIA GB10","1","1","closed/Dell/results/Dell_Pro_Max_GB10x1/yolo-95/offline/performance/run_1","CUDA 13.0","Ubuntu 24.04.3 LTS","Dell Pro Max with GB10. CPU only submission","closed","0","v6.0","1","False","Samples/s","fp4","","" +"Dell","available","closed","edge","Dell Pro Max (1x GB10)","Dell_Pro_Max_GB10x1","yolo-95","yolo-95","singlestream","611.926848","mAP: 53.108","1","Cortex-X925","20","1","NVIDIA GB10","1","1","closed/Dell/results/Dell_Pro_Max_GB10x1/yolo-95/singlestream/performance/run_1","CUDA 13.0","Ubuntu 24.04.3 LTS","Dell Pro Max with GB10. CPU only submission","closed","0","v6.0","1","False","Latency (ms)","fp4","","" +"Dell","available","closed","edge","Dell Pro Max (1x GB10)","Dell_Pro_Max_GB10x1","yolo-99","yolo-99","multistream","4891.380208","mAP: 53.108","1","Cortex-X925","20","1","NVIDIA GB10","1","1","closed/Dell/results/Dell_Pro_Max_GB10x1/yolo-99/multistream/performance/run_1","CUDA 13.0","Ubuntu 24.04.3 LTS","Dell Pro Max with GB10. CPU only submission","closed","0","v6.0","1","False","Latency (ms)","fp4","","" +"Dell","available","closed","edge","Dell Pro Max (1x GB10)","Dell_Pro_Max_GB10x1","yolo-99","yolo-99","offline","1.74887","mAP: 53.108","1","Cortex-X925","20","1","NVIDIA GB10","1","1","closed/Dell/results/Dell_Pro_Max_GB10x1/yolo-99/offline/performance/run_1","CUDA 13.0","Ubuntu 24.04.3 LTS","Dell Pro Max with GB10. CPU only submission","closed","0","v6.0","1","False","Samples/s","fp4","","" +"Dell","available","closed","edge","Dell Pro Max (1x GB10)","Dell_Pro_Max_GB10x1","yolo-99","yolo-99","singlestream","611.926848","mAP: 53.108","1","Cortex-X925","20","1","NVIDIA GB10","1","1","closed/Dell/results/Dell_Pro_Max_GB10x1/yolo-99/singlestream/performance/run_1","CUDA 13.0","Ubuntu 24.04.3 LTS","Dell Pro Max with GB10. CPU only submission","closed","0","v6.0","1","False","Latency (ms)","fp4","","" +"Dell","available","closed","edge","PowerEdge XE7745 (16x L4-PCIE-24GB, TensorRT)","XE7745_L4x16","yolo-95","yolo-95","multistream","81.093026","mAP: 53.110","1","AMD EPYC 9965","1","86","NVIDIA L4-PCIe-24GB","16","16","closed/Dell/results/XE7745_L4x16/yolo-95/multistream/performance/run_1","CUDA 13.1","Ubuntu 24.04.02","","closed","0","v6.0","1","False","Latency (ms)","fp4","","" +"Dell","available","closed","edge","PowerEdge XE7745 (16x L4-PCIE-24GB, TensorRT)","XE7745_L4x16","yolo-95","yolo-95","offline","102.232","mAP: 53.110","1","AMD EPYC 9965","1","86","NVIDIA L4-PCIe-24GB","16","16","closed/Dell/results/XE7745_L4x16/yolo-95/offline/performance/run_1","CUDA 13.1","Ubuntu 24.04.02","","closed","0","v6.0","1","False","Samples/s","fp4","","" +"Dell","available","closed","edge","PowerEdge XE7745 (16x L4-PCIE-24GB, TensorRT)","XE7745_L4x16","yolo-95","yolo-95","singlestream","10.54864","mAP: 53.110","1","AMD EPYC 9965","1","86","NVIDIA L4-PCIe-24GB","16","16","closed/Dell/results/XE7745_L4x16/yolo-95/singlestream/performance/run_1","CUDA 13.1","Ubuntu 24.04.02","","closed","0","v6.0","1","False","Latency (ms)","fp4","","" +"Dell","available","closed","edge","PowerEdge XE7745 (16x L4-PCIE-24GB, TensorRT)","XE7745_L4x16","yolo-99","yolo-99","multistream","81.093026","mAP: 53.110","1","AMD EPYC 9965","1","86","NVIDIA L4-PCIe-24GB","16","16","closed/Dell/results/XE7745_L4x16/yolo-99/multistream/performance/run_1","CUDA 13.1","Ubuntu 24.04.02","","closed","0","v6.0","1","False","Latency (ms)","fp4","","" +"Dell","available","closed","edge","PowerEdge XE7745 (16x L4-PCIE-24GB, TensorRT)","XE7745_L4x16","yolo-99","yolo-99","offline","102.232","mAP: 53.110","1","AMD EPYC 9965","1","86","NVIDIA L4-PCIe-24GB","16","16","closed/Dell/results/XE7745_L4x16/yolo-99/offline/performance/run_1","CUDA 13.1","Ubuntu 24.04.02","","closed","0","v6.0","1","False","Samples/s","fp4","","" +"Dell","available","closed","edge","PowerEdge XE7745 (16x L4-PCIE-24GB, TensorRT)","XE7745_L4x16","yolo-99","yolo-99","singlestream","10.54864","mAP: 53.110","1","AMD EPYC 9965","1","86","NVIDIA L4-PCIe-24GB","16","16","closed/Dell/results/XE7745_L4x16/yolo-99/singlestream/performance/run_1","CUDA 13.1","Ubuntu 24.04.02","","closed","0","v6.0","1","False","Latency (ms)","fp4","","" +"Dell","available","closed","edge","PowerEdge XE7745 (8x L40S, TensorRT)","XE7745_L40Sx8","yolo-95","yolo-95","multistream","54.674812","mAP: 53.116","1"," AMD EPYC 9655","2","96","NVIDIA L40S","8","8","closed/Dell/results/XE7745_L40Sx8/yolo-95/multistream/performance/run_1","TensorRT 10.14, CUDA 12.9","Ubuntu 24.04","L40S TGP 350W","closed","0","v6.0","1","False","Latency (ms)","fp4","","" +"Dell","available","closed","edge","PowerEdge XE7745 (8x L40S, TensorRT)","XE7745_L40Sx8","yolo-95","yolo-95","offline","153.544","mAP: 53.116","1"," AMD EPYC 9655","2","96","NVIDIA L40S","8","8","closed/Dell/results/XE7745_L40Sx8/yolo-95/offline/performance/run_1","TensorRT 10.14, CUDA 12.9","Ubuntu 24.04","L40S TGP 350W","closed","0","v6.0","1","False","Samples/s","fp4","","" +"Dell","available","closed","edge","PowerEdge XE7745 (8x L40S, TensorRT)","XE7745_L40Sx8","yolo-95","yolo-95","singlestream","7.124808","mAP: 53.116","1"," AMD EPYC 9655","2","96","NVIDIA L40S","8","8","closed/Dell/results/XE7745_L40Sx8/yolo-95/singlestream/performance/run_1","TensorRT 10.14, CUDA 12.9","Ubuntu 24.04","L40S TGP 350W","closed","0","v6.0","1","False","Latency (ms)","fp4","","" +"Dell","available","closed","edge","PowerEdge XE7745 (8x L40S, TensorRT)","XE7745_L40Sx8","yolo-99","yolo-99","multistream","54.674812","mAP: 53.116","1"," AMD EPYC 9655","2","96","NVIDIA L40S","8","8","closed/Dell/results/XE7745_L40Sx8/yolo-99/multistream/performance/run_1","TensorRT 10.14, CUDA 12.9","Ubuntu 24.04","L40S TGP 350W","closed","0","v6.0","1","False","Latency (ms)","fp4","","" +"Dell","available","closed","edge","PowerEdge XE7745 (8x L40S, TensorRT)","XE7745_L40Sx8","yolo-99","yolo-99","offline","153.544","mAP: 53.116","1"," AMD EPYC 9655","2","96","NVIDIA L40S","8","8","closed/Dell/results/XE7745_L40Sx8/yolo-99/offline/performance/run_1","TensorRT 10.14, CUDA 12.9","Ubuntu 24.04","L40S TGP 350W","closed","0","v6.0","1","False","Samples/s","fp4","","" +"Dell","available","closed","edge","PowerEdge XE7745 (8x L40S, TensorRT)","XE7745_L40Sx8","yolo-99","yolo-99","singlestream","7.124808","mAP: 53.116","1"," AMD EPYC 9655","2","96","NVIDIA L40S","8","8","closed/Dell/results/XE7745_L40Sx8/yolo-99/singlestream/performance/run_1","TensorRT 10.14, CUDA 12.9","Ubuntu 24.04","L40S TGP 350W","closed","0","v6.0","1","False","Latency (ms)","fp4","","" +"Dell_MangoBoost","available","closed","datacenter","Dell_MangoBoost Heterogeneous Cluster (8x MI300X + 8x MI325X + 8x MI355X, LLMBoost)","8xMI300X_2xEPYC_9534_8xMI325X_2xEPYC_9655_8xMI355X_2xEPYC_9965","llama2-70b-99","llama2-70b-99","Offline","151843.0","ROUGE1: 44.4975 ROUGE2: 22.0735 ROUGEL: 28.8141 TOKENS_PER_SAMPLE: 287.3","3","AMD EPYC 9534 (on MI300X node), AMD EPYC 9655 (on MI325X node), AMD EPYC 9965 (on MI355X node)","2","64(AMD EPYC 9534) and 96(AMD EPYC 9655) and 192(AMD EPYC 9965)","AMD Instinct MI300X 192GB HBM3 (x8) and AMD Instinct MI325X 256GB HBM3e (x8) and AMD Instinct MI355X 288GB HBM3e (x8)","8","24","closed/Dell_MangoBoost/results/8xMI300X_2xEPYC_9534_8xMI325X_2xEPYC_9655_8xMI355X_2xEPYC_9965/llama2-70b-99/Offline/performance/run_1","ROCm","Ubuntu 22.04.5 LTS (jammy)","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"Dell_MangoBoost","available","closed","datacenter","Dell_MangoBoost Heterogeneous Cluster (8x MI300X + 8x MI325X + 8x MI355X, LLMBoost)","8xMI300X_2xEPYC_9534_8xMI325X_2xEPYC_9655_8xMI355X_2xEPYC_9965","llama2-70b-99","llama2-70b-99","Server","141521.0","ROUGE1: 44.543 ROUGE2: 22.0969 ROUGEL: 28.8312 TOKENS_PER_SAMPLE: 287.0","3","AMD EPYC 9534 (on MI300X node), AMD EPYC 9655 (on MI325X node), AMD EPYC 9965 (on MI355X node)","2","64(AMD EPYC 9534) and 96(AMD EPYC 9655) and 192(AMD EPYC 9965)","AMD Instinct MI300X 192GB HBM3 (x8) and AMD Instinct MI325X 256GB HBM3e (x8) and AMD Instinct MI355X 288GB HBM3e (x8)","8","24","closed/Dell_MangoBoost/results/8xMI300X_2xEPYC_9534_8xMI325X_2xEPYC_9655_8xMI355X_2xEPYC_9965/llama2-70b-99/Server/performance/run_1","ROCm","Ubuntu 22.04.5 LTS (jammy)","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"Dell_MangoBoost","available","closed","datacenter","Dell_MangoBoost Heterogeneous Cluster (8x MI300X + 8x MI325X + 8x MI355X, LLMBoost)","8xMI300X_2xEPYC_9534_8xMI325X_2xEPYC_9655_8xMI355X_2xEPYC_9965","llama2-70b-99.9","llama2-70b-99.9","Offline","151843.0","ROUGE1: 44.4975 ROUGE2: 22.0735 ROUGEL: 28.8141 TOKENS_PER_SAMPLE: 287.3","3","AMD EPYC 9534 (on MI300X node), AMD EPYC 9655 (on MI325X node), AMD EPYC 9965 (on MI355X node)","2","64(AMD EPYC 9534) and 96(AMD EPYC 9655) and 192(AMD EPYC 9965)","AMD Instinct MI300X 192GB HBM3 (x8) and AMD Instinct MI325X 256GB HBM3e (x8) and AMD Instinct MI355X 288GB HBM3e (x8)","8","24","closed/Dell_MangoBoost/results/8xMI300X_2xEPYC_9534_8xMI325X_2xEPYC_9655_8xMI355X_2xEPYC_9965/llama2-70b-99.9/Offline/performance/run_1","ROCm","Ubuntu 22.04.5 LTS (jammy)","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"Dell_MangoBoost","available","closed","datacenter","Dell_MangoBoost Heterogeneous Cluster (8x MI300X + 8x MI325X + 8x MI355X, LLMBoost)","8xMI300X_2xEPYC_9534_8xMI325X_2xEPYC_9655_8xMI355X_2xEPYC_9965","llama2-70b-99.9","llama2-70b-99.9","Server","141521.0","ROUGE1: 44.543 ROUGE2: 22.0969 ROUGEL: 28.8312 TOKENS_PER_SAMPLE: 287.0","3","AMD EPYC 9534 (on MI300X node), AMD EPYC 9655 (on MI325X node), AMD EPYC 9965 (on MI355X node)","2","64(AMD EPYC 9534) and 96(AMD EPYC 9655) and 192(AMD EPYC 9965)","AMD Instinct MI300X 192GB HBM3 (x8) and AMD Instinct MI325X 256GB HBM3e (x8) and AMD Instinct MI355X 288GB HBM3e (x8)","8","24","closed/Dell_MangoBoost/results/8xMI300X_2xEPYC_9534_8xMI325X_2xEPYC_9655_8xMI355X_2xEPYC_9965/llama2-70b-99.9/Server/performance/run_1","ROCm","Ubuntu 22.04.5 LTS (jammy)","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"Dell_MangoBoost","available","closed","datacenter","PowerEdge XE9785","8xMI355X_2xEPYC_9965","llama2-70b-99","llama2-70b-99","Offline","97535.1","ROUGE1: 44.5433 ROUGE2: 22.1372 ROUGEL: 28.9134 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9965","2","192","AMD Instinct MI355X 288GB HBM3e","8","8","closed/Dell_MangoBoost/results/8xMI355X_2xEPYC_9965/llama2-70b-99/Offline/performance/run_1","ROCm","Ubuntu 22.04.5 LTS (jammy)","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"Dell_MangoBoost","available","closed","datacenter","PowerEdge XE9785","8xMI355X_2xEPYC_9965","llama2-70b-99","llama2-70b-99","Server","93272.4","ROUGE1: 44.5377 ROUGE2: 22.1338 ROUGEL: 28.9088 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9965","2","192","AMD Instinct MI355X 288GB HBM3e","8","8","closed/Dell_MangoBoost/results/8xMI355X_2xEPYC_9965/llama2-70b-99/Server/performance/run_1","ROCm","Ubuntu 22.04.5 LTS (jammy)","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"Dell_MangoBoost","available","closed","datacenter","PowerEdge XE9785","8xMI355X_2xEPYC_9965","llama2-70b-99.9","llama2-70b-99.9","Offline","97535.1","ROUGE1: 44.5433 ROUGE2: 22.1372 ROUGEL: 28.9134 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9965","2","192","AMD Instinct MI355X 288GB HBM3e","8","8","closed/Dell_MangoBoost/results/8xMI355X_2xEPYC_9965/llama2-70b-99.9/Offline/performance/run_1","ROCm","Ubuntu 22.04.5 LTS (jammy)","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"Dell_MangoBoost","available","closed","datacenter","PowerEdge XE9785","8xMI355X_2xEPYC_9965","llama2-70b-99.9","llama2-70b-99.9","Server","93272.4","ROUGE1: 44.5377 ROUGE2: 22.1338 ROUGEL: 28.9088 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9965","2","192","AMD Instinct MI355X 288GB HBM3e","8","8","closed/Dell_MangoBoost/results/8xMI355X_2xEPYC_9965/llama2-70b-99.9/Server/performance/run_1","ROCm","Ubuntu 22.04.5 LTS (jammy)","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"Dell_MangoBoost","available","closed","datacenter","PowerEdge XE9785 (MI355X Power Cap 1000W)","8xMI355X_2xEPYC_9965_Power_Cap_1000W","llama2-70b-99","llama2-70b-99","Offline","81870.5","ROUGE1: 44.5433 ROUGE2: 22.1372 ROUGEL: 28.9134 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9965","2","192","AMD Instinct MI355X 288GB HBM3e (Power Cap 1000 W)","8","8","closed/Dell_MangoBoost/results/8xMI355X_2xEPYC_9965_Power_Cap_1000W/llama2-70b-99/Offline/performance/run_1","ROCm","Ubuntu 22.04.5 LTS (jammy)","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"Dell_MangoBoost","available","closed","datacenter","PowerEdge XE9785 (MI355X Power Cap 1000W)","8xMI355X_2xEPYC_9965_Power_Cap_1000W","llama2-70b-99","llama2-70b-99","Server","76359.5","ROUGE1: 44.5427 ROUGE2: 22.135 ROUGEL: 28.9148 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9965","2","192","AMD Instinct MI355X 288GB HBM3e (Power Cap 1000 W)","8","8","closed/Dell_MangoBoost/results/8xMI355X_2xEPYC_9965_Power_Cap_1000W/llama2-70b-99/Server/performance/run_1","ROCm","Ubuntu 22.04.5 LTS (jammy)","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"Dell_MangoBoost","available","closed","datacenter","PowerEdge XE9785 (MI355X Power Cap 1000W)","8xMI355X_2xEPYC_9965_Power_Cap_1000W","llama2-70b-99.9","llama2-70b-99.9","Offline","81870.5","ROUGE1: 44.5433 ROUGE2: 22.1372 ROUGEL: 28.9134 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9965","2","192","AMD Instinct MI355X 288GB HBM3e (Power Cap 1000 W)","8","8","closed/Dell_MangoBoost/results/8xMI355X_2xEPYC_9965_Power_Cap_1000W/llama2-70b-99.9/Offline/performance/run_1","ROCm","Ubuntu 22.04.5 LTS (jammy)","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"Dell_MangoBoost","available","closed","datacenter","PowerEdge XE9785 (MI355X Power Cap 1000W)","8xMI355X_2xEPYC_9965_Power_Cap_1000W","llama2-70b-99.9","llama2-70b-99.9","Server","76359.5","ROUGE1: 44.5427 ROUGE2: 22.135 ROUGEL: 28.9148 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9965","2","192","AMD Instinct MI355X 288GB HBM3e (Power Cap 1000 W)","8","8","closed/Dell_MangoBoost/results/8xMI355X_2xEPYC_9965_Power_Cap_1000W/llama2-70b-99.9/Server/performance/run_1","ROCm","Ubuntu 22.04.5 LTS (jammy)","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"GATEOverflow","available","closed","edge","GATE Overflow Intel SPR i9","intel_spr_i9-reference-cpu-pytorch_v2.3.1-default_config","yolo-95","yolo-95","multistream","843.374268","mAP: 53.108","1","13th Gen Intel(R) Core(TM) i9-13900K","1","24","N/A","0","0","closed/GATEOverflow/results/intel_spr_i9-reference-cpu-pytorch_v2.3.1-default_config/yolo-95/multistream/performance/run_1","pytorch v2.3.1","Ubuntu 24.04 (linux-6.8.0-64-generic-glibc2.39)","","closed","0","v6.0","1","False","Latency (ms)","fp32","","" +"GATEOverflow","available","closed","edge","GATE Overflow Intel SPR i9","intel_spr_i9-reference-cpu-pytorch_v2.3.1-default_config","yolo-95","yolo-95","offline","12.0652","mAP: 53.108","1","13th Gen Intel(R) Core(TM) i9-13900K","1","24","N/A","0","0","closed/GATEOverflow/results/intel_spr_i9-reference-cpu-pytorch_v2.3.1-default_config/yolo-95/offline/performance/run_1","pytorch v2.3.1","Ubuntu 24.04 (linux-6.8.0-64-generic-glibc2.39)","","closed","0","v6.0","1","False","Samples/s","fp32","","" +"GATEOverflow","available","closed","edge","GATE Overflow Intel SPR i9","intel_spr_i9-reference-cpu-pytorch_v2.3.1-default_config","yolo-95","yolo-95","singlestream","90.105803","mAP: 53.108","1","13th Gen Intel(R) Core(TM) i9-13900K","1","24","N/A","0","0","closed/GATEOverflow/results/intel_spr_i9-reference-cpu-pytorch_v2.3.1-default_config/yolo-95/singlestream/performance/run_1","pytorch v2.3.1","Ubuntu 24.04 (linux-6.8.0-64-generic-glibc2.39)","","closed","0","v6.0","1","False","Latency (ms)","fp32","","" +"GATEOverflow","available","closed","edge","GATE Overflow Intel SPR i9","intel_spr_i9-reference-cpu-pytorch_v2.3.1-default_config","yolo-99","yolo-99","multistream","843.374268","mAP: 53.108","1","13th Gen Intel(R) Core(TM) i9-13900K","1","24","N/A","0","0","closed/GATEOverflow/results/intel_spr_i9-reference-cpu-pytorch_v2.3.1-default_config/yolo-99/multistream/performance/run_1","pytorch v2.3.1","Ubuntu 24.04 (linux-6.8.0-64-generic-glibc2.39)","","closed","0","v6.0","1","False","Latency (ms)","fp32","","" +"GATEOverflow","available","closed","edge","GATE Overflow Intel SPR i9","intel_spr_i9-reference-cpu-pytorch_v2.3.1-default_config","yolo-99","yolo-99","offline","12.0652","mAP: 53.108","1","13th Gen Intel(R) Core(TM) i9-13900K","1","24","N/A","0","0","closed/GATEOverflow/results/intel_spr_i9-reference-cpu-pytorch_v2.3.1-default_config/yolo-99/offline/performance/run_1","pytorch v2.3.1","Ubuntu 24.04 (linux-6.8.0-64-generic-glibc2.39)","","closed","0","v6.0","1","False","Samples/s","fp32","","" +"GATEOverflow","available","closed","edge","GATE Overflow Intel SPR i9","intel_spr_i9-reference-cpu-pytorch_v2.3.1-default_config","yolo-99","yolo-99","singlestream","90.105803","mAP: 53.108","1","13th Gen Intel(R) Core(TM) i9-13900K","1","24","N/A","0","0","closed/GATEOverflow/results/intel_spr_i9-reference-cpu-pytorch_v2.3.1-default_config/yolo-99/singlestream/performance/run_1","pytorch v2.3.1","Ubuntu 24.04 (linux-6.8.0-64-generic-glibc2.39)","","closed","0","v6.0","1","False","Latency (ms)","fp32","","" +"GATEOverflow","available","closed","edge","GATE Overflow Intel Sapphire Rapids (2x RTX 4090)","d986d93afc0b-nvidia-gpu-TensorRT-default_config","resnet50","resnet","multistream","0.399625","acc: 76.064","1","Intel(R) Xeon(R) w7-2495X","1","24","NVIDIA GeForce RTX 4090","2","2","closed/GATEOverflow/results/d986d93afc0b-nvidia-gpu-TensorRT-default_config/resnet50/multistream/performance/run_1","TensorRT","Ubuntu 24.04 (linux-6.17.0-14-generic-glibc2.39)","","closed","0","v6.0","1","False","Latency (ms)","int8","","" +"GATEOverflow","available","closed","edge","GATE Overflow Intel Sapphire Rapids (2x RTX 4090)","d986d93afc0b-nvidia-gpu-TensorRT-default_config","resnet50","resnet","offline","87945.5","acc: 76.078","1","Intel(R) Xeon(R) w7-2495X","1","24","NVIDIA GeForce RTX 4090","2","2","closed/GATEOverflow/results/d986d93afc0b-nvidia-gpu-TensorRT-default_config/resnet50/offline/performance/run_1","TensorRT","Ubuntu 24.04 (linux-6.17.0-14-generic-glibc2.39)","","closed","0","v6.0","1","False","Samples/s","int8","","" +"GATEOverflow","available","closed","edge","GATE Overflow Intel Sapphire Rapids (2x RTX 4090)","d986d93afc0b-nvidia-gpu-TensorRT-default_config","resnet50","resnet","singlestream","0.303116","acc: 76.064","1","Intel(R) Xeon(R) w7-2495X","1","24","NVIDIA GeForce RTX 4090","2","2","closed/GATEOverflow/results/d986d93afc0b-nvidia-gpu-TensorRT-default_config/resnet50/singlestream/performance/run_1","TensorRT","Ubuntu 24.04 (linux-6.17.0-14-generic-glibc2.39)","","closed","0","v6.0","1","False","Latency (ms)","int8","","" +"GATEOverflow","available","closed","edge","Lenovo IdeaPad Gaming 3 15ARH05","Lenovo_IdeaPad_Gaming_3_Ryzen_5_4600H-reference-gpu-pytorch_v2.5.0a0-cu126","yolo-95","yolo-95","multistream","344.438863","mAP: 53.108","1","AMD Ryzen 5 4600H","1","6","NVIDIA GeForce GTX 1650 Ti","1","1","closed/GATEOverflow/results/Lenovo_IdeaPad_Gaming_3_Ryzen_5_4600H-reference-gpu-pytorch_v2.5.0a0-cu126/yolo-95/multistream/performance/run_1","pytorch v2.5.0a0","Ubuntu 22.04 (linux-6.17.9-76061709-generic-glibc2.35)","","closed","0","v6.0","1","False","Latency (ms)","fp32","","" +"GATEOverflow","available","closed","edge","Lenovo IdeaPad Gaming 3 15ARH05","Lenovo_IdeaPad_Gaming_3_Ryzen_5_4600H-reference-gpu-pytorch_v2.5.0a0-cu126","yolo-95","yolo-95","offline","24.187","mAP: 53.108","1","AMD Ryzen 5 4600H","1","6","NVIDIA GeForce GTX 1650 Ti","1","1","closed/GATEOverflow/results/Lenovo_IdeaPad_Gaming_3_Ryzen_5_4600H-reference-gpu-pytorch_v2.5.0a0-cu126/yolo-95/offline/performance/run_1","pytorch v2.5.0a0","Ubuntu 22.04 (linux-6.17.9-76061709-generic-glibc2.35)","","closed","0","v6.0","1","False","Samples/s","fp32","","" +"GATEOverflow","available","closed","edge","Lenovo IdeaPad Gaming 3 15ARH05","Lenovo_IdeaPad_Gaming_3_Ryzen_5_4600H-reference-gpu-pytorch_v2.5.0a0-cu126","yolo-95","yolo-95","singlestream","43.150031","mAP: 53.108","1","AMD Ryzen 5 4600H","1","6","NVIDIA GeForce GTX 1650 Ti","1","1","closed/GATEOverflow/results/Lenovo_IdeaPad_Gaming_3_Ryzen_5_4600H-reference-gpu-pytorch_v2.5.0a0-cu126/yolo-95/singlestream/performance/run_1","pytorch v2.5.0a0","Ubuntu 22.04 (linux-6.17.9-76061709-generic-glibc2.35)","","closed","0","v6.0","1","False","Latency (ms)","fp32","","" +"GATEOverflow","available","closed","edge","Lenovo IdeaPad Gaming 3 15ARH05","Lenovo_IdeaPad_Gaming_3_Ryzen_5_4600H-reference-gpu-pytorch_v2.5.0a0-cu126","yolo-99","yolo-99","multistream","344.438863","mAP: 53.108","1","AMD Ryzen 5 4600H","1","6","NVIDIA GeForce GTX 1650 Ti","1","1","closed/GATEOverflow/results/Lenovo_IdeaPad_Gaming_3_Ryzen_5_4600H-reference-gpu-pytorch_v2.5.0a0-cu126/yolo-99/multistream/performance/run_1","pytorch v2.5.0a0","Ubuntu 22.04 (linux-6.17.9-76061709-generic-glibc2.35)","","closed","0","v6.0","1","False","Latency (ms)","fp32","","" +"GATEOverflow","available","closed","edge","Lenovo IdeaPad Gaming 3 15ARH05","Lenovo_IdeaPad_Gaming_3_Ryzen_5_4600H-reference-gpu-pytorch_v2.5.0a0-cu126","yolo-99","yolo-99","offline","24.187","mAP: 53.108","1","AMD Ryzen 5 4600H","1","6","NVIDIA GeForce GTX 1650 Ti","1","1","closed/GATEOverflow/results/Lenovo_IdeaPad_Gaming_3_Ryzen_5_4600H-reference-gpu-pytorch_v2.5.0a0-cu126/yolo-99/offline/performance/run_1","pytorch v2.5.0a0","Ubuntu 22.04 (linux-6.17.9-76061709-generic-glibc2.35)","","closed","0","v6.0","1","False","Samples/s","fp32","","" +"GATEOverflow","available","closed","edge","Lenovo IdeaPad Gaming 3 15ARH05","Lenovo_IdeaPad_Gaming_3_Ryzen_5_4600H-reference-gpu-pytorch_v2.5.0a0-cu126","yolo-99","yolo-99","singlestream","43.150031","mAP: 53.108","1","AMD Ryzen 5 4600H","1","6","NVIDIA GeForce GTX 1650 Ti","1","1","closed/GATEOverflow/results/Lenovo_IdeaPad_Gaming_3_Ryzen_5_4600H-reference-gpu-pytorch_v2.5.0a0-cu126/yolo-99/singlestream/performance/run_1","pytorch v2.5.0a0","Ubuntu 22.04 (linux-6.17.9-76061709-generic-glibc2.35)","","closed","0","v6.0","1","False","Latency (ms)","fp32","","" +"GigaComputing","available","closed","datacenter","DLB2-CB3","DLB2-CB3_GB300-288GB_aarch64x4_TRT","qwen3-vl-235b-a22b","qwen3-vl-235b-a22b","Offline","60.9525","F1_HIERARCHICAL: 0.7886741346065558","1","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","4","closed/GigaComputing/results/DLB2-CB3_GB300-288GB_aarch64x4_TRT/qwen3-vl-235b-a22b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04","NVIDIA GB300 NVL72","closed","0","v6.0","0","False","Samples/s","fp32","","" +"GigaComputing","available","closed","datacenter","DLB2-CB3","DLB2-CB3_GB300-288GB_aarch64x4_TRT","qwen3-vl-235b-a22b","qwen3-vl-235b-a22b","Server","41.6342","F1_HIERARCHICAL: 0.7871305769336737","1","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","4","closed/GigaComputing/results/DLB2-CB3_GB300-288GB_aarch64x4_TRT/qwen3-vl-235b-a22b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04","NVIDIA GB300 NVL72","closed","0","v6.0","0","False","Queries/s","fp32","","" +"GigaComputing","available","closed","datacenter","G893-ZX1-AAX4","8xMI355X_2xEPYC-9575F","gpt-oss-120b","gpt-oss-120b","Offline","92332.2","exact_match: 83.220","1","AMD EPYC 9575F","2","64","AMD Instinct MI355X 288GB HBM3e","8","8","closed/GigaComputing/results/8xMI355X_2xEPYC-9575F/gpt-oss-120b/Offline/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0","Ubuntu 22.04.5","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"GigaComputing","available","closed","datacenter","G893-ZX1-AAX4","8xMI355X_2xEPYC-9575F","gpt-oss-120b","gpt-oss-120b","Server","80089.6","exact_match: 82.853","1","AMD EPYC 9575F","2","64","AMD Instinct MI355X 288GB HBM3e","8","8","closed/GigaComputing/results/8xMI355X_2xEPYC-9575F/gpt-oss-120b/Server/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0","Ubuntu 22.04.5","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"GigaComputing","available","closed","datacenter","G893-ZX1-AAX4","8xMI355X_2xEPYC-9575F","llama2-70b-99","llama2-70b-99","Interactive","72547.3","ROUGE1: 44.5355 ROUGE2: 22.1293 ROUGEL: 28.91 TOKENS_PER_SAMPLE: 282.8","1","AMD EPYC 9575F","2","64","AMD Instinct MI355X 288GB HBM3e","8","8","closed/GigaComputing/results/8xMI355X_2xEPYC-9575F/llama2-70b-99/Interactive/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0","Ubuntu 22.04.5","","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"GigaComputing","available","closed","datacenter","G893-ZX1-AAX4","8xMI355X_2xEPYC-9575F","llama2-70b-99","llama2-70b-99","Offline","100310.0","ROUGE1: 44.4514 ROUGE2: 22.0869 ROUGEL: 28.8592 TOKENS_PER_SAMPLE: 284.3","1","AMD EPYC 9575F","2","64","AMD Instinct MI355X 288GB HBM3e","8","8","closed/GigaComputing/results/8xMI355X_2xEPYC-9575F/llama2-70b-99/Offline/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0","Ubuntu 22.04.5","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"GigaComputing","available","closed","datacenter","G893-ZX1-AAX4","8xMI355X_2xEPYC-9575F","llama2-70b-99","llama2-70b-99","Server","98157.2","ROUGE1: 44.5382 ROUGE2: 22.1338 ROUGEL: 28.9091 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9575F","2","64","AMD Instinct MI355X 288GB HBM3e","8","8","closed/GigaComputing/results/8xMI355X_2xEPYC-9575F/llama2-70b-99/Server/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0","Ubuntu 22.04.5","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"GigaComputing","available","closed","datacenter","G893-ZX1-AAX4","8xMI355X_2xEPYC-9575F","llama2-70b-99.9","llama2-70b-99.9","Interactive","72547.3","ROUGE1: 44.5355 ROUGE2: 22.1293 ROUGEL: 28.91 TOKENS_PER_SAMPLE: 282.8","1","AMD EPYC 9575F","2","64","AMD Instinct MI355X 288GB HBM3e","8","8","closed/GigaComputing/results/8xMI355X_2xEPYC-9575F/llama2-70b-99.9/Interactive/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0","Ubuntu 22.04.5","","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"GigaComputing","available","closed","datacenter","G893-ZX1-AAX4","8xMI355X_2xEPYC-9575F","llama2-70b-99.9","llama2-70b-99.9","Offline","100310.0","ROUGE1: 44.4514 ROUGE2: 22.0869 ROUGEL: 28.8592 TOKENS_PER_SAMPLE: 284.3","1","AMD EPYC 9575F","2","64","AMD Instinct MI355X 288GB HBM3e","8","8","closed/GigaComputing/results/8xMI355X_2xEPYC-9575F/llama2-70b-99.9/Offline/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0","Ubuntu 22.04.5","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"GigaComputing","available","closed","datacenter","G893-ZX1-AAX4","8xMI355X_2xEPYC-9575F","llama2-70b-99.9","llama2-70b-99.9","Server","98157.2","ROUGE1: 44.5382 ROUGE2: 22.1338 ROUGEL: 28.9091 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9575F","2","64","AMD Instinct MI355X 288GB HBM3e","8","8","closed/GigaComputing/results/8xMI355X_2xEPYC-9575F/llama2-70b-99.9/Server/performance/run_1","PyTorch 2.9.1+git8907517, ROCm 7.0.0","Ubuntu 22.04.5","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"GigaComputing","available","closed","datacenter","G894-SD3-AAX7","G894-SD3-AAX7_B300-SXM-270GBx8_TRT","deepseek-r1","deepseek-r1","Interactive","4934.65","exact_match: 80.94804010938924 TOKENS_PER_SAMPLE: 3773.6034639927075","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/GigaComputing/results/G894-SD3-AAX7_B300-SXM-270GBx8_TRT/deepseek-r1/Interactive/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04.4","B300 TGP 1100W","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"GigaComputing","available","closed","datacenter","G894-SD3-AAX7","G894-SD3-AAX7_B300-SXM-270GBx8_TRT","deepseek-r1","deepseek-r1","Offline","69617.4","exact_match: 80.67456700091158 TOKENS_PER_SAMPLE: 3756.651777575205","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/GigaComputing/results/G894-SD3-AAX7_B300-SXM-270GBx8_TRT/deepseek-r1/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04.4","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"GigaComputing","available","closed","datacenter","G894-SD3-AAX7","G894-SD3-AAX7_B300-SXM-270GBx8_TRT","deepseek-r1","deepseek-r1","Server","35552.6","exact_match: 81.22151321786691 TOKENS_PER_SAMPLE: 3733.6216955332725","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/GigaComputing/results/G894-SD3-AAX7_B300-SXM-270GBx8_TRT/deepseek-r1/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04.4","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"GigaComputing","available","closed","datacenter","G894-SD3-AAX7","G894-SD3-AAX7_B300-SXM-270GBx8_TRT","gpt-oss-120b","gpt-oss-120b","Offline","107696.0","exact_match: 83.622","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/GigaComputing/results/G894-SD3-AAX7_B300-SXM-270GBx8_TRT/gpt-oss-120b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04.4","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"GigaComputing","available","closed","datacenter","G894-SD3-AAX7","G894-SD3-AAX7_B300-SXM-270GBx8_TRT","gpt-oss-120b","gpt-oss-120b","Server","106397.0","exact_match: 83.855","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/GigaComputing/results/G894-SD3-AAX7_B300-SXM-270GBx8_TRT/gpt-oss-120b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04.4","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"GigaComputing","available","closed","datacenter","G894-SD3-AAX7","G894-SD3-AAX7_B300-SXM-270GBx8_TRT","llama2-70b-99","llama2-70b-99","Interactive","65464.8","ROUGE1: 44.7219 ROUGE2: 22.309 ROUGEL: 29.0918 TOKENS_PER_SAMPLE: 275.0","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/GigaComputing/results/G894-SD3-AAX7_B300-SXM-270GBx8_TRT/llama2-70b-99/Interactive/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04.4","B300 TGP 1100W","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"GigaComputing","available","closed","datacenter","G894-SD3-AAX7","G894-SD3-AAX7_B300-SXM-270GBx8_TRT","llama2-70b-99","llama2-70b-99","Offline","109343.0","ROUGE1: 44.7858 ROUGE2: 22.3212 ROUGEL: 29.1222 TOKENS_PER_SAMPLE: 276.3","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/GigaComputing/results/G894-SD3-AAX7_B300-SXM-270GBx8_TRT/llama2-70b-99/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04.4","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"GigaComputing","available","closed","datacenter","G894-SD3-AAX7","G894-SD3-AAX7_B300-SXM-270GBx8_TRT","llama2-70b-99","llama2-70b-99","Server","104623.0","ROUGE1: 44.7751 ROUGE2: 22.3149 ROUGEL: 29.1129 TOKENS_PER_SAMPLE: 277.3","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/GigaComputing/results/G894-SD3-AAX7_B300-SXM-270GBx8_TRT/llama2-70b-99/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04.4","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"GigaComputing","available","closed","datacenter","G894-SD3-AAX7","G894-SD3-AAX7_B300-SXM-270GBx8_TRT","llama2-70b-99.9","llama2-70b-99.9","Interactive","65464.8","ROUGE1: 44.7219 ROUGE2: 22.309 ROUGEL: 29.0918 TOKENS_PER_SAMPLE: 275.0","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/GigaComputing/results/G894-SD3-AAX7_B300-SXM-270GBx8_TRT/llama2-70b-99.9/Interactive/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04.4","B300 TGP 1100W","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"GigaComputing","available","closed","datacenter","G894-SD3-AAX7","G894-SD3-AAX7_B300-SXM-270GBx8_TRT","llama2-70b-99.9","llama2-70b-99.9","Offline","109343.0","ROUGE1: 44.7858 ROUGE2: 22.3212 ROUGEL: 29.1222 TOKENS_PER_SAMPLE: 276.3","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/GigaComputing/results/G894-SD3-AAX7_B300-SXM-270GBx8_TRT/llama2-70b-99.9/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04.4","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"GigaComputing","available","closed","datacenter","G894-SD3-AAX7","G894-SD3-AAX7_B300-SXM-270GBx8_TRT","llama2-70b-99.9","llama2-70b-99.9","Server","104623.0","ROUGE1: 44.7751 ROUGE2: 22.3149 ROUGEL: 29.1129 TOKENS_PER_SAMPLE: 277.3","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/GigaComputing/results/G894-SD3-AAX7_B300-SXM-270GBx8_TRT/llama2-70b-99.9/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04.4","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"GigaComputing","available","closed","datacenter","G894-SD3-AAX7","G894-SD3-AAX7_B300-SXM-270GBx8_TRT","llama3.1-405b","llama3.1-405b","Interactive","764.655","ROUGEL: 21.835240799978806 exact_match: 90.10374586366643 TOKENS_PER_SAMPLE: 636.2","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/GigaComputing/results/G894-SD3-AAX7_B300-SXM-270GBx8_TRT/llama3.1-405b/Interactive/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04.4","B300 TGP 1100W","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"GigaComputing","available","closed","datacenter","G894-SD3-AAX7","G894-SD3-AAX7_B300-SXM-270GBx8_TRT","llama3.1-405b","llama3.1-405b","Offline","1947.91","ROUGEL: 22.052524963008008 exact_match: 90.0969391131701 TOKENS_PER_SAMPLE: 652.3","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/GigaComputing/results/G894-SD3-AAX7_B300-SXM-270GBx8_TRT/llama3.1-405b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04.4","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"GigaComputing","available","closed","datacenter","G894-SD3-AAX7","G894-SD3-AAX7_B300-SXM-270GBx8_TRT","llama3.1-405b","llama3.1-405b","Server","1340.51","ROUGEL: 21.853317758658946 exact_match: 90.12565519523494 TOKENS_PER_SAMPLE: 636.3","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/GigaComputing/results/G894-SD3-AAX7_B300-SXM-270GBx8_TRT/llama3.1-405b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04.4","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"GigaComputing","available","closed","datacenter","G894-SD3-AAX7","G894-SD3-AAX7_B300-SXM-270GBx8_TRT","llama3.1-8b","llama3.1-8b","Interactive","128633.0","ROUGE1: 38.5225 ROUGE2: 15.8241 ROUGEL: 24.308 ROUGELSUM: 35.5704 GEN_LEN: 8194606","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/GigaComputing/results/G894-SD3-AAX7_B300-SXM-270GBx8_TRT/llama3.1-8b/Interactive/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04.4","B300 TGP 1100W","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"GigaComputing","available","closed","datacenter","G894-SD3-AAX7","G894-SD3-AAX7_B300-SXM-270GBx8_TRT","llama3.1-8b","llama3.1-8b","Offline","150707.0","ROUGE1: 38.5224 ROUGE2: 15.8225 ROUGEL: 24.3072 ROUGELSUM: 35.5714 GEN_LEN: 8194322","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/GigaComputing/results/G894-SD3-AAX7_B300-SXM-270GBx8_TRT/llama3.1-8b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04.4","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"GigaComputing","available","closed","datacenter","G894-SD3-AAX7","G894-SD3-AAX7_B300-SXM-270GBx8_TRT","llama3.1-8b","llama3.1-8b","Server","141484.0","ROUGE1: 38.5265 ROUGE2: 15.8301 ROUGEL: 24.3121 ROUGELSUM: 35.5742 GEN_LEN: 8194725","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/GigaComputing/results/G894-SD3-AAX7_B300-SXM-270GBx8_TRT/llama3.1-8b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04.4","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"GigaComputing","available","closed","datacenter","G894-SD3-AAX7","G894-SD3-AAX7_B300-SXM-270GBx8_TRT","wan-2.2-t2v-a14b","wan-2.2-t2v-a14b","Offline","0.0582849","vbench_score: 70.2730","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/GigaComputing/results/G894-SD3-AAX7_B300-SXM-270GBx8_TRT/wan-2.2-t2v-a14b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04.4","B300 TGP 1100W","closed","0","v6.0","0","False","Samples/s","fp32","","" +"GigaComputing","available","closed","datacenter","G894-SD3-AAX7","G894-SD3-AAX7_B300-SXM-270GBx8_TRT","wan-2.2-t2v-a14b","wan-2.2-t2v-a14b","SingleStream","20.897894239000003","vbench_score: 70.2024","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/GigaComputing/results/G894-SD3-AAX7_B300-SXM-270GBx8_TRT/wan-2.2-t2v-a14b/SingleStream/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04.4","B300 TGP 1100W","closed","0","v6.0","0","False","Latency (s)","fp32","","" +"GigaComputing","available","closed","datacenter","G894-SD3-AAX7","G894-SD3-AAX7_B300-SXM-270GBx8_TRT","whisper","whisper","Offline","48116.6","ACCURACY: 97.84824462061155","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/GigaComputing/results/G894-SD3-AAX7_B300-SXM-270GBx8_TRT/whisper/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 22.04.4","B300 TGP 1100W","closed","0","v6.0","0","False","Samples/s","fp16","","" +"Google","available","closed","datacenter","NVIDIA DGX B200 (8x B200-SXM-180GB, TensorRT)","B200-SXM-180GBx8_TRT","deepseek-r1","deepseek-r1","Offline","57116.9","exact_match: 81.0619872379216 TOKENS_PER_SAMPLE: 3785.250455788514","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B200-SXM-180GB","8","8","closed/Google/results/B200-SXM-180GBx8_TRT/deepseek-r1/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B200 TGP 1000W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Google","available","closed","datacenter","NVIDIA DGX B200 (8x B200-SXM-180GB, TensorRT)","B200-SXM-180GBx8_TRT","deepseek-r1","deepseek-r1","Server","41027.9","exact_match: 81.42661804922516 TOKENS_PER_SAMPLE: 3748.3705560619874","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B200-SXM-180GB","8","8","closed/Google/results/B200-SXM-180GBx8_TRT/deepseek-r1/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B200 TGP 1000W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Google","available","closed","datacenter","NVIDIA DGX B200 (8x B200-SXM-180GB, TensorRT)","B200-SXM-180GBx8_TRT","dlrm-v3","dlrm-v3","Offline","10447.3","DLRM_NE: 0.8675504587396468 DLRM_ACC: 0.6964261896038318 DLRM_AUC: 0.7862891896349031","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B200-SXM-180GB","8","8","closed/Google/results/B200-SXM-180GBx8_TRT/dlrm-v3/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B200 TGP 1000W","closed","0","v6.0","0","False","Samples/s","fp16","","" +"Google","available","closed","datacenter","NVIDIA DGX B200 (8x B200-SXM-180GB, TensorRT)","B200-SXM-180GBx8_TRT","dlrm-v3","dlrm-v3","Server","10007.0","DLRM_NE: 0.8675496448947934 DLRM_ACC: 0.6964279594720938 DLRM_AUC: 0.7862891760730971","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B200-SXM-180GB","8","8","closed/Google/results/B200-SXM-180GBx8_TRT/dlrm-v3/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B200 TGP 1000W","closed","0","v6.0","0","False","Queries/s","fp16","","" +"Google","available","closed","datacenter","NVIDIA DGX B200 (8x B200-SXM-180GB, TensorRT)","B200-SXM-180GBx8_TRT","llama3.1-405b","llama3.1-405b","Offline","1350.24","ROUGEL: 21.918853025027985 exact_match: 89.94308074123097 TOKENS_PER_SAMPLE: 654.5","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B200-SXM-180GB","8","8","closed/Google/results/B200-SXM-180GBx8_TRT/llama3.1-405b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B200 TGP 1000W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Google","available","closed","datacenter","NVIDIA DGX B200 (8x B200-SXM-180GB, TensorRT)","B200-SXM-180GBx8_TRT","llama3.1-405b","llama3.1-405b","Server","983.626","ROUGEL: 21.894180533777952 exact_match: 89.9182594308405 TOKENS_PER_SAMPLE: 654.1","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B200-SXM-180GB","8","8","closed/Google/results/B200-SXM-180GBx8_TRT/llama3.1-405b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B200 TGP 1000W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Google","available","closed","datacenter","NVIDIA DGX B200 (8x B200-SXM-180GB, TensorRT)","B200-SXM-180GBx8_TRT","whisper","whisper","Offline","49327.2","ACCURACY: 97.85490640197189","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B200-SXM-180GB","8","8","closed/Google/results/B200-SXM-180GBx8_TRT/whisper/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B200 TGP 1000W","closed","0","v6.0","0","False","Samples/s","fp16","","" +"Google","available","closed","datacenter","NVIDIA GB200 NVL72 (4x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x4_TRT","qwen3-vl-235b-a22b","qwen3-vl-235b-a22b","Offline","42.4736","F1_HIERARCHICAL: 0.7879394631035838","1","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","4","closed/Google/results/GB200-NVL72_GB200-186GB_aarch64x4_TRT/qwen3-vl-235b-a22b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","NVIDIA GB200 NVL72","closed","0","v6.0","0","False","Samples/s","fp32","","" +"Google","available","closed","datacenter","NVIDIA GB200 NVL72 (4x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x4_TRT","qwen3-vl-235b-a22b","qwen3-vl-235b-a22b","Server","35.5625","F1_HIERARCHICAL: 0.7874876598480153","1","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","4","closed/Google/results/GB200-NVL72_GB200-186GB_aarch64x4_TRT/qwen3-vl-235b-a22b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","NVIDIA GB200 NVL72","closed","0","v6.0","0","False","Queries/s","fp32","","" +"Google","available","closed","datacenter","NVIDIA GB200 NVL72 (4x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x4_TRT","whisper","whisper","Offline","22051.4","ACCURACY: 97.8501479867145","1","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","4","closed/Google/results/GB200-NVL72_GB200-186GB_aarch64x4_TRT/whisper/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","NVIDIA GB200 NVL72","closed","0","v6.0","0","False","Samples/s","fp16","","" +"Google","available","closed","datacenter","NVIDIA GB200 NVL72 (72x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x72_TRT","deepseek-r1","deepseek-r1","Offline","307874.0","exact_match: 81.22151321786691 TOKENS_PER_SAMPLE: 3723.8377392889697","18","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","72","closed/Google/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/deepseek-r1/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","NVIDIA GB200 NVL72","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Google","available","closed","datacenter","NVIDIA GB200 NVL72 (72x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x72_TRT","deepseek-r1","deepseek-r1","Server","156781.0","exact_match: 80.83409298085688 TOKENS_PER_SAMPLE: 3694.1697812215134","18","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","72","closed/Google/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/deepseek-r1/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","NVIDIA GB200 NVL72","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380 Gen12 (5x NVIDIA RTX PRO 4500 Blackwell, TensorRT)","HPE_Proliant_DL380_RTX_PRO_4500_PCIe_32GBx5_DC_TRT","llama3.1-8b","llama3.1-8b","Offline","11665.0","ROUGE1: 38.5883 ROUGE2: 15.86 ROUGEL: 24.3131 ROUGELSUM: 35.5397 GEN_LEN: 8181251","1","Intel(R) Xeon(R) 6747P","2","48","NVIDIA RTX PRO 4500 Blackwell","5","5","closed/HPE/results/HPE_Proliant_DL380_RTX_PRO_4500_PCIe_32GBx5_DC_TRT/llama3.1-8b/Offline/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380 Gen12 (5x NVIDIA RTX PRO 4500 Blackwell, TensorRT)","HPE_Proliant_DL380_RTX_PRO_4500_PCIe_32GBx5_DC_TRT","llama3.1-8b","llama3.1-8b","Server","11352.1","ROUGE1: 38.587 ROUGE2: 15.8597 ROUGEL: 24.3106 ROUGELSUM: 35.5382 GEN_LEN: 8181395","1","Intel(R) Xeon(R) 6747P","2","48","NVIDIA RTX PRO 4500 Blackwell","5","5","closed/HPE/results/HPE_Proliant_DL380_RTX_PRO_4500_PCIe_32GBx5_DC_TRT/llama3.1-8b/Server/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380a Gen12 (10x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx10_TRT_10_11_0_33_TRT","mixtral-8x7b","mixtral-8x7b","Offline","41242.4","ROUGE1: 45.5065 ROUGE2: 23.2835 ROUGEL: 30.3527 TOKENS_PER_SAMPLE: 144.3 gsm8k_accuracy: 73.92 mbxp_accuracy: 60.12","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA RTX PRO 6000 Blackwell Server Edition","10","10","closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx10_TRT_10_11_0_33_TRT/mixtral-8x7b/Offline/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3 LTS","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380a Gen12 (10x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx10_TRT_10_11_0_33_TRT","mixtral-8x7b","mixtral-8x7b","Server","39234.1","ROUGE1: 45.4809 ROUGE2: 23.3483 ROUGEL: 30.4976 TOKENS_PER_SAMPLE: 145.7 gsm8k_accuracy: 73.86 mbxp_accuracy: 59.94","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA RTX PRO 6000 Blackwell Server Edition","10","10","closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx10_TRT_10_11_0_33_TRT/mixtral-8x7b/Server/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3 LTS","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380a Gen12 (10x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx10_TRT_10_11_0_33_TRT","whisper","whisper","Offline","18889.4","ACCURACY: 97.86537491553813","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA RTX PRO 6000 Blackwell Server Edition","10","10","closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx10_TRT_10_11_0_33_TRT/whisper/Offline/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3 LTS","","closed","0","v6.0","0","False","Samples/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380a Gen12 (10x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx10_TRT_10_14_1_48_TRT","gpt-oss-120b","gpt-oss-120b","Offline","19040.8","exact_match: 84.096","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA RTX PRO 6000 Blackwell Server Edition","10","10","closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx10_TRT_10_14_1_48_TRT/gpt-oss-120b/Offline/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380a Gen12 (10x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx10_TRT_10_14_1_48_TRT","gpt-oss-120b","gpt-oss-120b","Server","17737.6","exact_match: 83.746","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA RTX PRO 6000 Blackwell Server Edition","10","10","closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx10_TRT_10_14_1_48_TRT/gpt-oss-120b/Server/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380a Gen12 (10x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx10_TRT_10_14_1_48_TRT","llama2-70b-99","llama2-70b-99","Offline","36710.8","ROUGE1: 44.8722 ROUGE2: 22.4111 ROUGEL: 29.1997 TOKENS_PER_SAMPLE: 271.9","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA RTX PRO 6000 Blackwell Server Edition","10","10","closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx10_TRT_10_14_1_48_TRT/llama2-70b-99/Offline/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380a Gen12 (10x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx10_TRT_10_14_1_48_TRT","llama2-70b-99","llama2-70b-99","Server","35590.8","ROUGE1: 44.87 ROUGE2: 22.409 ROUGEL: 29.197 TOKENS_PER_SAMPLE: 272.9","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA RTX PRO 6000 Blackwell Server Edition","10","10","closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx10_TRT_10_14_1_48_TRT/llama2-70b-99/Server/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380a Gen12 (10x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx10_TRT_10_14_1_48_TRT","llama2-70b-99.9","llama2-70b-99.9","Offline","36704.2","ROUGE1: 44.8721 ROUGE2: 22.411 ROUGEL: 29.1995 TOKENS_PER_SAMPLE: 271.9","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA RTX PRO 6000 Blackwell Server Edition","10","10","closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx10_TRT_10_14_1_48_TRT/llama2-70b-99.9/Offline/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380a Gen12 (10x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx10_TRT_10_14_1_48_TRT","llama2-70b-99.9","llama2-70b-99.9","Server","35522.3","ROUGE1: 44.8738 ROUGE2: 22.4117 ROUGEL: 29.1998 TOKENS_PER_SAMPLE: 272.8","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA RTX PRO 6000 Blackwell Server Edition","10","10","closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx10_TRT_10_14_1_48_TRT/llama2-70b-99.9/Server/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380a Gen12 (10x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx10_TRT_10_14_1_48_TRT","llama3.1-8b","llama3.1-8b","Interactive","53054.7","ROUGE1: 38.7806 ROUGE2: 16.0229 ROUGEL: 24.5438 ROUGELSUM: 35.7089 GEN_LEN: 8179467","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA RTX PRO 6000 Blackwell Server Edition","10","10","closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx10_TRT_10_14_1_48_TRT/llama3.1-8b/Interactive/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380a Gen12 (10x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx10_TRT_10_14_1_48_TRT","llama3.1-8b","llama3.1-8b","Offline","60679.9","ROUGE1: 38.7667 ROUGE2: 16.0132 ROUGEL: 24.5324 ROUGELSUM: 35.6922 GEN_LEN: 8178916","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA RTX PRO 6000 Blackwell Server Edition","10","10","closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx10_TRT_10_14_1_48_TRT/llama3.1-8b/Offline/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380a Gen12 (10x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx10_TRT_10_14_1_48_TRT","llama3.1-8b","llama3.1-8b","Server","59632.7","ROUGE1: 38.7625 ROUGE2: 16.0093 ROUGEL: 24.5327 ROUGELSUM: 35.6872 GEN_LEN: 8179131","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA RTX PRO 6000 Blackwell Server Edition","10","10","closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx10_TRT_10_14_1_48_TRT/llama3.1-8b/Server/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380a Gen12 (8x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8_TRT_10_11_0_33_TRT","mixtral-8x7b","mixtral-8x7b","Offline","33637.3","ROUGE1: 45.4424 ROUGE2: 23.2859 ROUGEL: 30.3282 TOKENS_PER_SAMPLE: 145.1 gsm8k_accuracy: 73.34 mbxp_accuracy: 60.28","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8_TRT_10_11_0_33_TRT/mixtral-8x7b/Offline/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3 LTS","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380a Gen12 (8x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8_TRT_10_11_0_33_TRT","mixtral-8x7b","mixtral-8x7b","Server","32151.0","ROUGE1: 45.5238 ROUGE2: 23.3068 ROUGEL: 30.4805 TOKENS_PER_SAMPLE: 144.9 gsm8k_accuracy: 73.72 mbxp_accuracy: 59.96","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8_TRT_10_11_0_33_TRT/mixtral-8x7b/Server/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3 LTS","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380a Gen12 (8x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8_TRT_10_14_1_48_TRT","gpt-oss-120b","gpt-oss-120b","Offline","15189.9","exact_match: 83.644","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8_TRT_10_14_1_48_TRT/gpt-oss-120b/Offline/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380a Gen12 (8x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8_TRT_10_14_1_48_TRT","gpt-oss-120b","gpt-oss-120b","Server","14258.9","exact_match: 83.633","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8_TRT_10_14_1_48_TRT/gpt-oss-120b/Server/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380a Gen12 (8x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8_TRT_10_14_1_48_TRT","llama2-70b-99","llama2-70b-99","Offline","29908.4","ROUGE1: 44.8698 ROUGE2: 22.4084 ROUGEL: 29.1974 TOKENS_PER_SAMPLE: 271.9","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8_TRT_10_14_1_48_TRT/llama2-70b-99/Offline/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380a Gen12 (8x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8_TRT_10_14_1_48_TRT","llama2-70b-99","llama2-70b-99","Server","28727.8","ROUGE1: 44.873 ROUGE2: 22.4095 ROUGEL: 29.199 TOKENS_PER_SAMPLE: 272.9","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8_TRT_10_14_1_48_TRT/llama2-70b-99/Server/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380a Gen12 (8x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8_TRT_10_14_1_48_TRT","llama2-70b-99.9","llama2-70b-99.9","Offline","29900.1","ROUGE1: 44.8712 ROUGE2: 22.4093 ROUGEL: 29.1983 TOKENS_PER_SAMPLE: 271.9","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8_TRT_10_14_1_48_TRT/llama2-70b-99.9/Offline/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380a Gen12 (8x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8_TRT_10_14_1_48_TRT","llama2-70b-99.9","llama2-70b-99.9","Server","28717.1","ROUGE1: 44.8687 ROUGE2: 22.4095 ROUGEL: 29.1984 TOKENS_PER_SAMPLE: 272.9","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8_TRT_10_14_1_48_TRT/llama2-70b-99.9/Server/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380a Gen12 (8x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8_TRT_10_14_1_48_TRT","llama3.1-8b","llama3.1-8b","Interactive","44087.3","ROUGE1: 38.7729 ROUGE2: 16.0189 ROUGEL: 24.5391 ROUGELSUM: 35.699 GEN_LEN: 8178708","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8_TRT_10_14_1_48_TRT/llama3.1-8b/Interactive/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380a Gen12 (8x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8_TRT_10_14_1_48_TRT","llama3.1-8b","llama3.1-8b","Offline","49580.2","ROUGE1: 38.7725 ROUGE2: 16.0187 ROUGEL: 24.5371 ROUGELSUM: 35.6975 GEN_LEN: 8178941","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8_TRT_10_14_1_48_TRT/llama3.1-8b/Offline/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380a Gen12 (8x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8_TRT_10_14_1_48_TRT","llama3.1-8b","llama3.1-8b","Server","48794.3","ROUGE1: 38.7506 ROUGE2: 16.0037 ROUGEL: 24.5269 ROUGELSUM: 35.6755 GEN_LEN: 8178736","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8_TRT_10_14_1_48_TRT/llama3.1-8b/Server/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute DL380a Gen12 (8x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8_TRT_10_14_1_48_TRT","whisper","whisper","Offline","18709.0","ACCURACY: 97.82254917822169","1","Intel(R) Xeon(R) 6787P","2","172","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/HPE/results/HPE_PROLIANT_DL380A_RTX_PRO_6000_PCIE_96GBx8_TRT_10_14_1_48_TRT/whisper/Offline/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","","closed","0","v6.0","0","False","Samples/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute XD685 (8x AMD Instinct MI355X 288GB, vLLM)","HPE_ProLiant_XD685_MI355X_288GBx8","gpt-oss-120b","gpt-oss-120b","Offline","91846.6","exact_match: 83.347","1","AMD EPYC 9575F","2","128","AMD Instinct MI355X 288GB HBM3e","8","8","closed/HPE/results/HPE_ProLiant_XD685_MI355X_288GBx8/gpt-oss-120b/Offline/performance/run_1","vLLM","Ubuntu 22.04","256 CUs per GPU, gfx950 architecture","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute XD685 (8x AMD Instinct MI355X 288GB, vLLM)","HPE_ProLiant_XD685_MI355X_288GBx8","gpt-oss-120b","gpt-oss-120b","Server","75354.9","exact_match: 82.871","1","AMD EPYC 9575F","2","128","AMD Instinct MI355X 288GB HBM3e","8","8","closed/HPE/results/HPE_ProLiant_XD685_MI355X_288GBx8/gpt-oss-120b/Server/performance/run_1","vLLM","Ubuntu 22.04","256 CUs per GPU, gfx950 architecture","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute XD685 (8x AMD Instinct MI355X 288GB, vLLM)","HPE_ProLiant_XD685_MI355X_288GBx8","llama2-70b-99","llama2-70b-99","Interactive","70954.1","ROUGE1: 44.5404 ROUGE2: 22.1352 ROUGEL: 28.9123 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9575F","2","128","AMD Instinct MI355X 288GB HBM3e","8","8","closed/HPE/results/HPE_ProLiant_XD685_MI355X_288GBx8/llama2-70b-99/Interactive/performance/run_1","vLLM","Ubuntu 22.04","256 CUs per GPU, gfx950 architecture","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute XD685 (8x AMD Instinct MI355X 288GB, vLLM)","HPE_ProLiant_XD685_MI355X_288GBx8","llama2-70b-99","llama2-70b-99","Offline","100019.0","ROUGE1: 44.4435 ROUGE2: 22.0815 ROUGEL: 28.8533 TOKENS_PER_SAMPLE: 284.2","1","AMD EPYC 9575F","2","128","AMD Instinct MI355X 288GB HBM3e","8","8","closed/HPE/results/HPE_ProLiant_XD685_MI355X_288GBx8/llama2-70b-99/Offline/performance/run_1","vLLM","Ubuntu 22.04","256 CUs per GPU, gfx950 architecture","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute XD685 (8x AMD Instinct MI355X 288GB, vLLM)","HPE_ProLiant_XD685_MI355X_288GBx8","llama2-70b-99","llama2-70b-99","Server","95544.3","ROUGE1: 44.5391 ROUGE2: 22.1346 ROUGEL: 28.91 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9575F","2","128","AMD Instinct MI355X 288GB HBM3e","8","8","closed/HPE/results/HPE_ProLiant_XD685_MI355X_288GBx8/llama2-70b-99/Server/performance/run_1","vLLM","Ubuntu 22.04","256 CUs per GPU, gfx950 architecture","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute XD685 (8x AMD Instinct MI355X 288GB, vLLM)","HPE_ProLiant_XD685_MI355X_288GBx8","llama2-70b-99.9","llama2-70b-99.9","Interactive","70954.1","ROUGE1: 44.5404 ROUGE2: 22.1352 ROUGEL: 28.9123 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9575F","2","128","AMD Instinct MI355X 288GB HBM3e","8","8","closed/HPE/results/HPE_ProLiant_XD685_MI355X_288GBx8/llama2-70b-99.9/Interactive/performance/run_1","vLLM","Ubuntu 22.04","256 CUs per GPU, gfx950 architecture","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute XD685 (8x AMD Instinct MI355X 288GB, vLLM)","HPE_ProLiant_XD685_MI355X_288GBx8","llama2-70b-99.9","llama2-70b-99.9","Offline","100019.0","ROUGE1: 44.4435 ROUGE2: 22.0815 ROUGEL: 28.8533 TOKENS_PER_SAMPLE: 284.2","1","AMD EPYC 9575F","2","128","AMD Instinct MI355X 288GB HBM3e","8","8","closed/HPE/results/HPE_ProLiant_XD685_MI355X_288GBx8/llama2-70b-99.9/Offline/performance/run_1","vLLM","Ubuntu 22.04","256 CUs per GPU, gfx950 architecture","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute XD685 (8x AMD Instinct MI355X 288GB, vLLM)","HPE_ProLiant_XD685_MI355X_288GBx8","llama2-70b-99.9","llama2-70b-99.9","Server","95544.3","ROUGE1: 44.5391 ROUGE2: 22.1346 ROUGEL: 28.91 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9575F","2","128","AMD Instinct MI355X 288GB HBM3e","8","8","closed/HPE/results/HPE_ProLiant_XD685_MI355X_288GBx8/llama2-70b-99.9/Server/performance/run_1","vLLM","Ubuntu 22.04","256 CUs per GPU, gfx950 architecture","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute XD685 (8x NVIDIA B200 180GB, TensorRT)","HPE_ProLiant_XD685_B200_SXM_192GBx8_TRT","llama2-70b-99","llama2-70b-99","Interactive","61644.7","ROUGE1: 44.7586 ROUGE2: 22.3423 ROUGEL: 29.1305 TOKENS_PER_SAMPLE: 274.4","1","AMD EPYC 9535","2","64","NVIDIA B200-SXM-180GB","8","8","closed/HPE/results/HPE_ProLiant_XD685_B200_SXM_192GBx8_TRT/llama2-70b-99/Interactive/performance/run_1","TensorRT 10.11, CUDA 13.0","Ubuntu 24.04.3","B200 TGP 1000W","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute XD685 (8x NVIDIA B200 180GB, TensorRT)","HPE_ProLiant_XD685_B200_SXM_192GBx8_TRT","llama2-70b-99","llama2-70b-99","Offline","103954.0","ROUGE1: 44.7474 ROUGE2: 22.3568 ROUGEL: 29.1367 TOKENS_PER_SAMPLE: 273.3","1","AMD EPYC 9535","2","64","NVIDIA B200-SXM-180GB","8","8","closed/HPE/results/HPE_ProLiant_XD685_B200_SXM_192GBx8_TRT/llama2-70b-99/Offline/performance/run_1","TensorRT 10.11, CUDA 13.0","Ubuntu 24.04.3","B200 TGP 1000W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute XD685 (8x NVIDIA B200 180GB, TensorRT)","HPE_ProLiant_XD685_B200_SXM_192GBx8_TRT","llama2-70b-99","llama2-70b-99","Server","103635.0","ROUGE1: 44.7537 ROUGE2: 22.361 ROUGEL: 29.1392 TOKENS_PER_SAMPLE: 274.3","1","AMD EPYC 9535","2","64","NVIDIA B200-SXM-180GB","8","8","closed/HPE/results/HPE_ProLiant_XD685_B200_SXM_192GBx8_TRT/llama2-70b-99/Server/performance/run_1","TensorRT 10.11, CUDA 13.0","Ubuntu 24.04.3","B200 TGP 1000W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute XD685 (8x NVIDIA B200 180GB, TensorRT)","HPE_ProLiant_XD685_B200_SXM_192GBx8_TRT","llama2-70b-99.9","llama2-70b-99.9","Interactive","61300.4","ROUGE1: 44.7627 ROUGE2: 22.343 ROUGEL: 29.1316 TOKENS_PER_SAMPLE: 274.4","1","AMD EPYC 9535","2","64","NVIDIA B200-SXM-180GB","8","8","closed/HPE/results/HPE_ProLiant_XD685_B200_SXM_192GBx8_TRT/llama2-70b-99.9/Interactive/performance/run_1","TensorRT 10.11, CUDA 13.0","Ubuntu 24.04.3","B200 TGP 1000W","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute XD685 (8x NVIDIA B200 180GB, TensorRT)","HPE_ProLiant_XD685_B200_SXM_192GBx8_TRT","llama2-70b-99.9","llama2-70b-99.9","Offline","104572.0","ROUGE1: 44.7474 ROUGE2: 22.3568 ROUGEL: 29.1367 TOKENS_PER_SAMPLE: 273.3","1","AMD EPYC 9535","2","64","NVIDIA B200-SXM-180GB","8","8","closed/HPE/results/HPE_ProLiant_XD685_B200_SXM_192GBx8_TRT/llama2-70b-99.9/Offline/performance/run_1","TensorRT 10.11, CUDA 13.0","Ubuntu 24.04.3","B200 TGP 1000W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute XD685 (8x NVIDIA B200 180GB, TensorRT)","HPE_ProLiant_XD685_B200_SXM_192GBx8_TRT","llama2-70b-99.9","llama2-70b-99.9","Server","103627.0","ROUGE1: 44.7454 ROUGE2: 22.3541 ROUGEL: 29.1337 TOKENS_PER_SAMPLE: 274.3","1","AMD EPYC 9535","2","64","NVIDIA B200-SXM-180GB","8","8","closed/HPE/results/HPE_ProLiant_XD685_B200_SXM_192GBx8_TRT/llama2-70b-99.9/Server/performance/run_1","TensorRT 10.11, CUDA 13.0","Ubuntu 24.04.3","B200 TGP 1000W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute XD685 (8x NVIDIA B200 180GB, TensorRT)","HPE_ProLiant_XD685_B200_SXM_192GBx8_TRT","llama3.1-8b","llama3.1-8b","Interactive","126143.0","ROUGE1: 38.4119 ROUGE2: 15.7662 ROUGEL: 24.2994 ROUGELSUM: 35.4992 GEN_LEN: 8201032","1","AMD EPYC 9535","2","64","NVIDIA B200-SXM-180GB","8","8","closed/HPE/results/HPE_ProLiant_XD685_B200_SXM_192GBx8_TRT/llama3.1-8b/Interactive/performance/run_1","TensorRT 10.11, CUDA 13.0","Ubuntu 24.04.3","B200 TGP 1000W","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute XD685 (8x NVIDIA B200 180GB, TensorRT)","HPE_ProLiant_XD685_B200_SXM_192GBx8_TRT","llama3.1-8b","llama3.1-8b","Offline","152625.0","ROUGE1: 38.4165 ROUGE2: 15.7673 ROUGEL: 24.3028 ROUGELSUM: 35.5051 GEN_LEN: 8201311","1","AMD EPYC 9535","2","64","NVIDIA B200-SXM-180GB","8","8","closed/HPE/results/HPE_ProLiant_XD685_B200_SXM_192GBx8_TRT/llama3.1-8b/Offline/performance/run_1","TensorRT 10.11, CUDA 13.0","Ubuntu 24.04.3","B200 TGP 1000W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute XD685 (8x NVIDIA B200 180GB, TensorRT)","HPE_ProLiant_XD685_B200_SXM_192GBx8_TRT","llama3.1-8b","llama3.1-8b","Server","131270.0","ROUGE1: 38.4148 ROUGE2: 15.7669 ROUGEL: 24.3024 ROUGELSUM: 35.5024 GEN_LEN: 8201083","1","AMD EPYC 9535","2","64","NVIDIA B200-SXM-180GB","8","8","closed/HPE/results/HPE_ProLiant_XD685_B200_SXM_192GBx8_TRT/llama3.1-8b/Server/performance/run_1","TensorRT 10.11, CUDA 13.0","Ubuntu 24.04.3","B200 TGP 1000W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute XD685 (8x NVIDIA B200 180GB, TensorRT)","HPE_ProLiant_XD685_B200_SXM_192GBx8_TRT","mixtral-8x7b","mixtral-8x7b","Offline","126468.0","ROUGE1: 45.4127 ROUGE2: 23.1305 ROUGEL: 30.2484 TOKENS_PER_SAMPLE: 144.4 gsm8k_accuracy: 73.42 mbxp_accuracy: 60.34","1","AMD EPYC 9535","2","64","NVIDIA B200-SXM-180GB","8","8","closed/HPE/results/HPE_ProLiant_XD685_B200_SXM_192GBx8_TRT/mixtral-8x7b/Offline/performance/run_1","TensorRT 10.11, CUDA 13.0","Ubuntu 24.04.3","B200 TGP 1000W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute XD685 (8x NVIDIA B200 180GB, TensorRT)","HPE_ProLiant_XD685_B200_SXM_192GBx8_TRT","mixtral-8x7b","mixtral-8x7b","Server","127209.0","ROUGE1: 45.4142 ROUGE2: 23.2853 ROUGEL: 30.2919 TOKENS_PER_SAMPLE: 146.0 gsm8k_accuracy: 73.02 mbxp_accuracy: 60.16","1","AMD EPYC 9535","2","64","NVIDIA B200-SXM-180GB","8","8","closed/HPE/results/HPE_ProLiant_XD685_B200_SXM_192GBx8_TRT/mixtral-8x7b/Server/performance/run_1","TensorRT 10.11, CUDA 13.0","Ubuntu 24.04.3","B200 TGP 1000W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant Compute XD685 (8x NVIDIA B200 180GB, TensorRT)","HPE_ProLiant_XD685_B200_SXM_192GBx8_TRT","whisper","whisper","Offline","46033.9","ACCURACY: 97.8377761070453","1","AMD EPYC 9535","2","64","NVIDIA B200-SXM-180GB","8","8","closed/HPE/results/HPE_ProLiant_XD685_B200_SXM_192GBx8_TRT/whisper/Offline/performance/run_1","TensorRT 10.11, CUDA 13.0","Ubuntu 24.04.3","B200 TGP 1000W","closed","0","v6.0","0","False","Samples/s","fp16","","" +"HPE","available","closed","datacenter","HPE ProLiant DL145 Gen11 (1x NVIDIA RTX PRO 4500 Blackwell, TensorRT)","HPE_PROLIANT_DL145_RTX_PRO_4500_PCIE_32GBx1_TRT_DC_TRT","llama3.1-8b","llama3.1-8b","Offline","2340.75","ROUGE1: 38.5879 ROUGE2: 15.8594 ROUGEL: 24.3118 ROUGELSUM: 35.5388 GEN_LEN: 8181331","1","AMD EPYC 8434P","1","48","NVIDIA RTX PRO 4500 Blackwell","1","1","closed/HPE/results/HPE_PROLIANT_DL145_RTX_PRO_4500_PCIE_32GBx1_TRT_DC_TRT/llama3.1-8b/Offline/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL145 Gen11 (1x NVIDIA RTX PRO 4500 Blackwell, TensorRT)","HPE_PROLIANT_DL145_RTX_PRO_4500_PCIE_32GBx1_TRT_DC_TRT","llama3.1-8b","llama3.1-8b","Server","2081.34","ROUGE1: 38.587 ROUGE2: 15.8592 ROUGEL: 24.3112 ROUGELSUM: 35.538 GEN_LEN: 8181380","1","AMD EPYC 8434P","1","48","NVIDIA RTX PRO 4500 Blackwell","1","1","closed/HPE/results/HPE_PROLIANT_DL145_RTX_PRO_4500_PCIE_32GBx1_TRT_DC_TRT/llama3.1-8b/Server/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (2x NVIDIA RTX PRO 6000 Blackwell Server Edition-600W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_11_0_33_TRT","llama2-70b-99","llama2-70b-99","Offline","6862.3","ROUGE1: 44.7589 ROUGE2: 22.3391 ROUGEL: 29.1509 TOKENS_PER_SAMPLE: 272.7","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","2","2","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_11_0_33_TRT/llama2-70b-99/Offline/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 600W / 600W (full power)","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (2x NVIDIA RTX PRO 6000 Blackwell Server Edition-600W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_11_0_33_TRT","llama2-70b-99","llama2-70b-99","Server","6662.33","ROUGE1: 44.7582 ROUGE2: 22.3381 ROUGEL: 29.1502 TOKENS_PER_SAMPLE: 273.8","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","2","2","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_11_0_33_TRT/llama2-70b-99/Server/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 600W / 600W (full power)","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (2x NVIDIA RTX PRO 6000 Blackwell Server Edition-600W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_11_0_33_TRT","llama3.1-8b","llama3.1-8b","Interactive","10459.6","ROUGE1: 38.5865 ROUGE2: 15.8596 ROUGEL: 24.3107 ROUGELSUM: 35.538 GEN_LEN: 8181239","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","2","2","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_11_0_33_TRT/llama3.1-8b/Interactive/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 600W / 600W (full power)","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (2x NVIDIA RTX PRO 6000 Blackwell Server Edition-600W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_11_0_33_TRT","llama3.1-8b","llama3.1-8b","Offline","12433.3","ROUGE1: 38.5864 ROUGE2: 15.8586 ROUGEL: 24.3112 ROUGELSUM: 35.5374 GEN_LEN: 8181316","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","2","2","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_11_0_33_TRT/llama3.1-8b/Offline/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 600W / 600W (full power)","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (2x NVIDIA RTX PRO 6000 Blackwell Server Edition-600W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_11_0_33_TRT","llama3.1-8b","llama3.1-8b","Server","11898.8","ROUGE1: 38.5854 ROUGE2: 15.8591 ROUGEL: 24.3111 ROUGELSUM: 35.5366 GEN_LEN: 8181448","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","2","2","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_11_0_33_TRT/llama3.1-8b/Server/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 600W / 600W (full power)","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (2x NVIDIA RTX PRO 6000 Blackwell Server Edition-600W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_11_0_33_TRT","mixtral-8x7b","mixtral-8x7b","Offline","8198.25","ROUGE1: 45.5367 ROUGE2: 23.3779 ROUGEL: 30.4572 TOKENS_PER_SAMPLE: 144.5 gsm8k_accuracy: 73.34 mbxp_accuracy: 60.06","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","2","2","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_11_0_33_TRT/mixtral-8x7b/Offline/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 600W / 600W (full power)","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (2x NVIDIA RTX PRO 6000 Blackwell Server Edition-600W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_11_0_33_TRT","mixtral-8x7b","mixtral-8x7b","Server","7742.84","ROUGE1: 45.3405 ROUGE2: 23.2703 ROUGEL: 30.3729 TOKENS_PER_SAMPLE: 146.0 gsm8k_accuracy: 73.52 mbxp_accuracy: 59.78","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","2","2","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_11_0_33_TRT/mixtral-8x7b/Server/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 600W / 600W (full power)","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (2x NVIDIA RTX PRO 6000 Blackwell Server Edition-600W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_14_1_48_TRT","llama2-70b-99","llama2-70b-99","Offline","6901.9","ROUGE1: 44.8714 ROUGE2: 22.41 ROUGEL: 29.1982 TOKENS_PER_SAMPLE: 271.8","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","2","2","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_14_1_48_TRT/llama2-70b-99/Offline/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 600W / 600W (full power)","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (2x NVIDIA RTX PRO 6000 Blackwell Server Edition-600W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_14_1_48_TRT","llama2-70b-99","llama2-70b-99","Server","6952.97","ROUGE1: 44.8708 ROUGE2: 22.4088 ROUGEL: 29.1982 TOKENS_PER_SAMPLE: 272.9","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","2","2","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_14_1_48_TRT/llama2-70b-99/Server/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 600W / 600W (full power)","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (2x NVIDIA RTX PRO 6000 Blackwell Server Edition-600W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_14_1_48_TRT","llama2-70b-99.9","llama2-70b-99.9","Offline","6893.36","ROUGE1: 44.8714 ROUGE2: 22.41 ROUGEL: 29.1982 TOKENS_PER_SAMPLE: 271.8","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","2","2","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_14_1_48_TRT/llama2-70b-99.9/Offline/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 600W / 600W (full power)","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (2x NVIDIA RTX PRO 6000 Blackwell Server Edition-600W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_14_1_48_TRT","llama2-70b-99.9","llama2-70b-99.9","Server","6953.48","ROUGE1: 44.8727 ROUGE2: 22.4101 ROUGEL: 29.1991 TOKENS_PER_SAMPLE: 272.9","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","2","2","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_14_1_48_TRT/llama2-70b-99.9/Server/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 600W / 600W (full power)","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (2x NVIDIA RTX PRO 6000 Blackwell Server Edition-600W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_14_1_48_TRT","llama3.1-8b","llama3.1-8b","Interactive","10589.3","ROUGE1: 38.7609 ROUGE2: 15.9725 ROUGEL: 24.5197 ROUGELSUM: 35.6577 GEN_LEN: 8148454","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","2","2","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_14_1_48_TRT/llama3.1-8b/Interactive/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 600W / 600W (full power)","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (2x NVIDIA RTX PRO 6000 Blackwell Server Edition-600W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_14_1_48_TRT","llama3.1-8b","llama3.1-8b","Offline","12495.2","ROUGE1: 38.7643 ROUGE2: 15.9777 ROUGEL: 24.5156 ROUGELSUM: 35.6656 GEN_LEN: 8148206","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","2","2","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_14_1_48_TRT/llama3.1-8b/Offline/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 600W / 600W (full power)","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (2x NVIDIA RTX PRO 6000 Blackwell Server Edition-600W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_14_1_48_TRT","llama3.1-8b","llama3.1-8b","Server","11899.7","ROUGE1: 38.7631 ROUGE2: 15.9835 ROUGEL: 24.5211 ROUGELSUM: 35.6664 GEN_LEN: 8148571","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","2","2","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_14_1_48_TRT/llama3.1-8b/Server/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 600W / 600W (full power)","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (2x NVIDIA RTX PRO 6000 Blackwell Server Edition-600W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_14_1_48_TRT","whisper","whisper","Offline","4712.66","ACCURACY: 97.82254917822169","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","2","2","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx2_TRT_10_14_1_48_TRT/whisper/Offline/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 600W / 600W (full power)","closed","0","v6.0","0","False","Samples/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (4x NVIDIA RTX PRO 6000 Blackwell Server Edition-450W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_11_0_33_TRT","llama3.1-8b","llama3.1-8b","Interactive","19991.1","ROUGE1: 38.5864 ROUGE2: 15.8598 ROUGEL: 24.3121 ROUGELSUM: 35.5377 GEN_LEN: 8181396","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","4","4","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_11_0_33_TRT/llama3.1-8b/Interactive/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 450W / 600W (reduced)","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (4x NVIDIA RTX PRO 6000 Blackwell Server Edition-450W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_11_0_33_TRT","llama3.1-8b","llama3.1-8b","Offline","23265.7","ROUGE1: 38.5869 ROUGE2: 15.8583 ROUGEL: 24.3109 ROUGELSUM: 35.5384 GEN_LEN: 8181409","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","4","4","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_11_0_33_TRT/llama3.1-8b/Offline/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 450W / 600W (reduced)","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (4x NVIDIA RTX PRO 6000 Blackwell Server Edition-450W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_11_0_33_TRT","llama3.1-8b","llama3.1-8b","Server","22916.3","ROUGE1: 38.5867 ROUGE2: 15.8599 ROUGEL: 24.3123 ROUGELSUM: 35.5373 GEN_LEN: 8181397","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","4","4","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_11_0_33_TRT/llama3.1-8b/Server/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 450W / 600W (reduced)","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (4x NVIDIA RTX PRO 6000 Blackwell Server Edition-450W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_11_0_33_TRT","mixtral-8x7b","mixtral-8x7b","Offline","14828.3","ROUGE1: 45.378 ROUGE2: 23.1949 ROUGEL: 30.2877 TOKENS_PER_SAMPLE: 144.9 gsm8k_accuracy: 73.8 mbxp_accuracy: 60.12","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","4","4","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_11_0_33_TRT/mixtral-8x7b/Offline/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 450W / 600W (reduced)","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (4x NVIDIA RTX PRO 6000 Blackwell Server Edition-450W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_11_0_33_TRT","mixtral-8x7b","mixtral-8x7b","Server","13846.1","ROUGE1: 45.3775 ROUGE2: 23.2622 ROUGEL: 30.3288 TOKENS_PER_SAMPLE: 145.9 gsm8k_accuracy: 73.52 mbxp_accuracy: 59.98","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","4","4","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_11_0_33_TRT/mixtral-8x7b/Server/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 450W / 600W (reduced)","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (4x NVIDIA RTX PRO 6000 Blackwell Server Edition-450W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_14_1_48_TRT","gpt-oss-120b","gpt-oss-120b","Offline","7297.05","exact_match: 83.972","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","4","4","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_14_1_48_TRT/gpt-oss-120b/Offline/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 450W / 600W (reduced)","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (4x NVIDIA RTX PRO 6000 Blackwell Server Edition-450W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_14_1_48_TRT","gpt-oss-120b","gpt-oss-120b","Server","6687.25","exact_match: 84.122","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","4","4","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_14_1_48_TRT/gpt-oss-120b/Server/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 450W / 600W (reduced)","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (4x NVIDIA RTX PRO 6000 Blackwell Server Edition-450W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_14_1_48_TRT","llama2-70b-99","llama2-70b-99","Offline","13361.9","ROUGE1: 44.8707 ROUGE2: 22.409 ROUGEL: 29.198 TOKENS_PER_SAMPLE: 271.9","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","4","4","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_14_1_48_TRT/llama2-70b-99/Offline/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 450W / 600W (reduced)","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (4x NVIDIA RTX PRO 6000 Blackwell Server Edition-450W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_14_1_48_TRT","llama2-70b-99","llama2-70b-99","Server","12822.2","ROUGE1: 44.8723 ROUGE2: 22.4108 ROUGEL: 29.1999 TOKENS_PER_SAMPLE: 272.8","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","4","4","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_14_1_48_TRT/llama2-70b-99/Server/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 450W / 600W (reduced)","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (4x NVIDIA RTX PRO 6000 Blackwell Server Edition-450W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_14_1_48_TRT","llama2-70b-99.9","llama2-70b-99.9","Offline","13200.3","ROUGE1: 44.8711 ROUGE2: 22.4097 ROUGEL: 29.1987 TOKENS_PER_SAMPLE: 271.9","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","4","4","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_14_1_48_TRT/llama2-70b-99.9/Offline/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 450W / 600W (reduced)","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (4x NVIDIA RTX PRO 6000 Blackwell Server Edition-450W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_14_1_48_TRT","llama2-70b-99.9","llama2-70b-99.9","Server","12572.6","ROUGE1: 44.8739 ROUGE2: 22.4139 ROUGEL: 29.2026 TOKENS_PER_SAMPLE: 272.8","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","4","4","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_14_1_48_TRT/llama2-70b-99.9/Server/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 450W / 600W (reduced)","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (4x NVIDIA RTX PRO 6000 Blackwell Server Edition-450W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_14_1_48_TRT","llama3.1-8b","llama3.1-8b","Interactive","20118.9","ROUGE1: 38.7669 ROUGE2: 16.003 ROUGEL: 24.5363 ROUGELSUM: 35.6914 GEN_LEN: 8180254","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","4","4","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_14_1_48_TRT/llama3.1-8b/Interactive/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 450W / 600W (reduced)","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (4x NVIDIA RTX PRO 6000 Blackwell Server Edition-450W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_14_1_48_TRT","llama3.1-8b","llama3.1-8b","Offline","23556.7","ROUGE1: 38.7715 ROUGE2: 16.0183 ROUGEL: 24.5357 ROUGELSUM: 35.696 GEN_LEN: 8178815","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","4","4","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_14_1_48_TRT/llama3.1-8b/Offline/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 450W / 600W (reduced)","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (4x NVIDIA RTX PRO 6000 Blackwell Server Edition-450W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_14_1_48_TRT","llama3.1-8b","llama3.1-8b","Server","23227.2","ROUGE1: 38.7528 ROUGE2: 16.008 ROUGEL: 24.5234 ROUGELSUM: 35.6804 GEN_LEN: 8178500","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","4","4","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_14_1_48_TRT/llama3.1-8b/Server/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 450W / 600W (reduced)","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"HPE","available","closed","datacenter","HPE ProLiant DL385 Gen11 (4x NVIDIA RTX PRO 6000 Blackwell Server Edition-450W, TensorRT)","HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_14_1_48_TRT","whisper","whisper","Offline","9228.36","ACCURACY: 97.81493571380987","1","AMD EPYC 9475F","2","96","NVIDIA RTX PRO 6000 Blackwell Server Edition","4","4","closed/HPE/results/HPE_Proliant_DL385_RTX_PRO_6000_PCIe_96GBx4_TRT_10_14_1_48_TRT/whisper/Offline/performance/run_1","TensorRT 10.14.1.48, CUDA 13.0","Ubuntu 24.04.3 LTS","GPU 450W / 600W (reduced)","closed","0","v6.0","0","False","Samples/s","fp4","","" +"HPE","available","closed","edge","HPE ProLiant Compute DL380 Gen12 (5x NVIDIA RTX PRO 4500 Blackwell, TensorRT)","HPE_Proliant_DL380_RTX_PRO_4500_PCIe_32GBx5_TRT","resnet","resnet","MultiStream","11.077851","acc: 76.138","1","Intel(R) Xeon(R) 6747P","2","48","NVIDIA RTX PRO 4500 Blackwell","5","5","closed/HPE/results/HPE_Proliant_DL380_RTX_PRO_4500_PCIe_32GBx5_TRT/resnet/MultiStream/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3","","closed","0","v6.0","0","False","Latency (ms)","int8","","" +"HPE","available","closed","edge","HPE ProLiant Compute DL380 Gen12 (5x NVIDIA RTX PRO 4500 Blackwell, TensorRT)","HPE_Proliant_DL380_RTX_PRO_4500_PCIe_32GBx5_TRT","resnet","resnet","Offline","94650.5","acc: 76.138","1","Intel(R) Xeon(R) 6747P","2","48","NVIDIA RTX PRO 4500 Blackwell","5","5","closed/HPE/results/HPE_Proliant_DL380_RTX_PRO_4500_PCIe_32GBx5_TRT/resnet/Offline/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3","","closed","0","v6.0","0","False","Samples/s","int8","","" +"HPE","available","closed","edge","HPE ProLiant Compute DL380 Gen12 (5x NVIDIA RTX PRO 4500 Blackwell, TensorRT)","HPE_Proliant_DL380_RTX_PRO_4500_PCIe_32GBx5_TRT","resnet","resnet","SingleStream","10.991706","acc: 76.138","1","Intel(R) Xeon(R) 6747P","2","48","NVIDIA RTX PRO 4500 Blackwell","5","5","closed/HPE/results/HPE_Proliant_DL380_RTX_PRO_4500_PCIe_32GBx5_TRT/resnet/SingleStream/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3","","closed","0","v6.0","0","False","Latency (ms)","int8","","" +"HPE","available","closed","edge","HPE ProLiant Compute DL380 Gen12 (5x NVIDIA RTX PRO 4500 Blackwell, TensorRT)","HPE_Proliant_DL380_RTX_PRO_4500_PCIe_32GBx5_TRT","whisper","whisper","Offline","4513.09","ACCURACY: 97.80922561550102","1","Intel(R) Xeon(R) 6747P","2","48","NVIDIA RTX PRO 4500 Blackwell","5","5","closed/HPE/results/HPE_Proliant_DL380_RTX_PRO_4500_PCIe_32GBx5_TRT/whisper/Offline/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3","","closed","0","v6.0","0","False","Samples/s","fp16","","" +"HPE","available","closed","edge","HPE ProLiant DL145 Gen11 (1x NVIDIA RTX PRO 4500 Blackwell, TensorRT)","HPE_PROLIANT_DL145_RTX_PRO_4500_PCIE_32GBx1_TRT_TRT","resnet","resnet","MultiStream","10.570551","acc: 76.138","1","AMD EPYC 8434P","1","48","NVIDIA RTX PRO 4500 Blackwell","1","1","closed/HPE/results/HPE_PROLIANT_DL145_RTX_PRO_4500_PCIE_32GBx1_TRT_TRT/resnet/MultiStream/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3","","closed","0","v6.0","0","False","Latency (ms)","int8","","" +"HPE","available","closed","edge","HPE ProLiant DL145 Gen11 (1x NVIDIA RTX PRO 4500 Blackwell, TensorRT)","HPE_PROLIANT_DL145_RTX_PRO_4500_PCIE_32GBx1_TRT_TRT","resnet","resnet","Offline","19210.7","acc: 76.138","1","AMD EPYC 8434P","1","48","NVIDIA RTX PRO 4500 Blackwell","1","1","closed/HPE/results/HPE_PROLIANT_DL145_RTX_PRO_4500_PCIE_32GBx1_TRT_TRT/resnet/Offline/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3","","closed","0","v6.0","0","False","Samples/s","int8","","" +"HPE","available","closed","edge","HPE ProLiant DL145 Gen11 (1x NVIDIA RTX PRO 4500 Blackwell, TensorRT)","HPE_PROLIANT_DL145_RTX_PRO_4500_PCIE_32GBx1_TRT_TRT","resnet","resnet","SingleStream","10.561798","acc: 76.138","1","AMD EPYC 8434P","1","48","NVIDIA RTX PRO 4500 Blackwell","1","1","closed/HPE/results/HPE_PROLIANT_DL145_RTX_PRO_4500_PCIE_32GBx1_TRT_TRT/resnet/SingleStream/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3","","closed","0","v6.0","0","False","Latency (ms)","int8","","" +"HPE","available","closed","edge","HPE ProLiant DL145 Gen11 (1x NVIDIA RTX PRO 4500 Blackwell, TensorRT)","HPE_PROLIANT_DL145_RTX_PRO_4500_PCIE_32GBx1_TRT_TRT","whisper","whisper","Offline","916.743","ACCURACY: 97.8425345223027","1","AMD EPYC 8434P","1","48","NVIDIA RTX PRO 4500 Blackwell","1","1","closed/HPE/results/HPE_PROLIANT_DL145_RTX_PRO_4500_PCIE_32GBx1_TRT_TRT/whisper/Offline/performance/run_1","TensorRT 10.11.0.33, CUDA 13.0","Ubuntu 24.04.3","","closed","0","v6.0","0","False","Samples/s","fp16","","" +"Intel","available","closed","datacenter","1-node-2S-GNR_128C","1-node-2S-GNR_128C","llama3_1-8b","llama3.1-8b","Offline","1229.56","ROUGE1: 38.8062 ROUGE2: 16.0222 ROUGEL: 24.5645 ROUGELSUM: 35.8805 GEN_LEN: 8167163","1","Intel(R) Xeon(R) 6980P","2","128","N/A","0","0","closed/Intel/results/1-node-2S-GNR_128C/llama3_1-8b/Offline/performance/run_1","PyTorch","CentOS Stream 9","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Intel","available","closed","datacenter","1-node-2S-GNR_128C","1-node-2S-GNR_128C","llama3_1-8b","llama3.1-8b","Server","471.541","ROUGE1: 38.792 ROUGE2: 15.9934 ROUGEL: 24.5338 ROUGELSUM: 35.8672 GEN_LEN: 8163459","1","Intel(R) Xeon(R) 6980P","2","128","N/A","0","0","closed/Intel/results/1-node-2S-GNR_128C/llama3_1-8b/Server/performance/run_1","PyTorch","CentOS Stream 9","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Intel","available","closed","datacenter","1-node-2S-GNR_128C","1-node-2S-GNR_128C","rgat","rgat","Offline","24149.8","acc: 72.565","1","Intel(R) Xeon(R) 6980P","2","128","N/A","0","0","closed/Intel/results/1-node-2S-GNR_128C/rgat/Offline/performance/run_1","PyTorch","CentOS Stream 9","N/A. N/A","closed","0","v6.0","0","False","Samples/s","INT8","","" +"Intel","available","closed","datacenter","1-node-2S-GNR_128C","1-node-2S-GNR_128C","whisper","whisper","Offline","2200.8","ACCURACY: 97.89147168805961","1","Intel(R) Xeon(R) 6980P","2","128","N/A","0","0","closed/Intel/results/1-node-2S-GNR_128C/whisper/Offline/performance/run_1","PyTorch","CentOS Stream 9","N/A. N/A","closed","0","v6.0","0","False","Samples/s","INT8","","" +"Intel","available","closed","datacenter","1-node-4x-BMG-B50","1-node-4x-BMG-B50","llama3_1-8b","llama3.1-8b","Offline","1721.84","ROUGE1: 38.7265 ROUGE2: 16.0947 ROUGEL: 24.6051 ROUGELSUM: 35.8654 GEN_LEN: 8149276","1","Intel(R) Xeon(R) Platinum 8480C","2","56","Intel(R) Arc Pro(R) B50","4","4","closed/Intel/results/1-node-4x-BMG-B50/llama3_1-8b/Offline/performance/run_1","PyTorch vLLM","Ubuntu 25.04","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Intel","available","closed","datacenter","1-node-4x-BMG-B50","1-node-4x-BMG-B50","llama3_1-8b","llama3.1-8b","Server","1187.13","ROUGE1: 38.7283 ROUGE2: 16.0975 ROUGEL: 24.6103 ROUGELSUM: 35.8691 GEN_LEN: 8149111","1","Intel(R) Xeon(R) Platinum 8480C","2","56","Intel(R) Arc Pro(R) B50","4","4","closed/Intel/results/1-node-4x-BMG-B50/llama3_1-8b/Server/performance/run_1","PyTorch vLLM","Ubuntu 25.04","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Intel","available","closed","datacenter","1-node-4x-BMG-B50","1-node-4x-BMG-B50","whisper","whisper","Offline","2112.35","ACCURACY: 97.94194045614135","1","Intel(R) Xeon(R) Platinum 8480C","2","56","Intel(R) Arc Pro(R) B50","4","4","closed/Intel/results/1-node-4x-BMG-B50/whisper/Offline/performance/run_1","PyTorch vLLM","Ubuntu 25.04","N/A. N/A","closed","0","v6.0","0","False","Samples/s","INT8","","" +"Intel","available","closed","datacenter","1-node-4x-BMG-B60","1-node-4x-BMG-B60","gpt-oss-120b","gpt-oss-120b","Offline","841.038","exact_match: 83.405","1","Intel(R) Xeon(R) 698X","1","86","Intel(R) Arc Pro(R) B60","4","4","closed/Intel/results/1-node-4x-BMG-B60/gpt-oss-120b/Offline/performance/run_1","PyTorch vLLM","Ubuntu 25.04","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","MXFP4","","" +"Intel","available","closed","datacenter","1-node-4x-BMG-B60","1-node-4x-BMG-B60","gpt-oss-120b","gpt-oss-120b","Server","452.191","exact_match: 83.582","1","Intel(R) Xeon(R) 698X","1","86","Intel(R) Arc Pro(R) B60","4","4","closed/Intel/results/1-node-4x-BMG-B60/gpt-oss-120b/Server/performance/run_1","PyTorch vLLM","Ubuntu 25.04","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","MXFP4","","" +"Intel","available","closed","datacenter","1-node-4x-BMG-B60","1-node-4x-BMG-B60","llama2-70b-99","llama2-70b-99","Offline","1697.66","ROUGE1: 44.8085 ROUGE2: 22.3222 ROUGEL: 29.0232 TOKENS_PER_SAMPLE: 284.5","1","Intel(R) Xeon(R) 698X","1","86","Intel(R) Arc Pro(R) B60","4","4","closed/Intel/results/1-node-4x-BMG-B60/llama2-70b-99/Offline/performance/run_1","PyTorch vLLM","Ubuntu 25.04","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Intel","available","closed","datacenter","1-node-4x-BMG-B60","1-node-4x-BMG-B60","llama2-70b-99","llama2-70b-99","Server","1106.26","ROUGE1: 44.7707 ROUGE2: 22.2808 ROUGEL: 28.9988 TOKENS_PER_SAMPLE: 283.4","1","Intel(R) Xeon(R) 698X","1","86","Intel(R) Arc Pro(R) B60","4","4","closed/Intel/results/1-node-4x-BMG-B60/llama2-70b-99/Server/performance/run_1","PyTorch vLLM","Ubuntu 25.04","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Intel","available","closed","datacenter","1-node-4x-BMG-B60","1-node-4x-BMG-B60","llama2-70b-99.9","llama2-70b-99.9","Offline","1697.66","ROUGE1: 44.8085 ROUGE2: 22.3222 ROUGEL: 29.0232 TOKENS_PER_SAMPLE: 284.5","1","Intel(R) Xeon(R) 698X","1","86","Intel(R) Arc Pro(R) B60","4","4","closed/Intel/results/1-node-4x-BMG-B60/llama2-70b-99.9/Offline/performance/run_1","PyTorch vLLM","Ubuntu 25.04","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Intel","available","closed","datacenter","1-node-4x-BMG-B60","1-node-4x-BMG-B60","llama2-70b-99.9","llama2-70b-99.9","Server","1106.26","ROUGE1: 44.7707 ROUGE2: 22.2808 ROUGEL: 28.9988 TOKENS_PER_SAMPLE: 283.4","1","Intel(R) Xeon(R) 698X","1","86","Intel(R) Arc Pro(R) B60","4","4","closed/Intel/results/1-node-4x-BMG-B60/llama2-70b-99.9/Server/performance/run_1","PyTorch vLLM","Ubuntu 25.04","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Intel","available","closed","datacenter","1-node-4x-BMG-B60","1-node-4x-BMG-B60","llama3_1-8b","llama3.1-8b","Offline","3347.43","ROUGE1: 38.7291 ROUGE2: 16.0967 ROUGEL: 24.6066 ROUGELSUM: 35.8688 GEN_LEN: 8149281","1","Intel(R) Xeon(R) 698X","1","86","Intel(R) Arc Pro(R) B60","4","4","closed/Intel/results/1-node-4x-BMG-B60/llama3_1-8b/Offline/performance/run_1","PyTorch vLLM","Ubuntu 25.04","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Intel","available","closed","datacenter","1-node-4x-BMG-B60","1-node-4x-BMG-B60","llama3_1-8b","llama3.1-8b","Server","3145.22","ROUGE1: 38.7301 ROUGE2: 16.0968 ROUGEL: 24.6083 ROUGELSUM: 35.8698 GEN_LEN: 8149242","1","Intel(R) Xeon(R) 698X","1","86","Intel(R) Arc Pro(R) B60","4","4","closed/Intel/results/1-node-4x-BMG-B60/llama3_1-8b/Server/performance/run_1","PyTorch vLLM","Ubuntu 25.04","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Intel","available","closed","datacenter","1-node-4x-BMG-B60","1-node-4x-BMG-B60","whisper","whisper","Offline","3995.63","ACCURACY: 97.45197476712876","1","Intel(R) Xeon(R) 698X","1","86","Intel(R) Arc Pro(R) B60","4","4","closed/Intel/results/1-node-4x-BMG-B60/whisper/Offline/performance/run_1","PyTorch vLLM","Ubuntu 25.04","N/A. N/A","closed","0","v6.0","0","False","Samples/s","INT4","","" +"Intel","available","closed","datacenter","1-node-4x-BMG-Pro-B60-Dual","1-node-4x-BMG-Pro-B60-Dual","gpt-oss-120b","gpt-oss-120b","Offline","1601.91","exact_match: 82.528","1","Intel(R) Xeon(R) 698X","1","86","MS-Intel Arc Pro B60 Dual 48G Turbo","4","4","closed/Intel/results/1-node-4x-BMG-Pro-B60-Dual/gpt-oss-120b/Offline/performance/run_1","PyTorch vLLM","Ubuntu 25.04","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","MXFP4","","" +"Intel","available","closed","datacenter","1-node-4x-BMG-Pro-B60-Dual","1-node-4x-BMG-Pro-B60-Dual","gpt-oss-120b","gpt-oss-120b","Server","884.238","exact_match: 83.413","1","Intel(R) Xeon(R) 698X","1","86","MS-Intel Arc Pro B60 Dual 48G Turbo","4","4","closed/Intel/results/1-node-4x-BMG-Pro-B60-Dual/gpt-oss-120b/Server/performance/run_1","PyTorch vLLM","Ubuntu 25.04","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","MXFP4","","" +"Intel","available","closed","datacenter","1-node-4x-BMG-Pro-B60-Dual","1-node-4x-BMG-Pro-B60-Dual","llama2-70b-99","llama2-70b-99","Offline","3270.66","ROUGE1: 44.7862 ROUGE2: 22.3133 ROUGEL: 29.0124 TOKENS_PER_SAMPLE: 284.7","1","Intel(R) Xeon(R) 698X","1","86","MS-Intel Arc Pro B60 Dual 48G Turbo","4","4","closed/Intel/results/1-node-4x-BMG-Pro-B60-Dual/llama2-70b-99/Offline/performance/run_1","PyTorch vLLM","Ubuntu 25.04","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Intel","available","closed","datacenter","1-node-4x-BMG-Pro-B60-Dual","1-node-4x-BMG-Pro-B60-Dual","llama2-70b-99","llama2-70b-99","Server","2199.5","ROUGE1: 44.7636 ROUGE2: 22.2774 ROUGEL: 28.9958 TOKENS_PER_SAMPLE: 283.4","1","Intel(R) Xeon(R) 698X","1","86","MS-Intel Arc Pro B60 Dual 48G Turbo","4","4","closed/Intel/results/1-node-4x-BMG-Pro-B60-Dual/llama2-70b-99/Server/performance/run_1","PyTorch vLLM","Ubuntu 25.04","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Intel","available","closed","datacenter","1-node-4x-BMG-Pro-B60-Dual","1-node-4x-BMG-Pro-B60-Dual","llama2-70b-99.9","llama2-70b-99.9","Offline","3270.66","ROUGE1: 44.7862 ROUGE2: 22.3133 ROUGEL: 29.0124 TOKENS_PER_SAMPLE: 284.7","1","Intel(R) Xeon(R) 698X","1","86","MS-Intel Arc Pro B60 Dual 48G Turbo","4","4","closed/Intel/results/1-node-4x-BMG-Pro-B60-Dual/llama2-70b-99.9/Offline/performance/run_1","PyTorch vLLM","Ubuntu 25.04","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Intel","available","closed","datacenter","1-node-4x-BMG-Pro-B60-Dual","1-node-4x-BMG-Pro-B60-Dual","llama2-70b-99.9","llama2-70b-99.9","Server","2199.5","ROUGE1: 44.7636 ROUGE2: 22.2774 ROUGEL: 28.9958 TOKENS_PER_SAMPLE: 283.4","1","Intel(R) Xeon(R) 698X","1","86","MS-Intel Arc Pro B60 Dual 48G Turbo","4","4","closed/Intel/results/1-node-4x-BMG-Pro-B60-Dual/llama2-70b-99.9/Server/performance/run_1","PyTorch vLLM","Ubuntu 25.04","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Intel","available","closed","datacenter","1-node-4x-BMG-Pro-B60-Dual","1-node-4x-BMG-Pro-B60-Dual","llama3_1-8b","llama3.1-8b","Offline","6762.3","ROUGE1: 38.7302 ROUGE2: 16.0981 ROUGEL: 24.6087 ROUGELSUM: 35.8702 GEN_LEN: 8149144","1","Intel(R) Xeon(R) 698X","1","86","MS-Intel Arc Pro B60 Dual 48G Turbo","4","4","closed/Intel/results/1-node-4x-BMG-Pro-B60-Dual/llama3_1-8b/Offline/performance/run_1","PyTorch vLLM","Ubuntu 25.04","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Intel","available","closed","datacenter","1-node-4x-BMG-Pro-B60-Dual","1-node-4x-BMG-Pro-B60-Dual","llama3_1-8b","llama3.1-8b","Server","6293.89","ROUGE1: 38.7315 ROUGE2: 16.0989 ROUGEL: 24.6089 ROUGELSUM: 35.8705 GEN_LEN: 8149219","1","Intel(R) Xeon(R) 698X","1","86","MS-Intel Arc Pro B60 Dual 48G Turbo","4","4","closed/Intel/results/1-node-4x-BMG-Pro-B60-Dual/llama3_1-8b/Server/performance/run_1","PyTorch vLLM","Ubuntu 25.04","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Intel","preview","closed","datacenter","1-node-4x-BMG-B70","1-node-4x-BMG-B70","gpt-oss-120b","gpt-oss-120b","Offline","1536.9","exact_match: 83.632","1","Intel(R) Xeon(R) 698X","1","86","Intel(R) Arc Pro(R) B70","4","4","closed/Intel/results/1-node-4x-BMG-B70/gpt-oss-120b/Offline/performance/run_1","PyTorch vLLM","Ubuntu 24.04.3 LTS","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","MXFP4","","" +"Intel","preview","closed","datacenter","1-node-4x-BMG-B70","1-node-4x-BMG-B70","gpt-oss-120b","gpt-oss-120b","Server","951.674","exact_match: 82.520","1","Intel(R) Xeon(R) 698X","1","86","Intel(R) Arc Pro(R) B70","4","4","closed/Intel/results/1-node-4x-BMG-B70/gpt-oss-120b/Server/performance/run_1","PyTorch vLLM","Ubuntu 24.04.3 LTS","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","MXFP4","","" +"Intel","preview","closed","datacenter","1-node-4x-BMG-B70","1-node-4x-BMG-B70","llama2-70b-99","llama2-70b-99","Offline","2459.18","ROUGE1: 44.7654 ROUGE2: 22.2793 ROUGEL: 28.9971 TOKENS_PER_SAMPLE: 283.3","1","Intel(R) Xeon(R) 698X","1","86","Intel(R) Arc Pro(R) B70","4","4","closed/Intel/results/1-node-4x-BMG-B70/llama2-70b-99/Offline/performance/run_1","PyTorch vLLM","Ubuntu 24.04.3 LTS","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Intel","preview","closed","datacenter","1-node-4x-BMG-B70","1-node-4x-BMG-B70","llama2-70b-99","llama2-70b-99","Server","1698.57","ROUGE1: 44.7712 ROUGE2: 22.2839 ROUGEL: 29.0009 TOKENS_PER_SAMPLE: 283.3","1","Intel(R) Xeon(R) 698X","1","86","Intel(R) Arc Pro(R) B70","4","4","closed/Intel/results/1-node-4x-BMG-B70/llama2-70b-99/Server/performance/run_1","PyTorch vLLM","Ubuntu 24.04.3 LTS","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Intel","preview","closed","datacenter","1-node-4x-BMG-B70","1-node-4x-BMG-B70","llama2-70b-99.9","llama2-70b-99.9","Offline","2459.18","ROUGE1: 44.7654 ROUGE2: 22.2793 ROUGEL: 28.9971 TOKENS_PER_SAMPLE: 283.3","1","Intel(R) Xeon(R) 698X","1","86","Intel(R) Arc Pro(R) B70","4","4","closed/Intel/results/1-node-4x-BMG-B70/llama2-70b-99.9/Offline/performance/run_1","PyTorch vLLM","Ubuntu 24.04.3 LTS","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Intel","preview","closed","datacenter","1-node-4x-BMG-B70","1-node-4x-BMG-B70","llama2-70b-99.9","llama2-70b-99.9","Server","1698.57","ROUGE1: 44.7712 ROUGE2: 22.2839 ROUGEL: 29.0009 TOKENS_PER_SAMPLE: 283.3","1","Intel(R) Xeon(R) 698X","1","86","Intel(R) Arc Pro(R) B70","4","4","closed/Intel/results/1-node-4x-BMG-B70/llama2-70b-99.9/Server/performance/run_1","PyTorch vLLM","Ubuntu 24.04.3 LTS","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Intel","preview","closed","datacenter","1-node-4x-BMG-B70","1-node-4x-BMG-B70","llama3_1-8b","llama3.1-8b","Offline","4617.4","ROUGE1: 38.7303 ROUGE2: 16.0979 ROUGEL: 24.6095 ROUGELSUM: 35.8705 GEN_LEN: 8149202","1","Intel(R) Xeon(R) 698X","1","86","Intel(R) Arc Pro(R) B70","4","4","closed/Intel/results/1-node-4x-BMG-B70/llama3_1-8b/Offline/performance/run_1","PyTorch vLLM","Ubuntu 24.04.3 LTS","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Intel","preview","closed","datacenter","1-node-4x-BMG-B70","1-node-4x-BMG-B70","llama3_1-8b","llama3.1-8b","Server","4170.13","ROUGE1: 38.711 ROUGE2: 16.0552 ROUGEL: 24.5943 ROUGELSUM: 35.8304 GEN_LEN: 8147007","1","Intel(R) Xeon(R) 698X","1","86","Intel(R) Arc Pro(R) B70","4","4","closed/Intel/results/1-node-4x-BMG-B70/llama3_1-8b/Server/performance/run_1","PyTorch vLLM","Ubuntu 24.04.3 LTS","N/A. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Intel","preview","closed","datacenter","1-node-4x-BMG-B70","1-node-4x-BMG-B70","whisper","whisper","Offline","5596.6","ACCURACY: 97.46051379638439","1","Intel(R) Xeon(R) 698X","1","86","Intel(R) Arc Pro(R) B70","4","4","closed/Intel/results/1-node-4x-BMG-B70/whisper/Offline/performance/run_1","PyTorch vLLM","Ubuntu 24.04.3 LTS","N/A. N/A","closed","0","v6.0","0","False","Samples/s","INT4","","" +"Inventec","available","open","datacenter","Inventec P9000AG7 (8x B200-SXM-180GB, TensorRT)","P9000AG7_B200-SXM-180GBx8_TRT","llama2-70b-99","llama2-70b-99","Offline","100291.0","ROUGE1: 44.7474 ROUGE2: 22.3568 ROUGEL: 29.1367 TOKENS_PER_SAMPLE: 273.3","1","AMD EPYC 9A75","2","64","NVIDIA B200-SXM-180GB","8","8","open/Inventec/results/P9000AG7_B200-SXM-180GBx8_TRT/llama2-70b-99/Offline/performance/run_1","TensorRT 10.11, CUDA 12.9","Ubuntu 24.04","B200 TGP 1000W","open","0","v6.0","0","False","Tokens/s","fp4","","" +"Inventec","available","open","datacenter","Inventec P9000AG7 (8x B200-SXM-180GB, TensorRT)","P9000AG7_B200-SXM-180GBx8_TRT","llama3.1-405b","llama3.1-405b","Offline","1632.1","ROUGEL: 21.979457539013907 exact_match: 90.03752812706816 TOKENS_PER_SAMPLE: 652.0","1","AMD EPYC 9A75","2","64","NVIDIA B200-SXM-180GB","8","8","open/Inventec/results/P9000AG7_B200-SXM-180GBx8_TRT/llama3.1-405b/Offline/performance/run_1","TensorRT 10.11, CUDA 12.9","Ubuntu 24.04","B200 TGP 1000W","open","0","v6.0","0","False","Tokens/s","fp4","","" +"Inventec","available","open","datacenter","Inventec P9000AG7 (8x B200-SXM-180GB, TensorRT)","P9000AG7_B200-SXM-180GBx8_TRT","llama3.1-8b","llama3.1-8b","Offline","142508.0","ROUGE1: 38.4843 ROUGE2: 15.9098 ROUGEL: 24.3232 ROUGELSUM: 35.6313 GEN_LEN: 8151646","1","AMD EPYC 9A75","2","64","NVIDIA B200-SXM-180GB","8","8","open/Inventec/results/P9000AG7_B200-SXM-180GBx8_TRT/llama3.1-8b/Offline/performance/run_1","TensorRT 10.11, CUDA 12.9","Ubuntu 24.04","B200 TGP 1000W","open","0","v6.0","0","False","Tokens/s","fp4","","" +"Krai","available","closed","datacenter","HPE Cray XD670 (8x H200-SXM-141GB, KISS-V)","h200_n1-kissv","wan-2.2-t2v-a14b","wan-2.2-t2v-a14b","offline","0.0181337","vbench_score: 69.78","1","Intel(R) Xeon(R) Platinum 8562Y+","2","32","NVIDIA H200-SXM-141GB","8","8","closed/Krai/results/h200_n1-kissv/wan-2.2-t2v-a14b/offline/performance/run_1","KISS-V: Krai Inference Serving Solution for Video","Red Hat Enterprise Linux release 9.4 (Plow) (Linux kernel: 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed Apr 10 10:29:16 EDT 2024 x86_64 x86_64 x86_64 GNU/Linux)","Powered by the Krai axs automation technology","closed","0","v6.0","1","False","Samples/s","bfloat16","","" +"Krai","available","closed","datacenter","HPE Cray XD670 (8x H200-SXM-141GB, KISS-V)","h200_n1-kissv","wan-2.2-t2v-a14b","wan-2.2-t2v-a14b","singlestream","57.401991155000005","vbench_score: 69.89","1","Intel(R) Xeon(R) Platinum 8562Y+","2","32","NVIDIA H200-SXM-141GB","8","8","closed/Krai/results/h200_n1-kissv/wan-2.2-t2v-a14b/singlestream/performance/run_1","KISS-V: Krai Inference Serving Solution for Video","Red Hat Enterprise Linux release 9.4 (Plow) (Linux kernel: 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed Apr 10 10:29:16 EDT 2024 x86_64 x86_64 x86_64 GNU/Linux)","Powered by the Krai axs automation technology","closed","0","v6.0","1","False","Latency (s)","bfloat16","","" +"Krai","available","open","datacenter","HPE Cray XD670 (8x H200-SXM-141GB, KISS-V Fast Mode)","h200_n1-kissv","wan-2.2-t2v-a14b","wan-2.2-t2v-a14b","offline","0.0241194","vbench_score: 70.09","1","Intel(R) Xeon(R) Platinum 8562Y+","2","32","NVIDIA H200-SXM-141GB","8","8","open/Krai/results/h200_n1-kissv/wan-2.2-t2v-a14b/offline/performance/run_1","KISS-V: Krai Inference Serving Solution for Video (Fast Mode)","Red Hat Enterprise Linux release 9.4 (Plow) (Linux kernel: 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed Apr 10 10:29:16 EDT 2024 x86_64 x86_64 x86_64 GNU/Linux)","Powered by the Krai axs automation technology","open","0","v6.0","1","False","Samples/s","bfloat16","","" +"Krai","available","open","datacenter","HPE Cray XD670 (8x H200-SXM-141GB, KISS-V Fast Mode)","h200_n1-kissv","wan-2.2-t2v-a14b","wan-2.2-t2v-a14b","singlestream","43.936000879","vbench_score: 69.78999999999999","1","Intel(R) Xeon(R) Platinum 8562Y+","2","32","NVIDIA H200-SXM-141GB","8","8","open/Krai/results/h200_n1-kissv/wan-2.2-t2v-a14b/singlestream/performance/run_1","KISS-V: Krai Inference Serving Solution for Video (Fast Mode)","Red Hat Enterprise Linux release 9.4 (Plow) (Linux kernel: 5.14.0-427.13.1.el9_4.x86_64 #1 SMP PREEMPT_DYNAMIC Wed Apr 10 10:29:16 EDT 2024 x86_64 x86_64 x86_64 GNU/Linux)","Powered by the Krai axs automation technology","open","0","v6.0","1","False","Latency (s)","bfloat16","","" +"Krai","available","open","datacenter","HPE Cray XD670 (8x H200-SXM-141GB, vLLM)","h200_n1-vllm","llama3_1-70b-fp8_dyn","llama2-70b-99.9","offline","25939.8","ROUGE1: 47.6127 ROUGE2: 25.255 ROUGEL: 31.5076 TOKENS_PER_SAMPLE: 298.2","1","Intel(R) Xeon(R) Platinum 8562Y+","2","32","NVIDIA H200-SXM-141GB","8","8","open/Krai/results/h200_n1-vllm/llama3_1-70b-fp8_dyn/offline/performance/run_1","vLLM v0.15.0-dev","host: Red Hat Enterprise Linux 9.4, container: Ubuntu 22.04.5 LTS","Powered by the Krai axs automation technology","open","0","v6.0","1","False","Tokens/s","fp16","","" +"Krai","available","open","datacenter","HPE Cray XD670 (8x H200-SXM-141GB, vLLM)","h200_n1-vllm","llama3_1-70b-fp8_dyn","llama2-70b-99.9","server","24821.6","ROUGE1: 47.6215 ROUGE2: 25.2626 ROUGEL: 31.5143 TOKENS_PER_SAMPLE: 298.2","1","Intel(R) Xeon(R) Platinum 8562Y+","2","32","NVIDIA H200-SXM-141GB","8","8","open/Krai/results/h200_n1-vllm/llama3_1-70b-fp8_dyn/server/performance/run_1","vLLM v0.15.0-dev","host: Red Hat Enterprise Linux 9.4, container: Ubuntu 22.04.5 LTS","Powered by the Krai axs automation technology","open","0","v6.0","1","False","Tokens/s","fp16","","" +"Lambda_SIT","available","closed","datacenter","NVIDIA DGX B200 (8x B200-SXM-180GB, TensorRT)","B200-SXM-180GBx8_TRT","llama3.1-8b","llama3.1-8b","Interactive","128750.0","ROUGE1: 38.6433 ROUGE2: 15.8347 ROUGEL: 24.3517 ROUGELSUM: 35.6515 GEN_LEN: 8192621","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B200-SXM-180GB","8","8","closed/Lambda_SIT/results/B200-SXM-180GBx8_TRT/llama3.1-8b/Interactive/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B200 TGP 1000W","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"Lambda_SIT","available","closed","datacenter","NVIDIA DGX B200 (8x B200-SXM-180GB, TensorRT)","B200-SXM-180GBx8_TRT","llama3.1-8b","llama3.1-8b","Offline","160403.0","ROUGE1: 38.7787 ROUGE2: 16.1964 ROUGEL: 24.6085 ROUGELSUM: 35.8317 GEN_LEN: 8187884","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B200-SXM-180GB","8","8","closed/Lambda_SIT/results/B200-SXM-180GBx8_TRT/llama3.1-8b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B200 TGP 1000W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Lambda_SIT","available","closed","datacenter","NVIDIA DGX B200 (8x B200-SXM-180GB, TensorRT)","B200-SXM-180GBx8_TRT","llama3.1-8b","llama3.1-8b","Server","130008.0","ROUGE1: 38.7757 ROUGE2: 16.1933 ROUGEL: 24.6056 ROUGELSUM: 35.8283 GEN_LEN: 8187968","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B200-SXM-180GB","8","8","closed/Lambda_SIT/results/B200-SXM-180GBx8_TRT/llama3.1-8b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B200 TGP 1000W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Lambda_SIT","available","closed","datacenter","NVIDIA GB300 NVL72 (4x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x4_TRT","gpt-oss-120b","gpt-oss-120b","Offline","60220.4","exact_match: 83.601","1","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","4","closed/Lambda_SIT/results/GB300-NVL72_GB300-288GB_aarch64x4_TRT/gpt-oss-120b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","NVIDIA GB300 NVL72","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Lambda_SIT","available","closed","datacenter","NVIDIA GB300 NVL72 (4x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x4_TRT","gpt-oss-120b","gpt-oss-120b","Server","53462.7","exact_match: 83.008","1","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","4","closed/Lambda_SIT/results/GB300-NVL72_GB300-288GB_aarch64x4_TRT/gpt-oss-120b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","NVIDIA GB300 NVL72","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Lambda_SIT","available","open","datacenter","NVIDIA DGX B200 (8x B200-SXM-180GB, TensorRT)","B200-SXM-180GBx8_TRT","gpt-oss-120b","gpt-oss-120b","Server","16167.4","exact_match: 83.661","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B200-SXM-180GB","8","8","open/Lambda_SIT/results/B200-SXM-180GBx8_TRT/gpt-oss-120b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B200 TGP 1000W","open","0","v6.0","0","False","Tokens/s","fp4","","" +"Lenovo","available","closed","datacenter","1-node-2S-GNR_86C","1-node-2S-GNR_86C","llama3_1-8b","llama3.1-8b","Offline","775.982","ROUGE1: 38.7771 ROUGE2: 16.0352 ROUGEL: 24.5468 ROUGELSUM: 35.8518 GEN_LEN: 8146140","1","Intel(R) Xeon(R) 6787P","2","86","N/A","0","0","closed/Lenovo/results/1-node-2S-GNR_86C/llama3_1-8b/Offline/performance/run_1","PyTorch","Ubuntu 24.04.3 LTS","ThinkSystem SR650 V4. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Lenovo","available","closed","datacenter","1-node-2S-GNR_86C","1-node-2S-GNR_86C","llama3_1-8b","llama3.1-8b","Server","281.779","ROUGE1: 38.851 ROUGE2: 16.086 ROUGEL: 24.6018 ROUGELSUM: 35.9286 GEN_LEN: 8145738","1","Intel(R) Xeon(R) 6787P","2","86","N/A","0","0","closed/Lenovo/results/1-node-2S-GNR_86C/llama3_1-8b/Server/performance/run_1","PyTorch","Ubuntu 24.04.3 LTS","ThinkSystem SR650 V4. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Lenovo","available","closed","datacenter","1-node-2S-GNR_86C","1-node-2S-GNR_86C","rgat","rgat","Offline","13570.3","acc: 72.574","1","Intel(R) Xeon(R) 6787P","2","86","N/A","0","0","closed/Lenovo/results/1-node-2S-GNR_86C/rgat/Offline/performance/run_1","PyTorch","Ubuntu 24.04.3 LTS","ThinkSystem SR650 V4. N/A","closed","0","v6.0","0","False","Samples/s","INT8","","" +"Lenovo","available","closed","datacenter","1-node-2S-GNR_86C","1-node-2S-GNR_86C","whisper","whisper","Offline","1411.42","ACCURACY: 97.88576267638467","1","Intel(R) Xeon(R) 6787P","2","86","N/A","0","0","closed/Lenovo/results/1-node-2S-GNR_86C/whisper/Offline/performance/run_1","PyTorch","Ubuntu 24.04.3 LTS","ThinkSystem SR650 V4. N/A","closed","0","v6.0","0","False","Samples/s","INT8","","" +"Lenovo","available","closed","datacenter","ThinkSystem SR675i V3 (8x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","SR675iV3","llama2-70b-99","llama2-70b-99","Offline","26889.6","ROUGE1: 44.7302 ROUGE2: 22.3372 ROUGEL: 29.1145 TOKENS_PER_SAMPLE: 273.4","1","AMD EPYC 9535","2","64","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/Lenovo/results/SR675iV3/llama2-70b-99/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Lenovo","available","closed","datacenter","ThinkSystem SR675i V3 (8x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","SR675iV3","llama2-70b-99","llama2-70b-99","Server","27160.2","ROUGE1: 44.787 ROUGE2: 22.329 ROUGEL: 29.1181 TOKENS_PER_SAMPLE: 274.1","1","AMD EPYC 9535","2","64","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/Lenovo/results/SR675iV3/llama2-70b-99/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Lenovo","available","closed","datacenter","ThinkSystem SR675i V3 (8x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","SR675iV3","llama2-70b-99.9","llama2-70b-99.9","Offline","26889.6","ROUGE1: 44.7302 ROUGE2: 22.3372 ROUGEL: 29.1145 TOKENS_PER_SAMPLE: 273.4","1","AMD EPYC 9535","2","64","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/Lenovo/results/SR675iV3/llama2-70b-99.9/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Lenovo","available","closed","datacenter","ThinkSystem SR675i V3 (8x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","SR675iV3","llama2-70b-99.9","llama2-70b-99.9","Server","27160.2","ROUGE1: 44.787 ROUGE2: 22.329 ROUGEL: 29.1181 TOKENS_PER_SAMPLE: 274.1","1","AMD EPYC 9535","2","64","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/Lenovo/results/SR675iV3/llama2-70b-99.9/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Lenovo","available","closed","datacenter","ThinkSystem SR675i V3 (8x NVIDIA RTX PRO 6000 Blackwell Server Edition, TensorRT)","SR675iV3","whisper","whisper","Offline","18401.1","ACCURACY: 97.84158283925122","1","AMD EPYC 9535","2","64","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/Lenovo/results/SR675iV3/whisper/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","","closed","0","v6.0","0","False","Samples/s","fp16","","" +"Lenovo","available","closed","datacenter","ThinkSystem SR680a V4 (8x B300-SXM-270GB, TensorRT)","SR680aV4","gpt-oss-120b","gpt-oss-120b","Offline","109141.0","exact_match: 83.731","1","Intel(R) Xeon(R) 6747P","2","56","NVIDIA B300-SXM-270GB","8","8","closed/Lenovo/results/SR680aV4/gpt-oss-120b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Lenovo","available","closed","datacenter","ThinkSystem SR680a V4 (8x B300-SXM-270GB, TensorRT)","SR680aV4","gpt-oss-120b","gpt-oss-120b","Server","100687.0","exact_match: 83.520","1","Intel(R) Xeon(R) 6747P","2","56","NVIDIA B300-SXM-270GB","8","8","closed/Lenovo/results/SR680aV4/gpt-oss-120b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Lenovo","available","closed","datacenter","ThinkSystem SR680a V4 (8x B300-SXM-270GB, TensorRT)","SR680aV4","whisper","whisper","Offline","47268.7","ACCURACY: 97.85109966976599","1","Intel(R) Xeon(R) 6747P","2","56","NVIDIA B300-SXM-270GB","8","8","closed/Lenovo/results/SR680aV4/whisper/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Samples/s","fp16","","" +"Lenovo","available","closed","edge","ThinkEdge SE100","SE100","resnet50","resnet","SingleStream","1.203352","acc: 76.218","1","Intel(R) Core(TM) Ultra 7 255H [GPU]","1","16","N/A","0","0","closed/Lenovo/results/SE100/resnet50/SingleStream/performance/run_1","OpenVINO","Ubuntu 24.04.3 LTS (Noble Numbat)","N/A. N/A","closed","0","v6.0","0","False","Latency (ms)","INT8","","" +"Lenovo","available","closed","edge","ThinkEdge SE100","SE100","resnet50","resnet","multistream","9.918944","acc: 76.218","1","Intel(R) Core(TM) Ultra 7 255H [GPU]","1","16","N/A","0","0","closed/Lenovo/results/SE100/resnet50/multistream/performance/run_1","OpenVINO","Ubuntu 24.04.3 LTS (Noble Numbat)","N/A. N/A","closed","0","v6.0","1","False","Latency (ms)","INT8","","" +"Lenovo","available","closed","edge","ThinkEdge SE100","SE100","resnet50","resnet","offline","834.804","acc: 76.218","1","Intel(R) Core(TM) Ultra 7 255H [GPU]","1","16","N/A","0","0","closed/Lenovo/results/SE100/resnet50/offline/performance/run_1","OpenVINO","Ubuntu 24.04.3 LTS (Noble Numbat)","N/A. N/A","closed","0","v6.0","1","False","Samples/s","INT8","","" +"Lenovo","available","closed","edge","ThinkEdge SE455i V3(2xL4, TensorRT","SE455I_V3_TRT","3d-unet-99","3d-unet-99","Offline","2.18794","DICE: 0.90862","1","AMD EPYC 8534P","1","64","NVIDIA L4-PCIe-24GB","2","2","closed/Lenovo/results/SE455I_V3_TRT/3d-unet-99/Offline/performance/run_1","TensorRT 10.8, CUDA 13.1","Ubuntu 24.04","","closed","0","v6.0","0","False","Samples/s","int8","","" +"Lenovo","available","closed","edge","ThinkEdge SE455i V3(2xL4, TensorRT","SE455I_V3_TRT","3d-unet-99","3d-unet-99","SingleStream","1880.729881","DICE: 0.90850","1","AMD EPYC 8534P","1","64","NVIDIA L4-PCIe-24GB","2","2","closed/Lenovo/results/SE455I_V3_TRT/3d-unet-99/SingleStream/performance/run_1","TensorRT 10.8, CUDA 13.1","Ubuntu 24.04","","closed","0","v6.0","0","False","Latency (ms)","int8","","" +"Lenovo","available","closed","edge","ThinkEdge SE455i V3(2xL4, TensorRT","SE455I_V3_TRT","3d-unet-99.9","3d-unet-99.9","Offline","2.18794","DICE: 0.90862","1","AMD EPYC 8534P","1","64","NVIDIA L4-PCIe-24GB","2","2","closed/Lenovo/results/SE455I_V3_TRT/3d-unet-99.9/Offline/performance/run_1","TensorRT 10.8, CUDA 13.1","Ubuntu 24.04","","closed","0","v6.0","0","False","Samples/s","int8","","" +"Lenovo","available","closed","edge","ThinkEdge SE455i V3(2xL4, TensorRT","SE455I_V3_TRT","3d-unet-99.9","3d-unet-99.9","SingleStream","1880.729881","DICE: 0.90850","1","AMD EPYC 8534P","1","64","NVIDIA L4-PCIe-24GB","2","2","closed/Lenovo/results/SE455I_V3_TRT/3d-unet-99.9/SingleStream/performance/run_1","TensorRT 10.8, CUDA 13.1","Ubuntu 24.04","","closed","0","v6.0","0","False","Latency (ms)","int8","","" +"Lenovo","available","closed","edge","ThinkEdge SE455i V3(2xL4, TensorRT","SE455I_V3_TRT","bert-99","bert-99","Offline","1521.11","F1: 90.279","1","AMD EPYC 8534P","1","64","NVIDIA L4-PCIe-24GB","2","2","closed/Lenovo/results/SE455I_V3_TRT/bert-99/Offline/performance/run_1","TensorRT 10.8, CUDA 13.1","Ubuntu 24.04","","closed","0","v6.0","0","False","Samples/s","int8","","" +"Lenovo","available","closed","edge","ThinkEdge SE455i V3(2xL4, TensorRT","SE455I_V3_TRT","bert-99","bert-99","SingleStream","2.166524","F1: 90.267","1","AMD EPYC 8534P","1","64","NVIDIA L4-PCIe-24GB","2","2","closed/Lenovo/results/SE455I_V3_TRT/bert-99/SingleStream/performance/run_1","TensorRT 10.8, CUDA 13.1","Ubuntu 24.04","","closed","0","v6.0","0","False","Latency (ms)","int8","","" +"Lenovo","available","closed","edge","ThinkEdge SE455i V3(2xL4, TensorRT","SE455I_V3_TRT","resnet50","resnet","MultiStream","0.80682","acc: 76.078","1","AMD EPYC 8534P","1","64","NVIDIA L4-PCIe-24GB","2","2","closed/Lenovo/results/SE455I_V3_TRT/resnet50/MultiStream/performance/run_1","TensorRT 10.8, CUDA 13.1","Ubuntu 24.04","","closed","0","v6.0","0","False","Latency (ms)","int8","","" +"Lenovo","available","closed","edge","ThinkEdge SE455i V3(2xL4, TensorRT","SE455I_V3_TRT","resnet50","resnet","Offline","25200.5","acc: 76.078","1","AMD EPYC 8534P","1","64","NVIDIA L4-PCIe-24GB","2","2","closed/Lenovo/results/SE455I_V3_TRT/resnet50/Offline/performance/run_1","TensorRT 10.8, CUDA 13.1","Ubuntu 24.04","","closed","0","v6.0","0","False","Samples/s","int8","","" +"Lenovo","available","closed","edge","ThinkEdge SE455i V3(2xL4, TensorRT","SE455I_V3_TRT","resnet50","resnet","SingleStream","0.400845","acc: 76.064","1","AMD EPYC 8534P","1","64","NVIDIA L4-PCIe-24GB","2","2","closed/Lenovo/results/SE455I_V3_TRT/resnet50/SingleStream/performance/run_1","TensorRT 10.8, CUDA 13.1","Ubuntu 24.04","","closed","0","v6.0","0","False","Latency (ms)","int8","","" +"MangoBoost","available","closed","datacenter","MangoBoost Mi300X (8x MI300X, LLMBoost)","8xMI300X_2xEPYC_9534","llama2-70b-99","llama2-70b-99","Offline","27598.7","ROUGE1: 44.4213 ROUGE2: 22.0278 ROUGEL: 28.6475 TOKENS_PER_SAMPLE: 296.1","1","AMD EPYC 9534","2","64","AMD Instinct MI300X 192GB HBM3","8","8","closed/MangoBoost/results/8xMI300X_2xEPYC_9534/llama2-70b-99/Offline/performance/run_1","ROCm 6.10.5","Ubuntu 22.04.5 LTS (jammy)","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"MangoBoost","available","closed","datacenter","MangoBoost Mi300X (8x MI300X, LLMBoost)","8xMI300X_2xEPYC_9534","llama2-70b-99","llama2-70b-99","Server","25463.1","ROUGE1: 44.4718 ROUGE2: 22.0358 ROUGEL: 28.668 TOKENS_PER_SAMPLE: 295.1","1","AMD EPYC 9534","2","64","AMD Instinct MI300X 192GB HBM3","8","8","closed/MangoBoost/results/8xMI300X_2xEPYC_9534/llama2-70b-99/Server/performance/run_1","ROCm 6.10.5","Ubuntu 22.04.5 LTS (jammy)","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"MangoBoost","available","closed","datacenter","MangoBoost Mi300X (8x MI300X, LLMBoost)","8xMI300X_2xEPYC_9534","llama2-70b-99.9","llama2-70b-99.9","Offline","27598.7","ROUGE1: 44.4213 ROUGE2: 22.0278 ROUGEL: 28.6475 TOKENS_PER_SAMPLE: 296.1","1","AMD EPYC 9534","2","64","AMD Instinct MI300X 192GB HBM3","8","8","closed/MangoBoost/results/8xMI300X_2xEPYC_9534/llama2-70b-99.9/Offline/performance/run_1","ROCm 6.10.5","Ubuntu 22.04.5 LTS (jammy)","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"MangoBoost","available","closed","datacenter","MangoBoost Mi300X (8x MI300X, LLMBoost)","8xMI300X_2xEPYC_9534","llama2-70b-99.9","llama2-70b-99.9","Server","25463.1","ROUGE1: 44.4718 ROUGE2: 22.0358 ROUGEL: 28.668 TOKENS_PER_SAMPLE: 295.1","1","AMD EPYC 9534","2","64","AMD Instinct MI300X 192GB HBM3","8","8","closed/MangoBoost/results/8xMI300X_2xEPYC_9534/llama2-70b-99.9/Server/performance/run_1","ROCm 6.10.5","Ubuntu 22.04.5 LTS (jammy)","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"MangoBoost","available","closed","datacenter","MangoBoost Mi325X (8x MI325X, LLMBoost)","8xMI325X_2xEPYC_9655","llama2-70b-99","llama2-70b-99","Offline","33880.4","ROUGE1: 44.4375 ROUGE2: 22.0426 ROUGEL: 28.5893 TOKENS_PER_SAMPLE: 300.6","1","AMD EPYC 9655","2","96","AMD Instinct MI325X 256GB HBM3e","8","8","closed/MangoBoost/results/8xMI325X_2xEPYC_9655/llama2-70b-99/Offline/performance/run_1","ROCm 6.10.5","Ubuntu 22.04.5 LTS (jammy)","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"MangoBoost","available","closed","datacenter","MangoBoost Mi325X (8x MI325X, LLMBoost)","8xMI325X_2xEPYC_9655","llama2-70b-99","llama2-70b-99","Server","31888.9","ROUGE1: 44.4754 ROUGE2: 22.0607 ROUGEL: 28.6289 TOKENS_PER_SAMPLE: 300.1","1","AMD EPYC 9655","2","96","AMD Instinct MI325X 256GB HBM3e","8","8","closed/MangoBoost/results/8xMI325X_2xEPYC_9655/llama2-70b-99/Server/performance/run_1","ROCm 6.10.5","Ubuntu 22.04.5 LTS (jammy)","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"MangoBoost","available","closed","datacenter","MangoBoost Mi325X (8x MI325X, LLMBoost)","8xMI325X_2xEPYC_9655","llama2-70b-99.9","llama2-70b-99.9","Offline","33880.4","ROUGE1: 44.4375 ROUGE2: 22.0426 ROUGEL: 28.5893 TOKENS_PER_SAMPLE: 300.6","1","AMD EPYC 9655","2","96","AMD Instinct MI325X 256GB HBM3e","8","8","closed/MangoBoost/results/8xMI325X_2xEPYC_9655/llama2-70b-99.9/Offline/performance/run_1","ROCm 6.10.5","Ubuntu 22.04.5 LTS (jammy)","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"MangoBoost","available","closed","datacenter","MangoBoost Mi325X (8x MI325X, LLMBoost)","8xMI325X_2xEPYC_9655","llama2-70b-99.9","llama2-70b-99.9","Server","31888.9","ROUGE1: 44.4754 ROUGE2: 22.0607 ROUGEL: 28.6289 TOKENS_PER_SAMPLE: 300.1","1","AMD EPYC 9655","2","96","AMD Instinct MI325X 256GB HBM3e","8","8","closed/MangoBoost/results/8xMI325X_2xEPYC_9655/llama2-70b-99.9/Server/performance/run_1","ROCm 6.10.5","Ubuntu 22.04.5 LTS (jammy)","","closed","0","v6.0","0","False","Tokens/s","fp8","","" +"MiTAC","available","closed","datacenter","G8825Z5","8xMI350X_2xEPYC-9755","llama2-70b-99","llama2-70b-99","Interactive","54730.5","ROUGE1: 44.6043 ROUGE2: 22.1796 ROUGEL: 28.9629 TOKENS_PER_SAMPLE: 281.8","1","AMD EPYC 9755","2","128","AMD Instinct MI350X 288GB HBM3e","8","8","closed/MiTAC/results/8xMI350X_2xEPYC-9755/llama2-70b-99/Interactive/performance/run_1","Pytorch 2.9.1 , ROCm 7.0","Ubuntu 22.04.5 LTS","","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"MiTAC","available","closed","datacenter","G8825Z5","8xMI350X_2xEPYC-9755","llama2-70b-99","llama2-70b-99","Offline","78988.6","ROUGE1: 44.4576 ROUGE2: 22.0944 ROUGEL: 28.8678 TOKENS_PER_SAMPLE: 284.0","1","AMD EPYC 9755","2","128","AMD Instinct MI350X 288GB HBM3e","8","8","closed/MiTAC/results/8xMI350X_2xEPYC-9755/llama2-70b-99/Offline/performance/run_1","Pytorch 2.9.1 , ROCm 7.0","Ubuntu 22.04.5 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"MiTAC","available","closed","datacenter","G8825Z5","8xMI350X_2xEPYC-9755","llama2-70b-99","llama2-70b-99","Server","77377.9","ROUGE1: 44.607 ROUGE2: 22.1794 ROUGEL: 28.9613 TOKENS_PER_SAMPLE: 281.7","1","AMD EPYC 9755","2","128","AMD Instinct MI350X 288GB HBM3e","8","8","closed/MiTAC/results/8xMI350X_2xEPYC-9755/llama2-70b-99/Server/performance/run_1","Pytorch 2.9.1 , ROCm 7.0","Ubuntu 22.04.5 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"MiTAC","available","closed","datacenter","G8825Z5","8xMI350X_2xEPYC-9755","llama2-70b-99.9","llama2-70b-99.9","Interactive","54730.5","ROUGE1: 44.6043 ROUGE2: 22.1796 ROUGEL: 28.9629 TOKENS_PER_SAMPLE: 281.8","1","AMD EPYC 9755","2","128","AMD Instinct MI350X 288GB HBM3e","8","8","closed/MiTAC/results/8xMI350X_2xEPYC-9755/llama2-70b-99.9/Interactive/performance/run_1","Pytorch 2.9.1 , ROCm 7.0","Ubuntu 22.04.5 LTS","","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"MiTAC","available","closed","datacenter","G8825Z5","8xMI350X_2xEPYC-9755","llama2-70b-99.9","llama2-70b-99.9","Offline","78988.6","ROUGE1: 44.4576 ROUGE2: 22.0944 ROUGEL: 28.8678 TOKENS_PER_SAMPLE: 284.0","1","AMD EPYC 9755","2","128","AMD Instinct MI350X 288GB HBM3e","8","8","closed/MiTAC/results/8xMI350X_2xEPYC-9755/llama2-70b-99.9/Offline/performance/run_1","Pytorch 2.9.1 , ROCm 7.0","Ubuntu 22.04.5 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"MiTAC","available","closed","datacenter","G8825Z5","8xMI350X_2xEPYC-9755","llama2-70b-99.9","llama2-70b-99.9","Server","77377.9","ROUGE1: 44.607 ROUGE2: 22.1794 ROUGEL: 28.9613 TOKENS_PER_SAMPLE: 281.7","1","AMD EPYC 9755","2","128","AMD Instinct MI350X 288GB HBM3e","8","8","closed/MiTAC/results/8xMI350X_2xEPYC-9755/llama2-70b-99.9/Server/performance/run_1","Pytorch 2.9.1 , ROCm 7.0","Ubuntu 22.04.5 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA DGX B200 (8x B200-SXM-180GB, TensorRT)","B200-SXM-180GBx8_TRT","dlrm-v3","dlrm-v3","Offline","10547.4","DLRM_NE: 0.8675504587396435 DLRM_ACC: 0.6964261896038318 DLRM_AUC: 0.7862891896349031","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B200-SXM-180GB","8","8","closed/NVIDIA/results/B200-SXM-180GBx8_TRT/dlrm-v3/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","B200 TGP 1000W","closed","0","v6.0","0","False","Samples/s","fp16","","" +"NVIDIA","available","closed","datacenter","NVIDIA DGX B200 (8x B200-SXM-180GB, TensorRT)","B200-SXM-180GBx8_TRT","dlrm-v3","dlrm-v3","Server","10007.0","DLRM_NE: 0.8675504587396445 DLRM_ACC: 0.6964261896038318 DLRM_AUC: 0.7862891896349031","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B200-SXM-180GB","8","8","closed/NVIDIA/results/B200-SXM-180GBx8_TRT/dlrm-v3/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","B200 TGP 1000W","closed","0","v6.0","0","False","Queries/s","fp16","","" +"NVIDIA","available","closed","datacenter","NVIDIA DGX B200 (8x B200-SXM-180GB, TensorRT)","B200-SXM-180GBx8_TRT","wan-2.2-t2v-a14b","wan-2.2-t2v-a14b","Offline","0.0464402","vbench_score: 69.8098","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B200-SXM-180GB","8","8","closed/NVIDIA/results/B200-SXM-180GBx8_TRT/wan-2.2-t2v-a14b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","B200 TGP 1000W","closed","0","v6.0","0","False","Samples/s","fp32","","" +"NVIDIA","available","closed","datacenter","NVIDIA DGX B200 (8x B200-SXM-180GB, TensorRT)","B200-SXM-180GBx8_TRT","wan-2.2-t2v-a14b","wan-2.2-t2v-a14b","SingleStream","25.458383354000002","vbench_score: 69.8853","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B200-SXM-180GB","8","8","closed/NVIDIA/results/B200-SXM-180GBx8_TRT/wan-2.2-t2v-a14b/SingleStream/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","B200 TGP 1000W","closed","0","v6.0","0","False","Latency (s)","fp32","","" +"NVIDIA","available","closed","datacenter","NVIDIA DGX B200 (8x B200-SXM-180GB, TensorRT)","B200-SXM-180GBx8_TRT","whisper","whisper","Offline","47433.9","ACCURACY: 97.83967947314827","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B200-SXM-180GB","8","8","closed/NVIDIA/results/B200-SXM-180GBx8_TRT/whisper/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","B200 TGP 1000W","closed","0","v6.0","0","False","Samples/s","fp16","","" +"NVIDIA","available","closed","datacenter","NVIDIA DGX B300 (8x B300-SXM-270GB, TensorRT)","B300-SXM-270GBx8_TRT","deepseek-r1","deepseek-r1","Offline","70325.7","exact_match: 81.10756608933455 TOKENS_PER_SAMPLE: 3775.6280765724705","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B300-SXM-270GB","8","8","closed/NVIDIA/results/B300-SXM-270GBx8_TRT/deepseek-r1/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA DGX B300 (8x B300-SXM-270GB, TensorRT)","B300-SXM-270GBx8_TRT","deepseek-r1","deepseek-r1","Server","42721.4","exact_match: 81.19872379216044 TOKENS_PER_SAMPLE: 3741.4772105742936","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B300-SXM-270GB","8","8","closed/NVIDIA/results/B300-SXM-270GBx8_TRT/deepseek-r1/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA DGX B300 (8x B300-SXM-270GB, TensorRT)","B300-SXM-270GBx8_TRT","gpt-oss-120b","gpt-oss-120b","Offline","103961.0","exact_match: 83.565","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B300-SXM-270GB","8","8","closed/NVIDIA/results/B300-SXM-270GBx8_TRT/gpt-oss-120b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA DGX B300 (8x B300-SXM-270GB, TensorRT)","B300-SXM-270GBx8_TRT","gpt-oss-120b","gpt-oss-120b","Server","100328.0","exact_match: 83.582","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B300-SXM-270GB","8","8","closed/NVIDIA/results/B300-SXM-270GBx8_TRT/gpt-oss-120b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA DGX B300 (8x B300-SXM-270GB, TensorRT)","B300-SXM-270GBx8_TRT","llama2-70b-99","llama2-70b-99","Offline","112954.0","ROUGE1: 44.7714 ROUGE2: 22.3739 ROUGEL: 29.1483 TOKENS_PER_SAMPLE: 273.1","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B300-SXM-270GB","8","8","closed/NVIDIA/results/B300-SXM-270GBx8_TRT/llama2-70b-99/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA DGX B300 (8x B300-SXM-270GB, TensorRT)","B300-SXM-270GBx8_TRT","llama2-70b-99","llama2-70b-99","Server","107318.0","ROUGE1: 44.856 ROUGE2: 22.4112 ROUGEL: 29.1899 TOKENS_PER_SAMPLE: 271.7","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B300-SXM-270GB","8","8","closed/NVIDIA/results/B300-SXM-270GBx8_TRT/llama2-70b-99/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA DGX B300 (8x B300-SXM-270GB, TensorRT)","B300-SXM-270GBx8_TRT","llama2-70b-99.9","llama2-70b-99.9","Offline","112954.0","ROUGE1: 44.7714 ROUGE2: 22.3739 ROUGEL: 29.1483 TOKENS_PER_SAMPLE: 273.1","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B300-SXM-270GB","8","8","closed/NVIDIA/results/B300-SXM-270GBx8_TRT/llama2-70b-99.9/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA DGX B300 (8x B300-SXM-270GB, TensorRT)","B300-SXM-270GBx8_TRT","llama2-70b-99.9","llama2-70b-99.9","Server","107318.0","ROUGE1: 44.856 ROUGE2: 22.4112 ROUGEL: 29.1899 TOKENS_PER_SAMPLE: 271.7","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B300-SXM-270GB","8","8","closed/NVIDIA/results/B300-SXM-270GBx8_TRT/llama2-70b-99.9/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA DGX B300 (8x B300-SXM-270GB, TensorRT)","B300-SXM-270GBx8_TRT","llama3.1-405b","llama3.1-405b","Offline","1951.6","ROUGEL: 21.84372254775757 exact_match: 90.04178689609532 TOKENS_PER_SAMPLE: 636.0","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B300-SXM-270GB","8","8","closed/NVIDIA/results/B300-SXM-270GBx8_TRT/llama3.1-405b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA DGX B300 (8x B300-SXM-270GB, TensorRT)","B300-SXM-270GBx8_TRT","llama3.1-405b","llama3.1-405b","Server","1460.25","ROUGEL: 21.822118834475546 exact_match: 90.01338186631372 TOKENS_PER_SAMPLE: 638.6","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B300-SXM-270GB","8","8","closed/NVIDIA/results/B300-SXM-270GBx8_TRT/llama3.1-405b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA DGX B300 (8x B300-SXM-270GB, TensorRT)","B300-SXM-270GBx8_TRT","llama3.1-8b","llama3.1-8b","Offline","165432.0","ROUGE1: 38.5159 ROUGE2: 15.9339 ROUGEL: 24.3491 ROUGELSUM: 35.6714 GEN_LEN: 8185045","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B300-SXM-270GB","8","8","closed/NVIDIA/results/B300-SXM-270GBx8_TRT/llama3.1-8b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA DGX B300 (8x B300-SXM-270GB, TensorRT)","B300-SXM-270GBx8_TRT","llama3.1-8b","llama3.1-8b","Server","141565.0","ROUGE1: 38.5059 ROUGE2: 15.9356 ROUGEL: 24.3441 ROUGELSUM: 35.6683 GEN_LEN: 8185542","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B300-SXM-270GB","8","8","closed/NVIDIA/results/B300-SXM-270GBx8_TRT/llama3.1-8b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA DGX B300 (8x B300-SXM-270GB, TensorRT)","B300-SXM-270GBx8_TRT","wan-2.2-t2v-a14b","wan-2.2-t2v-a14b","Offline","0.0585234","vbench_score: 70.3453","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B300-SXM-270GB","8","8","closed/NVIDIA/results/B300-SXM-270GBx8_TRT/wan-2.2-t2v-a14b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Samples/s","fp32","","" +"NVIDIA","available","closed","datacenter","NVIDIA DGX B300 (8x B300-SXM-270GB, TensorRT)","B300-SXM-270GBx8_TRT","wan-2.2-t2v-a14b","wan-2.2-t2v-a14b","SingleStream","23.778912625","vbench_score: 70.7974","1","Intel(R) Xeon(R) Platinum 8570","2","56","NVIDIA B300-SXM-270GB","8","8","closed/NVIDIA/results/B300-SXM-270GBx8_TRT/wan-2.2-t2v-a14b/SingleStream/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Latency (s)","fp32","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB200 NVL72 (4x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x4_TRT","qwen3-vl-235b-a22b","qwen3-vl-235b-a22b","Offline","43.7662","F1_HIERARCHICAL: 0.7884930210985771","1","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","4","closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x4_TRT/qwen3-vl-235b-a22b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB200 NVL72, 1200W TGP Per GPU","closed","0","v6.0","0","False","Samples/s","fp32","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB200 NVL72 (4x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x4_TRT","qwen3-vl-235b-a22b","qwen3-vl-235b-a22b","Server","38.0313","F1_HIERARCHICAL: 0.7883566298697722","1","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","4","closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x4_TRT/qwen3-vl-235b-a22b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB200 NVL72, 1200W TGP Per GPU","closed","0","v6.0","0","False","Queries/s","fp32","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB200 NVL72 (4x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x4_TRT","wan-2.2-t2v-a14b","wan-2.2-t2v-a14b","Offline","0.0269817","vbench_score: 69.8883","1","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","4","closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x4_TRT/wan-2.2-t2v-a14b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB200 NVL72, 1200W TGP Per GPU","closed","0","v6.0","0","False","Samples/s","fp32","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB200 NVL72 (4x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x4_TRT","wan-2.2-t2v-a14b","wan-2.2-t2v-a14b","SingleStream","40.243719116","vbench_score: 70.6549","1","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","4","closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x4_TRT/wan-2.2-t2v-a14b/SingleStream/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB200 NVL72, 1200W TGP Per GPU","closed","0","v6.0","0","False","Latency (s)","fp32","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB200 NVL72 (72x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x72_TRT","deepseek-r1","deepseek-r1","Interactive","240318.0","exact_match: 81.51777575205105 TOKENS_PER_SAMPLE: 3778.798541476755","18","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","72","closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/deepseek-r1/Interactive/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB200 NVL72, 1200W TGP Per GPU","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB200 NVL72 (72x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x72_TRT","deepseek-r1","deepseek-r1","Offline","486141.0","exact_match: 81.51777575205105 TOKENS_PER_SAMPLE: 3761.302415679125","18","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","72","closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/deepseek-r1/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB200 NVL72, 1200W TGP Per GPU","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB200 NVL72 (72x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x72_TRT","deepseek-r1","deepseek-r1","Server","336106.0","exact_match: 81.74567000911577 TOKENS_PER_SAMPLE: 3737.3990428441202","18","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","72","closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/deepseek-r1/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB200 NVL72, 1200W TGP Per GPU","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB200 NVL72 (72x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x72_TRT","dlrm-v3","dlrm-v3","Offline","104637.0","DLRM_NE: 0.8675504587396562 DLRM_ACC: 0.6964261896038318 DLRM_AUC: 0.7862891896349031","18","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","72","closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/dlrm-v3/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB200 NVL72, 1200W TGP Per GPU","closed","0","v6.0","0","False","Samples/s","fp16","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB200 NVL72 (72x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x72_TRT","dlrm-v3","dlrm-v3","Server","99996.8","DLRM_NE: 0.8675504587396344 DLRM_ACC: 0.6964261896038318 DLRM_AUC: 0.7862891896349031","18","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","72","closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/dlrm-v3/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB200 NVL72, 1200W TGP Per GPU","closed","0","v6.0","0","False","Queries/s","fp16","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB200 NVL72 (72x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x72_TRT","gpt-oss-120b","gpt-oss-120b","Interactive","624929.0","exact_match: 83.609","18","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","72","closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/gpt-oss-120b/Interactive/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB200 NVL72, 1200W TGP Per GPU","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB200 NVL72 (72x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x72_TRT","gpt-oss-120b","gpt-oss-120b","Offline","879542.0","exact_match: 83.630","18","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","72","closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/gpt-oss-120b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB200 NVL72, 1200W TGP Per GPU","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB200 NVL72 (72x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x72_TRT","gpt-oss-120b","gpt-oss-120b","Server","899218.0","exact_match: 83.181","18","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","72","closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/gpt-oss-120b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB200 NVL72, 1200W TGP Per GPU","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB200 NVL72 (72x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x72_TRT","llama2-70b-99","llama2-70b-99","Interactive","754855.0","ROUGE1: 44.8415 ROUGE2: 22.4159 ROUGEL: 29.2129 TOKENS_PER_SAMPLE: 271.5","18","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","72","closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/llama2-70b-99/Interactive/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB200 NVL72, 1200W TGP Per GPU","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB200 NVL72 (72x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x72_TRT","llama2-70b-99","llama2-70b-99","Offline","888054.0","ROUGE1: 44.8311 ROUGE2: 22.4118 ROUGEL: 29.2118 TOKENS_PER_SAMPLE: 271.5","18","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","72","closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/llama2-70b-99/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB200 NVL72, 1200W TGP Per GPU","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB200 NVL72 (72x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x72_TRT","llama2-70b-99","llama2-70b-99","Server","810104.0","ROUGE1: 44.8281 ROUGE2: 22.407 ROUGEL: 29.2059 TOKENS_PER_SAMPLE: 271.6","18","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","72","closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/llama2-70b-99/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB200 NVL72, 1200W TGP Per GPU","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB200 NVL72 (72x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x72_TRT","llama2-70b-99.9","llama2-70b-99.9","Interactive","754855.0","ROUGE1: 44.8415 ROUGE2: 22.4159 ROUGEL: 29.2129 TOKENS_PER_SAMPLE: 271.5","18","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","72","closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/llama2-70b-99.9/Interactive/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB200 NVL72, 1200W TGP Per GPU","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB200 NVL72 (72x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x72_TRT","llama2-70b-99.9","llama2-70b-99.9","Offline","888054.0","ROUGE1: 44.8311 ROUGE2: 22.4118 ROUGEL: 29.2118 TOKENS_PER_SAMPLE: 271.5","18","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","72","closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/llama2-70b-99.9/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB200 NVL72, 1200W TGP Per GPU","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB200 NVL72 (72x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x72_TRT","llama2-70b-99.9","llama2-70b-99.9","Server","810104.0","ROUGE1: 44.8281 ROUGE2: 22.407 ROUGEL: 29.2059 TOKENS_PER_SAMPLE: 271.6","18","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","72","closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/llama2-70b-99.9/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB200 NVL72, 1200W TGP Per GPU","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB200 NVL72 (72x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x72_TRT","llama3.1-405b","llama3.1-405b","Interactive","14010.1","ROUGEL: 21.784306887727258 exact_match: 89.97819986763734 TOKENS_PER_SAMPLE: 638.7","18","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","72","closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/llama3.1-405b/Interactive/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB200 NVL72, 1200W TGP Per GPU","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB200 NVL72 (72x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x72_TRT","llama3.1-405b","llama3.1-405b","Offline","15461.5","ROUGEL: 21.791103518649138 exact_match: 89.965337524818 TOKENS_PER_SAMPLE: 636.9","18","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","72","closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/llama3.1-405b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB200 NVL72, 1200W TGP Per GPU","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB200 NVL72 (72x GB200-186GB_aarch64, TensorRT)","GB200-NVL72_GB200-186GB_aarch64x72_TRT","llama3.1-405b","llama3.1-405b","Server","14133.9","ROUGEL: 21.78378090513782 exact_match: 90.00418266048975 TOKENS_PER_SAMPLE: 640.0","18","NVIDIA Grace CPU","2","144","NVIDIA GB200","4","72","closed/NVIDIA/results/GB200-NVL72_GB200-186GB_aarch64x72_TRT/llama3.1-405b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB200 NVL72, 1200W TGP Per GPU","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB300 NVL72 (4x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x4_TRT","qwen3-vl-235b-a22b","qwen3-vl-235b-a22b","Offline","61.2443","F1_HIERARCHICAL: 0.7882200623476316","1","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","4","closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x4_TRT/qwen3-vl-235b-a22b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72, 1400W TGP Per GPU","closed","0","v6.0","0","False","Samples/s","fp32","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB300 NVL72 (4x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x4_TRT","qwen3-vl-235b-a22b","qwen3-vl-235b-a22b","Server","41.6351","F1_HIERARCHICAL: 0.7883219387216303","1","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","4","closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x4_TRT/qwen3-vl-235b-a22b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72, 1400W TGP Per GPU","closed","0","v6.0","0","False","Queries/s","fp32","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB300 NVL72 (4x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x4_TRT","wan-2.2-t2v-a14b","wan-2.2-t2v-a14b","Offline","0.036622","vbench_score: 70.0891","1","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","4","closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x4_TRT/wan-2.2-t2v-a14b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72, 1400W TGP Per GPU","closed","0","v6.0","0","False","Samples/s","fp32","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB300 NVL72 (4x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x4_TRT","wan-2.2-t2v-a14b","wan-2.2-t2v-a14b","SingleStream","30.910508163","vbench_score: 70.2488","1","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","4","closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x4_TRT/wan-2.2-t2v-a14b/SingleStream/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72, 1400W TGP Per GPU","closed","0","v6.0","0","False","Latency (s)","fp32","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x288_TRT","deepseek-r1","deepseek-r1","Offline","2494310.0","exact_match: 81.60893345487693 TOKENS_PER_SAMPLE: 3773.291704649043","72","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","288","closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x288_TRT/deepseek-r1/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x288_TRT","deepseek-r1","deepseek-r1","Server","1555110.0","exact_match: 81.56335460346399 TOKENS_PER_SAMPLE: 3753.950774840474","72","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","288","closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x288_TRT/deepseek-r1/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x72_TRT","deepseek-r1","deepseek-r1","Interactive","250634.0","exact_match: 81.54056517775751 TOKENS_PER_SAMPLE: 3744.0936645396537","18","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","72","closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/deepseek-r1/Interactive/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72, 1400W TGP Per GPU","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x72_TRT","deepseek-r1","deepseek-r1","Offline","647076.0","exact_match: 81.10756608933455 TOKENS_PER_SAMPLE: 3775.2474931631723","18","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","72","closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/deepseek-r1/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72, 1400W TGP Per GPU","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x72_TRT","deepseek-r1","deepseek-r1","Server","400437.0","exact_match: 81.70009115770283 TOKENS_PER_SAMPLE: 3771.458979033728","18","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","72","closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/deepseek-r1/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72, 1400W TGP Per GPU","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x72_TRT","gpt-oss-120b","gpt-oss-120b","Interactive","677199.0","exact_match: 83.361","18","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","72","closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/gpt-oss-120b/Interactive/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72, 1400W TGP Per GPU","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x72_TRT","gpt-oss-120b","gpt-oss-120b","Offline","1042980.0","exact_match: 83.503","18","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","72","closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/gpt-oss-120b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72, 1400W TGP Per GPU","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x72_TRT","gpt-oss-120b","gpt-oss-120b","Server","1072250.0","exact_match: 82.918","18","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","72","closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/gpt-oss-120b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72, 1400W TGP Per GPU","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x72_TRT","llama2-70b-99","llama2-70b-99","Interactive","814128.0","ROUGE1: 44.8316 ROUGE2: 22.3762 ROUGEL: 29.1984 TOKENS_PER_SAMPLE: 272.2","18","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","72","closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/llama2-70b-99/Interactive/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72, 1400W TGP Per GPU","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x72_TRT","llama2-70b-99","llama2-70b-99","Offline","1126850.0","ROUGE1: 44.8406 ROUGE2: 22.4168 ROUGEL: 29.2108 TOKENS_PER_SAMPLE: 272.0","18","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","72","closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/llama2-70b-99/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72, 1400W TGP Per GPU","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x72_TRT","llama2-70b-99","llama2-70b-99","Server","868278.0","ROUGE1: 44.8072 ROUGE2: 22.3641 ROUGEL: 29.1722 TOKENS_PER_SAMPLE: 272.7","18","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","72","closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/llama2-70b-99/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72, 1400W TGP Per GPU","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x72_TRT","llama2-70b-99.9","llama2-70b-99.9","Interactive","814128.0","ROUGE1: 44.8316 ROUGE2: 22.3762 ROUGEL: 29.1984 TOKENS_PER_SAMPLE: 272.2","18","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","72","closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/llama2-70b-99.9/Interactive/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72, 1400W TGP Per GPU","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x72_TRT","llama2-70b-99.9","llama2-70b-99.9","Offline","1126850.0","ROUGE1: 44.8406 ROUGE2: 22.4168 ROUGEL: 29.2108 TOKENS_PER_SAMPLE: 272.0","18","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","72","closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/llama2-70b-99.9/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72, 1400W TGP Per GPU","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x72_TRT","llama2-70b-99.9","llama2-70b-99.9","Server","868278.0","ROUGE1: 44.8072 ROUGE2: 22.3641 ROUGEL: 29.1722 TOKENS_PER_SAMPLE: 272.7","18","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","72","closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/llama2-70b-99.9/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72, 1400W TGP Per GPU","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x72_TRT","llama3.1-405b","llama3.1-405b","Interactive","18365.4","ROUGEL: 21.756418050547687 exact_match: 90.05151555261416 TOKENS_PER_SAMPLE: 641.5","18","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","72","closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/llama3.1-405b/Interactive/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72, 1400W TGP Per GPU","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x72_TRT","llama3.1-405b","llama3.1-405b","Offline","19511.7","ROUGEL: 21.728067830579086 exact_match: 89.94727332892126 TOKENS_PER_SAMPLE: 638.8","18","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","72","closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/llama3.1-405b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72, 1400W TGP Per GPU","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","closed","datacenter","NVIDIA GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x72_TRT","llama3.1-405b","llama3.1-405b","Server","18627.6","ROUGEL: 21.801876398806673 exact_match: 90.001455989411 TOKENS_PER_SAMPLE: 642.7","18","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","72","closed/NVIDIA/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/llama3.1-405b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72, 1400W TGP Per GPU","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","open","edge","NVIDIA Jetson AGX Thor Developer Kit 128G (TensorRT)","Jetson-Thor","llama3.1-8b-edge","llama3.1-8b-edge","Offline","350.605","ROUGE1: 38.5299 ROUGE2: 16.2005 ROUGEL: 24.5013 ROUGELSUM: 35.6685 GEN_LEN: 3089315","1","14-core ARM Poseidon-AE CPU","1","14","NVIDIA Jetson AGX Thor 128G","1","1","open/NVIDIA/results/Jetson-Thor/llama3.1-8b-edge/Offline/performance/run_1","TensorRT 10.8, CUDA 12.8","Jetson L4T","","open","0","v6.0","0","False","Tokens/s","fp4","","" +"NVIDIA","available","open","edge","NVIDIA Jetson AGX Thor Developer Kit 128G (TensorRT)","Jetson-Thor","llama3.1-8b-edge","llama3.1-8b-edge","SingleStream","1357.552837","ROUGE1: 38.3796 ROUGE2: 15.9406 ROUGEL: 24.3855 ROUGELSUM: 35.5168 GEN_LEN: 3112369","1","14-core ARM Poseidon-AE CPU","1","14","NVIDIA Jetson AGX Thor 128G","1","1","open/NVIDIA/results/Jetson-Thor/llama3.1-8b-edge/SingleStream/performance/run_1","TensorRT 10.8, CUDA 12.8","Jetson L4T","","open","0","v6.0","0","False","Latency (ms)","fp4","","" +"Nebius","available","closed","datacenter","Nebius B200 n1 (8x B200-SXM-180GB, TensorRT)","nebius_b200_n1","deepseek-r1","deepseek-r1","Offline","58581.6","exact_match: 81.79124886052871 TOKENS_PER_SAMPLE: 3765.5856882406565","1","Intel(R) Xeon(R) Platinum 8580","2","60","NVIDIA B200-SXM-180GB","8","8","closed/Nebius/results/nebius_b200_n1/deepseek-r1/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04.3 LTS (Linux kernel: 6.11.0-1016-nvidia #16-Ubuntu SMP PREEMPT_DYNAMIC Sun Sep 21 17:06:33 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux)","B200 TGP 1000W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Nebius","available","closed","datacenter","Nebius B200 n1 (8x B200-SXM-180GB, TensorRT)","nebius_b200_n1","deepseek-r1","deepseek-r1","Server","51692.9","exact_match: 81.42661804922516 TOKENS_PER_SAMPLE: 3735.2408842297173","1","Intel(R) Xeon(R) Platinum 8580","2","60","NVIDIA B200-SXM-180GB","8","8","closed/Nebius/results/nebius_b200_n1/deepseek-r1/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04.3 LTS (Linux kernel: 6.11.0-1016-nvidia #16-Ubuntu SMP PREEMPT_DYNAMIC Sun Sep 21 17:06:33 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux)","B200 TGP 1000W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Nebius","available","closed","datacenter","Nebius B200 n1 (8x B200-SXM-180GB, TensorRT)","nebius_b200_n1","gpt-oss-120b","gpt-oss-120b","Interactive","13155.3","exact_match: 83.295","1","Intel(R) Xeon(R) Platinum 8580","2","60","NVIDIA B200-SXM-180GB","8","8","closed/Nebius/results/nebius_b200_n1/gpt-oss-120b/Interactive/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04.3 LTS (Linux kernel: 6.11.0-1016-nvidia #16-Ubuntu SMP PREEMPT_DYNAMIC Sun Sep 21 17:06:33 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux)","B200 TGP 1000W","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"Nebius","available","closed","datacenter","Nebius B200 n1 (8x B200-SXM-180GB, TensorRT)","nebius_b200_n1","gpt-oss-120b","gpt-oss-120b","Offline","85921.2","exact_match: 83.750","1","Intel(R) Xeon(R) Platinum 8580","2","60","NVIDIA B200-SXM-180GB","8","8","closed/Nebius/results/nebius_b200_n1/gpt-oss-120b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04.3 LTS (Linux kernel: 6.11.0-1016-nvidia #16-Ubuntu SMP PREEMPT_DYNAMIC Sun Sep 21 17:06:33 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux)","B200 TGP 1000W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Nebius","available","closed","datacenter","Nebius B200 n1 (8x B200-SXM-180GB, TensorRT)","nebius_b200_n1","gpt-oss-120b","gpt-oss-120b","Server","87444.2","exact_match: 83.833","1","Intel(R) Xeon(R) Platinum 8580","2","60","NVIDIA B200-SXM-180GB","8","8","closed/Nebius/results/nebius_b200_n1/gpt-oss-120b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04.3 LTS (Linux kernel: 6.11.0-1016-nvidia #16-Ubuntu SMP PREEMPT_DYNAMIC Sun Sep 21 17:06:33 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux)","B200 TGP 1000W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Nebius","available","closed","datacenter","Nebius B300 n1 (8x B300-SXM-270GB, TensorRT)","nebius_b300_n1","deepseek-r1","deepseek-r1","Offline","69318.9","exact_match: 81.58614402917047 TOKENS_PER_SAMPLE: 3722.1116681859617","1","Intel(R) Xeon(R) 6776P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/Nebius/results/nebius_b300_n1/deepseek-r1/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04.3 LTS (Linux kernel: 6.11.0-1016-nvidia #16-Ubuntu SMP PREEMPT_DYNAMIC Sun Sep 21 17:06:33 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux)","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Nebius","available","closed","datacenter","Nebius B300 n1 (8x B300-SXM-270GB, TensorRT)","nebius_b300_n1","deepseek-r1","deepseek-r1","Server","60413.4","exact_match: 81.58614402917047 TOKENS_PER_SAMPLE: 3721.7894257064722","1","Intel(R) Xeon(R) 6776P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/Nebius/results/nebius_b300_n1/deepseek-r1/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04.3 LTS (Linux kernel: 6.11.0-1016-nvidia #16-Ubuntu SMP PREEMPT_DYNAMIC Sun Sep 21 17:06:33 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux)","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Nebius","available","closed","datacenter","Nebius B300 n1 (8x B300-SXM-270GB, TensorRT)","nebius_b300_n1","gpt-oss-120b","gpt-oss-120b","offline","106885.0","exact_match: 82.959","1","Intel(R) Xeon(R) 6776P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/Nebius/results/nebius_b300_n1/gpt-oss-120b/offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04.3 LTS (Linux kernel: 6.11.0-1016-nvidia #16-Ubuntu SMP PREEMPT_DYNAMIC Sun Sep 21 17:06:33 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux)","B300 TGP 1100W","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"Nebius","available","closed","datacenter","Nebius B300 n1 (8x B300-SXM-270GB, TensorRT)","nebius_b300_n1","gpt-oss-120b","gpt-oss-120b","server","100437.0","exact_match: 83.337","1","Intel(R) Xeon(R) 6776P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/Nebius/results/nebius_b300_n1/gpt-oss-120b/server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04.3 LTS (Linux kernel: 6.11.0-1016-nvidia #16-Ubuntu SMP PREEMPT_DYNAMIC Sun Sep 21 17:06:33 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux)","B300 TGP 1100W","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"Nebius","available","closed","datacenter","Nebius B300 n1 (8x B300-SXM-270GB, TensorRT)","nebius_b300_n1","qwen3-vl-235b-a22b","qwen3-vl-235b-a22b","offline","78.2775","F1_HIERARCHICAL: 0.7880721729948121","1","Intel(R) Xeon(R) 6776P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/Nebius/results/nebius_b300_n1/qwen3-vl-235b-a22b/offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04.3 LTS (Linux kernel: 6.11.0-1016-nvidia #16-Ubuntu SMP PREEMPT_DYNAMIC Sun Sep 21 17:06:33 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux)","B300 TGP 1100W","closed","0","v6.0","1","False","Samples/s","BF16/F32/F8_E4M3/U8","","" +"Nebius","available","closed","datacenter","Nebius B300 n1 (8x B300-SXM-270GB, TensorRT)","nebius_b300_n1","qwen3-vl-235b-a22b","qwen3-vl-235b-a22b","server","45.1513","F1_HIERARCHICAL: 0.7865237675409595","1","Intel(R) Xeon(R) 6776P","2","64","NVIDIA B300-SXM-270GB","8","8","closed/Nebius/results/nebius_b300_n1/qwen3-vl-235b-a22b/server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04.3 LTS (Linux kernel: 6.11.0-1016-nvidia #16-Ubuntu SMP PREEMPT_DYNAMIC Sun Sep 21 17:06:33 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux)","B300 TGP 1100W","closed","0","v6.0","1","False","Queries/s","BF16/F32/F8_E4M3/U8","","" +"Nebius","available","closed","datacenter","Nebius GB300 NVL72 (4x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x1_TRT","gpt-oss-120b","gpt-oss-120b","Offline","15371.7","exact_match: 83.529","1","NVIDIA Grace CPU","2","144","NVIDIA GB300","1","1","closed/Nebius/results/GB300-NVL72_GB300-288GB_aarch64x1_TRT/gpt-oss-120b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, Driver 580.105.08, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Nebius","available","closed","datacenter","Nebius GB300 NVL72 (4x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x1_TRT","gpt-oss-120b","gpt-oss-120b","Server","14973.1","exact_match: 83.915","1","NVIDIA Grace CPU","2","144","NVIDIA GB300","1","1","closed/Nebius/results/GB300-NVL72_GB300-288GB_aarch64x1_TRT/gpt-oss-120b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, Driver 580.105.08, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Nebius","available","closed","datacenter","Nebius GB300 NVL72 (4x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x4_TRT","qwen3-vl-235b-a22b","qwen3-vl-235b-a22b","Offline","60.6154","F1_HIERARCHICAL: 0.7876592354860111","1","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","4","closed/Nebius/results/GB300-NVL72_GB300-288GB_aarch64x4_TRT/qwen3-vl-235b-a22b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, Driver 580.105.08, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72","closed","0","v6.0","0","False","Samples/s","fp32","","" +"Nebius","available","closed","datacenter","Nebius GB300 NVL72 (4x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x4_TRT","qwen3-vl-235b-a22b","qwen3-vl-235b-a22b","Server","43.1538","F1_HIERARCHICAL: 0.787752047599809","1","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","4","closed/Nebius/results/GB300-NVL72_GB300-288GB_aarch64x4_TRT/qwen3-vl-235b-a22b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, Driver 580.105.08, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72","closed","0","v6.0","0","False","Queries/s","fp32","","" +"Nebius","available","closed","datacenter","Nebius GB300 NVL72 (4x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x8_TRT","deepseek-r1","deepseek-r1","Offline","76346.6","exact_match: 81.76845943482225 TOKENS_PER_SAMPLE: 3735.315861440292","2","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","8","closed/Nebius/results/GB300-NVL72_GB300-288GB_aarch64x8_TRT/deepseek-r1/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, Driver 580.105.08, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Nebius","available","closed","datacenter","Nebius GB300 NVL72 (4x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x8_TRT","deepseek-r1","deepseek-r1","Server","64510.0","exact_match: 81.40382862351869 TOKENS_PER_SAMPLE: 3766.282133090246","2","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","8","closed/Nebius/results/GB300-NVL72_GB300-288GB_aarch64x8_TRT/deepseek-r1/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, Driver 580.105.08, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Nebius","available","closed","datacenter","Nebius GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x72_TRT","deepseek-r1","deepseek-r1","Offline","673936.0","exact_match: 81.44940747493163 TOKENS_PER_SAMPLE: 3806.084092980857","18","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","72","closed/Nebius/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/deepseek-r1/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, Driver 580.105.08, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Nebius","available","closed","datacenter","Nebius GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x72_TRT","deepseek-r1","deepseek-r1","Server","575580.0","exact_match: 80.78851412944394 TOKENS_PER_SAMPLE: 3751.325205104831","18","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","72","closed/Nebius/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/deepseek-r1/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, Driver 580.105.08, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Nebius","available","closed","datacenter","Nebius GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x72_TRT","gpt-oss-120b","gpt-oss-120b","Offline","1046150.0","exact_match: 83.036","18","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","72","closed/Nebius/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/gpt-oss-120b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, Driver 580.105.08, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Nebius","available","closed","datacenter","Nebius GB300 NVL72 (72x GB300-288GB_aarch64, TensorRT)","GB300-NVL72_GB300-288GB_aarch64x72_TRT","gpt-oss-120b","gpt-oss-120b","Server","1096770.0","exact_match: 83.217","18","NVIDIA Grace CPU","2","144","NVIDIA GB300","4","72","closed/Nebius/results/GB300-NVL72_GB300-288GB_aarch64x72_TRT/gpt-oss-120b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, Driver 580.105.08, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","NVIDIA GB300 NVL72","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Netweb_technologies","available","closed","datacenter","Camarero PDI200A2HG-810 (8x B200-SXM-180GB, TensorRT)","B200-SXM-180GBx8_TRT","dlrm-v3","dlrm-v3","Offline","10737.2","DLRM_NE: 0.8675504587396444 DLRM_ACC: 0.6964261896038318 DLRM_AUC: 0.7862891896349031","1","Intel(R) Xeon(R) Platinum 8562Y+","2","56","NVIDIA B200-SXM-180GB","8","8","closed/Netweb_technologies/results/B200-SXM-180GBx8_TRT/dlrm-v3/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","B200 TGP 1000W","closed","0","v6.0","0","False","Samples/s","fp16","","" +"Netweb_technologies","available","closed","datacenter","Camarero PDI200A2HG-810 (8x B200-SXM-180GB, TensorRT)","B200-SXM-180GBx8_TRT","dlrm-v3","dlrm-v3","Server","10007.0","DLRM_NE: 0.8675504587396445 DLRM_ACC: 0.6964261896038318 DLRM_AUC: 0.7862891896349031","1","Intel(R) Xeon(R) Platinum 8562Y+","2","56","NVIDIA B200-SXM-180GB","8","8","closed/Netweb_technologies/results/B200-SXM-180GBx8_TRT/dlrm-v3/Server/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","B200 TGP 1000W","closed","0","v6.0","0","False","Queries/s","fp16","","" +"Netweb_technologies","available","closed","datacenter","Camarero PDI200A2HG-810 (8x B200-SXM-180GB, TensorRT)","B200-SXM-180GBx8_TRT","whisper","whisper","Offline","45083.1","ACCURACY: 97.86632659858961","1","Intel(R) Xeon(R) Platinum 8562Y+","2","56","NVIDIA B200-SXM-180GB","8","8","closed/Netweb_technologies/results/B200-SXM-180GBx8_TRT/whisper/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1, cuDNN 9.17, TensorRT-LLM feat/1.2-mlpinf, NVIDIA Dynamo mlperf-v6.0-dynamo-v0.8.0, vLLM CentML:mlperf-inf-mm-q3vl-v6.0","Ubuntu 24.04","B200 TGP 1000W","closed","0","v6.0","0","False","Samples/s","fp16","","" +"ORACLE","available","closed","datacenter","BM.GPU.MI355X.8","8xMI355X_2xEPYC_9575F","gpt-oss-120b","gpt-oss-120b","Offline","91706.3","exact_match: 83.801","1","AMD EPYC 9575F","2","128","AMD Instinct MI355X 288GB HBM3e","8","8","closed/ORACLE/results/8xMI355X_2xEPYC_9575F/gpt-oss-120b/Offline/performance/run_1","N/A","Ubuntu 24.04.3 LTS","N/A. System typically deployed with ROCm stack and standard OCI GPU images for AI/HPC workloads.","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"ORACLE","available","closed","datacenter","BM.GPU.MI355X.8","8xMI355X_2xEPYC_9575F","gpt-oss-120b","gpt-oss-120b","Server","78186.3","exact_match: 82.597","1","AMD EPYC 9575F","2","128","AMD Instinct MI355X 288GB HBM3e","8","8","closed/ORACLE/results/8xMI355X_2xEPYC_9575F/gpt-oss-120b/Server/performance/run_1","N/A","Ubuntu 24.04.3 LTS","N/A. System typically deployed with ROCm stack and standard OCI GPU images for AI/HPC workloads.","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"ORACLE","available","closed","datacenter","BM.GPU.MI355X.8","8xMI355X_2xEPYC_9575F","llama2-70b-99","llama2-70b-99","Interactive","72116.7","ROUGE1: 44.5404 ROUGE2: 22.1362 ROUGEL: 28.9134 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9575F","2","128","AMD Instinct MI355X 288GB HBM3e","8","8","closed/ORACLE/results/8xMI355X_2xEPYC_9575F/llama2-70b-99/Interactive/performance/run_1","N/A","Ubuntu 24.04.3 LTS","N/A. System typically deployed with ROCm stack and standard OCI GPU images for AI/HPC workloads.","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"ORACLE","available","closed","datacenter","BM.GPU.MI355X.8","8xMI355X_2xEPYC_9575F","llama2-70b-99","llama2-70b-99","Offline","99874.3","ROUGE1: 44.4665 ROUGE2: 22.092 ROUGEL: 28.8666 TOKENS_PER_SAMPLE: 283.9","1","AMD EPYC 9575F","2","128","AMD Instinct MI355X 288GB HBM3e","8","8","closed/ORACLE/results/8xMI355X_2xEPYC_9575F/llama2-70b-99/Offline/performance/run_1","N/A","Ubuntu 24.04.3 LTS","N/A. System typically deployed with ROCm stack and standard OCI GPU images for AI/HPC workloads.","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"ORACLE","available","closed","datacenter","BM.GPU.MI355X.8","8xMI355X_2xEPYC_9575F","llama2-70b-99","llama2-70b-99","Server","98239.1","ROUGE1: 44.5401 ROUGE2: 22.1345 ROUGEL: 28.9098 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9575F","2","128","AMD Instinct MI355X 288GB HBM3e","8","8","closed/ORACLE/results/8xMI355X_2xEPYC_9575F/llama2-70b-99/Server/performance/run_1","N/A","Ubuntu 24.04.3 LTS","N/A. System typically deployed with ROCm stack and standard OCI GPU images for AI/HPC workloads.","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"ORACLE","available","closed","datacenter","BM.GPU.MI355X.8","8xMI355X_2xEPYC_9575F","llama2-70b-99.9","llama2-70b-99.9","Interactive","72116.7","ROUGE1: 44.5404 ROUGE2: 22.1362 ROUGEL: 28.9134 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9575F","2","128","AMD Instinct MI355X 288GB HBM3e","8","8","closed/ORACLE/results/8xMI355X_2xEPYC_9575F/llama2-70b-99.9/Interactive/performance/run_1","N/A","Ubuntu 24.04.3 LTS","N/A. System typically deployed with ROCm stack and standard OCI GPU images for AI/HPC workloads.","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"ORACLE","available","closed","datacenter","BM.GPU.MI355X.8","8xMI355X_2xEPYC_9575F","llama2-70b-99.9","llama2-70b-99.9","Offline","99874.3","ROUGE1: 44.4665 ROUGE2: 22.092 ROUGEL: 28.8666 TOKENS_PER_SAMPLE: 283.9","1","AMD EPYC 9575F","2","128","AMD Instinct MI355X 288GB HBM3e","8","8","closed/ORACLE/results/8xMI355X_2xEPYC_9575F/llama2-70b-99.9/Offline/performance/run_1","N/A","Ubuntu 24.04.3 LTS","N/A. System typically deployed with ROCm stack and standard OCI GPU images for AI/HPC workloads.","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"ORACLE","available","closed","datacenter","BM.GPU.MI355X.8","8xMI355X_2xEPYC_9575F","llama2-70b-99.9","llama2-70b-99.9","Server","98239.1","ROUGE1: 44.5401 ROUGE2: 22.1345 ROUGEL: 28.9098 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9575F","2","128","AMD Instinct MI355X 288GB HBM3e","8","8","closed/ORACLE/results/8xMI355X_2xEPYC_9575F/llama2-70b-99.9/Server/performance/run_1","N/A","Ubuntu 24.04.3 LTS","N/A. System typically deployed with ROCm stack and standard OCI GPU images for AI/HPC workloads.","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Quanta_Cloud_Technology","available","closed","datacenter","1-node-2S-GNR_86C","1-node-2S-GNR_86C","llama3_1-8b","llama3.1-8b","Offline","777.671","ROUGE1: 38.7399 ROUGE2: 15.8699 ROUGEL: 24.3997 ROUGELSUM: 35.7855 GEN_LEN: 8144909","1","Intel(R) Xeon(R) 6787P","2","86","N/A","0","0","closed/Quanta_Cloud_Technology/results/1-node-2S-GNR_86C/llama3_1-8b/Offline/performance/run_1","PyTorch","Rocky Linux 9.i5","QuantaGrid D54Q-2U. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Quanta_Cloud_Technology","available","closed","datacenter","1-node-2S-GNR_86C","1-node-2S-GNR_86C","llama3_1-8b","llama3.1-8b","Server","284.359","ROUGE1: 38.7265 ROUGE2: 15.8924 ROUGEL: 24.395 ROUGELSUM: 35.7669 GEN_LEN: 8146962","1","Intel(R) Xeon(R) 6787P","2","86","N/A","0","0","closed/Quanta_Cloud_Technology/results/1-node-2S-GNR_86C/llama3_1-8b/Server/performance/run_1","PyTorch","Rocky Linux 9.i5","QuantaGrid D54Q-2U. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Quanta_Cloud_Technology","available","closed","datacenter","1-node-2S-GNR_86C","1-node-2S-GNR_86C","rgat","rgat","Offline","13151.7","acc: 72.567","1","Intel(R) Xeon(R) 6787P","2","86","N/A","0","0","closed/Quanta_Cloud_Technology/results/1-node-2S-GNR_86C/rgat/Offline/performance/run_1","PyTorch","Rocky Linux 9.i5","QuantaGrid D54Q-2U. N/A","closed","0","v6.0","0","False","Samples/s","INT8","","" +"Quanta_Cloud_Technology","available","closed","datacenter","1-node-2S-GNR_86C","1-node-2S-GNR_86C","whisper","whisper","Offline","1407.49","ACCURACY: 97.88576267638467","1","Intel(R) Xeon(R) 6787P","2","86","N/A","0","0","closed/Quanta_Cloud_Technology/results/1-node-2S-GNR_86C/whisper/Offline/performance/run_1","PyTorch","Rocky Linux 9.i5","QuantaGrid D54Q-2U. N/A","closed","0","v6.0","0","False","Samples/s","INT8","","" +"Quanta_Cloud_Technology","available","closed","datacenter","QuantaGrid D75E-4U (8x RTX PRO6000 96GB, TensorRT)","D75E-4U_RTX_PRO_6000_PCIE_96GBx8_TRT","whisper","whisper","Offline","17838.3","ACCURACY: 97.8387277900968","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA RTX PRO 6000 Blackwell Server Edition","8","8","closed/Quanta_Cloud_Technology/results/D75E-4U_RTX_PRO_6000_PCIE_96GBx8_TRT/whisper/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Rocky Linux 9.5","QuantaGrid D75E-4U","closed","0","v6.0","0","False","Samples/s","fp16","","" +"Quanta_Cloud_Technology","available","closed","datacenter","QuantaGrid D75H-10U (8x B300-SXM-270GB, TensorRT)","D75H-10U_B300-270GBx8_TRT","deepseek-r1","deepseek-r1","Offline","69263.2","exact_match: 80.92525068368278 TOKENS_PER_SAMPLE: 3795.107793983592","1","Intel(R) Xeon(R) 6747P","2","48","NVIDIA B300-SXM-270GB","8","8","closed/Quanta_Cloud_Technology/results/D75H-10U_B300-270GBx8_TRT/deepseek-r1/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Rocky Linux 9.5","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Quanta_Cloud_Technology","available","closed","datacenter","QuantaGrid D75H-10U (8x B300-SXM-270GB, TensorRT)","D75H-10U_B300-270GBx8_TRT","deepseek-r1","deepseek-r1","Server","35435.7","exact_match: 81.38103919781221 TOKENS_PER_SAMPLE: 3759.358477666363","1","Intel(R) Xeon(R) 6747P","2","48","NVIDIA B300-SXM-270GB","8","8","closed/Quanta_Cloud_Technology/results/D75H-10U_B300-270GBx8_TRT/deepseek-r1/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Rocky Linux 9.5","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Quanta_Cloud_Technology","available","closed","datacenter","QuantaGrid D75H-10U (8x B300-SXM-270GB, TensorRT)","D75H-10U_B300-270GBx8_TRT","llama3.1-405b","llama3.1-405b","Offline","1953.77","ROUGEL: 21.9243802693401 exact_match: 90.1098841826605 TOKENS_PER_SAMPLE: 648.6","1","Intel(R) Xeon(R) 6747P","2","48","NVIDIA B300-SXM-270GB","8","8","closed/Quanta_Cloud_Technology/results/D75H-10U_B300-270GBx8_TRT/llama3.1-405b/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Rocky Linux 9.5","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Quanta_Cloud_Technology","available","closed","datacenter","QuantaGrid D75H-10U (8x B300-SXM-270GB, TensorRT)","D75H-10U_B300-270GBx8_TRT","llama3.1-405b","llama3.1-405b","Server","1484.4","ROUGEL: 21.924447125381427 exact_match: 90.12480476505624 TOKENS_PER_SAMPLE: 647.6","1","Intel(R) Xeon(R) 6747P","2","48","NVIDIA B300-SXM-270GB","8","8","closed/Quanta_Cloud_Technology/results/D75H-10U_B300-270GBx8_TRT/llama3.1-405b/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Rocky Linux 9.5","B300 TGP 1100W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"RedHat","available","closed","datacenter","Dell B200,8xB200-SXM-180GB,RHEL 10.1,vLLM CentML:mlperf-inf-mm-q3vl-v6.0","8xb200_NV_vllm_qwen","qwen3-vl-235b-a22b","qwen3-vl-235b-a22b","offline","79.0373","F1_HIERARCHICAL: 0.7890600786539296","1","Intel(R) Xeon(R) Platinum 8570","2","112","NVIDIA B200-SXM-180GB","8","8","closed/RedHat/results/8xb200_NV_vllm_qwen/qwen3-vl-235b-a22b/offline/performance/run_1","vLLM + NVIDIA-dynamo + MLPerf Inference Qwen3-VL harness, RHEL 10.1","Red Hat Enterprise Linux 10.1 (Coughlan)","1 node with 8x NVIDIA B200; all-to-all NVLink (NV18); 2-socket Intel Xeon Platinum 8570; NUMA=2; ~2.0 TiB RAM; 4x 7.68TB Dell NVMe ISE PS1010 RI U.2 + Dell BOSS-N1 OS disk. RHEL 10.1, vLLM via Dynamo backend; async scheduling enabled; SJF enabled with --scheduling-policy=sjf; CUDA Graph capture enabled via compilation config","closed","0","v6.0","1","False","Samples/s","fp4","","" +"RedHat","available","closed","datacenter","Dell B200,8xB200-SXM-180GB,RHEL 10.1,vLLM CentML:mlperf-inf-mm-q3vl-v6.0","8xb200_NV_vllm_qwen","qwen3-vl-235b-a22b","qwen3-vl-235b-a22b","server","67.8642","F1_HIERARCHICAL: 0.7887584531969497","1","Intel(R) Xeon(R) Platinum 8570","2","112","NVIDIA B200-SXM-180GB","8","8","closed/RedHat/results/8xb200_NV_vllm_qwen/qwen3-vl-235b-a22b/server/performance/run_1","vLLM + NVIDIA-dynamo + MLPerf Inference Qwen3-VL harness, RHEL 10.1","Red Hat Enterprise Linux 10.1 (Coughlan)","1 node with 8x NVIDIA B200; all-to-all NVLink (NV18); 2-socket Intel Xeon Platinum 8570; NUMA=2; ~2.0 TiB RAM; 4x 7.68TB Dell NVMe ISE PS1010 RI U.2 + Dell BOSS-N1 OS disk. RHEL 10.1, vLLM via Dynamo backend; async scheduling enabled; SJF enabled with --scheduling-policy=sjf; CUDA Graph capture enabled via compilation config","closed","0","v6.0","1","False","Queries/s","fp4","","" +"RedHat","available","closed","datacenter","IBM Cloud gx3-48x240x2l40s instance,2xL40S-PCIE-48GB,vLLM 0.15.1","2xL40S_RHEL9","whisper","whisper","Offline","3646.91","ACCURACY: 98.1099574597676","1","Intel Xeon Processor (SapphireRapids)","2","48","NVIDIA L40S","2","2","closed/RedHat/results/2xL40S_RHEL9/whisper/Offline/performance/run_1","vLLM 0.15.1, RHEL 9.6","Red Hat Enterprise Linux 9.6","IBM Cloud gx3-48x240x2l40s instance with 2xL40S-PCIE-48G. RHEL 9.6, vllm 0.15.1","closed","0","v6.0","0","False","Samples/s","bfloat16","","" +"RedHat","available","closed","datacenter","IBM Cloud gx3d-160x1792x8h200,8xH200-SXM-141GB,RHEL 9.6,vLLM CentML:mlperf-inf-mm-q3vl-v6.0","8xh200_NV_vllm_qwen","qwen3-vl-235b-a22b","qwen3-vl-235b-a22b","offline","18.218","F1_HIERARCHICAL: 0.7900114757860913","1","Intel Xeon Processor (SapphireRapids)","2","80","NVIDIA H200-SXM-141GB","8","8","closed/RedHat/results/8xh200_NV_vllm_qwen/qwen3-vl-235b-a22b/offline/performance/run_1","vLLM 0.10.0, RHEL 9.6","Red Hat Enterprise Linux 9.6","IBM Cloud instance gx3d-160x1792x8h200 8xH200-SXM-141GB. RHEL 9.6, vllm 6.0rc2+cu130","closed","0","v6.0","1","False","Samples/s","fp8","","" +"RedHat","available","closed","datacenter","IBM Cloud gx3d-160x1792x8h200,8xH200-SXM-141GB,RHEL 9.6,vLLM CentML:mlperf-inf-mm-q3vl-v6.0","8xh200_NV_vllm_qwen","qwen3-vl-235b-a22b","qwen3-vl-235b-a22b","server","11.0498","F1_HIERARCHICAL: 0.7898129513572557","1","Intel Xeon Processor (SapphireRapids)","2","80","NVIDIA H200-SXM-141GB","8","8","closed/RedHat/results/8xh200_NV_vllm_qwen/qwen3-vl-235b-a22b/server/performance/run_1","vLLM 0.10.0, RHEL 9.6","Red Hat Enterprise Linux 9.6","IBM Cloud instance gx3d-160x1792x8h200 8xH200-SXM-141GB. RHEL 9.6, vllm 6.0rc2+cu130","closed","0","v6.0","1","False","Queries/s","fp8","","" +"RedHat","available","closed","datacenter","IBM Cloud qx3d-160x1792x8h200,NVIDIA 8xH200-SXM-141GB,vLLM 0.15.1","8xH200_whisper","whisper","whisper","Offline","36395.7","ACCURACY: 98.11090914281908","1","Intel Xeon Processor (Sapphire Rapids)","2","80","NVIDIA H200-SXM-141GB","8","8","closed/RedHat/results/8xH200_whisper/whisper/Offline/performance/run_1","vLLM 0.15.1, PyTorch 2.9.1+cu128, CUDA 12.9.86","Red Hat Enterprise Linux 9.6 (Plow), Kernel 5.14.0-570.79.1.el9_6.x86_64","IBM cloud H200 system with 8x NVIDIA H200 GPUs (141GB HBM3e each), 2-socket Intel Sapphire Rapids CPUs (80 cores total), 1.7TB DDR5 ECC memory, and 56TB NVMe storage. Using vLLM 0.15.1 with Flash-Attention, torch.compile enabled, 8 parallel workers (1 per GPU), GPU memory utilization 95%, max batch tokens 32000, max sequences 256","closed","0","v6.0","0","False","Samples/s","bfloat16","","" +"RedHat","available","closed","datacenter","LLM-D v0.5.0,OpenShift 4.21,IBM cloud gx3d-160x1792x8h200,NVIDIA 8xH200-SXM-141GB","8xH200-LLM-D-Openshift","gpt-oss-120b","gpt-oss-120b","Offline","28680.0","exact_match: 84.206","1","Intel Xeon Processor (SapphireRapids)","2","80","NVIDIA H200-SXM-141GB","8","8","closed/RedHat/results/8xH200-LLM-D-Openshift/gpt-oss-120b/Offline/performance/run_1","LLM-D v0.5.0 ,OpenShift 4.21, RHEL 9.6","Red Hat Enterprise Linux 9.6","IBM Cloud gx3d-160x1792x8h200 instance with 8xH200-SXM-141G. LLM-D 0.5.0, RHEL 9.6, vllm 0.14.1","closed","0","v6.0","0","False","Tokens/s","mxfp4","","" +"RedHat","available","closed","datacenter","LLM-D v0.5.0,OpenShift 4.21,IBM cloud gx3d-160x1792x8h200,NVIDIA 8xH200-SXM-141GB","8xH200-LLM-D-Openshift","gpt-oss-120b","gpt-oss-120b","Server","24103.2","exact_match: 83.680","1","Intel Xeon Processor (SapphireRapids)","2","80","NVIDIA H200-SXM-141GB","8","8","closed/RedHat/results/8xH200-LLM-D-Openshift/gpt-oss-120b/Server/performance/run_1","LLM-D v0.5.0 ,OpenShift 4.21, RHEL 9.6","Red Hat Enterprise Linux 9.6","IBM Cloud gx3d-160x1792x8h200 instance with 8xH200-SXM-141G. LLM-D 0.5.0, RHEL 9.6, vllm 0.14.1","closed","0","v6.0","0","False","Tokens/s","mxfp4","","" +"RedHat","available","closed","datacenter","LLM-D v0.5.0,Openshift 4.20.12,NVIDIA 8xB200-SXM-180GB","8xB200-LLM-D-Openshift","gpt-oss-120b","gpt-oss-120b","Offline","93070.7","exact_match: 84.028","1","Intel(R) Xeon(R) Platinum 8570","2","112","NVIDIA B200-SXM-180GB","8","8","closed/RedHat/results/8xB200-LLM-D-Openshift/gpt-oss-120b/Offline/performance/run_1","LLM-D v0.5.0 ,vLLM 0.14.1, RHEL 9.6","Red Hat Enterprise Linux 9.6","NVIDIA 8xB200-SXM-180G. LLM-D v0.5.0,Openshift 4.20.12,RHEL 9.6,vllm 0.14.1","closed","0","v6.0","0","False","Tokens/s","mxfp4","","" +"RedHat","available","closed","datacenter","LLM-D v0.5.0,Openshift 4.20.12,NVIDIA 8xB200-SXM-180GB","8xB200-LLM-D-Openshift","gpt-oss-120b","gpt-oss-120b","Server","71588.1","exact_match: 83.724","1","Intel(R) Xeon(R) Platinum 8570","2","112","NVIDIA B200-SXM-180GB","8","8","closed/RedHat/results/8xB200-LLM-D-Openshift/gpt-oss-120b/Server/performance/run_1","LLM-D v0.5.0 ,vLLM 0.14.1, RHEL 9.6","Red Hat Enterprise Linux 9.6","NVIDIA 8xB200-SXM-180G. LLM-D v0.5.0,Openshift 4.20.12,RHEL 9.6,vllm 0.14.1","closed","0","v6.0","0","False","Tokens/s","mxfp4","","" +"Supermicro","available","closed","datacenter","1-node-2S-GNR_128C","1-node-2S-GNR_128C","llama3_1-8b","llama3.1-8b","Offline","1185.98","ROUGE1: 38.7203 ROUGE2: 15.8576 ROUGEL: 24.3722 ROUGELSUM: 35.7664 GEN_LEN: 8142319","1","Intel(R) Xeon(R) 6980P","2","128","N/A","0","0","closed/Supermicro/results/1-node-2S-GNR_128C/llama3_1-8b/Offline/performance/run_1","PyTorch: rgat: 2.7.0+cpu, whipser: 2.7.0+cpu, llama3.1-8b: 2.8.0.dev20250627+cpu","Ubuntu 22.04.5","SYS-822GA-NGR3. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Supermicro","available","closed","datacenter","1-node-2S-GNR_128C","1-node-2S-GNR_128C","llama3_1-8b","llama3.1-8b","Server","497.339","ROUGE1: 38.7405 ROUGE2: 15.8949 ROUGEL: 24.4131 ROUGELSUM: 35.7831 GEN_LEN: 8147194","1","Intel(R) Xeon(R) 6980P","2","128","N/A","0","0","closed/Supermicro/results/1-node-2S-GNR_128C/llama3_1-8b/Server/performance/run_1","PyTorch: rgat: 2.7.0+cpu, whipser: 2.7.0+cpu, llama3.1-8b: 2.8.0.dev20250627+cpu","Ubuntu 22.04.5","SYS-822GA-NGR3. N/A","closed","0","v6.0","0","False","Tokens/s","INT8","","" +"Supermicro","available","closed","datacenter","1-node-2S-GNR_128C","1-node-2S-GNR_128C","rgat","rgat","Offline","25192.7","acc: 72.557","1","Intel(R) Xeon(R) 6980P","2","128","N/A","0","0","closed/Supermicro/results/1-node-2S-GNR_128C/rgat/Offline/performance/run_1","PyTorch: rgat: 2.7.0+cpu, whipser: 2.7.0+cpu, llama3.1-8b: 2.8.0.dev20250627+cpu","Ubuntu 22.04.5","SYS-822GA-NGR3. N/A","closed","0","v6.0","0","False","Samples/s","INT8","","" +"Supermicro","available","closed","datacenter","1-node-2S-GNR_128C","1-node-2S-GNR_128C","whisper","whisper","Offline","2204.09","ACCURACY: 97.85666881011248","1","Intel(R) Xeon(R) 6980P","2","128","N/A","0","0","closed/Supermicro/results/1-node-2S-GNR_128C/whisper/Offline/performance/run_1","PyTorch: rgat: 2.7.0+cpu, whipser: 2.7.0+cpu, llama3.1-8b: 2.8.0.dev20250627+cpu","Ubuntu 22.04.5","SYS-822GA-NGR3. N/A","closed","0","v6.0","0","False","Samples/s","INT8","","" +"Supermicro","available","closed","datacenter","QuantaGrid D74A-7U","8xMI355X_2xEPYC-9575F","llama2-70b-99","llama2-70b-99","Interactive","71295.5","ROUGE1: 44.5421 ROUGE2: 22.1361 ROUGEL: 28.9128 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9575F","2","64","AMD Instinct MI355X 288GB HBM3e","8","8","closed/Supermicro/results/8xMI355X_2xEPYC-9575F/llama2-70b-99/Interactive/performance/run_1","TODO","Ubuntu 22.04.5 LTS","","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"Supermicro","available","closed","datacenter","QuantaGrid D74A-7U","8xMI355X_2xEPYC-9575F","llama2-70b-99","llama2-70b-99","Offline","100754.0","ROUGE1: 44.4357 ROUGE2: 22.0707 ROUGEL: 28.8441 TOKENS_PER_SAMPLE: 284.6","1","AMD EPYC 9575F","2","64","AMD Instinct MI355X 288GB HBM3e","8","8","closed/Supermicro/results/8xMI355X_2xEPYC-9575F/llama2-70b-99/Offline/performance/run_1","TODO","Ubuntu 22.04.5 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Supermicro","available","closed","datacenter","QuantaGrid D74A-7U","8xMI355X_2xEPYC-9575F","llama2-70b-99","llama2-70b-99","Server","97587.0","ROUGE1: 44.5381 ROUGE2: 22.1334 ROUGEL: 28.9085 TOKENS_PER_SAMPLE: 283.0","1","AMD EPYC 9575F","2","64","AMD Instinct MI355X 288GB HBM3e","8","8","closed/Supermicro/results/8xMI355X_2xEPYC-9575F/llama2-70b-99/Server/performance/run_1","TODO","Ubuntu 22.04.5 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Supermicro","available","closed","datacenter","QuantaGrid D74A-7U","8xMI355X_2xEPYC-9575F","llama2-70b-99.9","llama2-70b-99.9","Interactive","71295.5","ROUGE1: 44.5421 ROUGE2: 22.1361 ROUGEL: 28.9128 TOKENS_PER_SAMPLE: 282.9","1","AMD EPYC 9575F","2","64","AMD Instinct MI355X 288GB HBM3e","8","8","closed/Supermicro/results/8xMI355X_2xEPYC-9575F/llama2-70b-99.9/Interactive/performance/run_1","TODO","Ubuntu 22.04.5 LTS","","closed","0","v6.0","1","False","Tokens/s","fp4","","" +"Supermicro","available","closed","datacenter","QuantaGrid D74A-7U","8xMI355X_2xEPYC-9575F","llama2-70b-99.9","llama2-70b-99.9","Offline","100754.0","ROUGE1: 44.4357 ROUGE2: 22.0707 ROUGEL: 28.8441 TOKENS_PER_SAMPLE: 284.6","1","AMD EPYC 9575F","2","64","AMD Instinct MI355X 288GB HBM3e","8","8","closed/Supermicro/results/8xMI355X_2xEPYC-9575F/llama2-70b-99.9/Offline/performance/run_1","TODO","Ubuntu 22.04.5 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Supermicro","available","closed","datacenter","QuantaGrid D74A-7U","8xMI355X_2xEPYC-9575F","llama2-70b-99.9","llama2-70b-99.9","Server","97587.0","ROUGE1: 44.5381 ROUGE2: 22.1334 ROUGEL: 28.9085 TOKENS_PER_SAMPLE: 283.0","1","AMD EPYC 9575F","2","64","AMD Instinct MI355X 288GB HBM3e","8","8","closed/Supermicro/results/8xMI355X_2xEPYC-9575F/llama2-70b-99.9/Server/performance/run_1","TODO","Ubuntu 22.04.5 LTS","","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Supermicro","available","closed","datacenter","SYS-822GS-NBRT (8x B200-SXM-180GB, TensorRT)","SYS-822GS-NBRT_B200-SXM-180GB_TRT","llama2-70b-99","llama2-70b-99","Offline","101595.0","ROUGE1: 44.7467 ROUGE2: 22.3565 ROUGEL: 29.1362 TOKENS_PER_SAMPLE: 273.4","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B200-SXM-180GB","8","8","closed/Supermicro/results/SYS-822GS-NBRT_B200-SXM-180GB_TRT/llama2-70b-99/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B200 TGP 1000W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Supermicro","available","closed","datacenter","SYS-822GS-NBRT (8x B200-SXM-180GB, TensorRT)","SYS-822GS-NBRT_B200-SXM-180GB_TRT","llama2-70b-99","llama2-70b-99","Server","99301.5","ROUGE1: 44.744 ROUGE2: 22.3567 ROUGEL: 29.1286 TOKENS_PER_SAMPLE: 274.4","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B200-SXM-180GB","8","8","closed/Supermicro/results/SYS-822GS-NBRT_B200-SXM-180GB_TRT/llama2-70b-99/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B200 TGP 1000W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Supermicro","available","closed","datacenter","SYS-822GS-NBRT (8x B200-SXM-180GB, TensorRT)","SYS-822GS-NBRT_B200-SXM-180GB_TRT","llama2-70b-99.9","llama2-70b-99.9","Offline","101526.0","ROUGE1: 44.7465 ROUGE2: 22.356 ROUGEL: 29.1359 TOKENS_PER_SAMPLE: 273.3","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B200-SXM-180GB","8","8","closed/Supermicro/results/SYS-822GS-NBRT_B200-SXM-180GB_TRT/llama2-70b-99.9/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B200 TGP 1000W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Supermicro","available","closed","datacenter","SYS-822GS-NBRT (8x B200-SXM-180GB, TensorRT)","SYS-822GS-NBRT_B200-SXM-180GB_TRT","llama2-70b-99.9","llama2-70b-99.9","Server","99304.8","ROUGE1: 44.7501 ROUGE2: 22.3589 ROUGEL: 29.1383 TOKENS_PER_SAMPLE: 274.3","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B200-SXM-180GB","8","8","closed/Supermicro/results/SYS-822GS-NBRT_B200-SXM-180GB_TRT/llama2-70b-99.9/Server/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B200 TGP 1000W","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Supermicro","available","closed","datacenter","SYS-822GS-NBRT (8x B200-SXM-180GB, TensorRT)","SYS-822GS-NBRT_B200-SXM-180GB_TRT","whisper","whisper","Offline","42389.7","ACCURACY: 97.86918164774403","1","Intel(R) Xeon(R) 6767P","2","64","NVIDIA B200-SXM-180GB","8","8","closed/Supermicro/results/SYS-822GS-NBRT_B200-SXM-180GB_TRT/whisper/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B200 TGP 1000W","closed","0","v6.0","0","False","Samples/s","fp16","","" +"Supermicro","available","closed","datacenter","Supermicro AS-8126GS-NB3RT(8x B300-SXM-288GB, TensorRT)","AS-8126GS-NB3RT_B300-SXM-288GB_TRT","whisper","whisper","Offline","50562.2","ACCURACY: 97.88155352741323","1","AMD EPYC 9575F","2","56","NVIDIA B300-SXM-270GB","8","8","closed/Supermicro/results/AS-8126GS-NB3RT_B300-SXM-288GB_TRT/whisper/Offline/performance/run_1","TensorRT 10.14, CUDA 13.1","Ubuntu 24.04","B300 TGP 1100W","closed","0","v6.0","0","False","Samples/s","fp16","","" +"Supermicro_RedHat","available","closed","datacenter","AS-8126GS-TNMR,vLLM","8XMI350x_RHEL9","llama2-70b-99","llama2-70b-99","Offline","80478.1","ROUGE1: 44.4988 ROUGE2: 22.1091 ROUGEL: 28.8905 TOKENS_PER_SAMPLE: 282.7","1","AMD EPYC 9575F","2","64","AMD Instinct MI350X 288GB HBM3e","8","8","closed/Supermicro_RedHat/results/8XMI350x_RHEL9/llama2-70b-99/Offline/performance/run_1","vLLM 0.14.0, RHEL 9.6","Red Hat Enterprise Linux 9.6","Supermicro MI350x. RHEL 9.6, vllm 0.14.0","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Supermicro_RedHat","available","closed","datacenter","AS-8126GS-TNMR,vLLM","8XMI350x_RHEL9","llama2-70b-99","llama2-70b-99","Server","76393.1","ROUGE1: 44.6049 ROUGE2: 22.178 ROUGEL: 28.9591 TOKENS_PER_SAMPLE: 281.8","1","AMD EPYC 9575F","2","64","AMD Instinct MI350X 288GB HBM3e","8","8","closed/Supermicro_RedHat/results/8XMI350x_RHEL9/llama2-70b-99/Server/performance/run_1","vLLM 0.14.0, RHEL 9.6","Red Hat Enterprise Linux 9.6","Supermicro MI350x. RHEL 9.6, vllm 0.14.0","closed","0","v6.0","0","False","Tokens/s","fp4","","" +"Supermicro_RedHat","available","open","datacenter","AS-8126GS-TNMR,vLLM","8xMI350X_2xEPYC-9575F","gpt-oss-120b","gpt-oss-120b","Offline","64293.3","exact_match: 83.568","1","AMD EPYC 9575F","2","1","AMD Instinct MI350X","8","8","open/Supermicro_RedHat/results/8xMI350X_2xEPYC-9575F/gpt-oss-120b/Offline/performance/run_1","N/A","N/A","SMC MI350X AS -8126GS-TNMR. vLLM 0.14.0, RHEL 9.6","open","0","v6.0","0","False","Tokens/s","fp4","","" +"Supermicro_RedHat","available","open","datacenter","AS-8126GS-TNMR,vLLM","8xMI350X_2xEPYC-9575F","gpt-oss-120b","gpt-oss-120b","Server","58373.3","exact_match: 83.001","1","AMD EPYC 9575F","2","1","AMD Instinct MI350X","8","8","open/Supermicro_RedHat/results/8xMI350X_2xEPYC-9575F/gpt-oss-120b/Server/performance/run_1","N/A","N/A","SMC MI350X AS -8126GS-TNMR. vLLM 0.14.0, RHEL 9.6","open","0","v6.0","0","False","Tokens/s","fp4","","" diff --git a/tools/submission/tests/conftest.py b/tools/submission/tests/conftest.py new file mode 100644 index 0000000000..86c3dd71a3 --- /dev/null +++ b/tools/submission/tests/conftest.py @@ -0,0 +1,15 @@ +"""Pytest configuration for the submission-checker tests. + +The snapshot test (``test_snapshot.py``) needs a cloned submission repo, +supplied via ``MLPERF_SUBMISSION_DIR``. When that variable is not set we skip +*collecting* the module entirely rather than skipping the test at runtime: +a runtime skip would leave syrupy's snapshot marked "unused", which makes +pytest exit non-zero. Ignoring collection keeps a plain ``pytest`` run of this +directory green when the repo isn't available. +""" + +import os + +collect_ignore = [] +if not os.environ.get("MLPERF_SUBMISSION_DIR"): + collect_ignore.append("test_snapshot.py") diff --git a/tools/submission/tests/test_snapshot.py b/tools/submission/tests/test_snapshot.py new file mode 100644 index 0000000000..167838d632 --- /dev/null +++ b/tools/submission/tests/test_snapshot.py @@ -0,0 +1,158 @@ +"""Snapshot (golden-file) test for the MLPerf submission checker. + +This runs the submission checker end-to-end against a *pinned* commit of a +real submission repository and compares the produced ``summary.csv`` against a +committed snapshot using `syrupy `_. +It catches unintended changes to the checker's output (columns, computed +metrics, inferred scenarios, units, ...) without anyone hand-writing expected +values. + +How it works +------------ +* The submission repo is cloned in CI (see + ``.github/workflows/test-submission-checker-snapshot.yml``) and its path is + passed in via the ``MLPERF_SUBMISSION_DIR`` environment variable. +* We run against *all* submitters in the pinned repo. Because the commit is + fixed, the output is fully deterministic, so the snapshot only changes when + the checker's behavior changes. +* Volatile fields are normalized before comparison: the ``Location`` column + holds an absolute path, so we rewrite it to be relative to the submission + root, and rows are sorted so directory-iteration order never matters. +* The snapshot is stored as a plain, reviewable ``.csv`` (via syrupy's + single-file extension) under ``tests/__snapshots__/test_snapshot/``. + +Updating the snapshot +--------------------- +When a checker change *intentionally* alters the output, regenerate with +syrupy's standard flag:: + + export MLPERF_SUBMISSION_DIR=/path/to/cloned/submissions/repo + pytest tools/submission/tests/test_snapshot.py --snapshot-update + +then review and commit the updated file under ``tests/__snapshots__/``. + +If ``MLPERF_SUBMISSION_DIR`` is not set (e.g. a plain local ``pytest`` run +without the cloned repo), the test is skipped rather than failed. +""" + +import csv +import io +import os +import subprocess +import sys +from pathlib import Path + +import pytest +from syrupy.extensions.single_file import SingleFileSnapshotExtension, WriteMode + +# Directory layout: this file lives at tools/submission/tests/test_snapshot.py +TESTS_DIR = Path(__file__).resolve().parent +CHECKER_ROOT = TESTS_DIR.parent # tools/submission (parent of the package) + +# --- Pinned inputs. Keep these in sync with the CI workflow. ----------------- +SUBMISSION_REPO = "https://github.com/mlcommons/inference_results_v6.0" +PINNED_SHA = "4d3916ac9cf474b679cdfcf492d43a0559418ad1" +VERSION = "v6.0" +SKIP_FLAGS = [ + "--skip-power-check", + "--skip-extra-files-in-root-check", + "--skip-extra-accuracy-files-check", + "--skip-all-systems-have-results-check", + "--skip-calibration-check", +] + +LOCATION_COLUMN = "Location" + + +class CSVSnapshotExtension(SingleFileSnapshotExtension): + """Store each snapshot as a plain, diff-friendly ``.csv`` text file.""" + + _write_mode = WriteMode.TEXT + file_extension = "csv" + + +@pytest.fixture +def csv_snapshot(snapshot): + return snapshot.use_extension(CSVSnapshotExtension) + + +def _submission_dir() -> Path: + raw = os.environ.get("MLPERF_SUBMISSION_DIR") + if not raw: + pytest.skip( + "MLPERF_SUBMISSION_DIR is not set; clone " + f"{SUBMISSION_REPO} at {PINNED_SHA} and point this variable at it " + "to run the snapshot test." + ) + path = Path(raw).resolve() + if not path.is_dir(): + pytest.skip(f"MLPERF_SUBMISSION_DIR={path} does not exist.") + return path + + +def _run_checker(submission_dir: Path, out_csv: Path) -> None: + cmd = [ + sys.executable, + "-m", + "submission_checker.main", + "--input", + str(submission_dir), + "--version", + VERSION, + "--csv", + str(out_csv), + *SKIP_FLAGS, + ] + result = subprocess.run( + cmd, + cwd=CHECKER_ROOT, + capture_output=True, + text=True, + ) + assert result.returncode == 0, ( + "submission checker exited non-zero\n" + f"STDOUT:\n{result.stdout[-4000:]}\n" + f"STDERR:\n{result.stderr[-4000:]}" + ) + + +def _normalize(csv_text: str, submission_dir: Path) -> str: + """Make the CSV portable and order-independent. + + * Rewrite the absolute ``Location`` path to be relative to the submission + root (so the snapshot does not embed a machine-specific prefix). + * Sort the data rows so directory-iteration order does not affect the + comparison. + """ + prefix = str(submission_dir).rstrip("/") + "/" + reader = csv.DictReader(io.StringIO(csv_text)) + fieldnames = reader.fieldnames + assert fieldnames, "checker produced a CSV with no header" + + rows = [] + for row in reader: + loc = row.get(LOCATION_COLUMN, "") + if loc.startswith(prefix): + row[LOCATION_COLUMN] = loc[len(prefix):] + rows.append(row) + + rows.sort(key=lambda r: [r.get(c, "") for c in fieldnames]) + + buf = io.StringIO() + # match the checker's own quoting: every field wrapped in double quotes + writer = csv.DictWriter( + buf, fieldnames=fieldnames, quoting=csv.QUOTE_ALL, lineterminator="\n" + ) + writer.writeheader() + writer.writerows(rows) + return buf.getvalue() + + +def test_summary_csv_matches_snapshot(tmp_path, csv_snapshot): + submission_dir = _submission_dir() + out_csv = tmp_path / "summary.csv" + + _run_checker(submission_dir, out_csv) + actual = _normalize(out_csv.read_text(), submission_dir) + + assert actual == csv_snapshot