diff --git a/tools/submission/submission_checker/checks/accuracy_check.py b/tools/submission/submission_checker/checks/accuracy_check.py index ac94679c74..4d8fb00d4e 100644 --- a/tools/submission/submission_checker/checks/accuracy_check.py +++ b/tools/submission/submission_checker/checks/accuracy_check.py @@ -242,7 +242,7 @@ def dataset_check(self): ) return True expected_qsl_total_count = self.config.get_accuracy_sample_count( - self.model) + self.model, self.scenario_fixed) if "effective_accuracy_sample_count" in self.mlperf_log.get_keys(): qsl_total_count = self.mlperf_log["effective_accuracy_sample_count"] else: diff --git a/tools/submission/submission_checker/checks/compliance_check.py b/tools/submission/submission_checker/checks/compliance_check.py index e110af9c40..f18d98f000 100644 --- a/tools/submission/submission_checker/checks/compliance_check.py +++ b/tools/submission/submission_checker/checks/compliance_check.py @@ -8,6 +8,7 @@ from ..utils import * import re import os +import json class ComplianceCheck(BaseCheck): @@ -41,6 +42,7 @@ def __init__(self, log, path, config: Config, self.submission_logs = submission_logs self.config = config self.name = "compliance check" + self.perf_log = self.submission_logs.performance_log self.model = self.submission_logs.loader_data.get("benchmark", "") self.model_mapping = self.submission_logs.loader_data.get( "model_mapping", {}) @@ -60,14 +62,15 @@ def setup_checks(self): Appends the per-submission validation callables to `self.checks` in the order they should be executed by the checking framework. """ - self.checks.append(self.dir_exists_check) - self.checks.append(self.performance_check) - self.checks.append(self.accuracy_check) - self.checks.append(self.compliance_performance_check) - self.apply_checks = set(self.checks) - # No compliance tests for endpoints for now - if self.is_endpoints: - self.apply_checks = set() + if not self.is_endpoints: + self.checks.append(self.dir_exists_check) + self.checks.append(self.performance_check) + self.checks.append(self.accuracy_check) + self.checks.append(self.compliance_performance_check) + self.apply_checks = set(self.checks) + else: + self.checks.append(self.compliance_endpoints_check) + self.apply_checks = set(self.checks) def get_test_list(self, model): """Return the list of compliance tests applicable to `model`. @@ -494,3 +497,71 @@ def compliance_performance_check(self): diff) is_valid = False return is_valid + + def compliance_endpoints_check(self): + test_dir = os.path.join(self.compliance_dir, "audit") + is_valid = True + for test in self.test_list: + if test in ENDPOINTS_COMPLIANCE_MAPPING: + test_name = ENDPOINTS_COMPLIANCE_MAPPING.get(test, test) + fname = os.path.join(test_dir, f"audit_{test_name}.json") + if not os.path.exists(fname): + if self.endpoint_audit_file_required(test): + # if audit file is required, but missing, error out + self.log.error("%s is missing in %s", fname, test_dir) + is_valid = False + else: + # if audit file is not required, skip + continue + else: + try: + with open(fname) as f: + test_file = json.load(f) + + if test_file["test"] != test_name: + self.log.error( + "%s compliance test is incorrect %s. " + + "Expected name=%s" + + "Actual name=%s", + fname, + test_dir, + test_name, + test_file["test"] + ) + is_valid = False + elif not test_file["passed"]: + self.log.error( + "%s compliance test can not be loaded in %s", fname, test_dir) + is_valid = False + else: + self.log.info( + "%s compliance test %s passed for %s", fname, test_name, test_dir) + + except BaseException: + self.log.error( + "%s compliance test can not be loaded in %s", fname, test_dir) + is_valid = False + + if test in ["TEST09"]: + mean_output_tokens = self.perf_log["mean_output_tokens"] + if mean_output_tokens > TEST09_HIGH or mean_output_tokens < TEST09_LOW: + self.log.error("TEST 09 (Verify Output Token Length in Performance Mode)" + + " failed for %s. Mean output tokens need to be in the range" + + " [%s, %s]. Found %s", + self.compliance_dir, + TEST09_LOW, + TEST09_HIGH, + mean_output_tokens + ) + is_valid = False + + return is_valid + + def endpoint_audit_file_required(self, test): + # currently, only test04 is enabled in endpoints. and Wan is the only + # model that requires test04 + return ( + test == "TEST04" + and self.model in ENDPOINTS_ALLOWED_MODELS + and self.model in self.config.base.get("models_TEST04", []) + ) diff --git a/tools/submission/submission_checker/checks/performance_check.py b/tools/submission/submission_checker/checks/performance_check.py index ed8ad18bec..916c5fee27 100644 --- a/tools/submission/submission_checker/checks/performance_check.py +++ b/tools/submission/submission_checker/checks/performance_check.py @@ -56,10 +56,18 @@ def __init__(self, log, path, config: Config, self.is_endpoints = self.submission_logs.loader_data.get( "is_endpoints_submission", False) if self.is_endpoints: - if self.scenario.lower() == "online": - self.scenario = "Server" + self.scenario = self.map_endpoints_scenario( + self.scenario, self.scenario_fixed + ) self.setup_checks() + def map_endpoints_scenario(self, scenario, scenario_fixed): + if scenario_fixed.lower() == "singlestream": + return "SingleStream" + if scenario.lower() == "online": + return "Server" + return scenario + def setup_checks(self): """Register individual performance-related checks. @@ -251,6 +259,7 @@ def latency_check(self): bool: True if latency constraints are satisfied, False otherwise. """ uses_early_stopping = self.config.uses_early_stopping(self.scenario) + # traditional loadgen log path: if uses_early_stopping and not self.is_endpoints: # check if early_stopping condition was met if not self.mlperf_log["early_stopping_met"]: @@ -282,7 +291,14 @@ def latency_check(self): ) return False else: - # check if the benchmark meets latency constraint + if self.is_endpoints and self.model in self.config.get_llm_models(): + # Endpoint LLM latency is enforced by llm_check using + # TTFT/TPOT. + return True + + # check if the benchmark meets latency constraint, works for both Endpoint and non-Endpoint based logs + # Qwen3VL falls in this category, it has scenario specific e2e + # latency constraints latency_99_percentile = self.mlperf_log["result_99.00_percentile_latency_ns"] target_latency = self.config.latency_constraint.get( self.model, dict()).get(self.scenario) @@ -418,54 +434,63 @@ def llm_check(self): bool: True if LLM checks pass or model is not an LLM, False otherwise. """ - if self.model in self.config.get_llm_models(): - if self.is_endpoints: - # Endpoints don't use the loadgen use_token_latencies flag; - # check TTFT/TPOT directly from the endpoints result JSON. - if self.scenario not in ["Server", "Interactive"]: - return True - limits = LLM_LATENCY_LIMITS[self.model][self.scenario] - ttft = self.mlperf_log["result_first_token_99.00_percentile_latency_ns"] - tpot = self.mlperf_log["result_time_per_output_token_99.00_percentile_ns"] - if ttft is None or tpot is None: - self.log.warning( - "%s TTFT or TPOT percentile data missing for endpoints LLM check", - self.path) - return True - if ttft < limits["ttft"] and tpot < limits["tpot"]: - return True - self.log.error( - 'Failed extra check for TTFT and TPOT. Obtained: TTFT 99-tile: %.4f, TPOT 99-tile: %.4f. Required: TTFT 99-tile: %.4f, TPOT 99-tile: %.4f', - ttft, tpot, limits["ttft"], limits["tpot"]) - return False - if self.mlperf_log["requested_use_token_latencies"]: - if self.scenario not in ["Server", "Interactive"]: - # For offline, singlestream and multistream no further checks are - # necessary - return True - else: - limits = LLM_LATENCY_LIMITS[self.model][self.scenario] - if ( - self.mlperf_log["result_first_token_99.00_percentile_latency_ns"] - < limits["ttft"] - and self.mlperf_log["result_time_per_output_token_99.00_percentile_ns"] - < limits["tpot"] - ): - return True - else: - self.log.error( - f"use_token_latencies flag needs to be enabled for Llama2 benchmark") - return False + if self.model not in self.config.get_llm_models(): + return True + + # Endpoint logs do not carry LoadGen's requested_use_token_latencies + # setting, but the endpoint parser maps token latency fields to the + # same LoadGen-style result keys used below. + requested_use_token_latencies = ( + self.is_endpoints + or self.mlperf_log["requested_use_token_latencies"] + ) + if not requested_use_token_latencies: + self.log.error( + "use_token_latencies flag needs to be enabled for LLM benchmark") + return False + + if self.scenario not in ["Server", "Interactive"]: + # For offline, singlestream and multistream no further checks are + # necessary. + return True + limits = LLM_LATENCY_LIMITS.get(self.model, {}).get(self.scenario) + if limits is None: self.log.error( - 'Failed extra check for TTFT and TPOT. Obtained: TTFT 99-tile: %.4f, TPOT 99-tile: %.4f. Required: TTFT 99-tile: %.4f, TPOT 99-tile: %.4f', - self.mlperf_log["result_first_token_99.00_percentile_latency_ns"], - self.mlperf_log["result_time_per_output_token_99.00_percentile_ns"], - limits["ttft"], - limits["tpot"] + "%s TTFT/TPOT latency limits missing for model=%s scenario=%s", + self.path, + self.model, + self.scenario, ) return False - return True + + ttft = self.mlperf_log["result_first_token_99.00_percentile_latency_ns"] + tpot = self.mlperf_log["result_time_per_output_token_99.00_percentile_ns"] + if ttft is None or tpot is None: + self.log.error( + "%s TTFT or TPOT percentile data missing for LLM check", + self.path) + return False + + self.log.info( + "TTFT target: %s, TTFT: %s, TPOT target: %s, TPOT: %s, Scenario: %s", + limits["ttft"], + ttft, + limits["tpot"], + tpot, + self.scenario, + ) + if ttft < limits["ttft"] and tpot < limits["tpot"]: + return True + + self.log.error( + 'Failed extra check for TTFT and TPOT. Obtained: TTFT 99-tile: %.4f, TPOT 99-tile: %.4f. Required: TTFT 99-tile: %.4f, TPOT 99-tile: %.4f', + ttft, + tpot, + limits["ttft"], + limits["tpot"] + ) + return False def inferred_check(self): """Validate rules for inferring results across scenarios. @@ -519,13 +544,13 @@ def get_performance_metric_check(self): and self.mlperf_log["result_validity"] == "VALID" ): is_valid = True - scenario = self.mlperf_log["effective_scenario"] + scenario = self.scenario + res = None if self.is_endpoints: - if scenario.lower() == "online": - scenario = "Server" - scenario = scenario.capitalize() - - res = float(self.mlperf_log[RESULT_FIELD_NEW[version][scenario]]) + res = float( + self.mlperf_log[RESULT_FIELD_ENDPOINTS[version][scenario.lower()]]) + else: + res = float(self.mlperf_log[RESULT_FIELD_NEW[version][scenario]]) if ( not self.is_endpoints and version in RESULT_FIELD_BENCHMARK_OVERWRITE diff --git a/tools/submission/submission_checker/configuration/configuration.py b/tools/submission/submission_checker/configuration/configuration.py index 37abb6b802..31665c7499 100644 --- a/tools/submission/submission_checker/configuration/configuration.py +++ b/tools/submission/submission_checker/configuration/configuration.py @@ -158,11 +158,20 @@ def get_performance_sample_count(self, model): raise ValueError("model not known: " + model) return self.performance_sample_count[model] - def get_accuracy_sample_count(self, model): + def get_accuracy_sample_count(self, model, scenario=None): + # get expected accuracy sample count from config, qwen has scenario + # specific sample counts as special case model = self.get_mlperf_model(model) if model not in self.accuracy_sample_count: return self.get_dataset_size(model) - return self.accuracy_sample_count[model] + sample_count = self.accuracy_sample_count[model] + # handle Qwen's scenario specific sample counts + if isinstance(sample_count, dict): + if scenario in sample_count: + return sample_count[scenario] + if scenario is not None and scenario.lower() in sample_count: + return sample_count[scenario.lower()] + return sample_count def ignore_errors(self, line): for error in self.base["ignore_errors"]: @@ -231,5 +240,6 @@ def get_llm_models(self): "llama3.1-405b", "llama3.1-8b", "llama3.1-8b-edge", - "deepseek-r1" + "deepseek-r1", + "gpt-oss-120b" ] diff --git a/tools/submission/submission_checker/constants.py b/tools/submission/submission_checker/constants.py index 8278fe77a8..4b3de4a55b 100644 --- a/tools/submission/submission_checker/constants.py +++ b/tools/submission/submission_checker/constants.py @@ -17,6 +17,7 @@ "gpt-oss-120b", "wan-2.2-t2v-a14b", "qwen3-vl-235b-a22b", + "qwen3.6-27b", "dlrm-v3", "yolo-95", "yolo-99", @@ -58,6 +59,7 @@ "whisper": ["Offline"], "yolo-95": ["SingleStream", "MultiStream", "Offline"], "yolo-99": ["SingleStream", "MultiStream", "Offline"], + "qwen3.6-27b": ["SingleStream"], }, "optional-scenarios-edge": {}, "required-scenarios-datacenter-edge": { @@ -79,6 +81,7 @@ "dlrm-v3": ["Offline", "Server"], "yolo-95": ["SingleStream", "MultiStream", "Offline"], "yolo-99": ["SingleStream", "MultiStream", "Offline"], + "qwen3.6-27b": ["SingleStream"], }, "optional-scenarios-datacenter-edge": { "llama2-70b-99": ["Interactive", "Server"], @@ -164,6 +167,7 @@ # established "e2e": ("E2E_ACCURACY", ""), "e2e_vectorDB": ("E2E_ACCURACY", ""), + "qwen3.6-27b": ("mAP", 86.23 * 0.99), }, "accuracy-upper-limit": { "stable-diffusion-xl": ( @@ -203,11 +207,18 @@ "yolo-95": 64, "yolo-99": 64, "e2e": 824, - "e2e_vectorDB": 824 + "e2e_vectorDB": 824, + "qwen3.6-27b": 995, }, "accuracy-sample-count": { "gpt-oss-120b": 4395, "wan-2.2-t2v-a14b": 248, + "qwen3.6-27b": 995, + "qwen3-vl-235b-a22b": { + "Offline": 48289, + "Interactive": 8000, + "Server": 48289, + }, }, "dataset-size": { "resnet": 50000, @@ -231,6 +242,7 @@ "yolo-99": 1525, "e2e": 824, "e2e_vectorDB": 824, + "qwen3.6-27b": 995, }, "model_mapping": { "ssd-resnet34": "retinanet", @@ -291,6 +303,7 @@ "yolo-99": {"SingleStream": 1024, "MultiStream": 270336, "Offline": 1}, "e2e": {"Offline": 824}, "e2e_vectorDB": {"Offline": 824}, + "qwen3.6-27b": {"SingleStream": 995}, }, "models_TEST01": [ "resnet", @@ -1501,6 +1514,9 @@ "compliance_accuracy.txt", ] +TEST09_LOW = 1150.38 +TEST09_HIGH = 1406.02 + OFFLINE_MIN_SPQ_SINCE_V4 = { "resnet": 24576, "retinanet": 24576, @@ -1577,6 +1593,16 @@ }, } +RESULT_FIELD_ENDPOINTS = { + "v6.1": { + "offline": "result_samples_per_second", + "singlestream": "result_mean_latency_ns", + "multistream": "result_mean_latency_ns", + "server": "result_completed_samples_per_sec", + "interactive": "result_completed_samples_per_sec", + }, +} + RESULT_FIELD_BENCHMARK_OVERWRITE = { "v5.0": { "llama2-70b-99": { @@ -2030,17 +2056,19 @@ ENDPOINTS_ALLOWED_MODELS = [ "wan-2.2-t2v-a14b", "qwen3-vl-235b-a22b", + "qwen3.6-27b", "llama3.1-8b", "llama3.1-8b-edge", "gpt-oss-120b", "deepseek-r1" ] -PERFORMANCE_ENDPOINTS_DIR = { - "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", - "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", - "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", - "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/run_1/", +ENDPOINTS_SCENARIO_DIR = { + "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/", + "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/", + "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/", + "v6.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/", + "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/", } ACCURACY_LOG_PATH = { @@ -2067,14 +2095,6 @@ "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/mlperf_log_accuracy.json", } -ACCURACY_ENDPOINTS_DIR = { - "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", - "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", - "v6.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", - "default": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/accuracy/", -} - - POWER_DIR_PATH = { "v5.0": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/power", "v5.1": "{division}/{submitter}/results/{system}/{benchmark}/{scenario}/performance/power", @@ -2207,7 +2227,7 @@ "effective_sample_concatenate_permutation": "effective_sample_concatenate_permutation", "effective_samples_per_query": "effective_samples_per_query", "generated_query_count": "generated_query_count", - "generated_query_duration": "generated_query_duration", + "duration_ns": "generated_query_duration", "target_qps": "effective_target_qps", "result_scheduled_samples_per_sec": "result_scheduled_samples_per_sec", "qps": "result_completed_samples_per_sec", @@ -2216,30 +2236,31 @@ "latency.min": "result_min_latency_ns", "latency.max": "result_max_latency_ns", "latency.avg": "result_mean_latency_ns", - "latency.percentiles.50.0": "result_50.00_percentile_latency_ns", - "latency.percentiles.90.0": "result_90.00_percentile_latency_ns", - "latency.percentiles.95.0": "result_95.00_percentile_latency_ns", - "latency.percentiles.99.0": "result_99.00_percentile_latency_ns", - "latency.percentiles.99.9": "result_99.90_percentile_latency_ns", + "latency.early_stopping_percentiles.50.0": "result_50.00_percentile_latency_ns", + "latency.early_stopping_percentiles.90.0": "result_90.00_percentile_latency_ns", + "latency.early_stopping_percentiles.95.0": "result_95.00_percentile_latency_ns", + "latency.early_stopping_percentiles.99.0": "result_99.00_percentile_latency_ns", + "latency.early_stopping_percentiles.99.9": "result_99.90_percentile_latency_ns", "ttft.min": "result_first_token_min_latency_ns", "ttft.max": "result_first_token_max_latency_ns", "ttft.avg": "result_first_token_mean_latency_ns", - "ttft.percentiles.50.0": "result_first_token_50.00_percentile_latency_ns", - "ttft.percentiles.90.0": "result_first_token_90.00_percentile_latency_ns", - "ttft.percentiles.95.0": "result_first_token_95.00_percentile_latency_ns", - "ttft.percentiles.99.0": "result_first_token_99.00_percentile_latency_ns", - "ttft.percentiles.99.9": "result_first_token_99.90_percentile_latency_ns", - "tpot.percentiles.50.0": "result_time_per_output_token_50.00_percentile_ns", - "tpot.percentiles.90.0": "result_time_per_output_token_90.00_percentile_ns", - "tpot.percentiles.95.0": "result_time_per_output_token_95.00_percentile_ns", - "tpot.percentiles.99.0": "result_time_per_output_token_99.00_percentile_ns", - "tpot.percentiles.99.9": "result_time_per_output_token_99.90_percentile_ns", + "ttft.early_stopping_percentiles.50.0": "result_first_token_50.00_percentile_latency_ns", + "ttft.early_stopping_percentiles.90.0": "result_first_token_90.00_percentile_latency_ns", + "ttft.early_stopping_percentiles.95.0": "result_first_token_95.00_percentile_latency_ns", + "ttft.early_stopping_percentiles.99.0": "result_first_token_99.00_percentile_latency_ns", + "ttft.early_stopping_percentiles.99.9": "result_first_token_99.90_percentile_latency_ns", + "tpot.early_stopping_percentiles.50.0": "result_time_per_output_token_50.00_percentile_ns", + "tpot.early_stopping_percentiles.90.0": "result_time_per_output_token_90.00_percentile_ns", + "tpot.early_stopping_percentiles.95.0": "result_time_per_output_token_95.00_percentile_ns", + "tpot.early_stopping_percentiles.99.0": "result_time_per_output_token_99.00_percentile_ns", + "tpot.early_stopping_percentiles.99.9": "result_time_per_output_token_99.90_percentile_ns", "tpot.min": "result_time_per_output_token_min", "tpot.max": "result_time_per_output_token_max", "tpot.avg": "result_time_per_output_token_mean", "tps": "result_completed_tokens_per_second", "result.total": "result_query_count", "result.failed": "num_errors", + "output_sequence_lengths.avg": "mean_output_tokens", } @@ -2261,9 +2282,9 @@ # Alternative JSON paths for endpoints keys that don't directly match the # JSON structure ENDPOINTS_JSON_ALT_PATHS = { - "result.total": "results.total", - "result.failed": "results.failed", - "qps": "results.qps", + "result.total": "n_samples_completed", + "result.failed": "n_samples_failed", + "results.qps": "qps", "generated_query_count": "n_samples_issued", "generated_query_duration": "duration_ns", "test_datetime": "test_started_at", @@ -2272,5 +2293,13 @@ } ENDPOINTS_INFERRED_FIELDS = { - "effective_accuracy_sample_count": "result_query_count" + "generated_query_count": "qsl_reported_total_count", +} + +ENDPOINTS_COMPLIANCE_MAPPING = { + "TEST01": "accuracy_in_performance_test", + "TEST04": "output_caching_test", + "TEST06": "consistency_output_test", + "TEST07": "accuracy_in_performance_full_test", + "TEST08": "accuracy_in_performance_dlrmv3_test", } diff --git a/tools/submission/submission_checker/loader.py b/tools/submission/submission_checker/loader.py index 13c531bd93..c2f465587a 100644 --- a/tools/submission/submission_checker/loader.py +++ b/tools/submission/submission_checker/loader.py @@ -86,12 +86,9 @@ def __init__(self, root, version, config: Config) -> None: self.acc_json_path = os.path.join( self.root, ACCURACY_JSON_PATH.get( version, ACCURACY_JSON_PATH["default"])) - self.perf_endpoints_dir = os.path.join( - self.root, PERFORMANCE_ENDPOINTS_DIR.get( - version, PERFORMANCE_ENDPOINTS_DIR["default"])) - self.acc_endpoints_dir = os.path.join( - self.root, ACCURACY_ENDPOINTS_DIR.get( - version, ACCURACY_ENDPOINTS_DIR["default"])) + self.endpoints_scenario_dir = os.path.join( + self.root, ENDPOINTS_SCENARIO_DIR.get( + version, ENDPOINTS_SCENARIO_DIR["default"])) self.system_log_path = os.path.join( self.root, SYSTEM_PATH.get( version, SYSTEM_PATH["default"])) @@ -234,21 +231,16 @@ def load_single_log(self, path, log_type: Literal["Performance", "Accuracy", path) return log - def load_endpoints_logs(self, perf_dir, acc_dir): - perf_log = None - acc_log = None - if os.path.exists(acc_dir) and os.path.exists(perf_dir): - acc_log = EndpointsParser(acc_dir) - perf_log = EndpointsParser(perf_dir) - elif os.path.exists(perf_dir): - acc_log = EndpointsParser(perf_dir) - perf_log = EndpointsParser(perf_dir) - else: - self.logger.info( - "Could not load endpoints log from %s, path does not exist", - perf_dir - ) - return perf_log, acc_log + def load_endpoints_logs(self, scenario_dir): + if os.path.exists(scenario_dir): + perf_parser = EndpointsParser(scenario_dir, log_type="performance") + acc_parser = EndpointsParser(scenario_dir, log_type="accuracy") + return perf_parser, acc_parser + self.logger.info( + "Could not load endpoints log from %s, path does not exist", + scenario_dir + ) + return None, None def check_scenarios(self, benchmark, model_mapping, system_type, scenarios): @@ -362,13 +354,7 @@ def load(self) -> Generator[SubmissionLogs, None, None]: system=system, benchmark=benchmark, scenario=scenario) - perf_endpoints_dir = self.perf_endpoints_dir.format( - division=division, - submitter=submitter, - system=system, - benchmark=benchmark, - scenario=scenario) - acc_endpoints_dir = self.acc_endpoints_dir.format( + endpoints_scenario_dir = self.endpoints_scenario_dir.format( division=division, submitter=submitter, system=system, @@ -483,7 +469,7 @@ def load(self) -> Generator[SubmissionLogs, None, None]: if perf_log is None and acc_log is None: is_endpoints_submission = True perf_log, acc_log = self.load_endpoints_logs( - perf_endpoints_dir, acc_endpoints_dir + endpoints_scenario_dir ) # Load test logs diff --git a/tools/submission/submission_checker/parsers/endpoints_parser.py b/tools/submission/submission_checker/parsers/endpoints_parser.py index bb4b09f552..dbb1005ad8 100644 --- a/tools/submission/submission_checker/parsers/endpoints_parser.py +++ b/tools/submission/submission_checker/parsers/endpoints_parser.py @@ -23,8 +23,10 @@ "helper", "sample_logs") -_RESULT_SUMMARY_FILE = "results_summary.json" -_RESULTS_FILE = "results.json" +_RESULT_SUMMARY_FILE = "result_summary.json" +_ACCURACY_RESULTS_FILE = "accuracy_results.json" +_PERF_SUBDIR = "performance" +_ACC_SUBDIR = "accuracy" _CONFIG_FILES = ("config.yaml", "config.yml") @@ -93,23 +95,50 @@ def _resolve_value(stripped, summary_data, results_data, yaml_data): return _get_nested(yaml_data, stripped) +def _endpoint_accuracy_sample_count(results_data): + counts = [] + accuracy_scores = results_data.get("accuracy_scores", []) + + for result in accuracy_scores: + if not isinstance(result, dict): + continue + dataset_type = result.get("dataset_type") + if dataset_type is not None and dataset_type != "accuracy": + continue + if result.get("total_samples") is not None: + counts.append(int(result["total_samples"])) + elif result.get("response_counts", {}).get("issued") is not None: + counts.append(int(result["response_counts"]["issued"])) + + return sum(counts) if counts else None + + class EndpointsParser(BaseParser): - def __init__(self, run_dir): + def __init__(self, scenario_dir, log_type="performance"): """ - run_dir: path to the run directory containing: - - result_summary.json (highest priority) - - results.json - - config.yaml / config.yml (lowest priority) + scenario_dir: path to the scenario directory containing: + - config.yaml (scenario root) + - performance/result_summary.json (performance metrics) + - accuracy/accuracy_results.json (accuracy metrics) + log_type: which log to parse from scenario_dir, either + "performance" (reads performance/result_summary.json) or + "accuracy" (reads accuracy/accuracy_results.json). """ - super().__init__(run_dir) + super().__init__(scenario_dir) self.logger = logging.getLogger("MLPerfLog") self.messages = {} - - summary_data = self._load_json( - os.path.join(run_dir, _RESULT_SUMMARY_FILE)) - results_data = self._load_json(os.path.join(run_dir, _RESULTS_FILE)) - yaml_data = self._load_yaml(run_dir) + self.log_type = log_type + + if log_type == "accuracy": + summary_data = {} + results_data = self._load_json( + os.path.join(scenario_dir, _ACC_SUBDIR, _ACCURACY_RESULTS_FILE)) + else: + summary_data = self._load_json( + os.path.join(scenario_dir, _PERF_SUBDIR, _RESULT_SUMMARY_FILE)) + results_data = {} + yaml_data = self._load_yaml(scenario_dir) for endpoints_key, loadgen_key in ENDPOINTS_MAPPINGS.items(): stripped = endpoints_key.strip() @@ -120,6 +149,20 @@ def __init__(self, run_dir): {"key": loadgen_key, "value": value} ) + if log_type == "accuracy": + # load accuracy sample count from accuracy_results.json, 2 cases: + # gptoss has 3 sub dataset with 3 counts(aime, gpqa, lcb), sum them + # other which has 1 dataset with 1 sample count, use it + accuracy_count = _endpoint_accuracy_sample_count(results_data) + if accuracy_count is not None: + self.messages["effective_accuracy_sample_count"] = [ + {"key": "effective_accuracy_sample_count", + "value": accuracy_count} + ] + self.messages.setdefault("qsl_reported_total_count", [ + {"key": "qsl_reported_total_count", "value": accuracy_count} + ]) + self.keys = set(self.messages.keys()) # Inferred fields: copy the value of one loadgen key to another @@ -145,7 +188,7 @@ def __init__(self, run_dir): self.messages[qps_key] = [{"key": qps_key, "value": qps}] # Expose accuracy scores stored in results.json - for result in results_data.get("accuracy_scores", {}).values(): + for result in results_data.get("accuracy_scores", []): score = result.get("score") if score is not None: self.messages.setdefault("accuracy_score", []).append( @@ -153,7 +196,9 @@ def __init__(self, run_dir): ) self.keys = set(self.messages.keys()) - self.logger.info("Successfully loaded endpoints log from %s.", run_dir) + self.logger.info( + "Successfully loaded endpoints log from %s.", + scenario_dir) def _load_json(self, path): try: @@ -162,7 +207,6 @@ def _load_json(self, path): except BaseException: self.logger.error("Could not load json file from %s", path) return {} - return {} def _load_yaml(self, run_dir): for name in _CONFIG_FILES: @@ -197,7 +241,10 @@ def get_keys(self): return self.keys def num_errors(self): - return self["num_errors"] + return self["num_errors"] or 0 + + def get_errors(self): + return [] def has_error(self): return self.num_errors() != 0 @@ -212,18 +259,20 @@ def main(): backwards_map = _load_field_map("backwards.json") - # Collect all run directories (those containing at least one JSON and one - # YAML) + # Collect scenario-level directories (those containing config.yaml and + # the performance subdirectory with result_summary.json) run_dirs = [] - for root, _dirs, files in os.walk(_SAMPLE_LOGS_DIR): - has_json = any(f.endswith(".json") for f in files) - has_yaml = any(f.endswith(".yaml") or f.endswith(".yml") - for f in files) - if has_json and has_yaml: + for root, dirs, files in os.walk(_SAMPLE_LOGS_DIR): + has_yaml = any(f in _CONFIG_FILES for f in files) + has_perf = os.path.exists( + os.path.join(root, _PERF_SUBDIR, _RESULT_SUMMARY_FILE)) + if has_yaml and has_perf: run_dirs.append(root) if not run_dirs: - logger.error("No run directories found under %s.", _SAMPLE_LOGS_DIR) + logger.error( + "No scenario directories found under %s.", + _SAMPLE_LOGS_DIR) return 1 for run_dir in sorted(run_dirs): @@ -232,12 +281,15 @@ def main(): print(f"Directory: {folder}") print(f"{'=' * 70}") - parser = EndpointsParser(run_dir) + perf_parser = EndpointsParser(run_dir, log_type="performance") + acc_parser = EndpointsParser(run_dir, log_type="accuracy") found = [] not_found = [] for loadgen_key, endpoints_key in backwards_map.items(): - value = parser[loadgen_key] + value = perf_parser[loadgen_key] + if value is None: + value = acc_parser[loadgen_key] if value is not None: found.append((loadgen_key, endpoints_key, value)) else: diff --git a/tools/submission/submission_structure.md b/tools/submission/submission_structure.md index ac9e81ba7a..b0fb9ff4c7 100644 --- a/tools/submission/submission_structure.md +++ b/tools/submission/submission_structure.md @@ -65,7 +65,7 @@ The following diagram describes the standard submission structure. ## Endpoints submission structure -For endpoints submissions, the `mlperf_log_*.txt` files are replaced by structured JSON and YAML files produced by the endpoint harness. You can provide a performance+accuracy run in the performance folder or one performance run and one accuracy run +For endpoints submissions, the `mlperf_log_*.txt` files are replaced by structured JSON and YAML files produced by the endpoint harness. The `config.yaml` is placed at the scenario root, `result_summary.json` contains performance metrics, and `accuracy_results.json` contains accuracy metrics. ``` ... @@ -77,15 +77,11 @@ For endpoints submissions, the `mlperf_log_*.txt` files are replaced by structur │ │ │ ├── │ │ │ │ ├── │ │ │ │ │ ├── -│ │ │ │ │ │ ├── accuracy (optional) -│ │ │ │ │ │ │ ├── config.yaml -│ │ │ │ │ │ │ ├── results.json -│ │ │ │ │ │ │ └── results_summary.json +│ │ │ │ │ │ ├── config.yaml +│ │ │ │ │ │ ├── accuracy +│ │ │ │ │ │ │ └── accuracy_results.json │ │ │ │ │ │ ├── performance -│ │ │ │ │ │ │ ├── run_1 -│ │ │ │ │ │ │ │ ├── config.yaml -│ │ │ │ │ │ │ │ ├── results.json -│ │ │ │ │ │ │ │ └── results_summary.json +│ │ │ │ │ │ │ └── result_summary.json │ │ │ │ │ │ ├── │ │ │ │ │ │ │ ├── accuracy │ │ │ │ │ │ │ │ └── accuracy.txt diff --git a/tools/submission/truncate_accuracy_log.py b/tools/submission/truncate_accuracy_log.py index 62d8616927..b9d3d686a4 100755 --- a/tools/submission/truncate_accuracy_log.py +++ b/tools/submission/truncate_accuracy_log.py @@ -18,7 +18,6 @@ MAX_ACCURACY_LOG_SIZE = 10 * 1024 VIEWABLE_SIZE = 4096 -RESPONSES_LIMIT = 10 * 1024 # cap on truncated responses bytes HELP_TEXT = """ You can run this tool in 2 ways: @@ -124,84 +123,18 @@ def copy_submission_dir(src, dst, filter_submitter): ) -def _truncate_endpoints_results(results_path, acc_path, backup): - """Truncate the ``responses`` field in an endpoints accuracy results.json. +def _is_endpoints_submission(scenario_dir): + """Detect the endpoints submission layout via its scenario-root config.yaml. - Mirrors the behaviour of truncate_accuracy_log.py for traditional - submissions: backs up the original file (when --backup is given), - computes a SHA-256 hash of the original, appends ``hash=`` to - accuracy.txt, then writes back a copy with responses capped at - RESPONSES_LIMIT bytes. + Endpoints submissions have no mlperf_log_accuracy.json; the accuracy + artifact they produce (accuracy/accuracy_results.json) carries no large + per-sample payload to truncate, so callers should skip truncation + entirely rather than look for a file to shrink. """ - try: - with open(results_path, "r", encoding="utf-8") as f: - original_bytes = f.read().encode() - data = json.loads(original_bytes) - except Exception as exc: - log.error("Could not read %s: %s", results_path, exc) - return - - # Back up before any modification. - if backup: - backup_dir = os.path.join(backup, acc_path) - os.makedirs(backup_dir, exist_ok=True) - dst = os.path.join(backup_dir, "results.json") - if os.path.exists(dst): - log.error( - "not processing %s because %s already exists", - results_path, - dst, - ) - return - shutil.copy(results_path, dst) - - # Hash the original file for audit purposes (mirrors accuracy.txt hash - # written for traditional mlperf_log_accuracy.json truncation). - hash_val = hashlib.sha256(original_bytes).hexdigest() - acc_txt = os.path.join(acc_path, "accuracy.txt") - with open(acc_txt, "a", encoding="utf-8") as f: - f.write("\nhash={0}\n".format(hash_val)) - - # Truncate the responses collection so the file stays small. - responses = data.get("responses") - if responses is None or (isinstance( - responses, (list, dict)) and not responses): - log.info("%s has no responses to truncate", results_path) - return - - if isinstance(responses, list): - total = 2 # "[]" - idx = 0 - for i, r in enumerate(responses): - total += len(json.dumps(r).encode()) + (2 if i > 0 else 0) - if total > RESPONSES_LIMIT: - break - idx = i + 1 - data["responses"] = responses[:idx] - elif isinstance(responses, dict): - total = 2 # "{}" - kept = {} - for i, (k, v) in enumerate(responses.items()): - entry = len(json.dumps(k).encode()) + \ - len(json.dumps(v).encode()) + 2 - if i > 0: - entry += 2 - if total + entry > RESPONSES_LIMIT: - break - total += entry - kept[k] = v - data["responses"] = kept - else: - log.error( - "%s: responses has unexpected type %s; skipping truncation", - results_path, - type(responses).__name__, - ) - return - - with open(results_path, "w", encoding="utf-8") as f: - json.dump(data, f, indent=2) - log.info("%s responses truncated", results_path) + return any( + os.path.exists(os.path.join(scenario_dir, name)) + for name in ("config.yaml", "config.yml") + ) def truncate_results_dir(filter_submitter, backup, scenarios_to_skip): @@ -264,14 +197,10 @@ def truncate_results_dir(filter_submitter, backup, scenarios_to_skip): acc_txt = os.path.join( acc_path, "accuracy.txt") if not os.path.exists(acc_log): - # Endpoints submissions have results.json - # instead of mlperf_log_accuracy.json - endpoints_results = os.path.join( - acc_path, "results.json" - ) - if os.path.exists(endpoints_results): - _truncate_endpoints_results( - endpoints_results, acc_path, backup + if _is_endpoints_submission(name): + log.info( + "%s is an endpoints submission; nothing to truncate", + name, ) else: log.error("%s missing", acc_log)