From 98865d761db0c7da85770bc70245d3fc9d11e6cd Mon Sep 17 00:00:00 2001 From: iray-tno Date: Sun, 14 Jun 2026 10:39:44 +0900 Subject: [PATCH] feat: #9 add noise-aware suite runs --- src/commands/test_command.py | 83 +++++++++++++++++++++++++++++++++--- tests/test_test_command.py | 46 ++++++++++++++++++++ 2 files changed, 124 insertions(+), 5 deletions(-) diff --git a/src/commands/test_command.py b/src/commands/test_command.py index 7a5ae55..ba3216b 100644 --- a/src/commands/test_command.py +++ b/src/commands/test_command.py @@ -2,6 +2,7 @@ import click import concurrent.futures import json +import statistics import tempfile import sys import os @@ -53,6 +54,7 @@ def single_cmd(file, input_file, expected_file, preprocess, include_paths, rust) @click.argument('cases_dir', type=click.Path(exists=True, file_okay=False, resolve_path=True)) @click.option('--pattern', default='*.in', show_default=True, help='Input filename glob inside cases_dir') @click.option('-j', '--workers', default=1, show_default=True, type=click.IntRange(min=1), help='Parallel case worker count') +@click.option('--runs', default=1, show_default=True, type=click.IntRange(min=1), help='Runs per case for noise-aware averaging') @click.option('--timeout', type=float, help='Per-case timeout in seconds (defaults to prepkit_config.yaml test.timeout or 5)') @click.option('--preprocess', is_flag=True, help='Preprocess the file before compiling') @click.option('-I', '--include-path', 'include_paths', multiple=True, type=click.Path(exists=True, file_okay=False, resolve_path=True), help='Include paths for preprocessing') @@ -61,7 +63,7 @@ def single_cmd(file, input_file, expected_file, preprocess, include_paths, rust) @click.option('--update-best-known', is_flag=True, help='Update --best-known with improved numeric outputs from this run') @click.option('--score-mode', type=click.Choice(['min', 'max']), default='min', show_default=True, help='Whether lower or higher numeric outputs are better') @click.option('--relative-scale', type=float, default=1.0, show_default=True, help='Per-case relative score scale') -def suite_cmd(file, cases_dir, pattern, workers, timeout, preprocess, include_paths, rust, best_known_file, update_best_known, score_mode, relative_scale): +def suite_cmd(file, cases_dir, pattern, workers, runs, timeout, preprocess, include_paths, rust, best_known_file, update_best_known, score_mode, relative_scale): """Compile once and run an exact-match test suite over *.in/*.out cases.""" if update_best_known and not best_known_file: click.echo("❌ --update-best-known requires --best-known", err=True) @@ -90,27 +92,36 @@ def suite_cmd(file, cases_dir, pattern, workers, timeout, preprocess, include_pa source_file = _prepare_source_for_compile(file, preprocess, include_paths, is_rust, config) executable = _compile_for_suite(file, source_file, is_rust, config) - click.echo(f"Running {len(cases)} case(s) with {workers} worker(s)...") - results = [] + if runs == 1: + click.echo(f"Running {len(cases)} case(s) with {workers} worker(s)...") + else: + click.echo(f"Running {len(cases)} case(s) x {runs} run(s) with {workers} worker(s)...") + + raw_results = [] with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: futures = [ executor.submit(_run_suite_case, executable, input_file, expected_file, timeout) for input_file, expected_file in cases + for _ in range(runs) ] for future in concurrent.futures.as_completed(futures): - results.append(future.result()) + raw_results.append(future.result()) + results = _aggregate_suite_results(raw_results, runs) if runs > 1 else raw_results results.sort(key=lambda result: result["case"]) passed = sum(1 for result in results if result["passed"]) click.echo("\n--- Suite Results ---") for result in results: status = "PASS" if result["passed"] else "FAIL" - click.echo(f"{status} {result['case']} ({result['runtime']:.3f}s)") + click.echo(_format_suite_result_line(status, result, runs)) if not result["passed"] and result["error"]: click.echo(f" {result['error']}") click.echo(f"\nSummary: {passed}/{len(results)} passed") + if runs > 1: + _report_noise_summary(results, runs) + if best_known_file: best_known = _load_best_known(best_known_file) _report_relative_scores(results, best_known, score_mode, relative_scale) @@ -245,6 +256,68 @@ def _run_suite_case(executable, input_file, expected_file, timeout): } +def _aggregate_suite_results(raw_results, runs): + grouped = {} + for result in raw_results: + grouped.setdefault(result["case"], []).append(result) + + aggregate_results = [] + for case, case_results in grouped.items(): + numeric_outputs = [] + for result in case_results: + try: + numeric_outputs.append(_parse_numeric_output(result["output"])) + except ValueError: + pass + + mean_output = statistics.mean(numeric_outputs) if numeric_outputs else None + output = f"{mean_output}\n" if mean_output is not None else "" + failed_runs = [result for result in case_results if not result["passed"]] + aggregate_results.append({ + "case": case, + "passed": not failed_runs and len(case_results) == runs, + "runtime": statistics.mean(result["runtime"] for result in case_results), + "error": _format_aggregate_error(failed_runs, len(case_results), runs), + "output": output, + "runs": len(case_results), + "passed_runs": len(case_results) - len(failed_runs), + "score_mean": mean_output, + "score_stdev": statistics.stdev(numeric_outputs) if len(numeric_outputs) > 1 else 0.0, + "numeric_runs": len(numeric_outputs), + }) + return aggregate_results + + +def _format_aggregate_error(failed_runs, actual_runs, expected_runs): + if actual_runs != expected_runs: + return f"expected {expected_runs} run(s), got {actual_runs}" + if not failed_runs: + return "" + first_error = failed_runs[0]["error"] + return f"{len(failed_runs)}/{actual_runs} run(s) failed: {first_error}" + + +def _format_suite_result_line(status, result, runs): + if runs == 1: + return f"{status} {result['case']} ({result['runtime']:.3f}s)" + return ( + f"{status} {result['case']} " + f"({result['passed_runs']}/{result['runs']} runs, avg {result['runtime']:.3f}s)" + ) + + +def _report_noise_summary(results, runs): + click.echo("\n--- Noise Summary ---") + for result in results: + if result["numeric_runs"] == runs: + click.echo( + f"{result['case']}: mean={_format_score(result['score_mean'])} " + f"stdev={result['score_stdev']:.6f}" + ) + else: + click.echo(f"{result['case']}: N/A ({result['numeric_runs']}/{runs} numeric run(s))") + + def _load_best_known(best_known_file): if not os.path.exists(best_known_file): return {} diff --git a/tests/test_test_command.py b/tests/test_test_command.py index 55dfcbe..eb9131f 100644 --- a/tests/test_test_command.py +++ b/tests/test_test_command.py @@ -299,6 +299,52 @@ def test_cpp_suite_reports_relative_score_from_best_known(self, tmp_path): assert "002.in: your=20 best=10 relative=50.000000" in result.output assert "Total relative score: 100.000000 / 200.000000" in result.output + @pytest.mark.skipif(os.system("which g++ > /dev/null 2>&1") != 0, reason="g++ not available") + def test_cpp_suite_runs_multiple_times_for_noise_summary(self, tmp_path): + """Test repeated suite runs aggregate numeric outputs before scoring.""" + test_file = tmp_path / "echo_first.cpp" + test_file.write_text(""" +#include + +int main() { + int value; + std::cin >> value; + std::cout << value << std::endl; + return 0; +} +""") + + cases_dir = tmp_path / "cases" + cases_dir.mkdir() + (cases_dir / "001.in").write_text("10\n") + (cases_dir / "001.out").write_text("10\n") + best_known = tmp_path / "best.json" + best_known.write_text('{"001.in": 5}\n') + + runner = CliRunner() + result = runner.invoke( + cli, + [ + 'test', + 'suite', + str(test_file), + str(cases_dir), + '--runs', + '3', + '--best-known', + str(best_known), + '--relative-scale', + '100', + ], + ) + + assert result.exit_code == 0 + assert "Running 1 case(s) x 3 run(s) with 1 worker(s)..." in result.output + assert "PASS 001.in (3/3 runs" in result.output + assert "--- Noise Summary ---" in result.output + assert "001.in: mean=10 stdev=0.000000" in result.output + assert "001.in: your=10 best=5 relative=50.000000" in result.output + @pytest.mark.skipif(os.system("which g++ > /dev/null 2>&1") != 0, reason="g++ not available") def test_cpp_suite_updates_best_known(self, tmp_path): """Test best-known JSON updates when this run improves numeric values."""