diff --git a/src/commands/test_command.py b/src/commands/test_command.py index ba3216b..70964e5 100644 --- a/src/commands/test_command.py +++ b/src/commands/test_command.py @@ -19,7 +19,11 @@ def test(args): """Compile/run one case, or use `test suite` for multi-case runs.""" if args and args[0] == "suite": - suite_cmd.main(args=list(args[1:]), prog_name="test suite", standalone_mode=False) + suite_args = list(args[1:]) + if suite_args and suite_args[0] == "compare": + suite_compare_cmd.main(args=suite_args[1:], prog_name="test suite compare", standalone_mode=False) + else: + suite_cmd.main(args=suite_args, prog_name="test suite", standalone_mode=False) else: single_cmd.main(args=list(args), prog_name="test", standalone_mode=False) @@ -139,6 +143,75 @@ def suite_cmd(file, cases_dir, pattern, workers, runs, timeout, preprocess, incl os.remove(executable) +@click.command(name="compare") +@click.argument('file_a', type=click.Path(exists=True, resolve_path=True)) +@click.argument('file_b', type=click.Path(exists=True, resolve_path=True)) +@click.argument('cases_dir', type=click.Path(exists=True, file_okay=False, resolve_path=True)) +@click.option('--pattern', default='*.in', show_default=True, help='Input filename glob inside cases_dir') +@click.option('-j', '--workers', default=1, show_default=True, type=click.IntRange(min=1), help='Parallel paired worker count') +@click.option('--runs', default=1, show_default=True, type=click.IntRange(min=1), help='Paired runs per case') +@click.option('--timeout', type=float, help='Per-program timeout in seconds (defaults to prepkit_config.yaml test.timeout or 5)') +@click.option('--preprocess', is_flag=True, help='Preprocess both files before compiling') +@click.option('-I', '--include-path', 'include_paths', multiple=True, type=click.Path(exists=True, file_okay=False, resolve_path=True), help='Include paths for preprocessing') +@click.option('--rust', is_flag=True, help='Force Rust mode for both files (auto-detected from .rs extension)') +@click.option('--score-mode', type=click.Choice(['min', 'max']), default='min', show_default=True, help='Whether lower or higher numeric outputs are better') +def suite_compare_cmd(file_a, file_b, cases_dir, pattern, workers, runs, timeout, preprocess, include_paths, rust, score_mode): + """Compare two programs with paired repeated suite runs.""" + config = load_config() + timeout = timeout if timeout is not None else config.get("test", {}).get("timeout", 5) + is_rust_a = _detect_suite_language(file_a, rust) + is_rust_b = _detect_suite_language(file_b, rust) + + cases = _discover_suite_cases(cases_dir, pattern) + if not cases: + click.echo(f"❌ No cases found in {cases_dir} matching {pattern} with .out files", err=True) + sys.exit(1) + + executable_a = None + executable_b = None + source_a = None + source_b = None + try: + source_a = _prepare_source_for_compile(file_a, preprocess, include_paths, is_rust_a, config) + source_b = _prepare_source_for_compile(file_b, preprocess, include_paths, is_rust_b, config) + executable_a = _compile_for_suite(file_a, source_a, is_rust_a, config) + executable_b = _compile_for_suite(file_b, source_b, is_rust_b, config) + + click.echo(f"Comparing {len(cases)} case(s) x {runs} paired run(s) with {workers} worker(s)...") + pair_results = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: + futures = [ + executor.submit(_run_compare_pair, executable_a, executable_b, input_file, expected_file, timeout) + for input_file, expected_file in cases + for _ in range(runs) + ] + for future in concurrent.futures.as_completed(futures): + pair_results.append(future.result()) + + _report_compare_results(pair_results, score_mode) + if any(not result["valid"] for result in pair_results): + sys.exit(1) + finally: + for source_file in (source_a, source_b): + if preprocess and source_file and os.path.exists(source_file): + os.remove(source_file) + for executable in (executable_a, executable_b): + if executable and os.path.exists(executable): + os.remove(executable) + + +def _detect_suite_language(file, force_rust): + file_ext = os.path.splitext(file)[1].lower() + is_rust = force_rust or file_ext == '.rs' + is_cpp = file_ext in ['.cpp', '.cc', '.cxx', '.c++'] + + if not is_rust and not is_cpp: + click.echo(f"❌ Unsupported file extension: {file_ext}", err=True) + click.echo(" Supported: .cpp, .cc, .cxx, .c++, .rs", err=True) + sys.exit(1) + return is_rust + + def _discover_suite_cases(cases_dir, pattern): input_files = sorted( os.path.join(cases_dir, filename) @@ -256,6 +329,82 @@ def _run_suite_case(executable, input_file, expected_file, timeout): } +def _run_compare_pair(executable_a, executable_b, input_file, expected_file, timeout): + result_a = _run_suite_case(executable_a, input_file, expected_file, timeout) + result_b = _run_suite_case(executable_b, input_file, expected_file, timeout) + case = os.path.basename(input_file) + + try: + score_a = _parse_numeric_output(result_a["output"]) + score_b = _parse_numeric_output(result_b["output"]) + except ValueError as exc: + return { + "case": case, + "valid": False, + "error": str(exc), + "score_a": None, + "score_b": None, + "diff": None, + } + + return { + "case": case, + "valid": True, + "error": "", + "score_a": score_a, + "score_b": score_b, + "diff": score_a - score_b, + } + + +def _report_compare_results(pair_results, score_mode): + pair_results.sort(key=lambda result: result["case"]) + valid_results = [result for result in pair_results if result["valid"]] + + click.echo("\n--- Paired A/B Results ---") + for result in pair_results: + if not result["valid"]: + click.echo(f"{result['case']}: N/A ({result['error']})") + continue + winner = _compare_winner(result["score_a"], result["score_b"], score_mode) + click.echo( + f"{result['case']}: " + f"A={_format_score(result['score_a'])} " + f"B={_format_score(result['score_b'])} " + f"diff(A-B)={result['diff']:.6f} " + f"winner={winner}" + ) + + if not valid_results: + click.echo("\nSummary: 0 valid paired run(s)") + return + + diffs = [result["diff"] for result in valid_results] + a_wins = sum(1 for result in valid_results if _compare_winner(result["score_a"], result["score_b"], score_mode) == "A") + b_wins = sum(1 for result in valid_results if _compare_winner(result["score_a"], result["score_b"], score_mode) == "B") + ties = len(valid_results) - a_wins - b_wins + + click.echo( + "\nSummary: " + f"A wins {a_wins}/{len(valid_results)}, " + f"B wins {b_wins}/{len(valid_results)}, " + f"ties {ties}/{len(valid_results)}" + ) + click.echo(f"Mean diff(A-B): {statistics.mean(diffs):.6f}") + if len(diffs) > 1: + click.echo(f"Diff stdev: {statistics.stdev(diffs):.6f}") + else: + click.echo("Diff stdev: 0.000000") + + +def _compare_winner(score_a, score_b, score_mode): + if score_a == score_b: + return "tie" + if score_mode == "min": + return "A" if score_a < score_b else "B" + return "A" if score_a > score_b else "B" + + def _aggregate_suite_results(raw_results, runs): grouped = {} for result in raw_results: diff --git a/tests/test_test_command.py b/tests/test_test_command.py index eb9131f..e4a1681 100644 --- a/tests/test_test_command.py +++ b/tests/test_test_command.py @@ -345,6 +345,62 @@ def test_cpp_suite_runs_multiple_times_for_noise_summary(self, tmp_path): assert "001.in: mean=10 stdev=0.000000" in result.output assert "001.in: your=10 best=5 relative=50.000000" in result.output + @pytest.mark.skipif(os.system("which g++ > /dev/null 2>&1") != 0, reason="g++ not available") + def test_cpp_suite_compare_reports_paired_wins(self, tmp_path): + """Test paired A/B suite comparison for numeric outputs.""" + file_a = tmp_path / "solver_a.cpp" + file_a.write_text(""" +#include + +int main() { + int value; + std::cin >> value; + std::cout << value << std::endl; + return 0; +} +""") + file_b = tmp_path / "solver_b.cpp" + file_b.write_text(""" +#include + +int main() { + int value; + std::cin >> value; + std::cout << value + 5 << std::endl; + return 0; +} +""") + + cases_dir = tmp_path / "cases" + cases_dir.mkdir() + (cases_dir / "001.in").write_text("10\n") + (cases_dir / "001.out").write_text("10\n") + (cases_dir / "002.in").write_text("20\n") + (cases_dir / "002.out").write_text("20\n") + + runner = CliRunner() + result = runner.invoke( + cli, + [ + 'test', + 'suite', + 'compare', + str(file_a), + str(file_b), + str(cases_dir), + '--runs', + '2', + ], + ) + + assert result.exit_code == 0 + assert "Comparing 2 case(s) x 2 paired run(s) with 1 worker(s)..." in result.output + assert "--- Paired A/B Results ---" in result.output + assert "001.in: A=10 B=15 diff(A-B)=-5.000000 winner=A" in result.output + assert "002.in: A=20 B=25 diff(A-B)=-5.000000 winner=A" in result.output + assert "Summary: A wins 4/4, B wins 0/4, ties 0/4" in result.output + assert "Mean diff(A-B): -5.000000" in result.output + @pytest.mark.skipif(os.system("which g++ > /dev/null 2>&1") != 0, reason="g++ not available") def test_cpp_suite_updates_best_known(self, tmp_path): """Test best-known JSON updates when this run improves numeric values."""