From 956ebc1724bfa1d963c395ca0ed26739550ff658 Mon Sep 17 00:00:00 2001 From: iray-tno Date: Sun, 14 Jun 2026 12:04:39 +0900 Subject: [PATCH] feat: #11 improve relative score reporting --- src/commands/test_command.py | 37 ++++++++++++++++++++++++++--- tests/test_test_command.py | 46 ++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/src/commands/test_command.py b/src/commands/test_command.py index 70964e5..22d042d 100644 --- a/src/commands/test_command.py +++ b/src/commands/test_command.py @@ -67,7 +67,8 @@ def single_cmd(file, input_file, expected_file, preprocess, include_paths, rust) @click.option('--update-best-known', is_flag=True, help='Update --best-known with improved numeric outputs from this run') @click.option('--score-mode', type=click.Choice(['min', 'max']), default='min', show_default=True, help='Whether lower or higher numeric outputs are better') @click.option('--relative-scale', type=float, default=1.0, show_default=True, help='Per-case relative score scale') -def suite_cmd(file, cases_dir, pattern, workers, runs, timeout, preprocess, include_paths, rust, best_known_file, update_best_known, score_mode, relative_scale): +@click.option('--relative-round', is_flag=True, help='Round each relative score to contest-style integer points') +def suite_cmd(file, cases_dir, pattern, workers, runs, timeout, preprocess, include_paths, rust, best_known_file, update_best_known, score_mode, relative_scale, relative_round): """Compile once and run an exact-match test suite over *.in/*.out cases.""" if update_best_known and not best_known_file: click.echo("❌ --update-best-known requires --best-known", err=True) @@ -128,7 +129,7 @@ def suite_cmd(file, cases_dir, pattern, workers, runs, timeout, preprocess, incl if best_known_file: best_known = _load_best_known(best_known_file) - _report_relative_scores(results, best_known, score_mode, relative_scale) + _report_relative_scores(results, best_known, score_mode, relative_scale, relative_round) if update_best_known: updated = _update_best_known(results, best_known, score_mode) _write_best_known(best_known_file, best_known) @@ -486,10 +487,11 @@ def _write_best_known(best_known_file, best_known): f.write("\n") -def _report_relative_scores(results, best_known, score_mode, relative_scale): +def _report_relative_scores(results, best_known, score_mode, relative_scale, relative_round): click.echo("\n--- Relative Score ---") total = 0.0 scored = 0 + scored_cases = [] for result in results: case = result["case"] try: @@ -511,6 +513,14 @@ def _report_relative_scores(results, best_known, score_mode, relative_scale): total += relative_score scored += 1 + points = round(relative_score) if relative_round else relative_score + max_points = round(relative_scale) if relative_round else relative_scale + scored_cases.append({ + "case": case, + "points": points, + "max_points": max_points, + "loss": max(0, max_points - points), + }) click.echo( f"{case}: your={_format_score(your_score)} " f"best={_format_score(float(best_score))} " @@ -518,6 +528,27 @@ def _report_relative_scores(results, best_known, score_mode, relative_scale): ) click.echo(f"Total relative score: {total:.6f} / {(scored * relative_scale):.6f}") + _report_contest_score(scored_cases, relative_round) + + +def _report_contest_score(scored_cases, relative_round): + if not scored_cases: + return + + total_points = sum(case["points"] for case in scored_cases) + max_points = sum(case["max_points"] for case in scored_cases) + percent = total_points / max_points * 100 if max_points else 0.0 + if relative_round: + click.echo(f"Contest score: {total_points} / {max_points} ({percent:.2f}% of max, rounded per case)") + else: + click.echo(f"Contest score: {total_points:.6f} / {max_points:.6f} ({percent:.2f}% of max)") + + click.echo("\n--- Relative Loss Ranking ---") + for case in sorted(scored_cases, key=lambda item: item["loss"], reverse=True): + if relative_round: + click.echo(f"{case['case']}: loss={case['loss']} point(s)") + else: + click.echo(f"{case['case']}: loss={case['loss']:.6f} point(s)") def _update_best_known(results, best_known, score_mode): diff --git a/tests/test_test_command.py b/tests/test_test_command.py index e4a1681..9f2fa7c 100644 --- a/tests/test_test_command.py +++ b/tests/test_test_command.py @@ -298,6 +298,52 @@ def test_cpp_suite_reports_relative_score_from_best_known(self, tmp_path): assert "001.in: your=10 best=5 relative=50.000000" in result.output assert "002.in: your=20 best=10 relative=50.000000" in result.output assert "Total relative score: 100.000000 / 200.000000" in result.output + assert "Contest score: 100.000000 / 200.000000 (50.00% of max)" in result.output + assert "--- Relative Loss Ranking ---" in result.output + assert "001.in: loss=50.000000 point(s)" in result.output + + @pytest.mark.skipif(os.system("which g++ > /dev/null 2>&1") != 0, reason="g++ not available") + def test_cpp_suite_reports_rounded_contest_score(self, tmp_path): + """Test contest-style rounded per-case relative points.""" + test_file = tmp_path / "echo_first.cpp" + test_file.write_text(""" +#include + +int main() { + int value; + std::cin >> value; + std::cout << value << std::endl; + return 0; +} +""") + + cases_dir = tmp_path / "cases" + cases_dir.mkdir() + (cases_dir / "001.in").write_text("3\n") + (cases_dir / "001.out").write_text("3\n") + best_known = tmp_path / "best.json" + best_known.write_text('{"001.in": 2}\n') + + runner = CliRunner() + result = runner.invoke( + cli, + [ + 'test', + 'suite', + str(test_file), + str(cases_dir), + '--best-known', + str(best_known), + '--relative-scale', + '100', + '--relative-round', + ], + ) + + assert result.exit_code == 0 + assert "001.in: your=3 best=2 relative=66.666667" in result.output + assert "Contest score: 67 / 100 (67.00% of max, rounded per case)" in result.output + assert "001.in: loss=33 point(s)" in result.output @pytest.mark.skipif(os.system("which g++ > /dev/null 2>&1") != 0, reason="g++ not available") def test_cpp_suite_runs_multiple_times_for_noise_summary(self, tmp_path):