diff --git a/docs/QUALITY_SEED_UNCERTAINTY.md b/docs/QUALITY_SEED_UNCERTAINTY.md index 71f63da..23f3bc4 100644 --- a/docs/QUALITY_SEED_UNCERTAINTY.md +++ b/docs/QUALITY_SEED_UNCERTAINTY.md @@ -7,8 +7,12 @@ Each input must be an existing evaluation report from `kev_laya.evaluation.evalu with the same split, record/question IDs, groups, labels, question types and declared slice metadata. It requires the same frozen split hash and distinct checkpoint hashes from the CLI evaluation reports. It refuses missing or duplicate rows, mismatched -targets, inconsistent splits, nonfinite metrics and replacement of an existing -output. Input file hashes are included in the output. +targets, inconsistent splits and replacement of an existing output. Before +aggregation, each row must have finite numeric measurements: correctness and +answer confidence within [0, 1], and nonnegative NLL, Brier and ordinal error +(or no ordinal error for unordered tasks). Invalid measurements are rejected +before an output is created. Fractional soft-target correctness remains valid. +Input file hashes are included in the output. For each metric, the output gives individual seed values, their mean, sample standard deviation and observed range, including per-slice results. It is diff --git a/src/kev_laya/evaluation.py b/src/kev_laya/evaluation.py index 43249f7..f47c0d1 100644 --- a/src/kev_laya/evaluation.py +++ b/src/kev_laya/evaluation.py @@ -10,9 +10,27 @@ from .model import DecisionEngine +def _validate_metric_rows(rows: list[dict]) -> None: + """Reject invalid measurements before binning or bootstrap resampling.""" + for row in rows: + if not isinstance(row, dict): + raise ValueError("evaluation_row_metrics_invalid") + for name in ("correct", "answer_probability", "nll", "brier", "ordinal_mae"): + value = row.get(name) + if name == "ordinal_mae" and value is None and name in row: + continue + try: + valid = type(value) in (int, float) and math.isfinite(value) and value >= 0 + except OverflowError: + valid = False + if not valid or (name in ("correct", "answer_probability") and value > 1): + raise ValueError(f"evaluation_row_metric_invalid:{name}") + + def summarize(rows: list[dict], bins: int = 10) -> dict: if type(bins) is not int or bins < 1: raise ValueError("bins must be a positive integer") + _validate_metric_rows(rows) if not rows: return {"count": 0, "nll": None, "brier": None, "ece": None, "accuracy": None, "ordinal_mae": None, "reliability": [], "risk_coverage": []} @@ -183,6 +201,7 @@ def grouped_bootstrap(rows: list[dict], *, samples: int = 1000, seed: int = 42, if len(grouped) < 2: result["reason"] = "fewer than two source groups" return result + _validate_metric_rows(rows) groups = [grouped[key] for key in sorted(grouped)] rng = random.Random(seed) # Do not mutate training/evaluation global RNG. values = {key: [] for key in fields} diff --git a/tests/test_evaluation_uncertainty.py b/tests/test_evaluation_uncertainty.py index 58c5940..21b17b2 100644 --- a/tests/test_evaluation_uncertainty.py +++ b/tests/test_evaluation_uncertainty.py @@ -54,6 +54,35 @@ def test_invalid_bin_count_rejected(bins): summarize(rows(), bins=bins) +@pytest.mark.parametrize("metric,value", [ + ("correct", -0.1), ("correct", 1.1), ("correct", True), + ("answer_probability", -0.1), ("answer_probability", 1.1), + ("answer_probability", float("nan")), ("answer_probability", float("inf")), + ("nll", -0.1), ("nll", "0.3"), ("nll", 10 ** 400), + ("brier", -0.1), ("brier", float("inf")), + ("ordinal_mae", -0.1), ("ordinal_mae", float("nan")), +]) +def test_invalid_measurements_rejected_before_quality_aggregation(metric, value): + data = rows() + data[0][metric] = value + with pytest.raises(ValueError, match=f"evaluation_row_metric_invalid:{metric}"): + summarize(data) + with pytest.raises(ValueError, match=f"evaluation_row_metric_invalid:{metric}"): + grouped_bootstrap(data, samples=2, seed=1) + + +def test_soft_target_correctness_and_confidence_endpoints_remain_valid(): + data = rows()[:3] + for row, correct, confidence, ordinal in zip( + data, (0.0, 0.25, 1.0), (0.0, 0.5, 1.0), (None, 0.0, 1.25), strict=True): + row.update(correct=correct, answer_probability=confidence, ordinal_mae=ordinal) + result = summarize(data) + assert result["accuracy"] == pytest.approx(1.25 / 3) + assert result["ece"] == pytest.approx(0.25 / 3) + assert result["ordinal_mae"] == 0.625 + assert sum(bin_["count"] for bin_ in result["reliability"]) == len(data) + + def test_evaluation_retains_group_identity(tiny, suite): data, _ = suite report = evaluate(tiny, data["development"][:2], ByteTokenizer(), Limits(512, 8192), split="development") @@ -138,6 +167,25 @@ def test_seed_report_cli_reads_three_actual_evaluation_reports(tmp_path, tiny, s seed_report_cli([*args, "--out", str(output)]) +@pytest.mark.parametrize("metric,value", [("correct", 2.0), ("answer_probability", -0.1)]) +def test_seed_report_cli_refuses_impossible_quality_without_output( + tmp_path, tiny, suite, metric, value): + data, _ = suite + args = [] + for seed in (11, 12, 13): + path = tmp_path / f"seed-{seed}.json" + report = evaluate(tiny, data["development"][:1], ByteTokenizer(), Limits(512, 8192), + split="development") + report.update(checkpoint_sha256=f"{seed:064x}", split_sha256="a" * 64) + report["rows"][0][metric] = value + path.write_text(json.dumps(report), encoding="utf-8") + args.extend(("--run", f"{seed}={path}")) + output = tmp_path / "summary.json" + with pytest.raises(ValueError, match=f"evaluation_row_metric_invalid:{metric}"): + seed_report_cli([*args, "--out", str(output)]) + assert not output.exists() + + def test_calibration_without_type_does_not_retain_unreported_old_temperature(tiny, suite): data, manifest = suite datum = copy.deepcopy(data["calibration"][0]) diff --git a/tests/test_native_gradients.py b/tests/test_native_gradients.py index 8dfb558..de00d4a 100644 --- a/tests/test_native_gradients.py +++ b/tests/test_native_gradients.py @@ -7,6 +7,16 @@ from kev_laya.native_gradients import CanonicalFP32ShortAttention, causal_vjp +@pytest.fixture(params=[ + "cpu", + pytest.param("cuda", marks=pytest.mark.skipif( + not torch.cuda.is_available(), reason="CUDA device unavailable")), +]) +def attention_device(request): + """Exercise portable helper math without substituting CPU for CUDA proof.""" + return request.param + + @pytest.mark.parametrize("prefix", [0, 8]) def test_causal_vjp_matches_independent_double_autograd(prefix): generator = torch.Generator().manual_seed(2618 + prefix) @@ -28,15 +38,14 @@ def test_causal_vjp_matches_independent_double_autograd(prefix): torch.testing.assert_close(observed, oracle, atol=1e-12, rtol=1e-12) -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA device unavailable") @pytest.mark.parametrize("prefix", [0, 11]) -def test_fp32_short_grouped_double_cache_matches_explicit_eager_gqa(prefix): - generator = torch.Generator(device="cuda").manual_seed(4521 + prefix) - q0 = torch.randn(1, 4, 7, 16, generator=generator, device="cuda", dtype=torch.float32) +def test_fp32_short_grouped_double_cache_matches_explicit_eager_gqa(prefix, attention_device): + generator = torch.Generator(device=attention_device).manual_seed(4521 + prefix) + q0 = torch.randn(1, 4, 7, 16, generator=generator, device=attention_device, dtype=torch.float32) k0 = torch.randn(1, 2, prefix + 7, 16, generator=generator, - device="cuda", dtype=torch.float32).double() - v0 = torch.randn(k0.shape, generator=generator, device="cuda", dtype=torch.float32).double() - upstream = torch.randn(q0.shape, generator=generator, device="cuda", dtype=torch.float32) + device=attention_device, dtype=torch.float32).double() + v0 = torch.randn(k0.shape, generator=generator, device=attention_device, dtype=torch.float32).double() + upstream = torch.randn(q0.shape, generator=generator, device=attention_device, dtype=torch.float32) q, k, v = [tensor.detach().clone().requires_grad_(True) for tensor in (q0, k0, v0)] actual = CanonicalFP32ShortAttention.apply(q, k, v, 2) actual_vjp = torch.autograd.grad(actual, (q, k, v), upstream) @@ -49,7 +58,7 @@ def test_fp32_short_grouped_double_cache_matches_explicit_eager_gqa(prefix): padded_k = torch.nn.functional.pad(repeated_k, (0, 0, 0, 256 - total)) padded_v = torch.nn.functional.pad(repeated_v, (0, 0, 0, 256 - total)) scores = torch.matmul(padded_q, padded_k.transpose(-2, -1)) * (16 ** -0.5) - allowed = torch.ones((256, 256), device="cuda", dtype=torch.bool).tril() + allowed = torch.ones((256, 256), device=attention_device, dtype=torch.bool).tril() scores = scores.masked_fill(~allowed, torch.finfo(scores.dtype).min) probabilities = torch.nn.functional.softmax(scores, dim=-1, dtype=torch.float32).to(rq.dtype) expected = torch.matmul(probabilities, padded_v)[..., prefix:prefix + 7, :] @@ -76,14 +85,13 @@ def test_fp32_short_grouped_double_cache_matches_explicit_eager_gqa(prefix): atol=2e-5, rtol=2e-5) -@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA device unavailable") @pytest.mark.parametrize("prefix_length", [1, 11, 62, 127, 200, 254, 255]) -def test_fp32_short_fixed256_matches_any_cached_prefix(prefix_length): - generator = torch.Generator(device="cuda").manual_seed(6256 + prefix_length) - query = torch.randn(1, 4, 256, 16, generator=generator, device="cuda") - key = torch.randn(1, 2, 256, 16, generator=generator, device="cuda").double() - value = torch.randn(1, 2, 256, 16, generator=generator, device="cuda").double() - upstream = torch.randn(query.shape, generator=generator, device="cuda") +def test_fp32_short_fixed256_matches_any_cached_prefix(prefix_length, attention_device): + generator = torch.Generator(device=attention_device).manual_seed(6256 + prefix_length) + query = torch.randn(1, 4, 256, 16, generator=generator, device=attention_device) + key = torch.randn(1, 2, 256, 16, generator=generator, device=attention_device).double() + value = torch.randn(1, 2, 256, 16, generator=generator, device=attention_device).double() + upstream = torch.randn(query.shape, generator=generator, device=attention_device) full_inputs = [tensor.detach().clone().requires_grad_(True) for tensor in (query, key, value)] split_inputs = [tensor.detach().clone().requires_grad_(True)