From 172ce4456c6e1d87385f24ccdf7957a18ba08501 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ktu=C4=9F=20=C3=96zkan=2C=20MD?= Date: Sat, 11 Jul 2026 00:08:46 +0300 Subject: [PATCH 1/2] feat(monitor): add reward-hacking measurement schemas --- CHANGELOG.md | 1 + src/inspect_ai/monitor/__init__.py | 17 ++ src/inspect_ai/monitor/_types.py | 399 +++++++++++++++++++++++++++++ tests/monitor/test_types.py | 387 ++++++++++++++++++++++++++++ 4 files changed, 804 insertions(+) create mode 100644 src/inspect_ai/monitor/__init__.py create mode 100644 src/inspect_ai/monitor/_types.py create mode 100644 tests/monitor/test_types.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 102d0d350e..9b2d1598d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ ## 0.3.246 (10 July 2026) +- Monitoring: New public schema types represent oracle-isolated monitor context, signal derivation and availability, and append-only attempt/sample records. - Grok: Support for Grok 4.5 (model info database entry; accepts `reasoning_effort` with a documented default of `high`). - OpenAI: Support for GPT-5.6 (Sol, Terra, and Luna) — model info database entries and codename frontier aliasing now target `gpt-5.6`. - OpenAI: `reasoning_effort="max"` is now passed through natively for GPT-5.6+ models rather than being clamped to `xhigh`. diff --git a/src/inspect_ai/monitor/__init__.py b/src/inspect_ai/monitor/__init__.py new file mode 100644 index 0000000000..f49183f8b9 --- /dev/null +++ b/src/inspect_ai/monitor/__init__.py @@ -0,0 +1,17 @@ +from ._types import ( + ComponentRef, + MonitorContext, + MonitorRecord, + ProducerRef, + SignalDerivation, + SignalValue, +) + +__all__ = [ + "ComponentRef", + "MonitorContext", + "MonitorRecord", + "ProducerRef", + "SignalDerivation", + "SignalValue", +] diff --git a/src/inspect_ai/monitor/_types.py b/src/inspect_ai/monitor/_types.py new file mode 100644 index 0000000000..70736f2a79 --- /dev/null +++ b/src/inspect_ai/monitor/_types.py @@ -0,0 +1,399 @@ +import math +import re +from typing import Any, Literal, Mapping, cast + +from pydantic import ( + AwareDatetime, + BaseModel, + ConfigDict, + Field, + JsonValue, + SerializerFunctionWrapHandler, + model_serializer, + model_validator, +) +from typing_extensions import Self + + +class _FrozenDict(dict[str, Any]): + def _immutable(self, *args: Any, **kwargs: Any) -> None: + raise TypeError("monitor schema containers are immutable") + + __setitem__ = _immutable # type: ignore[assignment] + __delitem__ = _immutable # type: ignore[assignment] + __ior__ = _immutable # type: ignore[assignment] + clear = _immutable # type: ignore[assignment] + pop = _immutable # type: ignore[assignment] + popitem = _immutable # type: ignore[assignment] + setdefault = _immutable # type: ignore[assignment] + update = _immutable # type: ignore[assignment] + + def __copy__(self) -> Self: + return self + + def __deepcopy__(self, memo: dict[int, Any]) -> Self: + memo[id(self)] = self + return self + + def __reduce__(self) -> tuple[type[Self], tuple[dict[str, Any]]]: + return (type(self), (dict(self),)) + + +class _FrozenList(list[JsonValue]): + def _immutable(self, *args: Any, **kwargs: Any) -> None: + raise TypeError("monitor schema containers are immutable") + + __setitem__ = _immutable # type: ignore[assignment] + __delitem__ = _immutable # type: ignore[assignment] + __iadd__ = _immutable # type: ignore[assignment] + __imul__ = _immutable # type: ignore[assignment] + append = _immutable # type: ignore[assignment] + clear = _immutable # type: ignore[assignment] + extend = _immutable # type: ignore[assignment] + insert = _immutable # type: ignore[assignment] + pop = _immutable # type: ignore[assignment] + remove = _immutable # type: ignore[assignment] + reverse = _immutable # type: ignore[assignment] + sort = _immutable # type: ignore[assignment] + + def __copy__(self) -> Self: + return self + + def __deepcopy__(self, memo: dict[int, Any]) -> Self: + memo[id(self)] = self + return self + + def __reduce__(self) -> tuple[type[Self], tuple[list[JsonValue]]]: + return (type(self), (list(self),)) + + +class _MonitorModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + def model_copy( + self, *, update: Mapping[str, Any] | None = None, deep: bool = False + ) -> Self: + """Return a validated copy, including any field updates.""" + values = self.model_dump(round_trip=True) + if deep: + values = self.model_validate(values).model_dump(round_trip=True) + if update: + values.update(update) + return self.__class__.model_validate(values) + + def _validate_serialization_state(self) -> None: + pass + + @model_serializer(mode="wrap") + def validate_before_serialization( + self, handler: SerializerFunctionWrapHandler + ) -> Any: + self._validate_serialization_state() + return handler(self) + + +def _freeze_json(value: JsonValue) -> JsonValue: + if isinstance(value, dict): + return _FrozenDict({key: _freeze_json(item) for key, item in value.items()}) + if isinstance(value, list): + return _FrozenList([_freeze_json(item) for item in value]) + return value + + +def _validate_finite_json(value: JsonValue | None) -> None: + if isinstance(value, float) and not math.isfinite(value): + raise ValueError("JSON numbers must be finite") + if isinstance(value, list): + for item in value: + _validate_finite_json(item) + if isinstance(value, dict): + for item in value.values(): + _validate_finite_json(item) + + +_SECRET_KEY = re.compile( + r"(apikey|accesstoken|authtoken|token|secret|password|credential|credentials|privatekey|authorization)$" +) +_REDACTED_VALUES = {"[REDACTED]", "", "***"} + + +def _normalize_config_key(key: str) -> str: + return re.sub(r"[^a-z0-9]+", "", key.lower()) + + +def _is_redacted(value: JsonValue) -> bool: + return ( + value is None + or value is False + or value == "" + or (isinstance(value, str) and value in _REDACTED_VALUES) + ) + + +def _validate_redacted_config(value: JsonValue, path: str = "config") -> None: + if isinstance(value, dict): + for key, item in value.items(): + item_path = f"{path}.{key}" + normalized = _normalize_config_key(key) + if _SECRET_KEY.search(normalized) and not _is_redacted(item): + raise ValueError(f"{item_path} must be redacted before serialization") + _validate_redacted_config(item, item_path) + elif isinstance(value, list): + for index, item in enumerate(value): + _validate_redacted_config(item, f"{path}[{index}]") + + +class ComponentRef(_MonitorModel): + """Stable identity for a component referenced by monitor output.""" + + name: str = Field(min_length=1) + """Registry or provider-qualified component name.""" + + version: str | None = Field(default=None) + """Optional component version.""" + + +class ProducerRef(ComponentRef): + """Identity and configuration of a monitor producer.""" + + config: dict[str, JsonValue] = Field(default_factory=dict) + """Pre-redacted, non-secret, JSON-serializable producer configuration.""" + + def _validate_serialization_state(self) -> None: + _validate_finite_json(self.config) + _validate_redacted_config(self.config) + + @model_validator(mode="after") + def validate_config(self) -> Self: + self._validate_serialization_state() + object.__setattr__( + self, "config", cast(dict[str, JsonValue], _freeze_json(self.config)) + ) + return self + + +class SignalDerivation(_MonitorModel): + """How a signal value was derived from observations.""" + + method: Literal["instant", "ema", "rolling", "custom"] + """Signal derivation method.""" + + name: str | None = Field(default=None, min_length=1) + """Stable algorithm name required for custom derivations.""" + + version: str | None = Field(default=None, min_length=1) + """Optional custom derivation version.""" + + alpha: float | None = Field(default=None, gt=0, le=1) + """Smoothing factor for an exponential moving average.""" + + window_size: int | None = Field(default=None, gt=0) + """Observation count for a rolling window.""" + + parameters: dict[str, JsonValue] = Field(default_factory=dict) + """Method-specific JSON metadata such as a distance metric.""" + + def _validate_serialization_state(self) -> None: + _validate_finite_json(self.parameters) + reserved = {"method", "name", "version", "alpha", "window_size"} + shadowed = reserved.intersection(self.parameters) + if shadowed: + field = sorted(shadowed)[0] + raise ValueError(f"parameters cannot shadow '{field}'") + + @model_validator(mode="after") + def validate_parameters(self) -> Self: + self._validate_serialization_state() + + if self.method == "custom": + if self.name is None: + raise ValueError("name is required when method is 'custom'") + elif self.name is not None or self.version is not None: + field = "name" if self.name is not None else "version" + raise ValueError(f"{field} is only valid when method is 'custom'") + + if self.method == "ema": + if self.alpha is None: + raise ValueError("alpha is required when method is 'ema'") + if self.window_size is not None: + raise ValueError("window_size is not valid when method is 'ema'") + elif self.method == "rolling": + if self.window_size is None: + raise ValueError("window_size is required when method is 'rolling'") + if self.alpha is not None: + raise ValueError("alpha is not valid when method is 'rolling'") + elif self.alpha is not None or self.window_size is not None: + field = "alpha" if self.alpha is not None else "window_size" + raise ValueError(f"{field} is not valid when method is '{self.method}'") + object.__setattr__( + self, + "parameters", + cast(dict[str, JsonValue], _freeze_json(self.parameters)), + ) + return self + + +class SignalValue(_MonitorModel): + """A monitor signal together with its availability and derivation.""" + + value: JsonValue | None = Field(default=None) + """Observed JSON value, or `None` when no value is available.""" + + state: Literal["observed", "unavailable", "not_applicable", "error"] + """Availability state of the signal.""" + + reason: str | None = Field(default=None) + """Optional explanation for a non-observed state.""" + + derivation: SignalDerivation = Field( + default_factory=lambda: SignalDerivation(method="instant") + ) + """Description of how the value was derived.""" + + def _validate_serialization_state(self) -> None: + _validate_finite_json(self.value) + if self.state == "observed" and self.value is None: + raise ValueError("observed signals require a non-null value") + if self.state != "observed" and self.value is not None: + raise ValueError(f"value must be None when state is '{self.state}'") + + @model_validator(mode="after") + def validate_value_state(self) -> Self: + self._validate_serialization_state() + if self.value is not None: + object.__setattr__(self, "value", _freeze_json(self.value)) + return self + + +class MonitorContext(_MonitorModel): + """Oracle-free context exposed to a monitor producer.""" + + eval_set_id: str | None = Field(default=None) + """Evaluation set identifier, when the sample belongs to an eval set.""" + + run_id: str = Field(min_length=1) + """Evaluation run identifier.""" + + eval_id: str = Field(min_length=1) + """Task evaluation identifier.""" + + sample_uuid: str = Field(min_length=1) + """Stable sample UUID across attempts.""" + + epoch: int = Field(gt=0) + """One-based sample epoch.""" + + attempt: int = Field(gt=0) + """One-based sample attempt.""" + + model: ComponentRef + """Model used for the sample.""" + + +class MonitorRecord(_MonitorModel): + """Attempt or sample record for an append-only persistence stream. + + Append-only is a recorder invariant: persisted records are never updated. + Field reassignment and nested schema-container mutation are both rejected. + """ + + schema_version: Literal["1"] = "1" + """Monitor record schema version.""" + + record_id: str = Field(min_length=1) + """Unique record identifier.""" + + record_kind: Literal["attempt", "sample"] + """Lifecycle level represented by this record.""" + + emitted_at: AwareDatetime + """Timezone-aware record emission time.""" + + eval_set_id: str | None = Field(default=None) + run_id: str = Field(min_length=1) + eval_id: str = Field(min_length=1) + sample_uuid: str = Field(min_length=1) + dataset_sample_id: str | int + epoch: int = Field(gt=0) + attempt: int | None = Field(default=None, gt=0) + attempt_record_ids: tuple[str, ...] = Field(default_factory=tuple) + will_retry: bool | None = Field(default=None) + + producer: ProducerRef + model: ComponentRef + scorer: ComponentRef | None = Field(default=None) + + execution_status: Literal["completed", "errored", "cancelled"] + scoring_status: Literal[ + "scored", + "abstained", + "unscored", + "not_run", + "errored", + ] + status_reason: str | None = Field(default=None) + + signals: dict[str, SignalValue] = Field(default_factory=dict) + trace_ref: str | None = Field(default=None) + + def _validate_serialization_state(self) -> None: + for name, signal in self.signals.items(): + if not name: + raise ValueError("signal names must be non-empty") + if not isinstance(signal, SignalValue): + raise ValueError(f"signal '{name}' must be a SignalValue") + + @model_validator(mode="after") + def validate_record_shape(self) -> Self: + self._validate_serialization_state() + if self.record_kind == "attempt": + if self.attempt is None: + raise ValueError("attempt is required for attempt records") + if self.attempt_record_ids: + raise ValueError("attempt_record_ids must be empty for attempt records") + if self.will_retry is None: + raise ValueError("will_retry is required for attempt records") + if self.will_retry and self.execution_status != "errored": + raise ValueError( + "will_retry can be true only when execution_status is 'errored'" + ) + else: + if self.attempt is not None: + raise ValueError("attempt must be None for sample records") + if self.will_retry is not None: + raise ValueError("will_retry must be None for sample records") + if any(not record_id for record_id in self.attempt_record_ids): + raise ValueError("attempt_record_ids cannot contain empty IDs") + if len(set(self.attempt_record_ids)) != len(self.attempt_record_ids): + raise ValueError("attempt_record_ids cannot contain duplicates") + if self.record_id in self.attempt_record_ids: + raise ValueError( + "attempt_record_ids cannot contain the sample record_id" + ) + if not self.attempt_record_ids and not ( + self.execution_status in {"errored", "cancelled"} + and self.scoring_status == "not_run" + ): + raise ValueError( + "attempt_record_ids can be empty only for a terminal " + "initialization failure" + ) + + if self.scoring_status == "scored" and self.scorer is None: + raise ValueError("scorer is required when scoring_status is 'scored'") + object.__setattr__( + self, + "signals", + cast(dict[str, SignalValue], _FrozenDict(self.signals)), + ) + return self + + +__all__ = [ + "ComponentRef", + "MonitorContext", + "MonitorRecord", + "ProducerRef", + "SignalDerivation", + "SignalValue", +] diff --git a/tests/monitor/test_types.py b/tests/monitor/test_types.py new file mode 100644 index 0000000000..2f3254336d --- /dev/null +++ b/tests/monitor/test_types.py @@ -0,0 +1,387 @@ +import copy +import pickle +from datetime import datetime, timezone + +import pytest +from pydantic import ValidationError +from pydantic_core import PydanticSerializationError + +from inspect_ai.monitor import ( + ComponentRef, + MonitorContext, + MonitorRecord, + ProducerRef, + SignalDerivation, + SignalValue, +) + + +def component(name: str = "openai/gpt-4o") -> ComponentRef: + return ComponentRef(name=name) + + +def producer() -> ProducerRef: + return ProducerRef(name="flight-recorder", version="0.1.0") + + +def signal() -> SignalValue: + return SignalValue( + value=0.42, + state="observed", + derivation=SignalDerivation(method="ema", alpha=0.3), + ) + + +def attempt_record() -> MonitorRecord: + return MonitorRecord( + record_id="record-1", + record_kind="attempt", + emitted_at=datetime(2026, 1, 1, tzinfo=timezone.utc), + eval_set_id=None, + run_id="run-1", + eval_id="eval-1", + sample_uuid="sample-1", + dataset_sample_id=7, + epoch=1, + attempt=1, + will_retry=False, + producer=producer(), + model=component(), + scorer=component("match"), + execution_status="completed", + scoring_status="scored", + signals={"kl_acceleration": signal()}, + ) + + +def test_signal_derivation_represents_ema_and_rolling_windows() -> None: + ema = SignalDerivation(method="ema", alpha=0.3) + rolling = SignalDerivation( + method="rolling", + window_size=20, + parameters={"distance": "wasserstein"}, + ) + + assert ema.model_dump(exclude_none=True) == { + "method": "ema", + "alpha": 0.3, + "parameters": {}, + } + assert rolling.model_dump(exclude_none=True) == { + "method": "rolling", + "window_size": 20, + "parameters": {"distance": "wasserstein"}, + } + + +def test_custom_derivation_requires_a_stable_name() -> None: + with pytest.raises(ValidationError, match="name"): + SignalDerivation(method="custom") + + custom = SignalDerivation( + method="custom", + name="flightrecorder.advantage_drift", + version="1", + parameters={"distance": "wasserstein"}, + ) + assert custom.name == "flightrecorder.advantage_drift" + + +@pytest.mark.parametrize("reserved_key", ["method", "alpha", "window_size", "name"]) +def test_derivation_parameters_cannot_shadow_schema_fields( + reserved_key: str, +) -> None: + with pytest.raises(ValidationError, match=reserved_key): + SignalDerivation(method="instant", parameters={reserved_key: 0.3}) + + +@pytest.mark.parametrize( + ("kwargs", "message"), + [ + ({"method": "ema"}, "alpha"), + ({"method": "ema", "alpha": 0.0}, "alpha"), + ({"method": "ema", "alpha": 0.3, "window_size": 20}, "window_size"), + ({"method": "rolling"}, "window_size"), + ({"method": "rolling", "window_size": 0}, "window_size"), + ({"method": "instant", "alpha": 0.3}, "alpha"), + ], +) +def test_signal_derivation_rejects_ambiguous_parameters( + kwargs: dict[str, object], message: str +) -> None: + with pytest.raises(ValidationError, match=message): + SignalDerivation(**kwargs) # type: ignore[arg-type] + + +def test_signal_value_distinguishes_observed_and_unavailable_values() -> None: + observed = signal() + unavailable = SignalValue( + value=None, + state="unavailable", + reason="No token log probabilities", + derivation=SignalDerivation(method="instant"), + ) + + assert observed.value == 0.42 + assert unavailable.model_dump(exclude_none=True) == { + "state": "unavailable", + "reason": "No token log probabilities", + "derivation": {"method": "instant", "parameters": {}}, + } + + with pytest.raises(ValidationError, match="observed"): + SignalValue(value=None, state="observed") + + with pytest.raises(ValidationError, match="must be None"): + SignalValue(value=0.42, state="error", reason="extractor failed") + + +@pytest.mark.parametrize( + "value", + [float("nan"), float("inf"), float("-inf"), {"nested": [float("nan")]}], +) +def test_non_finite_json_values_are_rejected(value: object) -> None: + with pytest.raises(ValidationError, match="finite"): + SignalValue(value=value, state="observed") # type: ignore[arg-type] + + with pytest.raises(ValidationError, match="finite"): + ProducerRef(name="monitor", config={"value": value}) # type: ignore[dict-item] + + with pytest.raises(ValidationError, match="finite"): + SignalDerivation(method="instant", parameters={"value": value}) # type: ignore[dict-item] + + +@pytest.mark.parametrize( + "secret_key", + [ + "api_key", + "apiKey", + "APIKey", + "ClientAPIKey", + "clientSecret", + "authorization", + "HTTPAuthorization", + "JWTToken", + ], +) +def test_producer_config_rejects_unredacted_credentials(secret_key: str) -> None: + with pytest.raises(ValidationError, match=secret_key): + ProducerRef(name="monitor", config={"nested": {secret_key: "secret"}}) + + producer_ref = ProducerRef( + name="monitor", config={"nested": {secret_key: "[REDACTED]"}} + ) + assert producer_ref.config["nested"] == {secret_key: "[REDACTED]"} + + with pytest.raises(ValidationError, match="Authorization"): + ProducerRef( + name="monitor", + config={"headers": [{"Authorization": "Bearer secret"}]}, + ) + + +def test_monitor_context_is_an_oracle_free_allowlist() -> None: + context = MonitorContext( + eval_set_id=None, + run_id="run-1", + eval_id="eval-1", + sample_uuid="sample-1", + epoch=1, + attempt=1, + model=component(), + ) + + assert context.attempt == 1 + assert set(MonitorContext.model_fields) == { + "eval_set_id", + "run_id", + "eval_id", + "sample_uuid", + "epoch", + "attempt", + "model", + } + + with pytest.raises(ValidationError, match="target"): + MonitorContext( + eval_set_id=None, + run_id="run-1", + eval_id="eval-1", + sample_uuid="sample-1", + epoch=1, + attempt=1, + model=component(), + target="secret", + ) # type: ignore[call-arg] + + +@pytest.mark.parametrize( + "field", + ["dataset_sample_id", "score", "scorer"], +) +def test_monitor_context_rejects_runtime_owned_or_oracle_fields(field: str) -> None: + payload = { + "eval_set_id": None, + "run_id": "run-1", + "eval_id": "eval-1", + "sample_uuid": "sample-1", + "epoch": 1, + "attempt": 1, + "model": {"name": "openai/gpt-4o"}, + field: "forbidden", + } + + with pytest.raises(ValidationError): + MonitorContext.model_validate(payload) + + +def test_attempt_record_serializes_derivation_and_runtime_status() -> None: + record = attempt_record() + payload = record.model_dump(mode="json", exclude_none=True) + + assert payload["schema_version"] == "1" + assert payload["record_kind"] == "attempt" + assert payload["dataset_sample_id"] == 7 + assert payload["signals"]["kl_acceleration"]["derivation"] == { + "method": "ema", + "alpha": 0.3, + "parameters": {}, + } + assert MonitorRecord.model_validate_json(record.model_dump_json()) == record + + +def test_record_kind_enforces_attempt_and_sample_shapes() -> None: + attempt_payload = attempt_record().model_dump() + with pytest.raises(ValidationError, match="attempt"): + MonitorRecord.model_validate(attempt_payload | {"attempt": None}) + + sample_payload = attempt_payload | { + "record_id": "record-2", + "record_kind": "sample", + "attempt": None, + "attempt_record_ids": ("record-1",), + "will_retry": None, + } + validated = MonitorRecord.model_validate(sample_payload) + assert validated.attempt_record_ids == ("record-1",) + + with pytest.raises(ValidationError, match="will_retry"): + MonitorRecord.model_validate(sample_payload | {"will_retry": False}) + + +def test_record_lifecycle_rejects_ambiguous_references_and_statuses() -> None: + attempt = attempt_record() + sample_payload = attempt.model_dump() + sample_payload.update( + record_id="sample-record", + record_kind="sample", + attempt=None, + attempt_record_ids=("record-1",), + will_retry=None, + ) + + for attempt_record_ids in [ + ("",), + ("record-1", "record-1"), + ("sample-record",), + ]: + with pytest.raises(ValidationError, match="attempt_record_ids"): + MonitorRecord.model_validate( + sample_payload | {"attempt_record_ids": attempt_record_ids} + ) + + with pytest.raises(ValidationError, match="will_retry"): + MonitorRecord.model_validate(attempt.model_dump() | {"will_retry": True}) + + with pytest.raises(ValidationError, match="scorer"): + MonitorRecord.model_validate(attempt.model_dump() | {"scorer": None}) + + with pytest.raises(ValidationError, match="attempt_record_ids"): + MonitorRecord.model_validate(sample_payload | {"attempt_record_ids": ()}) + + initialization_failure = sample_payload | { + "attempt_record_ids": (), + "execution_status": "errored", + "scoring_status": "not_run", + "scorer": None, + "status_reason": "initialization failed", + } + assert MonitorRecord.model_validate(initialization_failure).attempt_record_ids == () + + +def test_model_copy_revalidates_updates() -> None: + signal = SignalValue(value=0.42, state="observed") + with pytest.raises(ValidationError, match="finite"): + signal.model_copy(update={"value": float("nan")}) + + context = MonitorContext( + eval_set_id=None, + run_id="run-1", + eval_id="eval-1", + sample_uuid="sample-1", + epoch=1, + attempt=1, + model=component(), + ) + with pytest.raises(ValidationError, match="target"): + context.model_copy(update={"target": "oracle"}) + + with pytest.raises(ValidationError, match="will_retry"): + attempt_record().model_copy(update={"will_retry": True}) + + +def test_nested_schema_containers_cannot_be_mutated() -> None: + producer_ref = ProducerRef( + name="monitor", + config={"headers": [{"Authorization": "[REDACTED]"}]}, + ) + with pytest.raises(TypeError, match="immutable"): + producer_ref.config["api_key"] = "secret" + + headers = producer_ref.config["headers"] + assert isinstance(headers, list) + with pytest.raises(TypeError, match="immutable"): + headers.append({"Authorization": "secret"}) + + record = attempt_record() + with pytest.raises(TypeError, match="immutable"): + record.signals["late_signal"] = SignalValue(value=1.0, state="observed") + + round_trip = ProducerRef.model_validate_json(producer_ref.model_dump_json()) + assert round_trip == producer_ref + + +def test_serialization_revalidates_base_class_mutation_bypasses() -> None: + component_ref = ComponentRef(name="model") + assert component_ref.model_dump(exclude={"name"}) == {"version": None} + + producer_ref = ProducerRef( + name="monitor", + config={"headers": [{"Authorization": "[REDACTED]"}]}, + ) + dict.__setitem__(producer_ref.config, "APIKey", "secret") + with pytest.raises(PydanticSerializationError, match="redacted"): + producer_ref.model_dump_json() + + signal_value = SignalValue(value=[0.42], state="observed") + assert isinstance(signal_value.value, list) + list.append(signal_value.value, float("nan")) + with pytest.raises(PydanticSerializationError, match="finite"): + signal_value.model_dump_json() + + +def test_immutable_models_support_deepcopy_and_pickle() -> None: + producer_ref = ProducerRef( + name="monitor", + config={"nested": [{"value": 1}]}, + ) + + assert copy.deepcopy(producer_ref) == producer_ref + assert pickle.loads(pickle.dumps(producer_ref)) == producer_ref + + +def test_monitor_model_fields_cannot_be_reassigned() -> None: + record = attempt_record() + + with pytest.raises(ValidationError, match="frozen"): + record.execution_status = "errored" From 1ac7eda9e76429a2788c8a67269d98fd08344b2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ktu=C4=9F=20=C3=96zkan=2C=20MD?= Date: Sat, 11 Jul 2026 14:59:53 +0300 Subject: [PATCH 2/2] test(monitor): cover unavailable rolling warmup --- tests/monitor/test_types.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/monitor/test_types.py b/tests/monitor/test_types.py index 2f3254336d..c7201d414e 100644 --- a/tests/monitor/test_types.py +++ b/tests/monitor/test_types.py @@ -136,6 +136,33 @@ def test_signal_value_distinguishes_observed_and_unavailable_values() -> None: SignalValue(value=0.42, state="error", reason="extractor failed") +def test_rolling_signal_marks_insufficient_history_as_unavailable() -> None: + cold_start = SignalValue( + value=None, + state="unavailable", + reason="insufficient_history", + derivation=SignalDerivation(method="rolling", window_size=20), + ) + + assert cold_start.model_dump(exclude_none=True) == { + "state": "unavailable", + "reason": "insufficient_history", + "derivation": { + "method": "rolling", + "window_size": 20, + "parameters": {}, + }, + } + + with pytest.raises(ValidationError, match="must be None"): + SignalValue( + value=0.0, + state="unavailable", + reason="insufficient_history", + derivation=SignalDerivation(method="rolling", window_size=20), + ) + + @pytest.mark.parametrize( "value", [float("nan"), float("inf"), float("-inf"), {"nested": [float("nan")]}],