From ddf524dccf8edbf65fcb46e5a1a288dda2702e08 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Tue, 14 Jul 2026 18:54:37 +0000 Subject: [PATCH 1/6] chore: make ty type checking blocking Signed-off-by: Aaron Gonzales --- .github/workflows/ci.yml | 5 +- .gitignore | 2 + .pre-commit-config.yaml | 4 +- AGENTS.md | 2 +- DEVELOPMENT.md | 6 +- Makefile | 6 +- pyproject.toml | 6 +- scripts/benchmark_sync_vs_async.py | 2 +- .../engine/detection/detection_workflow.py | 98 +++++++++--------- .../engine/detection/postprocess.py | 9 +- .../engine/evaluation/detection_judge.py | 3 +- .../engine/evaluation/judge_base.py | 22 ++++- .../replace/attribute_fidelity_judge.py | 5 +- .../replace/relational_consistency_judge.py | 5 +- .../evaluation/replace/type_fidelity_judge.py | 5 +- src/anonymizer/engine/ndd/adapter.py | 2 +- src/anonymizer/engine/ndd/model_loader.py | 29 +++--- .../engine/replace/llm_replace_workflow.py | 3 +- .../engine/replace/replace_runner.py | 9 +- src/anonymizer/engine/rewrite/parsers.py | 2 +- .../engine/rewrite/workflow_utils.py | 3 +- src/anonymizer/engine/schemas/shared.py | 16 ++- .../engine/workflow_columns/detection/impl.py | 4 +- src/anonymizer/interface/anonymizer.py | 99 ++++++++++--------- src/anonymizer/interface/cli/main.py | 2 + src/anonymizer/interface/display.py | 65 +++++++----- tests/config/test_anonymizer_config.py | 2 + tests/engine/test_chunked_validation.py | 10 +- tests/engine/test_io.py | 7 +- tests/engine/test_model_loader.py | 2 +- tests/engine/test_qa_generation.py | 2 + tests/engine/test_resolved_input.py | 2 +- tests/engine/test_rewrite_generation.py | 1 + tests/interface/test_anonymizer_interface.py | 9 +- tests/interface/test_anonymizer_logging.py | 17 ++-- tests/test_debug_logging.py | 3 +- tests/test_telemetry.py | 6 +- tests/tools/test_measurement_sweeps.py | 2 +- tools/codestyle/typecheck.sh | 6 +- uv.lock | 43 ++++---- 40 files changed, 306 insertions(+), 220 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9712d5b..e84bf57e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -69,9 +69,8 @@ jobs: run: make lock-check typecheck: - name: Type Check (advisory) + name: Type Check runs-on: ubuntu-latest - continue-on-error: true steps: # Required to use the reusable action - name: Checkout repository @@ -88,4 +87,4 @@ jobs: run: uv sync --group dev - name: Run type checks - run: uv run tools/codestyle/typecheck.sh || true + run: make typecheck diff --git a/.gitignore b/.gitignore index 8c8255c7..a5bc84b5 100644 --- a/.gitignore +++ b/.gitignore @@ -102,6 +102,8 @@ AGENTS.local.md CLAUDE.local.md .claude/settings.local.json .sandbox/ +.astnav/ +.worktrees/ ai/tmp/ # Claude worktrees diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f042d2fe..9e90e332 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,9 +28,9 @@ repos: pass_filenames: true # - id: typecheck -# name: Typecheck (advisory) +# name: Typecheck # language: system -# entry: bash -c 'uv run tools/codestyle/typecheck.sh || true' +# entry: uv run --group docs tools/codestyle/typecheck.sh # types: [python] # pass_filenames: false # verbose: true diff --git a/AGENTS.md b/AGENTS.md index 766c8781..30012cfc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -115,7 +115,7 @@ make test # run all tests make bootstrap # install dev dependencies make format # ruff format + sort imports make format-check # read-only lint check (used in CI) -make typecheck # ty type check (advisory) +make typecheck # blocking ty type check (warnings fail too) make docs-serve # local MkDocs server at http://127.0.0.1:8000 ``` diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index e39e117e..ff4f71e0 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -154,13 +154,13 @@ Run all read-only checks: make check ``` -Run the advisory type checker: +Run the blocking type checker: ```bash make typecheck ``` -`ty` is currently advisory. Treat new type-check findings in touched code as review-worthy even when CI does not block on them. +`ty` checks `src`, `tests`, `tests_e2e`, `docs`, and `scripts`. Errors and warnings fail locally and in CI. Check lockfile freshness: @@ -184,7 +184,7 @@ make install-pre-commit ``` Hooks run Ruff format and lint, uv lock verification, DCO signoff checks, and basic file hygiene. `ty` is installed -for `make typecheck` and `make check`, but it is not currently run as a pre-commit hook. +for the blocking `make typecheck` and `make check` targets, but it is not currently run as a pre-commit hook. If `pyproject.toml` changes and `uv.lock` is stale, the uv-lock hook may regenerate `uv.lock` and fail the commit. Add the updated `uv.lock` and retry. diff --git a/Makefile b/Makefile index 3ddf9023..4b117e0d 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ help: @echo "" @echo " format - Format and fix code" @echo " format-check - Check format and lint (read-only)" - @echo " typecheck - Run type checks (advisory, non-blocking)" + @echo " typecheck - Run blocking type checks" @echo " copyright - Add missing SPDX headers to source files" @echo " copyright-check - Check all source files have SPDX headers (read-only)" @echo " check - Run all read-only checks" @@ -67,8 +67,8 @@ format-check: uv run tools/codestyle/ruff_check.sh typecheck: - @echo "Running type checks (advisory -- see issue tracking full compliance)..." - -uv run tools/codestyle/typecheck.sh + @echo "Running type checks..." + uv run --group docs tools/codestyle/typecheck.sh copyright: @echo "Adding missing SPDX headers..." diff --git a/pyproject.toml b/pyproject.toml index 300be61c..37cf3331 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ dev = [ "pytest>=9.0.3,<10", "pytest-cov>=7.0,<8", "ruff>=0.12.3,<1", - "ty>=0.0.23,<0.1", + "ty==0.0.58", {include-group = "measurement"}, ] measurement = [ @@ -88,11 +88,11 @@ markers = [ ] [tool.ty.src] -root = "src" -exclude = ["tools/"] +include = ["src", "tests", "tests_e2e", "docs", "scripts"] [tool.ty.environment] python-version = "3.11" +root = ["src", ".", "tools/measurement"] [tool.coverage.run] omit = [] diff --git a/scripts/benchmark_sync_vs_async.py b/scripts/benchmark_sync_vs_async.py index 5e8866f4..416d4d39 100644 --- a/scripts/benchmark_sync_vs_async.py +++ b/scripts/benchmark_sync_vs_async.py @@ -268,7 +268,7 @@ def finalize(row: dict[str, Any]) -> dict[str, Any]: artifact_storage=artifact_storage, model_configs=builder.model_configs, secret_resolver=CompositeResolver([EnvironmentResolver(), PlaintextResolver()]), - model_provider_registry=resolve_model_provider_registry([provider], default_provider_name=provider.name), + model_provider_registry=resolve_model_provider_registry([provider]), seed_reader_registry=SeedReaderRegistry(readers=[]), seed_dataset_source=None, run_config=run_config, diff --git a/src/anonymizer/engine/detection/detection_workflow.py b/src/anonymizer/engine/detection/detection_workflow.py index c01ff781..a577a47b 100644 --- a/src/anonymizer/engine/detection/detection_workflow.py +++ b/src/anonymizer/engine/detection/detection_workflow.py @@ -7,6 +7,7 @@ from copy import deepcopy from dataclasses import dataclass from pathlib import Path +from typing import cast import pandas as pd from data_designer.config.column_configs import LLMStructuredColumnConfig, LLMTextColumnConfig @@ -174,54 +175,57 @@ def _build_detection_spec( validator_aliases, ) - columns: list[ColumnConfigT] = [ - LLMTextColumnConfig( - name=COL_RAW_DETECTED, - prompt=_jinja(COL_TEXT), - model_alias=detection_alias, - ), - DetectionTransformConfig( - name=COL_SEED_ENTITIES, - operation=DetectionTransformOperation.PARSE_DETECTED_ENTITIES, - ), - DetectionTransformConfig( - name=COL_SEED_VALIDATION_CANDIDATES, - operation=DetectionTransformOperation.PREPARE_VALIDATION_INPUTS, - ), - ChunkedValidationConfig( - name=COL_VALIDATION_DECISIONS, - pool=list(validator_aliases), - max_entities_per_call=validation_max_entities_per_call, - excerpt_window_chars=validation_excerpt_window_chars, - single_chunk_full_text=validation_single_chunk_full_text, - prompt_template=_get_validation_prompt(data_summary=data_summary, labels=labels), - drop=True, - ), - DetectionTransformConfig( - name=COL_VALIDATED_ENTITIES, - operation=DetectionTransformOperation.ENRICH_VALIDATION_DECISIONS, - ), - DetectionTransformConfig( - name=COL_SEED_ENTITIES_JSON, - operation=DetectionTransformOperation.APPLY_VALIDATION_TO_SEED_ENTITIES, - ), - LLMStructuredColumnConfig( - name=COL_AUGMENTED_ENTITIES, - prompt=_get_augment_prompt( - data_summary=data_summary, labels=labels, strict_labels=entity_labels is not None + columns = cast( + list[ColumnConfigT], + [ + LLMTextColumnConfig( + name=COL_RAW_DETECTED, + prompt=_jinja(COL_TEXT), + model_alias=detection_alias, ), - model_alias=augmenter_alias, - output_format=AugmentedEntitiesSchema, - ), - DetectionTransformConfig( - name=COL_MERGED_ENTITIES, - operation=DetectionTransformOperation.MERGE_AND_BUILD_CANDIDATES, - ), - DetectionTransformConfig( - name=COL_DETECTED_ENTITIES, - operation=DetectionTransformOperation.APPLY_VALIDATION_AND_FINALIZE, - ), - ] + DetectionTransformConfig( + name=COL_SEED_ENTITIES, + operation=DetectionTransformOperation.PARSE_DETECTED_ENTITIES, + ), + DetectionTransformConfig( + name=COL_SEED_VALIDATION_CANDIDATES, + operation=DetectionTransformOperation.PREPARE_VALIDATION_INPUTS, + ), + ChunkedValidationConfig( + name=COL_VALIDATION_DECISIONS, + pool=list(validator_aliases), + max_entities_per_call=validation_max_entities_per_call, + excerpt_window_chars=validation_excerpt_window_chars, + single_chunk_full_text=validation_single_chunk_full_text, + prompt_template=_get_validation_prompt(data_summary=data_summary, labels=labels), + drop=True, + ), + DetectionTransformConfig( + name=COL_VALIDATED_ENTITIES, + operation=DetectionTransformOperation.ENRICH_VALIDATION_DECISIONS, + ), + DetectionTransformConfig( + name=COL_SEED_ENTITIES_JSON, + operation=DetectionTransformOperation.APPLY_VALIDATION_TO_SEED_ENTITIES, + ), + LLMStructuredColumnConfig( + name=COL_AUGMENTED_ENTITIES, + prompt=_get_augment_prompt( + data_summary=data_summary, labels=labels, strict_labels=entity_labels is not None + ), + model_alias=augmenter_alias, + output_format=AugmentedEntitiesSchema, + ), + DetectionTransformConfig( + name=COL_MERGED_ENTITIES, + operation=DetectionTransformOperation.MERGE_AND_BUILD_CANDIDATES, + ), + DetectionTransformConfig( + name=COL_DETECTED_ENTITIES, + operation=DetectionTransformOperation.APPLY_VALIDATION_AND_FINALIZE, + ), + ], + ) return workflow_model_configs, columns def build_detection_config( diff --git a/src/anonymizer/engine/detection/postprocess.py b/src/anonymizer/engine/detection/postprocess.py index 9e3557e3..20c2e515 100644 --- a/src/anonymizer/engine/detection/postprocess.py +++ b/src/anonymizer/engine/detection/postprocess.py @@ -8,6 +8,7 @@ import re from dataclasses import dataclass from enum import Enum +from typing import SupportsFloat, SupportsIndex, SupportsInt logger = logging.getLogger(__name__) @@ -368,15 +369,19 @@ def _safe_json_loads(value: dict | str) -> dict: def _coerce_int(value: object) -> int | None: + if not isinstance(value, (str, bytes, bytearray, SupportsInt, SupportsIndex)): + return None try: - return int(value) # type: ignore[arg-type] + return int(value) except (TypeError, ValueError): return None def _coerce_float(value: object, default: float) -> float: + if not isinstance(value, (str, bytes, bytearray, SupportsFloat, SupportsIndex)): + return default try: - return float(value) # type: ignore[arg-type] + return float(value) except (TypeError, ValueError): return default diff --git a/src/anonymizer/engine/evaluation/detection_judge.py b/src/anonymizer/engine/evaluation/detection_judge.py index 4dc888cf..0c5d5033 100644 --- a/src/anonymizer/engine/evaluation/detection_judge.py +++ b/src/anonymizer/engine/evaluation/detection_judge.py @@ -5,7 +5,7 @@ import json import logging -from typing import ClassVar +from typing import ClassVar, cast import pandas as pd from pydantic import BaseModel, Field @@ -227,4 +227,5 @@ def _build_prompt(cls) -> str: @classmethod def _extract_invalid(cls, parsed: BaseModel) -> list[dict[str, object]]: + parsed = cast(DetectionJudgmentSchema, parsed) return [entry.model_dump() for entry in parsed.invalid_entities] diff --git a/src/anonymizer/engine/evaluation/judge_base.py b/src/anonymizer/engine/evaluation/judge_base.py index 5367764f..357293ea 100644 --- a/src/anonymizer/engine/evaluation/judge_base.py +++ b/src/anonymizer/engine/evaluation/judge_base.py @@ -19,15 +19,16 @@ import logging from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import ClassVar +from typing import Any, ClassVar, Protocol, cast import pandas as pd from data_designer.config.column_configs import LLMStructuredColumnConfig +from data_designer.config.column_types import ColumnConfigT from data_designer.config.models import ModelConfig from pydantic import BaseModel from anonymizer.config.models import EvaluateModelSelection -from anonymizer.engine.ndd.adapter import FailedRecord, NddAdapter +from anonymizer.engine.ndd.adapter import FailedRecord from anonymizer.engine.ndd.model_loader import resolve_model_alias from anonymizer.engine.row_partitioning import ROW_ORDER_COL, merge_and_reorder @@ -42,6 +43,19 @@ class JudgeResult: failed_records: list[FailedRecord] +class _JudgeAdapter(Protocol): + def run_workflow( + self, + dataframe: pd.DataFrame, + /, + *, + model_configs: list[ModelConfig], + columns: list[ColumnConfigT], + workflow_name: str, + preview_num_records: int | None = None, + ) -> Any: ... + + class _BaseJudgeWorkflow(ABC): """Common scaffolding for the four LLM-as-judge workflows.""" @@ -63,7 +77,7 @@ class _BaseJudgeWorkflow(ABC): # Logical workflow name surfaced in logs and FailedRecord entries. WORKFLOW_NAME: ClassVar[str] - def __init__(self, adapter: NddAdapter) -> None: + def __init__(self, adapter: _JudgeAdapter) -> None: self._adapter = adapter # ------------------------------------------------------------------ hooks @@ -191,7 +205,7 @@ def evaluate( run_result = self._adapter.run_workflow( with_content, model_configs=model_configs, - columns=[self.column_config(selected_models)], + columns=cast(list[ColumnConfigT], [self.column_config(selected_models)]), workflow_name=self.WORKFLOW_NAME, preview_num_records=effective_preview_num_records, ) diff --git a/src/anonymizer/engine/evaluation/replace/attribute_fidelity_judge.py b/src/anonymizer/engine/evaluation/replace/attribute_fidelity_judge.py index 3dd67692..4bd9778c 100644 --- a/src/anonymizer/engine/evaluation/replace/attribute_fidelity_judge.py +++ b/src/anonymizer/engine/evaluation/replace/attribute_fidelity_judge.py @@ -6,7 +6,7 @@ import json import logging from datetime import datetime -from typing import ClassVar +from typing import ClassVar, cast import pandas as pd from pydantic import BaseModel, Field @@ -184,7 +184,7 @@ def _replacements_for_judge(raw_map: object) -> list[dict[str, str]]: """Flatten COL_REPLACEMENT_MAP into Jinja-friendly dicts.""" if raw_map is None: return [] - if hasattr(raw_map, "model_dump"): + if isinstance(raw_map, BaseModel): raw_map = raw_map.model_dump(mode="python") if isinstance(raw_map, str): try: @@ -238,4 +238,5 @@ def _build_prompt(cls) -> str: @classmethod def _extract_invalid(cls, parsed: BaseModel) -> list[dict[str, object]]: + parsed = cast(AttributeFidelityJudgmentSchema, parsed) return [e.model_dump() for e in parsed.entities if not e.passes] diff --git a/src/anonymizer/engine/evaluation/replace/relational_consistency_judge.py b/src/anonymizer/engine/evaluation/replace/relational_consistency_judge.py index ae190d3e..76f3e936 100644 --- a/src/anonymizer/engine/evaluation/replace/relational_consistency_judge.py +++ b/src/anonymizer/engine/evaluation/replace/relational_consistency_judge.py @@ -5,7 +5,7 @@ import json import logging -from typing import ClassVar +from typing import ClassVar, cast import pandas as pd from pydantic import BaseModel, Field @@ -241,7 +241,7 @@ def _replacements_for_judge(raw_map: object) -> list[dict[str, str]]: """Flatten COL_REPLACEMENT_MAP into Jinja-friendly dicts.""" if raw_map is None: return [] - if hasattr(raw_map, "model_dump"): + if isinstance(raw_map, BaseModel): raw_map = raw_map.model_dump(mode="python") if isinstance(raw_map, str): try: @@ -296,4 +296,5 @@ def _build_prompt(cls) -> str: @classmethod def _extract_invalid(cls, parsed: BaseModel) -> list[dict[str, object]]: + parsed = cast(RelationalConsistencyJudgmentSchema, parsed) return [r.model_dump() for r in parsed.relations if not r.passes] diff --git a/src/anonymizer/engine/evaluation/replace/type_fidelity_judge.py b/src/anonymizer/engine/evaluation/replace/type_fidelity_judge.py index 8fe2a113..03819865 100644 --- a/src/anonymizer/engine/evaluation/replace/type_fidelity_judge.py +++ b/src/anonymizer/engine/evaluation/replace/type_fidelity_judge.py @@ -5,7 +5,7 @@ import json import logging -from typing import ClassVar +from typing import ClassVar, cast import pandas as pd from pydantic import BaseModel, Field @@ -234,7 +234,7 @@ def _replacements_for_judge(raw_map: object) -> list[dict[str, str]]: """ if raw_map is None: return [] - if hasattr(raw_map, "model_dump"): + if isinstance(raw_map, BaseModel): raw_map = raw_map.model_dump(mode="python") if isinstance(raw_map, str): try: @@ -298,4 +298,5 @@ def _build_prompt(cls) -> str: @classmethod def _extract_invalid(cls, parsed: BaseModel) -> list[dict[str, object]]: + parsed = cast(TypeFidelityJudgmentSchema, parsed) return [entry.model_dump() for entry in parsed.invalid_replacements] diff --git a/src/anonymizer/engine/ndd/adapter.py b/src/anonymizer/engine/ndd/adapter.py index 4dbc4ae9..0de78a36 100644 --- a/src/anonymizer/engine/ndd/adapter.py +++ b/src/anonymizer/engine/ndd/adapter.py @@ -1034,7 +1034,7 @@ def traced_generator(*args: Any, **kwargs: Any) -> Any: finally: _MODEL_TRACE_COLUMN.reset(token) - traced_generator.custom_column_metadata = getattr(generator, "custom_column_metadata", {}) # type: ignore[attr-defined] + setattr(traced_generator, "custom_column_metadata", getattr(generator, "custom_column_metadata", {})) return cast(ColumnConfigT, column.model_copy(update={"generator_function": traced_generator})) diff --git a/src/anonymizer/engine/ndd/model_loader.py b/src/anonymizer/engine/ndd/model_loader.py index 123c4be9..ce27f93d 100644 --- a/src/anonymizer/engine/ndd/model_loader.py +++ b/src/anonymizer/engine/ndd/model_loader.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from enum import Enum from pathlib import Path -from typing import Any +from typing import Any, cast from data_designer.config.models import ModelConfig, ModelProvider, load_model_configs from data_designer.config.utils.io_helpers import load_config_file @@ -48,7 +48,7 @@ def parse_model_configs(raw: str | Path | None) -> ParsedModelConfigs: if raw is None: default_yaml = _load_yaml_dict(DEFAULT_CONFIG_DIR / "models.yaml") return ParsedModelConfigs( - model_configs=load_model_configs(default_yaml), + model_configs=load_model_configs(default_yaml), # ty: ignore[invalid-argument-type] selected_models=load_default_model_selection(), ) @@ -63,7 +63,7 @@ def parse_model_configs(raw: str | Path | None) -> ParsedModelConfigs: user_selections = parsed.pop("selected_models", None) _validate_raw_model_configs_have_provider(parsed) - model_configs = load_model_configs(parsed) + model_configs = load_model_configs(parsed) # ty: ignore[invalid-argument-type] return ParsedModelConfigs( model_configs=model_configs, selected_models=_merge_selections(user_selections), @@ -77,10 +77,12 @@ def load_default_model_selection(config_dir: Path | None = None) -> ModelSelecti """ resolved_dir = config_dir or DEFAULT_CONFIG_DIR return ModelSelection( - detection=DetectionModelSelection(**load_workflow_selections(WorkflowName.detection, resolved_dir)), - replace=ReplaceModelSelection(**load_workflow_selections(WorkflowName.replace, resolved_dir)), - rewrite=RewriteModelSelection(**load_workflow_selections(WorkflowName.rewrite, resolved_dir)), - evaluate=EvaluateModelSelection(**load_workflow_selections(WorkflowName.evaluate, resolved_dir)), + detection=DetectionModelSelection.model_validate( + load_workflow_selections(WorkflowName.detection, resolved_dir) + ), + replace=ReplaceModelSelection.model_validate(load_workflow_selections(WorkflowName.replace, resolved_dir)), + rewrite=RewriteModelSelection.model_validate(load_workflow_selections(WorkflowName.rewrite, resolved_dir)), + evaluate=EvaluateModelSelection.model_validate(load_workflow_selections(WorkflowName.evaluate, resolved_dir)), ) @@ -197,8 +199,12 @@ def resolve_model_aliases( """ value = getattr(selection_model, role) if isinstance(value, list): - return list(value) - return [value] + if all(isinstance(alias, str) for alias in value): + return value + raise TypeError(f"Role {role!r} contains a non-string model alias.") + if isinstance(value, str): + return [value] + raise TypeError(f"Role {role!r} does not contain a model alias.") def _merge_selections(user_selections: dict[str, dict[str, str]] | None) -> ModelSelection: @@ -359,8 +365,9 @@ def _validate_raw_model_configs_have_provider(parsed: dict[str, Any]) -> None: for idx, entry in enumerate(raw_configs): if not isinstance(entry, dict): raise ValueError(f"model_configs[{idx}] must be a mapping.") - if _provider_field_missing(entry): - missing.append(str(entry.get("alias", f""))) + typed_entry = cast(dict[str, Any], entry) + if _provider_field_missing(typed_entry): + missing.append(str(typed_entry.get("alias", f""))) if missing: aliases = ", ".join(repr(alias) for alias in missing) raise ValueError( diff --git a/src/anonymizer/engine/replace/llm_replace_workflow.py b/src/anonymizer/engine/replace/llm_replace_workflow.py index 531b6827..0a24c582 100644 --- a/src/anonymizer/engine/replace/llm_replace_workflow.py +++ b/src/anonymizer/engine/replace/llm_replace_workflow.py @@ -11,6 +11,7 @@ import pandas as pd from data_designer.config.column_configs import LLMStructuredColumnConfig from data_designer.config.models import ModelConfig +from pydantic import BaseModel from anonymizer.config.models import ReplaceModelSelection from anonymizer.engine.constants import ( @@ -146,7 +147,7 @@ def _filter_replacement_map_to_input_entities( record_id: str = "", ) -> dict[str, list[dict[str, str]]]: """Keep only replacement entries that correspond to actual requested entities.""" - if hasattr(raw_map, "model_dump"): + if isinstance(raw_map, BaseModel): raw_map = raw_map.model_dump(mode="python") if not isinstance(raw_map, dict): logger.warning( diff --git a/src/anonymizer/engine/replace/replace_runner.py b/src/anonymizer/engine/replace/replace_runner.py index f3a95adc..e3ba6c8b 100644 --- a/src/anonymizer/engine/replace/replace_runner.py +++ b/src/anonymizer/engine/replace/replace_runner.py @@ -5,6 +5,7 @@ import logging from dataclasses import dataclass +from typing import cast import pandas as pd from data_designer.config.models import ModelConfig @@ -22,6 +23,7 @@ COL_REPLACEMENT_MAP, ) from anonymizer.engine.evaluation.detection_judge import DetectionJudgeWorkflow +from anonymizer.engine.evaluation.judge_base import _BaseJudgeWorkflow from anonymizer.engine.evaluation.replace.attribute_fidelity_judge import AttributeFidelityJudgeWorkflow from anonymizer.engine.evaluation.replace.relational_consistency_judge import RelationalConsistencyJudgeWorkflow from anonymizer.engine.evaluation.replace.type_fidelity_judge import TypeFidelityJudgeWorkflow @@ -173,7 +175,7 @@ def _run_merged_judges( + apply passthrough defaults). The adapter sees one workflow with N columns and lets DD schedule them in parallel. """ - active = [j for j in [self._detection_judge] if j is not None] + active: list[_BaseJudgeWorkflow] = [j for j in [self._detection_judge] if j is not None] if is_substitute: active.extend( j @@ -195,10 +197,11 @@ def _run_merged_judges( # these itself; doing it here lets us treat evaluation as non-critical: # rows the LLM drops still appear in the result with "Unavailable" # verdicts instead of disappearing from a previously successful run. - prepared = self._adapter._attach_record_ids(prepared) # type: ignore[union-attr] + adapter = cast(NddAdapter, self._adapter) + prepared = adapter._attach_record_ids(prepared) try: - run_result = self._adapter.run_workflow( # type: ignore[union-attr] + run_result = adapter.run_workflow( prepared, model_configs=model_configs, columns=[judge.column_config(selected_models) for judge in active], diff --git a/src/anonymizer/engine/rewrite/parsers.py b/src/anonymizer/engine/rewrite/parsers.py index 05f88b02..b8ace228 100644 --- a/src/anonymizer/engine/rewrite/parsers.py +++ b/src/anonymizer/engine/rewrite/parsers.py @@ -25,7 +25,7 @@ logger = logging.getLogger("anonymizer.rewrite.parsers") -def field(model: type, name: str) -> str: +def field(model: type[BaseModel], name: str) -> str: """Return *name* after verifying it exists on *model* as a Pydantic field. Called at module import time so a renamed schema field raises diff --git a/src/anonymizer/engine/rewrite/workflow_utils.py b/src/anonymizer/engine/rewrite/workflow_utils.py index 6f29318d..43adc5f0 100644 --- a/src/anonymizer/engine/rewrite/workflow_utils.py +++ b/src/anonymizer/engine/rewrite/workflow_utils.py @@ -4,6 +4,7 @@ from __future__ import annotations import json +from collections.abc import Sequence import pandas as pd from data_designer.config.column_types import ColumnConfigT @@ -12,7 +13,7 @@ def derive_seed_columns( - columns: list[ColumnConfigT], + columns: Sequence[ColumnConfigT], df: pd.DataFrame, ) -> list[str]: """Compute the minimal seed column set from column config dependencies. diff --git a/src/anonymizer/engine/schemas/shared.py b/src/anonymizer/engine/schemas/shared.py index f2eae732..3eb1e7fb 100644 --- a/src/anonymizer/engine/schemas/shared.py +++ b/src/anonymizer/engine/schemas/shared.py @@ -4,13 +4,21 @@ from __future__ import annotations import json -from typing import TypeVar +from typing import Protocol, TypeGuard, TypeVar, cast from pydantic import BaseModel, ValidationError T = TypeVar("T", bound=BaseModel) +class _ListConvertible(Protocol): + def tolist(self) -> object: ... + + +def _is_list_convertible(value: object) -> TypeGuard[_ListConvertible]: + return callable(getattr(value, "tolist", None)) + + def _parse_raw_wrapper( model_cls: type[T], raw: object, @@ -47,9 +55,9 @@ def _safe_validate(candidate_list: list[object]) -> T: if isinstance(candidate, BaseModel): candidate = candidate.model_dump(mode="python") if isinstance(candidate, list): - return _safe_validate(candidate) - if hasattr(candidate, "tolist"): + return _safe_validate(cast(list[object], candidate)) + if _is_list_convertible(candidate): as_list = candidate.tolist() if isinstance(as_list, list): - return _safe_validate(as_list) + return _safe_validate(cast(list[object], as_list)) return model_cls() diff --git a/src/anonymizer/engine/workflow_columns/detection/impl.py b/src/anonymizer/engine/workflow_columns/detection/impl.py index 7b0dae38..54a92405 100644 --- a/src/anonymizer/engine/workflow_columns/detection/impl.py +++ b/src/anonymizer/engine/workflow_columns/detection/impl.py @@ -119,12 +119,12 @@ def _params(self, models: dict[str, Any]) -> ChunkedValidationParams: system_prompt=self.config.system_prompt, ) - def generate(self, data: dict[str, Any]) -> dict[str, Any]: + def generate(self, data: dict[str, Any]) -> dict[str, Any]: # ty: ignore[invalid-method-override] raw_models = {alias: self.get_model(alias) for alias in self.config.pool} models = {alias: _AsyncBridgedModelFacade(model) for alias, model in raw_models.items()} return chunked_validate_row(data, self._params(raw_models), models) - async def agenerate(self, data: dict[str, Any]) -> dict[str, Any]: + async def agenerate(self, data: dict[str, Any]) -> dict[str, Any]: # ty: ignore[invalid-method-override] models = {alias: self.get_model(alias) for alias in self.config.pool} return await chunked_validate_row_async(data, self._params(models), models) diff --git a/src/anonymizer/interface/anonymizer.py b/src/anonymizer/interface/anonymizer.py index bcfee0c5..e31d01a9 100644 --- a/src/anonymizer/interface/anonymizer.py +++ b/src/anonymizer/interface/anonymizer.py @@ -21,6 +21,7 @@ AnonymizerConfig, AnonymizerInput, EvaluateConfig, + Rewrite, ) from anonymizer.config.replace_strategies import Substitute from anonymizer.engine.constants import ( @@ -593,12 +594,14 @@ def _run_internal_impl( elif config.rewrite is not None: logger.info("✏️ Running rewrite pipeline") t0 = time.perf_counter() + privacy_goal = config.rewrite.privacy_goal + assert privacy_goal is not None # populated by Rewrite's model validator rewrite_result = self._rewrite_runner.run( detection_result.dataframe, model_configs=self._model_configs, selected_models=self._selected_models.rewrite, replace_model_selection=self._selected_models.replace, - privacy_goal=config.rewrite.privacy_goal, + privacy_goal=privacy_goal, evaluation=config.rewrite.evaluation, data_summary=data.data_summary, preview_num_records=preview_num_records, @@ -730,57 +733,63 @@ def _build_telemetry_event( failure_counts = _collect_failure_counts(failed) hosts = _resolve_model_hosts(self._resolved_providers) - return AnonymizerEvent( - task=task, - task_status=status, - job_duration_sec=duration_sec, - num_input_records=total_records, - num_success_records=success_count, - num_failure_records=failure_count, - avg_tokens_per_record=avg_tokens, - transformation_type=transformation_type, - custom_data_summary_provided=bool(data.data_summary), - custom_privacy_goal_provided=_custom_privacy_goal_provided(rewrite), - custom_substitute_instructions_provided=bool(substitute is not None and substitute.instructions), - max_repair_iterations=(rewrite.max_repair_iterations if rewrite is not None else -1), - strict_entity_protection=(rewrite.strict_entity_protection if rewrite is not None else False), - repair_iterations_triggered=_repair_iterations_triggered(failed, rewrite is not None), - entity_detector_model=models["entity_detector"], - entity_validator_model=models["entity_validator"], - entity_augmenter_model=models["entity_augmenter"], - latent_detector_model=models["latent_detector"], - replacement_generator_model=models["replacement_generator"], - domain_classifier_model=models["domain_classifier"], - disposition_analyzer_model=models["disposition_analyzer"], - meaning_extractor_model=models["meaning_extractor"], - qa_generator_model=models["qa_generator"], - rewriter_model=models["rewriter"], - evaluator_model=models["evaluator"], - repairer_model=models["repairer"], - model_hosts=hosts, - entity_detection_failure_count=failure_counts["entity_detection"], - latent_detection_failure_count=failure_counts["latent_detection"], - replace_map_generation_failure_count=failure_counts["replace_map_generation"], - rewrite_pipeline_failure_count=failure_counts["rewrite_pipeline"], - rewrite_evaluate_failure_count=failure_counts["rewrite_evaluate"], - rewrite_repair_failure_count=failure_counts["rewrite_repair"], - rewrite_final_judge_failure_count=failure_counts["rewrite_final_judge"], - unknown_step_failure_count=failure_counts["unknown"], + return AnonymizerEvent.model_validate( + { + "task": task, + "task_status": status, + "job_duration_sec": duration_sec, + "num_input_records": total_records, + "num_success_records": success_count, + "num_failure_records": failure_count, + "avg_tokens_per_record": avg_tokens, + "transformation_type": transformation_type, + "custom_data_summary_provided": bool(data.data_summary), + "custom_privacy_goal_provided": _custom_privacy_goal_provided(rewrite), + "custom_substitute_instructions_provided": bool(substitute is not None and substitute.instructions), + "max_repair_iterations": rewrite.max_repair_iterations if rewrite is not None else -1, + "strict_entity_protection": rewrite.strict_entity_protection if rewrite is not None else False, + "repair_iterations_triggered": _repair_iterations_triggered(failed, rewrite is not None), + "entity_detector_model": models["entity_detector"], + "entity_validator_model": models["entity_validator"], + "entity_augmenter_model": models["entity_augmenter"], + "latent_detector_model": models["latent_detector"], + "replacement_generator_model": models["replacement_generator"], + "domain_classifier_model": models["domain_classifier"], + "disposition_analyzer_model": models["disposition_analyzer"], + "meaning_extractor_model": models["meaning_extractor"], + "qa_generator_model": models["qa_generator"], + "rewriter_model": models["rewriter"], + "evaluator_model": models["evaluator"], + "repairer_model": models["repairer"], + "model_hosts": hosts, + "entity_detection_failure_count": failure_counts["entity_detection"], + "latent_detection_failure_count": failure_counts["latent_detection"], + "replace_map_generation_failure_count": failure_counts["replace_map_generation"], + "rewrite_pipeline_failure_count": failure_counts["rewrite_pipeline"], + "rewrite_evaluate_failure_count": failure_counts["rewrite_evaluate"], + "rewrite_repair_failure_count": failure_counts["rewrite_repair"], + "rewrite_final_judge_failure_count": failure_counts["rewrite_final_judge"], + "unknown_step_failure_count": failure_counts["unknown"], + } ) def _unwrap_entities(raw: object) -> list: if isinstance(raw, dict): - return raw.get("entities", []) + entities = raw.get("entities", []) + return entities if isinstance(entities, list) else [] if isinstance(raw, list): return raw - return getattr(raw, "entities", []) + entities = getattr(raw, "entities", []) + return entities if isinstance(entities, list) else [] def _entity_label(entity: object) -> str | None: if isinstance(entity, dict): - return entity.get("label") - return getattr(entity, "label", None) + label = entity.get("label") + else: + label = getattr(entity, "label", None) + return label if isinstance(label, str) else None def _count_labels_for_row(raw: object) -> Counter[str]: @@ -820,7 +829,7 @@ def _resolve_model_providers( if not candidate.is_file(): raise FileNotFoundError(f"Providers config file not found: {candidate}") model_providers = candidate - config_dict = load_config_file(model_providers) + config_dict = load_config_file(model_providers) # ty: ignore[invalid-argument-type] raw_providers = config_dict.get("providers") if not isinstance(raw_providers, list): raise ValueError("model_providers YAML must contain a top-level 'providers' list.") @@ -940,17 +949,17 @@ def _transformation_type_string(config: AnonymizerConfig) -> str: return type(config.replace).__name__.lower() -def _custom_privacy_goal_provided(rewrite: object | None) -> bool: +def _custom_privacy_goal_provided(rewrite: Rewrite | None) -> bool: """Detect whether the user supplied a non-default privacy_goal. ``Rewrite.populate_default_privacy_goal`` always populates a default if the user passed None, so we treat the default protect/preserve text as "not custom". """ - if rewrite is None or rewrite.privacy_goal is None: # type: ignore[union-attr] + if rewrite is None or rewrite.privacy_goal is None: return False from anonymizer.config.rewrite import DEFAULT_PRESERVE_TEXT, DEFAULT_PROTECT_TEXT - goal = rewrite.privacy_goal # type: ignore[union-attr] + goal = rewrite.privacy_goal return goal.protect != DEFAULT_PROTECT_TEXT or goal.preserve != DEFAULT_PRESERVE_TEXT diff --git a/src/anonymizer/interface/cli/main.py b/src/anonymizer/interface/cli/main.py index d6490d2f..54c1b79a 100644 --- a/src/anonymizer/interface/cli/main.py +++ b/src/anonymizer/interface/cli/main.py @@ -170,6 +170,8 @@ def _build_replace_strategy(opts: CliOpts) -> Substitute | Redact | Hash | Annot Only passes non-default kwargs so Pydantic field defaults are preserved. """ + if opts.replace is None: + raise ValueError("A replacement strategy is required in replace mode.") cls = _STRATEGY_CLS[opts.replace] kw: dict = {} if opts.format_template is not None and "format_template" in cls.model_fields: diff --git a/src/anonymizer/interface/display.py b/src/anonymizer/interface/display.py index 5cdf53ae..59e29fb6 100644 --- a/src/anonymizer/interface/display.py +++ b/src/anonymizer/interface/display.py @@ -8,8 +8,10 @@ import logging import re from dataclasses import dataclass +from typing import Protocol, TypeGuard, cast import pandas as pd +from pydantic import BaseModel from anonymizer.engine.constants import ( COL_ATTRIBUTE_FIDELITY_INVALID_ENTITIES, @@ -38,6 +40,15 @@ logger = logging.getLogger(__name__) + +class _ListConvertible(Protocol): + def tolist(self) -> object: ... + + +def _is_list_convertible(value: object) -> TypeGuard[_ListConvertible]: + return callable(getattr(value, "tolist", None)) + + ENTITY_COLORS: list[str] = [ "#dbeafe", # blue "#dcfce7", # green @@ -385,7 +396,7 @@ def _normalize_replacement_map(raw: str | dict | object) -> list[dict[str, str]] """ if raw is None: return [] - if hasattr(raw, "model_dump"): + if isinstance(raw, BaseModel): raw = raw.model_dump(mode="python") if isinstance(raw, str): try: @@ -400,16 +411,16 @@ def _normalize_replacement_map(raw: str | dict | object) -> list[dict[str, str]] except Exception: pass replacements = raw.get("replacements", []) - if hasattr(replacements, "tolist"): + if _is_list_convertible(replacements): replacements = replacements.tolist() if not isinstance(replacements, list): return [] result: list[dict[str, str]] = [] for r in replacements: - if hasattr(r, "model_dump"): + if isinstance(r, BaseModel): r = r.model_dump() if isinstance(r, dict): - result.append(r) + result.append(cast(dict[str, str], r)) return result @@ -535,8 +546,8 @@ def _extract_judge_scores(raw: object) -> list[tuple[str, int | str]]: for name, value in raw.items(): if not isinstance(value, dict) or "score" not in value: continue - score = value["score"] - if score is None: + score = cast(dict[str, object], value)["score"] + if not isinstance(score, (int, str)): continue result.append((str(name), score)) return result @@ -740,7 +751,7 @@ def _render_attribute_entries_table(entries: list[dict[str, object]]) -> str: label = html.escape(str(entry.get("label", ""))) synthetic = html.escape(str(entry.get("synthetic", ""))) attrs = entry.get("attributes_checked", []) - if hasattr(attrs, "tolist"): + if _is_list_convertible(attrs): attrs = attrs.tolist() if not isinstance(attrs, list): attrs = [] @@ -784,7 +795,7 @@ def _extract_all_attribute_entries(row: pd.Series) -> list[dict[str, object]]: raw = row.get(COL_ATTRIBUTE_FIDELITY_JUDGE) if COL_ATTRIBUTE_FIDELITY_JUDGE in row.index else None if raw is None: return [] - if hasattr(raw, "model_dump"): + if isinstance(raw, BaseModel): raw = raw.model_dump(mode="python") if isinstance(raw, str): try: @@ -794,16 +805,16 @@ def _extract_all_attribute_entries(row: pd.Series) -> list[dict[str, object]]: if not isinstance(raw, dict): return [] entities = raw.get("entities", []) - if hasattr(entities, "tolist"): + if _is_list_convertible(entities): entities = entities.tolist() if not isinstance(entities, list): return [] out: list[dict[str, object]] = [] for entry in entities: - if hasattr(entry, "model_dump"): + if isinstance(entry, BaseModel): entry = entry.model_dump() if isinstance(entry, dict): - out.append(entry) + out.append(cast(dict[str, object], entry)) return out @@ -816,16 +827,16 @@ def _normalize_attribute_entries(raw: object) -> list[dict[str, object]]: raw = json.loads(raw) except (json.JSONDecodeError, ValueError): return [] - if hasattr(raw, "tolist"): + if _is_list_convertible(raw): raw = raw.tolist() if not isinstance(raw, list): return [] out: list[dict[str, object]] = [] for entry in raw: - if hasattr(entry, "model_dump"): + if isinstance(entry, BaseModel): entry = entry.model_dump() if isinstance(entry, dict): - out.append(entry) + out.append(cast(dict[str, object], entry)) return out @@ -876,7 +887,7 @@ def _render_relations_table(relations: list[dict[str, object]]) -> str: for entry in relations: description = html.escape(str(entry.get("description", ""))) entities = entry.get("entities", []) - if hasattr(entities, "tolist"): + if _is_list_convertible(entities): entities = entities.tolist() if not isinstance(entities, list): entities = [] @@ -924,7 +935,7 @@ def _extract_all_relations(row: pd.Series) -> list[dict[str, object]]: raw = row.get(COL_RELATIONAL_CONSISTENCY_JUDGE) if COL_RELATIONAL_CONSISTENCY_JUDGE in row.index else None if raw is None: return [] - if hasattr(raw, "model_dump"): + if isinstance(raw, BaseModel): raw = raw.model_dump(mode="python") if isinstance(raw, str): try: @@ -934,16 +945,16 @@ def _extract_all_relations(row: pd.Series) -> list[dict[str, object]]: if not isinstance(raw, dict): return [] relations = raw.get("relations", []) - if hasattr(relations, "tolist"): + if _is_list_convertible(relations): relations = relations.tolist() if not isinstance(relations, list): return [] out: list[dict[str, object]] = [] for entry in relations: - if hasattr(entry, "model_dump"): + if isinstance(entry, BaseModel): entry = entry.model_dump() if isinstance(entry, dict): - out.append(entry) + out.append(cast(dict[str, object], entry)) return out @@ -956,16 +967,16 @@ def _normalize_relations(raw: object) -> list[dict[str, object]]: raw = json.loads(raw) except (json.JSONDecodeError, ValueError): return [] - if hasattr(raw, "tolist"): + if _is_list_convertible(raw): raw = raw.tolist() if not isinstance(raw, list): return [] out: list[dict[str, object]] = [] for entry in raw: - if hasattr(entry, "model_dump"): + if isinstance(entry, BaseModel): entry = entry.model_dump() if isinstance(entry, dict): - out.append(entry) + out.append(cast(dict[str, object], entry)) return out @@ -996,7 +1007,7 @@ def _count_replacement_triples(row: pd.Series, *, fallback: list[dict[str, str]] """ raw = row.get(COL_REPLACEMENT_MAP) if COL_REPLACEMENT_MAP in row.index else None if raw is not None: - if hasattr(raw, "model_dump"): + if isinstance(raw, BaseModel): raw = raw.model_dump(mode="python") if isinstance(raw, str): try: @@ -1020,16 +1031,16 @@ def _normalize_invalid_entities(raw: object) -> list[dict[str, str]]: raw = json.loads(raw) except (json.JSONDecodeError, ValueError): return [] - if hasattr(raw, "tolist"): + if _is_list_convertible(raw): raw = raw.tolist() if not isinstance(raw, list): return [] out: list[dict[str, str]] = [] for entry in raw: - if hasattr(entry, "model_dump"): + if isinstance(entry, BaseModel): entry = entry.model_dump() if isinstance(entry, dict): - out.append(entry) + out.append(cast(dict[str, str], entry)) return out @@ -1080,7 +1091,7 @@ def _normalize_disposition(raw: object) -> list[dict[str, str]]: entries = raw.get("sensitivity_disposition", []) if not isinstance(entries, list): return [] - return [e for e in entries if isinstance(e, dict)] + return [cast(dict[str, str], e) for e in entries if isinstance(e, dict)] # --------------------------------------------------------------------------- diff --git a/tests/config/test_anonymizer_config.py b/tests/config/test_anonymizer_config.py index ee8c8594..0738208c 100644 --- a/tests/config/test_anonymizer_config.py +++ b/tests/config/test_anonymizer_config.py @@ -89,11 +89,13 @@ def test_entity_labels_defaults_to_none() -> None: def test_entity_labels_accepts_list() -> None: config = AnonymizerConfig(detect={"entity_labels": ["FIRST_NAME", "email"]}, replace=Redact()) + assert config.detect.entity_labels is not None assert set(config.detect.entity_labels) == {"first_name", "email"} def test_entity_labels_strips_whitespace() -> None: config = AnonymizerConfig(detect={"entity_labels": [" first_name ", "email"]}, replace=Redact()) + assert config.detect.entity_labels is not None assert "first_name" in config.detect.entity_labels assert "email" in config.detect.entity_labels diff --git a/tests/engine/test_chunked_validation.py b/tests/engine/test_chunked_validation.py index 4a4207a0..c88e30c0 100644 --- a/tests/engine/test_chunked_validation.py +++ b/tests/engine/test_chunked_validation.py @@ -100,7 +100,7 @@ def _call(self, *, prompt, parser, system_prompt, purpose, kwargs): raise RuntimeError(f"forced failure from {self.alias}") response = self._response if callable(response): - response = response(prompt) + response = response(prompt) # ty: ignore[call-top-callable] raw = response if isinstance(response, str) else f"```json\n{json.dumps(response)}\n```" return parser(raw), [] @@ -209,7 +209,7 @@ def test_missing_seed_raises_with_triage_hint(self) -> None: class TestChunkCandidates: def test_splits_into_chunks_of_at_most_n(self) -> None: ordered = [(i, None) for i in range(5)] # type: ignore[misc] - chunks = chunk_candidates(ordered, max_entities_per_call=2) + chunks = chunk_candidates(ordered, max_entities_per_call=2) # ty: ignore[invalid-argument-type] assert [len(c) for c in chunks] == [2, 2, 1] def test_empty_input_yields_no_chunks(self) -> None: @@ -220,7 +220,7 @@ def test_tuple_input_returns_list_of_lists(self) -> None: # the declared ``list[list[...]]`` return contract must hold even # when the caller hands in a tuple. ordered: tuple[tuple[int, None], ...] = tuple((i, None) for i in range(3)) # type: ignore[misc] - chunks = chunk_candidates(ordered, max_entities_per_call=2) + chunks = chunk_candidates(ordered, max_entities_per_call=2) # ty: ignore[invalid-argument-type] assert chunks == [[(0, None), (1, None)], [(2, None)]] assert all(isinstance(c, list) for c in chunks) @@ -711,7 +711,9 @@ def test_generate_bridges_sync_unavailable_to_agenerate(self) -> None: excerpt_window_chars=50, prompt_template=_MINIMAL_TEMPLATE, ), - resource_provider=SimpleNamespace(model_registry=FakeModelRegistry({"v0": facade})), + resource_provider=SimpleNamespace( # ty: ignore[invalid-argument-type] + model_registry=FakeModelRegistry({"v0": facade}) + ), ) out = generator.generate(row) diff --git a/tests/engine/test_io.py b/tests/engine/test_io.py index 799b3ed6..b1cdee29 100644 --- a/tests/engine/test_io.py +++ b/tests/engine/test_io.py @@ -4,6 +4,7 @@ from __future__ import annotations from pathlib import Path +from typing import Any import pandas as pd import pytest @@ -25,7 +26,7 @@ def _write_input( df: pd.DataFrame, tmp_path: Path, suffix: str = ".csv", - **input_kwargs: object, + **input_kwargs: Any, ) -> AnonymizerInput: """Write *df* to a temp file and return a ready-to-use ``AnonymizerInput``.""" file_path = tmp_path / f"data{suffix}" @@ -59,7 +60,7 @@ def test_write_output_unsupported_format_raises(stub_dataframe: pd.DataFrame, tm (".parquet", lambda df, p: df.to_parquet(p, index=False)), ], ) -def test_read_input_from_file(suffix: str, writer: object, tmp_path: Path) -> None: +def test_read_input_from_file(suffix: str, writer: Any, tmp_path: Path) -> None: input_df = pd.DataFrame({"text": ["Alice works at Acme"]}) file_path = tmp_path / f"data{suffix}" writer(input_df, file_path) @@ -437,7 +438,7 @@ def test_read_input_nrows_remote_csv(monkeypatch: pytest.MonkeyPatch) -> None: def _read_csv(url: str, *args: object, **kwargs: object) -> pd.DataFrame: nrows = kwargs.get("nrows") - if nrows is not None: + if isinstance(nrows, int): return full_df.head(nrows) return full_df diff --git a/tests/engine/test_model_loader.py b/tests/engine/test_model_loader.py index 204bf141..27de6b63 100644 --- a/tests/engine/test_model_loader.py +++ b/tests/engine/test_model_loader.py @@ -565,7 +565,7 @@ def test_non_string_non_list_rejected(self) -> None: with pytest.raises((ValueError, TypeError)): DetectionModelSelection( entity_detector="d", - entity_validator=42, # type: ignore[arg-type] + entity_validator=42, # ty: ignore[invalid-argument-type] entity_augmenter="a", latent_detector="l", ) diff --git a/tests/engine/test_qa_generation.py b/tests/engine/test_qa_generation.py index 74831f9d..3d0368dd 100644 --- a/tests/engine/test_qa_generation.py +++ b/tests/engine/test_qa_generation.py @@ -110,6 +110,7 @@ def test_meaning_extractor_alias_used( stub_rewrite_model_selection: RewriteModelSelection, ) -> None: cols = QAGenerationWorkflow().columns(selected_models=stub_rewrite_model_selection) + assert isinstance(cols[1], LLMStructuredColumnConfig) assert cols[1].model_alias == stub_rewrite_model_selection.meaning_extractor @@ -117,6 +118,7 @@ def test_qa_generator_alias_used( stub_rewrite_model_selection: RewriteModelSelection, ) -> None: cols = QAGenerationWorkflow().columns(selected_models=stub_rewrite_model_selection) + assert isinstance(cols[3], LLMStructuredColumnConfig) assert cols[3].model_alias == stub_rewrite_model_selection.qa_generator diff --git a/tests/engine/test_resolved_input.py b/tests/engine/test_resolved_input.py index 4c982134..4ba848c9 100644 --- a/tests/engine/test_resolved_input.py +++ b/tests/engine/test_resolved_input.py @@ -40,7 +40,7 @@ def test_is_frozen() -> None: resolved_text_column="bio", ) with pytest.raises(FrozenInstanceError): - resolved.resolved_text_column = "other" # type: ignore[misc] + resolved.resolved_text_column = "other" # ty: ignore[invalid-assignment] def test_with_dataframe_swaps_frame_keeps_metadata() -> None: diff --git a/tests/engine/test_rewrite_generation.py b/tests/engine/test_rewrite_generation.py index be32ad44..336ade0a 100644 --- a/tests/engine/test_rewrite_generation.py +++ b/tests/engine/test_rewrite_generation.py @@ -309,6 +309,7 @@ def test_columns_full_rewrite_uses_rewrite_output_schema( workflow = RewriteGenerationWorkflow() cols = workflow.columns(selected_models=stub_rewrite_model_selection, privacy_goal=privacy_goal) full_rewrite_col = next(c for c in cols if c.name == COL_FULL_REWRITE) + assert isinstance(full_rewrite_col, LLMStructuredColumnConfig) assert full_rewrite_col.output_format == RewriteOutputSchema.model_json_schema() diff --git a/tests/interface/test_anonymizer_interface.py b/tests/interface/test_anonymizer_interface.py index f307a609..59930bee 100644 --- a/tests/interface/test_anonymizer_interface.py +++ b/tests/interface/test_anonymizer_interface.py @@ -370,7 +370,7 @@ def test_run_ignores_workflow_output_attrs_for_text_column_resolution( COL_FINAL_ENTITIES: [{"entities": [{"value": "Alice", "label": "first_name"}]}], } ) - detection_df.attrs = {"resolved_text_column": "WRONG_COL", "requested_text_column": "WRONG_COL"} + detection_df.attrs.update({"resolved_text_column": "WRONG_COL", "requested_text_column": "WRONG_COL"}) replace_df = pd.DataFrame( { COL_TEXT: ["Alice bio text"], @@ -384,7 +384,7 @@ def test_run_ignores_workflow_output_attrs_for_text_column_resolution( ], } ) - replace_df.attrs = {"resolved_text_column": "WRONG_COL", "extra": "noise"} + replace_df.attrs.update({"resolved_text_column": "WRONG_COL", "extra": "noise"}) anonymizer, _, _, _ = _make_anonymizer( detection_return=EntityDetectionResult(dataframe=detection_df, failed_records=[]), @@ -715,6 +715,7 @@ def test_run_rewrite_calls_rewrite_runner(stub_input: AnonymizerInput) -> None: rewrite_runner.run.assert_called_once() call_kwargs = rewrite_runner.run.call_args.kwargs + assert config.rewrite is not None assert call_kwargs["privacy_goal"] == config.rewrite.privacy_goal assert call_kwargs["evaluation"] == config.rewrite.evaluation @@ -833,7 +834,7 @@ def test_evaluate_raises_value_error_on_legacy_result_without_replace_method() - ) with pytest.raises(ValueError, match="replace_method"): - anonymizer.evaluate(legacy_result) # type: ignore[arg-type] + anonymizer.evaluate(legacy_result) # ty: ignore[invalid-argument-type] # --------------------------------------------------------------------------- @@ -916,7 +917,7 @@ def test_evaluate_rewrite_raises_without_rewrite_config() -> None: ) with pytest.raises(ValueError): - anonymizer.evaluate(bare_result) # type: ignore[arg-type] + anonymizer.evaluate(bare_result) # ty: ignore[invalid-argument-type] def test_evaluate_rewrite_raises_on_bad_rewrite_judge_alias( diff --git a/tests/interface/test_anonymizer_logging.py b/tests/interface/test_anonymizer_logging.py index 26a52b21..a507d98b 100644 --- a/tests/interface/test_anonymizer_logging.py +++ b/tests/interface/test_anonymizer_logging.py @@ -6,6 +6,7 @@ import logging import re from pathlib import Path +from typing import Any, cast from unittest.mock import Mock import pandas as pd @@ -27,6 +28,10 @@ from anonymizer.interface.anonymizer import Anonymizer +def _call_kwargs(method: object) -> dict[str, Any]: + return dict(cast(Mock, method).call_args.kwargs) + + @pytest.fixture def stub_input(tmp_path: Path) -> AnonymizerInput: csv_path = tmp_path / "input.csv" @@ -162,10 +167,10 @@ def test_preview_set_preview_num_records_capped(stub_input: AnonymizerInput) -> # stub_input has 2 rows; num_records=5 is clamped to min(5, 2) = 2 anonymizer.preview(config=config, data=stub_input, num_records=5) - det_call_kwargs = anonymizer._detection_workflow.run.call_args.kwargs + det_call_kwargs = _call_kwargs(anonymizer._detection_workflow.run) assert det_call_kwargs["preview_num_records"] == 2 - rep_call_kwargs = anonymizer._replace_runner.run.call_args.kwargs + rep_call_kwargs = _call_kwargs(anonymizer._replace_runner.run) assert rep_call_kwargs["preview_num_records"] == 2 @@ -177,10 +182,10 @@ def test_preview_set_preview_num_records_not_capped(stub_input: AnonymizerInput) # stub_input has 2 rows; num_records=1 fits, so no clamping anonymizer.preview(config=config, data=stub_input, num_records=1) - det_call_kwargs = anonymizer._detection_workflow.run.call_args.kwargs + det_call_kwargs = _call_kwargs(anonymizer._detection_workflow.run) assert det_call_kwargs["preview_num_records"] == 1 - rep_call_kwargs = anonymizer._replace_runner.run.call_args.kwargs + rep_call_kwargs = _call_kwargs(anonymizer._replace_runner.run) assert rep_call_kwargs["preview_num_records"] == 1 @@ -191,10 +196,10 @@ def test_run_set_preview_num_records(stub_input: AnonymizerInput) -> None: anonymizer.run(config=config, data=stub_input) - det_call_kwargs = anonymizer._detection_workflow.run.call_args.kwargs + det_call_kwargs = _call_kwargs(anonymizer._detection_workflow.run) assert det_call_kwargs["preview_num_records"] is None - rep_call_kwargs = anonymizer._replace_runner.run.call_args.kwargs + rep_call_kwargs = _call_kwargs(anonymizer._replace_runner.run) assert rep_call_kwargs["preview_num_records"] is None diff --git a/tests/test_debug_logging.py b/tests/test_debug_logging.py index c7848dc9..a7f78767 100644 --- a/tests/test_debug_logging.py +++ b/tests/test_debug_logging.py @@ -4,6 +4,7 @@ from __future__ import annotations import logging +from pathlib import Path from unittest.mock import Mock import pandas as pd @@ -60,7 +61,7 @@ def _stub_anonymizer() -> Anonymizer: @pytest.fixture def debug_messages( - tmp_path: pytest.TempPathFactory, + tmp_path: Path, _stub_anonymizer: Anonymizer, caplog: pytest.LogCaptureFixture, ) -> list[str]: diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index d30d2040..7bb756c4 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -210,7 +210,7 @@ def test_transformation_type_is_required(self) -> None: import pydantic with pytest.raises(pydantic.ValidationError): - AnonymizerEvent( + AnonymizerEvent( # ty: ignore[missing-argument] task=TaskEnum.BATCH, task_status=TaskStatusEnum.COMPLETED, entity_detector_model="x", @@ -223,7 +223,7 @@ def test_detector_validator_augmenter_required(self) -> None: import pydantic with pytest.raises(pydantic.ValidationError): - AnonymizerEvent( + AnonymizerEvent( # ty: ignore[missing-argument] task=TaskEnum.BATCH, task_status=TaskStatusEnum.COMPLETED, transformation_type="redact", @@ -315,7 +315,7 @@ def test_enqueue_noop_when_disabled(self, monkeypatch: pytest.MonkeyPatch) -> No def test_enqueue_noop_for_non_event(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("NEMO_TELEMETRY_ENABLED", "true") handler = TelemetryHandler() - handler.enqueue("not an event") # type: ignore[arg-type] + handler.enqueue("not an event") # ty: ignore[invalid-argument-type] assert handler._events == [] def test_enqueue_when_enabled(self, monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/tools/test_measurement_sweeps.py b/tests/tools/test_measurement_sweeps.py index b91ad6ef..707f1586 100644 --- a/tests/tools/test_measurement_sweeps.py +++ b/tests/tools/test_measurement_sweeps.py @@ -59,7 +59,7 @@ def _patch_sweep_runner( status: Any, ) -> None: class FakeCase: - pass + status: Any FakeCase.status = status result = SimpleNamespace(suite_id="base-suite", cases=[FakeCase()]) diff --git a/tools/codestyle/typecheck.sh b/tools/codestyle/typecheck.sh index 90176d17..34673c33 100755 --- a/tools/codestyle/typecheck.sh +++ b/tools/codestyle/typecheck.sh @@ -6,7 +6,7 @@ # typecheck.sh -- run ty type checks on Python files # # Usage: -# ./typecheck.sh # all files (ty discovers and excludes via pyproject.toml) +# ./typecheck.sh # configured repository paths # ./typecheck.sh src/foo.py bar.py # specific files # @@ -19,7 +19,7 @@ source "$REPO_ROOT/tools/codestyle/_lib.sh" require_tool ty if [[ $# -eq 0 ]]; then - ty check + ty check --error-on-warning else - ty check "$@" + ty check --error-on-warning "$@" fi diff --git a/uv.lock b/uv.lock index 2ce3c508..a6f82b96 100644 --- a/uv.lock +++ b/uv.lock @@ -2508,7 +2508,7 @@ dev = [ { name = "pytest", specifier = ">=9.0.3,<10" }, { name = "pytest-cov", specifier = ">=7.0,<8" }, { name = "ruff", specifier = ">=0.12.3,<1" }, - { name = "ty", specifier = ">=0.0.23,<0.1" }, + { name = "ty", specifier = "==0.0.58" }, { name = "wandb", extras = ["workspaces"], specifier = ">=0.19,<1" }, ] docs = [ @@ -4295,26 +4295,27 @@ wheels = [ [[package]] name = "ty" -version = "0.0.23" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/75/ba/d3c998ff4cf6b5d75b39356db55fe1b7caceecc522b9586174e6a5dee6f7/ty-0.0.23.tar.gz", hash = "sha256:5fb05db58f202af366f80ef70f806e48f5237807fe424ec787c9f289e3f3a4ef", size = 5341461, upload-time = "2026-03-13T12:34:23.125Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/21/aab32603dfdfacd4819e52fa8c6074e7bd578218a5142729452fc6a62db6/ty-0.0.23-py3-none-linux_armv6l.whl", hash = "sha256:e810eef1a5f1cfc0731a58af8d2f334906a96835829767aed00026f1334a8dd7", size = 10329096, upload-time = "2026-03-13T12:34:09.432Z" }, - { url = "https://files.pythonhosted.org/packages/9f/a9/dd3287a82dce3df546ec560296208d4905dcf06346b6e18c2f3c63523bd1/ty-0.0.23-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e43d36bd89a151ddcad01acaeff7dcc507cb73ff164c1878d2d11549d39a061c", size = 10156631, upload-time = "2026-03-13T12:34:53.122Z" }, - { url = "https://files.pythonhosted.org/packages/0f/01/3f25909b02fac29bb0a62b2251f8d62e65d697781ffa4cf6b47a4c075c85/ty-0.0.23-py3-none-macosx_11_0_arm64.whl", hash = "sha256:bd6a340969577b4645f231572c4e46012acba2d10d4c0c6570fe1ab74e76ae00", size = 9653211, upload-time = "2026-03-13T12:34:15.049Z" }, - { url = "https://files.pythonhosted.org/packages/d5/60/bfc0479572a6f4b90501c869635faf8d84c8c68ffc5dd87d04f049affabc/ty-0.0.23-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:341441783e626eeb7b1ec2160432956aed5734932ab2d1c26f94d0c98b229937", size = 10156143, upload-time = "2026-03-13T12:34:34.468Z" }, - { url = "https://files.pythonhosted.org/packages/3a/81/8a93e923535a340f54bea20ff196f6b2787782b2f2f399bd191c4bc132d6/ty-0.0.23-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8ce1dc66c26d4167e2c78d12fa870ef5a7ec9cc344d2baaa6243297cfa88bd52", size = 10136632, upload-time = "2026-03-13T12:34:28.832Z" }, - { url = "https://files.pythonhosted.org/packages/da/cb/2ac81c850c58acc9f976814404d28389c9c1c939676e32287b9cff61381e/ty-0.0.23-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bae1e7a294bf8528836f7617dc5c360ea2dddb63789fc9471ae6753534adca05", size = 10655025, upload-time = "2026-03-13T12:34:37.105Z" }, - { url = "https://files.pythonhosted.org/packages/b5/9b/bac771774c198c318ae699fc013d8cd99ed9caf993f661fba11238759244/ty-0.0.23-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d2b162768764d9dc177c83fb497a51532bb67cbebe57b8fa0f2668436bf53f3c", size = 11230107, upload-time = "2026-03-13T12:34:20.751Z" }, - { url = "https://files.pythonhosted.org/packages/14/09/7644fb0e297265e18243f878aca343593323b9bb19ed5278dcbc63781be0/ty-0.0.23-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d28384e48ca03b34e4e2beee0e230c39bbfb68994bb44927fec61ef3642900da", size = 10934177, upload-time = "2026-03-13T12:34:17.904Z" }, - { url = "https://files.pythonhosted.org/packages/18/14/69a25a0cad493fb6a947302471b579a03516a3b00e7bece77fdc6b4afb9b/ty-0.0.23-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:559d9a299df793cb7a7902caed5eda8a720ff69164c31c979673e928f02251ee", size = 10752487, upload-time = "2026-03-13T12:34:31.785Z" }, - { url = "https://files.pythonhosted.org/packages/9d/2a/42fc3cbccf95af0a62308ebed67e084798ab7a85ef073c9986ef18032743/ty-0.0.23-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:32a7b8a14a98e1d20a9d8d2af23637ed7efdb297ac1fa2450b8e465d05b94482", size = 10133007, upload-time = "2026-03-13T12:34:42.838Z" }, - { url = "https://files.pythonhosted.org/packages/e1/69/307833f1b52fa3670e0a1d496e43ef7df556ecde838192d3fcb9b35e360d/ty-0.0.23-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:6f803b9b9cca87af793467973b9abdd4b83e6b96d9b5e749d662cff7ead70b6d", size = 10169698, upload-time = "2026-03-13T12:34:12.351Z" }, - { url = "https://files.pythonhosted.org/packages/89/ae/5dd379ec22d0b1cba410d7af31c366fcedff191d5b867145913a64889f66/ty-0.0.23-py3-none-musllinux_1_2_i686.whl", hash = "sha256:4a0bf086ec8e2197b7ea7ebfcf4be36cb6a52b235f8be61647ef1b2d99d6ffd3", size = 10346080, upload-time = "2026-03-13T12:34:40.012Z" }, - { url = "https://files.pythonhosted.org/packages/98/c7/dfc83203d37998620bba9c4873a080c8850a784a8a46f56f8163c5b4e320/ty-0.0.23-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:252539c3fcd7aeb9b8d5c14e2040682c3e1d7ff640906d63fd2c4ce35865a4ba", size = 10848162, upload-time = "2026-03-13T12:34:45.421Z" }, - { url = "https://files.pythonhosted.org/packages/89/08/05481511cfbcc1fd834b6c67aaae090cb609a079189ddf2032139ccfc490/ty-0.0.23-py3-none-win32.whl", hash = "sha256:51b591d19eef23bbc3807aef77d38fa1f003c354e1da908aa80ea2dca0993f77", size = 9748283, upload-time = "2026-03-13T12:34:50.607Z" }, - { url = "https://files.pythonhosted.org/packages/31/2e/eaed4ff5c85e857a02415084c394e02c30476b65e158eec1938fdaa9a205/ty-0.0.23-py3-none-win_amd64.whl", hash = "sha256:1e137e955f05c501cfbb81dd2190c8fb7d01ec037c7e287024129c722a83c9ad", size = 10698355, upload-time = "2026-03-13T12:34:26.134Z" }, - { url = "https://files.pythonhosted.org/packages/91/29/b32cb7b4c7d56b9ed50117f8ad6e45834aec293e4cb14749daab4e9236d5/ty-0.0.23-py3-none-win_arm64.whl", hash = "sha256:a0399bd13fd2cd6683fd0a2d59b9355155d46546d8203e152c556ddbdeb20842", size = 10155890, upload-time = "2026-03-13T12:34:48.082Z" }, +version = "0.0.58" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/09/4c/26c90732658903aeb1d289208f7b7b492fa21029e0c4d6c51bdd6f8f5e51/ty-0.0.58.tar.gz", hash = "sha256:8f22484174e65c630660a454bf81b80cae7a3a7e70479f19c170d6cd87949258", size = 6133665, upload-time = "2026-07-10T03:09:30.542Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/e1/5d1aa2a75829459834689f080e4be7a9d8828ce14b939ebed69161a35811/ty-0.0.58-py3-none-linux_armv6l.whl", hash = "sha256:47412850b6fbef61c42f244f6a51aa2f2c9e91f08cfbafd2d1e3730d2419d317", size = 11706915, upload-time = "2026-07-10T03:08:51.028Z" }, + { url = "https://files.pythonhosted.org/packages/96/e1/929eda9cc72a9afe39a03c76f946a503508d37343cc8ff2e64226afda105/ty-0.0.58-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:79deb7bb4e5b3a1eee6ab9abc724d6ce3559d4977982707f310a139ee11fc703", size = 11532079, upload-time = "2026-07-10T03:08:53.771Z" }, + { url = "https://files.pythonhosted.org/packages/07/43/ebc58b3fc7d86a7abba2829f1674f7d4ae3a08f9794c1f31b707950c871f/ty-0.0.58-py3-none-macosx_11_0_arm64.whl", hash = "sha256:5a28af3187e661708a386d44a4fc32896a5f589fb07b734a11ab2f516e7572b7", size = 11092983, upload-time = "2026-07-10T03:08:56.025Z" }, + { url = "https://files.pythonhosted.org/packages/92/6e/9547dbb8e51e47749cfb721a02b4fc862f9a932fa0f66a34a4d6dc429bb1/ty-0.0.58-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21a6977e34bc362fb378add46e59d5d56331c1c36727e6904767217ca8479718", size = 11490492, upload-time = "2026-07-10T03:08:58.49Z" }, + { url = "https://files.pythonhosted.org/packages/d9/76/9f83b51b5e7795d6c8d76b4bb1bb7cebf4c10d609e846c02ab138e404556/ty-0.0.58-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3ac23e6bf6105ceca46632debf1b10b98125aaf60202aeae02f6abfeea242b3d", size = 11503696, upload-time = "2026-07-10T03:09:00.741Z" }, + { url = "https://files.pythonhosted.org/packages/47/9d/9fd48a0696c680f74e50f31cd54e524eeb58c97ea9bb1c3b8e04230ba215/ty-0.0.58-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bb6df6a8c6a21894807a49851370ec7fc64aa910296c78ada31db0ef19359112", size = 12158653, upload-time = "2026-07-10T03:09:02.998Z" }, + { url = "https://files.pythonhosted.org/packages/bc/ea/b5de845d2d8edae04d901c7585af7f04a057e18b26311f7fa4ca62b2da30/ty-0.0.58-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9df5847eebc026cde088b44420a03f7c6c169a7db747176bdf0a656eb1144713", size = 12723019, upload-time = "2026-07-10T03:09:05.291Z" }, + { url = "https://files.pythonhosted.org/packages/17/1c/54083b23eeff1e101f50b6df6a2c7f1e14b31abe0577c91bcae9e2c9395d/ty-0.0.58-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:53a331a7f1f85872c810676a0f16096ef98c1b95c8c9a573fe7fde64d0a93e7c", size = 12275715, upload-time = "2026-07-10T03:09:07.564Z" }, + { url = "https://files.pythonhosted.org/packages/22/88/16925434b06faa49d36aa7e7508a6821ec6feacb429ca2fd80a3d52716b4/ty-0.0.58-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb0b05cd479fdcedc2e6781d376ee1a33569f37ae7f58357004635f615c4374c", size = 12033075, upload-time = "2026-07-10T03:09:10.222Z" }, + { url = "https://files.pythonhosted.org/packages/58/2b/b55708dd483982ae03d14da3667e3f0346cd384e11cca3dd3674a8b598c1/ty-0.0.58-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:9a0012786077e5becbb6add9fe51eda1d4d36d249afd3ae6cd141d554af15ae3", size = 12367729, upload-time = "2026-07-10T03:09:12.338Z" }, + { url = "https://files.pythonhosted.org/packages/91/a3/99ad66652956408f7e9ac3db6b4a416199d773b6c073cf95b0eea126d340/ty-0.0.58-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4ede96d7f6d149156da254e784b062b817736b68d4c6d660555a3d02d96966fb", size = 11439798, upload-time = "2026-07-10T03:09:14.518Z" }, + { url = "https://files.pythonhosted.org/packages/ee/23/344ceed4fe02ed498711e1f4a47b6e311fb1e9c4fecc19d31894be7a3472/ty-0.0.58-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:47608e58f73901b989402e6f283249cda4c2314282ec368aa74ab61761c62bd5", size = 11512695, upload-time = "2026-07-10T03:09:16.852Z" }, + { url = "https://files.pythonhosted.org/packages/55/cf/3801831812c468f3fd0b3043a80f557a9aa90e6c27375763d7c3121e03f2/ty-0.0.58-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9757de17cc17e4c6bc26e18d4e26dea52ffee53d41a10d9087c6465e6ab12e2b", size = 11812253, upload-time = "2026-07-10T03:09:19.151Z" }, + { url = "https://files.pythonhosted.org/packages/7b/80/bce4f245787b77d1ec9feec7d9161eade5e01a77dbc132e016b24df83b0d/ty-0.0.58-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:f3776d54c1d935fcb8f814bd58efba402c86c555f93e1144c59d087e7aa8b906", size = 12123918, upload-time = "2026-07-10T03:09:21.636Z" }, + { url = "https://files.pythonhosted.org/packages/46/00/bab3d6268e7e88c792bf7cdde81bd11b31aa587eaba75196502b729747c8/ty-0.0.58-py3-none-win32.whl", hash = "sha256:8f50ec0ac3b42baa4c75895dc367071dc86b00ab440d29fdc72c633286a94815", size = 11230897, upload-time = "2026-07-10T03:09:23.871Z" }, + { url = "https://files.pythonhosted.org/packages/9c/68/d9504c895864aaa84a840dce6ac7f8e681f6938be1882c8d4f60832dbe57/ty-0.0.58-py3-none-win_amd64.whl", hash = "sha256:de8847b3a65475ae4773bddd3126bfcf29f017e88967c4dcc9c75d743a4d3e5c", size = 12299376, upload-time = "2026-07-10T03:09:26.292Z" }, + { url = "https://files.pythonhosted.org/packages/26/04/c9847cb680b5fe8e1f7d7b483edd5cedcc29394496e7b8ed40d96be796ba/ty-0.0.58-py3-none-win_arm64.whl", hash = "sha256:7334bb38789878f60677f2eb9c1de4bfdf4583e2443790989c77da9b10fe0989", size = 11710495, upload-time = "2026-07-10T03:09:28.478Z" }, ] [[package]] From 1c60bef87ff2bcba3afbb05631ae1909be9009eb Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Tue, 14 Jul 2026 22:31:13 +0000 Subject: [PATCH 2/6] fix: address blocking typecheck review findings Signed-off-by: Aaron Gonzales --- DEVELOPMENT.md | 2 +- pyproject.toml | 2 +- .../engine/evaluation/judge_base.py | 12 ++++++-- src/anonymizer/interface/anonymizer.py | 20 +++++++++---- tests/interface/test_anonymizer_logging.py | 23 ++++++++++++++- tools/measurement/analyze_benchmark_output.py | 29 +++++++++++++------ .../analyze_detection_artifacts.py | 23 +++++++++++---- tools/measurement/export_measurements.py | 10 +++++-- tools/measurement/measurement_tools/tables.py | 2 +- tools/serve_gliner.py | 6 ++-- 10 files changed, 99 insertions(+), 30 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index ff4f71e0..0ccbaf1c 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -160,7 +160,7 @@ Run the blocking type checker: make typecheck ``` -`ty` checks `src`, `tests`, `tests_e2e`, `docs`, and `scripts`. Errors and warnings fail locally and in CI. +`ty` checks `src`, `tests`, `tests_e2e`, `docs`, `scripts`, and `tools`. Errors and warnings fail locally and in CI. Check lockfile freshness: diff --git a/pyproject.toml b/pyproject.toml index 37cf3331..a05521c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,7 +88,7 @@ markers = [ ] [tool.ty.src] -include = ["src", "tests", "tests_e2e", "docs", "scripts"] +include = ["src", "tests", "tests_e2e", "docs", "scripts", "tools"] [tool.ty.environment] python-version = "3.11" diff --git a/src/anonymizer/engine/evaluation/judge_base.py b/src/anonymizer/engine/evaluation/judge_base.py index 357293ea..78b8ff92 100644 --- a/src/anonymizer/engine/evaluation/judge_base.py +++ b/src/anonymizer/engine/evaluation/judge_base.py @@ -19,7 +19,7 @@ import logging from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import Any, ClassVar, Protocol, cast +from typing import ClassVar, Protocol, cast import pandas as pd from data_designer.config.column_configs import LLMStructuredColumnConfig @@ -43,6 +43,14 @@ class JudgeResult: failed_records: list[FailedRecord] +class _JudgeRunResult(Protocol): + @property + def dataframe(self) -> pd.DataFrame: ... + + @property + def failed_records(self) -> list[FailedRecord]: ... + + class _JudgeAdapter(Protocol): def run_workflow( self, @@ -53,7 +61,7 @@ def run_workflow( columns: list[ColumnConfigT], workflow_name: str, preview_num_records: int | None = None, - ) -> Any: ... + ) -> _JudgeRunResult: ... class _BaseJudgeWorkflow(ABC): diff --git a/src/anonymizer/interface/anonymizer.py b/src/anonymizer/interface/anonymizer.py index e31d01a9..349e34b7 100644 --- a/src/anonymizer/interface/anonymizer.py +++ b/src/anonymizer/interface/anonymizer.py @@ -10,7 +10,7 @@ import uuid from collections import Counter from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Protocol, TypeGuard from data_designer.config.models import ModelProvider from data_designer.config.run_config import RunConfig @@ -774,13 +774,23 @@ def _build_telemetry_event( ) +class _ListConvertible(Protocol): + def tolist(self) -> object: ... + + +def _is_list_convertible(value: object) -> TypeGuard[_ListConvertible]: + return callable(getattr(value, "tolist", None)) + + def _unwrap_entities(raw: object) -> list: if isinstance(raw, dict): entities = raw.get("entities", []) - return entities if isinstance(entities, list) else [] - if isinstance(raw, list): - return raw - entities = getattr(raw, "entities", []) + elif isinstance(raw, list): + entities = raw + else: + entities = getattr(raw, "entities", []) + if _is_list_convertible(entities): + entities = entities.tolist() return entities if isinstance(entities, list) else [] diff --git a/tests/interface/test_anonymizer_logging.py b/tests/interface/test_anonymizer_logging.py index a507d98b..c8155361 100644 --- a/tests/interface/test_anonymizer_logging.py +++ b/tests/interface/test_anonymizer_logging.py @@ -9,6 +9,7 @@ from typing import Any, cast from unittest.mock import Mock +import numpy as np import pandas as pd import pytest @@ -41,7 +42,7 @@ def stub_input(tmp_path: Path) -> AnonymizerInput: def _make_logging_anonymizer( *, - detection_entities: list[list[dict]] | None = None, + detection_entities: list[object] | None = None, detection_failures: list[FailedRecord] | None = None, replace_failures: list[FailedRecord] | None = None, ) -> Anonymizer: @@ -114,6 +115,26 @@ def test_run_logs_pipeline_stages(stub_input: AnonymizerInput, caplog: pytest.Lo assert "2 records processed" in messages +def test_run_logs_numpy_wrapped_entity_counts(stub_input: AnonymizerInput, caplog: pytest.LogCaptureFixture) -> None: + detection_entities: list[object] = [ + { + "entities": np.array( + [{"value": "Alice", "label": "first_name"}, {"value": "Acme", "label": "organization"}], + dtype=object, + ) + }, + {"entities": np.array([{"value": "Bob", "label": "first_name"}], dtype=object)}, + ] + + with caplog.at_level(logging.INFO, logger="anonymizer"): + anonymizer = _make_logging_anonymizer(detection_entities=detection_entities) + anonymizer.run(config=AnonymizerConfig(replace=Redact()), data=stub_input) + + assert "3 entities" in caplog.text + assert "first_name=2" in caplog.text + assert "organization=1" in caplog.text + + def test_run_logs_failure_counts(stub_input: AnonymizerInput, caplog: pytest.LogCaptureFixture) -> None: anonymizer = _make_logging_anonymizer( detection_failures=[FailedRecord(record_id="r1", step="detection", reason="timeout")], diff --git a/tools/measurement/analyze_benchmark_output.py b/tools/measurement/analyze_benchmark_output.py index 434cef6f..ecd50594 100644 --- a/tools/measurement/analyze_benchmark_output.py +++ b/tools/measurement/analyze_benchmark_output.py @@ -18,7 +18,7 @@ import sys from dataclasses import dataclass from pathlib import Path -from typing import Annotated, Any, cast +from typing import Annotated, Any, Protocol, TypeGuard, cast import cyclopts import pandas as pd @@ -505,7 +505,10 @@ def _build_case_row( repetition=_first_int([measurement_rows, artifact_rows, trace_rows], ["run_tags.repetition", "repetition"]), case_id=case_id, run_id=_first_value([measurement_rows, artifact_rows, trace_rows], ["run_id"]) or case_id, - **_case_failure_metrics(stage_rows=stage_rows, ndd_rows=ndd_rows, model_rows=model_rows), + **cast( + dict[str, Any], + _case_failure_metrics(stage_rows=stage_rows, ndd_rows=ndd_rows, model_rows=model_rows), + ), pipeline_elapsed_sec=pipeline_elapsed_sec, ndd_workflow_count=len(ndd_rows), ndd_elapsed_sec_total=ndd_elapsed_sec_total, @@ -785,7 +788,7 @@ def _case_artifact_metrics( artifact_rows: pd.DataFrame, *, validation_max_entities_per_call: int | None, -) -> dict[str, int | float | list[str] | dict[str, str] | None]: +) -> dict[str, int | float | list[str] | dict[str, str] | dict[str, dict[str, Any]] | None]: signature_hashes = _artifact_signature_hashes(artifact_rows) return { "detection_artifact_rows": len(artifact_rows), @@ -900,7 +903,7 @@ def build_model_usage_rows(measurements: pd.DataFrame) -> list[ModelUsageAnalysi model_name=metadata.get("model_name") or model_usage_key, model_provider_name=metadata.get("model_provider_name"), ndd_elapsed_sec=_float_from_row(data, ["elapsed_sec"]), - **usage, + **cast(dict[str, Any], usage), ) ) return rows @@ -1077,8 +1080,16 @@ def _coerce_detail_map(raw: object) -> dict[str, dict[str, Any]]: return details +class _ScalarConvertible(Protocol): + def item(self) -> object: ... + + +def _is_scalar_convertible(value: object) -> TypeGuard[_ScalarConvertible]: + return callable(getattr(value, "item", None)) + + def _json_scalar(value: object) -> Any: - if hasattr(value, "item"): + if _is_scalar_convertible(value): try: return value.item() except ValueError: @@ -1179,7 +1190,7 @@ def build_group_rows(cases: list[CaseAnalysisRow]) -> list[GroupAnalysisRow]: "gliner_threshold", ] for keys, group in table.groupby(group_columns, dropna=False): - rows.append(_build_group_row(keys, group)) + rows.append(_build_group_row(cast(tuple[Any, ...], keys), group)) return rows @@ -1200,7 +1211,7 @@ def build_model_usage_group_rows(model_usage: list[ModelUsageAnalysisRow]) -> li "model_provider_name", ] for keys, group in table.groupby(group_columns, dropna=False): - rows.append(_build_model_usage_group_row(keys, group)) + rows.append(_build_model_usage_group_row(cast(tuple[Any, ...], keys), group)) return rows @@ -1379,7 +1390,7 @@ def _build_group_row(keys: tuple[Any, ...], group: pd.DataFrame) -> GroupAnalysi sum_original_value_leak_count=_sum_or_none(group, "original_value_leak_count"), leaking_case_count=_positive_count(group, "original_value_leak_count"), median_original_value_leak_count=_median_or_none(group, "original_value_leak_count"), - **evaluation_metrics, + **cast(dict[str, Any], evaluation_metrics), median_seed_entity_count=_median_or_none(group, "seed_entity_count"), median_seed_validation_candidate_count=_median_or_none(group, "seed_validation_candidate_count"), median_estimated_seed_validation_chunk_count=_median_or_none(group, "estimated_seed_validation_chunk_count"), @@ -1467,7 +1478,7 @@ def _f1(precision: float | None, recall: float | None) -> float | None: def _optional_number(value: object) -> float | None: if value is None or pd.isna(value): return None - return float(value) + return float(cast(Any, value)) def _estimated_validation_chunk_count( diff --git a/tools/measurement/analyze_detection_artifacts.py b/tools/measurement/analyze_detection_artifacts.py index 2d340c55..42d84a96 100644 --- a/tools/measurement/analyze_detection_artifacts.py +++ b/tools/measurement/analyze_detection_artifacts.py @@ -20,7 +20,7 @@ import sys from collections import Counter from pathlib import Path -from typing import Annotated, Iterable +from typing import Annotated, Iterable, Protocol, TypeGuard, cast import cyclopts import pandas as pd @@ -102,7 +102,12 @@ def _analyze_parquet_file(parquet_file: Path, *, artifact_root: Path) -> list[De workflow_name = parquet_file.parents[1].name batch_file = str(parquet_file.relative_to(artifact_root)) return [ - _analyze_dataframe_row(row, workflow_name=workflow_name, batch_file=batch_file, row_index=row_index) + _analyze_dataframe_row( + row, + workflow_name=workflow_name, + batch_file=batch_file, + row_index=cast(int, row_index), + ) for row_index, row in dataframe.iterrows() ] @@ -199,10 +204,18 @@ def _extract_payload_list(raw: object, *, key: str) -> list[object]: return _coerce_list(payload) +class _ListConvertible(Protocol): + def tolist(self) -> object: ... + + +def _is_list_convertible(value: object) -> TypeGuard[_ListConvertible]: + return callable(getattr(value, "tolist", None)) + + def _coerce_payload(raw: object) -> object: if _is_missing(raw): return {} - if hasattr(raw, "tolist"): + if _is_list_convertible(raw): raw = raw.tolist() if not isinstance(raw, str): return raw @@ -221,7 +234,7 @@ def _coerce_payload(raw: object) -> object: def _coerce_list(value: object) -> list[object]: value = _coerce_payload(value) if isinstance(value, list): - return value + return cast(list[object], value) return [] @@ -244,7 +257,7 @@ def _entity_signature_labels(entities: list[EntitySchema], *, row_index: int) -> def _entity_signature_details(entities: list[EntitySchema], *, row_index: int) -> dict[str, dict[str, object]]: - details = { + details: dict[str, dict[str, object]] = { _entity_signature_hash(entity, row_index=row_index): { "label": entity.label, "source": entity.source, diff --git a/tools/measurement/export_measurements.py b/tools/measurement/export_measurements.py index 6480bbf9..1ab48dc1 100755 --- a/tools/measurement/export_measurements.py +++ b/tools/measurement/export_measurements.py @@ -13,7 +13,7 @@ import logging import sys from pathlib import Path -from typing import Annotated +from typing import Annotated, cast import cyclopts import pandas as pd @@ -86,7 +86,13 @@ def export_tables( ) -> ExportResult: output_dir.mkdir(parents=True, exist_ok=True) tables = [ - _export_one_table(record_type, rows, output_dir=output_dir, export_format=export_format, overwrite=overwrite) + _export_one_table( + cast(str, record_type), + rows, + output_dir=output_dir, + export_format=export_format, + overwrite=overwrite, + ) for record_type, rows in dataframe.groupby("record_type", sort=False) ] result = ExportResult( diff --git a/tools/measurement/measurement_tools/tables.py b/tools/measurement/measurement_tools/tables.py index 4e697aa8..a77ec2ae 100644 --- a/tools/measurement/measurement_tools/tables.py +++ b/tools/measurement/measurement_tools/tables.py @@ -76,7 +76,7 @@ def rows_to_table(rows: Sequence[BaseModel], row_model: type[BaseModel] | None = return pd.json_normalize([row.model_dump() for row in rows], sep=".") if row_model is None: return pd.DataFrame() - return pd.DataFrame(columns=list(row_model.model_fields)) + return pd.DataFrame(columns=pd.Index(list(row_model.model_fields))) def write_table(table: pd.DataFrame, path: Path, export_format: ExportFormat) -> None: diff --git a/tools/serve_gliner.py b/tools/serve_gliner.py index e73c2f9d..6f8ecaf2 100644 --- a/tools/serve_gliner.py +++ b/tools/serve_gliner.py @@ -32,10 +32,10 @@ from dataclasses import dataclass from typing import Any -import torch +import torch # ty: ignore[unresolved-import] -- optional server dependency import uvicorn -from fastapi import FastAPI, HTTPException, Request -from gliner import GLiNER +from fastapi import FastAPI, HTTPException, Request # ty: ignore[unresolved-import] -- optional server dependency +from gliner import GLiNER # ty: ignore[unresolved-import] -- optional server dependency MODEL_NAME = "nvidia/gliner-pii" DEFAULT_HOST = "127.0.0.1" From 91fa7e5cefe574362189b3b84baa2b9034de23d0 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Tue, 14 Jul 2026 22:59:35 +0000 Subject: [PATCH 3/6] ci: align local and CI checks Signed-off-by: Aaron Gonzales --- .github/workflows/ci.yml | 32 +++++++------------------------- .pre-commit-config.yaml | 15 +++++++-------- CONTRIBUTING.md | 28 +++++++++++++++++++++++++--- DEVELOPMENT.md | 11 ++++++++--- 4 files changed, 47 insertions(+), 39 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e84bf57e..cfb14f1e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,8 +41,8 @@ jobs: - name: Run tests with coverage run: make coverage - lint: - name: Lint and Format Check + check: + name: Check runs-on: ubuntu-latest steps: # Required to use the reusable action @@ -59,32 +59,14 @@ jobs: - name: Install dependencies run: uv sync --group dev - - name: Run formatting check + - name: Check format and lint run: make format-check - - name: Check copyright headers - run: make copyright-check + - name: Run type checks + run: make typecheck - name: Check uv.lock is up to date run: make lock-check - typecheck: - name: Type Check - runs-on: ubuntu-latest - steps: - # Required to use the reusable action - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: "0" - - - uses: ./.github/actions/setup-python-env - with: - fetch-depth: "0" - python-version: ${{ env.DEFAULT_PYTHON_VERSION }} - - - name: Install dependencies - run: uv sync --group dev - - - name: Run type checks - run: make typecheck + - name: Check copyright headers + run: make copyright-check diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9e90e332..4329c4ac 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,14 +27,6 @@ repos: types: [python] pass_filenames: true -# - id: typecheck -# name: Typecheck -# language: system -# entry: uv run --group docs tools/codestyle/typecheck.sh -# types: [python] -# pass_filenames: false -# verbose: true - - id: copyright-fix name: Copyright Fix language: system @@ -50,6 +42,13 @@ repos: stages: [pre-commit] pass_filenames: false + - id: check + name: Check + language: system + entry: make check + pass_filenames: false + stages: [pre-commit] + - id: check-signoff name: Check commit is signed off (DCO) entry: "grep -qE '^Signed-off-by: .+ <.+>'" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6bc5deaa..27ddb45c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,6 +12,7 @@ This document covers contribution policy and pull request expectations. For loca - [Conventional Commits](#conventional-commits) - [Release Tags](#release-tags) - [Branch Protection](#branch-protection) + - [Local Commit Safety](#local-commit-safety) - [Pull Request Expectations](#pull-request-expectations) - [Agent-Assisted Development](#agent-assisted-development) - [Developer Certificate of Origin](#developer-certificate-of-origin) @@ -24,9 +25,11 @@ This document covers contribution policy and pull request expectations. For loca 3. Create a branch from the latest `main` using the [branch naming](#branch-naming) convention. 4. For non-trivial changes, write a plan document before implementation. See [Agent-Assisted Development](#agent-assisted-development). 5. Implement the change following [AGENTS.md](AGENTS.md), [STYLEGUIDE.md](STYLEGUIDE.md), and [DEVELOPMENT.md](DEVELOPMENT.md). -6. Run the relevant local checks from [DEVELOPMENT.md](DEVELOPMENT.md#validation-before-opening-a-pr). -7. Open a pull request with a conventional title, a linked issue, and a completed checklist. -8. Address review feedback. CODEOWNERS are assigned automatically. +6. Install the repository hooks once per clone with `make install-pre-commit`. +7. Run the relevant tests from [DEVELOPMENT.md](DEVELOPMENT.md#validation-before-opening-a-pr). +8. Commit with DCO signoff. The pre-commit hooks run the repository checks before accepting the commit. +9. Open a pull request with a conventional title, a linked issue, and a completed checklist. +10. Address review feedback. CODEOWNERS are assigned automatically. ## Repository Rules @@ -147,6 +150,25 @@ The `main` branch has these protections: | Deletions | Blocked | | Merge strategy | Squash only | +### Local Commit Safety + +Install the repository hooks once per clone: + +```bash +make install-pre-commit +``` + +Before Git records a commit, the hooks check file hygiene, format and lint staged Python files, repair SPDX headers, +verify `uv.lock` when `pyproject.toml` changes, and run `make check`. The aggregate check covers four read-only stages: + +- `make format-check` +- `make typecheck` +- `make lock-check` +- `make copyright-check` + +The commit-message hook rejects commits without a `Signed-off-by` line. If a hook changes a file, review the change, +stage it again, and retry the commit. Do not bypass repository hooks with `git commit --no-verify`. + ## Pull Request Expectations Every PR should include: diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 0ccbaf1c..ece79800 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -154,6 +154,9 @@ Run all read-only checks: make check ``` +`make check` runs `make format-check`, `make typecheck`, `make lock-check`, and `make copyright-check`. CI runs the +same four Make targets as separate steps in its `Check` job so failures identify the affected stage. + Run the blocking type checker: ```bash @@ -183,10 +186,12 @@ Install hooks once: make install-pre-commit ``` -Hooks run Ruff format and lint, uv lock verification, DCO signoff checks, and basic file hygiene. `ty` is installed -for the blocking `make typecheck` and `make check` targets, but it is not currently run as a pre-commit hook. +Before Git records a commit, the hooks check file hygiene, format and lint staged Python files, repair SPDX headers, +verify `uv.lock` when `pyproject.toml` changes, and run the repository-wide `make check`. The commit-message hook +rejects commits without a DCO `Signed-off-by` line. -If `pyproject.toml` changes and `uv.lock` is stale, the uv-lock hook may regenerate `uv.lock` and fail the commit. Add the updated `uv.lock` and retry. +If a hook changes a file, review the change, stage it again, and retry the commit. In particular, the uv-lock hook may +regenerate `uv.lock` when `pyproject.toml` changes. Do not bypass repository hooks with `git commit --no-verify`. ## Secrets and Credentials From 0b8ad9fcac757f3086e27059dbd01490fe807225 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Wed, 15 Jul 2026 15:07:39 +0000 Subject: [PATCH 4/6] Use native Data Designer config transport Signed-off-by: Aaron Gonzales --- src/anonymizer/distributed.py | 87 ----------- .../engine/detection/chunked_validation.py | 16 +- src/anonymizer/engine/ndd/adapter.py | 7 +- src/anonymizer/interface/anonymizer.py | 14 +- tests/engine/test_chunked_validation.py | 6 +- .../test_detection_config_serialization.py | 140 ++++++++++++++++++ tests/engine/test_detection_workflow.py | 2 +- 7 files changed, 163 insertions(+), 109 deletions(-) delete mode 100644 src/anonymizer/distributed.py create mode 100644 tests/engine/test_detection_config_serialization.py diff --git a/src/anonymizer/distributed.py b/src/anonymizer/distributed.py deleted file mode 100644 index 2bcae407..00000000 --- a/src/anonymizer/distributed.py +++ /dev/null @@ -1,87 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Distributed-executor entrypoint for running the detection workflow on a SLURM cluster. - -For running detection at scale, an external DataDesigner runtime (e.g. a SLURM -orchestrator) provisions the model servers, partitions the dataset across workers, and -runs the workflow. Such runtimes usually ship a *serialized* config to each worker and -rebuild it with ``from_config`` — but the detection workflow can't go through that path: -it uses ``CustomColumnConfig`` columns whose ``generator_function`` is a live Python -callable (DataDesigner custom columns are "library only"), which do not survive JSON -serialization. - -This module is the alternative: a factory the runtime imports and calls **in-process on -each worker** to get the live ``DataDesignerConfigBuilder`` (callables intact). The custom -columns reference their LLM by *alias* and receive model facades injected by the -DataDesigner runtime, so the runtime's provider wiring (alias → provisioned server) still -routes their calls correctly. The seed parquet is read from the path the runtime provides -(not rewritten — workers may share it), and ``num_jobs > 1`` selects this worker's ordered -partition. - -The runtime calls: - build_detection_builder(seed_path=..., job_index=..., num_jobs=..., spec={...}) -where ``spec`` is the JSON-serializable detection spec produced by the submitting side. -Requires ``nemo-anonymizer`` installed in the worker environment. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from data_designer.config.config_builder import DataDesignerConfigBuilder - -# Placeholder provider endpoint; the distributed runtime overrides providers at run time -# (the workflow is only *built* here, never run against this endpoint). -_PLACEHOLDER_ENDPOINT = "http://overridden-by-runtime:8000/v1" - - -def build_detection_builder( - *, - seed_path: str, - spec: dict[str, Any], - job_index: int = 0, - num_jobs: int = 1, -) -> DataDesignerConfigBuilder: - """Return the live detection ``DataDesignerConfigBuilder`` for one distributed worker. - - Args: - seed_path: Path to the seed parquet the runtime placed on this worker (read, not - written). Record ids are assumed already attached by the submitting side. - spec: JSON-serializable detection spec with keys: - ``model_configs_yaml`` (str): the Anonymizer model_configs YAML (selected_models - + model_configs aliases) — the alias ``model`` ids must match the served - model names the runtime provisions, so its provider wiring can map them. - ``provider_names`` (list[str]): provider names referenced by the YAML; placeholder - ``ModelProvider``s are created for them (the runtime supplies the real ones). - ``detect`` (dict): ``gliner_threshold`` (float) and optional ``entity_labels`` - (list[str] | None). - ``data_summary`` (str | None): optional dataset description for prompts. - job_index: index of this worker's ordered partition of the seed. - num_jobs: total number of partitions the seed is split across. - """ - from anonymizer import Anonymizer, AnonymizerConfig, ModelProvider, Redact # noqa: PLC0415 - from anonymizer.config.anonymizer_config import Detect # noqa: PLC0415 - - providers = [ - ModelProvider(name=name, endpoint=_PLACEHOLDER_ENDPOINT, provider_type="openai", api_key="EMPTY") - for name in spec["provider_names"] - ] - anonymizer = Anonymizer(model_configs=spec["model_configs_yaml"], model_providers=providers) - - if "detect" not in spec: - raise KeyError("spec must include required 'detect' section") - detect = spec["detect"] - detect_kwargs: dict[str, Any] = {"gliner_threshold": detect["gliner_threshold"]} - if detect.get("entity_labels") is not None: - detect_kwargs["entity_labels"] = detect["entity_labels"] - config = AnonymizerConfig(detect=Detect(**detect_kwargs), replace=Redact()) - - return anonymizer.export_detection_builder_for_seed( - config=config, - seed_path=seed_path, - job_index=job_index, - num_jobs=num_jobs, - data_summary=spec.get("data_summary"), - ) diff --git a/src/anonymizer/engine/detection/chunked_validation.py b/src/anonymizer/engine/detection/chunked_validation.py index f74e54bb..711c26ae 100644 --- a/src/anonymizer/engine/detection/chunked_validation.py +++ b/src/anonymizer/engine/detection/chunked_validation.py @@ -22,11 +22,11 @@ ``NddAdapter._detect_missing_records`` surfaces it as a ``FailedRecord``. Raw text never silently leaks through as unscrubbed output. -Concurrency. Plugin columns dispatch chunks with ``asyncio.gather`` and -``facade.agenerate()``. The legacy custom-column wrapper still uses a +Concurrency. Async plugin execution dispatches chunks with ``asyncio.gather`` +and ``facade.agenerate()``. The plugin's synchronous execution path uses a ``ThreadPoolExecutor`` and ``facade.generate()``. Per-alias concurrency is -enforced downstream by DataDesigner's request-admission layer, so the -optional row-level cap only bounds local fan-out pressure. +enforced downstream by DataDesigner's request-admission layer, so the optional +row-level cap only bounds local fan-out pressure. """ from __future__ import annotations @@ -100,8 +100,8 @@ class ChunkedValidationParams(BaseModel): excerpt_window_chars: Chars of surrounding raw text included in each chunk's excerpt on either side of the chunk span. max_parallel_chunks: Optional row-local cap on concurrently dispatched - chunks. Defaults to all chunks for the legacy custom-column path; - plugin generators derive a cap from validator model capacity. + chunks. Plugin generators derive a default cap from validator model + capacity. single_chunk_full_text: If True, a row with one validation chunk sees the full tagged document. If False, even a single chunk uses the excerpt window. The default preserves production parity with the @@ -578,8 +578,8 @@ def chunked_validate_row( ) -> dict[str, Any]: """Run chunked validation for a single row and write ``COL_VALIDATION_DECISIONS``. - This sync path is kept for the legacy DataDesigner custom-column wrapper. - Plugin columns should call :func:`chunked_validate_row_async`. + The plugin's synchronous generator uses this path. Its asynchronous generator + calls :func:`chunked_validate_row_async`. """ candidates, dispatch_kwargs_per_chunk = _build_dispatch_kwargs_per_chunk(row, params, models) diff --git a/src/anonymizer/engine/ndd/adapter.py b/src/anonymizer/engine/ndd/adapter.py index 0de78a36..a0c1eb07 100644 --- a/src/anonymizer/engine/ndd/adapter.py +++ b/src/anonymizer/engine/ndd/adapter.py @@ -476,10 +476,9 @@ def build_config_for_seed( Like :meth:`build_config` but the seed dataset points at an already-written ``seed_path`` (record ids assumed already attached) instead of materializing a - DataFrame. Use this on a distributed worker that received the seed from an - orchestrator and must NOT rewrite the shared file. ``num_jobs > 1`` selects this - worker's ordered partition (``job_index`` of ``num_jobs``), matching how the - orchestrator shards the seed. + DataFrame. Use this on the submitting side to build a serializable config for an + existing shared seed. ``num_jobs > 1`` selects one worker's ordered partition + (``job_index`` of ``num_jobs``), matching how the orchestrator shards the seed. """ from data_designer.config.seed import PartitionBlock # noqa: PLC0415 diff --git a/src/anonymizer/interface/anonymizer.py b/src/anonymizer/interface/anonymizer.py index 349e34b7..ca415250 100644 --- a/src/anonymizer/interface/anonymizer.py +++ b/src/anonymizer/interface/anonymizer.py @@ -273,12 +273,14 @@ def export_detection_builder_for_seed( ) -> DataDesignerConfigBuilder: """Build the detection workflow reading an EXISTING seed parquet (no write). - Like :meth:`export_detection_config`, but for a distributed executor that builds - the workflow *in-process on each worker* (so the custom-column callables stay - live — they cannot survive JSON serialization) and received the seed dataset from - the orchestrator. The seed is read from ``seed_path`` (not rewritten), and - ``num_jobs > 1`` selects this worker's ordered partition (``job_index`` of - ``num_jobs``). See ``anonymizer.distributed`` for the worker factory entrypoint. + Like :meth:`export_detection_config`, but points at a seed dataset that the + submitting side has already materialized. ``num_jobs > 1`` selects one ordered + partition (``job_index`` of ``num_jobs``). Serialize the returned builder with + ``builder.get_builder_config().to_json()`` and reconstruct it on a worker with + ``DataDesignerConfigBuilder.from_config()``. The worker must have + ``nemo-anonymizer`` installed so Data Designer can discover the detection plugins, + and ``seed_path`` must resolve to an existing file when the worker reconstructs + the config. Runtime model providers remain external to the serialized config. """ self._validate_preflight_config(config) return self._detection_workflow.build_detection_builder_for_seed( diff --git a/tests/engine/test_chunked_validation.py b/tests/engine/test_chunked_validation.py index c88e30c0..31b74e03 100644 --- a/tests/engine/test_chunked_validation.py +++ b/tests/engine/test_chunked_validation.py @@ -59,8 +59,8 @@ class FakeFacade: """Test double for ``ModelFacade`` recording invocations and replaying canned responses. - Exposes both ``generate()`` and ``agenerate()`` so the sync custom-column - wrapper and async plugin path can share assertions. + Exposes both ``generate()`` and ``agenerate()`` so the plugin's synchronous + and asynchronous paths can share assertions. A canned response can be a ``dict`` (auto-wrapped in a ```json fence), a raw string, or a callable that receives the prompt and returns either. @@ -1043,7 +1043,7 @@ class TestChunkedValidationRegression: """Partitioning must not change outcomes when the LLM is deterministic per candidate. Guards the most important property we promised when we switched from a - single ``LLMStructuredColumnConfig`` to chunked ``CustomColumnConfig`` + single ``LLMStructuredColumnConfig`` to the chunked validation plugin dispatch: given the same set of per-id decisions, chunk sizing is an implementation detail of *how* we talk to the validator, not *what* entities survive validation. diff --git a/tests/engine/test_detection_config_serialization.py b/tests/engine/test_detection_config_serialization.py new file mode 100644 index 00000000..0acda66a --- /dev/null +++ b/tests/engine/test_detection_config_serialization.py @@ -0,0 +1,140 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path +from typing import cast +from unittest.mock import Mock + +import pandas as pd +import pytest +from data_designer.config.config_builder import DataDesignerConfigBuilder +from data_designer.config.seed import PartitionBlock, SamplingStrategy +from data_designer.config.seed_source import LocalFileSeedSource +from data_designer.engine.testing.utils import assert_valid_plugin +from data_designer.interface.data_designer import DataDesigner +from data_designer.plugins import Plugin + +from anonymizer.engine.constants import COL_TEXT, COL_VALIDATION_DECISIONS +from anonymizer.engine.detection.detection_workflow import EntityDetectionWorkflow +from anonymizer.engine.ndd.adapter import NddAdapter +from anonymizer.engine.ndd.model_loader import parse_model_configs +from anonymizer.engine.workflow_columns.detection.config import ( + ChunkedValidationConfig, + DetectionTransformConfig, + DetectionTransformOperation, +) +from anonymizer.engine.workflow_columns.detection.plugins import ( + chunked_validation_plugin, + detection_transform_plugin, +) + + +@pytest.mark.parametrize("plugin", [detection_transform_plugin, chunked_validation_plugin]) +def test_detection_plugin_satisfies_data_designer_contract(plugin: Plugin) -> None: + assert_valid_plugin(plugin) + + +def test_detection_builder_round_trips_through_native_data_designer_config(tmp_path: Path) -> None: + seed_path = tmp_path / "seed.parquet" + pd.DataFrame({COL_TEXT: ["Alice", "Bob", "Carol"]}).to_parquet(seed_path, index=False) + + parsed_models = parse_model_configs(None) + workflow = EntityDetectionWorkflow(adapter=NddAdapter(data_designer=cast(DataDesigner, Mock()))) + builder = workflow.build_detection_builder_for_seed( + seed_path=seed_path, + model_configs=parsed_models.model_configs, + selected_models=parsed_models.selected_models.detection, + gliner_detection_threshold=0.42, + validation_max_entities_per_call=7, + validation_excerpt_window_chars=321, + entity_labels=["first_name", "email"], + data_summary="Customer support messages", + job_index=1, + num_jobs=3, + ) + + payload = builder.get_builder_config().to_json() + assert payload is not None + restored = DataDesignerConfigBuilder.from_config(payload) + + assert restored.get_builder_config().to_dict() == builder.get_builder_config().to_dict() + + seed_config = restored.get_seed_config() + assert seed_config is not None + assert isinstance(seed_config.source, LocalFileSeedSource) + assert seed_config.source.path == str(seed_path) + assert seed_config.sampling_strategy == SamplingStrategy.ORDERED + assert seed_config.selection_strategy == PartitionBlock(index=1, num_partitions=3) + + columns = restored.get_column_configs() + assert len(columns) == 9 + assert all(column.column_type != "custom" for column in columns) + + transforms = [column for column in columns if isinstance(column, DetectionTransformConfig)] + assert {DetectionTransformOperation(column.operation) for column in transforms} == set(DetectionTransformOperation) + + validation = next(column for column in columns if column.name == COL_VALIDATION_DECISIONS) + assert isinstance(validation, ChunkedValidationConfig) + assert validation.max_entities_per_call == 7 + assert validation.excerpt_window_chars == 321 + assert validation.pool == parsed_models.selected_models.detection.entity_validator + assert "Customer support messages" in validation.prompt_template + + serialized = json.loads(payload) + serialized_text = json.dumps(serialized) + assert "anonymizer-detection-transform" in serialized_text + assert "anonymizer-chunked-validation" in serialized_text + assert "generator_function" not in serialized_text + assert "generator_params" not in serialized_text + + +def test_fresh_process_discovers_plugins_when_loading_native_config(tmp_path: Path) -> None: + seed_path = tmp_path / "seed.parquet" + pd.DataFrame({COL_TEXT: ["Alice"]}).to_parquet(seed_path, index=False) + + parsed_models = parse_model_configs(None) + workflow = EntityDetectionWorkflow(adapter=NddAdapter(data_designer=cast(DataDesigner, Mock()))) + builder = workflow.build_detection_builder_for_seed( + seed_path=seed_path, + model_configs=parsed_models.model_configs, + selected_models=parsed_models.selected_models.detection, + gliner_detection_threshold=0.3, + ) + config_path = tmp_path / "detection.json" + output_path = tmp_path / "restored-columns.json" + builder.get_builder_config().to_json(config_path) + + script = """ +import json +import sys +from pathlib import Path + +from data_designer.config.config_builder import DataDesignerConfigBuilder + +builder = DataDesignerConfigBuilder.from_config(Path(sys.argv[1])) +columns = [ + { + "name": column.name, + "column_type": column.column_type, + "class_name": type(column).__name__, + } + for column in builder.get_column_configs() +] +Path(sys.argv[2]).write_text(json.dumps(columns)) +""" + subprocess.run( + [sys.executable, "-c", script, str(config_path), str(output_path)], + check=True, + capture_output=True, + text=True, + ) + + restored_columns = json.loads(output_path.read_text()) + restored_types = {(column["column_type"], column["class_name"]) for column in restored_columns} + assert ("anonymizer-detection-transform", "DetectionTransformConfig") in restored_types + assert ("anonymizer-chunked-validation", "ChunkedValidationConfig") in restored_types diff --git a/tests/engine/test_detection_workflow.py b/tests/engine/test_detection_workflow.py index 84857314..aed0928a 100644 --- a/tests/engine/test_detection_workflow.py +++ b/tests/engine/test_detection_workflow.py @@ -482,7 +482,7 @@ def test_detection_workflow_plugins_are_discoverable() -> None: for plugin in PluginRegistry().get_plugins(PluginType.COLUMN_GENERATOR) if plugin.name.startswith("anonymizer-") } - assert names == {"anonymizer-detection-transform", "anonymizer-chunked-validation"} + assert {"anonymizer-detection-transform", "anonymizer-chunked-validation"} <= names finally: PluginRegistry.reset() From 41fe7aedd9b3d740a93bb00d6ea52e9d0502d4d2 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Wed, 15 Jul 2026 18:05:47 +0000 Subject: [PATCH 5/6] ci: use the DCO GitHub App Signed-off-by: Aaron Gonzales --- .github/workflows/dco-assistant.yml | 58 ----------------------------- CONTRIBUTING.md | 4 ++ 2 files changed, 4 insertions(+), 58 deletions(-) delete mode 100644 .github/workflows/dco-assistant.yml diff --git a/.github/workflows/dco-assistant.yml b/.github/workflows/dco-assistant.yml deleted file mode 100644 index 248594bd..00000000 --- a/.github/workflows/dco-assistant.yml +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (c) 2025-2026, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: "DCO Assistant" - -on: - issue_comment: - types: [created] - pull_request_target: - types: [opened, closed, synchronize] - -permissions: - actions: write - contents: write - pull-requests: write - statuses: write - -jobs: - DCOAssistant: - if: github.repository_owner == 'NVIDIA-NeMo' - runs-on: ubuntu-latest - steps: - - name: "DCO Assistant" - if: | - github.event.comment.body == 'recheck' || - github.event.comment.body == 'I have read the DCO document and I hereby sign the DCO.' || - github.event_name == 'pull_request_target' - uses: contributor-assistant/github-action@ca4a40a7d1004f18d9960b404b97e5f30a505a08 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PERSONAL_ACCESS_TOKEN: ${{ secrets.DCO_ASSISTANT_TOKEN }} - with: - path-to-signatures: "dco-signatures.json" - path-to-document: "https://github.com/NVIDIA-NeMo/anonymizer/blob/main/DCO" - branch: "signatures" - allowlist: dependabot - create-file-commit-message: "chore: create file to store dco signatures" - signed-commit-message: "chore: $contributorName has signed the dco in #$pullRequestNo" - custom-notsigned-prcomment: | - Thank you for your submission! We ask that $you sign our - [Developer Certificate of Origin](https://github.com/NVIDIA-NeMo/anonymizer/blob/main/DCO) - before we can accept your contribution. - - You can sign the DCO by adding a comment below using this text: - custom-pr-sign-comment: "I have read the DCO document and I hereby sign the DCO." - lock-pullrequest-aftermerge: false - use-dco-flag: true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 27ddb45c..655031b5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -238,6 +238,10 @@ Signed-off-by: Your Name By signing off, you certify the [Developer Certificate of Origin](DCO). See the full [DCO](DCO) file for details. +The DCO GitHub App verifies the signoff on every non-bot, non-merge commit in a pull request. The email address in +each `Signed-off-by` line must match that commit's author email. If the check is stale after an outage, request a +fresh check by submitting a pull request review whose summary contains `@dcoapp recheck` on its own line. + ## Issues and Discussions We provide structured issue templates for bug reports, feature requests, and development tasks. From fee84cfbf1196eaad2ff0f9d802e26474ce8a40b Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Wed, 15 Jul 2026 18:07:59 +0000 Subject: [PATCH 6/6] revert: move DCO app migration to a separate PR Signed-off-by: Aaron Gonzales --- .github/workflows/dco-assistant.yml | 58 +++++++++++++++++++++++++++++ CONTRIBUTING.md | 4 -- 2 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/dco-assistant.yml diff --git a/.github/workflows/dco-assistant.yml b/.github/workflows/dco-assistant.yml new file mode 100644 index 00000000..248594bd --- /dev/null +++ b/.github/workflows/dco-assistant.yml @@ -0,0 +1,58 @@ +# Copyright (c) 2025-2026, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: "DCO Assistant" + +on: + issue_comment: + types: [created] + pull_request_target: + types: [opened, closed, synchronize] + +permissions: + actions: write + contents: write + pull-requests: write + statuses: write + +jobs: + DCOAssistant: + if: github.repository_owner == 'NVIDIA-NeMo' + runs-on: ubuntu-latest + steps: + - name: "DCO Assistant" + if: | + github.event.comment.body == 'recheck' || + github.event.comment.body == 'I have read the DCO document and I hereby sign the DCO.' || + github.event_name == 'pull_request_target' + uses: contributor-assistant/github-action@ca4a40a7d1004f18d9960b404b97e5f30a505a08 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PERSONAL_ACCESS_TOKEN: ${{ secrets.DCO_ASSISTANT_TOKEN }} + with: + path-to-signatures: "dco-signatures.json" + path-to-document: "https://github.com/NVIDIA-NeMo/anonymizer/blob/main/DCO" + branch: "signatures" + allowlist: dependabot + create-file-commit-message: "chore: create file to store dco signatures" + signed-commit-message: "chore: $contributorName has signed the dco in #$pullRequestNo" + custom-notsigned-prcomment: | + Thank you for your submission! We ask that $you sign our + [Developer Certificate of Origin](https://github.com/NVIDIA-NeMo/anonymizer/blob/main/DCO) + before we can accept your contribution. + + You can sign the DCO by adding a comment below using this text: + custom-pr-sign-comment: "I have read the DCO document and I hereby sign the DCO." + lock-pullrequest-aftermerge: false + use-dco-flag: true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 655031b5..27ddb45c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -238,10 +238,6 @@ Signed-off-by: Your Name By signing off, you certify the [Developer Certificate of Origin](DCO). See the full [DCO](DCO) file for details. -The DCO GitHub App verifies the signoff on every non-bot, non-merge commit in a pull request. The email address in -each `Signed-off-by` line must match that commit's author email. If the check is stale after an outage, request a -fresh check by submitting a pull request review whose summary contains `@dcoapp recheck` on its own line. - ## Issues and Discussions We provide structured issue templates for bug reports, feature requests, and development tasks.