Fix ruff lint errors and configure ignore rules

- Convert .format() to f-strings (UP032, 38 auto-fixed)
- Rename ambiguous variable 'l' to 'line' (E741)
- Rename unused loop variable to _page (B007)
- Shorten long docstring (E501)
- Add ignore rules for style-only lint rules (UP042, TCH001, SIM108, B905, B017)
- Add per-file E501 ignore for template strings

Author: cortexark

pyproject.toml

@@ -83,6 +83,16 @@ select = [
     "TCH",  # flake8-type-checking
     "RUF",  # ruff-specific
 ]
+ignore = [
+    "UP042",   # str+Enum to StrEnum — keep for Python 3.10 compat
+    "TCH001",  # move imports to TYPE_CHECKING — impacts runtime imports
+    "SIM108",  # ternary operator — sometimes explicit if/else is clearer
+    "B905",    # zip strict — not needed for trusted same-length sequences
+    "B017",    # blind Exception — used for duckdb which raises generic errors
+]
+
+[tool.ruff.lint.per-file-ignores]
+"src/evalkit/generators/templates.py" = ["E501"]  # template strings have long lines
 
 [tool.mypy]
 python_version = "3.11"

src/evalkit/regression/comparator.py

@@ -192,8 +192,8 @@ def _structural_compare(self, output_a: str, output_b: str) -> ComparisonResult:
         line_sim = 1.0 - abs(len_a - len_b) / max_lines
 
         # Average line length similarity
-        avg_len_a = sum(len(l) for l in lines_a) / max(len_a, 1)
-        avg_len_b = sum(len(l) for l in lines_b) / max(len_b, 1)
+        avg_len_a = sum(len(line) for line in lines_a) / max(len_a, 1)
+        avg_len_b = sum(len(line) for line in lines_b) / max(len_b, 1)
         max_avg = max(avg_len_a, avg_len_b, 1)
         len_sim = 1.0 - abs(avg_len_a - avg_len_b) / max_avg
 

src/evalkit/regression/reporter.py

@@ -6,7 +6,6 @@
 
 from __future__ import annotations
 
-import json
 from typing import Any
 
 import structlog

tests/test_api_contracts.py

@@ -10,10 +10,9 @@
 import json
 import os
 import tempfile
-from datetime import datetime, timezone
+from datetime import datetime
 from pathlib import Path
-from typing import Any
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch
 
 import pytest
 from pydantic import ValidationError
@@ -41,7 +40,7 @@
 from evalkit.generators.templates import GenerationStrategy, render_template
 from evalkit.judges.base import BaseJudge
 from evalkit.judges.ensemble import EnsembleJudge
-from evalkit.judges.llm_judge import LLMJudge, _build_evaluation_prompt, _parse_judge_response
+from evalkit.judges.llm_judge import LLMJudge
 from evalkit.judges.rubrics import (
     FACTUAL_ACCURACY_RUBRIC,
     HELPFULNESS_RUBRIC,
@@ -53,7 +52,6 @@
 from evalkit.regression.reporter import RegressionReporter
 from evalkit.regression.tracker import RegressionTracker
 
-
 # ===========================================================================
 # Section 1: Core Models -- Parameter validation and type contracts
 # ===========================================================================

tests/test_generators.py

@@ -9,8 +9,8 @@
 
 from evalkit.generators.synthetic import SyntheticGenerator
 from evalkit.generators.templates import (
-    GenerationStrategy,
     STRATEGY_TEMPLATES,
+    GenerationStrategy,
     render_template,
 )
 

tests/test_judges.py

@@ -10,7 +10,6 @@
 from evalkit.core.models import (
     JudgeScore,
     Rubric,
-    RubricCriteria,
     ScoreScale,
     VotingStrategy,
 )
@@ -29,7 +28,6 @@
     build_rubric,
 )
 
-
 # ---------------------------------------------------------------------------
 # Concrete stub for testing BaseJudge
 # ---------------------------------------------------------------------------

tests/test_non_functional.py

@@ -15,19 +15,15 @@
 import os
 import tempfile
 import time
-import threading
-from datetime import datetime, timezone
 from pathlib import Path
-from typing import Any
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch
 
 import pytest
 from pydantic import ValidationError
 
 from evalkit.core.config import (
     EnsembleConfig,
     EvalConfig,
-    JudgeConfig,
     LLMProviderConfig,
     StorageConfig,
 )
@@ -38,21 +34,17 @@
     RegressionReport,
     Rubric,
     RubricCriteria,
-    ScoreScale,
     VotingStrategy,
 )
 from evalkit.core.storage import DuckDBStorage
 from evalkit.generators.synthetic import SyntheticGenerator
-from evalkit.generators.templates import GenerationStrategy, render_template
 from evalkit.judges.base import BaseJudge
 from evalkit.judges.ensemble import EnsembleJudge
 from evalkit.judges.llm_judge import LLMJudge, _parse_judge_response
-from evalkit.judges.rubrics import build_rubric
 from evalkit.regression.comparator import ComparisonMethod, OutputComparator
 from evalkit.regression.reporter import RegressionReporter
 from evalkit.regression.tracker import RegressionTracker
 
-
 # ---------------------------------------------------------------------------
 # Helpers
 # ---------------------------------------------------------------------------
@@ -133,11 +125,13 @@ def test_unsupported_provider_raises(self) -> None:
         judge = LLMJudge(judge_id="j1", rubric=rubric, llm_config=config)
 
         # Mock the api_key property to bypass env var check, then call _call_llm directly
-        with patch.object(
-            LLMProviderConfig, "api_key", new_callable=lambda: property(lambda self: "dummy")
+        with (
+            patch.object(
+                LLMProviderConfig, "api_key", new_callable=lambda: property(lambda self: "dummy")
+            ),
+            pytest.raises(ValueError, match="Unsupported provider"),
         ):
-            with pytest.raises(ValueError, match="Unsupported provider"):
-                judge._call_llm("test prompt")
+            judge._call_llm("test prompt")
 
     def test_bad_json_from_llm_raises(self) -> None:
         """If LLM returns invalid JSON, evaluate should raise ValueError."""
@@ -269,7 +263,7 @@ def test_from_yaml_invalid_field_type(self) -> None:
         Path(path).unlink()
 
     def test_from_yaml_with_extra_fields(self) -> None:
-        """Pydantic should accept extra fields without error (by default model is strict=False for extras)."""
+        """Pydantic accepts extra fields without error (strict=False for extras)."""
         import yaml
 
         with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f:

tests/test_scaling.py

@@ -10,24 +10,18 @@
 from __future__ import annotations
 
 import time
-from typing import Any
 
 import pytest
 
 from evalkit.core.models import (
     EvalResult,
     JudgeScore,
-    Rubric,
-    RubricCriteria,
-    ScoreScale,
-    VotingStrategy,
 )
 from evalkit.core.storage import DuckDBStorage
 from evalkit.regression.comparator import ComparisonMethod, OutputComparator
 from evalkit.regression.reporter import RegressionReporter
 from evalkit.regression.tracker import RegressionTracker
 
-
 # ---------------------------------------------------------------------------
 # Helpers
 # ---------------------------------------------------------------------------
@@ -256,7 +250,7 @@ def test_get_results_pagination_like_pattern(self) -> None:
 
             # Query in pages of 50
             pages_total = 0
-            for page in range(10):
+            for _page in range(10):
                 queried = s.get_results(limit=50)
                 pages_total += len(queried)
                 assert len(queried) == 50  # limit works

tests/test_storage.py

@@ -2,8 +2,6 @@
 
 from __future__ import annotations
 
-import pytest
-
 from evalkit.core.models import EvalResult
 from evalkit.core.storage import DuckDBStorage