feat: multi-Python CI and callback serialization

.github/workflows/ci.yml

@@ -0,0 +1,114 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+    paths-ignore:
+      - "docs/**"
+      - "*.md"
+  pull_request:
+    branches: [main]
+    paths-ignore:
+      - "docs/**"
+      - "*.md"
+
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+
+      - uses: astral-sh/setup-uv@v8.3.2
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: uv sync
+
+      - name: Ruff check
+        run: uv run ruff check llmeter tests
+
+      - name: Ruff format check
+        run: uv run ruff format --check llmeter tests
+
+      - name: Bandit security scan
+        run: uv run bandit -c pyproject.toml -r llmeter
+
+  # Full matrix on PRs, single version on push to main
+  resolve-python-versions:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    outputs:
+      versions: ${{ steps.parse.outputs.versions }}
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Extract Python versions from classifiers
+        id: parse
+        run: |
+          versions=$(python3 -c "
+          import tomllib, json
+          with open('pyproject.toml', 'rb') as f:
+              data = tomllib.load(f)
+          classifiers = data['project'].get('classifiers', [])
+          versions = []
+          for c in classifiers:
+              # Match 'Programming Language :: Python :: 3.X'
+              parts = [p.strip() for p in c.split('::')]
+              if len(parts) == 3 and parts[0] == 'Programming Language' and parts[1] == 'Python':
+                  v = parts[2]
+                  # Only include specific minor versions (e.g. '3.10'), not just '3'
+                  if '.' in v:
+                      versions.append(v)
+          print(json.dumps(versions))
+          ")
+          echo "versions=$versions" >> "$GITHUB_OUTPUT"
+          echo "Resolved Python versions: $versions"
+
+  test:
+    runs-on: ubuntu-latest
+    needs: resolve-python-versions
+    if: github.event_name == 'pull_request'
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ${{ fromJson(needs.resolve-python-versions.outputs.versions) }}
+    steps:
+      - uses: actions/checkout@v7
+
+      - uses: astral-sh/setup-uv@v8.3.2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: uv sync
+
+      - name: Run tests
+        run: uv run pytest --cov=llmeter -m "not integ"
+        env:
+          AWS_DEFAULT_REGION: us-east-1
+          AWS_ACCESS_KEY_ID: testing
+          AWS_SECRET_ACCESS_KEY: testing
+
+  test-push:
+    if: github.event_name == 'push'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+
+      - uses: astral-sh/setup-uv@v8.3.2
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: uv sync
+
+      - name: Run tests
+        run: uv run pytest --cov=llmeter -m "not integ"
+        env:
+          AWS_DEFAULT_REGION: us-east-1
+          AWS_ACCESS_KEY_ID: testing
+          AWS_SECRET_ACCESS_KEY: testing

llmeter/callbacks/base.py

@@ -4,14 +4,21 @@
 
 from __future__ import annotations
 
+import importlib
+import json
+import logging
 from abc import ABC
-from typing import final
+from typing import Any, final
 
 from upath.types import ReadablePathLike, WritablePathLike
 
 from ..endpoints.base import InvocationResponse
+from ..json_utils import llmeter_default_serializer
 from ..results import Result
 from ..runner import _RunConfig
+from ..utils import ensure_path
+
+logger = logging.getLogger(__name__)
 
 
 class Callback(ABC):
@@ -22,8 +29,17 @@ class Callback(ABC):
     associated with test runs or individual model invocations.
 
     A Callback object may implement multiple of the defined lifecycle hooks (such as
-    `before_invoke`, `after_run`, etc). Callbacks must support serializing their configuration to
-    a file (by implementing `save_to_file`), and loading back (via `load_from_file`).
+    `before_invoke`, `after_run`, etc). Callbacks support serializing their configuration via
+    ``to_dict()`` / ``from_dict()`` (and the convenience wrappers ``save_to_file()`` /
+    ``load_from_file()``).
+
+    Serialization uses a ``_callback_type`` marker (``"module:ClassName"``) so that
+    ``Callback.from_dict()`` can dynamically import and reconstruct the correct subclass
+    without a hardcoded registry. This means third-party callbacks round-trip through JSON
+    automatically, as long as the defining module is importable at load time.
+
+    Subclasses with complex nested state (like ``CostModel``) can override ``to_dict()`` and
+    ``from_dict()`` while preserving the type marker by calling ``super()``.
     """
 
     async def before_invoke(self, payload: dict) -> None:
@@ -71,46 +87,149 @@ async def after_run(self, result: Result) -> None:
         """
         pass
 
-    def save_to_file(self, path: WritablePathLike) -> None:
-        """Save this Callback to file
+    # -- Serialization -----------------------------------------------------------------
 
-        Individual Callbacks implement this method to save their configuration to a file that will
-        be re-loadable with the equivalent `_load_from_file()` method.
+    def to_dict(self) -> dict:
+        """Serialize this callback's configuration to a JSON-safe dict.
 
-        Args:
-            path: (Local or Cloud) path where the callback is saved
+        The returned dict includes a ``_callback_type`` key with the fully-qualified
+        class path (``"module:ClassName"``), enabling ``Callback.from_dict`` to
+        reconstruct the correct subclass without a hardcoded registry.
+
+        By default, all public (non-underscore-prefixed) instance attributes are
+        included. Subclasses with richer state should override this method and call
+        ``super().to_dict()`` to preserve the type marker.
+
+        Returns:
+            dict: A JSON-serializable dictionary representation of this callback.
+
+        Example::
+
+            >>> from llmeter.callbacks import CostModel
+            >>> from llmeter.callbacks.cost.dimensions import InputTokens
+            >>> model = CostModel(request_dims=[InputTokens(price_per_million=3.0)])
+            >>> d = model.to_dict()
+            >>> d["_callback_type"]
+            'llmeter.callbacks.cost.model:CostModel'
         """
-        raise NotImplementedError("TODO: Callback.save_to_file is not yet implemented!")
+        cls = self.__class__
+        data: dict[str, Any] = {
+            "_callback_type": f"{cls.__module__}:{cls.__qualname__}",
+        }
+        data.update({k: v for k, v in vars(self).items() if not k.startswith("_")})
+        return data
 
-    @staticmethod
-    @final
-    def load_from_file(path: ReadablePathLike) -> Callback:
-        """Load (any type of) Callback from file
+    @classmethod
+    def from_dict(cls, raw: dict, **kwargs: Any) -> Callback:
+        """Reconstruct a Callback from a dict produced by ``to_dict()``.
 
-        `Callback.load_from_file()` attempts to detect the type of Callback saved in a given file,
-        and use the relevant implementation's `_load_from_file` method to load it.
+        Uses the ``_callback_type`` field to dynamically import and instantiate
+        the correct subclass. If called on a concrete subclass (e.g.
+        ``CostModel.from_dict(...)``), the ``_callback_type`` is still respected
+        so that the dict always controls which class is created.
+
+        Args:
+            raw: A dictionary previously produced by ``to_dict()`` (or loaded from
+                JSON). Must contain a ``_callback_type`` key.
+            **kwargs: Extra keyword arguments forwarded to the resolved class
+                constructor (or its own ``from_dict`` if it overrides this method).
+
+        Returns:
+            Callback: An instance of the appropriate Callback subclass.
+
+        Raises:
+            ValueError: If ``_callback_type`` is missing from *raw*.
+            ImportError: If the module referenced by ``_callback_type`` cannot be
+                imported.
+            AttributeError: If the class name cannot be found in the referenced
+                module.
+
+        Example::
+
+            >>> from llmeter.callbacks.base import Callback
+            >>> d = {
+            ...     "_callback_type": "llmeter.callbacks.mlflow:MlflowCallback",
+            ...     "step": 1,
+            ...     "nested": False,
+            ... }
+            >>> cb = Callback.from_dict(d)  # returns an MlflowCallback instance
+        """
+        raw = dict(raw)  # shallow copy — don't mutate caller's dict
+        callback_type = raw.pop("_callback_type", None)
+        if callback_type is None:
+            raise ValueError(
+                "Cannot deserialize Callback: dict is missing '_callback_type' key. "
+                f"Got keys: {list(raw.keys())}"
+            )
+
+        module_path, class_name = callback_type.rsplit(":", 1)
+        module = importlib.import_module(module_path)
+        callback_cls = getattr(module, class_name)
+
+        # If the resolved class has its own from_dict (e.g. CostModel), delegate to it
+        # so that subclass-specific deserialization logic is honoured.
+        if callback_cls is not cls and "from_dict" in callback_cls.__dict__:
+            # Re-inject _callback_type so the subclass from_dict can pop it if needed
+            return callback_cls.from_dict(raw, **kwargs)
+
+        # Remove any keys the constructor doesn't expect (e.g. _type from JSONableBase)
+        raw.pop("_type", None)
+        return callback_cls(**raw, **kwargs)
+
+    def to_json(self, **kwargs: Any) -> str:
+        """Serialize this callback to a JSON string.
 
         Args:
-            path: (Local or Cloud) path where the callback is saved
+            **kwargs: Extra keyword arguments forwarded to ``json.dumps``
+                (e.g. ``indent``).
+
         Returns:
-            callback: A loaded Callback - for example an `MlflowCallback`.
+            str: JSON representation of this callback.
         """
-        raise NotImplementedError(
-            "TODO: Callback.load_from_file is not yet implemented!"
-        )
+        kwargs.setdefault("default", llmeter_default_serializer)
+        return json.dumps(self.to_dict(), **kwargs)
 
     @classmethod
-    def _load_from_file(cls, path: ReadablePathLike) -> Callback:
-        """Load this Callback from file
+    def from_json(cls, json_string: str, **kwargs: Any) -> Callback:
+        """Reconstruct a Callback from a JSON string produced by ``to_json()``.
+
+        Args:
+            json_string: A valid JSON string.
+            **kwargs: Extra keyword arguments forwarded to ``from_dict``.
+
+        Returns:
+            Callback: An instance of the appropriate Callback subclass.
+        """
+        return cls.from_dict(json.loads(json_string), **kwargs)
+
+    def save_to_file(self, path: WritablePathLike) -> None:
+        """Save this Callback's configuration to a JSON file.
 
-        Individual Callbacks implement this method to define how they can be loaded from files
-        created by the equivalent `save_to_file()` method.
+        The file can be loaded back with ``Callback.load_from_file(path)``.
 
         Args:
-            path: (Local or Cloud) path where the callback is saved
+            path: (Local or Cloud) path where the callback should be saved.
+        """
+        path = ensure_path(path)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with path.open("w") as f:
+            f.write(self.to_json(indent=4))
+
+    @staticmethod
+    @final
+    def load_from_file(path: ReadablePathLike) -> Callback:
+        """Load (any type of) Callback from a JSON file.
+
+        The ``_callback_type`` field inside the file determines which subclass is
+        instantiated, so callers don't need to know the concrete type in advance.
+
+        Args:
+            path: (Local or Cloud) path to a JSON file previously created by
+                ``save_to_file()``.
+
         Returns:
-            callback: The loaded Callback object
+            Callback: The deserialized callback instance.
         """
-        raise NotImplementedError(
-            "TODO: Callback._load_from_file is not yet implemented!"
-        )
+        path = ensure_path(path)
+        with path.open("r") as f:
+            return Callback.from_dict(json.load(f))

llmeter/callbacks/cost/model.py

@@ -15,7 +15,7 @@
 from ..base import Callback
 from .dimensions import IRequestCostDimension, IRunCostDimension
 from .results import CalculatedCostWithDimensions
-from .serde import JSONableBase, from_dict_with_class_map
+from .serde import JSONableBase, from_dict_with_class, from_dict_with_class_map
 
 
 @dataclass
@@ -203,6 +203,18 @@ async def after_run(self, result: Result) -> None:
             result, recalculate_request_costs=False, save=True
         )
 
+    def to_dict(self, **kwargs) -> dict:
+        """Serialize the cost model to a JSON-safe dict.
+
+        Injects the ``_callback_type`` marker from ``Callback.to_dict()`` into the
+        dict produced by ``JSONableBase.to_dict()``, so that
+        ``Callback.from_dict()`` can reconstruct this ``CostModel`` dynamically.
+        """
+        data = JSONableBase.to_dict(self, **kwargs)
+        cls = self.__class__
+        data["_callback_type"] = f"{cls.__module__}:{cls.__qualname__}"
+        return data
+
     def save_to_file(self, path: WritablePathLike) -> None:
         """Save the cost model (including all dimensions) to a JSON file"""
         path = ensure_path(path)
@@ -217,13 +229,16 @@ def from_dict(cls, raw: dict, alt_classes: dict = {}, **kwargs) -> "CostModel":
             **alt_classes,
         }
         raw_args = {**raw}
+        # Strip callback/serde type markers — they're not constructor args
+        raw_args.pop("_callback_type", None)
+        raw_args.pop("_type", None)
         for key in ("request_dims", "run_dims"):
             if key in raw_args:
                 raw_args[key] = {
                     name: from_dict_with_class_map(d, class_map=dim_classes)
                     for name, d in raw_args[key].items()
                 }
-        return super().from_dict(raw_args, alt_classes=alt_classes, **kwargs)
+        return from_dict_with_class(raw=raw_args, cls=cls, **kwargs)
 
     @classmethod
     def _load_from_file(cls, path: ReadablePathLike):