From ac7a9f6fef43abbfa7dfa493ac10ff57cbe7bd34 Mon Sep 17 00:00:00 2001 From: "Paul S. Schweigert" Date: Fri, 31 Jul 2026 12:52:59 -0400 Subject: [PATCH 1/2] feat: add SIMBAUQSamplingStrategy Signed-off-by: Paul S. Schweigert Signed-off-by: Radu Marinescu Co-authored-by: Radu Marinescu --- agent-utilities/README.md | 32 + agent-utilities/docs/simbauq.mdx | 100 +++ agent-utilities/examples/README.md | 10 +- agent-utilities/examples/simbauq/README.md | 282 ++++++++ .../examples/simbauq/simbauq_example.py | 651 ++++++++++++++++++ .../agent_utilities/core/simbauq.py | 634 +++++++++++++++++ agent-utilities/pyproject.toml | 5 + agent-utilities/tests/test_simbauq.py | 431 ++++++++++++ agent-utilities/uv.lock | 288 +++++++- 9 files changed, 2411 insertions(+), 22 deletions(-) create mode 100644 agent-utilities/docs/simbauq.mdx create mode 100644 agent-utilities/examples/simbauq/README.md create mode 100644 agent-utilities/examples/simbauq/simbauq_example.py create mode 100644 agent-utilities/mellea_contribs/agent_utilities/core/simbauq.py create mode 100644 agent-utilities/tests/test_simbauq.py diff --git a/agent-utilities/README.md b/agent-utilities/README.md index e99a44b6..4866bd70 100644 --- a/agent-utilities/README.md +++ b/agent-utilities/README.md @@ -12,6 +12,7 @@ generation, evaluation, or testing of m-programs. | `top_k` | Generic Top-K LLM-as-judge selector. Pick the best K of N candidate items using a comparison prompt. | | `double_round_robin` | Pairwise tournament selector. Runs A-vs-B and B-vs-A across all pairs and ranks by accumulated wins. | | `benchdrift_runner` | BenchDrift integration for robustness testing of Mellea m-programs against semantic problem variations. | +| `simbauq` | SIMBA-UQ confidence-aware sampling strategy. Generates samples across temperatures and selects the most confident one via similarity-based uncertainty quantification. | ## Install @@ -20,6 +21,9 @@ pip install mellea-contribs-agent-utilities # With BenchDrift robustness extras pip install "mellea-contribs-agent-utilities[robustness]" + +# With SIMBA-UQ sampling extras (scikit-learn, sentence-transformers, tqdm, datasets) +pip install "mellea-contribs-agent-utilities[simbauq]" ``` ## Usage @@ -60,6 +64,34 @@ for item, score in ranked: print(item["name"], score) ``` +### SIMBA-UQ sampling + +Confidence-aware sample selection. Requires the `simbauq` extra for the +`sbert` metric and the `classifier` confidence method: + +```python +from mellea import start_session +from mellea_contribs.agent_utilities.core.simbauq import SIMBAUQSamplingStrategy + +m = start_session() +result = m.instruct( + "What is the capital of France?", + strategy=SIMBAUQSamplingStrategy( + temperatures=[0.3, 0.5, 0.7, 1.0], + n_per_temp=3, + similarity_metric="rouge", + confidence_method="aggregation", + aggregation="mean", + ), + return_sampling_results=True, +) + +best = result.result +print(best._meta["simba_uq"]["confidence"], str(best)) +``` + +See `docs/simbauq.mdx` and `examples/simbauq/` for the full guide. + ### BenchDrift robustness Install with the `robustness` extra and have an Ollama server running: diff --git a/agent-utilities/docs/simbauq.mdx b/agent-utilities/docs/simbauq.mdx new file mode 100644 index 00000000..ccc62a34 --- /dev/null +++ b/agent-utilities/docs/simbauq.mdx @@ -0,0 +1,100 @@ +--- +title: SIMBA-UQ Sampling Strategy +description: Confidence-aware sample selection for Mellea via similarity-based uncertainty quantification. +--- + +# SIMBA-UQ Sampling + +`SIMBAUQSamplingStrategy` is a confidence-aware sample selector. It generates +multiple samples across a range of temperatures, computes a pairwise similarity +matrix between them, and selects the sample with the highest estimated +confidence. + +Based on the SIMBA-UQ framework (Bhattacharjya et al., 2025), +[SIMBA UQ: Similarity-Based Aggregation for Uncertainty Quantification in Large Language Models](https://arxiv.org/abs/2510.13836). + +--- + +## What SIMBA-UQ Does + +- Generates `len(temperatures) * n_per_temp` samples for a single instruction. +- Builds an `N x N` pairwise similarity matrix (rouge, jaccard, sbert, difflib, or levenshtein). +- Estimates per-sample confidence with one of two methods: + - **aggregation** (data-free): aggregates each sample's similarity to the others. + - **classifier**: a trained probabilistic classifier predicts `P(correct)` from similarity features. +- Returns the most confident sample, with metadata stored on the selected + `ModelOutputThunk` under `mot._meta["simba_uq"]`. + + +Use SIMBA-UQ when you can afford multiple generations and want the answer the +model is most self-consistent about, rather than a single greedy sample. + + +--- + +## Install + +The strategy's `sbert` and `classifier` paths need extra dependencies: + +```bash +pip install "mellea-contribs-agent-utilities[simbauq]" +``` + +`rouge`, `jaccard`, `difflib`, and `levenshtein` metrics with the +`aggregation` method work without the extra. + +--- + +## When to Use SIMBA-UQ + +### Recommended for: +- Factual / short-answer questions where self-consistency signals correctness. +- Selecting among several candidate generations without a reference answer. +- Confidence-gated pipelines (abstain when confidence is low). + +### Avoid for: +- Single-sample deterministic generation (`temperatures` of length 1, `n_per_temp=1`). +- Long, open-ended outputs where surface similarity is a poor proxy for agreement. + +--- + +## Core API + +### Aggregation (data-free) + +```python +from mellea import start_session +from mellea_contribs.agent_utilities.core.simbauq import SIMBAUQSamplingStrategy + +m = start_session() +result = m.instruct( + "What is the capital of France?", + strategy=SIMBAUQSamplingStrategy( + temperatures=[0.3, 0.5, 0.7, 1.0], + n_per_temp=3, + similarity_metric="rouge", + confidence_method="aggregation", + aggregation="mean", + ), + return_sampling_results=True, +) + +best = result.result +meta = best._meta["simba_uq"] +print(meta["confidence"], str(best)) +``` + +### Classifier (trained) + +Provide either `training_samples` + `training_labels` (each group sized +`len(temperatures) * n_per_temp`) or a pre-fitted `classifier` with +`predict_proba`. See `examples/simbauq/` for the full four-variant walkthrough, +including Hugging Face training-data generation. + +--- + +## Attribution + +Original author: Radu Marinescu ([@radum2275](https://github.com/radum2275)), +IBM Research. Ported into mellea-contribs from upstream mellea +[PR #785](https://github.com/generative-computing/mellea/pull/785). diff --git a/agent-utilities/examples/README.md b/agent-utilities/examples/README.md index 3763795b..4f02b977 100644 --- a/agent-utilities/examples/README.md +++ b/agent-utilities/examples/README.md @@ -1,5 +1,9 @@ # Examples -Runnable examples for `mellea-contribs-agent-utilities` will live here. For now, -see the README at the package root and the `tests/` directory for working usage -of `top_k`, `double_round_robin`, and `benchdrift_runner`. +Runnable examples for `mellea-contribs-agent-utilities`. + +- `simbauq/` — SIMBA-UQ confidence-aware sampling strategy, demonstrating all + four confidence-estimation variants against Ollama. See `simbauq/README.md`. + +For `top_k`, `double_round_robin`, and `benchdrift_runner`, see the README at +the package root and the `tests/` directory for working usage. diff --git a/agent-utilities/examples/simbauq/README.md b/agent-utilities/examples/simbauq/README.md new file mode 100644 index 00000000..82fba2a5 --- /dev/null +++ b/agent-utilities/examples/simbauq/README.md @@ -0,0 +1,282 @@ +# SIMBA-UQ Sampling Strategy + +Confidence-aware sample selection using the SIMBA-UQ framework +(Bhattacharjya et al., 2025). Generates multiple samples across a range of +temperatures and selects the one with the highest estimated confidence. + +**Paper:** [SIMBA UQ: Similarity-Based Aggregation for Uncertainty Quantification in Large Language Models](https://arxiv.org/abs/2510.13836) + +## Install + +This example needs the `simbauq` extra: + +```bash +pip install "mellea-contribs-agent-utilities[simbauq]" +``` + +## Files + +### simbauq_example.py + +Complete example demonstrating all four confidence estimation variants with +Ollama and granite4:micro: + +1. **Aggregation** — data-free, no training data required. +2. **Classifier (synthetic)** — trained on hand-coded labeled groups. +3. **Classifier (HF data)** — training data generated live from a Hugging Face + dataset via Ollama. Calls `generate_training_data()` which streams items + from TriviaQA or SAMSum, generates `len(temperatures) * n_per_temp` responses + per item at the configured temperature schedule, and labels each response by + similarity to the ground-truth reference. Groups where all labels are + identical are discarded. Requires the `simbauq` extra (`pip install "mellea-contribs-agent-utilities[simbauq]"`). +4. **Classifier (pre-trained)** — same HF-generated training data, but the + `RandomForestClassifier` is trained externally via `train_classifier()` and + passed directly to `SIMBAUQSamplingStrategy` via the `classifier=` argument. + Useful when you want to persist, inspect, or swap the classifier independently + of the sampling strategy. + +## Running the example + +``` +ollama serve +uv run python examples/simbauq/simbauq_example.py +``` + +The script runs against the demo query +`"Which magazine was started first Arthur's Magazine or First for Women?"`. + +Which variant runs is controlled by the **CONFIG block** at the top of the +script (immediately below the imports). Edit the values in place: + +| Variable | Purpose | Allowed values | +|----------|---------|----------------| +| `EXAMPLE` | Which variant(s) to run | `"aggregation"`, `"synthetic"`, `"hf"`, `"pretrained"`, `"all"` | +| `DATASET` | HF dataset for the `hf` and `pretrained` variants | `"triviaqa"`, `"samsum"` | +| `METRIC` | Pairwise similarity metric (used by both the strategy and HF labelling) | `"rouge"`, `"jaccard"`, `"sbert"`, `"difflib"`, `"levenshtein"` | +| `AGGREGATION` | Aggregation function for the `aggregation` confidence method | `"mean"`, `"geometric_mean"`, `"harmonic_mean"`, `"median"`, `"max"`, `"min"` | +| `THRESHOLD` | Similarity score above which an HF-generated response is labelled correct (1) | float; tune per metric/dataset | + +The shipped defaults are `EXAMPLE="aggregation"`, `DATASET="triviaqa"`, +`METRIC="sbert"`, `AGGREGATION="mean"`, `THRESHOLD=0.2`. Set `EXAMPLE="all"` +to run all four variants sequentially. + +Reasonable starting points for `THRESHOLD`: + +- `sbert` + `triviaqa`: 0.5 - 0.7 +- `sbert` + `samsum`: 0.2 - 0.4 +- `rouge` + `triviaqa`: 0.3 - 0.5 + +Too strict drops every group; too loose makes every response "correct" and +the classifier sees no negatives. Groups where every response receives the +same label are discarded automatically. + +The number of HF training groups is controlled by the module-level constant +`N_TRAINING_GROUPS=5` — increase for stronger classifier signal at the cost +of more LLM calls. + +## Architecture + +``` +User Query + | + v +Generate N samples (across temperatures) + | + v +Compute pairwise similarity matrix (N x N) + | + +---> [Aggregation] Aggregate similarities per sample -> confidence + | + +---> [Classifier] Extract features per sample -> RF predicts P(correct) + | + v +Select sample with highest confidence + | + v +Result (with confidence metadata in mot.meta["simba_uq"]) +``` + +## Confidence Methods + +### 1. Aggregation (data-free) + +No training data required. For each sample, computes its similarity to every +other sample, then aggregates those values into a confidence score. Samples +that are more similar to the majority get higher confidence. + +```python +from mellea_contribs.agent_utilities.core.simbauq import SIMBAUQSamplingStrategy + +strategy = SIMBAUQSamplingStrategy( + temperatures=[0.3, 0.5, 0.7, 1.0], + n_per_temp=3, + similarity_metric="sbert", + confidence_method="aggregation", + aggregation="mean", +) + +result = m.instruct("Your query here", strategy=strategy, return_sampling_results=True) +``` + +### 2. Classifier (trained) + +Uses a random forest classifier trained on labeled examples. The classifier +learns to predict P(correct) from pairwise similarity features. Provide +either training data or a pre-trained sklearn classifier. + +Each training group must have exactly `len(temperatures) * n_per_temp` samples +so the feature vectors match at inference time. + +**Option A — synthetic training data:** + +With `temperatures=[0.3, 0.5, 0.7, 1.0]` and `n_per_temp=3`, each group must +contain exactly 12 samples (4 temperatures × 3 per temp). See +`run_classifier_synthetic_example()` in `simbauq_example.py` for a complete, +hand-coded example. + +```python +strategy = SIMBAUQSamplingStrategy( + temperatures=[0.3, 0.5, 0.7, 1.0], + n_per_temp=3, + similarity_metric="rouge", + confidence_method="classifier", + training_samples=[ + ["correct answer 1", "correct answer 2", ..., "wrong answer"], # group 1 (12 samples) + ["correct answer 1", "correct answer 2", ..., "wrong answer"], # group 2 (12 samples) + ], + training_labels=[ + [1, 1, ..., 0], # labels for group 1 (12 entries) + [1, 1, ..., 0], # labels for group 2 (12 entries) + ], +) +``` + +**Option B — HF-generated training data (requires the `simbauq` extra):** + +`generate_training_data()` in `simbauq_example.py` streams items from a HF +dataset, generates responses at each temperature, and labels them by similarity +to the ground-truth reference. Supported datasets: `"triviaqa"` (short QA) +and `"samsum"` (dialogue summarization). + +```python +from simbauq_example import generate_training_data, make_session + +m = make_session() +temperatures = [0.3, 0.5, 0.7, 1.0] +n_per_temp = 3 + +training_samples, training_labels = generate_training_data( + m, + temperatures, + n_per_temp, + dataset="triviaqa", # or "samsum" + similarity_metric="rouge", + threshold=0.5, # similarity >= threshold → label 1 +) + +strategy = SIMBAUQSamplingStrategy( + temperatures=temperatures, + n_per_temp=n_per_temp, + similarity_metric="rouge", + confidence_method="classifier", + training_samples=training_samples, + training_labels=training_labels, +) +``` + +Groups where all responses receive the same label are discarded automatically. +If no valid groups are collected, lower the `threshold` (scores are too low) +or raise it (all responses score above threshold). + +**Option C — pre-trained classifier:** + +Train the classifier externally with `train_classifier()`, then pass the fitted +object via `classifier=`. The feature extraction reuses +`SIMBAUQSamplingStrategy._compute_similarity_matrix` and `_extract_features` +internally, so the feature space is identical to what the strategy uses at +inference time. + +```python +from simbauq_example import generate_training_data, train_classifier, make_session + +m = make_session() +temperatures = [0.3, 0.5, 0.7, 1.0] +n_per_temp = 3 + +training_samples, training_labels = generate_training_data( + m, temperatures, n_per_temp, dataset="triviaqa", similarity_metric="rouge", threshold=0.5 +) + +clf = train_classifier(training_samples, training_labels, similarity_metric="rouge") + +strategy = SIMBAUQSamplingStrategy( + temperatures=temperatures, + n_per_temp=n_per_temp, + similarity_metric="rouge", + confidence_method="classifier", + classifier=clf, +) +``` + +## Constructor Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `temperatures` | `list[float]` | `[0.3, 0.5, 0.7, 1.0]` | Temperature values to sample at | +| `n_per_temp` | `int` | `4` | Number of samples per temperature | +| `similarity_metric` | `"rouge"`, `"jaccard"`, `"sbert"`, `"difflib"`, `"levenshtein"` | `"rouge"` | Pairwise similarity metric | +| `confidence_method` | `"aggregation"`, `"classifier"` | `"aggregation"` | Confidence estimation method | +| `aggregation` | `"mean"`, `"geometric_mean"`, `"harmonic_mean"`, `"median"`, `"max"`, `"min"` | `"mean"` | Aggregation function (for `aggregation` method) | +| `classifier` | sklearn classifier | `None` | Pre-trained classifier with `predict_proba` | +| `training_samples` | `list[list[str]]` | `None` | Training data for classifier | +| `training_labels` | `list[list[int]]` | `None` | Binary correctness labels (0/1) | +| `clf_max_depth` | `int` | `4` | Max tree depth for random forest | +| `rouge_type` | `str` | `"rougeL"` | Rouge variant | +| `sbert_model` | `str` | `"all-MiniLM-L6-v2"` | Sentence-BERT model name | +| `requirements` | `list[Requirement]` | `None` | Requirements to validate the selected sample | + +## Similarity Metrics + +- **rouge** (default): RougeL F-measure. Good general-purpose text similarity. + No extra dependencies beyond `rouge-score` (already in Mellea). +- **jaccard**: Word-level set overlap (intersection / union). Fast, no + external dependencies, works well for short structured answers. +- **sbert**: Cosine similarity of Sentence-BERT embeddings. Best semantic + similarity but requires `sentence-transformers`. +- **difflib**: `difflib.SequenceMatcher` ratio. Character-level similarity + from the Python standard library; no extra dependencies. +- **levenshtein**: Normalized Levenshtein edit distance (`1 - dist / max_len`). + Exact character-level metric; no extra dependencies. + +## Inspecting Results + +The selected sample's `ModelOutputThunk` stores confidence metadata: + +```python +result = m.instruct(..., strategy=strategy, return_sampling_results=True) + +# Best sample +best_mot = result.result +meta = best_mot._meta["simba_uq"] + +meta["confidence"] # float: confidence of the selected sample +meta["all_confidences"] # list[float]: confidence for every sample +meta["similarity_matrix"] # list[list[float]]: N x N pairwise similarity matrix +meta["temperatures_used"] # list[float]: temperature used for each sample +meta["confidence_method"] # "aggregation" or "classifier" +meta["similarity_metric"] # "rouge", "jaccard", "sbert", "difflib", or "levenshtein" +meta["aggregation"] # aggregation function name + +# All generated samples +for i, mot in enumerate(result.sample_generations): + print(f"Sample {i}: {mot.value}") +``` + +## Related Files + +- `mellea_contribs/agent_utilities/core/simbauq.py` -- Strategy implementation +- `tests/test_simbauq.py` -- Unit and integration tests + +## Attribution + +Original author: Radu Marinescu ([@radum2275](https://github.com/radum2275)), IBM Research. Ported into mellea-contribs from upstream mellea [PR #785](https://github.com/generative-computing/mellea/pull/785). diff --git a/agent-utilities/examples/simbauq/simbauq_example.py b/agent-utilities/examples/simbauq/simbauq_example.py new file mode 100644 index 00000000..506fbed3 --- /dev/null +++ b/agent-utilities/examples/simbauq/simbauq_example.py @@ -0,0 +1,651 @@ +# pytest: ollama, llm, qualitative + +"""SIMBA-UQ Sampling Strategy Example. + +Original author: Radu Marinescu (@radum2275), IBM Research. +Ported into mellea-contribs from upstream mellea PR #785. + +This example demonstrates the SIMBAUQSamplingStrategy using both confidence +estimation methods: + +1. **Aggregation** (data-free) - Computes pairwise similarity between all + generated samples and aggregates them into per-sample confidence scores. + The sample with the highest confidence is selected. + +2. **Classifier with synthetic data** - Uses a random forest classifier + trained on hand-coded labeled examples. + +3. **Classifier with HF data** - Same classifier method, but training data + is generated live via Ollama from one of two supported HF datasets: + ``"triviaqa"`` (short factoid QA) or ``"samsum"`` (dialogue summarization). + No other datasets are supported by ``generate_training_data()``. + +4. **Classifier with pre-trained classifier** - Trains a RandomForestClassifier + externally using HF-generated data, then passes the fitted object directly + to SIMBAUQSamplingStrategy via the ``classifier=`` argument. + +All variants generate multiple samples across different temperature values, +compute a pairwise similarity matrix, and select the most confident response. + +Available similarity metrics (set via ``METRIC`` in the CONFIG block below): +``"rouge"``, ``"jaccard"``, ``"sbert"``, ``"difflib"``, ``"levenshtein"``. + +To control which example runs, edit the CONFIG block immediately below the +imports — set ``EXAMPLE``, ``DATASET``, ``METRIC``, and ``THRESHOLD`` there. + +The example uses OllamaModelBackend with granite4:micro. To run: + + ollama serve + uv run python examples/simbauq/simbauq_example.py +""" + +from typing import Literal + +import numpy as np +from sklearn.ensemble import RandomForestClassifier # type: ignore[import-not-found] +from tqdm import tqdm + +from mellea import MelleaSession +from mellea.backends import ModelOption +from mellea.backends.ollama import OllamaModelBackend +from mellea.core import SamplingResult +from mellea.stdlib.context import ChatContext +from mellea_contribs.agent_utilities.core.simbauq import SIMBAUQSamplingStrategy + +# ============================================================================ +# CONFIG — edit these to control which example(s) run. +# ============================================================================ + +# Which example(s) to run. +# "aggregation" — data-free similarity aggregation +# "synthetic" — classifier with hand-coded labeled groups +# "hf" — classifier trained on data generated from an HF dataset +# "pretrained" — classifier trained externally, passed in via `classifier=` +# "all" — run all four sequentially +EXAMPLE = "aggregation" + +# HF dataset for the `hf` and `pretrained` examples. Only two datasets are +# supported by `generate_training_data()`: +# "triviaqa" — short factoid QA (rc.nocontext split) +# "samsum" — dialogue summarization +DATASET = "triviaqa" + +# Pairwise similarity metric. Used by both the strategy and the labelling +# pass in `generate_training_data()`. Available metrics: +# "rouge" — RougeL F-measure (default; needs `rouge-score`) +# "jaccard" — word-level set overlap, fast, no extra deps +# "sbert" — Sentence-BERT cosine; needs `sentence-transformers` +# "difflib" — `difflib.SequenceMatcher` ratio; stdlib only +# "levenshtein" — normalized edit distance; stdlib only +METRIC: Literal["rouge", "jaccard", "sbert", "difflib", "levenshtein"] = "sbert" + +# Aggregation method for the "aggregation" confidence method. Options: +# "mean" — Arithmetic mean (default) +# "geometric_mean" — Geometric mean +# "harmonic_mean" — Harmonic mean +# "median" — Median +# "max" — Maximum +# "min" — Minimum +AGGREGATION: Literal[ + "mean", "geometric_mean", "harmonic_mean", "median", "max", "min" +] = "mean" + +# Similarity threshold for labelling generated responses against the HF +# reference: score >= THRESHOLD → label 1 (correct), else 0. Tune per +# dataset/metric/model — too strict drops every group, too loose makes every +# response "correct" and the classifier sees no negatives. Groups where every +# response receives the same label are discarded automatically. +# Reasonable starting points: +# sbert + triviaqa: 0.5 - 0.7 +# sbert + samsum: 0.2 - 0.4 +# rouge + triviaqa: 0.3 - 0.5 +THRESHOLD = 0.2 + +# ============================================================================ + +# Allowed values, used for CONFIG validation in `main()`. +_VALID_EXAMPLES = ("aggregation", "synthetic", "hf", "pretrained", "all") +_VALID_DATASETS = ("triviaqa", "samsum") +_VALID_METRICS = ("rouge", "jaccard", "sbert", "difflib", "levenshtein") +_VALID_AGGREGATIONS = ( + "mean", + "geometric_mean", + "harmonic_mean", + "median", + "max", + "min", +) + +# Number of training groups collected per dataset. +# Each group has len(temperatures) * n_per_temp samples. +# Increase for better classifier signal at the cost of more LLM calls. +N_TRAINING_GROUPS = 5 + + +def make_session() -> MelleaSession: + """Create a MelleaSession with OllamaModelBackend.""" + backend = OllamaModelBackend(model_options={ModelOption.MAX_NEW_TOKENS: 150}) + return MelleaSession(backend, ctx=ChatContext()) + + +def print_results(result: SamplingResult) -> None: + """Print detailed results from a SIMBA-UQ sampling run.""" + meta = result.result._meta["simba_uq"] + confidences = meta["all_confidences"] + temperatures = meta["temperatures_used"] + sim_matrix = np.array(meta["similarity_matrix"]) + + # --- Best response --- + print("=" * 70) + print("BEST RESPONSE") + print("=" * 70) + print(f" Index: {result.result_index}") + print(f" Confidence: {meta['confidence']:.4f}") + print(f" Method: {meta['confidence_method']}") + print(f" Metric: {meta['similarity_metric']}") + print(f" Aggregation: {meta['aggregation']}") + print(f" Text:\n {result.result!s}") + print() + + # --- All samples --- + print("=" * 70) + print("ALL SAMPLES") + print("=" * 70) + print(f"{'Idx':>4} {'Temp':>5} {'Conf':>8} {'Text'}") + print("-" * 70) + for i, mot in enumerate(result.sample_generations): + text = str(mot).replace("\n", " ") + truncated = (text[:100] + "...") if len(text) > 100 else text + marker = " <-- best" if i == result.result_index else "" + print( + f"{i:>4} {temperatures[i]:>5.2f} {confidences[i]:>8.4f} " + f"{truncated}{marker}" + ) + print() + + # --- Similarity matrix --- + n = sim_matrix.shape[0] + print("=" * 70) + print("SIMILARITY MATRIX") + print("=" * 70) + header = " " + "".join(f" [{i:>2}] " for i in range(n)) + print(header) + for i in range(n): + row = f"[{i:>2}] " + "".join(f" {sim_matrix[i, j]:.3f} " for j in range(n)) + print(row) + print() + + +def generate_training_data( + session: MelleaSession, + temperatures: list[float], + n_per_temp: int, + dataset: str = "triviaqa", + similarity_metric: Literal[ + "rouge", "jaccard", "sbert", "difflib", "levenshtein" + ] = "rouge", + threshold: float = 0.5, + n_groups: int = N_TRAINING_GROUPS, +) -> tuple[list[list[str]], list[list[int]]]: + """Generate classifier training data from a single HF dataset via Ollama. + + For each dataset item, generates one group of ``len(temperatures) * + n_per_temp`` responses at the configured temperature schedule. Each + response is labelled 1 if its similarity to the ground-truth reference + meets ``threshold``, 0 otherwise. Groups where all labels are identical + are discarded as they provide no classifier signal. + + Args: + session: Active MelleaSession to use for generation. + temperatures: Temperature schedule (must match inference-time schedule). + n_per_temp: Samples per temperature (must match inference-time value). + dataset: HF dataset to use. One of ``"triviaqa"`` (short QA) or + ``"samsum"`` (dialogue summarization). + similarity_metric: Metric used for labelling (should match the + ``similarity_metric`` passed to SIMBAUQSamplingStrategy). + threshold: Similarity score >= threshold → label 1. + n_groups: Target number of valid groups to collect. + + Returns: + Tuple of (training_samples, training_labels), each a list of groups + with exactly ``len(temperatures) * n_per_temp`` entries per group. + """ + if dataset not in _VALID_DATASETS: + raise ValueError( + f"Unknown dataset {dataset!r}. Supported: {list(_VALID_DATASETS)}." + ) + + try: + from datasets import load_dataset # type: ignore[import-not-found] + except ImportError: + raise ImportError( + "The 'datasets' package is required for HF training data generation. " + "Install it with: pip install datasets" + ) + + group_size = len(temperatures) * n_per_temp + print(f"Generating training data with {group_size} samples per group ") + + # Reused solely for _compute_similarity — no training data needed at init. + scorer = SIMBAUQSamplingStrategy( + similarity_metric=similarity_metric, confidence_method="aggregation" + ) + + def _load_triviaqa(n: int) -> list[dict]: + ds = load_dataset("trivia_qa", "rc.nocontext", split="train", streaming=True) + items = [] + for row in ds: + ref = row.get("answer", {}).get("value", "") + if not ref: + continue + items.append( + { + "prompt": f"Answer the following question briefly: {row['question']}", + "reference": ref, + } + ) + if len(items) >= n: + break + return items + + def _load_samsum(n: int) -> list[dict]: + ds = load_dataset("samsum", split="train", streaming=True) + items = [] + for row in ds: + dialogue = row.get("dialogue", "")[:1000] + ref = row.get("summary", "") + if not dialogue or not ref: + continue + items.append( + { + "prompt": f"Summarize the following dialogue in one sentence:\n\n{dialogue}", + "reference": ref, + } + ) + if len(items) >= n: + break + return items + + loaders = {"triviaqa": _load_triviaqa, "samsum": _load_samsum} + print(f" Collecting {n_groups} training groups from {dataset}...") + items = loaders[dataset](n_groups * 3) + + training_samples: list[list[str]] = [] + training_labels: list[list[int]] = [] + collected = 0 + + with tqdm(total=n_groups, desc=f"Generating [{dataset}]", unit="group") as pbar: + for item in items: + if collected >= n_groups: + break + + responses: list[str] = [] + for temp in temperatures: + for _ in range(n_per_temp): + try: + mot = session.instruct( + item["prompt"], + model_options={ + ModelOption.TEMPERATURE: temp, + ModelOption.MAX_NEW_TOKENS: 150, + }, + ) + responses.append(str(mot)) + except Exception: + print( + f" Warning: generation failed for prompt: {item['prompt']!r}" + ) + responses.append("") + + scores = [ + scorer._compute_similarity(r, item["reference"]) for r in responses + ] + labels = [1 if s >= threshold else 0 for s in scores] + + print( + "Responses:\n " + + "\n ".join( + f"{i}. {r!r} (score={s:.4f}, label={label})" + for i, (r, s, label) in enumerate(zip(responses, scores, labels)) + ) + ) + if len(set(labels)) < 2: + pbar.set_postfix_str( + f"discarded (scores {min(scores):.2f}–{max(scores):.2f}, threshold={threshold})" + ) + continue + + training_samples.append(responses) + training_labels.append(labels) + collected += 1 + pbar.update(1) + + return training_samples, training_labels + + +def train_classifier( + training_samples: list[list[str]], + training_labels: list[list[int]], + similarity_metric: Literal[ + "rouge", "jaccard", "sbert", "difflib", "levenshtein" + ] = "rouge", + clf_max_depth: int = 4, +) -> RandomForestClassifier: + """Train a RandomForestClassifier on similarity features extracted from training data. + + Uses the same feature extraction as SIMBAUQSamplingStrategy internally + (_compute_similarity_matrix + _extract_features), ensuring the feature + space is identical at train and inference time. + + Args: + training_samples: List of groups, each with the same number of samples + as ``len(temperatures) * n_per_temp`` used at inference time. + training_labels: Binary correctness labels (0/1) matching + ``training_samples``. + similarity_metric: Similarity metric for feature extraction. + clf_max_depth: Maximum tree depth for the random forest. + + Returns: + Fitted RandomForestClassifier. + """ + extractor = SIMBAUQSamplingStrategy( + similarity_metric=similarity_metric, confidence_method="aggregation" + ) + + x_train: list[np.ndarray] = [] + y_train: list[int] = [] + for samples, labels in zip(training_samples, training_labels): + sim_matrix = extractor._compute_similarity_matrix(samples) + for i, label in enumerate(labels): + x_train.append(extractor._extract_features(sim_matrix, i)) + y_train.append(label) + + clf = RandomForestClassifier(max_depth=clf_max_depth, random_state=0) + clf.fit(x_train, y_train) + return clf + + +def run_aggregation_example( + session: MelleaSession, + similarity_metric: Literal["rouge", "jaccard", "sbert", "difflib", "levenshtein"], + aggregation: Literal[ + "mean", "geometric_mean", "harmonic_mean", "median", "max", "min" + ], +) -> None: + """Run SIMBA-UQ with data-free similarity aggregation.""" + print("\n>>> AGGREGATION CONFIDENCE METHOD <<<\n") + + strategy = SIMBAUQSamplingStrategy( + temperatures=[0.3, 0.5, 0.7, 1.0], + n_per_temp=3, + similarity_metric=similarity_metric, + confidence_method="aggregation", + aggregation=aggregation, + ) + + result: SamplingResult = session.instruct( + "Which magazine was started first Arthur's Magazine or First for Women?", + strategy=strategy, + return_sampling_results=True, + ) + + print(f"Total samples generated: {len(result.sample_generations)}") + print_results(result) + + +def run_classifier_synthetic_example( + session: MelleaSession, + similarity_metric: Literal["rouge", "jaccard", "sbert", "difflib", "levenshtein"], +) -> None: + """Run SIMBA-UQ classifier with hand-coded synthetic training data.""" + print("\n>>> CLASSIFIER CONFIDENCE METHOD (synthetic training data) <<<\n") + + temperatures = [0.3, 0.5, 0.7, 1.0] + n_per_temp = 3 + + # Synthetic training data: 3 groups of 12 samples (4 temps * 3 per temp). + # Each group has mostly "correct" similar answers and a few outliers. + training_samples = [ + [ + "Paris is the capital of France.", + "The capital of France is Paris.", + "France's capital city is Paris.", + "Paris, the capital of France.", + "The capital city of France is Paris.", + "France has Paris as its capital.", + "Paris serves as France's capital.", + "In France, Paris is the capital.", + "The French capital is Paris.", + "Bananas are a yellow fruit.", + "Dogs are loyal pets.", + "The ocean is very deep.", + ], + [ + "Water boils at 100 degrees Celsius.", + "At 100C water reaches boiling point.", + "The boiling point of water is 100 degrees.", + "Water boils when heated to 100C.", + "100 degrees Celsius is water's boiling point.", + "Boiling occurs at 100C for water.", + "Water starts boiling at one hundred degrees.", + "At 100 degrees water boils.", + "The temperature for boiling water is 100C.", + "Cats like to sleep a lot.", + "Mountains can be very high.", + "Stars shine in the night sky.", + ], + [ + "Python is a programming language.", + "Python is a popular programming language.", + "The Python programming language is widely used.", + "Python is used for programming.", + "Programming in Python is common.", + "Python is a well-known language for coding.", + "Many developers use Python.", + "Python is a general-purpose language.", + "The language Python is popular.", + "Pizza originated in Italy.", + "Rain falls from clouds.", + "Books contain many pages.", + ], + ] + training_labels = [ + [1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0], + [1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0], + [1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0], + ] + + strategy = SIMBAUQSamplingStrategy( + temperatures=temperatures, + n_per_temp=n_per_temp, + similarity_metric=similarity_metric, + confidence_method="classifier", + training_samples=training_samples, + training_labels=training_labels, + ) + + result: SamplingResult = session.instruct( + "Which magazine was started first Arthur's Magazine or First for Women?", + strategy=strategy, + return_sampling_results=True, + ) + + print(f"Total samples generated: {len(result.sample_generations)}") + print_results(result) + + +def run_classifier_hf_example( + session: MelleaSession, + dataset: str, + similarity_metric: Literal["rouge", "jaccard", "sbert", "difflib", "levenshtein"], + threshold: float, +) -> None: + """Run SIMBA-UQ classifier with training data generated from an HF dataset.""" + print(f"\n>>> CLASSIFIER CONFIDENCE METHOD (HF / {dataset}) <<<\n") + + temperatures = [0.3, 0.5, 0.7, 1.0] + n_per_temp = 3 + + print(f"Generating training data from {dataset}...") + training_samples, training_labels = generate_training_data( + session, + temperatures, + n_per_temp, + dataset=dataset, + similarity_metric=similarity_metric, + threshold=threshold, + ) + print( + f"Training data ready: {len(training_samples)} groups of {len(temperatures) * n_per_temp} samples each.\n" + ) + + if not training_samples: + print( + f" No valid training groups collected (threshold={threshold} may be " + "too strict or too loose for this model/dataset combination). " + "Try adjusting --threshold." + ) + return + + print("--- Training examples sample ---") + for group_idx, (samples, labels) in enumerate( + zip(training_samples, training_labels) + ): + correct = [s for s, lab in zip(samples, labels) if lab == 1] + incorrect = [s for s, lab in zip(samples, labels) if lab == 0] + print(f" Group {group_idx}:") + if correct: + print(f" [correct] {correct[0]!r}") + if incorrect: + print(f" [incorrect] {incorrect[0]!r}") + print() + + strategy = SIMBAUQSamplingStrategy( + temperatures=temperatures, + n_per_temp=n_per_temp, + similarity_metric=similarity_metric, + confidence_method="classifier", + training_samples=training_samples, + training_labels=training_labels, + ) + + result: SamplingResult = session.instruct( + "Which magazine was started first Arthur's Magazine or First for Women?", + strategy=strategy, + return_sampling_results=True, + ) + + print(f"Total samples generated: {len(result.sample_generations)}") + print_results(result) + + +def run_classifier_pretrained_example( + session: MelleaSession, + dataset: str, + similarity_metric: Literal["rouge", "jaccard", "sbert", "difflib", "levenshtein"], + threshold: float, +) -> None: + """Run SIMBA-UQ classifier with a pre-trained RandomForestClassifier.""" + print(f"\n>>> CLASSIFIER CONFIDENCE METHOD (pre-trained / {dataset}) <<<\n") + + temperatures = [0.3, 0.5, 0.7, 1.0] + n_per_temp = 3 + + print(f"Generating training data from {dataset}...") + training_samples, training_labels = generate_training_data( + session, + temperatures, + n_per_temp, + dataset=dataset, + similarity_metric=similarity_metric, + threshold=threshold, + ) + print( + f"Training data ready: {len(training_samples)} groups of {len(temperatures) * n_per_temp} samples each.\n" + ) + + if not training_samples: + print( + f" No valid training groups collected (threshold={threshold} may be " + "too strict or too loose for this model/dataset combination). " + "Try adjusting the threshold." + ) + return + + print("--- Training examples sample ---") + for group_idx, (samples, labels) in enumerate( + zip(training_samples, training_labels) + ): + correct = [s for s, lab in zip(samples, labels) if lab == 1] + incorrect = [s for s, lab in zip(samples, labels) if lab == 0] + print(f" Group {group_idx}:") + if correct: + print(f" [correct] {correct[0]!r}") + if incorrect: + print(f" [incorrect] {incorrect[0]!r}") + print() + + clf = train_classifier( + training_samples, training_labels, similarity_metric=similarity_metric + ) + print(f"Classifier trained: {clf}\n") + + strategy = SIMBAUQSamplingStrategy( + temperatures=temperatures, + n_per_temp=n_per_temp, + similarity_metric=similarity_metric, + confidence_method="classifier", + classifier=clf, + ) + + result: SamplingResult = session.instruct( + "Which magazine was started first Arthur's Magazine or First for Women?", + strategy=strategy, + return_sampling_results=True, + ) + + print(f"Total samples generated: {len(result.sample_generations)}") + print_results(result) + + +def main() -> None: + """Run the SIMBA-UQ example(s) selected via the CONFIG block at the top of this file.""" + if EXAMPLE not in _VALID_EXAMPLES: + raise ValueError( + f"Unknown EXAMPLE={EXAMPLE!r}. Choose one of: {list(_VALID_EXAMPLES)}." + ) + if DATASET not in _VALID_DATASETS: + raise ValueError( + f"Unknown DATASET={DATASET!r}. Choose one of: {list(_VALID_DATASETS)}." + ) + if METRIC not in _VALID_METRICS: + raise ValueError( + f"Unknown METRIC={METRIC!r}. Choose one of: {list(_VALID_METRICS)}." + ) + if AGGREGATION not in _VALID_AGGREGATIONS: + raise ValueError( + f"Unknown AGGREGATION={AGGREGATION!r}. Choose one of: {list(_VALID_AGGREGATIONS)}." + ) + + # Start a Mellea session with OllamaModelBackend. + m = make_session() + + runners = { + "aggregation": lambda: run_aggregation_example(m, METRIC, AGGREGATION), + "synthetic": lambda: run_classifier_synthetic_example(m, METRIC), + "hf": lambda: run_classifier_hf_example(m, DATASET, METRIC, THRESHOLD), + "pretrained": lambda: run_classifier_pretrained_example( + m, DATASET, METRIC, THRESHOLD + ), + } + + to_run = list(runners.values()) if EXAMPLE == "all" else [runners[EXAMPLE]] + + for i, run in enumerate(to_run): + if i > 0: + print("\n" + "=" * 70 + "\n") + run() + + +if __name__ == "__main__": + main() diff --git a/agent-utilities/mellea_contribs/agent_utilities/core/simbauq.py b/agent-utilities/mellea_contribs/agent_utilities/core/simbauq.py new file mode 100644 index 00000000..81d153ef --- /dev/null +++ b/agent-utilities/mellea_contribs/agent_utilities/core/simbauq.py @@ -0,0 +1,634 @@ +"""SIMBA-UQ Sampling Strategy. + +Original author: Radu Marinescu (@radum2275), IBM Research. +Ported into mellea-contribs from upstream mellea PR #785 +(https://github.com/generative-computing/mellea/pull/785). + +Implements confidence-aware sample selection using the SIMBA-UQ framework +(Bhattacharjya et al., 2025). Generates multiple samples across a range of +temperatures and selects the most confident one. + +Two confidence estimation methods are supported: + +* **aggregation** (data-free) — computes pairwise similarity between all + samples, then aggregates per-sample similarities into a confidence score. +* **classifier** — extracts pairwise similarity features and feeds them into + a trained probabilistic classifier (e.g. random forest) that predicts + P(correct) for each sample. + +Reference: + Bhattacharjya et al. (2025), "SIMBA UQ: Similarity-Based Aggregation for + Uncertainty Quantification in Large Language Models", https://arxiv.org/abs/2510.13836 +""" + +import asyncio +from copy import deepcopy +from difflib import SequenceMatcher +from typing import Literal, Protocol, runtime_checkable + +import numpy as np +from rouge_score.rouge_scorer import RougeScorer + +from mellea.core.base import ComponentParseError +from mellea.core.utils import MelleaLogger + +from mellea.core import ( + Backend, + BaseModelSubclass, + Component, + Context, + Requirement, + S, + SamplingResult, + SamplingStrategy, + ValidationResult, +) +from mellea.stdlib import functional as mfuncs + + +@runtime_checkable +class ProbabilisticClassifier(Protocol): + """Protocol for sklearn-compatible probabilistic classifiers.""" + + def predict_proba(self, X: list[np.ndarray]) -> np.ndarray: + """Return class probability estimates for the given samples.""" + ... + + +class SIMBAUQSamplingStrategy(SamplingStrategy): + """Sampling strategy that selects the most confident sample using SIMBA-UQ. + + Generates ``len(temperatures) * n_per_temp`` samples across a range of + temperature values, computes pairwise similarity between all samples, and + uses either similarity aggregation or a trained classifier to estimate + per-sample confidence. The sample with the highest confidence is returned. + + Confidence metadata is stored on the selected ``ModelOutputThunk`` in + ``mot.meta['simba_uq']``. + + Unlike BaseSamplingStrategy, merges both global and per-call requirements. + + Args: + temperatures (list[float]): Temperature values to sample at. + n_per_temp (int): Number of samples to generate per temperature value. + similarity_metric (Literal['rouge', 'jaccard', 'sbert', 'difflib', + 'levenshtein']): Pairwise similarity metric. ``'rouge'`` uses + RougeL F-measure; ``'jaccard'`` uses word-level Jaccard index; + ``'sbert'`` uses cosine similarity of Sentence-BERT embeddings + (requires ``sentence-transformers``); ``'difflib'`` uses + ``difflib.SequenceMatcher`` ratio; ``'levenshtein'`` uses + normalized Levenshtein edit distance. + confidence_method (Literal['aggregation', 'classifier']): How to + compute confidence from the similarity matrix. ``'aggregation'`` + uses a data-free aggregation function; ``'classifier'`` uses a + trained probabilistic classifier. + aggregation (Literal['mean', 'geometric_mean', 'harmonic_mean', + 'median', 'max', 'min']): Aggregation function used when + ``confidence_method='aggregation'``. + classifier (ProbabilisticClassifier | None): Pre-trained + sklearn-compatible probabilistic classifier (any estimator with a + ``predict_proba`` method). Used when + ``confidence_method='classifier'``. If not provided, a random + forest is trained from ``training_samples`` and + ``training_labels``. + training_samples (list[list[str]] | None): Training data for the + classifier — a list of query groups, each containing sample + strings. Each group must have the same number of samples as + ``len(temperatures) * n_per_temp``. + training_labels (list[list[int]] | None): Binary correctness labels + (0/1) matching ``training_samples``. + clf_max_depth (int): Maximum tree depth for the random forest when + training from data. + rouge_type (str): Rouge variant when ``similarity_metric='rouge'``. + sbert_model (str): Sentence-BERT model name when + ``similarity_metric='sbert'``. + requirements (list[Requirement] | None): Optional global requirements + to validate the selected sample against. + """ + + _CLF_EPS = 1e-6 + + def __init__( + self, + *, + temperatures: list[float] | None = None, + n_per_temp: int = 4, + similarity_metric: Literal[ + "rouge", "jaccard", "sbert", "difflib", "levenshtein" + ] = "rouge", + confidence_method: Literal["aggregation", "classifier"] = "aggregation", + aggregation: Literal[ + "mean", "geometric_mean", "harmonic_mean", "median", "max", "min" + ] = "mean", + classifier: ProbabilisticClassifier | None = None, + training_samples: list[list[str]] | None = None, + training_labels: list[list[int]] | None = None, + clf_max_depth: int = 4, + clf_random_state: int | None = 0, + rouge_type: str = "rougeL", + sbert_model: str = "all-MiniLM-L6-v2", + requirements: list[Requirement] | None = None, + ) -> None: + """Initialize SIMBAUQSamplingStrategy with temperature schedule and confidence parameters.""" + if temperatures is None: + temperatures = [0.3, 0.5, 0.7, 1.0] + + if len(temperatures) == 0: + raise ValueError("Temperatures must be a non-empty list") + if n_per_temp <= 0: + raise ValueError("n_per_temp must be > 0") + if confidence_method == "classifier" and len(temperatures) * n_per_temp <= 1: + raise ValueError( + "classifier mode requires len(temperatures) * n_per_temp >= 2" + ) + + self.temperatures = temperatures + self.n_per_temp = n_per_temp + self.similarity_metric = similarity_metric + self.confidence_method = confidence_method + self.aggregation = aggregation + self.clf_max_depth = clf_max_depth + self.clf_random_state = clf_random_state + self.rouge_type = rouge_type + self.sbert_model = sbert_model + self.requirements = requirements + + # --- Similarity metric initialization --- + if similarity_metric == "rouge": + self._rouge_scorer = RougeScorer([rouge_type], use_stemmer=True) + elif similarity_metric == "sbert": + try: + import sentence_transformers # type: ignore[import-not-found] + except ImportError: + msg = ( + "sentence-transformers is required for sbert similarity. " + "Please install with extra dependencies: `pip install 'mellea-contribs-agent-utilities[simbauq]'`." + ) + raise ImportError(msg) + self._sbert_model_obj = sentence_transformers.SentenceTransformer( + sbert_model + ) + + # --- Classifier initialization --- + self._classifier: ProbabilisticClassifier | None = None + if confidence_method == "classifier": + if classifier is not None: + self._classifier = classifier + + # If a classifier is provided, do a sanity check to ensure the feature + # dimensionality matches the expected number of samples. + expected = len(temperatures) * n_per_temp - 1 + n_features = getattr(classifier, "n_features_in_", None) + if n_features is not None and n_features != expected: + raise ValueError( + f"Classifier expects {n_features} features but this configuration " + f"produces {expected} (len(temperatures) * n_per_temp - 1)." + ) + elif training_samples is not None and training_labels is not None: + n_samples = len(temperatures) * n_per_temp + for i, group in enumerate(training_samples): + msg = ( + f"Training group {i} has {len(group)} samples, " + f"expected {n_samples} " + f"(len(temperatures) * n_per_temp)" + ) + if len(group) != n_samples: + raise ValueError(msg) + + msg = ( + f"Training labels group {i} has " + f"{len(training_labels[i])} labels, " + f"expected {n_samples}" + ) + if len(training_labels[i]) != n_samples: + raise ValueError(msg) + + self._classifier = self._train_classifier( + training_samples, training_labels + ) + else: + msg = ( + "confidence_method='classifier' requires either a " + "'classifier' or both 'training_samples' and " + "'training_labels'" + ) + raise ValueError(msg) + + async def sample( + self, + action: Component[S], + context: Context, + backend: Backend, + requirements: list[Requirement] | None, + *, + validation_ctx: Context | None = None, + format: type[BaseModelSubclass] | None = None, + model_options: dict | None = None, + tool_calls: bool = False, + ) -> SamplingResult[S]: + """Sample across temperatures and select the most confident result. + + Args: + action: The action object to be sampled. + context: The context to be passed to the sampling strategy. + backend: The backend used for generating samples. + requirements: List of requirements to test against (merged with + global requirements). + validation_ctx: Optional context to use for validation. + format: Output format for structured outputs. + model_options: Model options to pass to the backend during + generation. + tool_calls: True if tool calls should be used during this sampling + strategy. + + Returns: + SamplingResult with the most confident sample selected. + """ + if model_options is None: + model_options = {} + + # Merge requirements: global requirements override local. + reqs = self._merge_requirements(requirements) + + # --- Phase 1: Generate samples across temperatures --- + generation_tasks: list[asyncio.Task] = [] + task_actions: list[Component[S]] = [] + task_temps: list[float] = [] + + for temp in self.temperatures: + for _ in range(self.n_per_temp): + opts = {**model_options, "temperature": temp} + task_action = deepcopy(action) + task = asyncio.create_task( + backend.generate_from_context( + task_action, + ctx=context, + format=format, + model_options=opts, + tool_calls=tool_calls, + ) + ) + generation_tasks.append(task) + task_actions.append(task_action) + task_temps.append(temp) + + generation_results = await asyncio.gather( + *generation_tasks, return_exceptions=True + ) + + # Resolve all thunks and parse. Skip failed tasks but keep + # all_mots / all_contexts / all_actions / temp_assignments aligned positionally. + all_mots = [] + all_contexts = [] + all_actions: list[Component[S]] = [] + temp_assignments: list[float] = [] + for gen_result, task_action, task_temp in zip( + generation_results, task_actions, task_temps + ): + if isinstance(gen_result, BaseException): + continue # Skip failed generations. + result_mot, result_ctx = gen_result + await result_mot.avalue() + try: + result_mot.parsed_repr = task_action.parse(result_mot) + except ComponentParseError as e: + print(f"Error parsing result: {e}") + continue # Skip unparsable results. + all_mots.append(result_mot) + all_contexts.append(result_ctx) + all_actions.append(task_action) + temp_assignments.append(task_temp) + + flog = MelleaLogger.get_logger() + + # --- Phase 2: Compute SIMBA-UQ confidence scores --- + sample_strings = [str(mot) for mot in all_mots] + degraded = False + n = len(sample_strings) + if n == 0: + raise RuntimeError("No successful samples were generated.") + elif n == 1: + sim_matrix = np.ones((1, 1)) + confidences = np.array([0.5]) + degraded = True + flog.warning( + "Only one successful sample generated; SIMBA-UQ confidence estimation is degraded." + ) + else: + sim_matrix = self._compute_similarity_matrix(sample_strings) + if self.confidence_method == "classifier": + confidences = self._compute_confidences_classifier(sim_matrix, n) + else: + confidences = self._compute_confidences(sim_matrix) + + # Select the sample with the highest confidence. + best_index = int(np.argmax(confidences)) + best_confidence = float(confidences[best_index]) + + # Store confidence metadata in the selected MOT's meta dict. + # TODO: At the moment the SIMBAUQ sampling strategy metadata is stored + # in the _meta dictionary under the `simba_uq` key; this may lead to silent + # conflicts later on if other strategies also use the same key. + best_mot = all_mots[best_index] + if best_mot._meta is None: + best_mot._meta = {} + best_mot._meta["simba_uq"] = { + "degraded": degraded, + "confidence": best_confidence if not degraded else None, + "all_confidences": confidences.tolist(), + "similarity_matrix": sim_matrix.tolist(), + "temperatures_used": temp_assignments, + "confidence_method": self.confidence_method, + "similarity_metric": self.similarity_metric, + "aggregation": self.aggregation, + } + + # Mark as final result. + if best_mot._generate_log is not None: + best_mot._generate_log.is_final_result = True + + # --- Phase 3: Validate selected sample (if requirements exist) --- + success = True + all_validations: list[list[tuple[Requirement, ValidationResult]]] = [ + [] for _ in all_mots + ] + + validation_ctx = ( + validation_ctx if validation_ctx is not None else all_contexts[best_index] + ) + + if reqs: + val_results = await mfuncs.avalidate( + reqs=reqs, + context=validation_ctx, + backend=backend, + output=best_mot, + format=None, + model_options=model_options, + ) + scored = list(zip(reqs, val_results)) + all_validations[best_index] = scored + success = all(vr.as_bool() for vr in val_results) + + return SamplingResult( + result_index=best_index, + success=success, + sample_generations=all_mots, + sample_validations=all_validations, + sample_actions=all_actions, + sample_contexts=all_contexts, + ) + + def _merge_requirements(self, local: list[Requirement] | None) -> list[Requirement]: + """Merge global and local requirements, deduplicating by identity.""" + combined: list[Requirement] = [] + seen: set[int] = set() + for req_list in (self.requirements, local): + if req_list is None: + continue + for req in req_list: + if id(req) not in seen: + combined.append(req) + seen.add(id(req)) + return combined + + @staticmethod + def _levenshtein_distance(s1: str, s2: str) -> int: + """Compute the Levenshtein edit distance between two strings.""" + m, n = len(s1), len(s2) + dp = list(range(n + 1)) + for i in range(1, m + 1): + prev = dp[0] + dp[0] = i + for j in range(1, n + 1): + temp = dp[j] + if s1[i - 1] == s2[j - 1]: + dp[j] = prev + else: + dp[j] = 1 + min(prev, dp[j], dp[j - 1]) + prev = temp + return dp[n] + + def _compute_similarity(self, text1: str, text2: str) -> float: + """Compute pairwise similarity between two text strings. + + Args: + text1: First text. + text2: Second text. + + Returns: + Similarity score in [0.0, 1.0]. + """ + if self.similarity_metric == "rouge": + scores = self._rouge_scorer.score(text1, text2) + return scores[self.rouge_type].fmeasure + + if self.similarity_metric == "sbert": + try: + from sklearn.metrics.pairwise import ( + cosine_similarity, # type: ignore[import-not-found] + ) + except ImportError: + msg = ( + "sklearn.metrics.pairwise.cosine_similarity is required for sbert similarity. " + "Please install with extra dependencies: `pip install 'mellea-contribs-agent-utilities[simbauq]'`." + ) + raise ImportError(msg) + + embs = self._sbert_model_obj.encode([text1, text2]) + return float(cosine_similarity([embs[0]], [embs[1]])[0, 0]) + + if self.similarity_metric == "difflib": + return SequenceMatcher(None, text1, text2).ratio() + + if self.similarity_metric == "levenshtein": + dist = self._levenshtein_distance(text1, text2) + max_len = max(len(text1), len(text2)) + return 1.0 - dist / max_len if max_len > 0 else 1.0 + + if self.similarity_metric == "jaccard": + # Jaccard: word-level set overlap. + words1 = set(text1.lower().split()) + words2 = set(text2.lower().split()) + if len(words1) == 0 and len(words2) == 0: + return 1.0 + union = len(words1 | words2) + return len(words1 & words2) / union if union > 0 else 0.0 + + msg = f"Unknown similarity metric: {self.similarity_metric!r}" + raise ValueError(msg) + + def _compute_similarity_matrix(self, samples: list[str]) -> np.ndarray: + """Build a symmetric pairwise similarity matrix. + + For ``sbert``, batch-encodes all samples once and computes cosine + similarity in a single matrix operation. For ``rouge`` and ``jaccard``, + computes pairwise similarities individually (upper triangle, mirrored). + + Args: + samples: List of sample strings. + + Returns: + Symmetric (N, N) matrix with self-similarity = 1.0. + """ + if self.similarity_metric == "sbert": + try: + from sklearn.metrics.pairwise import ( + cosine_similarity, # type: ignore[import-not-found] + ) + except ImportError: + msg = ( + "sklearn.metrics.pairwise.cosine_similarity is required for sbert similarity. " + "Please install with extra dependencies: `pip install 'mellea-contribs-agent-utilities[simbauq]'`." + ) + raise ImportError(msg) + + embeddings = self._sbert_model_obj.encode(samples) + matrix = cosine_similarity(embeddings) + np.fill_diagonal(matrix, 1.0) + return matrix + + n = len(samples) + matrix = np.eye(n) + for i in range(n): + for j in range(i + 1, n): + sim = self._compute_similarity(samples[i], samples[j]) + matrix[i, j] = sim + matrix[j, i] = sim + return matrix + + def _aggregate(self, similarities: np.ndarray) -> float: + """Aggregate a vector of similarity scores into a single confidence value. + + Args: + similarities: 1-D array of similarity scores. + + Returns: + Aggregated confidence score. + """ + epsilon = 1e-10 + if len(similarities) == 0: + return 0.0 + + if self.aggregation == "mean": + return float(np.mean(similarities)) + + if self.aggregation == "geometric_mean": + similarities = np.clip(similarities, 0.0, 1.0) + log_sims = np.log(similarities + epsilon) + return float(np.exp(np.mean(log_sims))) + + if self.aggregation == "harmonic_mean": + similarities = np.clip(similarities, 0.0, 1.0) + return float(len(similarities) / np.sum(1.0 / (similarities + epsilon))) + + if self.aggregation == "median": + return float(np.median(similarities)) + + if self.aggregation == "max": + return float(np.max(similarities)) + + if self.aggregation == "min": + return float(np.min(similarities)) + + msg = f"Unknown aggregation method: {self.aggregation}" + raise ValueError(msg) + + def _extract_features( + self, sim_matrix: np.ndarray, sample_index: int + ) -> np.ndarray: + """Extract pairwise similarity features for a single sample. + + Returns the similarity row with self-similarity removed and values + clipped to ``(eps, 1 - eps)`` for numerical stability. + + Args: + sim_matrix: Symmetric (N, N) similarity matrix. + sample_index: Index of the sample to extract features for. + + Returns: + 1-D feature array of length ``N - 1``. + """ + row = np.delete(sim_matrix[sample_index, :], sample_index) + return np.clip(row, self._CLF_EPS, 1.0 - self._CLF_EPS) + + def _train_classifier( + self, training_samples: list[list[str]], training_labels: list[list[int]] + ) -> ProbabilisticClassifier: + """Train a random forest classifier on similarity features. + + Args: + training_samples: List of query groups, each a list of sample + strings with the same length as the inference-time sample + count. + training_labels: Binary correctness labels (0/1) matching + ``training_samples``. + + Returns: + Trained ``RandomForestClassifier``. + """ + try: + from sklearn.ensemble import ( + RandomForestClassifier, # type: ignore[import-not-found] + ) + except ImportError: + msg = ( + "sklearn is required for training a Random Forest classifier. " + "Please install with extra dependencies: `pip install 'mellea-contribs-agent-utilities[simbauq]'`." + ) + raise ImportError(msg) + + x_train: list[np.ndarray] = [] + y_train: list[int] = [] + for samples, labels in zip(training_samples, training_labels): + sim_matrix = self._compute_similarity_matrix(samples) + for i, label in enumerate(labels): + x_train.append(self._extract_features(sim_matrix, i)) + y_train.append(label) + clf = RandomForestClassifier( + max_depth=self.clf_max_depth, random_state=self.clf_random_state + ) + clf.fit(x_train, y_train) + return clf + + def _compute_confidences_classifier( + self, sim_matrix: np.ndarray, n: int + ) -> np.ndarray: + """Compute per-sample confidence using the trained classifier. + + Args: + sim_matrix: Pre-computed (N, N) similarity matrix. + n: Number of samples. + + Returns: + Array of P(correct) confidence scores with shape ``(n,)``. + """ + x_test = [self._extract_features(sim_matrix, i) for i in range(n)] + if self._classifier is None: + raise RuntimeError( + "Classifier is not initialised — this is a bug in SIMBAUQSamplingStrategy." + ) + probs = self._classifier.predict_proba(x_test) + return probs[:, 1] + + def _compute_confidences(self, sim_matrix: np.ndarray) -> np.ndarray: + """Compute per-sample confidence using similarity-based aggregation. + + For each sample, aggregates its similarities to every other sample + into a single confidence score. + + Args: + sim_matrix: Symmetric (N, N) pairwise similarity matrix. + + Returns: + Array of confidence scores with shape ``(N,)``. + """ + n = sim_matrix.shape[0] + if n == 1: + return np.array([0.5]) + + confidences = np.zeros(n) + for i in range(n): + others = np.concatenate([sim_matrix[i, :i], sim_matrix[i, i + 1 :]]) + confidences[i] = self._aggregate(others) + return confidences diff --git a/agent-utilities/pyproject.toml b/agent-utilities/pyproject.toml index ba9bebbe..beeb85e2 100644 --- a/agent-utilities/pyproject.toml +++ b/agent-utilities/pyproject.toml @@ -32,6 +32,10 @@ Repository = "https://github.com/generative-computing/mellea-contribs" [project.optional-dependencies] robustness = ["benchdrift"] +# SIMBA-UQ confidence-aware sample selector. scikit-learn + sentence-transformers +# are needed by the strategy (classifier and sbert paths); tqdm + datasets are +# used only by the example's Hugging Face training-data generation. +simbauq = ["scikit-learn", "sentence-transformers", "tqdm", "datasets"] [build-system] requires = ["hatchling"] @@ -76,4 +80,5 @@ markers = [ "integration: requires local services", "e2e: end-to-end against an LLM", "qualitative: checks LLM output quality", + "ollama: requires a local Ollama server", ] diff --git a/agent-utilities/tests/test_simbauq.py b/agent-utilities/tests/test_simbauq.py new file mode 100644 index 00000000..d023616c --- /dev/null +++ b/agent-utilities/tests/test_simbauq.py @@ -0,0 +1,431 @@ +"""Tests for SIMBAUQSamplingStrategy.""" + +import numpy as np +import pytest + +from mellea_contribs.agent_utilities.core.simbauq import SIMBAUQSamplingStrategy + +# --- Unit tests (no LLM required) --- + + +class TestComputeSimilarity: + def test_rouge_identical(self): + strategy = SIMBAUQSamplingStrategy(similarity_metric="rouge") + assert strategy._compute_similarity( + "hello world", "hello world" + ) == pytest.approx(1.0) + + def test_rouge_different(self): + strategy = SIMBAUQSamplingStrategy(similarity_metric="rouge") + score = strategy._compute_similarity("the cat sat on the mat", "dogs run fast") + assert 0.0 <= score < 0.5 + + def test_jaccard_identical(self): + strategy = SIMBAUQSamplingStrategy(similarity_metric="jaccard") + assert strategy._compute_similarity( + "hello world", "hello world" + ) == pytest.approx(1.0) + + def test_jaccard_partial_overlap(self): + strategy = SIMBAUQSamplingStrategy(similarity_metric="jaccard") + score = strategy._compute_similarity("hello world foo", "hello world bar") + # intersection = {"hello", "world"}, union = {"hello", "world", "foo", "bar"} + assert score == pytest.approx(2.0 / 4.0) + + def test_jaccard_no_overlap(self): + strategy = SIMBAUQSamplingStrategy(similarity_metric="jaccard") + score = strategy._compute_similarity("alpha beta", "gamma delta") + assert score == pytest.approx(0.0) + + def test_jaccard_empty_strings(self): + strategy = SIMBAUQSamplingStrategy(similarity_metric="jaccard") + assert strategy._compute_similarity("", "") == pytest.approx(1.0) + + def test_difflib_identical(self): + strategy = SIMBAUQSamplingStrategy(similarity_metric="difflib") + assert strategy._compute_similarity( + "hello world", "hello world" + ) == pytest.approx(1.0) + + def test_difflib_different(self): + strategy = SIMBAUQSamplingStrategy(similarity_metric="difflib") + score = strategy._compute_similarity("the cat sat on the mat", "dogs run fast") + assert 0.0 <= score < 0.5 + + def test_difflib_partial(self): + strategy = SIMBAUQSamplingStrategy(similarity_metric="difflib") + score = strategy._compute_similarity("hello world foo", "hello world bar") + assert 0.5 < score < 1.0 + + def test_difflib_empty(self): + strategy = SIMBAUQSamplingStrategy(similarity_metric="difflib") + assert strategy._compute_similarity("", "") == pytest.approx(1.0) + + def test_levenshtein_identical(self): + strategy = SIMBAUQSamplingStrategy(similarity_metric="levenshtein") + assert strategy._compute_similarity( + "hello world", "hello world" + ) == pytest.approx(1.0) + + def test_levenshtein_different(self): + strategy = SIMBAUQSamplingStrategy(similarity_metric="levenshtein") + score = strategy._compute_similarity("abc", "xyz") + assert score == pytest.approx(0.0) + + def test_levenshtein_partial(self): + strategy = SIMBAUQSamplingStrategy(similarity_metric="levenshtein") + score = strategy._compute_similarity("kitten", "sitting") + assert 0.0 < score < 1.0 + + def test_levenshtein_empty(self): + strategy = SIMBAUQSamplingStrategy(similarity_metric="levenshtein") + assert strategy._compute_similarity("", "") == pytest.approx(1.0) + + def test_levenshtein_single_edit(self): + strategy = SIMBAUQSamplingStrategy(similarity_metric="levenshtein") + score = strategy._compute_similarity("abcd", "abce") + assert score == pytest.approx(0.75) + + +class TestLevenshteinDistance: + def test_empty_strings(self): + assert SIMBAUQSamplingStrategy._levenshtein_distance("", "") == 0 + + def test_one_empty(self): + assert SIMBAUQSamplingStrategy._levenshtein_distance("a", "") == 1 + + def test_classic_example(self): + assert SIMBAUQSamplingStrategy._levenshtein_distance("kitten", "sitting") == 3 + + def test_identical(self): + assert SIMBAUQSamplingStrategy._levenshtein_distance("abc", "abc") == 0 + + +class TestAggregate: + def setup_method(self): + self.sims = np.array([0.8, 0.6, 0.4]) + + def test_mean(self): + strategy = SIMBAUQSamplingStrategy(aggregation="mean") + assert strategy._aggregate(self.sims) == pytest.approx(0.6) + + def test_median(self): + strategy = SIMBAUQSamplingStrategy(aggregation="median") + assert strategy._aggregate(self.sims) == pytest.approx(0.6) + + def test_max(self): + strategy = SIMBAUQSamplingStrategy(aggregation="max") + assert strategy._aggregate(self.sims) == pytest.approx(0.8) + + def test_min(self): + strategy = SIMBAUQSamplingStrategy(aggregation="min") + assert strategy._aggregate(self.sims) == pytest.approx(0.4) + + def test_geometric_mean(self): + strategy = SIMBAUQSamplingStrategy(aggregation="geometric_mean") + expected = (0.8 * 0.6 * 0.4) ** (1.0 / 3.0) + assert strategy._aggregate(self.sims) == pytest.approx(expected, abs=1e-3) + + def test_harmonic_mean(self): + strategy = SIMBAUQSamplingStrategy(aggregation="harmonic_mean") + expected = 3.0 / (1.0 / 0.8 + 1.0 / 0.6 + 1.0 / 0.4) + assert strategy._aggregate(self.sims) == pytest.approx(expected, abs=1e-3) + + def test_empty(self): + strategy = SIMBAUQSamplingStrategy(aggregation="mean") + assert strategy._aggregate(np.array([])) == 0.0 + + +class TestComputeConfidences: + def test_single_sample(self): + strategy = SIMBAUQSamplingStrategy(similarity_metric="jaccard") + sim_matrix = strategy._compute_similarity_matrix(["hello world"]) + confs = strategy._compute_confidences(sim_matrix) + assert len(confs) == 1 + assert confs[0] == pytest.approx(0.5) + + def test_identical_samples_high_confidence(self): + strategy = SIMBAUQSamplingStrategy( + similarity_metric="jaccard", aggregation="mean" + ) + samples = ["the cat sat on the mat"] * 5 + sim_matrix = strategy._compute_similarity_matrix(samples) + confs = strategy._compute_confidences(sim_matrix) + assert len(confs) == 5 + for c in confs: + assert c == pytest.approx(1.0) + + def test_outlier_has_lower_confidence(self): + strategy = SIMBAUQSamplingStrategy( + similarity_metric="jaccard", aggregation="mean" + ) + samples = [ + "the capital of france is paris", + "paris is the capital of france", + "france capital is paris", + "bananas are yellow fruit", # outlier + ] + sim_matrix = strategy._compute_similarity_matrix(samples) + confs = strategy._compute_confidences(sim_matrix) + assert len(confs) == 4 + # The outlier (index 3) should have the lowest confidence. + assert confs[3] < confs[0] + assert confs[3] < confs[1] + assert confs[3] < confs[2] + + def test_similarity_matrix_symmetric(self): + strategy = SIMBAUQSamplingStrategy(similarity_metric="rouge") + samples = ["hello world", "world hello", "foo bar"] + matrix = strategy._compute_similarity_matrix(samples) + assert matrix.shape == (3, 3) + np.testing.assert_array_almost_equal(matrix, matrix.T) + np.testing.assert_array_equal(np.diag(matrix), [1.0, 1.0, 1.0]) + + +class TestSBERTSimilarity: + @pytest.fixture(autouse=True) + def _require_sbert(self): + pytest.importorskip("sentence_transformers") + + def test_sbert_identical(self): + strategy = SIMBAUQSamplingStrategy(similarity_metric="sbert") + score = strategy._compute_similarity("hello world", "hello world") + assert score == pytest.approx(1.0, abs=0.01) + + def test_sbert_similar(self): + strategy = SIMBAUQSamplingStrategy(similarity_metric="sbert") + score = strategy._compute_similarity( + "The capital of France is Paris.", "Paris is the capital city of France." + ) + assert score > 0.7 + + def test_sbert_different(self): + strategy = SIMBAUQSamplingStrategy(similarity_metric="sbert") + score = strategy._compute_similarity( + "The capital of France is Paris.", "Bananas are a yellow tropical fruit." + ) + assert score < 0.4 + + def test_sbert_matrix_symmetric(self): + strategy = SIMBAUQSamplingStrategy(similarity_metric="sbert") + samples = ["hello world", "world hello", "foo bar baz"] + matrix = strategy._compute_similarity_matrix(samples) + assert matrix.shape == (3, 3) + np.testing.assert_array_almost_equal(matrix, matrix.T) + np.testing.assert_array_almost_equal( + np.diag(matrix), [1.0, 1.0, 1.0], decimal=2 + ) + + def test_sbert_outlier_confidence(self): + strategy = SIMBAUQSamplingStrategy( + similarity_metric="sbert", aggregation="mean" + ) + samples = [ + "The capital of France is Paris.", + "Paris is the capital of France.", + "France has Paris as its capital.", + "Bananas are a yellow tropical fruit.", # outlier + ] + sim_matrix = strategy._compute_similarity_matrix(samples) + confs = strategy._compute_confidences(sim_matrix) + assert len(confs) == 4 + assert confs[3] < confs[0] + assert confs[3] < confs[1] + assert confs[3] < confs[2] + + +class TestClassifierConfidence: + """Tests for the classifier-based confidence estimation method.""" + + @pytest.fixture(autouse=True) + def _require_sklearn(self): + pytest.importorskip("sklearn") + + # Synthetic training data: 3 groups of 4 samples each. + # Groups have 3 "correct" similar answers and 1 "incorrect" outlier. + TRAINING_SAMPLES = [ + [ + "The capital of France is Paris.", + "Paris is the capital of France.", + "France's capital city is Paris.", + "Bananas are a yellow tropical fruit.", + ], + [ + "Water boils at 100 degrees Celsius.", + "At 100 degrees Celsius water boils.", + "The boiling point of water is 100C.", + "The sky is often blue on clear days.", + ], + [ + "Python is a programming language.", + "Python is a popular programming language.", + "The Python programming language is widely used.", + "Mount Everest is very tall.", + ], + ] + TRAINING_LABELS = [[1, 1, 1, 0], [1, 1, 1, 0], [1, 1, 1, 0]] + + def test_extract_features(self): + strategy = SIMBAUQSamplingStrategy(similarity_metric="jaccard") + sim_matrix = np.array([[1.0, 0.8, 0.2], [0.8, 1.0, 0.3], [0.2, 0.3, 1.0]]) + features = strategy._extract_features(sim_matrix, 0) + assert len(features) == 2 + assert features[0] == pytest.approx(0.8) + assert features[1] == pytest.approx(0.2) + + def test_extract_features_clipping(self): + strategy = SIMBAUQSamplingStrategy(similarity_metric="jaccard") + sim_matrix = np.array([[1.0, 0.0, 1.0], [0.0, 1.0, 0.5], [1.0, 0.5, 1.0]]) + features = strategy._extract_features(sim_matrix, 0) + # 0.0 clipped to eps, 1.0 clipped to 1-eps + assert features[0] > 0.0 + assert features[1] < 1.0 + + def test_train_classifier(self): + strategy = SIMBAUQSamplingStrategy( + temperatures=[0.5], + n_per_temp=4, + similarity_metric="jaccard", + confidence_method="classifier", + training_samples=self.TRAINING_SAMPLES, + training_labels=self.TRAINING_LABELS, + ) + assert strategy._classifier is not None + assert hasattr(strategy._classifier, "predict_proba") + + def test_classifier_confidence_produces_valid_scores(self): + strategy = SIMBAUQSamplingStrategy( + temperatures=[0.5], + n_per_temp=4, + similarity_metric="jaccard", + confidence_method="classifier", + training_samples=self.TRAINING_SAMPLES, + training_labels=self.TRAINING_LABELS, + ) + # Test samples: 3 similar + 1 outlier (same structure as training) + test_samples = [ + "The capital of Germany is Berlin.", + "Berlin is the capital of Germany.", + "Germany's capital is Berlin.", + "Cats like to chase mice around.", + ] + sim_matrix = strategy._compute_similarity_matrix(test_samples) + confs = strategy._compute_confidences_classifier(sim_matrix, len(test_samples)) + assert len(confs) == 4 + for c in confs: + assert 0.0 <= c <= 1.0 + + def test_classifier_outlier_lower_confidence(self): + strategy = SIMBAUQSamplingStrategy( + temperatures=[0.5], + n_per_temp=4, + similarity_metric="jaccard", + confidence_method="classifier", + training_samples=self.TRAINING_SAMPLES, + training_labels=self.TRAINING_LABELS, + ) + test_samples = [ + "The capital of Italy is Rome.", + "Rome is the capital of Italy.", + "Italy has Rome as its capital.", + "Elephants are the largest land animals.", + ] + sim_matrix = strategy._compute_similarity_matrix(test_samples) + confs = strategy._compute_confidences_classifier(sim_matrix, len(test_samples)) + # Outlier (index 3) should have lower confidence than the similar ones. + assert confs[3] < confs[0] + + def test_pretrained_classifier(self): + + # Train a classifier manually. + strategy_train = SIMBAUQSamplingStrategy( + temperatures=[0.5], + n_per_temp=4, + similarity_metric="jaccard", + confidence_method="classifier", + training_samples=self.TRAINING_SAMPLES, + training_labels=self.TRAINING_LABELS, + ) + trained_clf = strategy_train._classifier + + # Use pre-trained classifier in a new strategy. + strategy = SIMBAUQSamplingStrategy( + temperatures=[0.5], + n_per_temp=4, + similarity_metric="jaccard", + confidence_method="classifier", + classifier=trained_clf, + ) + assert strategy._classifier is trained_clf + + def test_classifier_requires_data_or_clf(self): + with pytest.raises(ValueError, match="requires either"): + SIMBAUQSamplingStrategy(confidence_method="classifier") + + +class TestInit: + def test_default_temperatures(self): + strategy = SIMBAUQSamplingStrategy() + assert strategy.temperatures == [0.3, 0.5, 0.7, 1.0] + + def test_custom_temperatures(self): + strategy = SIMBAUQSamplingStrategy(temperatures=[0.1, 0.9]) + assert strategy.temperatures == [0.1, 0.9] + + def test_empty_temperatures_raises(self): + with pytest.raises(ValueError): + SIMBAUQSamplingStrategy(temperatures=[]) + + def test_zero_n_per_temp_raises(self): + with pytest.raises(ValueError): + SIMBAUQSamplingStrategy(n_per_temp=0) + + +# --- Integration test (requires Ollama) --- + + +@pytest.mark.ollama +@pytest.mark.e2e +@pytest.mark.qualitative +class TestSIMBAUQIntegration: + def test_simbauq_sampling(self): + from mellea import MelleaSession, start_session + from mellea.backends import ModelOption + from mellea.core import SamplingResult + + m: MelleaSession = start_session(model_options={ModelOption.MAX_NEW_TOKENS: 30}) + + result: SamplingResult = m.instruct( + "What is the capital of France?", + strategy=SIMBAUQSamplingStrategy( + temperatures=[0.3, 0.7], + n_per_temp=2, + similarity_metric="rouge", + aggregation="mean", + ), + return_sampling_results=True, + ) + + assert isinstance(result, SamplingResult) + assert result.success is True + assert len(result.sample_generations) == 4 # 2 temps * 2 per temp + + # Check that the selected MOT has confidence metadata. + best_mot = result.result + assert best_mot._meta is not None + simba_meta = best_mot._meta["simba_uq"] + assert "confidence" in simba_meta + assert 0.0 <= simba_meta["confidence"] <= 1.0 + assert len(simba_meta["all_confidences"]) == 4 + assert simba_meta["similarity_metric"] == "rouge" + assert simba_meta["aggregation"] == "mean" + + output = str(best_mot) + print(f"Best output (confidence={simba_meta['confidence']:.3f}): {output}") + assert output + + del m + + +if __name__ == "__main__": + pytest.main(["-s", __file__]) diff --git a/agent-utilities/uv.lock b/agent-utilities/uv.lock index 5cd58e54..191bfb8d 100644 --- a/agent-utilities/uv.lock +++ b/agent-utilities/uv.lock @@ -594,7 +594,7 @@ name = "cuda-bindings" version = "13.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-pathfinder" }, + { name = "cuda-pathfinder", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/51/6b/457ca12dad3ee9bfcc9a545cfd6b64b359ba49de40f776f6e028e678f262/cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474", size = 6053539, upload-time = "2026-05-29T23:11:43.19Z" }, @@ -627,34 +627,34 @@ wheels = [ [package.optional-dependencies] cudart = [ - { name = "nvidia-cuda-runtime" }, + { name = "nvidia-cuda-runtime", marker = "sys_platform == 'linux'" }, ] cufft = [ - { name = "nvidia-cufft" }, + { name = "nvidia-cufft", marker = "sys_platform == 'linux'" }, ] cufile = [ - { name = "nvidia-cufile" }, + { name = "nvidia-cufile", marker = "sys_platform == 'linux'" }, ] cupti = [ - { name = "nvidia-cuda-cupti" }, + { name = "nvidia-cuda-cupti", marker = "sys_platform == 'linux'" }, ] curand = [ - { name = "nvidia-curand" }, + { name = "nvidia-curand", marker = "sys_platform == 'linux'" }, ] cusolver = [ - { name = "nvidia-cusolver" }, + { name = "nvidia-cusolver", marker = "sys_platform == 'linux'" }, ] cusparse = [ - { name = "nvidia-cusparse" }, + { name = "nvidia-cusparse", marker = "sys_platform == 'linux'" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux'" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc" }, + { name = "nvidia-cuda-nvrtc", marker = "sys_platform == 'linux'" }, ] nvtx = [ - { name = "nvidia-nvtx" }, + { name = "nvidia-nvtx", marker = "sys_platform == 'linux'" }, ] [[package]] @@ -722,6 +722,40 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/66/66/150e406a2db5535533aa3c946de58f0371f2e412e23f050c704588023e6e/cymem-2.0.13-cp314-cp314t-win_arm64.whl", hash = "sha256:e9027764dc5f1999fb4b4cabee1d0322c59e330c0a6485b436a68275f614277f", size = 39715, upload-time = "2025-11-14T14:58:24.773Z" }, ] +[[package]] +name = "datasets" +version = "5.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dill" }, + { name = "filelock" }, + { name = "fsspec", extra = ["http"] }, + { name = "httpx" }, + { name = "huggingface-hub" }, + { name = "multiprocess" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "pyarrow" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "tqdm" }, + { name = "xxhash" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0a/5b/836516269d4f618efe621661cfb6f9acc57e6f95265db3efaee48a5ffe04/datasets-5.0.1.tar.gz", hash = "sha256:ce22bb851efd7494f08aad33b940803784434f6e77763d00679a0dc45fcf686a", size = 641498, upload-time = "2026-07-28T11:09:12.016Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/0b/98fc6eb83333508ca5f44c52b3e287ea8137a0ad582714e2cbc67a02154b/datasets-5.0.1-py3-none-any.whl", hash = "sha256:9fbf73688f8c18f7529b4fe592abd04015f81d1e58001e4bac73ffb2b39d7cc4", size = 559079, upload-time = "2026-07-28T11:09:10.266Z" }, +] + +[[package]] +name = "dill" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/81/e1/56027a71e31b02ddc53c7d65b01e68edf64dea2932122fe7746a516f75d5/dill-0.4.1.tar.gz", hash = "sha256:423092df4182177d4d8ba8290c8a5b640c66ab35ec7da59ccfa00f6fa3eea5fa", size = 187315, upload-time = "2026-01-19T02:36:56.85Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/77/dc8c558f7593132cf8fefec57c4f60c83b16941c574ac5f619abb3ae7933/dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", size = 120019, upload-time = "2026-01-19T02:36:55.663Z" }, +] + [[package]] name = "distro" version = "1.9.0" @@ -1030,6 +1064,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d5/0c/043d5e551459da400957a1395e0febbf771446ff34291afcbe3d8be2a279/fsspec-2026.4.0-py3-none-any.whl", hash = "sha256:11ef7bb35dab8a394fde6e608221d5cf3e8499401c249bebaeaad760a1a8dec2", size = 203402, upload-time = "2026-04-29T20:42:36.842Z" }, ] +[package.optional-dependencies] +http = [ + { name = "aiohttp" }, +] + [[package]] name = "greenlet" version = "3.5.1" @@ -1916,6 +1955,12 @@ dependencies = [ robustness = [ { name = "benchdrift" }, ] +simbauq = [ + { name = "datasets" }, + { name = "scikit-learn" }, + { name = "sentence-transformers" }, + { name = "tqdm" }, +] [package.dev-dependencies] dev = [ @@ -1929,13 +1974,17 @@ dev = [ requires-dist = [ { name = "benchdrift", marker = "extra == 'robustness'", git = "https://github.com/IBM/BenchDrift.git?rev=95db1c95b6abbc3b9f9bc177f6a7933c555c7175" }, { name = "citeurl" }, + { name = "datasets", marker = "extra == 'simbauq'" }, { name = "eyecite" }, { name = "markdown" }, { name = "mellea", extras = ["litellm"], specifier = ">=0.3.2" }, { name = "playwright" }, { name = "rapidfuzz", specifier = ">=3.14.3" }, + { name = "scikit-learn", marker = "extra == 'simbauq'" }, + { name = "sentence-transformers", marker = "extra == 'simbauq'" }, + { name = "tqdm", marker = "extra == 'simbauq'" }, ] -provides-extras = ["robustness"] +provides-extras = ["robustness", "simbauq"] [package.metadata.requires-dev] dev = [ @@ -2080,6 +2129,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" }, ] +[[package]] +name = "multiprocess" +version = "0.70.19" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dill" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a2/f2/e783ac7f2aeeed14e9e12801f22529cc7e6b7ab80928d6dcce4e9f00922d/multiprocess-0.70.19.tar.gz", hash = "sha256:952021e0e6c55a4a9fe4cd787895b86e239a40e76802a789d6305398d3975897", size = 2079989, upload-time = "2026-01-19T06:47:39.744Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/aa/714635c727dbfc251139226fa4eaf1b07f00dc12d9cd2eb25f931adaf873/multiprocess-0.70.19-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1bbf1b69af1cf64cd05f65337d9215b88079ec819cd0ea7bac4dab84e162efe7", size = 144743, upload-time = "2026-01-19T06:47:24.562Z" }, + { url = "https://files.pythonhosted.org/packages/0f/e1/155f6abf5e6b5d9cef29b6d0167c180846157a4aca9b9bee1a217f67c959/multiprocess-0.70.19-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:5be9ec7f0c1c49a4f4a6fd20d5dda4aeabc2d39a50f4ad53720f1cd02b3a7c2e", size = 144738, upload-time = "2026-01-19T06:47:26.636Z" }, + { url = "https://files.pythonhosted.org/packages/af/cb/f421c2869d75750a4f32301cc20c4b63fab6376e9a75c8e5e655bdeb3d9b/multiprocess-0.70.19-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1c3dce098845a0db43b32a0b76a228ca059a668071cfeaa0f40c36c0b1585d45", size = 144741, upload-time = "2026-01-19T06:47:27.985Z" }, + { url = "https://files.pythonhosted.org/packages/e3/45/8004d1e6b9185c1a444d6b55ac5682acf9d98035e54386d967366035a03a/multiprocess-0.70.19-py310-none-any.whl", hash = "sha256:97404393419dcb2a8385910864eedf47a3cadf82c66345b44f036420eb0b5d87", size = 134948, upload-time = "2026-01-19T06:47:32.325Z" }, + { url = "https://files.pythonhosted.org/packages/86/c2/dec9722dc3474c164a0b6bcd9a7ed7da542c98af8cabce05374abab35edd/multiprocess-0.70.19-py311-none-any.whl", hash = "sha256:928851ae7973aea4ce0eaf330bbdafb2e01398a91518d5c8818802845564f45c", size = 144457, upload-time = "2026-01-19T06:47:33.711Z" }, + { url = "https://files.pythonhosted.org/packages/71/70/38998b950a97ea279e6bd657575d22d1a2047256caf707d9a10fbce4f065/multiprocess-0.70.19-py312-none-any.whl", hash = "sha256:3a56c0e85dd5025161bac5ce138dcac1e49174c7d8e74596537e729fd5c53c28", size = 150281, upload-time = "2026-01-19T06:47:35.037Z" }, + { url = "https://files.pythonhosted.org/packages/7f/74/d2c27e03cb84251dfe7249b8e82923643c6d48fa4883b9476b025e7dc7eb/multiprocess-0.70.19-py313-none-any.whl", hash = "sha256:8d5eb4ec5017ba2fab4e34a747c6d2c2b6fecfe9e7236e77988db91580ada952", size = 156414, upload-time = "2026-01-19T06:47:35.915Z" }, + { url = "https://files.pythonhosted.org/packages/a0/61/af9115673a5870fd885247e2f1b68c4f1197737da315b520a91c757a861a/multiprocess-0.70.19-py314-none-any.whl", hash = "sha256:e8cc7fbdff15c0613f0a1f1f8744bef961b0a164c0ca29bdff53e9d2d93c5e5f", size = 160318, upload-time = "2026-01-19T06:47:37.497Z" }, + { url = "https://files.pythonhosted.org/packages/7e/82/69e539c4c2027f1e1697e09aaa2449243085a0edf81ae2c6341e84d769b6/multiprocess-0.70.19-py39-none-any.whl", hash = "sha256:0d4b4397ed669d371c81dcd1ef33fd384a44d6c3de1bd0ca7ac06d837720d3c5", size = 133477, upload-time = "2026-01-19T06:47:38.619Z" }, +] + [[package]] name = "murmurhash" version = "1.0.15" @@ -2313,7 +2382,7 @@ name = "nvidia-cublas" version = "13.1.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cuda-nvrtc" }, + { name = "nvidia-cuda-nvrtc", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" }, @@ -2352,7 +2421,7 @@ name = "nvidia-cudnn-cu13" version = "9.20.0.48" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas" }, + { name = "nvidia-cublas", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/56/c5/83384d846b2fd17c44bd499b36c75a45ed4f095fbbb2252294e89cea5c5c/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1", size = 444574296, upload-time = "2026-03-09T19:28:27.751Z" }, @@ -2364,7 +2433,7 @@ name = "nvidia-cufft" version = "12.0.0.61" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, @@ -2394,9 +2463,9 @@ name = "nvidia-cusolver" version = "12.0.4.66" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas" }, - { name = "nvidia-cusparse" }, - { name = "nvidia-nvjitlink" }, + { name = "nvidia-cublas", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-cusparse", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-nvjitlink", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, @@ -2408,7 +2477,7 @@ name = "nvidia-cusparse" version = "12.6.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, @@ -2908,6 +2977,49 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c5/96/37c50ac951bb0260ec38d8d12e5b51587ef1ef4035c279088f2771544b28/pyahocorasick-2.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:4acb11a0a2ff10519465749d22ad70789e9fe7f81dc8fe9957a8868e499e18ab", size = 35987, upload-time = "2026-04-27T16:32:07.08Z" }, ] +[[package]] +name = "pyarrow" +version = "25.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/f3/95428098d1fa7d04432fb750eed06b41304c2f6a5d3319985e64db2d9d41/pyarrow-25.0.0.tar.gz", hash = "sha256:d2d697008b5ec06d75952ef260c2e9a8a0f6ccfce24266c04c9c8ade927cb3b4", size = 1199181, upload-time = "2026-07-10T08:29:50.116Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/98/ae2b5acf9876dbeffa6f320776242c52caab062df55c8ac5501ed2679e74/pyarrow-25.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:2e3b6544e26e393fe2cd530f523e36c1c8d3c345bbbb60cca3fd866be8322517", size = 35939080, upload-time = "2026-07-10T08:26:04.53Z" }, + { url = "https://files.pythonhosted.org/packages/80/09/3de2a968edbd496c86cb8b932cdbee2d4b08c4a28e9884a15e5c705a646b/pyarrow-25.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:b724d127783b4c19f088fcdfc844cbc318809246a30307bcabd5ed02045e890e", size = 37633420, upload-time = "2026-07-10T08:26:10.354Z" }, + { url = "https://files.pythonhosted.org/packages/19/86/8399243a4ce080426ec37db18d5e29148b7ec960a8a8c7f9059a7bf6ef0a/pyarrow-25.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:244f98a595f70fa4fd35faa7508c4ae67e14a173397a4b3b49d2b3c360fb0062", size = 46861050, upload-time = "2026-07-10T08:26:16.397Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/72d704b02bc5fc6d06954d76a0208c1e79cad3ab370f6d6a91ffe5078870/pyarrow-25.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:0222f0071d13313962a88d21bf28b80d355ac39d81bfa6ff3fe00eeaf748e4be", size = 50056458, upload-time = "2026-07-10T08:26:23.271Z" }, + { url = "https://files.pythonhosted.org/packages/06/5d/3c31a60b6403d63cad2e0f829096f5fc5763a129ead4207a5d4690b96448/pyarrow-25.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b58726f118c079f9d4ed7e904975d4f15fd69d0741ba511a4e2dcaa4ef16354f", size = 49957793, upload-time = "2026-07-10T08:26:30.232Z" }, + { url = "https://files.pythonhosted.org/packages/34/f7/8f8a019061f9863a831915329264372a87ed25eaf9109ce56eb0e84012c5/pyarrow-25.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:38a2c887cb3883e241b70201688db34133b6dfadd04f03c8f9213df53770c18e", size = 53100544, upload-time = "2026-07-10T08:26:36.414Z" }, + { url = "https://files.pythonhosted.org/packages/f1/e2/738071e95c5ddad7b3dfc12f569ffa992db89d7d7b4a95258fd184191249/pyarrow-25.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:161649d60a7a46c613a19fd795763ea8a88c36ba997dd99d9bc66e6794ee36e8", size = 27848311, upload-time = "2026-07-10T08:26:41.429Z" }, + { url = "https://files.pythonhosted.org/packages/73/44/fdd3a4377807b7dcabe2d4b5aa99dbbc98e2e5df3f1ca4e7f0aec492d987/pyarrow-25.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:149730a3d1f0fb59d663a0b8aa210adfd9c17c27cd94a0d143e60daea8320d4e", size = 35850884, upload-time = "2026-07-10T08:26:47.357Z" }, + { url = "https://files.pythonhosted.org/packages/bf/71/9f053177a7709b8c90abb00a2375b916286f9f0d6cfb21a5cadd4ef811e8/pyarrow-25.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:0721332c30fdd453fdd1fc203b2ac1f4c9db5aea28fa38d41f2574c4b068b9ec", size = 37616197, upload-time = "2026-07-10T08:26:53.564Z" }, + { url = "https://files.pythonhosted.org/packages/95/1a/22bfb6597dcdc861fa83c39c06e1457cb56f698940eff42fbb25de30e8e5/pyarrow-25.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fa1482b3da10cac2d4db6e26b81da543e237616af2ef6d466018b31ca586496f", size = 46841966, upload-time = "2026-07-10T08:27:07.685Z" }, + { url = "https://files.pythonhosted.org/packages/55/0e/cd705c042bc4fe7022478db577fcab4abdcfabb9bc37ab7a75556b3fcb2b/pyarrow-25.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5d1dbf24e151042f2fa3c129563f65d66674128868496fb008c4272b16bdf778", size = 50088993, upload-time = "2026-07-10T08:27:14.268Z" }, + { url = "https://files.pythonhosted.org/packages/98/ee/d822e1ee31fe31ec5d057210e0605c950b975dcd8d9a332976cc859a9df8/pyarrow-25.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:20887a762dd61dcc530f93a140840ab1f6aa7836b33270e42d627ab3cf11e537", size = 49941005, upload-time = "2026-07-10T08:27:21.274Z" }, + { url = "https://files.pythonhosted.org/packages/33/1b/207a90cc64619a095eb75a263ae069735f2810056d43c667befd573ec083/pyarrow-25.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58d1ab556b0cea1c93fdb799b24ad58adb2f2a2788dbce782a94f64ae1a5cc9b", size = 53112355, upload-time = "2026-07-10T08:27:27.911Z" }, + { url = "https://files.pythonhosted.org/packages/7e/fe/81d1e5f8beed15c01e98649d5c6e2167b67fd395884a2488f18bf1cf0dba/pyarrow-25.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:3f356afe61186395c861d5cd63dc21ff7d5fa335012a4668d979257df7fea0f5", size = 27945954, upload-time = "2026-07-10T08:27:32.903Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c8/098ce17d778fd9d29e40bb8c5f19a40cc90c3f0b46c9057b0d7993f42f54/pyarrow-25.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:8831a3ba52fa7cdb78d368d968b1dcd06171e6dff5461e16d90de91d371e47bc", size = 35844549, upload-time = "2026-07-10T08:27:37.956Z" }, + { url = "https://files.pythonhosted.org/packages/bc/66/24c28877219abf6263d909b1592c97ff82c59f13a59acbed11fc87c0654f/pyarrow-25.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:5f4bacb60f91dd2fca6c52f1b9a0012cd090e0294f1f781dc1881a247a352f8e", size = 37610397, upload-time = "2026-07-10T08:27:43.803Z" }, + { url = "https://files.pythonhosted.org/packages/53/55/6d1d5f5aff317ec5de9421594679ed51ed828fe7e2ce209327f819d801e4/pyarrow-25.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:59516c822d5fd8e544aaa0dfe72f36fed5d4c24ea8390aab1bcd31d7e959c6be", size = 46841701, upload-time = "2026-07-10T08:27:49.741Z" }, + { url = "https://files.pythonhosted.org/packages/b5/5d/f790fb6965ab54c9da0dda7856abc75fd0d7648d865f8d603c111d203a64/pyarrow-25.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f9dbd83e91c239a1f5ee7ce13f108b5f6c0efbe40a4375260d8f08b43ad05e9", size = 50090118, upload-time = "2026-07-10T08:27:56.051Z" }, + { url = "https://files.pythonhosted.org/packages/0c/8c/faf025357ebf31bc96777f234277aa31e2aeca6dd4ecaa391f29085473c2/pyarrow-25.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:18dcc8cc50b5e72eae6fcbfc6c8776c21a007176b27a3cdec5c2f5bcf126708d", size = 49945559, upload-time = "2026-07-10T08:28:01.927Z" }, + { url = "https://files.pythonhosted.org/packages/07/a1/bd051871708ea99a5e0fc711926c26c6f2c6d0130c7aaac8093e34998af6/pyarrow-25.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4ec1895a87aa834c3b99b7a1e758747eb8bb57f922b32c0e0fa04afb8d6998b1", size = 53114238, upload-time = "2026-07-10T08:28:08.594Z" }, + { url = "https://files.pythonhosted.org/packages/7c/31/737f0c3cffcd6af647849477d1dd68045deac2e3963c3f9f211bedc48540/pyarrow-25.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:77c8d1ae46a44b4006e8db1cc977bbcc6ce4873c92f74137d68e45503b97fb18", size = 27861162, upload-time = "2026-07-10T08:28:12.975Z" }, + { url = "https://files.pythonhosted.org/packages/55/c7/581ccbcdb3d897eb2893328d68db3d52eca373bf2a7e964d0a6276b8e85b/pyarrow-25.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:72132b9a8a0a1840197794d4dea26080069b6b0981c116bc078762dc9691b21b", size = 35878945, upload-time = "2026-07-10T08:28:18.222Z" }, + { url = "https://files.pythonhosted.org/packages/64/d1/ccb01db7329ea0411ef4fbd9b62a04d3268b36777d4e758d5e39b91ddeab/pyarrow-25.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e009ef945e498dca2f050ea10d2e9764cb44017254826fc4574fdb8d2530173b", size = 37630854, upload-time = "2026-07-10T08:28:23.452Z" }, + { url = "https://files.pythonhosted.org/packages/af/9f/2d81ba89d1e4198d0cb25fe7529de936830fdaec0db926bb52a1ef7080d4/pyarrow-25.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f57a39dbcb416345401c2e77a4373669b45fd111a1768e6cf267a7a0607ff0ec", size = 46905617, upload-time = "2026-07-10T08:28:29.376Z" }, + { url = "https://files.pythonhosted.org/packages/6a/29/0ed312ec800fb536f93783215126cee4b8977dcfeccba6f0f44df0cc87d7/pyarrow-25.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:447df764beb07c544f0178a5f6b70ef44b9ecf382b3cdfad4c2d7867353c3887", size = 50119765, upload-time = "2026-07-10T08:28:35.826Z" }, + { url = "https://files.pythonhosted.org/packages/ca/88/cab5063ba0c4d46a9f6b4b7eb1c9029dc0302d65cd5ab3510c949a386568/pyarrow-25.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ac5dfeee59f9ceb4d45ba76e83b026c38c24334135bb329d8274baa49cec3c62", size = 50027563, upload-time = "2026-07-10T08:28:43.848Z" }, + { url = "https://files.pythonhosted.org/packages/7b/fb/4d24f1b7fe2e042dc4ef315ef75e4e702d8e46fe10c37e63caff00502b03/pyarrow-25.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f0f100dacf2c0f400601664a79d1a907ced4740514bb2b00917341038e2ce76f", size = 53162437, upload-time = "2026-07-10T08:28:52.819Z" }, + { url = "https://files.pythonhosted.org/packages/fa/65/da20806de93ca6ee91e72cb6a9b08b3ac890b46efc8d94a7326c651c4c81/pyarrow-25.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:2e093efbecb5317372f819228fa4b4e6157eee48d3f0a7b0303705ebf81a7104", size = 28613262, upload-time = "2026-07-10T08:29:47.544Z" }, + { url = "https://files.pythonhosted.org/packages/86/9f/c632afb1d3ef4a7814cee236718235f3a47eac46e97eb87df40f550b6b48/pyarrow-25.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:26be35b80780d2d21f4bae3d568b1666337c3a89722cc1794c956a77017cb24e", size = 36120702, upload-time = "2026-07-10T08:28:59.577Z" }, + { url = "https://files.pythonhosted.org/packages/36/0a/093d53a0e72ad06e45d6443e00651bbc2d21af4211295086cbf4d873d3b9/pyarrow-25.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:6f4812bfbf11ca7d8faf59eb8fff8bf4dd25ce3a38b62baa010cc17a0926d1b2", size = 37750674, upload-time = "2026-07-10T08:29:06.916Z" }, + { url = "https://files.pythonhosted.org/packages/8a/18/b37fc31a69cff4bdfb8842683def5612f551b93fff6f44375e4a4a6a5535/pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b8af8ceedf0c9c160fd2b63440f2d205b9404db85866c1217bfea601de7cfb50", size = 46912304, upload-time = "2026-07-10T08:29:14.656Z" }, + { url = "https://files.pythonhosted.org/packages/32/35/5cae19ba72493e5598022468b56f6a5571f399f485bf412f157356476caa/pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c70a5fd9a82bd1a702fd482bdc62d38dcb672fb2b449b1d7c0d7d1f4be7b7bfe", size = 50073652, upload-time = "2026-07-10T08:29:22.467Z" }, + { url = "https://files.pythonhosted.org/packages/2e/a5/ddd508424bdfd5e6945765e9e2ffc687e2f6115972badc8ecf423076c407/pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0490a7f8b38ffe11cc26526b50c65d111cb54ddac3717cec781806793f1244dc", size = 50058654, upload-time = "2026-07-10T08:29:29.689Z" }, + { url = "https://files.pythonhosted.org/packages/5f/a4/324d0db203ff5eebe8694ec2d6ec5a23f9aaa5d02e5b8c692914c518c33c/pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e83916bbcf380866b4e14255850b33323ff678dc9758411d0409cdd2523880b0", size = 53140153, upload-time = "2026-07-10T08:29:36.041Z" }, + { url = "https://files.pythonhosted.org/packages/bd/8d/d236e9c82fe315f9128885c8be3ec719f41965a1eb6b6f4b42470904cd41/pyarrow-25.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:13240f0d3dc5932ccd0bfa90cd76d835680b9d94a7661c635df4b703d40ce849", size = 28743657, upload-time = "2026-07-10T08:29:42.742Z" }, +] + [[package]] name = "pydantic" version = "2.13.4" @@ -4337,6 +4449,144 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/53/46/29ac9daf11a86c22a8c38cd9236c62928ccae83f7ceb06bd3b0467cf9d05/wrapt-2.2.1-py3-none-any.whl", hash = "sha256:3aafea2975caef8ca49400640dde02cc7426e798f24870ed01f490bc3cffd32f", size = 61000, upload-time = "2026-05-22T14:49:41.593Z" }, ] +[[package]] +name = "xxhash" +version = "3.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/63/71aa56b151a1b28770037a61bd4e461c2619cfc8866a4fcaf1548605e325/xxhash-3.8.1.tar.gz", hash = "sha256:b0de4bf3aa66363552d52c6a89003c479911f12098cd48a53d44a0f7a25f7c46", size = 86223, upload-time = "2026-07-06T10:49:58.937Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/5a/05eaa129555f85476a3e16ff869e95f81a78bbe4647eef9d0229f515a317/xxhash-3.8.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602efcad4a42c184e81d43a2b7e6e4f524d619878f2b6ee2ba469011f47c8147", size = 34699, upload-time = "2026-07-06T10:44:10.14Z" }, + { url = "https://files.pythonhosted.org/packages/80/59/0df1133958b2228929355e022aab1e958c7b2c43e27bf7f59bc9edfa8a54/xxhash-3.8.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:131324f719957b988861714de7d6ddf57b47abec3b0cc691302ffeaba0e05e10", size = 32373, upload-time = "2026-07-06T10:44:11.353Z" }, + { url = "https://files.pythonhosted.org/packages/3e/bf/1cfda5b5e6bf26617812b4a31662ef2220d2ad04e0a55b8ff9eb36e56a5c/xxhash-3.8.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:db77278a6eddadbf44ce5aae2fee5ebb4d061f026b1ce2130d058cd4d7a7b670", size = 220284, upload-time = "2026-07-06T10:44:12.683Z" }, + { url = "https://files.pythonhosted.org/packages/70/93/45dc0ad7913b69e5b08bd039236cf628380e4c9cc76a8a4c6625a328e058/xxhash-3.8.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c332dd48b8cb050da2bb2a3c96d72b1664168650a250ef9718e423df7989e05", size = 240980, upload-time = "2026-07-06T10:44:14.297Z" }, + { url = "https://files.pythonhosted.org/packages/e9/02/f28ba7d17f2c1410ee397982c817ab1bd5b2701070c2d2c373539aad000a/xxhash-3.8.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a5cd96f6dcdf4fa657b2d95668d71d58455248f98712ecffaa9c528edf40ccae", size = 264526, upload-time = "2026-07-06T10:44:16.017Z" }, + { url = "https://files.pythonhosted.org/packages/5c/d0/f10651cec2c7981b20d693deae6bdfc438427d92be2db4ccabb6181f0021/xxhash-3.8.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c959f88160b13b4e730b0d75b459b7929fc0d2225c284c9683ac95d6feeeac6a", size = 241369, upload-time = "2026-07-06T10:44:17.698Z" }, + { url = "https://files.pythonhosted.org/packages/ff/40/136e0cbaf5db51e191423b1c98643593189f02b6cd90837bf64b19113d70/xxhash-3.8.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:027dee4355f3fcc41481650d846cf6cfc895c85a1ab7acd063063821a0df5b4c", size = 473186, upload-time = "2026-07-06T10:44:19.354Z" }, + { url = "https://files.pythonhosted.org/packages/4b/3f/6aa808a96bdc43dba9a740dec56c744526ee3c0019e32c75e810fa90ae4d/xxhash-3.8.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad52a0e4bcc0ba956a953a169d1feec2734a64981d689e4fc8f490f7bf91af60", size = 220092, upload-time = "2026-07-06T10:44:20.956Z" }, + { url = "https://files.pythonhosted.org/packages/47/28/a8675e78a9ced96dab853416162268e10e05b452e95db7888cf69f58ac5f/xxhash-3.8.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5d3dfb1f0ff146da7952867a9414f0c7a29762f8825a84879592612fd6139342", size = 309846, upload-time = "2026-07-06T10:44:22.543Z" }, + { url = "https://files.pythonhosted.org/packages/89/0f/7fe4d4ef4e69f0033e012396ee2a115886bca7b10b7e45ce398626436bfc/xxhash-3.8.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4482380b462ca9e59994d072a877ecadd1cf51102daeeab2db696f96ab763723", size = 237659, upload-time = "2026-07-06T10:44:24.135Z" }, + { url = "https://files.pythonhosted.org/packages/38/8f/83e9e31d4ed57fe963b99cb5b13a23e3e0f0dad1885aa0ebd2a7819dd423/xxhash-3.8.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:950ac754d16daea42038f38e7465eb84cda4d08d7343c1c915771b29470f065a", size = 268737, upload-time = "2026-07-06T10:44:25.875Z" }, + { url = "https://files.pythonhosted.org/packages/57/79/7e7de46dbe5d1f49afc96a0bc42e6b8df24eae3d6bad6007b99e42f48430/xxhash-3.8.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0418ec8b2331b9d4d575fc9284427e8e69449d7172e99e1a86fcdd1f51a0a937", size = 224955, upload-time = "2026-07-06T10:44:27.777Z" }, + { url = "https://files.pythonhosted.org/packages/ec/34/b8540839e958d5ef5c6101af6f16032109e7099698ae8edbc8dcefe4d8f4/xxhash-3.8.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:32a94ad2763e0263d9102037d349002c3d3c401e42770542c3eeb4801f311661", size = 239653, upload-time = "2026-07-06T10:44:29.422Z" }, + { url = "https://files.pythonhosted.org/packages/ce/87/a735d05f7f859354acadabe470ff40e2c46672275f96dcf096a761904def/xxhash-3.8.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:89b11a5cdd441aa463f6d34ca0241602bc09b001a76994b6059828494108c673", size = 300213, upload-time = "2026-07-06T10:44:31.401Z" }, + { url = "https://files.pythonhosted.org/packages/98/31/3e1cb020237b68117fc212dc5f9753b87f865b4dfee7c1ce62d0836955b5/xxhash-3.8.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:09a204dd4bb0823daf938cdd0dc8057d5f1e14fe3cbde929424255f23f9de872", size = 442508, upload-time = "2026-07-06T10:44:33.023Z" }, + { url = "https://files.pythonhosted.org/packages/23/bf/f80090622141cc734b039ce1d15ce3ff6dced375e9680249bf5b9b8c6bf9/xxhash-3.8.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e710ad822c493fb80a4fbc1e3d0a807b1422cb90adbe64378f98291b7fa48fef", size = 216853, upload-time = "2026-07-06T10:44:34.983Z" }, + { url = "https://files.pythonhosted.org/packages/a6/a3/60157acecc307b238d3651c2483168e224b48b23a36ae6d6903588341d80/xxhash-3.8.1-cp311-cp311-win32.whl", hash = "sha256:5013be3bea7612852c62a7437f3302c1cfb91ca7e703b194459db0b2b2e0d792", size = 31936, upload-time = "2026-07-06T10:44:36.542Z" }, + { url = "https://files.pythonhosted.org/packages/59/5c/ef70c418d878d187b8da56d4cdc06aea6cf5e456b301e96e51e1d2cc8625/xxhash-3.8.1-cp311-cp311-win_amd64.whl", hash = "sha256:f377012b86c0a23a1df0cf5a1b05aa7187649e472f71c7892e5f2c2815bbe74f", size = 32724, upload-time = "2026-07-06T10:44:38.177Z" }, + { url = "https://files.pythonhosted.org/packages/2c/25/f008db952cec6b2a26445b456eeed2ebebd65e08e848ebe09ed6ac0634e6/xxhash-3.8.1-cp311-cp311-win_arm64.whl", hash = "sha256:836f11d4474d3228e9909d97216faa4f7505df41cfaf3927eb29809de785a78d", size = 29212, upload-time = "2026-07-06T10:44:39.577Z" }, + { url = "https://files.pythonhosted.org/packages/42/91/f65c34a7aa7b4e7cf4854f8e6ef3f7ee32ceac41d4f008da0780db0612f6/xxhash-3.8.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e6e49370822c1f4d8d90e678b06dbcb08b51a026a7c4b55479e7d467f2e813bc", size = 34680, upload-time = "2026-07-06T10:44:40.932Z" }, + { url = "https://files.pythonhosted.org/packages/57/04/b10a245a4c09a9cfa88f8e9ae755029413ad1ac17047f9a61906e5ae0799/xxhash-3.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:220d68130f83f7cc86d6edfdeab176adc73d7200bf3a8ec10c629e8cf605c215", size = 32397, upload-time = "2026-07-06T10:44:42.196Z" }, + { url = "https://files.pythonhosted.org/packages/3a/75/45ab795b5945b6388583bd75202106af505537935566c15a1577797a0e08/xxhash-3.8.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4d365ee1892c1fa803536f8c6ce21d24b29c9718ec75eb856095c07830f8c478", size = 220549, upload-time = "2026-07-06T10:44:43.603Z" }, + { url = "https://files.pythonhosted.org/packages/13/44/5ba2bd0a14ddf4193fc7d8ec29625f659f22c06d60b28f04bf46305d8330/xxhash-3.8.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:852bfe059720632e2f16a6a4745e41d20937b2bf2a42a401e2412046bb6971cc", size = 241186, upload-time = "2026-07-06T10:44:45.534Z" }, + { url = "https://files.pythonhosted.org/packages/23/32/c4147def4d1e4538b906f82731e0ba23424377fc50a7cddd03cd284c8f63/xxhash-3.8.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f8c25a7061d952de589bd0ea0eaadee32378ff83dd6a677b267f9cd86f401f8", size = 264852, upload-time = "2026-07-06T10:44:47.199Z" }, + { url = "https://files.pythonhosted.org/packages/6c/bd/71ed14f4f0318bb7fd7b2ec51999413487fa8da8d41208e84d50d1ef0f98/xxhash-3.8.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:868a8dcaff1a84ba78038e1cef14fc88ccf84d9b4d12ea604696e0693296aa56", size = 242663, upload-time = "2026-07-06T10:44:48.846Z" }, + { url = "https://files.pythonhosted.org/packages/91/09/70af22c565a8473b3f2ae73f88e7721af281bc4a575236dbd1970c9f76f6/xxhash-3.8.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6536d8677d2fff7e64cd0b98b976df9de7aee0e69590044c2af5f51b76b7a170", size = 473510, upload-time = "2026-07-06T10:44:50.695Z" }, + { url = "https://files.pythonhosted.org/packages/18/96/34db781c8f0cf99c544ca1f2bc2e5bf55426e1eb4ca6de8ea5da56a9f352/xxhash-3.8.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:82c0cedd280eab2e8291270e6c04894dbc096f8159a39dcf1807429f026ca3cc", size = 220469, upload-time = "2026-07-06T10:44:52.422Z" }, + { url = "https://files.pythonhosted.org/packages/93/5f/9a184f615fa5a4dce30c01534f62946ce5a11ce40f73785cbd356ccabaa9/xxhash-3.8.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daa86e4b68221d38e669bb236ba112d0335353829fb627c82e5909e4bbe8694c", size = 310290, upload-time = "2026-07-06T10:44:54.142Z" }, + { url = "https://files.pythonhosted.org/packages/a9/dc/9b9a9789011ee153723a5eb9e7dd7fcbae2ba9b3fe7a729249ca7c252056/xxhash-3.8.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2bc7113e6f2b6b3922dd61796ca9f36af09da3773898e7003038dc992fc83b8d", size = 238173, upload-time = "2026-07-06T10:44:55.693Z" }, + { url = "https://files.pythonhosted.org/packages/ec/4d/71c6005ada9dcb608a4e1902e8475ecadb5f3fbfa04e1e244d276a2d0c43/xxhash-3.8.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5eed32dad81d6ba8e62dc7b9ffa0500199385d7810a8dd9d4eafaceb8c6e20bb", size = 269026, upload-time = "2026-07-06T10:44:57.424Z" }, + { url = "https://files.pythonhosted.org/packages/2f/87/d6c036ba25dfbd9c8633be5aa86fc9474bbb9e2c68212a841d090abe7344/xxhash-3.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:83697b0ea1f10e7f5d8b26a4906fa851393c61546c63839643a2b7fe2d868061", size = 224970, upload-time = "2026-07-06T10:44:59.085Z" }, + { url = "https://files.pythonhosted.org/packages/48/62/4c1f035a41c5752aa05e195b6c904c07b94fe9061a16de61e72a6e6b135f/xxhash-3.8.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:36fc69160465ae75c6ec4ac9f781bb2aa16ae7ff869e73c26fee85fbb11b9887", size = 240820, upload-time = "2026-07-06T10:45:00.746Z" }, + { url = "https://files.pythonhosted.org/packages/da/14/d39d565069b87e86d21a2af2a31d04db79249d25aa8d5b62959056a89857/xxhash-3.8.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:445e0f5a31f2f3546ae0895d4811e159518cdc9d824c11419898d40cfadb677e", size = 300619, upload-time = "2026-07-06T10:45:02.716Z" }, + { url = "https://files.pythonhosted.org/packages/13/22/75467acc887edc8cf71c97ab1708feb3df7a88bda589b9f399765c6387d2/xxhash-3.8.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:dfe0580fbfd5e4af87d0cc52d2044f155d55ebd8c8a93568758a2ea7d8e15975", size = 443267, upload-time = "2026-07-06T10:45:04.653Z" }, + { url = "https://files.pythonhosted.org/packages/a4/b6/1da3baa5fa6ef705e3425fddd382be7dfc4dfba2686df90a20f16e9c7b1b/xxhash-3.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:095e1323fa108be1292c54c86da3ef3c7a7dc015b105a52133973bc07a6ad11a", size = 217338, upload-time = "2026-07-06T10:45:06.304Z" }, + { url = "https://files.pythonhosted.org/packages/78/dd/b5295a9f97484e7a1c2b283a742ca45e3104991c55a1ef670dde161829ba/xxhash-3.8.1-cp312-cp312-win32.whl", hash = "sha256:bf28f55e427e0483acb1f666bd0d869b6d5e5a716680c216ad7befe3d4cfba2e", size = 31970, upload-time = "2026-07-06T10:45:07.823Z" }, + { url = "https://files.pythonhosted.org/packages/ec/31/3fa0b807d7e21515cd975e7fe5c039d52ac3e9401a96d6ad68dae6305215/xxhash-3.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:2256e80e4960ee282f63428adb349cb7f8bd8efe4db770d88eb815f4b9860724", size = 32741, upload-time = "2026-07-06T10:45:09.42Z" }, + { url = "https://files.pythonhosted.org/packages/b8/05/86feada74e239600e6875aa507afb40482a89b92700aa74a92da83bdcb77/xxhash-3.8.1-cp312-cp312-win_arm64.whl", hash = "sha256:9df56e6df96a60590935e22373041cccc91fd55858763dcffb55bf63b3a2b396", size = 29234, upload-time = "2026-07-06T10:45:10.809Z" }, + { url = "https://files.pythonhosted.org/packages/6b/8c/446bb782cd0d27007a917b5569a08dd73219c3e8d6e459014db104b27bdb/xxhash-3.8.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:3c682fcd96eb4bf64be32a4d95f96107e1588005831bd8a741b324fdda01b913", size = 38562, upload-time = "2026-07-06T10:45:12.425Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ec/c0c45627eaa6be7a5d6117423adf8f7a15b17ee74b4b17072cca5959a225/xxhash-3.8.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:036a024d8b9c01f70782e09ed98d532e76fd23f950ae7154bd950fe94e90ebec", size = 36656, upload-time = "2026-07-06T10:45:13.932Z" }, + { url = "https://files.pythonhosted.org/packages/f6/94/8324c04cc7597154caaeba6c094e01fbd2e7601d01e7a13eea9f5420e77b/xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d6a5c0bce213b23b0166fe0d35bcbbe23ce4b968f257cc7eb6fd57cb8e1e6297", size = 31169, upload-time = "2026-07-06T10:45:15.687Z" }, + { url = "https://files.pythonhosted.org/packages/40/a4/beb6bb26e1184e126dbe7a5682330214ef54dcfbf882078aa9f4b5428d42/xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:5177aa44eddaa97c6ef0cc00c6d540edb64d51781d2f8fb941612ec61a92c9ed", size = 32177, upload-time = "2026-07-06T10:45:17.035Z" }, + { url = "https://files.pythonhosted.org/packages/56/0f/fc4c92a5a528f839b34b6419b2e53c8597f2a629d5a1f5d721f65bfa1fd6/xxhash-3.8.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7801b7223db017b9c0c9ccf37e44524edb35a1544a1c032add22c061c6af0276", size = 34642, upload-time = "2026-07-06T10:45:18.39Z" }, + { url = "https://files.pythonhosted.org/packages/d4/58/edbfb141d4000767ac6a9694f8ac0763e2c2e983e65c9e31620ba56e2667/xxhash-3.8.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e80238259655bf69d7bcd08226a970d7f42605f3157786bfa76dd13472d7fa0", size = 34684, upload-time = "2026-07-06T10:45:20.033Z" }, + { url = "https://files.pythonhosted.org/packages/07/3f/5072f1f0f5714186f0ac2a0b5a4929ce30d4b845e94886b6c01b6ebda0be/xxhash-3.8.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bcab50a389cc04d87f90092af78a6adba2ab3deca63175a3344ca83514045315", size = 32401, upload-time = "2026-07-06T10:45:21.414Z" }, + { url = "https://files.pythonhosted.org/packages/49/c7/802ea2f9c2ed59219934d6d65c470d502b1788043eae277a52af8658bda6/xxhash-3.8.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a2489d3a776fa380cb8e71f54c7fda268a9baf3de9b1395093fd280f95735907", size = 220617, upload-time = "2026-07-06T10:45:23.234Z" }, + { url = "https://files.pythonhosted.org/packages/99/a8/e10488efd31fcb13fcd6acbc6e788f10c6f8e3a0cc4ae3eb89dc19c55a12/xxhash-3.8.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32ab1e5432690276e71192be7401b55f96db2d0eedea5d44eb1f164505669cc0", size = 241295, upload-time = "2026-07-06T10:45:25.364Z" }, + { url = "https://files.pythonhosted.org/packages/18/cc/14180b17d44892a631f8ae7323c30bfbb1328efc8209e528a480293528ac/xxhash-3.8.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b30e01a0b97a4bc3f519a4d7a82da3dc53251fb0de5eeea8660dcd4ff094c0c2", size = 264688, upload-time = "2026-07-06T10:45:27.09Z" }, + { url = "https://files.pythonhosted.org/packages/a9/72/a14019d0c5f6c41ee407a503036ae32787c91325ca218a96a9b5627be651/xxhash-3.8.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1f44275ddb0978b67a58a951501903f04d49335a91f7681c9ce122ecb8ccb329", size = 242740, upload-time = "2026-07-06T10:45:28.753Z" }, + { url = "https://files.pythonhosted.org/packages/68/08/92550e556c6fcfcb96c6a336945eb53a431ed43120ed749636debb16c5cf/xxhash-3.8.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e3b87cbd974512c0c5fc7b469c36b2cdc9ee6d76e4ec78bccb2c7184611c49b0", size = 473599, upload-time = "2026-07-06T10:45:30.524Z" }, + { url = "https://files.pythonhosted.org/packages/29/83/e361d3c1acd1b21e1d489616de6fa4aaf843365d8179f612e3743eac20a9/xxhash-3.8.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98ee81b4b7f3023c9cb04a78cc67610baffcb5812d92f2096cb5a5efc6f19437", size = 220559, upload-time = "2026-07-06T10:45:32.979Z" }, + { url = "https://files.pythonhosted.org/packages/05/01/006a4243c2c2a6831827f9999f6d1c23feeef100eb023c1f886022a00bf3/xxhash-3.8.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2666f059a1588a99267e33605365ed89cea92f424b3522806a9f4bd8ad2e3d62", size = 310383, upload-time = "2026-07-06T10:45:35.875Z" }, + { url = "https://files.pythonhosted.org/packages/d8/20/af388e8bf9f9a0f89eeef7d2a1935d176ee1c20bc6adeda05035879379cf/xxhash-3.8.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b0093cf7eeb91b84776e8742113afa4bdf47533d36cf719179aaaf1f56f6f8bf", size = 238228, upload-time = "2026-07-06T10:45:38.02Z" }, + { url = "https://files.pythonhosted.org/packages/63/6b/4666579a87eebd1744663c404297355fa0658617b015cedfa58810ee7036/xxhash-3.8.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3a800912a2e5e975d4128969d645c4a2a80aa886ccd6c9b1c6f44529e327e8cf", size = 269137, upload-time = "2026-07-06T10:45:39.954Z" }, + { url = "https://files.pythonhosted.org/packages/de/d3/e963a8a46f900a137d91b02144d8ea07a8f812971b138204a3b2f8b8e55c/xxhash-3.8.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0fe37f72a207223d22a4eddc3149d4298993385aa9daef25c039246ca5a309f3", size = 225068, upload-time = "2026-07-06T10:45:41.718Z" }, + { url = "https://files.pythonhosted.org/packages/aa/80/9d181dbcde4b0fe48375f48833a5832d4b8cd2b349b15110c92ee472d874/xxhash-3.8.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5db43f249b4be9f99ef4b967863f37094fb40e67effafb78ba4f0356b6396104", size = 240874, upload-time = "2026-07-06T10:45:43.414Z" }, + { url = "https://files.pythonhosted.org/packages/39/15/ce3ab5a1cd27ead25a5196e55a7284220f6ad6e316da494ffd900b2b600f/xxhash-3.8.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c4ed42965c2cd9081f011be22f69d0e65d3b6165fe7734072fd0c232840bbd4e", size = 300702, upload-time = "2026-07-06T10:45:45.135Z" }, + { url = "https://files.pythonhosted.org/packages/96/c0/2281a8ab5f2a62dbf57a23c58a01ccc1d98abf40f71193c8a81f59e759b5/xxhash-3.8.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3557bec8fcb11738a8920eeb68974bc76b75262f6947998d3147954ce0a4b893", size = 443351, upload-time = "2026-07-06T10:45:47.188Z" }, + { url = "https://files.pythonhosted.org/packages/81/2e/071a58c1a53a52d4f7a3aa0987be0c396dffd40da8204805fe1b130a81f4/xxhash-3.8.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:00de40f3b42240db23a82a5c682b55d7263d84a26a953240c1aee463409660e3", size = 217396, upload-time = "2026-07-06T10:45:48.925Z" }, + { url = "https://files.pythonhosted.org/packages/68/44/36ab58134badd9d3433fc7b53c4ca8d113d8e807782885628640f8297a4d/xxhash-3.8.1-cp313-cp313-win32.whl", hash = "sha256:b5196cc2574cfec572a5f3fb7cfa5ade27305ae3d06516a082132441aff4c83a", size = 31974, upload-time = "2026-07-06T10:45:50.591Z" }, + { url = "https://files.pythonhosted.org/packages/96/2a/2a0b84798448e766f7b89ceed073cb0cb5a43fc9ebbacbdea74a38de18e3/xxhash-3.8.1-cp313-cp313-win_amd64.whl", hash = "sha256:538f5f865df6cd8c32dd63158a0e5b4f5dd08d732a7da8b7228a5a0776c8ce55", size = 32739, upload-time = "2026-07-06T10:45:52.221Z" }, + { url = "https://files.pythonhosted.org/packages/d4/60/bb51dbf7c363ff88a7cbd50b7959718219577ef44d7cf255929ffc4a2194/xxhash-3.8.1-cp313-cp313-win_arm64.whl", hash = "sha256:a6617f30641ba0d8baa1635fbefb1dffc5165ec36d26921bd5cee13497cd937a", size = 29239, upload-time = "2026-07-06T10:45:53.714Z" }, + { url = "https://files.pythonhosted.org/packages/56/d3/827ca123c2ee5443a6aaed3c5dd199237dc2f010e2bebd7ec09ef36f3a5f/xxhash-3.8.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:bfcd82852c62a60e314670a9602de354c4460f8adad916e2e42a20860c7870bc", size = 34964, upload-time = "2026-07-06T10:45:55.535Z" }, + { url = "https://files.pythonhosted.org/packages/05/67/67ae2a3ccdeb8b8ef025d35aee9edd1d26c3abe5051d47da9286232afbf8/xxhash-3.8.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:08ea2081f5e88615fec8622a9f87fbe21b8ea58d88cfc02163ca11026ee62a92", size = 32697, upload-time = "2026-07-06T10:45:57.288Z" }, + { url = "https://files.pythonhosted.org/packages/38/5a/3d3994346e1f45493679cb5c1ffc2bf454e410e9d1e8a662d253becee91e/xxhash-3.8.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2e32855b6f9e5b18f449e59d45e3d5778bdeb660632ef2693cca267a11246c75", size = 225954, upload-time = "2026-07-06T10:45:58.897Z" }, + { url = "https://files.pythonhosted.org/packages/3f/2c/53169270309b7cd8e05504e07fe123bac053b89d00ac63617faacf0a2ec0/xxhash-3.8.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6e088bd7870775624256a0d84c2a6714afd223b2eeb56b0ca58398e52a32fda", size = 249776, upload-time = "2026-07-06T10:46:00.977Z" }, + { url = "https://files.pythonhosted.org/packages/70/e0/5c551d8d592f944506f7c5185e210255c15e672a3c6008c156a1bd9b775e/xxhash-3.8.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:72eb5ae575cc7ae2b23f6f8064a8b10f638c7149819ae9cc6d20ebd4d37a1629", size = 274776, upload-time = "2026-07-06T10:46:02.869Z" }, + { url = "https://files.pythonhosted.org/packages/a0/2a/d3a762270cee2d7bcd0e25e28c623e5f3f5c0dc637b66e3e47dd5b0bb3f0/xxhash-3.8.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d0b48cdf690a64cedf7258c3dc9506cc41fc86edd7739c40e3098952265dc068", size = 252056, upload-time = "2026-07-06T10:46:04.688Z" }, + { url = "https://files.pythonhosted.org/packages/c1/8f/b78e4373b2cb6d1c42af60ea2d7e9146ad0710b239ac7f706d5d31d5bb98/xxhash-3.8.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fb9e256a357dfcede7818c6d34e70db2d6b664394803d1de4b6984d2de76c0f1", size = 482108, upload-time = "2026-07-06T10:46:06.498Z" }, + { url = "https://files.pythonhosted.org/packages/e6/0d/642d923336ea61a15f8ce64fc7e078729e6e06c3a026e517fa79b2c23b7a/xxhash-3.8.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51f71a6e2ad071e70c937e41fcb6c19f82c3f9f49831eba850ed4a106ffbb647", size = 226739, upload-time = "2026-07-06T10:46:08.598Z" }, + { url = "https://files.pythonhosted.org/packages/a6/0a/a37d6da6427d45a8d23e3ee3a0ca9c9d4a90364849c6637fe2963a755f9b/xxhash-3.8.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e4a6443968c4e8dc69967e12776776a5952c119cc1bd94168ad1c5ad667c2be1", size = 319658, upload-time = "2026-07-06T10:46:10.504Z" }, + { url = "https://files.pythonhosted.org/packages/4a/51/ebbd40da8a3f1bc53b4b7a9a87f8e28bd95c5f21bc14b8a57860cf367d1b/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:714503083a1f2065c9ad15340dd49ac8a8e948a505a705ffa1750cb951519113", size = 246059, upload-time = "2026-07-06T10:46:12.634Z" }, + { url = "https://files.pythonhosted.org/packages/24/4c/d9014030147e1f0bb26e7da47aa240dd9ec61c763c573e558111d869f8e1/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:77f74e45a1e5574bbbf80181c8027b3a4c65c2248fffbd557bd596fff13102f9", size = 275535, upload-time = "2026-07-06T10:46:14.614Z" }, + { url = "https://files.pythonhosted.org/packages/84/86/caee2db41fadcd5a25aa4323213f9afec5a8586d4e419241e3d659362bd7/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:4e0e1b0fb0259c1b75d1251ac0bb4d7ab675d36f7a6bf4ba6aa630dae94f9ffa", size = 231292, upload-time = "2026-07-06T10:46:16.452Z" }, + { url = "https://files.pythonhosted.org/packages/0b/60/f52f08bcdc904c4514ea5c25caa19e9f3214144434a6ff96dc82dc1cbddd/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:10e4393ec33633c2f05ad01869e546ad080b1a18f2650503731f153774608b31", size = 250490, upload-time = "2026-07-06T10:46:18.318Z" }, + { url = "https://files.pythonhosted.org/packages/24/a0/94dc7ae310838f250669c6ad7168e6d6fca17d49dac1053f06dc232c4a56/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:b3ba794c3d885803db6c3116686923f1ec13bc86e621e169a375282b63ea1cc6", size = 309861, upload-time = "2026-07-06T10:46:20.503Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f9/adeead7d0eb28cdfc2832544ea639ffbc6749ccde47a8e228d667459182e/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:57189a69c0891e4818853feaa521c972d22c880a001453addea015f48e3c3398", size = 448739, upload-time = "2026-07-06T10:46:22.79Z" }, + { url = "https://files.pythonhosted.org/packages/04/a4/22ec0e07db57d901c9298ae98aa3cf2be45bafded6f07c13131e85b89032/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d59e71153fe9ff85648d00e18649b07e9b22c797291abb7e27274fa06df8b838", size = 223657, upload-time = "2026-07-06T10:46:24.831Z" }, + { url = "https://files.pythonhosted.org/packages/94/32/8a9531f37b59e5a013003db7cb7414baf4ce7e0e1268e0d5947cd3d6a2df/xxhash-3.8.1-cp313-cp313t-win32.whl", hash = "sha256:5b96f0024e9840f449bd91b2d005c921a4b666055a0d1b6492463799f32aae22", size = 32377, upload-time = "2026-07-06T10:46:26.86Z" }, + { url = "https://files.pythonhosted.org/packages/e7/ab/2ca45fd7f671de5f81fc297ef1c95080b40c86ec6be0cc6034b8f7707ac8/xxhash-3.8.1-cp313-cp313t-win_amd64.whl", hash = "sha256:37d5a56c36dcc0b9a87b814cd992598d33863ff683749de6c86081f278d5e629", size = 33274, upload-time = "2026-07-06T10:46:28.39Z" }, + { url = "https://files.pythonhosted.org/packages/5a/54/20d7163463ddb6438b73a427d1655a77a502cf9b9b0c3ada3599629d9c0a/xxhash-3.8.1-cp313-cp313t-win_arm64.whl", hash = "sha256:6696c8752aded28ff3b16f33ef28ce28fb5d209b80c206746f943199fcf5fd65", size = 29375, upload-time = "2026-07-06T10:46:29.962Z" }, + { url = "https://files.pythonhosted.org/packages/c2/8b/df2ba04f22a6cd6b39f96a6577329a8471a55c90ef8d8e2f7c102363613f/xxhash-3.8.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:9db455cb649dcfe4504d6d68a6d83a7315a99a3ca59871dc3ff840671f99adba", size = 38430, upload-time = "2026-07-06T10:46:31.496Z" }, + { url = "https://files.pythonhosted.org/packages/b2/4f/6a059e8ad3ca8deedc91dfe335b211204900895152212c03ebbe721de68b/xxhash-3.8.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:affb37f152e55b5e4494bb9d0107f7bb08515c6704fbed82d9f61214d74adc17", size = 36558, upload-time = "2026-07-06T10:46:33.078Z" }, + { url = "https://files.pythonhosted.org/packages/cb/95/40be178205acce092ae418feb20ac737b32a02c7b864926ed0717354c9f8/xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:460261045936975193bfd20549a0de1cd52a33b405cbb972f0d80940c42266cd", size = 31181, upload-time = "2026-07-06T10:46:34.793Z" }, + { url = "https://files.pythonhosted.org/packages/3f/89/2da4dbf051bafa156c0e3f12012db2b0ac3b84ff37ca1f021f6bfffcdfbb/xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:38c887aedb696ef8bca19983206d270848558cfae4a91afa6a2fb05dde58ffc5", size = 32192, upload-time = "2026-07-06T10:46:36.393Z" }, + { url = "https://files.pythonhosted.org/packages/7c/4e/e000bbae3566bc8e0be771a8a0f294aa99075e3f0bc4ef43922ebffdebc8/xxhash-3.8.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:594131ce1aad18db3689781f806db1b065cdaa04f4df36b4c038d2013aefd0bf", size = 34691, upload-time = "2026-07-06T10:46:38.1Z" }, + { url = "https://files.pythonhosted.org/packages/b4/4a/ea954aacc7d1c8711880ac2b55da94429a9b4296b151c4fc0966549ca1ee/xxhash-3.8.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:78c794b643d214f1522e7a288bcf5a2de120d26cd170516749a4009dc92722c9", size = 34807, upload-time = "2026-07-06T10:46:39.647Z" }, + { url = "https://files.pythonhosted.org/packages/ca/29/df598e738ff37558ac627264deb2e560902d9bf7f46d3bd5175c9eee593e/xxhash-3.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:af0c9fedc4a2c24e8664953882fe8185f3790b8338c9c700f76f5ad660817711", size = 32410, upload-time = "2026-07-06T10:46:41.359Z" }, + { url = "https://files.pythonhosted.org/packages/59/9c/81ab40e7d33ada0b3df5d1bc884894d15dbf4f805cd645b685e4606bb8e0/xxhash-3.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:115772daeb71b2f3b9381177017f53e6cf3f3439c840737fdabd21aba6e54920", size = 220564, upload-time = "2026-07-06T10:46:43.463Z" }, + { url = "https://files.pythonhosted.org/packages/fd/6f/62ae6f5c8606320a0e2a41c2dc8c6d91cc5d63d0f84dd9582e9543779dd8/xxhash-3.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:000435984a0469b0f822fe76f35bddea0f96a4d6521b3339a60a6428cdee1edc", size = 241462, upload-time = "2026-07-06T10:46:45.509Z" }, + { url = "https://files.pythonhosted.org/packages/15/a1/9c3a0ec6cb524396f551eddd102a76690a795494eb9784fc67542b0daa37/xxhash-3.8.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f1c68394818e0595569c2ff3cbc1e6d5a36a434e796f5c526b987b80c8a8c62", size = 264491, upload-time = "2026-07-06T10:46:47.655Z" }, + { url = "https://files.pythonhosted.org/packages/64/f2/700a4674e4308eb59d2fdb973977e82eae231bea5044753fee5c9eec0e0c/xxhash-3.8.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:46b39976d008e2a845758650f0ff7136bca004f40da0c8798bd37ac37860154f", size = 242905, upload-time = "2026-07-06T10:46:49.857Z" }, + { url = "https://files.pythonhosted.org/packages/f3/8a/72d9874375c8d4cbc64a8cd1d659d5695a8765c3db82efa82dc5bd9f14d0/xxhash-3.8.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d5006c65ec507a333479e76e00e2c368781f16c24ededa764763956b32a0e93e", size = 473873, upload-time = "2026-07-06T10:46:51.953Z" }, + { url = "https://files.pythonhosted.org/packages/03/f0/6db07590ed7e0a77f186ef0bcea8d52553bf1ba57833e09467a2411f0f2d/xxhash-3.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c31a2649bcf1fe97cf11c79848d761df33ac46b3896942d31b640557b486ff6b", size = 220765, upload-time = "2026-07-06T10:46:55.41Z" }, + { url = "https://files.pythonhosted.org/packages/8f/10/00d12d8b8beabbf49a8bbc626fb9f40445145a8887eb41a6acfb69149ac4/xxhash-3.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8f759eed402448c2bdbb492e4fba1f20668ffe29688605ea61f0f67f9e4e386d", size = 310478, upload-time = "2026-07-06T10:46:57.729Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f9/12a82394eefb0f185d15a7f7b9f627c61c475a72dd83718436a5b84b42ac/xxhash-3.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b5f97ecfede10d5b2870383620e2d25c8561e217c7bf9081073802b54248d2b", size = 238393, upload-time = "2026-07-06T10:46:59.87Z" }, + { url = "https://files.pythonhosted.org/packages/20/f3/53f963e320b9ce678337aa7273f39ce692ded8b99e3d22a866ec722159ab/xxhash-3.8.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1da930bbcac3e8fbe2191850e2abb57977a99348c12c4b385e1058ac1b0a9ecc", size = 268704, upload-time = "2026-07-06T10:47:01.806Z" }, + { url = "https://files.pythonhosted.org/packages/0a/50/5b5badbd87c82d9f9b5f58ac74a3f29ef08f6fc387b324b8fd482450b862/xxhash-3.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:747476436f6891b9773374ce8d48edcc8b12cb5b61b67c6fb6289633747d088f", size = 225015, upload-time = "2026-07-06T10:47:03.784Z" }, + { url = "https://files.pythonhosted.org/packages/30/93/3ca68265afe7b4e69435e08a7b6a1d9d0f2a071e889da1f8041ed00fe878/xxhash-3.8.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4ef09bbc2519a93cd0f95f2ceb5f7b85919dffea643278e02362bf40e3c4bed1", size = 240951, upload-time = "2026-07-06T10:47:05.816Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a6/27e19670c40f46b5e76e11f2f4713d21054804568425d870670e757172ad/xxhash-3.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a5eed9d41995a83f3332b4e3396abb7f433cac584222bd7e305b606d8353861e", size = 300751, upload-time = "2026-07-06T10:47:07.95Z" }, + { url = "https://files.pythonhosted.org/packages/bc/fb/b33e27689959fe7ed2ae0b830af41560d65213943983afa9db3a8d481bce/xxhash-3.8.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:53f3ed9118397074ff63a79b66b7fec1c84c782eecde35c5bc94e420a971c231", size = 443480, upload-time = "2026-07-06T10:47:10Z" }, + { url = "https://files.pythonhosted.org/packages/26/60/0e0d973be5fe280753ef02fbc89349492ad6e903bf1dcb870b668f94b662/xxhash-3.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d247b34bf433c92b41689318fd25d246313cab2275a6a47e2efac178b80d6efe", size = 217657, upload-time = "2026-07-06T10:47:12.196Z" }, + { url = "https://files.pythonhosted.org/packages/ad/68/c9e3ecef4a9a417d464cb5bd200aa12f73192dee677901b9e08e0ad0d1bb/xxhash-3.8.1-cp314-cp314-win32.whl", hash = "sha256:d58ce8b6cfa9c4d2f230557f69caf7c06369e318015d0b19485095bc2c5963ab", size = 32690, upload-time = "2026-07-06T10:47:14.204Z" }, + { url = "https://files.pythonhosted.org/packages/d7/99/e9e44588c0b62837bbec5ba7927816de0afa03406b1a0b6c7a7e1d1a30a0/xxhash-3.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:6cee733fe4ccb1737e0997135283c82341e5cfa9cf214b165f9087fb663aaf4f", size = 33460, upload-time = "2026-07-06T10:47:16.021Z" }, + { url = "https://files.pythonhosted.org/packages/45/2b/64f36d86380b3657ad9031967ab814f3ef31307174650853f69c18932ebc/xxhash-3.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:58346024d47e84f7d8b3e7f5d6faa1d58acbbe49a8771497872059f58c1d8ea5", size = 30092, upload-time = "2026-07-06T10:47:17.81Z" }, + { url = "https://files.pythonhosted.org/packages/92/cb/18b64bff88c58a0ca209dc533e63cf02d7ae5aa6b1b9a9fd14e81b5dbd60/xxhash-3.8.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:01cab782f8a0a05ecad2c63d7ef10f7ab475f660e0d6419d069418c14d88de7c", size = 35024, upload-time = "2026-07-06T10:47:19.821Z" }, + { url = "https://files.pythonhosted.org/packages/af/1d/72d8a70520e5dcddb472ea0486d299da3240745a10658290cd7b5690ede2/xxhash-3.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:717b12fdc51819833704e85e6926d76981ffa3f780ef92e33ebb8b26d46bb230", size = 32697, upload-time = "2026-07-06T10:47:21.649Z" }, + { url = "https://files.pythonhosted.org/packages/9c/b8/e041f555903c56db3d0a731b3d72a6575d75e0ed868b1bd2e5176111ca44/xxhash-3.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ec55d80e9b8a519d742669e0b49e8ce9e6747be42bf3c138158b6543a9c8e489", size = 226044, upload-time = "2026-07-06T10:47:23.612Z" }, + { url = "https://files.pythonhosted.org/packages/3a/7e/5cdcf06bf6ec4b5d2ac073feb23432ec1d603fd438864cbd2c09c7cb45e1/xxhash-3.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98d8ac1129b4dd39098cffed94d1284aceb61c3aa396757ccc736ac392e4cee5", size = 249899, upload-time = "2026-07-06T10:47:25.812Z" }, + { url = "https://files.pythonhosted.org/packages/c0/c0/eb7e059cb5e1dba11fd30d2fdf882f56e5a417a3eaa43669d43623767f45/xxhash-3.8.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3bc0fa90830df1e1277f33cc6e55de9990b83c0319fd8c7412866cfde38b025e", size = 274892, upload-time = "2026-07-06T10:47:27.931Z" }, + { url = "https://files.pythonhosted.org/packages/66/74/a600aaf7cd39957fd1510adeedb1749c1e7eb82bd632a1153d9c664c3135/xxhash-3.8.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c73b6f652f0745425aa6378319c331293b5341756262e9408ed3d45f183375e6", size = 252243, upload-time = "2026-07-06T10:47:30.288Z" }, + { url = "https://files.pythonhosted.org/packages/ad/04/78d88fa75a6763e5d09bf1b947a392a27988903381b219006f92f3c68fc8/xxhash-3.8.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6114692261eff4266386cdec0f7d87eee24e317ab397c218b7ae6a76b4c6339", size = 482191, upload-time = "2026-07-06T10:47:32.45Z" }, + { url = "https://files.pythonhosted.org/packages/7f/06/07a8aea1108d682de8791ce608cdf367d75ff4e7e57cd3c154bdc6f47b23/xxhash-3.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df57c0b161ec1b3ed0526a67b0db0914b557e86ee8aae51887aec941b261542", size = 226877, upload-time = "2026-07-06T10:47:34.705Z" }, + { url = "https://files.pythonhosted.org/packages/ed/b5/86bade5618a524d2c06c4041aa2fe8e5749ce16e88afba60d67c1684a21f/xxhash-3.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9043877a917be88ccf230aa5667c1bd059bce80f4c2727e4defa1b29b7f48b08", size = 319794, upload-time = "2026-07-06T10:47:37.08Z" }, + { url = "https://files.pythonhosted.org/packages/23/69/9b1a2b89b1621bb740fbcb7beb512f60f99480c1bdc680c0c90e1f56ff75/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:559e3cabe522231909f9de98ef06929edbd53782046bd21aae0c72db6f2a0775", size = 246202, upload-time = "2026-07-06T10:47:39.676Z" }, + { url = "https://files.pythonhosted.org/packages/08/ea/662ed6cb49f1d34078b6a3a3e0f3d29ff93fd7b5a03c0bc9ecfd9b2159c3/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:264710bd335016f303763ce1275c6486df30bb57c2245c91b224c983d7ac39b8", size = 275628, upload-time = "2026-07-06T10:47:41.99Z" }, + { url = "https://files.pythonhosted.org/packages/13/f5/49fc9e4c6728a5a3bd8fe639199d2fa67609b3a84f938aff6e8568dd3e4f/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e14800b9b10bb39d7a60ad4a310e403164d7b8988a27ae933d4e40618a44088e", size = 231390, upload-time = "2026-07-06T10:47:44.233Z" }, + { url = "https://files.pythonhosted.org/packages/64/9d/3acaf8f599c0e0b30e910a3a11ba32929da53c86dc73c7c55fe6a010b4e9/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:ea6a3e734b0fd41b82784a400be946821900daebe610c050a5e0760838a34f99", size = 250600, upload-time = "2026-07-06T10:47:47.611Z" }, + { url = "https://files.pythonhosted.org/packages/23/64/8acab4c5ec60dbe664b5b9858fd44c2413b07e535b09556a0a5022e78aa6/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cf399fac542a1c7a4734a435b93df2c55e858c7d31abf6c1bdf46f9ae67fbfd0", size = 310032, upload-time = "2026-07-06T10:47:49.88Z" }, + { url = "https://files.pythonhosted.org/packages/56/47/a0288d7329b1fe63e2734a32d19d444a96ae2b4810f545bc61e561224917/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:44c89d915a75c11d2547eaee9098fcd80398987c4bff2974a0497a925bf92c07", size = 448882, upload-time = "2026-07-06T10:47:52.631Z" }, + { url = "https://files.pythonhosted.org/packages/01/e7/3071dfd3beb5c38204ce1cf56bf7749fce08de900fa92714b81d1d8ca1f2/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:358650d5bda9c635da699c53adf4e8134af492ecc79c960f917eebf088bb6799", size = 223728, upload-time = "2026-07-06T10:47:55.093Z" }, + { url = "https://files.pythonhosted.org/packages/12/11/b99949f0ba2b07e9f9ffe83b9c86faa685f9080725dc21a916a607313be5/xxhash-3.8.1-cp314-cp314t-win32.whl", hash = "sha256:c240939e963653054fc7e4a17c382829cda4aa88a7daf0af841715dbded1b497", size = 33150, upload-time = "2026-07-06T10:47:57.274Z" }, + { url = "https://files.pythonhosted.org/packages/54/1c/09703eb341f8416e74e58d6c6732d4b5c46de59c942363203cb237cc95b0/xxhash-3.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:7258ee276e8772599bc19e14b36f6260306e21b637190cd7cb489a2449d48684", size = 34005, upload-time = "2026-07-06T10:47:59.434Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f9/6ed7251bb6a8af10ac73b1821c60583d2826e5b2064e45a979c935287c98/xxhash-3.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:8f454166c2ffed45636c8d501741e649851ba2f346c4eb73a64c07ac00428f20", size = 30239, upload-time = "2026-07-06T10:48:01.874Z" }, + { url = "https://files.pythonhosted.org/packages/99/e4/4d8040435aeac814fc69ba63621565fbeb19229a138e2568324a26b2a45c/xxhash-3.8.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:39c9d5b61508b0bb68f29e54546de0ed2a74943c6a18585535a7e37356f1dd12", size = 32687, upload-time = "2026-07-06T10:49:42.803Z" }, + { url = "https://files.pythonhosted.org/packages/da/6a/975f1f2318c760e5bcec109ed379713ae645d8d856c2a3b9ec5d26857087/xxhash-3.8.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:83b9130b80b216d56fdf9e87131946b353c9627930c061955a101ea82b09fed9", size = 29879, upload-time = "2026-07-06T10:49:45.172Z" }, + { url = "https://files.pythonhosted.org/packages/08/0b/40a2a55ff52cf635bfdc5eae67a772bec85b4f44c6c737f73f6f528d51d1/xxhash-3.8.1-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8304be0982130954b7fd3aad18e2c6f8ee40254bc3d2e635991c16d77c91e2bd", size = 43246, upload-time = "2026-07-06T10:49:47.905Z" }, + { url = "https://files.pythonhosted.org/packages/9c/6d/56ed2b6b200f26fb474f3fd387d95d0601efcd5bb33430c90c68924bdd77/xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b512261801b1e5fde7b6ebf2fef7977339c620cbbca88a0040ad9ad134f4d02", size = 38202, upload-time = "2026-07-06T10:49:50.59Z" }, + { url = "https://files.pythonhosted.org/packages/0d/a3/56864d895d1161a9f17502088e9c1fb7c06bde2c2efdde620d22bb7a9c43/xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49aa8692507835dcc1e8ad8021f20c74c2dc13d83b5112e87877faa2a0035b20", size = 34448, upload-time = "2026-07-06T10:49:53.242Z" }, + { url = "https://files.pythonhosted.org/packages/6b/57/5c6e0908a47f61dca96d01c8ee6fce01ed1050611eb779083ba8758fed81/xxhash-3.8.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:345b07b78e2bf583d71682aa34ae5b5fab575f7a1cb31e10263ebbc6f89f8c42", size = 32869, upload-time = "2026-07-06T10:49:55.972Z" }, +] + [[package]] name = "yarl" version = "1.24.2" From dc53351c69c3cd695d7985b54fe71e38fcd95bd0 Mon Sep 17 00:00:00 2001 From: "Paul S. Schweigert" Date: Tue, 4 Aug 2026 18:07:36 -0400 Subject: [PATCH 2/2] review comments Signed-off-by: Paul S. Schweigert --- .github/CODEOWNERS | 6 + .../examples/simbauq/simbauq_example.py | 2 +- .../agent_utilities/core/simbauq.py | 10 +- agent-utilities/tests/test_simbauq.py | 127 ++++++++++++++++++ 4 files changed, 141 insertions(+), 4 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 78f309e2..e9024f6d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,8 @@ # Default: request review from maintainers * @generative-computing/mellea-maintainers + +# SIMBA-UQ sampling strategy: pulls in scikit-learn / sentence-transformers and +# has confidence-estimation footguns worth a specialist's eye on changes. +/agent-utilities/mellea_contribs/agent_utilities/core/simbauq.py @radum2275 +/agent-utilities/tests/test_simbauq.py @radum2275 +/agent-utilities/examples/simbauq/ @radum2275 diff --git a/agent-utilities/examples/simbauq/simbauq_example.py b/agent-utilities/examples/simbauq/simbauq_example.py index 506fbed3..e4913d90 100644 --- a/agent-utilities/examples/simbauq/simbauq_example.py +++ b/agent-utilities/examples/simbauq/simbauq_example.py @@ -1,4 +1,4 @@ -# pytest: ollama, llm, qualitative +# pytest: ollama, e2e, qualitative """SIMBA-UQ Sampling Strategy Example. diff --git a/agent-utilities/mellea_contribs/agent_utilities/core/simbauq.py b/agent-utilities/mellea_contribs/agent_utilities/core/simbauq.py index 81d153ef..33703c4d 100644 --- a/agent-utilities/mellea_contribs/agent_utilities/core/simbauq.py +++ b/agent-utilities/mellea_contribs/agent_utilities/core/simbauq.py @@ -282,25 +282,28 @@ async def sample( all_contexts = [] all_actions: list[Component[S]] = [] temp_assignments: list[float] = [] + failed_count = 0 + flog = MelleaLogger.get_logger() for gen_result, task_action, task_temp in zip( generation_results, task_actions, task_temps ): if isinstance(gen_result, BaseException): + failed_count += 1 + flog.warning(f"Sample generation failed: {gen_result}") continue # Skip failed generations. result_mot, result_ctx = gen_result await result_mot.avalue() try: result_mot.parsed_repr = task_action.parse(result_mot) except ComponentParseError as e: - print(f"Error parsing result: {e}") + failed_count += 1 + flog.warning(f"Error parsing result: {e}") continue # Skip unparsable results. all_mots.append(result_mot) all_contexts.append(result_ctx) all_actions.append(task_action) temp_assignments.append(task_temp) - flog = MelleaLogger.get_logger() - # --- Phase 2: Compute SIMBA-UQ confidence scores --- sample_strings = [str(mot) for mot in all_mots] degraded = False @@ -341,6 +344,7 @@ async def sample( "confidence_method": self.confidence_method, "similarity_metric": self.similarity_metric, "aggregation": self.aggregation, + "failed_count": failed_count, } # Mark as final result. diff --git a/agent-utilities/tests/test_simbauq.py b/agent-utilities/tests/test_simbauq.py index d023616c..fa2439cd 100644 --- a/agent-utilities/tests/test_simbauq.py +++ b/agent-utilities/tests/test_simbauq.py @@ -3,6 +3,10 @@ import numpy as np import pytest +from mellea.core import Component +from mellea.core.base import ModelOutputThunk +from mellea.stdlib.context import SimpleContext + from mellea_contribs.agent_utilities.core.simbauq import SIMBAUQSamplingStrategy # --- Unit tests (no LLM required) --- @@ -381,6 +385,129 @@ def test_zero_n_per_temp_raises(self): SIMBAUQSamplingStrategy(n_per_temp=0) +# --- sample() tests (mocked backend, no LLM required) --- + + +class _FakeAction(Component): + """Minimal Component that echoes the model output, rejecting ``BAD*`` text. + + Used to exercise the parse-error branch of ``sample()`` without an LLM. + """ + + def _parse(self, computed: ModelOutputThunk) -> str: + text = str(computed) + if text.startswith("BAD"): + raise ValueError("unparsable output") + return text + + def format_for_llm(self) -> str: + return "prompt" + + def parts(self) -> list: + return [] + + +class _ScriptedBackend: + """Backend stub returning a queued sequence of outcomes, one per generate call. + + Each outcome is either a string (yields a ``ModelOutputThunk`` with that + value) or an ``Exception`` (raised so ``asyncio.gather`` records it as a + failed generation). + """ + + def __init__(self, outcomes: list): + self._outcomes = list(outcomes) + self._i = 0 + + async def generate_from_context( + self, + action, + ctx, + *, + format=None, + model_options=None, + tool_calls=False, + ): + outcome = self._outcomes[self._i] + self._i += 1 + if isinstance(outcome, Exception): + raise outcome + return ModelOutputThunk(value=outcome), ctx + + +class TestSampleMockedBackend: + """Exercise the async ``sample()`` path with a scripted backend.""" + + async def test_selects_most_confident_and_counts_failures(self): + # 6 requested samples: 4 usable, 1 generation failure, 1 parse error. + outcomes = [ + "the capital of france is paris", + "paris is the capital of france", + RuntimeError("rate limited"), + "france capital is paris", + "BAD unparsable output", + "bananas are yellow fruit", # outlier + ] + strategy = SIMBAUQSamplingStrategy( + temperatures=[0.3, 0.7, 1.0], + n_per_temp=2, + similarity_metric="jaccard", + ) + result = await strategy.sample( + _FakeAction(), + SimpleContext(), + _ScriptedBackend(outcomes), + requirements=None, + ) + + # Only the 4 usable samples survive; the failure and parse error are dropped. + assert len(result.sample_generations) == 4 + assert result.success is True + + meta = result.result._meta["simba_uq"] + assert meta["failed_count"] == 2 + assert meta["degraded"] is False + assert len(meta["all_confidences"]) == 4 + # The outlier should not be the selected (most confident) sample. + assert str(result.result) != "bananas are yellow fruit" + + async def test_single_successful_sample_is_degraded(self): + outcomes = [ + "the only good answer", + RuntimeError("boom"), + RuntimeError("boom"), + RuntimeError("boom"), + ] + strategy = SIMBAUQSamplingStrategy( + temperatures=[0.5], n_per_temp=4, similarity_metric="jaccard" + ) + result = await strategy.sample( + _FakeAction(), + SimpleContext(), + _ScriptedBackend(outcomes), + requirements=None, + ) + + meta = result.result._meta["simba_uq"] + assert meta["degraded"] is True + assert meta["confidence"] is None + assert meta["failed_count"] == 3 + assert len(result.sample_generations) == 1 + + async def test_all_samples_failing_raises(self): + outcomes = [RuntimeError("x"), RuntimeError("y")] + strategy = SIMBAUQSamplingStrategy( + temperatures=[0.5], n_per_temp=2, similarity_metric="jaccard" + ) + with pytest.raises(RuntimeError, match="No successful samples"): + await strategy.sample( + _FakeAction(), + SimpleContext(), + _ScriptedBackend(outcomes), + requirements=None, + ) + + # --- Integration test (requires Ollama) ---