feat: add seeded Thompson-sampling router policy (#48)

README.md

@@ -393,7 +393,7 @@ Forks, issues, and amendments welcome.
 The protocol layer is stable. Work in flight, in rough order of expected impact:
 
 - **`examples/` gallery** — 3 worked polises (research team, product team, OSS maintainer trio) to teach by example. Contributions welcome.
-- **Alternate routers** — a deterministic UCB1 variant ships behind `route_contract.py --policy ucb` (see `references/routing.md`); a Thompson-sampling variant is still to come, side-by-side with the default ε-greedy bandit. Benchmark harness on synthetic capability traces.
+- **Alternate routers** — a deterministic UCB1 variant ships behind `route_contract.py --policy ucb` and a seeded Thompson-sampling variant behind `--policy thompson` (both documented in `references/routing.md`), side-by-side with the default ε-greedy bandit. Benchmark harness on synthetic capability traces.
 - **Contextual bandit** — incorporate per-contract features (deadline pressure, stakes level, language) into the routing decision, not just per-tag history.
 - **Auto-rollover** — quarterly chronicle rollover and 90-day settled-contract archival as a one-line cron, so a year-long polis stays bounded without manual hygiene.
 - **Bridge expansions** — first-class entry pointers for Aider, opencode, Zed, Devin, Cursor agent mode. Each is a 30-line markdown stub.

polis/routing.py

@@ -12,7 +12,7 @@
 Optional:
     --explain                 # print the score breakdown for every citizen
     --apply                   # write the recommendation into the contract's routing.recommended_by_router field
-    --policy P                # selection policy: epsilon-greedy (default) or ucb
+    --policy P                # selection policy: epsilon-greedy (default), ucb, or thompson
     --adaptive                # use adaptive explore_rate per-tag based on leader_confidence (epsilon-greedy only)
     --reconcile               # rebuild routing_stats.yml from settled contracts; ignores --contract
 
@@ -49,14 +49,28 @@
 LESSON_CAP = 0.15
 
 # Selection policies. epsilon-greedy is the historical default; ucb is an
-# optional deterministic UCB1 variant (see references/routing.md).
-POLICIES = ("epsilon-greedy", "ucb")
+# optional deterministic UCB1 variant; thompson is a seeded Thompson-sampling
+# variant (see references/routing.md).
+POLICIES = ("epsilon-greedy", "ucb", "thompson")
 
 # UCB1 exploration constant (the `c` in mean + c*sqrt(ln(total)/n)). sqrt(2) is
 # the textbook value; larger explores more. Fixed rather than CLI-tunable to
 # keep the surface small — tune weights/explore_rate in polis.yml instead.
 UCB_EXPLORATION_C = math.sqrt(2)
 
+# Thompson-sampling posterior widths. Each citizen's combined score is the mean
+# of a Gaussian posterior whose standard deviation is THOMPSON_EXPLORATION_C /
+# sqrt(n): wide when little evidence exists, shrinking as sample count n grows,
+# so early picks explore and later ones exploit. Fixed (not CLI-tunable) to keep
+# the flag surface small, mirroring UCB_EXPLORATION_C.
+THOMPSON_EXPLORATION_C = math.sqrt(2)
+
+# Unplayed citizens (n == 0) get this wide fixed prior instead of an undefined
+# c/sqrt(0). It is wider than the n == 1 posterior (sqrt(2) ~ 1.414), so the
+# posterior width is monotonically decreasing in n and every untried citizen is
+# sampled broadly — the cold-start guarantee that all arms still get a real shot.
+THOMPSON_PRIOR_SIGMA = 2.0
+
 COST_FIT = {
     # cost_ceiling -> citizen.cost_envelope.relative -> score
     ("low", "low"): 1.0,
@@ -339,6 +353,67 @@ def ucb_value(s: dict) -> float:
     return chosen["citizen"], is_exploration, chosen["total"]
 
 
+def thompson_posterior_sigma(n: int) -> float:
+    """Posterior standard deviation for a citizen with sample count ``n``.
+
+    Unplayed citizens (``n <= 0``) get the wide ``THOMPSON_PRIOR_SIGMA`` prior;
+    otherwise the width is ``THOMPSON_EXPLORATION_C / sqrt(n)``, shrinking as
+    evidence accumulates. Monotonically decreasing in ``n`` (the prior is wider
+    than the ``n == 1`` width), so more-sampled arms have tighter posteriors.
+    """
+    if n <= 0:
+        return THOMPSON_PRIOR_SIGMA
+    return THOMPSON_EXPLORATION_C / math.sqrt(n)
+
+
+def thompson_sample_values(scores: list, counts: dict, rng: random.Random) -> dict:
+    """Draw one Gaussian posterior sample per citizen: id -> sampled value.
+
+    Each citizen's combined score is the posterior mean; the width comes from
+    ``thompson_posterior_sigma(n)``. Citizens are sampled in citizen-id order so
+    the sequence of ``rng`` draws — and therefore every sample — is fully
+    determined by the seed, independent of dict/insertion order. That is what
+    makes a seeded run reproducible.
+    """
+    samples = {}
+    for s in sorted(scores, key=lambda s: s["citizen"]):
+        n = counts.get(s["citizen"], 0)
+        samples[s["citizen"]] = rng.gauss(s["total"], thompson_posterior_sigma(n))
+    return samples
+
+
+def pick_from_thompson_samples(scores: list, samples: dict) -> tuple:
+    """Return (chosen_citizen_id, exploration_bool, score_of_choice) for samples.
+
+    Picks the citizen with the highest sampled posterior value; sampled-value
+    ties (vanishingly rare with continuous draws) break by citizen id ascending.
+    ``exploration`` is True when the sampled winner differs from the pure-exploit
+    pick (the highest-mean citizen), i.e. sampling changed the decision.
+    """
+    if not scores:
+        return None, False, 0.0
+    chosen = min(scores, key=lambda s: (-samples[s["citizen"]], s["citizen"]))
+    exploit_leader = sorted(scores, key=lambda s: (-s["total"], s["citizen"]))[0]
+    is_exploration = chosen["citizen"] != exploit_leader["citizen"]
+    return chosen["citizen"], is_exploration, chosen["total"]
+
+
+def pick_thompson(scores: list, counts: dict, rng: random.Random) -> tuple:
+    """Return (chosen_citizen_id, exploration_bool, score_of_choice) via Thompson.
+
+    Thompson sampling over a Gaussian posterior per citizen: sample one value
+    from each citizen's ``N(score, sigma(n))`` posterior and pick the max. Unlike
+    UCB this is stochastic, so ``rng`` (a seeded ``random.Random``) is injected —
+    the same seed always yields the same pick, which is what keeps it reproducible
+    in tests and audits. Unplayed citizens (``n == 0``) get the wide prior so
+    cold-start still samples every untried citizen broadly.
+    """
+    if not scores:
+        return None, False, 0.0
+    samples = thompson_sample_values(scores, counts, rng)
+    return pick_from_thompson_samples(scores, samples)
+
+
 def recommend(polis_root, contract_id, seed=None, adaptive=False, policy="epsilon-greedy") -> dict:
     """Score every citizen for an open contract and pick a recommendation.
 
@@ -388,6 +463,10 @@ def recommend(polis_root, contract_id, seed=None, adaptive=False, policy="epsilo
         required_tags = contract_fm.get("required_tags") or []
         counts = {cid: citizen_sample_count(cid, required_tags, stats) for cid in citizens}
         chosen, is_exploration, chosen_score = pick_ucb(scores, counts)
+    elif policy == "thompson":
+        required_tags = contract_fm.get("required_tags") or []
+        counts = {cid: citizen_sample_count(cid, required_tags, stats) for cid in citizens}
+        chosen, is_exploration, chosen_score = pick_thompson(scores, counts, random.Random(seed))
     else:
         chosen, is_exploration, chosen_score = pick_recommendation(
             scores, explore_rate, random.Random(seed))
@@ -531,7 +610,8 @@ def main():
     parser.add_argument("--explain", action="store_true", help="Print score breakdown for every citizen.")
     parser.add_argument("--apply", action="store_true", help="Write the recommendation into the contract.")
     parser.add_argument("--policy", choices=POLICIES, default="epsilon-greedy",
-                        help="Selection policy: epsilon-greedy (default) or ucb (deterministic UCB1).")
+                        help="Selection policy: epsilon-greedy (default), ucb (deterministic UCB1), "
+                             "or thompson (seeded Thompson sampling).")
     parser.add_argument("--adaptive", action="store_true", help="Use adaptive explore_rate per-tag (epsilon-greedy only).")
     parser.add_argument("--reconcile", action="store_true", help="Rebuild routing_stats.yml from settled contracts.")
     parser.add_argument("--seed", type=int, default=None, help="Random seed for reproducible exploration.")
@@ -585,10 +665,16 @@ def main():
         status = load_citizen_status(polis_root, cid)
         scores.append(score_citizen(cid, card, status, contract_fm, stats, weights, lessons))
 
+    thompson_sampled = None
     if args.policy == "ucb":
         required_tags = contract_fm.get("required_tags") or []
         counts = {cid: citizen_sample_count(cid, required_tags, stats) for cid in citizens}
         chosen, is_exploration, chosen_score = pick_ucb(scores, counts)
+    elif args.policy == "thompson":
+        required_tags = contract_fm.get("required_tags") or []
+        counts = {cid: citizen_sample_count(cid, required_tags, stats) for cid in citizens}
+        thompson_sampled = thompson_sample_values(scores, counts, random.Random(args.seed))
+        chosen, is_exploration, chosen_score = pick_from_thompson_samples(scores, thompson_sampled)
     else:
         rng = random.Random(args.seed)
         chosen, is_exploration, chosen_score = pick_recommendation(scores, explore_rate, rng)
@@ -605,6 +691,11 @@ def main():
         print(f"\nPolicy: {args.policy}")
         if args.policy == "ucb":
             print(f"UCB exploration constant c: {UCB_EXPLORATION_C:.3f}")
+        elif args.policy == "thompson":
+            print(f"Thompson exploration scale c: {THOMPSON_EXPLORATION_C:.3f}  (seed: {args.seed})")
+            print("Sampled posterior values (sorted):")
+            for cid in sorted(thompson_sampled, key=lambda k: thompson_sampled[k], reverse=True):
+                print(f"  {cid:30s}  sample={thompson_sampled[cid]:+.3f}")
         else:
             print(f"Explore rate: {explore_rate:.2f}")
 

references/routing.md

@@ -131,6 +131,28 @@ An **unplayed citizen** (`n = 0`) has an infinite bonus, so UCB tries every untr
 
 **Tradeoff.** UCB is fully deterministic: the same state always yields the same pick (ties, including ties between unplayed citizens, break by citizen id ascending), which makes it reproducible in tests and audits. Epsilon-greedy is stochastic, so it can spread work more evenly across near-equal citizens over many contracts but needs a seed to reproduce. UCB explores more aggressively early (every new citizen gets a turn) and then commits harder to leaders; epsilon-greedy keeps a constant background exploration rate. `--adaptive` applies only to epsilon-greedy.
 
+### Thompson sampling: a posterior-based alternative
+
+Thompson sampling is the third bandit policy, available via `scripts/route_contract.py --policy thompson`. Instead of a confidence bonus (UCB) or an ε-coin-flip (epsilon-greedy), it explores by **sampling from each citizen's posterior** and picking whoever draws highest.
+
+Each citizen's combined `score` is treated as the mean of a Gaussian posterior whose standard deviation shrinks as their sample count `n` grows:
+
+```
+sample(citizen) ~ Normal(mean = score(citizen), sigma = c / sqrt(n))
+pick = argmax over citizens of sample(citizen)
+```
+
+- `score(citizen)` is the same combined score used everywhere else — the posterior mean.
+- `n` is the citizen's total `contracts_completed` across the contract's `required_tags` (the same count UCB uses).
+- `c` is the exploration scale (`sqrt(2)`, mirroring the UCB constant). Fixed rather than CLI-tunable to keep the flag surface small.
+- An **unplayed citizen** (`n = 0`) gets a wide fixed prior (`sigma = 2.0`, wider than the `n = 1` width of `sqrt(2)`), so cold-start samples every untried citizen broadly and each has a real shot at the top draw. The posterior width is monotonically decreasing in `n`.
+
+**Cold-start behavior.** With everyone unplayed, the wide prior means picks spread across *all* citizens over successive contracts (weighted toward higher means but never locked to the top one) — similar in spirit to UCB trying every arm, but smoothed: a slightly-lower-mean newcomer still wins a healthy share of early draws instead of strictly waiting its turn. As evidence accumulates the posteriors tighten and the policy commits to the leader.
+
+**Determinism.** Thompson sampling is stochastic, so like epsilon-greedy it needs a seed to reproduce: pass `--seed N` (or `seed=` through `recommend(...)`). Citizens are sampled in citizen-id order, so a given `(state, seed)` always yields the same pick and the same `--explain` sampled values — reproducible in tests and audits despite the randomness. Without `--seed`, the RNG is entropy-seeded (same as epsilon-greedy).
+
+**Tradeoff vs epsilon-greedy and UCB.** Epsilon-greedy explores a *fixed fraction* of the time regardless of how certain the ranking is. UCB is deterministic and explores via an additive bonus that decays like `sqrt(ln(total_n)/n)`. Thompson explores in proportion to *posterior overlap*: when two citizens are genuinely close it splits work between them smoothly, and when one is a clear, well-sampled leader the overlap vanishes and it exploits. It tends to spread early work across near-equal citizens more evenly than UCB's turn-taking, at the cost of being non-deterministic without a seed. `--adaptive` applies only to epsilon-greedy.
+
 ## Updating the policy when a contract settles
 
 When a contract moves to `_polis/contracts/settled/`, the router updates `routing_stats.yml`:

scripts/test_lessons_routing.py

@@ -7,23 +7,30 @@
   a structured routing_effect changes a later recommendation and is named in the
   explanation, the effect is bounded, and un-accepted lessons are ignored; and
 * the deterministic UCB1 variant — cold-start (unplayed arms first) and
-  learned-history (mean + confidence bonus) selection, with no randomness.
+  learned-history (mean + confidence bonus) selection, with no randomness; and
+* the seeded Thompson-sampling variant — seeded determinism, cold-start spread
+  across unplayed arms, exploit-on-ample-evidence, and empty-input safety.
 """
+import random
 import sys
 import tempfile
 import unittest
+from collections import Counter
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
 
 from polis.routing import (  # noqa: E402
     LESSON_CAP,
+    THOMPSON_PRIOR_SIGMA,
     WEIGHTS_DEFAULT,
     citizen_sample_count,
     load_lessons,
     lessons_score,
+    pick_thompson,
     pick_ucb,
     score_citizen,
+    thompson_posterior_sigma,
 )
 
 CONTRACT = {"required_tags": ["spanish-translation"], "cost_ceiling": "medium"}
@@ -180,5 +187,78 @@ def test_empty_scores_is_safe(self):
         self.assertEqual(pick_ucb([], {}), (None, False, 0.0))
 
 
+class ThompsonRoutingTest(unittest.TestCase):
+    """The optional Thompson policy is stochastic but seed-reproducible."""
+
+    def test_posterior_sigma_shrinks_with_sample_count(self):
+        # Unplayed arms get the wide prior; width then shrinks monotonically as
+        # evidence accumulates, so more-sampled arms have tighter posteriors.
+        self.assertEqual(thompson_posterior_sigma(0), THOMPSON_PRIOR_SIGMA)
+        self.assertLess(thompson_posterior_sigma(1), THOMPSON_PRIOR_SIGMA)
+        self.assertLess(thompson_posterior_sigma(100), thompson_posterior_sigma(4))
+        self.assertLess(thompson_posterior_sigma(4), thompson_posterior_sigma(1))
+
+    def test_same_seed_gives_same_pick_across_calls(self):
+        # Reproducibility: a fixed seed pins the pick, call after call.
+        scores = [{"citizen": "a", "total": 0.60}, {"citizen": "b", "total": 0.55}]
+        counts = {"a": 3, "b": 3}
+        first = pick_thompson(scores, counts, random.Random(7))
+        for _ in range(20):
+            self.assertEqual(pick_thompson(scores, counts, random.Random(7)), first)
+
+    def test_pick_is_independent_of_score_order(self):
+        # Sampling in citizen-id order makes the pick a function of the seed
+        # alone, not of dict/list insertion order.
+        counts = {"a": 3, "b": 3}
+        ab = [{"citizen": "a", "total": 0.60}, {"citizen": "b", "total": 0.55}]
+        ba = [{"citizen": "b", "total": 0.55}, {"citizen": "a", "total": 0.60}]
+        self.assertEqual(
+            pick_thompson(ab, counts, random.Random(7)),
+            pick_thompson(ba, counts, random.Random(7)),
+        )
+
+    def test_cold_start_spreads_across_unplayed_arms(self):
+        # Three unplayed citizens (n == 0) with differing means: the wide prior
+        # spreads picks across ALL of them over many seeds — even the lowest
+        # mean arm wins sometimes — so cold-start reaches everyone.
+        scores = [{"citizen": "a", "total": 0.9},
+                  {"citizen": "b", "total": 0.5},
+                  {"citizen": "c", "total": 0.1}]
+        counts = {"a": 0, "b": 0, "c": 0}
+        picks = Counter(pick_thompson(scores, counts, random.Random(s))[0]
+                        for s in range(200))
+        self.assertEqual(set(picks), {"a", "b", "c"})  # every arm gets a turn
+        self.assertGreater(picks["c"], 0)  # even the weakest cold arm is tried
+
+    def test_exploits_the_leader_on_ample_evidence(self):
+        # Strong positive evidence (high mean, ample n -> tight posterior) beats
+        # weak evidence, and the pick matches the pure-exploit leader (not an
+        # exploration pick). Robust: 'a' wins for every seed in a wide sweep.
+        scores = [{"citizen": "a", "total": 0.90}, {"citizen": "b", "total": 0.30}]
+        counts = {"a": 100, "b": 100}
+        for s in range(50):
+            self.assertEqual(pick_thompson(scores, counts, random.Random(s))[0], "a")
+        chosen, is_exploration, score = pick_thompson(scores, counts, random.Random(7))
+        self.assertEqual(chosen, "a")
+        self.assertFalse(is_exploration)  # 'a' is also the highest-mean pick
+        self.assertEqual(score, 0.90)
+
+    def test_strong_evidence_pick_depends_on_the_mean_accounting(self):
+        # Decision-boundary / mutation-biting scenario. At seed 7 the correct
+        # posterior (centered on each citizen's combined score) picks 'a', the
+        # strong-evidence arm. This is the assertion the mutation transcript
+        # flips: dropping the mean from the sample (rng.gauss(0.0, sigma)) centers
+        # both arms at 0 and at seed 7 flips the pick to 'b'. So a broken
+        # success/failure accounting — sampling that ignores the evidence — makes
+        # this fail, while a correct one keeps 'a'.
+        scores = [{"citizen": "a", "total": 0.90}, {"citizen": "b", "total": 0.30}]
+        counts = {"a": 100, "b": 100}
+        chosen, _, _ = pick_thompson(scores, counts, random.Random(7))
+        self.assertEqual(chosen, "a")
+
+    def test_empty_scores_is_safe(self):
+        self.assertEqual(pick_thompson([], {}, random.Random(7)), (None, False, 0.0))
+
+
 if __name__ == "__main__":
     unittest.main()