diff --git a/README.md b/README.md index 5dc0040..18022bf 100644 --- a/README.md +++ b/README.md @@ -393,7 +393,7 @@ Forks, issues, and amendments welcome. The protocol layer is stable. Work in flight, in rough order of expected impact: - **`examples/` gallery** — 3 worked polises (research team, product team, OSS maintainer trio) to teach by example. Contributions welcome. -- **Alternate routers** — a deterministic UCB1 variant ships behind `route_contract.py --policy ucb` (see `references/routing.md`); a Thompson-sampling variant is still to come, side-by-side with the default ε-greedy bandit. Benchmark harness on synthetic capability traces. +- **Alternate routers** — a deterministic UCB1 variant ships behind `route_contract.py --policy ucb` and a seeded Thompson-sampling variant behind `--policy thompson` (both documented in `references/routing.md`), side-by-side with the default ε-greedy bandit. Benchmark harness on synthetic capability traces. - **Contextual bandit** — incorporate per-contract features (deadline pressure, stakes level, language) into the routing decision, not just per-tag history. - **Auto-rollover** — quarterly chronicle rollover and 90-day settled-contract archival as a one-line cron, so a year-long polis stays bounded without manual hygiene. - **Bridge expansions** — first-class entry pointers for Aider, opencode, Zed, Devin, Cursor agent mode. Each is a 30-line markdown stub. diff --git a/polis/routing.py b/polis/routing.py index 4c53cbc..b42ee71 100755 --- a/polis/routing.py +++ b/polis/routing.py @@ -12,7 +12,7 @@ Optional: --explain # print the score breakdown for every citizen --apply # write the recommendation into the contract's routing.recommended_by_router field - --policy P # selection policy: epsilon-greedy (default) or ucb + --policy P # selection policy: epsilon-greedy (default), ucb, or thompson --adaptive # use adaptive explore_rate per-tag based on leader_confidence (epsilon-greedy only) --reconcile # rebuild routing_stats.yml from settled contracts; ignores --contract @@ -49,14 +49,28 @@ LESSON_CAP = 0.15 # Selection policies. epsilon-greedy is the historical default; ucb is an -# optional deterministic UCB1 variant (see references/routing.md). -POLICIES = ("epsilon-greedy", "ucb") +# optional deterministic UCB1 variant; thompson is a seeded Thompson-sampling +# variant (see references/routing.md). +POLICIES = ("epsilon-greedy", "ucb", "thompson") # UCB1 exploration constant (the `c` in mean + c*sqrt(ln(total)/n)). sqrt(2) is # the textbook value; larger explores more. Fixed rather than CLI-tunable to # keep the surface small — tune weights/explore_rate in polis.yml instead. UCB_EXPLORATION_C = math.sqrt(2) +# Thompson-sampling posterior widths. Each citizen's combined score is the mean +# of a Gaussian posterior whose standard deviation is THOMPSON_EXPLORATION_C / +# sqrt(n): wide when little evidence exists, shrinking as sample count n grows, +# so early picks explore and later ones exploit. Fixed (not CLI-tunable) to keep +# the flag surface small, mirroring UCB_EXPLORATION_C. +THOMPSON_EXPLORATION_C = math.sqrt(2) + +# Unplayed citizens (n == 0) get this wide fixed prior instead of an undefined +# c/sqrt(0). It is wider than the n == 1 posterior (sqrt(2) ~ 1.414), so the +# posterior width is monotonically decreasing in n and every untried citizen is +# sampled broadly — the cold-start guarantee that all arms still get a real shot. +THOMPSON_PRIOR_SIGMA = 2.0 + COST_FIT = { # cost_ceiling -> citizen.cost_envelope.relative -> score ("low", "low"): 1.0, @@ -339,6 +353,67 @@ def ucb_value(s: dict) -> float: return chosen["citizen"], is_exploration, chosen["total"] +def thompson_posterior_sigma(n: int) -> float: + """Posterior standard deviation for a citizen with sample count ``n``. + + Unplayed citizens (``n <= 0``) get the wide ``THOMPSON_PRIOR_SIGMA`` prior; + otherwise the width is ``THOMPSON_EXPLORATION_C / sqrt(n)``, shrinking as + evidence accumulates. Monotonically decreasing in ``n`` (the prior is wider + than the ``n == 1`` width), so more-sampled arms have tighter posteriors. + """ + if n <= 0: + return THOMPSON_PRIOR_SIGMA + return THOMPSON_EXPLORATION_C / math.sqrt(n) + + +def thompson_sample_values(scores: list, counts: dict, rng: random.Random) -> dict: + """Draw one Gaussian posterior sample per citizen: id -> sampled value. + + Each citizen's combined score is the posterior mean; the width comes from + ``thompson_posterior_sigma(n)``. Citizens are sampled in citizen-id order so + the sequence of ``rng`` draws — and therefore every sample — is fully + determined by the seed, independent of dict/insertion order. That is what + makes a seeded run reproducible. + """ + samples = {} + for s in sorted(scores, key=lambda s: s["citizen"]): + n = counts.get(s["citizen"], 0) + samples[s["citizen"]] = rng.gauss(s["total"], thompson_posterior_sigma(n)) + return samples + + +def pick_from_thompson_samples(scores: list, samples: dict) -> tuple: + """Return (chosen_citizen_id, exploration_bool, score_of_choice) for samples. + + Picks the citizen with the highest sampled posterior value; sampled-value + ties (vanishingly rare with continuous draws) break by citizen id ascending. + ``exploration`` is True when the sampled winner differs from the pure-exploit + pick (the highest-mean citizen), i.e. sampling changed the decision. + """ + if not scores: + return None, False, 0.0 + chosen = min(scores, key=lambda s: (-samples[s["citizen"]], s["citizen"])) + exploit_leader = sorted(scores, key=lambda s: (-s["total"], s["citizen"]))[0] + is_exploration = chosen["citizen"] != exploit_leader["citizen"] + return chosen["citizen"], is_exploration, chosen["total"] + + +def pick_thompson(scores: list, counts: dict, rng: random.Random) -> tuple: + """Return (chosen_citizen_id, exploration_bool, score_of_choice) via Thompson. + + Thompson sampling over a Gaussian posterior per citizen: sample one value + from each citizen's ``N(score, sigma(n))`` posterior and pick the max. Unlike + UCB this is stochastic, so ``rng`` (a seeded ``random.Random``) is injected — + the same seed always yields the same pick, which is what keeps it reproducible + in tests and audits. Unplayed citizens (``n == 0``) get the wide prior so + cold-start still samples every untried citizen broadly. + """ + if not scores: + return None, False, 0.0 + samples = thompson_sample_values(scores, counts, rng) + return pick_from_thompson_samples(scores, samples) + + def recommend(polis_root, contract_id, seed=None, adaptive=False, policy="epsilon-greedy") -> dict: """Score every citizen for an open contract and pick a recommendation. @@ -388,6 +463,10 @@ def recommend(polis_root, contract_id, seed=None, adaptive=False, policy="epsilo required_tags = contract_fm.get("required_tags") or [] counts = {cid: citizen_sample_count(cid, required_tags, stats) for cid in citizens} chosen, is_exploration, chosen_score = pick_ucb(scores, counts) + elif policy == "thompson": + required_tags = contract_fm.get("required_tags") or [] + counts = {cid: citizen_sample_count(cid, required_tags, stats) for cid in citizens} + chosen, is_exploration, chosen_score = pick_thompson(scores, counts, random.Random(seed)) else: chosen, is_exploration, chosen_score = pick_recommendation( scores, explore_rate, random.Random(seed)) @@ -531,7 +610,8 @@ def main(): parser.add_argument("--explain", action="store_true", help="Print score breakdown for every citizen.") parser.add_argument("--apply", action="store_true", help="Write the recommendation into the contract.") parser.add_argument("--policy", choices=POLICIES, default="epsilon-greedy", - help="Selection policy: epsilon-greedy (default) or ucb (deterministic UCB1).") + help="Selection policy: epsilon-greedy (default), ucb (deterministic UCB1), " + "or thompson (seeded Thompson sampling).") parser.add_argument("--adaptive", action="store_true", help="Use adaptive explore_rate per-tag (epsilon-greedy only).") parser.add_argument("--reconcile", action="store_true", help="Rebuild routing_stats.yml from settled contracts.") parser.add_argument("--seed", type=int, default=None, help="Random seed for reproducible exploration.") @@ -585,10 +665,16 @@ def main(): status = load_citizen_status(polis_root, cid) scores.append(score_citizen(cid, card, status, contract_fm, stats, weights, lessons)) + thompson_sampled = None if args.policy == "ucb": required_tags = contract_fm.get("required_tags") or [] counts = {cid: citizen_sample_count(cid, required_tags, stats) for cid in citizens} chosen, is_exploration, chosen_score = pick_ucb(scores, counts) + elif args.policy == "thompson": + required_tags = contract_fm.get("required_tags") or [] + counts = {cid: citizen_sample_count(cid, required_tags, stats) for cid in citizens} + thompson_sampled = thompson_sample_values(scores, counts, random.Random(args.seed)) + chosen, is_exploration, chosen_score = pick_from_thompson_samples(scores, thompson_sampled) else: rng = random.Random(args.seed) chosen, is_exploration, chosen_score = pick_recommendation(scores, explore_rate, rng) @@ -605,6 +691,11 @@ def main(): print(f"\nPolicy: {args.policy}") if args.policy == "ucb": print(f"UCB exploration constant c: {UCB_EXPLORATION_C:.3f}") + elif args.policy == "thompson": + print(f"Thompson exploration scale c: {THOMPSON_EXPLORATION_C:.3f} (seed: {args.seed})") + print("Sampled posterior values (sorted):") + for cid in sorted(thompson_sampled, key=lambda k: thompson_sampled[k], reverse=True): + print(f" {cid:30s} sample={thompson_sampled[cid]:+.3f}") else: print(f"Explore rate: {explore_rate:.2f}") diff --git a/references/routing.md b/references/routing.md index d95f0d5..7346d10 100644 --- a/references/routing.md +++ b/references/routing.md @@ -131,6 +131,28 @@ An **unplayed citizen** (`n = 0`) has an infinite bonus, so UCB tries every untr **Tradeoff.** UCB is fully deterministic: the same state always yields the same pick (ties, including ties between unplayed citizens, break by citizen id ascending), which makes it reproducible in tests and audits. Epsilon-greedy is stochastic, so it can spread work more evenly across near-equal citizens over many contracts but needs a seed to reproduce. UCB explores more aggressively early (every new citizen gets a turn) and then commits harder to leaders; epsilon-greedy keeps a constant background exploration rate. `--adaptive` applies only to epsilon-greedy. +### Thompson sampling: a posterior-based alternative + +Thompson sampling is the third bandit policy, available via `scripts/route_contract.py --policy thompson`. Instead of a confidence bonus (UCB) or an ε-coin-flip (epsilon-greedy), it explores by **sampling from each citizen's posterior** and picking whoever draws highest. + +Each citizen's combined `score` is treated as the mean of a Gaussian posterior whose standard deviation shrinks as their sample count `n` grows: + +``` +sample(citizen) ~ Normal(mean = score(citizen), sigma = c / sqrt(n)) +pick = argmax over citizens of sample(citizen) +``` + +- `score(citizen)` is the same combined score used everywhere else — the posterior mean. +- `n` is the citizen's total `contracts_completed` across the contract's `required_tags` (the same count UCB uses). +- `c` is the exploration scale (`sqrt(2)`, mirroring the UCB constant). Fixed rather than CLI-tunable to keep the flag surface small. +- An **unplayed citizen** (`n = 0`) gets a wide fixed prior (`sigma = 2.0`, wider than the `n = 1` width of `sqrt(2)`), so cold-start samples every untried citizen broadly and each has a real shot at the top draw. The posterior width is monotonically decreasing in `n`. + +**Cold-start behavior.** With everyone unplayed, the wide prior means picks spread across *all* citizens over successive contracts (weighted toward higher means but never locked to the top one) — similar in spirit to UCB trying every arm, but smoothed: a slightly-lower-mean newcomer still wins a healthy share of early draws instead of strictly waiting its turn. As evidence accumulates the posteriors tighten and the policy commits to the leader. + +**Determinism.** Thompson sampling is stochastic, so like epsilon-greedy it needs a seed to reproduce: pass `--seed N` (or `seed=` through `recommend(...)`). Citizens are sampled in citizen-id order, so a given `(state, seed)` always yields the same pick and the same `--explain` sampled values — reproducible in tests and audits despite the randomness. Without `--seed`, the RNG is entropy-seeded (same as epsilon-greedy). + +**Tradeoff vs epsilon-greedy and UCB.** Epsilon-greedy explores a *fixed fraction* of the time regardless of how certain the ranking is. UCB is deterministic and explores via an additive bonus that decays like `sqrt(ln(total_n)/n)`. Thompson explores in proportion to *posterior overlap*: when two citizens are genuinely close it splits work between them smoothly, and when one is a clear, well-sampled leader the overlap vanishes and it exploits. It tends to spread early work across near-equal citizens more evenly than UCB's turn-taking, at the cost of being non-deterministic without a seed. `--adaptive` applies only to epsilon-greedy. + ## Updating the policy when a contract settles When a contract moves to `_polis/contracts/settled/`, the router updates `routing_stats.yml`: diff --git a/scripts/test_lessons_routing.py b/scripts/test_lessons_routing.py index 9f11ea4..08c3696 100644 --- a/scripts/test_lessons_routing.py +++ b/scripts/test_lessons_routing.py @@ -7,23 +7,30 @@ a structured routing_effect changes a later recommendation and is named in the explanation, the effect is bounded, and un-accepted lessons are ignored; and * the deterministic UCB1 variant — cold-start (unplayed arms first) and - learned-history (mean + confidence bonus) selection, with no randomness. + learned-history (mean + confidence bonus) selection, with no randomness; and +* the seeded Thompson-sampling variant — seeded determinism, cold-start spread + across unplayed arms, exploit-on-ample-evidence, and empty-input safety. """ +import random import sys import tempfile import unittest +from collections import Counter from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from polis.routing import ( # noqa: E402 LESSON_CAP, + THOMPSON_PRIOR_SIGMA, WEIGHTS_DEFAULT, citizen_sample_count, load_lessons, lessons_score, + pick_thompson, pick_ucb, score_citizen, + thompson_posterior_sigma, ) CONTRACT = {"required_tags": ["spanish-translation"], "cost_ceiling": "medium"} @@ -180,5 +187,78 @@ def test_empty_scores_is_safe(self): self.assertEqual(pick_ucb([], {}), (None, False, 0.0)) +class ThompsonRoutingTest(unittest.TestCase): + """The optional Thompson policy is stochastic but seed-reproducible.""" + + def test_posterior_sigma_shrinks_with_sample_count(self): + # Unplayed arms get the wide prior; width then shrinks monotonically as + # evidence accumulates, so more-sampled arms have tighter posteriors. + self.assertEqual(thompson_posterior_sigma(0), THOMPSON_PRIOR_SIGMA) + self.assertLess(thompson_posterior_sigma(1), THOMPSON_PRIOR_SIGMA) + self.assertLess(thompson_posterior_sigma(100), thompson_posterior_sigma(4)) + self.assertLess(thompson_posterior_sigma(4), thompson_posterior_sigma(1)) + + def test_same_seed_gives_same_pick_across_calls(self): + # Reproducibility: a fixed seed pins the pick, call after call. + scores = [{"citizen": "a", "total": 0.60}, {"citizen": "b", "total": 0.55}] + counts = {"a": 3, "b": 3} + first = pick_thompson(scores, counts, random.Random(7)) + for _ in range(20): + self.assertEqual(pick_thompson(scores, counts, random.Random(7)), first) + + def test_pick_is_independent_of_score_order(self): + # Sampling in citizen-id order makes the pick a function of the seed + # alone, not of dict/list insertion order. + counts = {"a": 3, "b": 3} + ab = [{"citizen": "a", "total": 0.60}, {"citizen": "b", "total": 0.55}] + ba = [{"citizen": "b", "total": 0.55}, {"citizen": "a", "total": 0.60}] + self.assertEqual( + pick_thompson(ab, counts, random.Random(7)), + pick_thompson(ba, counts, random.Random(7)), + ) + + def test_cold_start_spreads_across_unplayed_arms(self): + # Three unplayed citizens (n == 0) with differing means: the wide prior + # spreads picks across ALL of them over many seeds — even the lowest + # mean arm wins sometimes — so cold-start reaches everyone. + scores = [{"citizen": "a", "total": 0.9}, + {"citizen": "b", "total": 0.5}, + {"citizen": "c", "total": 0.1}] + counts = {"a": 0, "b": 0, "c": 0} + picks = Counter(pick_thompson(scores, counts, random.Random(s))[0] + for s in range(200)) + self.assertEqual(set(picks), {"a", "b", "c"}) # every arm gets a turn + self.assertGreater(picks["c"], 0) # even the weakest cold arm is tried + + def test_exploits_the_leader_on_ample_evidence(self): + # Strong positive evidence (high mean, ample n -> tight posterior) beats + # weak evidence, and the pick matches the pure-exploit leader (not an + # exploration pick). Robust: 'a' wins for every seed in a wide sweep. + scores = [{"citizen": "a", "total": 0.90}, {"citizen": "b", "total": 0.30}] + counts = {"a": 100, "b": 100} + for s in range(50): + self.assertEqual(pick_thompson(scores, counts, random.Random(s))[0], "a") + chosen, is_exploration, score = pick_thompson(scores, counts, random.Random(7)) + self.assertEqual(chosen, "a") + self.assertFalse(is_exploration) # 'a' is also the highest-mean pick + self.assertEqual(score, 0.90) + + def test_strong_evidence_pick_depends_on_the_mean_accounting(self): + # Decision-boundary / mutation-biting scenario. At seed 7 the correct + # posterior (centered on each citizen's combined score) picks 'a', the + # strong-evidence arm. This is the assertion the mutation transcript + # flips: dropping the mean from the sample (rng.gauss(0.0, sigma)) centers + # both arms at 0 and at seed 7 flips the pick to 'b'. So a broken + # success/failure accounting — sampling that ignores the evidence — makes + # this fail, while a correct one keeps 'a'. + scores = [{"citizen": "a", "total": 0.90}, {"citizen": "b", "total": 0.30}] + counts = {"a": 100, "b": 100} + chosen, _, _ = pick_thompson(scores, counts, random.Random(7)) + self.assertEqual(chosen, "a") + + def test_empty_scores_is_safe(self): + self.assertEqual(pick_thompson([], {}, random.Random(7)), (None, False, 0.0)) + + if __name__ == "__main__": unittest.main()