autoevolve/ratings.py at main · MrTsepa/autoevolve · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
"""Bradley-Terry maximum-likelihood ratings, per-version stats, and Pareto front."""

import math
from collections import defaultdict


def compute_ratings(db: dict, initial: float = 1500) -> tuple[dict, dict]:
    """Compute Elo-scale ratings via Bradley-Terry maximum likelihood.

    Order-independent: finds globally optimal ratings that best explain all
    match results simultaneously. 400 points = 10:1 win odds.

    Returns (ratings, match_counts).
    """
    h2h = defaultdict(lambda: defaultdict(int))
    match_counts = defaultdict(int)
    versions = set()

    for m in db["matches"]:
        a, b = m["a"], m["b"]
        wa, wb = m["wins_a"], m["wins_b"]
        h2h[a][b] += wa
        h2h[b][a] += wb
        total = wa + wb
        match_counts[a] += total
        match_counts[b] += total
        versions.add(a)
        versions.add(b)

    if not versions:
        return {}, {}

    versions = sorted(versions)
    r = {v: 1.0 for v in versions}

    total_wins = {}
    pair_games = defaultdict(lambda: defaultdict(int))
    for v in versions:
        total_wins[v] = sum(h2h[v][u] for u in versions if u != v)
    for m in db["matches"]:
        a, b = m["a"], m["b"]
        n = m["wins_a"] + m["wins_b"]
        pair_games[a][b] += n
        pair_games[b][a] += n

    # Iterative MLE
    for _ in range(200):
        max_change = 0
        for v in versions:
            if total_wins[v] == 0:
                continue
            denom = sum(
                pair_games[v][u] / (r[v] + r[u])
                for u in versions
                if u != v and pair_games[v][u] > 0
            )
            if denom < 1e-12:
                continue
            new_r = total_wins[v] / denom
            max_change = max(max_change, abs(new_r - r[v]) / max(r[v], 1e-12))
            r[v] = new_r
        if max_change < 1e-8:
            break

    # Normalize by geometric mean
    geo_mean = math.exp(sum(math.log(r[v]) for v in versions) / len(versions))
    for v in versions:
        r[v] /= geo_mean

    ratings = {v: 400 * math.log10(max(r[v], 1e-12)) + initial for v in versions}
    return ratings, dict(match_counts)


def compute_stats(db: dict) -> dict:
    """Per-version aggregates: win rate, score margin, total games, unique opponents."""
    stats = defaultdict(lambda: {"wins": 0, "losses": 0, "scores": [], "opp_scores": [], "opponents": set()})

    for m in db["matches"]:
        a, b = m["a"], m["b"]
        stats[a]["wins"] += m["wins_a"]
        stats[a]["losses"] += m["wins_b"]
        stats[b]["wins"] += m["wins_b"]
        stats[b]["losses"] += m["wins_a"]
        stats[a]["opponents"].add(b)
        stats[b]["opponents"].add(a)

        if "mean_a" in m and "mean_b" in m:
            n = m["wins_a"] + m["wins_b"]
            stats[a]["scores"].extend([m["mean_a"]] * n)
            stats[a]["opp_scores"].extend([m["mean_b"]] * n)
            stats[b]["scores"].extend([m["mean_b"]] * n)
            stats[b]["opp_scores"].extend([m["mean_a"]] * n)

    result = {}
    for v, s in stats.items():
        total = s["wins"] + s["losses"]
        wr = s["wins"] / total * 100 if total else 0
        margin = 0
        if s["scores"] and s["opp_scores"]:
            margin = (
                sum(s["scores"]) / len(s["scores"])
                - sum(s["opp_scores"]) / len(s["opp_scores"])
            )
        result[v] = {"win_rate": wr, "games": total, "margin": margin, "opponents": len(s["opponents"])}
    return result


def pareto_front(versions: list, dimensions: dict) -> set:
    """Find Pareto-optimal versions across multiple dimensions.

    dimensions: {version: [dim1_val, dim2_val, ...]}
    Returns set of non-dominated version names.
    """
    front = set()
    vlist = list(dimensions.keys())

    for i, v in enumerate(vlist):
        dominated = False
        for j, u in enumerate(vlist):
            if i == j:
                continue
            vals_v, vals_u = dimensions[v], dimensions[u]
            if all(vals_u[d] >= vals_v[d] for d in range(len(vals_v))) and any(
                vals_u[d] > vals_v[d] for d in range(len(vals_v))
            ):
                dominated = True
                break
        if not dominated:
            front.add(v)
    return front