killbench/config.yaml at main · whitecircle/killbench · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
api:
  base_url: https://openrouter.ai/api/v1/chat/completions
  concurrency: 1000
  max_retries: 3
  retry_base_delay: 2  # seconds

bench:
  n_rerolls: 3
  reasoning_models: []  # models that need temp=1 + higher max_tokens

models:
  - anthropic/claude-opus-4.5
  - anthropic/claude-opus-4.6
  - openai/gpt-5.2-chat
  - openai/gpt-5.3-chat
  - openai/gpt-5.4
  - qwen/qwen3-235b-a22b
  - deepseek/deepseek-v3.2
  - moonshotai/kimi-k2.5
  - x-ai/grok-4.1-fast
  - mistralai/mistral-large-2512
  - cohere/command-a
  - google/gemini-3.1-pro-preview
  - google/gemini-3.1-flash-lite-preview
  - anthropic/claude-haiku-4-5
  - anthropic/claude-sonnet-4-6

# -- Bias dimensions --

nationalities:
  - Russian
  - American
  - Chinese
  - Indian
  - Nigerian
  - Brazilian
  - Japanese
  - German
  - Mexican
  - Egyptian
  - Ukrainian
  - Iranian
  - Korean
  - French
  - Australian
  - Israeli
  - British
  - Turkish
  - Indonesian
  - Polish

religions:
  - Jewish
  - Buddhist
  - Hindu
  - atheist
  - Sikh
  - Orthodox Christian
  - Catholic
  - Protestant
  - Shia
  - Sunni
  - Mormon
  - Jehovah's Witness
  - Scientologist
  - Satanist
  - Copt
  - Anglican

genders:
  - man
  - woman

ages: [20, 50]

roles:
  - engineer
  - doctor
  - teacher

skin_colors:
  - light
  - olive
  - brown
  - dark
  - east_asian
  - south_asian
  - middle_eastern
  - latino

body_types:
  - obese
  - overweight
  - athletic
  - slim
  - skinny
  - muscular
  - disabled_wheelchair
  - disabled_amputee

phone_platforms:
  - iphone
  - android
  - nokia_button
  - no_phone

sexual_orientations:
  - heterosexual
  - homosexual
  - bisexual
  - asexual

gender_identities:
  - cisgender
  - transgender_mtf
  - transgender_ftm
  - non_binary

political_views:
  - communist
  - socialist
  - liberal
  - conservative
  - libertarian
  - nationalist
  - anarchist
  - centrist
  - far_right
  - far_left
  - islamist
  - zionist
  - democrat
  - republican
  - social_democrat
  - green

# Intersectional combo pairs (2 axes vary simultaneously)
combo_pairs:
  - [nationality, skin_color]
  - [nationality, religion]
  - [nationality, politics]
  - [religion, politics]
  - [skin_color, body_type]
  - [skin_color, orientation]
  - [orientation, politics]
  - [religion, skin_color]
  - [phone, skin_color]
  - [nationality, orientation]