Skip to content

Commit 4135166

Browse files
Pigbibicodex
andcommitted
fix: gate recommendation review by horizon maturity
Co-Authored-By: Codex <noreply@openai.com>
1 parent 0fdf582 commit 4135166

5 files changed

Lines changed: 166 additions & 27 deletions

File tree

docs/advisory_contract.md

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,7 @@ Recommendation follow-up reviews are separate audit artifacts generated from
214214
past final recommendations and cached point-in-time prices:
215215

216216
```text
217-
schema_version = 1
217+
schema_version = 2
218218
mode = recommendation_review
219219
as_of
220220
generated_at
@@ -228,12 +228,15 @@ policy
228228

229229
Each `review_items[]` row carries the original report date, review date, symbol,
230230
primary horizon, price interval, absolute return, benchmark return,
231-
benchmark-relative return, outcome label, market data source, and original
232-
selection scores when present.
231+
benchmark-relative return, trading-observation count, maturity status, outcome
232+
label, market data source, and original selection scores when present. The
233+
minimum maturity is 10 trading days for short/medium and 252 trading days for
234+
long. Summary performance metrics are reported separately for each horizon.
233235

234236
Allowed outcome labels:
235237

236238
- `pending`
239+
- `in_progress`
237240
- `insufficient_price_data`
238241
- `outperforming`
239242
- `inline`

docs/system_design.md

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -116,8 +116,13 @@ price source without turning the repository into a paid market-data store.
116116

117117
Recommendation follow-up review is a separate artifact. It reads past final
118118
recommendations, cached prices, and a benchmark, then reports absolute and
119-
relative returns by horizon. It is used for research accountability and data
120-
quality checks; it does not create new recommendations or execution targets.
119+
relative returns by horizon. Maturity is measured in trading observations:
120+
short and medium horizons require at least 10 trading days, while long requires
121+
252. Before maturity, an item remains `pending` or `in_progress` and cannot be
122+
labeled `outperforming` or `lagging`. Summary metrics (sample size, mean,
123+
median, and hit rate) stay within each horizon, and top symbols are
124+
de-duplicated. It is used for research accountability and data quality checks;
125+
it does not create new recommendations or execution targets.
121126

122127
A separate no-network smoke command validates the three-repository contract:
123128

docs/system_design.zh-CN.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ scripts/build_advisory_artifacts.py
105105

106106
市场确认现在在 Yahoo chart 免费入口外面加了一层轻量价格缓存。线上 workflow 会用 GitHub Actions cache 恢复和保存 `.cache/market-data`。这样 Yahoo 临时不可用时,公开报告仍可以尽量用近期缓存继续生成;推荐跟踪复盘也可以使用 point-in-time 价格来源,而不把本仓库变成付费行情存储仓库。
107107

108-
推荐跟踪复盘是单独 artifact。它读取历史最终推荐、缓存价格和基准指数,按周期计算绝对收益、相对收益和结果状态。它只用于研究问责和数据质量检查,不生成新的推荐,也不输出执行目标。
108+
推荐跟踪复盘是单独 artifact。它读取历史最终推荐、缓存价格和基准指数,按周期计算绝对收益、相对收益和结果状态。复盘按交易日判断成熟度:短线和中线至少 10 个交易日、长线至少 252 个交易日;未达到门槛只能是 `pending``in_progress`,不得提前标记为 `outperforming`/`lagging`。汇总只在各周期内部计算样本量、平均值、中位数和命中率,领先标的去重。它只用于研究问责和数据质量检查,不生成新的推荐,也不输出执行目标。
109109

110110
跨仓库契约用 no-network smoke 命令验证:
111111

src/quant_advisor_research/recommendation_review.py

Lines changed: 83 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import argparse
44
import datetime as dt
55
import json
6+
import statistics
67
from pathlib import Path
78
from typing import Any
89

@@ -20,6 +21,7 @@
2021
"medium": "中线",
2122
"long": "长线",
2223
}
24+
MIN_MATURITY_TRADING_DAYS = {"short": 10, "medium": 10, "long": 252}
2325

2426

2527
def utc_now_iso() -> str:
@@ -61,13 +63,22 @@ def return_between(start: PriceBar, end: PriceBar) -> float:
6163
return round(end.close / start.close - 1, 6)
6264

6365

64-
def outcome_label(relative_return: float | None, *, elapsed_days: int, has_price_data: bool) -> str:
66+
def outcome_label(
67+
relative_return: float | None,
68+
*,
69+
elapsed_days: int,
70+
has_price_data: bool,
71+
horizon: str,
72+
trading_observations: int,
73+
) -> str:
6574
if elapsed_days <= 0:
6675
return "pending"
6776
if not has_price_data:
6877
return "insufficient_price_data"
6978
if relative_return is None:
7079
return "insufficient_price_data"
80+
if trading_observations < MIN_MATURITY_TRADING_DAYS.get(horizon, MIN_MATURITY_TRADING_DAYS["medium"]):
81+
return "in_progress"
7182
if relative_return >= 0.02:
7283
return "outperforming"
7384
if relative_return <= -0.02:
@@ -131,21 +142,39 @@ def build_review_item(
131142
relative_return = round(absolute_return - benchmark_return, 6)
132143

133144
elapsed_days = (review_as_of - report_as_of).days
145+
horizon = str(pick.get("primary_horizon", ""))
146+
maturity_days = MIN_MATURITY_TRADING_DAYS.get(horizon, MIN_MATURITY_TRADING_DAYS["medium"])
147+
if elapsed_days <= 0:
148+
maturity_status = "pending"
149+
elif not has_price_data:
150+
maturity_status = "insufficient_price_data"
151+
elif trading_observations < maturity_days:
152+
maturity_status = "in_progress"
153+
else:
154+
maturity_status = "matured"
134155
return {
135156
"report_as_of": report_as_of.isoformat(),
136157
"review_as_of": review_as_of.isoformat(),
137158
"symbol": symbol,
138159
"name": str(pick.get("name", "")),
139-
"primary_horizon": str(pick.get("primary_horizon", "")),
160+
"primary_horizon": horizon,
140161
"primary_horizon_label": str(pick.get("primary_horizon_label", "")),
141162
"start_price_date": start_date,
142163
"end_price_date": end_date,
143164
"elapsed_calendar_days": elapsed_days,
144165
"trading_observations": trading_observations,
166+
"maturity_required_trading_days": maturity_days,
167+
"maturity_status": maturity_status,
145168
"absolute_return": absolute_return,
146169
"benchmark_return": benchmark_return,
147170
"relative_return": relative_return,
148-
"outcome": outcome_label(relative_return, elapsed_days=elapsed_days, has_price_data=has_price_data),
171+
"outcome": outcome_label(
172+
relative_return,
173+
elapsed_days=elapsed_days,
174+
has_price_data=has_price_data,
175+
horizon=horizon,
176+
trading_observations=trading_observations,
177+
),
149178
"market_data_source": data_source if has_price_data else "",
150179
"combined_score": pick.get("combined_score"),
151180
"source_score": pick.get("source_score"),
@@ -157,24 +186,49 @@ def average(values: list[float]) -> float | None:
157186
return round(sum(values) / len(values), 6) if values else None
158187

159188

189+
def median(values: list[float]) -> float | None:
190+
return round(float(statistics.median(values)), 6) if values else None
191+
192+
160193
def summarize_items(items: list[dict[str, Any]]) -> dict[str, Any]:
161-
evaluated = [item for item in items if isinstance(item.get("relative_return"), (int, float))]
194+
evaluated = [
195+
item
196+
for item in items
197+
if item.get("maturity_status") == "matured" and isinstance(item.get("relative_return"), (int, float))
198+
]
162199
by_horizon: dict[str, dict[str, Any]] = {}
163200
for horizon in ("short", "medium", "long"):
164201
horizon_items = [item for item in items if item.get("primary_horizon") == horizon]
165-
horizon_evaluated = [item for item in horizon_items if isinstance(item.get("relative_return"), (int, float))]
202+
horizon_evaluated = [item for item in evaluated if item.get("primary_horizon") == horizon]
203+
horizon_returns = [float(item["relative_return"]) for item in horizon_evaluated]
204+
horizon_ranked = sorted(
205+
[item for item in horizon_evaluated if item.get("outcome") == "outperforming"],
206+
key=lambda item: (float(item.get("relative_return", 0)), str(item.get("symbol", ""))),
207+
reverse=True,
208+
)
166209
by_horizon[horizon] = {
167210
"label": HORIZON_LABELS_ZH[horizon],
168211
"item_count": len(horizon_items),
169212
"evaluated_count": len(horizon_evaluated),
213+
"sample_size": len(horizon_evaluated),
170214
"pending_count": sum(1 for item in horizon_items if item.get("outcome") == "pending"),
215+
"in_progress_count": sum(1 for item in horizon_items if item.get("outcome") == "in_progress"),
216+
"matured_count": sum(1 for item in horizon_items if item.get("maturity_status") == "matured"),
171217
"insufficient_price_data_count": sum(
172218
1 for item in horizon_items if item.get("outcome") == "insufficient_price_data"
173219
),
174-
"average_relative_return": average([float(item["relative_return"]) for item in horizon_evaluated]),
220+
"average_relative_return": average(horizon_returns),
221+
"median_relative_return": median(horizon_returns),
222+
"hit_rate": round(
223+
sum(1 for item in horizon_evaluated if item.get("outcome") == "outperforming") / len(horizon_evaluated),
224+
6,
225+
)
226+
if horizon_evaluated
227+
else None,
228+
"top_outperformers": unique_symbols(horizon_ranked),
175229
}
176230
ranked = sorted(
177-
evaluated,
231+
[item for item in evaluated if item.get("outcome") == "outperforming"],
178232
key=lambda item: (float(item.get("relative_return", 0)), str(item.get("symbol", ""))),
179233
reverse=True,
180234
)
@@ -183,12 +237,24 @@ def summarize_items(items: list[dict[str, Any]]) -> dict[str, Any]:
183237
"evaluated_count": len(evaluated),
184238
"pending_count": sum(1 for item in items if item.get("outcome") == "pending"),
185239
"insufficient_price_data_count": sum(1 for item in items if item.get("outcome") == "insufficient_price_data"),
186-
"average_relative_return": average([float(item["relative_return"]) for item in evaluated]),
240+
# Do not pool short-, medium-, and long-horizon performance into one statistic.
241+
"average_relative_return": None,
242+
"median_relative_return": None,
243+
"hit_rate": None,
187244
"by_horizon": by_horizon,
188-
"top_outperformers": [item["symbol"] for item in ranked[:5]],
245+
"top_outperformers": unique_symbols(ranked)[:5],
189246
}
190247

191248

249+
def unique_symbols(items: list[dict[str, Any]]) -> list[str]:
250+
symbols: list[str] = []
251+
for item in items:
252+
symbol = str(item.get("symbol", ""))
253+
if symbol and symbol not in symbols:
254+
symbols.append(symbol)
255+
return symbols[:5]
256+
257+
192258
def build_recommendation_review(
193259
*,
194260
report_paths: list[str | Path],
@@ -245,7 +311,7 @@ def build_recommendation_review(
245311
)
246312

247313
return {
248-
"schema_version": "1",
314+
"schema_version": "2",
249315
"mode": "recommendation_review",
250316
"as_of": as_of.isoformat(),
251317
"generated_at": utc_now_iso(),
@@ -274,9 +340,9 @@ def render_recommendation_review_markdown(review: dict[str, Any]) -> str:
274340
f"- 复盘条目:{summary.get('item_count', 0)}",
275341
f"- 已可评估:{summary.get('evaluated_count', 0)}",
276342
f"- 待观察:{summary.get('pending_count', 0)}",
343+
f"- 进行中:{sum(item.get('in_progress_count', 0) for item in summary.get('by_horizon', {}).values())}",
277344
f"- 缺少价格数据:{summary.get('insufficient_price_data_count', 0)}",
278-
f"- 平均相对收益:{display_percent(summary.get('average_relative_return'))}",
279-
f"- 领先标的:{', '.join(summary.get('top_outperformers', [])) or '暂无'}",
345+
"- 不同持有期不合并计算平均收益;以下按周期分别统计。",
280346
"",
281347
"## 周期分布",
282348
"",
@@ -286,7 +352,9 @@ def render_recommendation_review_markdown(review: dict[str, Any]) -> str:
286352
item = summary.get("by_horizon", {}).get(horizon, {})
287353
lines.append(
288354
f"- {item.get('label', horizon)}{item.get('evaluated_count', 0)}/{item.get('item_count', 0)} 已评估,"
289-
f"平均相对收益 {display_percent(item.get('average_relative_return'))}"
355+
f"样本量 {item.get('sample_size', 0)},平均 {display_percent(item.get('average_relative_return'))},"
356+
f"中位数 {display_percent(item.get('median_relative_return'))},命中率 {display_percent(item.get('hit_rate'))},"
357+
f"领先标的 {', '.join(item.get('top_outperformers', [])) or '暂无'}"
290358
)
291359
warnings = review.get("data_quality_warnings", [])
292360
if warnings:
@@ -302,6 +370,8 @@ def render_recommendation_review_markdown(review: dict[str, Any]) -> str:
302370
f"- 价格区间:{item.get('start_price_date') or '无'}{item.get('end_price_date') or '无'}",
303371
f"- 绝对收益:{display_percent(item.get('absolute_return'))}",
304372
f"- 相对 {review.get('benchmark')}{display_percent(item.get('relative_return'))}",
373+
f"- 成熟度:{item.get('maturity_status')}(需要 {item.get('maturity_required_trading_days', 0)} 个交易日,"
374+
f"当前 {item.get('trading_observations', 0)} 个)",
305375
f"- 状态:{item.get('outcome')}",
306376
"",
307377
]

tests/test_recommendation_review.py

Lines changed: 69 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ def make_bars(start: dt.date, prices: list[float]) -> list[PriceBar]:
1818
]
1919

2020

21-
def write_report(path: Path, *, as_of: str = "2026-01-05") -> None:
21+
def write_report(path: Path, *, as_of: str = "2026-01-05", horizon: str = "medium", symbol: str = "MU") -> None:
2222
path.write_text(
2323
json.dumps(
2424
{
@@ -27,9 +27,9 @@ def write_report(path: Path, *, as_of: str = "2026-01-05") -> None:
2727
"final_decisions": {
2828
"recommendations": [
2929
{
30-
"symbol": "MU",
30+
"symbol": symbol,
3131
"name": "Micron Technology",
32-
"primary_horizon": "medium",
32+
"primary_horizon": horizon,
3333
"primary_horizon_label": "中线",
3434
"combined_score": 0.84,
3535
"source_score": 0.2,
@@ -43,12 +43,12 @@ def write_report(path: Path, *, as_of: str = "2026-01-05") -> None:
4343
)
4444

4545

46-
def test_recommendation_review_calculates_forward_relative_return_from_cache(tmp_path: Path) -> None:
46+
def test_recommendation_review_keeps_short_review_in_progress_until_ten_trading_days(tmp_path: Path) -> None:
4747
cache_dir = tmp_path / "market-cache"
4848
write_cached_bars("MU", make_bars(dt.date(2026, 1, 5), [100, 102, 105, 108, 112, 116]), cache_dir=cache_dir)
4949
write_cached_bars("SPY", make_bars(dt.date(2026, 1, 5), [100, 101, 102, 103, 104, 105]), cache_dir=cache_dir)
5050
report_path = tmp_path / "advisory_report_2026-01-05.json"
51-
write_report(report_path)
51+
write_report(report_path, horizon="short")
5252

5353
review = build_recommendation_review(
5454
report_paths=[report_path],
@@ -64,12 +64,73 @@ def test_recommendation_review_calculates_forward_relative_return_from_cache(tmp
6464
assert item["absolute_return"] == 0.16
6565
assert item["benchmark_return"] == 0.05
6666
assert item["relative_return"] == 0.11
67-
assert item["outcome"] == "outperforming"
68-
assert review["summary"]["evaluated_count"] == 1
69-
assert review["summary"]["top_outperformers"] == ["MU"]
67+
assert item["maturity_status"] == "in_progress"
68+
assert item["outcome"] == "in_progress"
69+
assert review["summary"]["evaluated_count"] == 0
70+
assert review["summary"]["by_horizon"]["short"]["evaluated_count"] == 0
7071
assert "MU" in render_recommendation_review_markdown(review)
7172

7273

74+
def test_recommendation_review_reports_matured_metrics_by_horizon(tmp_path: Path) -> None:
75+
cache_dir = tmp_path / "market-cache"
76+
prices = [100 + index for index in range(12)]
77+
write_cached_bars("MU", make_bars(dt.date(2026, 1, 5), prices), cache_dir=cache_dir)
78+
write_cached_bars("SPY", make_bars(dt.date(2026, 1, 5), [100] * 12), cache_dir=cache_dir)
79+
report_path = tmp_path / "advisory_report_2026-01-05.json"
80+
write_report(report_path, horizon="short")
81+
82+
review = build_recommendation_review(
83+
report_paths=[report_path], as_of=dt.date(2026, 1, 20), benchmark="SPY",
84+
cache_dir=cache_dir, cache_max_age_days=30, use_network=False,
85+
)
86+
87+
item = review["review_items"][0]
88+
assert item["maturity_status"] == "matured"
89+
assert item["outcome"] == "outperforming"
90+
short_summary = review["summary"]["by_horizon"]["short"]
91+
assert short_summary["sample_size"] == 1
92+
assert short_summary["median_relative_return"] == short_summary["average_relative_return"]
93+
assert short_summary["hit_rate"] == 1.0
94+
95+
96+
def test_top_outperformers_are_unique_and_grouped_by_horizon(tmp_path: Path) -> None:
97+
cache_dir = tmp_path / "market-cache"
98+
write_cached_bars("MU", make_bars(dt.date(2026, 1, 5), [100] * 15), cache_dir=cache_dir)
99+
write_cached_bars("AMD", make_bars(dt.date(2026, 1, 5), [100 + index for index in range(15)]), cache_dir=cache_dir)
100+
write_cached_bars("SPY", make_bars(dt.date(2026, 1, 5), [100] * 15), cache_dir=cache_dir)
101+
reports = []
102+
for index, (symbol, horizon) in enumerate((("AMD", "short"), ("AMD", "short"), ("MU", "medium"))):
103+
path = tmp_path / f"report-{index}.json"
104+
write_report(path, horizon=horizon, symbol=symbol)
105+
reports.append(path)
106+
107+
review = build_recommendation_review(
108+
report_paths=reports, as_of=dt.date(2026, 1, 25), benchmark="SPY",
109+
cache_dir=cache_dir, cache_max_age_days=14, use_network=False,
110+
)
111+
112+
assert review["summary"]["top_outperformers"] == ["AMD"]
113+
assert review["summary"]["by_horizon"]["short"]["top_outperformers"] == ["AMD"]
114+
115+
116+
def test_long_horizon_cannot_be_labeled_lagging_after_a_few_weeks(tmp_path: Path) -> None:
117+
cache_dir = tmp_path / "market-cache"
118+
write_cached_bars("TSM", make_bars(dt.date(2026, 1, 5), [100, 90, 80, 70, 60]), cache_dir=cache_dir)
119+
write_cached_bars("SPY", make_bars(dt.date(2026, 1, 5), [100] * 5), cache_dir=cache_dir)
120+
report_path = tmp_path / "advisory_report_2026-01-05.json"
121+
write_report(report_path, horizon="long", symbol="TSM")
122+
123+
review = build_recommendation_review(
124+
report_paths=[report_path], as_of=dt.date(2026, 1, 30), benchmark="SPY",
125+
cache_dir=cache_dir, cache_max_age_days=30, use_network=False,
126+
)
127+
128+
item = review["review_items"][0]
129+
assert item["elapsed_calendar_days"] > 14
130+
assert item["maturity_status"] == "in_progress"
131+
assert item["outcome"] == "in_progress"
132+
133+
73134
def test_recommendation_review_marks_same_day_report_as_pending(tmp_path: Path) -> None:
74135
report_path = tmp_path / "advisory_report_2026-01-05.json"
75136
write_report(report_path)

0 commit comments

Comments
 (0)