From 9b8003213ab2d3aec341bf1504b7530a5d33cd11 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Sat, 11 Jul 2026 07:45:03 +0800 Subject: [PATCH 1/5] feat: publish crypto lifecycle inputs Co-Authored-By: Codex --- .../workflows/publish-lifecycle-inputs.yml | 55 ++++++++++++++ scripts/export_lifecycle_preflight_inputs.py | 75 +++++++++++++++++++ .../test_export_lifecycle_preflight_inputs.py | 29 +++++++ .../test_publish_lifecycle_inputs_workflow.py | 13 ++++ 4 files changed, 172 insertions(+) create mode 100644 .github/workflows/publish-lifecycle-inputs.yml create mode 100644 scripts/export_lifecycle_preflight_inputs.py create mode 100644 tests/test_export_lifecycle_preflight_inputs.py create mode 100644 tests/test_publish_lifecycle_inputs_workflow.py diff --git a/.github/workflows/publish-lifecycle-inputs.yml b/.github/workflows/publish-lifecycle-inputs.yml new file mode 100644 index 0000000..7bb1926 --- /dev/null +++ b/.github/workflows/publish-lifecycle-inputs.yml @@ -0,0 +1,55 @@ +name: Publish Lifecycle Inputs + +on: + schedule: + - cron: "0 4 * * 1" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: false + +jobs: + publish: + if: github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + runs-on: [self-hosted, Linux, X64] + timeout-minutes: 60 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + set -euo pipefail + python -m pip install --upgrade pip + requirements_file=requirements-lock.txt + if [ ! -f "${requirements_file}" ]; then requirements_file=requirements.txt; fi + python -m pip install -r "${requirements_file}" + + - name: Download real Binance history + run: | + set -euo pipefail + python scripts/download_history.py --top-liquid 30 --force-exchange-info + + - name: Export production research inputs + run: | + set -euo pipefail + python scripts/export_lifecycle_preflight_inputs.py \ + --universe-mode broad_liquid \ + --output-dir "${RUNNER_TEMP}/crypto-lifecycle-inputs" + + - name: Upload lifecycle inputs + uses: actions/upload-artifact@v7 + with: + name: crypto-lifecycle-inputs-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/crypto-lifecycle-inputs + if-no-files-found: error + retention-days: 30 diff --git a/scripts/export_lifecycle_preflight_inputs.py b/scripts/export_lifecycle_preflight_inputs.py new file mode 100644 index 0000000..4e6b87d --- /dev/null +++ b/scripts/export_lifecycle_preflight_inputs.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +import pandas as pd + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from src.config import load_config +from src.pipeline import run_research_pipeline + +PANEL_COLUMNS = ("in_universe", "open", "final_score") +COMBO_SYMBOLS = ("BTCUSDT", "ETHUSDT") + + +def export_lifecycle_inputs(panel: pd.DataFrame, output_dir: Path) -> dict[str, object]: + if list(panel.index.names) != ["date", "symbol"]: + raise ValueError("research panel must use a date/symbol MultiIndex") + missing = sorted(set((*PANEL_COLUMNS, "close")) - set(panel.columns)) + if missing: + raise ValueError(f"research panel is missing columns: {', '.join(missing)}") + + frame = panel.reset_index().copy() + frame["date"] = pd.to_datetime(frame["date"], errors="coerce").dt.tz_localize(None).dt.normalize() + frame["symbol"] = frame["symbol"].astype(str).str.strip().str.upper() + lifecycle_panel = frame[["date", "symbol", *PANEL_COLUMNS]].dropna(subset=["date", "open", "final_score"]) + if lifecycle_panel.empty: + raise ValueError("research panel has no scored lifecycle rows") + + market_history = frame.loc[frame["symbol"].isin(COMBO_SYMBOLS), ["date", "symbol", "close"]].dropna() + missing_combo = sorted(set(COMBO_SYMBOLS) - set(market_history["symbol"])) + if missing_combo: + raise ValueError(f"research panel is missing combo symbols: {', '.join(missing_combo)}") + + output_dir.mkdir(parents=True, exist_ok=True) + panel_path = output_dir / "research_panel.csv.gz" + market_path = output_dir / "market_history.csv.gz" + manifest_path = output_dir / "manifest.json" + lifecycle_panel.to_csv(panel_path, index=False, compression="gzip") + market_history.to_csv(market_path, index=False, compression="gzip") + manifest = { + "contract_version": "crypto.lifecycle_preflight.v1", + "panel_rows": int(len(lifecycle_panel)), + "panel_symbols": sorted(lifecycle_panel["symbol"].unique().tolist()), + "market_rows": int(len(market_history)), + "market_symbols": sorted(market_history["symbol"].unique().tolist()), + "start_date": lifecycle_panel["date"].min().date().isoformat(), + "end_date": lifecycle_panel["date"].max().date().isoformat(), + } + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return manifest + + +def main() -> int: + parser = argparse.ArgumentParser(description="Export real production research inputs for lifecycle drift preflight.") + parser.add_argument("--config", default="config/default.yaml") + parser.add_argument("--universe-mode", default="broad_liquid") + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + + config = load_config(args.config) + result = run_research_pipeline(config, universe_mode=args.universe_mode) + manifest = export_lifecycle_inputs(result["panel"], args.output_dir) + print(manifest) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_export_lifecycle_preflight_inputs.py b/tests/test_export_lifecycle_preflight_inputs.py new file mode 100644 index 0000000..ed4798c --- /dev/null +++ b/tests/test_export_lifecycle_preflight_inputs.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pandas as pd + +from scripts.export_lifecycle_preflight_inputs import export_lifecycle_inputs + + +def test_export_lifecycle_inputs_writes_real_panel_contract(tmp_path: Path) -> None: + dates = pd.date_range("2024-01-01", periods=10, freq="D") + symbols = ("BTCUSDT", "ETHUSDT", "SOLUSDT") + index = pd.MultiIndex.from_product([dates, symbols], names=["date", "symbol"]) + panel = pd.DataFrame(index=index) + panel["in_universe"] = True + panel["open"] = range(1, len(panel) + 1) + panel["close"] = panel["open"] + 0.5 + panel["final_score"] = 0.75 + + manifest = export_lifecycle_inputs(panel, tmp_path) + + exported_panel = pd.read_csv(tmp_path / "research_panel.csv.gz") + market_history = pd.read_csv(tmp_path / "market_history.csv.gz") + persisted_manifest = json.loads((tmp_path / "manifest.json").read_text()) + assert set(exported_panel.columns) == {"date", "symbol", "in_universe", "open", "final_score"} + assert set(market_history["symbol"]) == {"BTCUSDT", "ETHUSDT"} + assert manifest == persisted_manifest + assert manifest["contract_version"] == "crypto.lifecycle_preflight.v1" diff --git a/tests/test_publish_lifecycle_inputs_workflow.py b/tests/test_publish_lifecycle_inputs_workflow.py new file mode 100644 index 0000000..4a98667 --- /dev/null +++ b/tests/test_publish_lifecycle_inputs_workflow.py @@ -0,0 +1,13 @@ +from pathlib import Path + + +def test_publish_lifecycle_inputs_workflow_uses_real_research_pipeline() -> None: + workflow = ( + Path(__file__).resolve().parents[1] / ".github" / "workflows" / "publish-lifecycle-inputs.yml" + ).read_text(encoding="utf-8") + + assert "scripts/download_history.py --top-liquid 30" in workflow + assert "scripts/export_lifecycle_preflight_inputs.py" in workflow + assert "--universe-mode broad_liquid" in workflow + assert "crypto-lifecycle-inputs-${{ github.run_id }}-${{ github.run_attempt }}" in workflow + assert "if-no-files-found: error" in workflow From 36dafc773a0e511e0e56a62649d5221ea433d480 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Sat, 11 Jul 2026 07:55:47 +0800 Subject: [PATCH 2/5] fix: validate crypto lifecycle input coverage Co-Authored-By: Codex --- .../workflows/publish-lifecycle-inputs.yml | 4 ++- scripts/export_lifecycle_preflight_inputs.py | 27 +++++++++++++++++-- .../test_export_lifecycle_preflight_inputs.py | 2 +- .../test_publish_lifecycle_inputs_workflow.py | 3 ++- 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish-lifecycle-inputs.yml b/.github/workflows/publish-lifecycle-inputs.yml index 7bb1926..eb5b276 100644 --- a/.github/workflows/publish-lifecycle-inputs.yml +++ b/.github/workflows/publish-lifecycle-inputs.yml @@ -17,6 +17,8 @@ jobs: if: github.ref == format('refs/heads/{0}', github.event.repository.default_branch) runs-on: [self-hosted, Linux, X64] timeout-minutes: 60 + env: + DOWNLOAD_TOP_LIQUID: "90" steps: - name: Checkout uses: actions/checkout@v6 @@ -37,7 +39,7 @@ jobs: - name: Download real Binance history run: | set -euo pipefail - python scripts/download_history.py --top-liquid 30 --force-exchange-info + python scripts/download_history.py --top-liquid "${DOWNLOAD_TOP_LIQUID}" --force-exchange-info - name: Export production research inputs run: | diff --git a/scripts/export_lifecycle_preflight_inputs.py b/scripts/export_lifecycle_preflight_inputs.py index 4e6b87d..0f21728 100644 --- a/scripts/export_lifecycle_preflight_inputs.py +++ b/scripts/export_lifecycle_preflight_inputs.py @@ -17,6 +17,9 @@ PANEL_COLUMNS = ("in_universe", "open", "final_score") COMBO_SYMBOLS = ("BTCUSDT", "ETHUSDT") +MIN_PANEL_DAYS = 730 +MIN_MARKET_DAYS = 900 +MAX_FRESHNESS_DAYS = 3 def export_lifecycle_inputs(panel: pd.DataFrame, output_dir: Path) -> dict[str, object]: @@ -32,11 +35,31 @@ def export_lifecycle_inputs(panel: pd.DataFrame, output_dir: Path) -> dict[str, lifecycle_panel = frame[["date", "symbol", *PANEL_COLUMNS]].dropna(subset=["date", "open", "final_score"]) if lifecycle_panel.empty: raise ValueError("research panel has no scored lifecycle rows") + panel_dates = pd.DatetimeIndex(sorted(lifecycle_panel["date"].unique())) + if len(panel_dates) < MIN_PANEL_DAYS: + raise ValueError(f"research panel requires at least {MIN_PANEL_DAYS} scored dates") + in_universe_counts = lifecycle_panel.loc[lifecycle_panel["in_universe"]].groupby("date")["symbol"].nunique() + if in_universe_counts.empty or int(in_universe_counts.min()) < 2: + raise ValueError("research panel requires at least two in-universe symbols per scored date") market_history = frame.loc[frame["symbol"].isin(COMBO_SYMBOLS), ["date", "symbol", "close"]].dropna() missing_combo = sorted(set(COMBO_SYMBOLS) - set(market_history["symbol"])) if missing_combo: raise ValueError(f"research panel is missing combo symbols: {', '.join(missing_combo)}") + reference_dates = set(market_history.loc[market_history["symbol"] == "BTCUSDT", "date"]) + if len(reference_dates) < MIN_MARKET_DAYS: + raise ValueError(f"BTC market history requires at least {MIN_MARKET_DAYS} dates") + for symbol in COMBO_SYMBOLS: + symbol_dates = set(market_history.loc[market_history["symbol"] == symbol, "date"]) + if ( + len(symbol_dates & reference_dates) / len(reference_dates) < 0.99 + or min(symbol_dates) > min(reference_dates) + or max(symbol_dates) < max(reference_dates) + ): + raise ValueError(f"market history has incomplete symbol coverage: {symbol}") + latest_date = max(max(reference_dates), panel_dates.max()) + if pd.Timestamp.now(tz="UTC").tz_localize(None).normalize() - latest_date > pd.Timedelta(days=MAX_FRESHNESS_DAYS): + raise ValueError("lifecycle inputs are stale") output_dir.mkdir(parents=True, exist_ok=True) panel_path = output_dir / "research_panel.csv.gz" @@ -50,8 +73,8 @@ def export_lifecycle_inputs(panel: pd.DataFrame, output_dir: Path) -> dict[str, "panel_symbols": sorted(lifecycle_panel["symbol"].unique().tolist()), "market_rows": int(len(market_history)), "market_symbols": sorted(market_history["symbol"].unique().tolist()), - "start_date": lifecycle_panel["date"].min().date().isoformat(), - "end_date": lifecycle_panel["date"].max().date().isoformat(), + "start_date": panel_dates.min().date().isoformat(), + "end_date": panel_dates.max().date().isoformat(), } manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") return manifest diff --git a/tests/test_export_lifecycle_preflight_inputs.py b/tests/test_export_lifecycle_preflight_inputs.py index ed4798c..9b645a2 100644 --- a/tests/test_export_lifecycle_preflight_inputs.py +++ b/tests/test_export_lifecycle_preflight_inputs.py @@ -9,7 +9,7 @@ def test_export_lifecycle_inputs_writes_real_panel_contract(tmp_path: Path) -> None: - dates = pd.date_range("2024-01-01", periods=10, freq="D") + dates = pd.date_range(end=pd.Timestamp.now().normalize(), periods=1000, freq="D") symbols = ("BTCUSDT", "ETHUSDT", "SOLUSDT") index = pd.MultiIndex.from_product([dates, symbols], names=["date", "symbol"]) panel = pd.DataFrame(index=index) diff --git a/tests/test_publish_lifecycle_inputs_workflow.py b/tests/test_publish_lifecycle_inputs_workflow.py index 4a98667..f75d531 100644 --- a/tests/test_publish_lifecycle_inputs_workflow.py +++ b/tests/test_publish_lifecycle_inputs_workflow.py @@ -6,7 +6,8 @@ def test_publish_lifecycle_inputs_workflow_uses_real_research_pipeline() -> None Path(__file__).resolve().parents[1] / ".github" / "workflows" / "publish-lifecycle-inputs.yml" ).read_text(encoding="utf-8") - assert "scripts/download_history.py --top-liquid 30" in workflow + assert 'DOWNLOAD_TOP_LIQUID: "90"' in workflow + assert 'scripts/download_history.py --top-liquid "${DOWNLOAD_TOP_LIQUID}"' in workflow assert "scripts/export_lifecycle_preflight_inputs.py" in workflow assert "--universe-mode broad_liquid" in workflow assert "crypto-lifecycle-inputs-${{ github.run_id }}-${{ github.run_attempt }}" in workflow From 76e6739f9eda7c5aa6ab179fdab4fbc8e155ecaf Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Sat, 11 Jul 2026 08:12:22 +0800 Subject: [PATCH 3/5] fix: validate crypto input dates independently Co-Authored-By: Codex --- scripts/export_lifecycle_preflight_inputs.py | 11 ++++--- .../test_export_lifecycle_preflight_inputs.py | 33 +++++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/scripts/export_lifecycle_preflight_inputs.py b/scripts/export_lifecycle_preflight_inputs.py index 0f21728..97e8a9b 100644 --- a/scripts/export_lifecycle_preflight_inputs.py +++ b/scripts/export_lifecycle_preflight_inputs.py @@ -39,7 +39,8 @@ def export_lifecycle_inputs(panel: pd.DataFrame, output_dir: Path) -> dict[str, if len(panel_dates) < MIN_PANEL_DAYS: raise ValueError(f"research panel requires at least {MIN_PANEL_DAYS} scored dates") in_universe_counts = lifecycle_panel.loc[lifecycle_panel["in_universe"]].groupby("date")["symbol"].nunique() - if in_universe_counts.empty or int(in_universe_counts.min()) < 2: + in_universe_counts = in_universe_counts.reindex(panel_dates, fill_value=0) + if int(in_universe_counts.min()) < 2: raise ValueError("research panel requires at least two in-universe symbols per scored date") market_history = frame.loc[frame["symbol"].isin(COMBO_SYMBOLS), ["date", "symbol", "close"]].dropna() @@ -57,9 +58,11 @@ def export_lifecycle_inputs(panel: pd.DataFrame, output_dir: Path) -> dict[str, or max(symbol_dates) < max(reference_dates) ): raise ValueError(f"market history has incomplete symbol coverage: {symbol}") - latest_date = max(max(reference_dates), panel_dates.max()) - if pd.Timestamp.now(tz="UTC").tz_localize(None).normalize() - latest_date > pd.Timedelta(days=MAX_FRESHNESS_DAYS): - raise ValueError("lifecycle inputs are stale") + today = pd.Timestamp.now(tz="UTC").tz_localize(None).normalize() + if today - panel_dates.max() > pd.Timedelta(days=MAX_FRESHNESS_DAYS): + raise ValueError("research panel is stale") + if today - max(reference_dates) > pd.Timedelta(days=MAX_FRESHNESS_DAYS): + raise ValueError("BTC/ETH market history is stale") output_dir.mkdir(parents=True, exist_ok=True) panel_path = output_dir / "research_panel.csv.gz" diff --git a/tests/test_export_lifecycle_preflight_inputs.py b/tests/test_export_lifecycle_preflight_inputs.py index 9b645a2..69418bb 100644 --- a/tests/test_export_lifecycle_preflight_inputs.py +++ b/tests/test_export_lifecycle_preflight_inputs.py @@ -4,6 +4,7 @@ from pathlib import Path import pandas as pd +import pytest from scripts.export_lifecycle_preflight_inputs import export_lifecycle_inputs @@ -27,3 +28,35 @@ def test_export_lifecycle_inputs_writes_real_panel_contract(tmp_path: Path) -> N assert set(market_history["symbol"]) == {"BTCUSDT", "ETHUSDT"} assert manifest == persisted_manifest assert manifest["contract_version"] == "crypto.lifecycle_preflight.v1" + + +def _valid_panel() -> pd.DataFrame: + dates = pd.date_range(end=pd.Timestamp.now().normalize(), periods=1000, freq="D") + symbols = ("BTCUSDT", "ETHUSDT", "SOLUSDT") + index = pd.MultiIndex.from_product([dates, symbols], names=["date", "symbol"]) + panel = pd.DataFrame(index=index) + panel["in_universe"] = True + panel["open"] = range(1, len(panel) + 1) + panel["close"] = panel["open"] + 0.5 + panel["final_score"] = 0.75 + return panel + + +def test_export_rejects_scored_date_without_universe(tmp_path: Path) -> None: + panel = _valid_panel() + latest_date = panel.index.get_level_values("date").max() + panel.loc[(latest_date, slice(None)), "in_universe"] = False + + with pytest.raises(ValueError, match="at least two in-universe symbols per scored date"): + export_lifecycle_inputs(panel, tmp_path) + + +def test_export_rejects_stale_combo_history_even_when_panel_is_fresh(tmp_path: Path) -> None: + panel = _valid_panel() + cutoff = panel.index.get_level_values("date").max() - pd.Timedelta(days=10) + combo_mask = panel.index.get_level_values("symbol").isin({"BTCUSDT", "ETHUSDT"}) + stale_mask = combo_mask & (panel.index.get_level_values("date") > cutoff) + panel.loc[stale_mask, "close"] = pd.NA + + with pytest.raises(ValueError, match="BTC/ETH market history is stale"): + export_lifecycle_inputs(panel, tmp_path) From 4fa153877707b0b92e531c428b4db753acf515be Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Sat, 11 Jul 2026 08:24:49 +0800 Subject: [PATCH 4/5] test: run lifecycle export coverage under unittest Co-Authored-By: Codex --- .../test_export_lifecycle_preflight_inputs.py | 108 +++++++++--------- 1 file changed, 56 insertions(+), 52 deletions(-) diff --git a/tests/test_export_lifecycle_preflight_inputs.py b/tests/test_export_lifecycle_preflight_inputs.py index 69418bb..f801186 100644 --- a/tests/test_export_lifecycle_preflight_inputs.py +++ b/tests/test_export_lifecycle_preflight_inputs.py @@ -1,62 +1,66 @@ from __future__ import annotations import json +import tempfile +import unittest from pathlib import Path import pandas as pd -import pytest from scripts.export_lifecycle_preflight_inputs import export_lifecycle_inputs -def test_export_lifecycle_inputs_writes_real_panel_contract(tmp_path: Path) -> None: - dates = pd.date_range(end=pd.Timestamp.now().normalize(), periods=1000, freq="D") - symbols = ("BTCUSDT", "ETHUSDT", "SOLUSDT") - index = pd.MultiIndex.from_product([dates, symbols], names=["date", "symbol"]) - panel = pd.DataFrame(index=index) - panel["in_universe"] = True - panel["open"] = range(1, len(panel) + 1) - panel["close"] = panel["open"] + 0.5 - panel["final_score"] = 0.75 - - manifest = export_lifecycle_inputs(panel, tmp_path) - - exported_panel = pd.read_csv(tmp_path / "research_panel.csv.gz") - market_history = pd.read_csv(tmp_path / "market_history.csv.gz") - persisted_manifest = json.loads((tmp_path / "manifest.json").read_text()) - assert set(exported_panel.columns) == {"date", "symbol", "in_universe", "open", "final_score"} - assert set(market_history["symbol"]) == {"BTCUSDT", "ETHUSDT"} - assert manifest == persisted_manifest - assert manifest["contract_version"] == "crypto.lifecycle_preflight.v1" - - -def _valid_panel() -> pd.DataFrame: - dates = pd.date_range(end=pd.Timestamp.now().normalize(), periods=1000, freq="D") - symbols = ("BTCUSDT", "ETHUSDT", "SOLUSDT") - index = pd.MultiIndex.from_product([dates, symbols], names=["date", "symbol"]) - panel = pd.DataFrame(index=index) - panel["in_universe"] = True - panel["open"] = range(1, len(panel) + 1) - panel["close"] = panel["open"] + 0.5 - panel["final_score"] = 0.75 - return panel - - -def test_export_rejects_scored_date_without_universe(tmp_path: Path) -> None: - panel = _valid_panel() - latest_date = panel.index.get_level_values("date").max() - panel.loc[(latest_date, slice(None)), "in_universe"] = False - - with pytest.raises(ValueError, match="at least two in-universe symbols per scored date"): - export_lifecycle_inputs(panel, tmp_path) - - -def test_export_rejects_stale_combo_history_even_when_panel_is_fresh(tmp_path: Path) -> None: - panel = _valid_panel() - cutoff = panel.index.get_level_values("date").max() - pd.Timedelta(days=10) - combo_mask = panel.index.get_level_values("symbol").isin({"BTCUSDT", "ETHUSDT"}) - stale_mask = combo_mask & (panel.index.get_level_values("date") > cutoff) - panel.loc[stale_mask, "close"] = pd.NA - - with pytest.raises(ValueError, match="BTC/ETH market history is stale"): - export_lifecycle_inputs(panel, tmp_path) +class ExportLifecyclePreflightInputsTests(unittest.TestCase): + @staticmethod + def _valid_panel() -> pd.DataFrame: + dates = pd.date_range(end=pd.Timestamp.now().normalize(), periods=1000, freq="D") + symbols = ("BTCUSDT", "ETHUSDT", "SOLUSDT") + index = pd.MultiIndex.from_product([dates, symbols], names=["date", "symbol"]) + panel = pd.DataFrame(index=index) + panel["in_universe"] = True + panel["open"] = range(1, len(panel) + 1) + panel["close"] = panel["open"] + 0.5 + panel["final_score"] = 0.75 + return panel + + def test_export_lifecycle_inputs_writes_real_panel_contract(self) -> None: + panel = self._valid_panel() + with tempfile.TemporaryDirectory() as tmpdir: + output_dir = Path(tmpdir) + manifest = export_lifecycle_inputs(panel, output_dir) + + exported_panel = pd.read_csv(output_dir / "research_panel.csv.gz") + market_history = pd.read_csv(output_dir / "market_history.csv.gz") + persisted_manifest = json.loads((output_dir / "manifest.json").read_text()) + + self.assertEqual( + set(exported_panel.columns), + {"date", "symbol", "in_universe", "open", "final_score"}, + ) + self.assertEqual(set(market_history["symbol"]), {"BTCUSDT", "ETHUSDT"}) + self.assertEqual(manifest, persisted_manifest) + self.assertEqual(manifest["contract_version"], "crypto.lifecycle_preflight.v1") + + def test_export_rejects_scored_date_without_universe(self) -> None: + panel = self._valid_panel() + latest_date = panel.index.get_level_values("date").max() + panel.loc[(latest_date, slice(None)), "in_universe"] = False + + with tempfile.TemporaryDirectory() as tmpdir, self.assertRaisesRegex( + ValueError, + "at least two in-universe symbols per scored date", + ): + export_lifecycle_inputs(panel, Path(tmpdir)) + + def test_export_rejects_stale_combo_history_even_when_panel_is_fresh(self) -> None: + panel = self._valid_panel() + cutoff = panel.index.get_level_values("date").max() - pd.Timedelta(days=10) + combo_mask = panel.index.get_level_values("symbol").isin({"BTCUSDT", "ETHUSDT"}) + stale_mask = combo_mask & (panel.index.get_level_values("date") > cutoff) + panel.loc[stale_mask, "close"] = pd.NA + + with tempfile.TemporaryDirectory() as tmpdir, self.assertRaisesRegex( + ValueError, + "BTC/ETH market history is stale", + ): + export_lifecycle_inputs(panel, Path(tmpdir)) From 466174224e6d611008a3b85f264eea0ca600239a Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Sat, 11 Jul 2026 08:38:16 +0800 Subject: [PATCH 5/5] fix: preserve complete crypto lifecycle inputs Co-Authored-By: Codex --- .../workflows/publish-lifecycle-inputs.yml | 6 +++++- scripts/export_lifecycle_preflight_inputs.py | 12 ++++++----- .../test_export_lifecycle_preflight_inputs.py | 21 +++++++++++++++++-- .../test_publish_lifecycle_inputs_workflow.py | 2 ++ 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/.github/workflows/publish-lifecycle-inputs.yml b/.github/workflows/publish-lifecycle-inputs.yml index eb5b276..cf9094a 100644 --- a/.github/workflows/publish-lifecycle-inputs.yml +++ b/.github/workflows/publish-lifecycle-inputs.yml @@ -39,7 +39,11 @@ jobs: - name: Download real Binance history run: | set -euo pipefail - python scripts/download_history.py --top-liquid "${DOWNLOAD_TOP_LIQUID}" --force-exchange-info + completed_date="$(python -c 'from datetime import datetime, timedelta, timezone; print((datetime.now(timezone.utc) - timedelta(days=1)).date())')" + python scripts/download_history.py --top-liquid "${DOWNLOAD_TOP_LIQUID}" \ + --end-date "${completed_date}" \ + --force-exchange-info + python scripts/download_history.py --symbols ETHUSDT --end-date "${completed_date}" - name: Export production research inputs run: | diff --git a/scripts/export_lifecycle_preflight_inputs.py b/scripts/export_lifecycle_preflight_inputs.py index 97e8a9b..5b1a17e 100644 --- a/scripts/export_lifecycle_preflight_inputs.py +++ b/scripts/export_lifecycle_preflight_inputs.py @@ -32,13 +32,16 @@ def export_lifecycle_inputs(panel: pd.DataFrame, output_dir: Path) -> dict[str, frame = panel.reset_index().copy() frame["date"] = pd.to_datetime(frame["date"], errors="coerce").dt.tz_localize(None).dt.normalize() frame["symbol"] = frame["symbol"].astype(str).str.strip().str.upper() - lifecycle_panel = frame[["date", "symbol", *PANEL_COLUMNS]].dropna(subset=["date", "open", "final_score"]) - if lifecycle_panel.empty: + today = pd.Timestamp.now(tz="UTC").tz_localize(None).normalize() + frame = frame.loc[frame["date"] < today] + lifecycle_panel = frame[["date", "symbol", *PANEL_COLUMNS]].dropna(subset=["date", "open"]) + scored_panel = lifecycle_panel.dropna(subset=["final_score"]) + if scored_panel.empty: raise ValueError("research panel has no scored lifecycle rows") - panel_dates = pd.DatetimeIndex(sorted(lifecycle_panel["date"].unique())) + panel_dates = pd.DatetimeIndex(sorted(scored_panel["date"].unique())) if len(panel_dates) < MIN_PANEL_DAYS: raise ValueError(f"research panel requires at least {MIN_PANEL_DAYS} scored dates") - in_universe_counts = lifecycle_panel.loc[lifecycle_panel["in_universe"]].groupby("date")["symbol"].nunique() + in_universe_counts = scored_panel.loc[scored_panel["in_universe"]].groupby("date")["symbol"].nunique() in_universe_counts = in_universe_counts.reindex(panel_dates, fill_value=0) if int(in_universe_counts.min()) < 2: raise ValueError("research panel requires at least two in-universe symbols per scored date") @@ -58,7 +61,6 @@ def export_lifecycle_inputs(panel: pd.DataFrame, output_dir: Path) -> dict[str, or max(symbol_dates) < max(reference_dates) ): raise ValueError(f"market history has incomplete symbol coverage: {symbol}") - today = pd.Timestamp.now(tz="UTC").tz_localize(None).normalize() if today - panel_dates.max() > pd.Timedelta(days=MAX_FRESHNESS_DAYS): raise ValueError("research panel is stale") if today - max(reference_dates) > pd.Timedelta(days=MAX_FRESHNESS_DAYS): diff --git a/tests/test_export_lifecycle_preflight_inputs.py b/tests/test_export_lifecycle_preflight_inputs.py index f801186..d6180a5 100644 --- a/tests/test_export_lifecycle_preflight_inputs.py +++ b/tests/test_export_lifecycle_preflight_inputs.py @@ -43,8 +43,8 @@ def test_export_lifecycle_inputs_writes_real_panel_contract(self) -> None: def test_export_rejects_scored_date_without_universe(self) -> None: panel = self._valid_panel() - latest_date = panel.index.get_level_values("date").max() - panel.loc[(latest_date, slice(None)), "in_universe"] = False + latest_completed_date = panel.index.get_level_values("date").unique()[-2] + panel.loc[(latest_completed_date, slice(None)), "in_universe"] = False with tempfile.TemporaryDirectory() as tmpdir, self.assertRaisesRegex( ValueError, @@ -52,6 +52,23 @@ def test_export_rejects_scored_date_without_universe(self) -> None: ): export_lifecycle_inputs(panel, Path(tmpdir)) + def test_export_preserves_open_rows_without_scores(self) -> None: + panel = self._valid_panel() + date = panel.index.get_level_values("date").unique()[500] + panel.loc[(date, "BTCUSDT"), ["in_universe", "final_score"]] = [False, pd.NA] + + with tempfile.TemporaryDirectory() as tmpdir: + output_dir = Path(tmpdir) + export_lifecycle_inputs(panel, output_dir) + exported = pd.read_csv(output_dir / "research_panel.csv.gz") + + row = exported.loc[ + (exported["date"] == date.date().isoformat()) + & (exported["symbol"] == "BTCUSDT") + ] + self.assertEqual(len(row), 1) + self.assertTrue(pd.isna(row.iloc[0]["final_score"])) + def test_export_rejects_stale_combo_history_even_when_panel_is_fresh(self) -> None: panel = self._valid_panel() cutoff = panel.index.get_level_values("date").max() - pd.Timedelta(days=10) diff --git a/tests/test_publish_lifecycle_inputs_workflow.py b/tests/test_publish_lifecycle_inputs_workflow.py index f75d531..a039b74 100644 --- a/tests/test_publish_lifecycle_inputs_workflow.py +++ b/tests/test_publish_lifecycle_inputs_workflow.py @@ -8,6 +8,8 @@ def test_publish_lifecycle_inputs_workflow_uses_real_research_pipeline() -> None assert 'DOWNLOAD_TOP_LIQUID: "90"' in workflow assert 'scripts/download_history.py --top-liquid "${DOWNLOAD_TOP_LIQUID}"' in workflow + assert '--end-date "${completed_date}"' in workflow + assert 'scripts/download_history.py --symbols ETHUSDT' in workflow assert "scripts/export_lifecycle_preflight_inputs.py" in workflow assert "--universe-mode broad_liquid" in workflow assert "crypto-lifecycle-inputs-${{ github.run_id }}-${{ github.run_attempt }}" in workflow