diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21d386d..67c205a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,3 +37,26 @@ jobs: - name: ChatGPT instructions build (must stay under the 8000-char Action limit) run: uv run python chatgpt/build_instructions.py + + doc-sync: + # Fail the build when a dataset documented in FinMind-Doc is missing from + # knowledge/datasets.md (the catalog silently drifting behind the docs is + # what let several datasets go unlisted for months). Only needs stdlib, so + # no uv/deps — just checkout both public repos and diff. + runs-on: ubuntu-latest + steps: + - name: Checkout FinMind-MCP + uses: actions/checkout@v4 + + - name: Checkout FinMind-Doc (public) + uses: actions/checkout@v4 + with: + repository: FinMind/FinMind-Doc + path: FinMind-Doc + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Catalog vs FinMind-Doc sync check + run: python scripts/check_doc_sync.py --doc ./FinMind-Doc diff --git a/knowledge/datasets.md b/knowledge/datasets.md index 3441718..10cbfe0 100644 --- a/knowledge/datasets.md +++ b/knowledge/datasets.md @@ -37,7 +37,7 @@ FinMind 會員方案由低到高為 **Free → Backer → Sponsor → Sponsor Pr - **Required:** `dataset=TaiwanStockInfoWithWarrantSummary`, `data_id`, `start_date` - **Optional:** `end_date` - **Key columns:** stock_id, date, close, target_stock_id, target_close, type, exercise_ratio, fulfillment_price -- **描述:** 台股權證標的對照表(含履約價、行使比例) +- **描述:** 台股權證標的對照表(含履約價、行使比例);涵蓋上市 (TWSE) 與上櫃 (TPEX) 權證,可用母股代碼 `target_stock_id` 反查其對應權證(含已到期、代碼重用的歷史權證),上櫃對照歷史回溯至 2011-01-03 ### TaiwanStockTradingDate - **Endpoint:** `/api/v4/data` @@ -203,6 +203,14 @@ FinMind 會員方案由低到高為 **Free → Backer → Sponsor → Sponsor Pr - **Key columns:** date, stock_id, buy, name, sell - **描述:** 個股三大法人買賣超 +### TaiwanStockInstitutionalInvestorsBuySellWide +- **Endpoint:** `/api/v4/data` +- **Tier:** Free(w/ data_id) +- **Required:** `dataset=TaiwanStockInstitutionalInvestorsBuySellWide`, `data_id` (股票代號), `start_date` +- **Optional:** `end_date` +- **Key columns:** date, stock_id, Foreign_Investor_buy, Foreign_Investor_sell, Foreign_Dealer_Self_buy, Foreign_Dealer_Self_sell, Investment_Trust_buy, Investment_Trust_sell, Dealer_buy, Dealer_sell, Dealer_self_buy, Dealer_self_sell, Dealer_Hedging_buy, Dealer_Hedging_sell +- **描述:** 個股三大法人買賣(寬表);與 TaiwanStockInstitutionalInvestorsBuySell 同資料,改為每交易日一列、各法人別買賣攤平成獨立欄位(免自行轉置),欄位涵蓋所有歷史分類、尚未存在的年代為 0 + ### TaiwanStockTotalInstitutionalInvestors - **Endpoint:** `/api/v4/data` - **Tier:** Free @@ -276,10 +284,10 @@ FinMind 會員方案由低到高為 **Free → Backer → Sponsor → Sponsor Pr - **Key columns:** securities_trader, price, buy, sell, securities_trader_id, stock_id, date - **描述:** 權證分點進出(單日) -### TaiwanstockGovernmentBankBuySell +### TaiwanStockGovernmentBankBuySell - **Endpoint:** `/api/v4/data` - **Tier:** Sponsor -- **Required:** `dataset=TaiwanstockGovernmentBankBuySell`, `data_id` (股票代號), `start_date` +- **Required:** `dataset=TaiwanStockGovernmentBankBuySell`, `data_id` (股票代號), `start_date` - **Optional:** `end_date` - **Key columns:** date, stock_id, buy_amount, sell_amount, buy, sell, bank_name - **描述:** 八大行庫買賣 @@ -308,6 +316,38 @@ FinMind 會員方案由低到高為 **Free → Backer → Sponsor → Sponsor Pr - **Key columns:** date, stock_id, stock_name, disposition_cnt, condition, measure, period_start, period_end - **描述:** 處置有價證券(含處置原因與期間) +### TaiwanStockBlockTrade +- **Endpoint:** `/api/v4/data` +- **Tier:** Sponsor +- **Required:** `dataset=TaiwanStockBlockTrade`, `data_id` (股票代號), `start_date` +- **Optional:** `end_date` +- **Key columns:** date, stock_id, trade_type, price, volume, trading_money +- **描述:** 鉅額交易日成交資訊(逐筆,含交易別、成交價量金額);資料區間 2005-04-04 ~ now + +### TaiwanStockBlockTradingDailyReport +- **Endpoint:** `/api/v4/data` +- **Tier:** Sponsor +- **Required:** `dataset=TaiwanStockBlockTradingDailyReport`, `start_date` +- **Optional:** `data_id` (股票代號), `end_date` +- **Key columns:** date, stock_id, securities_trader, securities_trader_id, price, buy, sell, trade_type +- **描述:** 鉅額交易買賣日報表(含券商別);資料區間 2026-04-28 ~ now + +### TaiwanStockDayTradingBorrowingFeeRate +- **Endpoint:** `/api/v4/data` +- **Tier:** Backer +- **Required:** `dataset=TaiwanStockDayTradingBorrowingFeeRate`, `start_date` +- **Optional:** `data_id` (股票代號), `end_date` +- **Key columns:** date, stock_id, stock_name, InvestorBorrowedShares, InvestorBorrowingFeeRate +- **描述:** 現股當日沖銷券差借券費率(借券股數、借券費率);資料區間 2015-06-01 ~ now + +### TaiwanStockLoanCollateralBalance +- **Endpoint:** `/api/v4/data` +- **Tier:** Sponsor +- **Required:** `dataset=TaiwanStockLoanCollateralBalance`, `data_id` (股票代號), `start_date` +- **Optional:** `end_date` +- **Key columns:** date, stock_id, market, MarginCurrentDayBalance, SecuritiesFirmLoanCurrentDayBalance, UnrestrictedLoanCurrentDayBalance(融資/證券商借貸/不限用途借貸各前日餘額、買進、賣出、現償、更換、今日餘額、次日限額,共 37 欄) +- **描述:** 借貸款項擔保品餘額表(集中市場 / 櫃買中心,融資與證券商借貸款項各項餘額);資料區間 2006-10-02 ~ now + ## 台股 - 基本面 ### TaiwanStockFinancialStatements @@ -538,6 +578,22 @@ FinMind 會員方案由低到高為 **Free → Backer → Sponsor → Sponsor Pr - **Key columns:** date, contract_month, option_id, settlement_price - **描述:** 選擇權最後結算價 +### TaiwanFuturesSpreadTick +- **Endpoint:** `/api/v4/data` +- **Tier:** Sponsor +- **Required:** `dataset=TaiwanFuturesSpreadTick`, `data_id` (期貨代號,如 `CAF`), `start_date` (single day) +- **Optional:** (無) +- **Key columns:** date, time, futures_id, contract_date, price, volume, near_price, far_price, spread_to_spread +- **描述:** 期貨價差(近月/遠月)每筆成交;單次請求只提供一天資料,自 2026-04-27 起逐交易日累積(暫不含更早歷史回補) + +### TaiwanOptionVix +- **Endpoint:** `/api/v4/data` +- **Tier:** Backer +- **Required:** `dataset=TaiwanOptionVix`, `start_date` +- **Optional:** `end_date` +- **Key columns:** date, time, vix +- **描述:** 臺指選擇權波動率指數(VIX);含日期、時間、波動率指數,資料區間 2026-03-01 ~ now + ## 台股 - 即時資料 > 此分類所有 dataset 都需要 **Sponsor** 方案。 @@ -610,6 +666,14 @@ FinMind 會員方案由低到高為 **Free → Backer → Sponsor → Sponsor Pr - **Key columns:** cb_id, ConversionPrice, IssuanceAmount, OutstandingAmount, date - **描述:** 可轉債每日總覽(轉換價、發行 / 流通在外金額) +### TaiwanStockConvertibleBondMonthlyAnalysis +- **Endpoint:** `/api/v4/data` +- **Tier:** Backer +- **Required:** `dataset=TaiwanStockConvertibleBondMonthlyAnalysis`, `start_date` +- **Optional:** `data_id` (可轉債代號), `end_date` +- **Key columns:** date, cb_id, cb_name, cb_name_en, custody_balance, last_month_balance, change, change_percent, issued_units, custody_accounts, pledged_units +- **描述:** 可轉換公司債月份分析表(每月保管餘額、上月餘額、增減、發行單位數、保管戶數、設質單位數);資料區間 2026-05-01 ~ now + ## 台股 - 其他 ### TaiwanStockNews diff --git a/scripts/check_doc_sync.py b/scripts/check_doc_sync.py new file mode 100644 index 0000000..5e58a34 --- /dev/null +++ b/scripts/check_doc_sync.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""CI gate: keep the MCP dataset catalog in sync with FinMind-Doc. + +The MCP knowledge pack (`knowledge/datasets.md`) is the single source of truth +for "what datasets exist", shared by the `list_datasets` tool and the Custom GPT +knowledge bundle. FinMind ships new datasets by documenting them in FinMind-Doc, +and historically the MCP catalog silently drifted behind (datasets added to the +docs months earlier were missing here). Unit tests can't catch that — it is a +content gap, not a code bug. + +This script fails CI when a dataset is documented in FinMind-Doc (i.e. appears in +an example `dataset="..."` call under docs/tutor/) but is absent from the MCP +catalog and not in the explicit EXCLUDE allowlist below. When it fails, the fix +is one of: + * add the dataset to knowledge/datasets.md (it belongs in the catalog), or + * add it to EXCLUDE with a reason (documented in FinMind-Doc but intentionally + outside the MCP generic-endpoint catalog, or a known doc typo). + +Usage: + python scripts/check_doc_sync.py --doc ./FinMind-Doc + FINMIND_DOC_DIR=./FinMind-Doc python scripts/check_doc_sync.py +""" + +from __future__ import annotations + +import argparse +import os +import re +import sys +from pathlib import Path + +# Datasets that appear in FinMind-Doc example code but are intentionally NOT in +# the MCP generic-endpoint catalog. Every entry needs a reason so this list +# stays auditable rather than becoming a dumping ground that hides real drift. +EXCLUDE: dict[str, str] = { + # Doc example typo: the documented dataset section is `TaiwanStockDividend` + # (股利政策表, already in the catalog); a couple of example snippets in + # Fundamental.md write `dataset="TaiwanStockStockDividend"` with a stray + # extra "Stock". The API enum has no such dataset. + "TaiwanStockStockDividend": "Doc example typo for TaiwanStockDividend", +} + +# Matches `dataset="Name"`, `dataset='Name'`, and `"dataset": "Name"`. +_DATASET_RE = re.compile( + r"""(?:"dataset"\s*:\s*|dataset\s*=\s*)["']([A-Za-z][A-Za-z0-9]+)["']""" +) + + +def documented_datasets(doc_dir: Path) -> set[str]: + """Dataset ids that appear in example calls under docs/tutor/.""" + tutor = doc_dir / "docs" / "tutor" + if not tutor.is_dir(): + sys.exit( + f"error: {tutor} not found — is --doc pointing at a FinMind-Doc " + f"checkout? (got {doc_dir})" + ) + found: set[str] = set() + for md in tutor.rglob("*.md"): + text = md.read_text(encoding="utf-8", errors="replace") + found.update(_DATASET_RE.findall(text)) + return found + + +def catalog_datasets(catalog_md: Path) -> set[str]: + """Dataset names from the `### Name` headers in knowledge/datasets.md.""" + names: set[str] = set() + for line in catalog_md.read_text(encoding="utf-8").splitlines(): + if line.startswith("### "): + names.add(line[4:].strip()) + return names + + +def main() -> int: + repo_root = Path(__file__).resolve().parent.parent + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--doc", + default=os.environ.get("FINMIND_DOC_DIR", "./FinMind-Doc"), + help="Path to a FinMind-Doc checkout (default: ./FinMind-Doc or " + "$FINMIND_DOC_DIR)", + ) + parser.add_argument( + "--catalog", + default=str(repo_root / "knowledge" / "datasets.md"), + help="Path to knowledge/datasets.md", + ) + args = parser.parse_args() + + doc = documented_datasets(Path(args.doc)) + catalog = catalog_datasets(Path(args.catalog)) + + missing = sorted(doc - catalog - set(EXCLUDE)) + # Excludes that no longer appear in the docs — clean them up so the list + # keeps reflecting reality (warning only, does not fail the build). + stale_excludes = sorted(set(EXCLUDE) - doc) + + print(f"FinMind-Doc documented datasets : {len(doc)}") + print(f"MCP catalog datasets : {len(catalog)}") + print(f"EXCLUDE allowlist : {len(EXCLUDE)}") + + if stale_excludes: + print() + print("warning: EXCLUDE entries no longer found in FinMind-Doc " + "(consider removing):") + for name in stale_excludes: + print(f" - {name}") + + if missing: + print() + print("FAIL: datasets documented in FinMind-Doc but missing from " + "knowledge/datasets.md:") + for name in missing: + print(f" - {name}") + print() + print("Fix: add each to knowledge/datasets.md (with Endpoint/Tier/" + "Required/Optional/Key columns/描述), or add it to EXCLUDE in " + "scripts/check_doc_sync.py with a reason.") + return 1 + + print() + print("OK: MCP catalog is in sync with FinMind-Doc.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())