diff --git a/AGENTS.md b/AGENTS.md index 0a07fcff..2d97fd22 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -71,3 +71,23 @@ pre-commit run --all-files For packaging or release changes, also run the verification scripts used by `.github/workflows/release.yml`. Never bypass hooks or weaken provider-specific checks to make a shared gate pass. + +## Maintainer-directed change batching + +For our own fixes and agent work directed by the maintainer, use a bounded, coherent PR rather +than one PR per small defect or implementation step. One PR may close several related issues +when the changes serve the same user workflow, subsystem repair or compatibility objective and +can be reviewed and reverted together. Keep implementation, regressions, documentation and +follow-up repairs together; use focused commits and retain evidence for each defect. Do not +create separate issues or PRs merely for individual files, tests, assertions or platform variants. + +Split unrelated changes, independently risky or breaking changes, separate ownership boundaries, +and work that needs independent release or rollback decisions. A general maintenance label alone +does not make a batch coherent. Reuse or safely consolidate existing owned submissions instead of +opening overlapping PRs; preserve contributor attribution, recoverable history and other owners' +active work. Do not rewrite or close someone else's work merely to reduce the queue count. + +Run focused checks while iterating, then qualify the complete final revision with every applicable +required gate. Group related work before expensive CI; do not weaken checks, bypass review or +claim old-head checks qualify changed code. This maintainer-directed batching rule takes precedence +over a literal one-issue-per-PR reading of contributor intake guidance. diff --git a/docs/providers/databento.md b/docs/providers/databento.md index 564d7b95..851e2963 100644 --- a/docs/providers/databento.md +++ b/docs/providers/databento.md @@ -146,6 +146,37 @@ df = provider.fetch_continuous_futures( --- +## Exact Timestamp Ranges + +`fetch_ohlcv_range()` requests one series between timezone-aware instants, including `start` and +excluding `end`. Unlike the date-based API, it never rounds to dates or applies +`adjust_session_dates`. The wrapper supports native minute, hourly and daily OHLCV schemas and +rejects unsupported frequencies, parent expansion and `ALL_SYMBOLS` before a request. + +```python +from datetime import UTC, datetime + +with DataBentoProvider(stype_in="continuous") as provider: + rows = provider.fetch_ohlcv_range( + "BTC.c.0", + datetime(2026, 9, 5, 12, 1, tzinfo=UTC), + datetime(2026, 9, 5, 12, 2, tzinfo=UTC), + "minute", + ) +``` + +The returned `symbol` identifies the requested series; `contract` retains the resolved contract +when the SDK supplies it. Select the same dataset, continuous roll rule and price convention used +for the stored series. This method does not reconstruct adjusted continuous prices or select another +contract to fill an absence. Databento timestamps identify interval starts, and no-trade intervals +can have no OHLCV record. A download may therefore leave gaps unresolved. + +For a stored report, see [targeted source recovery](../user-guide/incremental-updates.md#download-real-observations-for-a-selected-report). +For raw Parquet files, this method returns a frame for explicit review and merging; it does not +rewrite those files. Requests are metered. Use the native client's cost estimate before broad +requests. The endpoint and bar semantics follow the [historical range API](https://databento.com/docs/api-reference-historical/timeseries/timeseries-get-range) +and [OHLCV specification](https://databento.com/docs/schemas-and-data-formats/ohlcv). + ## API Key Setup ```bash diff --git a/docs/tutorials/03_incremental_updates.md b/docs/tutorials/03_incremental_updates.md index a4f4880f..9ded7f92 100644 --- a/docs/tutorials/03_incremental_updates.md +++ b/docs/tutorials/03_incremental_updates.md @@ -283,7 +283,10 @@ Today: Jan 15 Missing: [Jan 11] [Jan 12] [Jan 13] [Jan 14] ``` -**Solution**: Incremental updates automatically detect and fill gaps! +Incremental updates request actual observations from the configured source. Missing periods remain +missing when the source supplies no row; they are never replaced with synthetic market values. +Use the [stored calendar report](../user-guide/incremental-updates.md#inspect-missing-observations-on-disk) +to inspect internal gaps independently of acquisition. ```python # Even after 5 days without updates, incremental mode works @@ -296,7 +299,7 @@ print(result) # 'records_added': 5, # 'gap_detected': True, # 'gap_days': 5, -# 'message': 'Filled 5-day gap' +# 'message': 'Downloaded available source observations' # } ``` @@ -410,7 +413,7 @@ def catch_up_all_symbols(): if result.get('gap_detected'): logger.warning( - f"{symbol}: Filled {result['gap_days']}-day gap " + f"{symbol}: Downloaded source observations for {result['gap_days']} days " f"({result['records_added']} records)" ) elif result.get('skip_reason') == 'already_up_to_date': diff --git a/docs/tutorials/04_data_quality.md b/docs/tutorials/04_data_quality.md index f979bbb4..cfa14660 100644 --- a/docs/tutorials/04_data_quality.md +++ b/docs/tutorials/04_data_quality.md @@ -274,23 +274,13 @@ data = provider.fetch_ohlcv("AAPL", start, end) data = clean_ohlcv_data(data) ``` -### Option 3: Fill Forward (Last Resort) +### Preserve and Investigate Missing Observations -```python -def fill_missing_data(df): - """Fill missing values with forward fill (use sparingly).""" - # Fill nulls with previous valid value - df = df.with_columns([ - pl.col("open").fill_null(strategy="forward"), - pl.col("high").fill_null(strategy="forward"), - pl.col("low").fill_null(strategy="forward"), - pl.col("close").fill_null(strategy="forward"), - pl.col("volume").fill_null(0), # Volume = 0 for missing days - ]) - - logger.warning("Applied forward fill to missing data") - return df -``` +Do not replace absent OHLCV values with copied prices or zero volume. Preserve the missing data, +use the [calendar-based disk report](../user-guide/incremental-updates.md#inspect-missing-observations-on-disk) +to identify absent scheduled observations, and request actual rows from the original source when +its API and coverage permit. Intervals without trades may legitimately have no candle. A null price +inside a present row remains a validation finding, not permission to invent a replacement. ## Provider-Specific Validation diff --git a/docs/user-guide/cli-reference.md b/docs/user-guide/cli-reference.md index dd409347..edfc1a47 100644 --- a/docs/user-guide/cli-reference.md +++ b/docs/user-guide/cli-reference.md @@ -75,7 +75,8 @@ ml4t-data update -s AAPL --config ml4t-data.yaml | `--frequency` | `daily`, `hourly`, or `weekly` | | `--asset-class` | `equities`, `crypto`, `forex`, or `futures` | | `--lookback-days` | Existing-data overlap used for revisions | -| `--fill-gaps`, `--no-fill-gaps` | Enable or disable gap filling | +| `--check-gaps`, `--no-check-gaps` | Enable or disable read-only calendar gap checks | +| `--fill-gaps`, `--no-fill-gaps` | Legacy: filling is rejected; `--no-fill-gaps` disables checks | | `--provider`, `-p` | Provider to use for fetching | | `--start`, `--end` | Date bounds used only for a first load | | `--initial-load-days` | First-load history when `--start` is omitted | diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index d1f59c90..79afc418 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -166,7 +166,8 @@ provider, frequency, asset class, and bootstrap range. | `frequency` | `daily` | Bar frequency passed to `DataManager.update()` | | `asset_class` | `equity` | Storage key prefix and validation context | | `lookback_days` | `7` | Overlap fetched for incremental updates | -| `fill_gaps` | `true` | Enable gap detection and filling after merge | +| `check_gaps` | `true` | Report calendar gaps without changing observations | +| `fill_gaps` | unset | Legacy `false` disables checks; `true` is rejected | | `start` / `start_date` | unset | Explicit first-load start date when storage is empty | | `end` / `end_date` | today | Explicit first-load end date when storage is empty | | `initial_load_days` | `365` | First-load history length when no start date is set | diff --git a/docs/user-guide/data-quality.md b/docs/user-guide/data-quality.md index ba4e4d8c..cbf8c8db 100644 --- a/docs/user-guide/data-quality.md +++ b/docs/user-guide/data-quality.md @@ -59,11 +59,11 @@ from ml4t.data.validation.rules import ValidationRulePresets # Crypto is more volatile - wider thresholds crypto_rules = ValidationRulePresets.crypto_rules() -# max_return_threshold=0.5, staleness_threshold=3, no market hours check +# max_return_threshold=0.5, staleness_threshold=3 # Forex is less volatile - tighter thresholds forex_rules = ValidationRulePresets.forex_rules() -# max_return_threshold=0.1, staleness_threshold=10, 24/5 market +# max_return_threshold=0.1, staleness_threshold=10 # Strict mode for production data strict_rules = ValidationRulePresets.strict_rules() @@ -74,6 +74,57 @@ Available presets: `equity_rules()`, `crypto_rules()`, `forex_rules()`, `commodity_rules()`, `futures_rules()`, `strict_rules()`, `relaxed_rules()`. Commodity and futures presets report negative prices as warnings. The `negative_price_policy` setting accepts `"forbid"`, `"warn"`, or `"allow"`. +Presets adjust numerical thresholds, not calendar selection. Crypto and forex presets no longer +disable schedule checks implicitly; declare the market's actual schedule. Explicit per-check +opt-outs remain available. + +### Validate Calendar Membership and Continuity + +`CrossValidator`, `AssetValidator` and `CryptoSessionValidator` accept the canonical `calendar` +keyword and a shared `feed_spec` on construction and validation. A call's explicit calendar +overrides its constructor setting; the effective explicit setting overrides feed metadata, then +NYSE applies. Version-2 metadata can also explicitly disable calendars. Neither a descriptive +asset class nor an exchange name selects a schedule. + +```python +from ml4t.data.providers import SyntheticProvider +from ml4t.data.validation.cross_validation import CrossValidator +from ml4t.specs import FeedSpec + +with SyntheticProvider(calendar_mode="continuous") as provider: + spot = provider.fetch_ohlcv("BTCUSD", "2025-07-12", "2025-07-12", "minute") + +feed = FeedSpec(calendar="24/7", data_frequency="minute", timestamp_semantics="event_time") +report = CrossValidator(feed_spec=feed).validate(spot) +row_only = CrossValidator(calendar=None).validate(spot) +``` + +Calendar membership uses the same assignment as [native sessions](sessions.md), including DST, +holidays, early closes, overnight dates, breaks and registered custom schedules. Nanosecond instants +retain their precision in membership comparisons. Daily session +labels are not wall-clock observations; explicit event or close semantics override the daily-label +heuristic. Invalid calendars and incompatible temporal declarations fail clearly. Reported sample +indices refer to the original input positions. `calendar=None` disables temporal checks without +disabling price, volume, schema or reference-price checks. + +`CrossValidator` checks membership, not missing-bar completeness. `CryptoSessionValidator` checks +continuity through the shared gap detector: explicit `"24/7"` still detects real outages, whereas +`None` performs no schedule-derived gap check. Intraday continuity requires the complete version-2 +bar declaration described in [incremental updates](incremental-updates.md). An explicitly supplied +legacy `CryptoCalendar` object retains its elapsed continuous-gap and maintenance behavior; omission +no longer constructs that object automatically. `CrossValidator(is_crypto=True)` remains a legacy +24/7 hint only when neither an explicit calendar nor a feed calendar is declared. It cannot override +a canonical selection. + +`DataManager.load()` and `import_data()` pass their effective declaration and acquisition frequency +to validation before storage. For spot data, declare `"24/7"`; use the selected exchange calendar +for futures. Explicit calendar disable does not bypass OHLCV validation. + +`ValidationRuleConfig(calendar=...)` preserves omitted, named and explicitly disabled selections +through model copying, JSON and `ValidationRuleSet` YAML. New rule configurations use version 2; +version-1 payloads with a null calendar still inherit the default. Pass a restored configuration +as `CrossValidator(feed_spec=rules)` to apply its declaration, or override it with `calendar=...`. +Use ordinary `model_dump_json()` or `model_dump(mode="json")` without excluding explicit nulls. ### Persistent Rule Sets diff --git a/docs/user-guide/incremental-updates.md b/docs/user-guide/incremental-updates.md index 99c3318e..b76a1d63 100644 --- a/docs/user-guide/incremental-updates.md +++ b/docs/user-guide/incremental-updates.md @@ -41,26 +41,34 @@ metadata. Supplying the option overrides that value. ## Gap Handling -Gap detection and forward filling are enabled by default. Disable them when missing periods must -remain explicit: +Updates preserve absent observations and null market values. Calendar gap checks are enabled by +default and never create rows or substitute prices or volumes. To skip those checks: ```bash -ml4t-data update --symbol AAPL --no-fill-gaps --storage-path ./data +ml4t-data update --symbol AAPL --no-check-gaps --storage-path ./data ``` Daily detection counts missing sessions from the selected exchange schedule, not elapsed days. The Python updater resolves its explicit `calendar`, then supplied feed or stored metadata, then NYSE. Asset class and descriptive exchange metadata do not select a schedule. Continuous feeds -must declare `calendar="24/7"`; `calendar=None` disables schedule-derived detection and filling. +must declare `calendar="24/7"`; `calendar=None` disables schedule-derived detection. Intraday detection requires the complete version-2 grid declaration described below, supplied through `feed_spec` or retained in stored metadata. Undeclared intraday datasets must migrate their -metadata or use `--no-fill-gaps` / `fill_gaps=False` for updates. -[Trading-session completion](sessions.md) is a separate one-minute operation. The acquisition-range -planner's intraday limitations are described separately below. - -Detection does not itself fill or modify observations. The updater's separately enabled forward -fill uses a regular elapsed-time grid, not the selected exchange schedule, and can insert closed -dates when a real gap is present. Disable filling when those synthetic rows are inappropriate. +metadata or use `--no-check-gaps` / `check_gaps=False` for updates. +[Minute-grid expansion](sessions.md) is a separate operation that exposes absent positions as nulls. + +Synthetic imputation is unsupported. An explicit `fill_gaps=True` or `--fill-gaps` request fails +before acquisition or a dataset write. The legacy `fill_gaps=False` / `--no-fill-gaps` controls remain +compatible aliases for disabling checks. Configurations should use `check_gaps` instead. +The updater records whether checking ran and the detected missing-observation count; disabled +checks are not reported as proof of zero gaps. Source-history limits do not suppress detection. + +For an incremental tail, `IncrementalUpdater.determine_update_range()` uses the stored feed and +the same bounded calendar report to select the first missing scheduled observation after the +latest real row. It does not advance minute data by a day or request an exchange closure. Its +keyword-only `calendar` and `feed_spec` arguments override stored declarations. With an explicitly +disabled calendar, it overlaps the latest real row rather than guessing the next timestamp. +Use the bounded report below for older interior gaps, not the incremental tail planner. ### Inspect Daily Gaps Without Filling @@ -94,8 +102,8 @@ mapping, even with daily frequency or midnight timestamps. Right-labeled continu to the preceding session; event time belongs to the new session. Out-of-session instants fail clearly. Gap classification does not add a new off-session rejection policy for legacy daily labels. The requested daily frequency cannot silently override a declared intraday frequency. -Unknown calendars and malformed daily timestamps fail without modifying the input. `fill_gaps()` -remains a separate elapsed-grid operation, not calendar-aware imputation. +Unknown calendars and malformed daily timestamps fail without modifying the input. +The retired `fill_gaps()` helper rejects nonempty gap lists rather than manufacturing observations. ### Inspect Intraday Gaps Against a Declared Grid @@ -135,16 +143,141 @@ Explicit `calendar=None` disables classification, while `"24/7"` still expects c This replaces the old fixed-US-clock approximation. Unqualified legacy intraday calls now require a complete feed declaration or explicit `calendar=None`; the detector does not invent an anchor, -label side or partial-bar policy. Daily classification is unchanged. The separate intraday/weekly -backfill range planner below still has its documented limitations. +label side or partial-bar policy. Daily classification is unchanged. Use the bounded range report +below to inspect leading/trailing absences; weekly grids are not supported. -### Plan Daily Backfill Acquisition +### Inspect Missing Observations on Disk -`ml4t.data.update_manager.GapDetector` is a separate acquisition-range planner. Its constructor and +`ml4t.data.update_manager.GapDetector` compares a stored dataset with its expected calendar grid. +It reads timestamps, not prices, and neither contacts a provider nor changes stored rows or metadata. +The report works with timestamped measurements as well as OHLCV data. Its constructor and `detect_gaps()` / `detect_gaps_in_storage()` accept the canonical `calendar` and shared `feed_spec` declaration. Explicit selection overrides the declaration, then NYSE applies. Storage reads use the -dataset's persisted declaration unless a feed is supplied. Explicit `None` disables schedule-based -planning, including for absent datasets; 24/7 still expects weekend observations. +dataset's persisted declaration and frequency unless supplied explicitly. Explicit `None` disables +schedule-based detection, including for absent datasets; 24/7 still expects weekend observations. + +For an existing spot-crypto minute dataset, supply or retain its complete grid declaration: + +```python +from datetime import UTC, datetime + +from ml4t.data.storage import create_storage +from ml4t.data.update_manager import GapDetector +from ml4t.specs import FeedSpec + +storage = create_storage("./data", strategy="hive") +feed = FeedSpec( + time_contract_version=2, + calendar="24/7", + data_frequency="minute", + timestamp_semantics="event_time", + bar_grid_anchor="session_open", + bar_label="left", + partial_bar_policy="include", +) +report = GapDetector(feed_spec=feed).detect_gaps_in_storage( + storage, + "crypto/minute/coinbase-btc-usd-spot", + datetime(2026, 10, 16, 0, 0, tzinfo=UTC), + datetime(2026, 10, 16, 23, 59, tzinfo=UTC), +) +for gap in report: + print(gap["start"], gap["end"], gap["size_bars"]) +``` + +Intraday bounds are inclusive UTC instants. Each returned range starts at its first absent bar +timestamp, ends at its last, and reports `size_bars`, an exact count of scheduled bars. Bounds need +not land on a bar: only expected timestamps inside them count. The report includes missing boundary +bars, wholly absent sessions and absent datasets. Breaks, maintenance, holidays, DST, shortened +sessions and partial-bar policy follow the selected schedule. It shares the arithmetic expected grid +with in-memory detection rather than constructing a second calendar or every absent row. + +Existing Parquet files do not need to be imported into a Data storage backend. Read the file with +Polars and supply the same declaration and explicit coverage bounds: + +```python +import polars as pl + +report = GapDetector(feed_spec=feed).detect_gaps( + pl.read_parquet("./prices.parquet"), + start_date=datetime(2026, 10, 16, 0, 0, tzinfo=UTC), + end_date=datetime(2026, 10, 16, 23, 59, tzinfo=UTC), +) +``` + +Without explicit bounds, frame detection checks between the first and last stored timestamps. +Explicit bounds also reveal missing boundary bars and wholly empty intervals. Use `timestamp_col` +in the feed declaration when the file names its time column differently. A Boolean `is_imputed=True` +marker identifies a known placeholder, which does not satisfy an expected observation. Zero volume +alone does not identify a placeholder. Previously fabricated rows without a marker cannot be +distinguished from real observations by timestamps alone. + +An intraday calendar alone is insufficient. Frequency, timestamp side, anchor and partial-bar policy +must be declared, as in the example; contradictory or missing declarations fail clearly. Multi-asset +datasets are compared separately by the declared `entity_col`, or `symbol` when present; their gap +ranges include an `entity` value. The stored entity column is inspected across the dataset so an +instrument absent throughout the requested interval is not overlooked. Instruments that have never +been stored cannot be inferred: inspect an explicit absent dataset key and supply its declaration. +For CME futures select the product's actual calendar, such as `CME_Equity`, rather than assuming +every CME product uses the crypto schedule. CME cryptocurrency futures use `CME Globex Crypto`; +spot crypto normally uses `24/7`. See [CME and spot session conventions](sessions.md#cme-crypto-and-spot-crypto). + +A missing scheduled observation is not proof of a provider outage. Databento OHLCV and Coin Metrics +market candles can omit no-trade intervals. Keep these absences in the report; a source returning no +row does not justify creating a replacement candle. Targeted source downloads are a separate step, +not an effect of inspecting the dataset. Values inside present rows are checked by validation, not +by this timestamp-completeness report. + +### Download Real Observations for a Selected Report + +This is an explicit, potentially billable operation. For an existing one-instrument CME minute +dataset, first inspect the desired interval. The storage key and vendor symbol are separate: + +```python +from ml4t.data.providers import DataBentoProvider +from ml4t.data.storage.metadata_tracker import MetadataTracker +from ml4t.data.update_manager import BackfillManager + +cme_feed = feed.with_overrides(calendar="CME Globex Crypto") +key = "futures/minute/BTC" +gaps = GapDetector(feed_spec=cme_feed).detect_gaps_in_storage( + storage, + key, + datetime(2026, 9, 5, 12, 0, tzinfo=UTC), + datetime(2026, 9, 5, 12, 2, tzinfo=UTC), +) +with DataBentoProvider(stype_in="continuous") as provider: + result = BackfillManager(storage, MetadataTracker(storage.base_path)).execute_backfill( + key, gaps, provider, source_symbol="BTC.c.0", feed_spec=cme_feed + ) +print(result.rows_added, result.remaining_gaps) +``` + +The consumer rechecks the selected ranges against disk before requesting data. It downloads only +currently missing ranges, preserves real stored observations and identity/calendar metadata, and replaces a marked +null placeholder only when an actual source observation is returned. Intraday requests use the +optional `TimestampRangeProvider` capability; unsupported providers fail without falling back to a +full-day download. `calendar=None` disables calendar-based recovery. The first supported adapter is +Databento, with a left-labeled grid and one instrument per storage key. Multi-asset inspection is +supported, but targeted recovery of a panel requires separate per-instrument datasets. + +`result.success` means processing completed without errors, not that every observation was recovered. +An empty input report succeeds without downloading or writing; use `rows_added` and `gaps_filled` +to distinguish recovered data from a no-op. Previously an empty report returned `success=False`. +Inspect `errors` and `remaining_gaps`. Empty no-trade responses do not change storage or trigger +unbounded retries. Partial responses retain their residual gaps; `gaps_filled` counts only completely +recovered ranges. A history-recording failure reports an error even if observations were stored; +the residual report still reflects disk. A failed operation that cannot establish coverage returns +`remaining_gaps=None`, not an unverified clean report. Existing observed null values are a separate +validation concern, not permission to overwrite a source observation. `identify_candidates(min_gap_bars=...)` uses bar counts for +intraday data; its daily `min_gap_days` remains a session count. + +Coin Metrics is not currently a Data provider. Its candle and metric endpoints use different +identities and schemas; this Databento workflow does not claim to download them. See the +[Databento range contract](../providers/databento.md#exact-timestamp-ranges) before selecting a +continuous symbol or price convention. + +### Daily Range Compatibility Daily plans include missing sessions at either boundary and within the inclusive requested date range. Closed dates never become missing sessions. Returned `start` and `end` are inclusive session @@ -156,16 +289,16 @@ it does not manufacture observations or insert closed-date bars. The legacy `exclude_weekends=False` argument selects continuous days only when no canonical calendar or feed declaration is present. `exclude_weekends=True` now follows NYSE holidays as well as weekends. Omission follows NYSE, so callers requiring the old elapsed-day counts should pass `calendar="24/7"`. -Intraday/weekly range planning retains its legacy elapsed-time behavior; named calendar/feed -declarations for those frequencies are not supported by this planner yet. `calendar=None` disables -classification. Session-label dates use their declared timezone, or their existing timezone when +The legacy undeclared weekly path is elapsed-time based, not an exchange-grid completeness report; +declared weekly grids are not supported. `calendar=None` disables classification. +Session-label dates use their declared timezone, or their existing timezone when undeclared. Explicit event-time/bar-close/bar-side declarations map to authoritative session labels before planning, including for provider-row selection during backfill. Out-of-session instants fail clearly; original provider timestamps are retained in storage. Declare the feed's `timezone` when a non-UTC label must survive storage's canonical UTC reads. If a provider's history limit cannot reach the stored dataset, the updater records -`provider_history_limited` and skips gap filling across the unavailable interval. +`provider_history_limited`. Missing observations remain detectable even when they cannot be retrieved. ## Python API @@ -184,7 +317,7 @@ key = manager.update( frequency="daily", asset_class="equities", lookback_days=7, - fill_gaps=True, + check_gaps=True, initial_start="2023-01-01", initial_end="2024-01-01", ) @@ -204,7 +337,7 @@ Feed timezone, timestamp semantics and bar-grid declarations survive ingestion. `session_start_time` override is not supported by the native schedule adapter and is rejected. The requested frequency and bar type must agree with declared feed metadata. These declarations do not themselves validate OHLCV completeness or transform timestamps. Gap detection remains -separate from the updater's optional elapsed-grid imputation described above. +separate from acquisition; neither validation nor gap checking imputes missing market values. ## Inspect Stored State @@ -245,7 +378,7 @@ fetching or writing data. - Run an update only after the provider is expected to have finalized the latest requested period. - Reuse one storage directory and the same asset-class and frequency values so updates address the intended key. -- Treat forward-filled rows as synthetic observations. For minute-session completion, the - `is_imputed` column identifies inserted rows; the general updater does not add that marker. +- Preserve missing market data. Optional minute-grid expansion marks null placeholder rows with + the legacy `is_imputed` column; those rows are not recovered source observations. - Use `ml4t-data validate` after an update when schema, OHLC relationships, duplicates, or anomaly checks are part of the ingestion contract. diff --git a/docs/user-guide/sessions.md b/docs/user-guide/sessions.md index 528f39f0..13634000 100644 --- a/docs/user-guide/sessions.md +++ b/docs/user-guide/sessions.md @@ -135,10 +135,14 @@ Ordinary spot crypto uses `calendar="24/7"`, with civil-day labels and no CME ma `calendar=None` disables session handling instead. Select the schedule from the feed's actual market, not from its symbol or asset-class name. -## Complete Minute Sessions + -`SessionCompleter` expands minute bars to the exchange schedule. It requires `timestamp`, `open`, -`high`, `low`, `close`, and `volume`. The result adds `session_date` and `is_imputed`. +## Inspect a Nullable Minute Grid + +`SessionCompleter` optionally expands a timestamped series to scheduled minutes for inspection. +Missing positions have null market values; they are not recovered observations. It requires a +Datetime `timestamp` column and retains supplied value columns. The result adds `session_date` and +the legacy `is_imputed` marker, which identifies null placeholder rows. ```python from ml4t.data.sessions import SessionCompleter @@ -146,13 +150,15 @@ from ml4t.data.sessions import SessionCompleter completer = SessionCompleter("NYSE") complete = completer.complete_sessions( bars, - fill_method="forward", - zero_volume=True, + fill_method="none", + zero_volume=False, ) ``` -The supported fill methods are `"forward"`, `"backward"`, and `"none"`. Forward and backward -filling apply only within a session. `zero_volume=True` sets volume to zero for inserted rows. +Only `fill_method="none"` and `zero_volume=False` are supported. Requests to copy prices or invent +zero volume fail clearly. Observed nulls and all other original values remain unchanged. +Use the [disk gap report](incremental-updates.md#inspect-missing-observations-on-disk) when only a list +of missing observations is needed; it does not expand the data. Session completion rejects duplicate or non-minute-aligned timestamps and observations outside the selected schedule. Lunch breaks and declared calendar interruptions are excluded from the template; diff --git a/examples/cme_futures_sessions.py b/examples/cme_futures_sessions.py index 92d5bfc5..d35b7656 100644 --- a/examples/cme_futures_sessions.py +++ b/examples/cme_futures_sessions.py @@ -3,25 +3,26 @@ Demonstrates how to: 1. Load minute-level CME futures data 2. Assign session dates based on CME calendar -3. Fill gaps with forward-filled OHLC and zero volume +3. Inspect missing minute positions with null market values 4. Prepare data for session-aware cross-validation -CME Bitcoin futures trade Sunday 5pm CT → Friday 4pm CT (23 hours/day). -Sessions span multiple calendar days, requiring special handling for: +CME crypto sessions span multiple calendar days. The selected authoritative schedule +preserves historical rules and the May2026 transition. Relevant operations include: - Cross-validation (split by session, not calendar date) -- Gap filling (some providers skip zero-volume periods) +- Missing-observation inspection (no-trade intervals may legitimately have no candle) - Session aggregation (daily/weekly session summaries) """ import polars as pl from ml4t.data import DataManager +from ml4t.data.storage import create_storage # Constants SYMBOL = "BTC" PROVIDER = "databento" # Institutional futures data EXCHANGE = "CME" -CALENDAR = "CME_Globex_Crypto" +CALENDAR = "CME Globex Crypto" START_DATE = "2024-01-01" END_DATE = "2024-12-31" @@ -30,7 +31,7 @@ def main(): """Complete CME futures workflow with session management.""" # Initialize data manager - manager = DataManager(storage_path="./data") + manager = DataManager(storage=create_storage("./data", strategy="hive")) print("=" * 80) print("CME Bitcoin Futures - Session Management Workflow") @@ -47,7 +48,7 @@ def main(): start=START_DATE, end=END_DATE, provider=PROVIDER, - frequency="1min", + frequency="minute", exchange=EXCHANGE, calendar=CALENDAR, ) @@ -93,22 +94,22 @@ def main(): print("\n Sample sessions:") print(sample_session) - # Step 4: Fill gaps for complete sessions - print("\n4. Filling gaps with forward-filled OHLC and zero volume...") + # Step 4: Expose absent minutes without inventing market values. + print("\n4. Inspecting the nullable minute grid...") print(" Strategy:") - print(" - Forward-fill OHLC prices from last close") - print(" - Set volume=0 for filled rows") - print(" - Ensures continuous minute-level data") + print(" - Preserve observed prices and volumes") + print(" - Leave absent market values null") + print(" - Mark placeholders, not recovered observations") df_complete = manager.complete_sessions( df_with_sessions, exchange=EXCHANGE, fill_gaps=True, - zero_volume=True, + zero_volume=False, ) rows_added = len(df_complete) - len(df_with_sessions) - print(f" ✓ Added {rows_added:,} rows to fill gaps") + print(f" ✓ Exposed {rows_added:,} null placeholder rows") print(f" Total rows: {len(df_complete):,}") # Verify completeness @@ -163,7 +164,7 @@ def main(): print(f"Symbol: {SYMBOL}") print(f"Exchange: {EXCHANGE}") print(f"Original rows: {len(df):,}") - print(f"Rows added (gap filling): {rows_added:,}") + print(f"Null placeholder rows: {rows_added:,}") print(f"Final rows: {len(df_complete):,}") print(f"Unique sessions: {unique_sessions}") print(f"Average bars per session: {len(df_complete) / unique_sessions:,.0f}") @@ -172,12 +173,12 @@ def main(): def demonstrate_without_session_filling(): - """Show the problem when NOT using session filling.""" + """Compare observed rows with an optional nullable minute grid.""" print("\n" + "=" * 80) print("Why Session Filling Matters") print("=" * 80) - manager = DataManager(storage_path="./data") + manager = DataManager(storage=create_storage("./data", strategy="hive")) # Load data key = manager.load( @@ -185,13 +186,13 @@ def demonstrate_without_session_filling(): start=START_DATE, end=END_DATE, provider=PROVIDER, - frequency="1min", + frequency="minute", exchange=EXCHANGE, ) df = manager.storage.read(key).collect() - print("\nWithout gap filling:") + print("\nObserved rows:") print(f" Total rows: {len(df):,}") # Check for gaps @@ -202,13 +203,8 @@ def demonstrate_without_session_filling(): print(f" Gaps > 1 minute: {len(gaps)}") print(f" Largest gap: {gaps.max():.0f} minutes") - print("\nProblems without filling:") - print(" ✗ Missing bars distort technical indicators (e.g., moving averages)") - print(" ✗ Session aggregation incorrect (incomplete sessions)") - print(" ✗ Cross-validation splits may have imbalanced sessions") - print(" ✗ Backtest slippage calculations affected by gaps") - - print("\nWith gap filling (forward-fill OHLC, zero volume):") + print("\nUse a calendar report to distinguish missing bars from scheduled closures.") + print("\nOptional nullable grid, with no synthetic prices or volumes:") df_complete = manager.complete_sessions(df, exchange=EXCHANGE) print(f" Total rows: {len(df_complete):,}") print(f" Rows added: {len(df_complete) - len(df):,}") @@ -217,18 +213,15 @@ def demonstrate_without_session_filling(): remaining_gaps = time_diffs_complete[time_diffs_complete > 1].drop_nulls() print(f" Remaining gaps: {len(remaining_gaps)}") - print("\nBenefits:") - print(" ✓ Continuous time series for indicators") - print(" ✓ Complete sessions for aggregation") - print(" ✓ Balanced cross-validation splits") - print(" ✓ Realistic backtest execution") + print("\nNull placeholders do not establish complete market-data coverage.") + print("Download available source observations separately before treating a gap as repaired.") if __name__ == "__main__": # Run main workflow main() - # Demonstrate importance of gap filling + # Compare observations with null placeholders. demonstrate_without_session_filling() print("\n" + "=" * 80) diff --git a/examples/configs/advanced.yaml b/examples/configs/advanced.yaml index 6ed5b5f2..8189d9eb 100644 --- a/examples/configs/advanced.yaml +++ b/examples/configs/advanced.yaml @@ -184,7 +184,7 @@ validation: strict: true # Fail on any violation missing_data: max_gap_hours: 24 - fill_method: "forward" # forward, backward, or none + fill_method: "none" # Preserve missing observations; never invent market values outlier_detection: method: "zscore" threshold: 4.0 diff --git a/examples/wyden_cme_sessions_complete_workflow.py b/examples/wyden_cme_sessions_complete_workflow.py index 380ffd3d..8b57a517 100644 --- a/examples/wyden_cme_sessions_complete_workflow.py +++ b/examples/wyden_cme_sessions_complete_workflow.py @@ -1,31 +1,27 @@ #!/usr/bin/env python3 """Complete workflow: CME crypto futures with session management. -This example demonstrates the full workflow for working with Wyden CME futures data: +This historical example demonstrates a stored CME crypto futures workflow: 1. Load data from ml4t.data storage (already migrated from crypto-data-pipeline) 2. Assign CME session dates -3. Complete sessions (fill gaps) +3. Expose absent minutes as null placeholders, without inventing market values 4. Session-level analysis 5. Cross-validation with sessions -This is production data stored at ~/clients/wyden/long-short/data/ -- BTC futures: 2.79M rows (Dec 2017 → present) -- ETH futures: 1.68M rows (Feb 2021 → present) +Select the existing storage root with ML4T_DATA_PATH; source observations remain unchanged. Usage: python wyden_cme_sessions_complete_workflow.py """ -from pathlib import Path - import polars as pl from ml4t.data import DataManager +from ml4t.data.core.config import resolve_data_root from ml4t.data.storage.backend import StorageConfig from ml4t.data.storage.hive import HiveStorage -# Wyden data location -DATA_PATH = Path.home() / "clients/wyden/long-short/data" +DATA_PATH = resolve_data_root() # Symbols to process SYMBOLS = ["BTC", "ETH"] @@ -34,10 +30,10 @@ def main(): """Complete session management workflow for CME futures.""" print("=" * 80) - print("Wyden CME Crypto Futures - Complete Session Workflow") + print("Stored CME Crypto Futures - Session Workflow") print("=" * 80) - # 1. Initialize DataManager with Wyden storage + # 1. Initialize DataManager with the configured storage. print(f"\n[1] Initializing DataManager with storage: {DATA_PATH}") storage = HiveStorage(config=StorageConfig(base_path=str(DATA_PATH))) manager = DataManager(storage=storage) @@ -75,20 +71,20 @@ def main(): n_sessions = df_with_sessions["session_date"].n_unique() print(f" ✓ Assigned {n_sessions:,} unique sessions") - # 4. Complete sessions (fill gaps) - print("\n[4] Completing sessions (filling gaps)...") - print(" Strategy: Forward-fill OHLC, zero volume") + # 4. Expose absent minute positions without manufacturing observations. + print("\n[4] Inspecting the nullable minute grid...") + print(" Missing market values remain null") df_complete = manager.complete_sessions( df_with_sessions, exchange="CME", - fill_gaps=True, # Fill missing minutes within sessions - zero_volume=True, # Set volume=0 for filled bars + fill_gaps=True, # Legacy name for null grid expansion, not price imputation + zero_volume=False, ) n_filled = len(df_complete) - len(df_with_sessions) print( - f" ✓ Filled {n_filled:,} missing bars ({n_filled / len(df_with_sessions) * 100:.2f}% of original)" + f" ✓ Exposed {n_filled:,} null placeholders ({n_filled / len(df_with_sessions) * 100:.2f}% of original)" ) print(f" Total bars: {len(df_complete):,}") @@ -173,7 +169,7 @@ def main(): print(f"{'=' * 80}") print(f"✓ Processed {len(SYMBOLS)} symbols: {', '.join(SYMBOLS)}") print("✓ Assigned CME session dates") - print("✓ Filled gaps for complete sessions") + print("✓ Exposed absent minutes as null placeholders, not recovered observations") print("✓ Saved output with session_date column") print("\nNext steps:") diff --git a/src/ml4t/data/assets/validation.py b/src/ml4t/data/assets/validation.py index 670b78ae..450c6e6b 100644 --- a/src/ml4t/data/assets/validation.py +++ b/src/ml4t/data/assets/validation.py @@ -2,12 +2,15 @@ from __future__ import annotations +from typing import Any + import polars as pl import structlog from ml4t.data.assets.asset_class import AssetClass, AssetInfo from ml4t.data.assets.schemas import AssetSchema -from ml4t.data.calendar.crypto import CryptoCalendar, CryptoSessionValidator +from ml4t.data.calendar.crypto import CryptoSessionValidator +from ml4t.specs import UNSET_CALENDAR, CalendarUnset, FeedSpec logger = structlog.get_logger() @@ -15,7 +18,13 @@ class AssetValidator: """Validator for asset-specific data quality rules.""" - def __init__(self, asset_info: AssetInfo): + def __init__( + self, + asset_info: AssetInfo, + *, + calendar: str | None | CalendarUnset = UNSET_CALENDAR, + feed_spec: Any | None = None, + ) -> None: """ Initialize validator for specific asset. @@ -24,19 +33,23 @@ def __init__(self, asset_info: AssetInfo): """ self.asset_info = asset_info self.asset_class = asset_info.asset_class + self._calendar = calendar + self.feed_spec = FeedSpec.from_any(feed_spec) + self.calendar = self.feed_spec.resolve_calendar(calendar=calendar) # Initialize specialized validators if self.asset_class == AssetClass.CRYPTO: - self.calendar = CryptoCalendar(exchange=asset_info.exchange) - self.session_validator = CryptoSessionValidator(self.calendar) + self.session_validator = CryptoSessionValidator(self.calendar, feed_spec=self.feed_spec) else: - self.calendar = None self.session_validator = None def validate( self, df: pl.DataFrame, - frequency: str = "daily", + frequency: str | None = None, + *, + calendar: str | None | CalendarUnset = UNSET_CALENDAR, + feed_spec: Any | None = None, ) -> dict[str, list[str]]: """ Perform comprehensive validation. @@ -48,6 +61,11 @@ def validate( Returns: Dictionary of validation category to issues """ + feed = self.feed_spec if feed_spec is None else FeedSpec.from_any(feed_spec) + if calendar is UNSET_CALENDAR: + calendar = self._calendar + selected = feed.resolve_calendar(calendar=calendar) + frequency = frequency or feed.data_frequency or "daily" results = {} # Schema validation @@ -57,7 +75,7 @@ def validate( # Asset-specific validation if self.asset_class == AssetClass.CRYPTO: - results.update(self._validate_crypto(df, frequency)) + results.update(self._validate_crypto(df, frequency, selected, feed)) elif self.asset_class == AssetClass.EQUITY: results.update(self._validate_equity(df)) elif self.asset_class == AssetClass.FOREX: @@ -65,17 +83,45 @@ def validate( # Common validations results.update(self._validate_common(df)) + if not df.is_empty() and "timestamp" in df.columns: + from ml4t.data.validation.cross_validation import CrossValidator + + temporal = CrossValidator( + check_price_continuity=False, + check_volume_spikes=False, + calendar=selected, + feed_spec=feed, + ).validate(df, frequency=frequency) + category = { + AssetClass.EQUITY: "equity_specific", + AssetClass.FOREX: "forex_specific", + }.get(self.asset_class, "sessions") + for issue in temporal.issues: + if issue.check == "weekend_trading": + message = f"Found {issue.row_count} weekend data points" + else: + count = issue.row_count - issue.details["weekend_count"] + if count == 0 or ( + self.asset_class == AssetClass.EQUITY and count <= len(df) * 0.1 + ): + continue + message = f"Found {count} data points outside market hours" + results.setdefault(category, []).append(message) # Remove empty categories return {k: v for k, v in results.items() if v} - def _validate_crypto(self, df: pl.DataFrame, frequency: str) -> dict[str, list[str]]: + def _validate_crypto( + self, df: pl.DataFrame, frequency: str, calendar: str | None, feed: FeedSpec + ) -> dict[str, list[str]]: """Validate crypto-specific rules.""" results = {} # 24/7 continuity check if self.session_validator: - continuity_issues = self.session_validator.validate_continuity(df, frequency) + continuity_issues = self.session_validator.validate_continuity( + df, frequency, calendar=calendar, feed_spec=feed + ) if continuity_issues: results["continuity"] = continuity_issues @@ -109,29 +155,6 @@ def _validate_equity(self, df: pl.DataFrame) -> dict[str, list[str]]: results = {} issues = [] - # Check for trading during market hours only - if "timestamp" in df.columns: - timestamps = df["timestamp"].to_list() - - # Check for after-hours trading (simplified) - after_hours = [ - ts - for ts in timestamps - if ts.weekday() < 5 - and ( # Weekday - ts.hour < 9 or ts.hour >= 16 or (ts.hour == 9 and ts.minute < 30) - ) - ] - - if after_hours and len(after_hours) > len(timestamps) * 0.1: - issues.append(f"Found {len(after_hours)} data points outside market hours") - - # Check for weekend data (should not exist for regular equities) - if "timestamp" in df.columns: - weekend_data = df.filter(pl.col("timestamp").dt.weekday() >= 6) - if len(weekend_data) > 0: - issues.append(f"Found {len(weekend_data)} weekend data points") - # Check for reasonable price ranges if "close" in df.columns: prices = df["close"].to_numpy() @@ -151,20 +174,6 @@ def _validate_forex(self, df: pl.DataFrame) -> dict[str, list[str]]: results = {} issues = [] - # Check for weekend gaps (forex closes Friday evening, opens Sunday evening) - if "timestamp" in df.columns: - timestamps = df["timestamp"].to_list() - - # Check for Saturday data (should not exist) - saturday_data = [ - ts - for ts in timestamps - if ts.weekday() == 5 # Saturday - ] - - if saturday_data: - issues.append(f"Found {len(saturday_data)} Saturday data points") - # Check for reasonable exchange rates if "close" in df.columns: rates = df["close"].to_numpy() diff --git a/src/ml4t/data/calendar/crypto.py b/src/ml4t/data/calendar/crypto.py index c2ecce77..7bc54f1a 100644 --- a/src/ml4t/data/calendar/crypto.py +++ b/src/ml4t/data/calendar/crypto.py @@ -3,10 +3,13 @@ from __future__ import annotations from datetime import UTC, datetime, timedelta +from typing import Any import polars as pl import structlog +from ml4t.specs import UNSET_CALENDAR, CalendarUnset, FeedSpec + logger = structlog.get_logger() @@ -274,19 +277,28 @@ def _get_interval_seconds(self, frequency: str) -> int: class CryptoSessionValidator: """Validator for crypto trading sessions.""" - def __init__(self, calendar: CryptoCalendar | None = None) -> None: + def __init__( + self, + calendar: CryptoCalendar | str | None | CalendarUnset = UNSET_CALENDAR, + *, + feed_spec: Any | None = None, + ) -> None: """ Initialize validator. Args: calendar: Crypto calendar to use """ - self.calendar = calendar or CryptoCalendar() + self.calendar = calendar + self.feed_spec = FeedSpec.from_any(feed_spec) def validate_continuity( self, df: pl.DataFrame, - frequency: str = "daily", + frequency: str | None = None, + *, + calendar: CryptoCalendar | str | None | CalendarUnset = UNSET_CALENDAR, + feed_spec: Any | None = None, ) -> list[str]: """ Validate data continuity for 24/7 market. @@ -299,6 +311,19 @@ def validate_continuity( List of validation issues """ issues = [] + feed = self.feed_spec if feed_spec is None else FeedSpec.from_any(feed_spec) + if calendar is UNSET_CALENDAR: + calendar = self.calendar + selected = ( + calendar + if isinstance(calendar, CryptoCalendar) + else feed.resolve_calendar(calendar=calendar) + ) + if selected is None: + return issues + frequency = frequency or feed.data_frequency or "daily" + if frequency.lower() in {"1d", "1day"}: + frequency = "daily" if df.is_empty(): return ["No data to validate"] @@ -307,7 +332,17 @@ def validate_continuity( return ["Missing timestamp column"] # Find gaps - gaps = self.calendar.find_gaps(df, frequency) + if isinstance(selected, CryptoCalendar): + gaps = selected.find_gaps(df, frequency) + else: + from ml4t.data.utils.gaps import GapDetector + + gaps = [ + (gap.start, gap.end) + for gap in GapDetector().detect_gaps( + df, frequency, calendar=selected, feed_spec=feed + ) + ] for gap_start, gap_end in gaps: duration = gap_end - gap_start diff --git a/src/ml4t/data/cli/batch.py b/src/ml4t/data/cli/batch.py index 326ae616..7e31a5de 100644 --- a/src/ml4t/data/cli/batch.py +++ b/src/ml4t/data/cli/batch.py @@ -179,6 +179,7 @@ def update_all(ctx, config, dataset, dry_run): provider=provider, lookback_days=lookback_days, fill_gaps=fill_gaps, + check_gaps=ds_config.check_gaps, initial_start=initial_start, initial_end=initial_end, initial_load_days=initial_load_days, diff --git a/src/ml4t/data/cli/core.py b/src/ml4t/data/cli/core.py index 1bf94567..3e563e5a 100644 --- a/src/ml4t/data/cli/core.py +++ b/src/ml4t/data/cli/core.py @@ -287,7 +287,12 @@ def fetch( show_default=True, ) @click.option("--lookback-days", type=click.IntRange(min=0), default=7, show_default=True) -@click.option("--fill-gaps/--no-fill-gaps", default=True, show_default=True) +@click.option( + "--fill-gaps/--no-fill-gaps", + default=None, + help="Legacy control: --no-fill-gaps disables checks; synthetic filling is unsupported", +) +@click.option("--check-gaps/--no-check-gaps", default=True, show_default=True) @click.option("--provider", "-p", help="Provider to use for fetching") @click.option( "--initial-start", @@ -314,6 +319,7 @@ def update( asset_class, lookback_days, fill_gaps, + check_gaps, provider, initial_start, initial_end, @@ -341,6 +347,7 @@ def update( asset_class=asset_class, lookback_days=lookback_days, fill_gaps=fill_gaps, + check_gaps=check_gaps, provider=provider, initial_start=initial_start, initial_end=initial_end, diff --git a/src/ml4t/data/config/models.py b/src/ml4t/data/config/models.py index ebba2c21..ab96a24b 100644 --- a/src/ml4t/data/config/models.py +++ b/src/ml4t/data/config/models.py @@ -259,10 +259,20 @@ class DatasetConfig(BaseModel): validation: dict[str, Any] = Field(default_factory=dict, description="Validation settings") storage: dict[str, Any] = Field(default_factory=dict, description="Storage settings") lookback_days: int = Field(default=7, ge=0, description="Days to revisit during updates") - fill_gaps: bool = Field(default=True, description="Fill gaps during updates") + fill_gaps: bool | None = Field( + default=None, description="Legacy False disables checks; synthetic filling is unsupported" + ) + check_gaps: bool = Field(default=True, description="Report calendar gaps without imputation") initial_load_days: int = Field(default=365, ge=1, description="Initial history length") extra: dict[str, Any] = Field(default_factory=dict, description="Dataset-specific settings") + @field_validator("fill_gaps") + @classmethod + def reject_synthetic_imputation(cls, value: bool | None) -> bool | None: + if value is True: + raise ValueError("Synthetic gap imputation is unsupported; use check_gaps for reports") + return value + @model_validator(mode="before") @classmethod def normalize_beta_dates(cls, value: Any) -> Any: diff --git a/src/ml4t/data/data_manager.py b/src/ml4t/data/data_manager.py index 15c85080..ddf1347d 100644 --- a/src/ml4t/data/data_manager.py +++ b/src/ml4t/data/data_manager.py @@ -400,7 +400,7 @@ def update( frequency: str = "daily", asset_class: str = "equities", lookback_days: int = 7, - fill_gaps: bool = True, + fill_gaps: bool | None = None, provider: str | None = None, initial_start: str | None = None, initial_end: str | None = None, @@ -408,6 +408,7 @@ def update( *, calendar: str | None | CalendarUnset = UNSET_CALENDAR, feed_spec: Any | None = None, + check_gaps: bool = True, ) -> str: """Update existing data with incremental fetch.""" if not self._storage_manager: @@ -426,6 +427,7 @@ def update( initial_load_days, calendar=calendar, feed_spec=feed_spec, + check_gaps=check_gaps, ) # ======================================================================== @@ -480,12 +482,12 @@ def complete_sessions( exchange: str | None = None, calendar: str | None | CalendarUnset = UNSET_CALENDAR, fill_gaps: bool = True, - fill_method: str = "forward", - zero_volume: bool = True, + fill_method: str = "none", + zero_volume: bool = False, *, feed_spec: Any | None = None, ) -> pl.DataFrame: - """Complete sessions by filling gaps.""" + """Expose absent minute positions as nulls without inventing market values.""" manager = self._metadata_manager or MetadataManager(None) return manager.complete_sessions( df, exchange, calendar, fill_gaps, fill_method, zero_volume, feed_spec=feed_spec @@ -545,10 +547,21 @@ def _report_progress(self, message: str, progress: float) -> None: if self._storage_manager: self._storage_manager._report_progress(message, progress) - def _validate_data(self, df: pl.DataFrame, symbol: str, asset_class: str = "equity") -> Any: + def _validate_data( + self, + df: pl.DataFrame, + symbol: str, + asset_class: str = "equity", + *, + frequency: str | None = None, + calendar: str | None | CalendarUnset = UNSET_CALENDAR, + feed_spec: Any | None = None, + ) -> Any: """Validate data (backward compatibility).""" if self._storage_manager: - return self._storage_manager._validate_data(df, symbol, asset_class) + return self._storage_manager._validate_data( + df, symbol, asset_class, frequency=frequency, calendar=calendar, feed_spec=feed_spec + ) return None def _merge_data(self, existing: pl.DataFrame, new: pl.DataFrame) -> pl.DataFrame: diff --git a/src/ml4t/data/managers/bulk_manager.py b/src/ml4t/data/managers/bulk_manager.py index 745fd1c2..8804a569 100644 --- a/src/ml4t/data/managers/bulk_manager.py +++ b/src/ml4t/data/managers/bulk_manager.py @@ -146,10 +146,12 @@ def update_symbols( asset_class: str = "equities", provider: str | None = None, lookback_days: int = 7, - fill_gaps: bool = True, + fill_gaps: bool | None = None, initial_start: str | None = None, initial_end: str | None = None, initial_load_days: int = 365, + *, + check_gaps: bool = True, ) -> dict[str, str]: """Update specific symbols. @@ -159,7 +161,8 @@ def update_symbols( asset_class: Asset class provider: Optional provider override lookback_days: Days to look back for updates - fill_gaps: If True, detect and fill gaps + fill_gaps: Legacy False disables checks; True is rejected. + check_gaps: Report calendar gaps without creating observations. initial_start: Optional start date for first load when no data exists initial_end: Optional end date for first load when no data exists initial_load_days: Days to fetch on first load when initial_start is not set @@ -178,6 +181,7 @@ def update_symbols( asset_class=asset_class, lookback_days=lookback_days, fill_gaps=fill_gaps, + check_gaps=check_gaps, provider=provider, initial_start=initial_start, initial_end=initial_end, diff --git a/src/ml4t/data/managers/metadata_manager.py b/src/ml4t/data/managers/metadata_manager.py index 1605fd83..a7a2a175 100644 --- a/src/ml4t/data/managers/metadata_manager.py +++ b/src/ml4t/data/managers/metadata_manager.py @@ -188,20 +188,20 @@ def complete_sessions( exchange: str | None = None, calendar: str | None | CalendarUnset = UNSET_CALENDAR, fill_gaps: bool = True, - fill_method: str = "forward", - zero_volume: bool = True, + fill_method: str = "none", + zero_volume: bool = False, *, feed_spec: Any | None = None, ) -> pl.DataFrame: - """Complete sessions by filling gaps. + """Expand the minute grid with null placeholders, never synthetic market values. Args: df: DataFrame with timestamp column exchange: Exchange code calendar: Calendar name override - fill_gaps: If True, fill gaps - fill_method: Fill method ("forward", "backward", "none") - zero_volume: If True, set volume=0 for filled rows + fill_gaps: Legacy name for grid expansion; False performs assignment only. + fill_method: Only "none" is supported; missing market values remain null. + zero_volume: Must be False; a missing observation has no known volume. Returns: DataFrame with complete minute sessions and an is_imputed column. Input must diff --git a/src/ml4t/data/managers/storage_manager.py b/src/ml4t/data/managers/storage_manager.py index 4d62b2d1..be158021 100644 --- a/src/ml4t/data/managers/storage_manager.py +++ b/src/ml4t/data/managers/storage_manager.py @@ -95,6 +95,10 @@ def _validate_data( df: pl.DataFrame, symbol: str, asset_class: str = "equity", + *, + frequency: str | None = None, + calendar: str | None | CalendarUnset = UNSET_CALENDAR, + feed_spec: Any | None = None, ) -> Any: """Validate data using OHLCV and cross-validation. @@ -106,7 +110,6 @@ def _validate_data( Returns: ValidationReport with results """ - from ml4t.data.assets.asset_class import AssetClass from ml4t.data.validation import OHLCVValidator, ValidationReport from ml4t.data.validation.cross_validation import CrossValidator from ml4t.data.validation.rules import ValidationRulePresets @@ -114,7 +117,10 @@ def _validate_data( report = ValidationReport(symbol=symbol, provider="unknown") rules = ValidationRulePresets.for_asset_class(asset_class) - is_crypto = rules.asset_class == AssetClass.CRYPTO + if calendar is UNSET_CALENDAR: + calendar = rules.calendar + if feed_spec is None: + feed_spec = rules # Run OHLCV validation ohlcv_validator = OHLCVValidator( @@ -141,10 +147,11 @@ def _validate_data( check_market_hours=rules.check_market_hours, volume_spike_threshold=rules.volume_spike_threshold, price_gap_threshold=rules.price_gap_threshold, - is_crypto=is_crypto, + calendar=calendar, + feed_spec=feed_spec, ) - result = cross_validator.validate(df) + result = cross_validator.validate(df, frequency=frequency) report.add_result(result, "CrossValidator") return report @@ -265,7 +272,7 @@ def load( if not self.storage: raise ValueError("Storage not configured") - _, temporal_values = self._ingestion_feed(calendar, feed_spec, frequency, bar_type) + feed, temporal_values = self._ingestion_feed(calendar, feed_spec, frequency, bar_type) logger.info( "Starting data load", @@ -290,9 +297,13 @@ def load( logger.info(f"Fetched {len(df)} rows of data") self._report_progress(f"Fetched {len(df)} rows", 0.5) + min_ts, max_ts = timestamp_bounds(df) + # Validate data if enabled if self.enable_validation: - validation_report = self._validate_data(df, symbol, asset_class) + validation_report = self._validate_data( + df, symbol, asset_class, frequency=frequency, feed_spec=feed + ) if not validation_report.passed: logger.warning( "Data validation issues found", @@ -305,8 +316,6 @@ def load( # Create metadata from ml4t.data.core.models import DataObject, Metadata - min_ts, max_ts = timestamp_bounds(df) - bar_params = {} if bar_type == "time": bar_params["frequency"] = frequency @@ -393,7 +402,7 @@ def import_data( if not self.storage: raise ValueError("Storage not configured") - _, temporal_values = self._ingestion_feed(calendar, feed_spec, frequency, bar_type) + feed, temporal_values = self._ingestion_feed(calendar, feed_spec, frequency, bar_type) logger.info( "Importing external data", @@ -414,9 +423,13 @@ def import_data( if missing: raise ValueError(f"Missing required columns: {missing}") + min_ts, max_ts = timestamp_bounds(df) + # Validate data if enabled if self.enable_validation: - validation_report = self._validate_data(df, symbol, asset_class) + validation_report = self._validate_data( + df, symbol, asset_class, frequency=frequency, feed_spec=feed + ) if not validation_report.passed: logger.warning( "Imported data has validation issues", @@ -427,8 +440,6 @@ def import_data( # Create metadata from ml4t.data.core.models import DataObject, Metadata - min_ts, max_ts = timestamp_bounds(df) - bar_params = {} if bar_type == "time": bar_params["frequency"] = frequency @@ -472,7 +483,7 @@ def update( frequency: str = "daily", asset_class: str = "equities", lookback_days: int = 7, - fill_gaps: bool = True, + fill_gaps: bool | None = None, provider: str | None = None, initial_start: str | None = None, initial_end: str | None = None, @@ -480,6 +491,7 @@ def update( *, calendar: str | None | CalendarUnset = UNSET_CALENDAR, feed_spec: Any | None = None, + check_gaps: bool = True, ) -> str: """Update existing data with incremental fetch. @@ -488,7 +500,8 @@ def update( frequency: Data frequency asset_class: Asset class lookback_days: Days to look back for updates - fill_gaps: If True, detect and fill gaps in data + fill_gaps: Legacy False disables checks; True is rejected, never imputed. + check_gaps: Report calendar gaps without changing observations. provider: Optional provider override initial_start: Optional start date for first load when no data exists initial_end: Optional end date for first load when no data exists @@ -500,6 +513,12 @@ def update( Raises: ValueError: If storage not configured """ + if fill_gaps is True: + raise ValueError( + "Synthetic gap imputation is unsupported; inspect gaps and download source observations" + ) + if fill_gaps is False: + check_gaps = False if not self.storage: raise ValueError("Storage not configured") @@ -638,16 +657,17 @@ def update( added_rows=len(merged_df) - len(existing_df), ) - # Detect and optionally fill gaps + # Gap checks never change observations, even outside provider history limits. gaps = [] - if fill_gaps and not provider_history_limited: + gap_check_performed = check_gaps and effective_feed.resolve_calendar() is not None + if gap_check_performed: self._report_progress(f"Checking for gaps in {symbol}", 0.6) from ml4t.data.utils.gaps import GapDetector selected_calendar = effective_feed.resolve_calendar() if selected_calendar is not None and effective_bar_type != "time": - raise NotImplementedError("calendar gap filling requires fixed time bars") + raise NotImplementedError("calendar gap checks require fixed time bars") gap_detector = GapDetector() gaps = gap_detector.detect_gaps( merged_df, @@ -664,16 +684,6 @@ def update( total_missing=gap_summary["total_missing_periods"], ) - merged_df = gap_detector.fill_gaps(merged_df, gaps, method="forward") - logger.info("Filled gaps using forward fill", final_rows=len(merged_df)) - elif fill_gaps and provider_history_limited: - logger.warning( - "Gap filling skipped because provider history cannot cover the missing range", - provider=provider, - last_timestamp=last_timestamp, - resumed_at=fetch_start, - ) - self._report_progress(f"Storing updated data for {symbol}", 0.8) # Update metadata with new range @@ -690,7 +700,12 @@ def update( # BulkManager compares this legacy field with naive local cutoffs. "last_update": updated_at.astimezone().replace(tzinfo=None).isoformat(), "update_type": "incremental", - "gaps_filled": fill_gaps and len(gaps) > 0, + "gaps_filled": False, + "gap_check_performed": gap_check_performed, + "gap_count": len(gaps) if gap_check_performed else None, + "missing_observations": ( + sum(gap.missing_periods for gap in gaps) if gap_check_performed else None + ), "provider_history_limited": provider_history_limited, } ) diff --git a/src/ml4t/data/providers/databento.py b/src/ml4t/data/providers/databento.py index bedf2b13..16013104 100644 --- a/src/ml4t/data/providers/databento.py +++ b/src/ml4t/data/providers/databento.py @@ -25,7 +25,7 @@ NetworkError, RateLimitError, ) -from ml4t.data.providers.base import BaseProvider +from ml4t.data.providers.base import BaseProvider, provider_retry logger = structlog.get_logger() @@ -91,6 +91,8 @@ def __init__( rate_limit: tuple[int, float] | None = None, adjust_session_dates: bool = False, session_start_hour_utc: int = 0, + *, + stype_in: str = "raw_symbol", ): """Initialize Databento provider. @@ -100,6 +102,7 @@ def __init__( rate_limit: Optional custom rate limit (calls, period_seconds) adjust_session_dates: Whether to adjust dates for CME session logic session_start_hour_utc: Hour in UTC when trading session starts (for futures) + stype_in: Input symbology for OHLCV requests, such as continuous for BTC.c.0. """ super().__init__(rate_limit=rate_limit or self.DEFAULT_RATE_LIMIT) @@ -123,6 +126,7 @@ def __init__( self.default_schema = "ohlcv-1m" self.adjust_session_dates = adjust_session_dates self.session_start_hour_utc = session_start_hour_utc + self.stype_in = stype_in self.logger.info( "Initialized Databento provider", @@ -149,15 +153,15 @@ def _create_empty_dataframe(self) -> pl.DataFrame: } ) - def _map_frequency_to_schema(self, frequency: str) -> str: + def _map_frequency_to_schema(self, frequency: str, *, strict: bool = False) -> str: """Map frequency parameter to Databento schema.""" freq_lower = frequency.lower() - if freq_lower in ["daily", "day", "1d", "d"]: + if freq_lower in ["daily", "day", "1day", "1d", "d"]: return "ohlcv-1d" - if freq_lower in ["hourly", "hour", "1h", "h"]: + if freq_lower in ["hourly", "hour", "1hour", "1h", "h"]: return "ohlcv-1h" - if freq_lower in ["minute", "min", "1m", "m"]: + if freq_lower in ["minute", "min", "1minute", "1min", "1m", "m"]: return "ohlcv-1m" if freq_lower in ["tick", "trades"]: return "trades" @@ -166,6 +170,8 @@ def _map_frequency_to_schema(self, frequency: str) -> str: if freq_lower in ["mbo"]: return "mbo" + if strict: + raise ValueError(f"Unsupported targeted OHLCV frequency: {frequency}") # Default to daily return "ohlcv-1d" @@ -303,7 +309,7 @@ def _fetch_raw_data( end=end_dt, symbols=symbol, schema=schema, - stype_in="raw_symbol", + stype_in=self.stype_in, ) return response @@ -320,6 +326,56 @@ def _fetch_raw_data( message=f"Failed to fetch data from Databento: {e}", ) + @provider_retry + def fetch_ohlcv_range( + self, + symbol: str, + start: datetime, + end: datetime, + frequency: str = "minute", + *, + stype_in: str | None = None, + ) -> pl.DataFrame: + """Fetch one series within exact UTC instants, including start and excluding end. + + Session-date adjustment is never applied. Only native one-minute, one-hour and + daily OHLCV schemas are supported; no reaggregation or synthetic bars are produced. + """ + if ( + not isinstance(start, datetime) + or not isinstance(end, datetime) + or start.utcoffset() is None + or end.utcoffset() is None + ): + raise ValueError("start and end must be timezone-aware datetimes") + start, end = start.astimezone(UTC), end.astimezone(UTC) + if start >= end: + raise ValueError("start must be before end for a half-open range") + schema = self._map_frequency_to_schema(frequency, strict=True) + if schema not in {"ohlcv-1m", "ohlcv-1h", "ohlcv-1d"}: + raise ValueError(f"Unsupported targeted OHLCV frequency: {frequency}") + selected_stype = self.stype_in if stype_in is None else stype_in + if selected_stype not in {"raw_symbol", "continuous", "instrument_id"}: + raise ValueError("stype_in must select a single series, not a parent or universe") + if not isinstance(symbol, str) or symbol.strip().upper() == "ALL_SYMBOLS" or "," in symbol: + raise ValueError("symbol must identify a single instrument, not a universe") + self._validate_inputs(symbol, start.date().isoformat(), end.date().isoformat(), frequency) + self._acquire_rate_limit() + + def fetch() -> pl.DataFrame: + raw = self._fetch_timeseries( + dataset=self.dataset, + symbols=symbol, + schema=schema, + stype_in=selected_stype, + start=start, + end=end, + ) + frame = self._validate_ohlcv(self._transform_data(raw, symbol), self.name, symbol) + return frame.filter((pl.col("timestamp") >= start) & (pl.col("timestamp") < end)) + + return self._with_circuit_breaker(fetch) + def _transform_data(self, raw_data: Any, symbol: str) -> pl.DataFrame: """Transform Databento data to standard schema.""" try: diff --git a/src/ml4t/data/providers/protocols.py b/src/ml4t/data/providers/protocols.py index fe8ce7aa..8e308cc9 100644 --- a/src/ml4t/data/providers/protocols.py +++ b/src/ml4t/data/providers/protocols.py @@ -26,6 +26,8 @@ def fetch_data(provider: OHLCVProvider, symbol: str) -> pl.DataFrame: from typing import TYPE_CHECKING, Protocol, runtime_checkable if TYPE_CHECKING: + from datetime import datetime + import polars as pl @@ -105,6 +107,21 @@ def capabilities(self) -> ProviderCapabilities: ... +@runtime_checkable +class TimestampRangeProvider(Protocol): + """Optional single-series OHLCV capability with half-open aware timestamp bounds.""" + + def fetch_ohlcv_range( + self, + symbol: str, + start: datetime, + end: datetime, + frequency: str = "minute", + ) -> pl.DataFrame: + """Return actual source observations with start <= timestamp < end.""" + ... + + @runtime_checkable class FactorProvider(Protocol): """Protocol for academic factor data providers. diff --git a/src/ml4t/data/sessions/assigner.py b/src/ml4t/data/sessions/assigner.py index 9193a38d..723aa680 100644 --- a/src/ml4t/data/sessions/assigner.py +++ b/src/ml4t/data/sessions/assigner.py @@ -80,6 +80,22 @@ def _schedule_for_instants(calendar: Any, start: Any, end: Any, *, padding_days: } +def _uses_session_labels(df: pl.DataFrame, feed: FeedSpec, bar_frequency: str) -> bool: + if feed.timestamp_semantics is not None: + return feed.timestamp_semantics == TimestampSemantics.SESSION_LABEL + if feed.bar_label is not None: + return False + if df.schema["timestamp"] == pl.Date: + return True + timestamps = df["timestamp"].drop_nulls().to_list() + inferred_daily = bool(timestamps) and all( + value.hour == value.minute == value.second == value.microsecond == 0 + for value in timestamps + if isinstance(value, datetime) + ) + return bar_frequency == "daily" or (bar_frequency == "auto" and inferred_daily) + + def _daily_session_labels( df: pl.DataFrame, assigner: SessionAssigner, timestamp_col: str = "timestamp" ) -> pl.Series: @@ -320,24 +336,24 @@ def assign_sessions( timestamp_expr = timestamp_expr.dt.replace_time_zone( self.feed_spec.timezone or "UTC" ) - timestamp_expr = timestamp_expr.dt.convert_time_zone("UTC").cast( - pl.Datetime("us", "UTC") + comparison_unit = ( + "ns" + if isinstance(timestamp_dtype, pl.Datetime) and timestamp_dtype.time_unit == "ns" + else "us" ) + comparison_dtype = pl.Datetime(comparison_unit, "UTC") + timestamp_expr = timestamp_expr.dt.convert_time_zone("UTC").cast(comparison_dtype) - session_df = pl.DataFrame(session_map).sort("_calendar_session_end") - non_null_timestamps = df.get_column("timestamp").drop_nulls().to_list() - inferred_daily = bool(non_null_timestamps) and all( - value.hour == value.minute == value.second == value.microsecond == 0 - for value in non_null_timestamps - if isinstance(value, datetime) + session_df = ( + pl.DataFrame(session_map) + .with_columns( + pl.col("_calendar_session_start", "_calendar_session_end").cast( + comparison_dtype + ) + ) + .sort("_calendar_session_end") ) - daily_labels = bar_frequency == "daily" or (bar_frequency == "auto" and inferred_daily) - if semantics is not None: - daily_labels = semantics == TimestampSemantics.SESSION_LABEL - elif self.feed_spec.bar_label is not None: - daily_labels = False - elif timestamp_dtype == pl.Date: - daily_labels = True + daily_labels = _uses_session_labels(df, self.feed_spec, bar_frequency) if daily_labels: label_expr = pl.col("timestamp") if ( @@ -383,7 +399,7 @@ def assign_sessions( .with_columns(timestamp_expr.alias("_session_timestamp")) .with_columns( pl.col("_session_timestamp") - .fill_null(pl.lit(after_last_session)) + .fill_null(pl.lit(after_last_session, dtype=comparison_dtype)) .alias("_session_join_timestamp") ) .sort("_session_join_timestamp") diff --git a/src/ml4t/data/sessions/completer.py b/src/ml4t/data/sessions/completer.py index 36838df7..4ad3cdf2 100644 --- a/src/ml4t/data/sessions/completer.py +++ b/src/ml4t/data/sessions/completer.py @@ -1,7 +1,4 @@ -"""Session completion with gap filling for trading data. - -Ported from crypto-data-pipeline with enhancements for ml4t.data. -""" +"""Expose absent minute positions without manufacturing market observations.""" from __future__ import annotations @@ -28,21 +25,10 @@ def _datetime_scalar(value: object, name: str) -> datetime: class SessionCompleter: - """Fill gaps in trading data to create complete sessions. - - For each trading session: - 1. Generate all minute timestamps in session (e.g., 1380 for 23-hour CME sessions) - 2. Left join with actual data - 3. Forward fill OHLC prices from last close - 4. Set volume=0 for filled rows - 5. Add session_date column - - Example: - ```python - completer = SessionCompleter("CME_Globex_Crypto") - df_complete = completer.complete_sessions(df) - # Now has continuous timestamps with no gaps - ``` + """Left join observations to scheduled minutes, retaining absent values as nulls. + + The legacy is_imputed marker identifies placeholder rows, not fabricated prices. + Use the disk gap report instead when expansion into a nullable grid is unnecessary. """ def __init__( @@ -73,17 +59,17 @@ def complete_sessions( df: pl.DataFrame, start_date: datetime | None = None, end_date: datetime | None = None, - fill_method: str = "forward", - zero_volume: bool = True, + fill_method: str = "none", + zero_volume: bool = False, ) -> pl.DataFrame: - """Fill gaps in data to create complete trading sessions. + """Represent missing scheduled minutes with nulls, preserving observed values. Args: df: Input DataFrame with timestamp, open, high, low, close, volume start_date: Optional start date (auto-detected if not provided) end_date: Optional exclusive end date (auto-detected if not provided) - fill_method: Method for filling prices ("forward", "backward", "none") - zero_volume: If True, set volume=0 for filled rows; if False, use NaN + fill_method: Only "none" is supported; synthetic prices are rejected. + zero_volume: Must be False; an absent observation has no known volume. Returns: DataFrame with complete sessions, an is_imputed column, and timestamp order @@ -117,8 +103,11 @@ def complete_sessions( ) if "timestamp" not in df.columns: raise ValueError("DataFrame must have 'timestamp' column") - if fill_method not in {"forward", "backward", "none"}: - raise ValueError("fill_method must be 'forward', 'backward', or 'none'") + if fill_method != "none" or zero_volume: + raise ValueError( + "Synthetic price/volume imputation is unsupported; " + "use fill_method='none' and zero_volume=False" + ) if df.is_empty(): logger.warning("DataFrame is empty, cannot complete sessions") @@ -258,14 +247,6 @@ def complete_sessions( pl.col("_is_observed").is_null().alias("is_imputed") ) - # Fill missing data based on method - if fill_method != "none": - complete_df = self._fill_missing_data( - complete_df, - method=fill_method, - zero_volume=zero_volume, - ) - complete_df = complete_df.drop("_is_observed") rows_added = complete_df.get_column("is_imputed").sum() logger.info(f"Completed sessions: added {rows_added} rows ({len(complete_df)} total)") @@ -276,82 +257,6 @@ def complete_sessions( logger.error(f"Failed to complete sessions: {e}", exc_info=True) raise - def _fill_missing_data( - self, - df: pl.DataFrame, - method: str = "forward", - zero_volume: bool = True, - ) -> pl.DataFrame: - """Fill missing data with forward-filled prices and zero/NaN volume. - - Strategy: - 1. Forward/backward fill OHLC from last/next available close - 2. Set volume=0 (or NaN) for filled rows - 3. Preserve all other columns - - Args: - df: DataFrame with nulls in OHLCV columns - method: Fill method ("forward" or "backward") - zero_volume: If True, use 0 for missing volume; if False, keep NaN - - Returns: - DataFrame with filled data - """ - price_columns = ["open", "high", "low", "close"] - - reference_price: pl.Expr | None = None - reference_column = next((column for column in price_columns if column in df.columns), None) - if method == "forward": - if "close" in df.columns: - reference_column = "close" - if reference_column is not None: - reference_price = pl.col(reference_column).forward_fill() - elif method == "backward": - if "close" in df.columns: - reference_column = "close" - if reference_column is not None: - reference_price = pl.col(reference_column).backward_fill().over("session_date") - - filled_df = df - if reference_price is not None: - filled_df = filled_df.with_columns( - *[ - pl.when(pl.col("is_imputed")) - .then(reference_price) - .otherwise(pl.col(column)) - .alias(column) - for column in price_columns - if column in df.columns - ] - ) - - # Handle volume - if "volume" in df.columns: - if zero_volume: - filled_df = filled_df.with_columns( - pl.when(pl.col("is_imputed")) - .then(pl.lit(0.0)) - .otherwise(pl.col("volume")) - .alias("volume") - ) - # else: keep NaN for missing volume - - # Forward fill common metadata columns if present - metadata_columns = [ - "instrument_id", - "symbol", - "raw_symbol", - "base_symbol", - "rtype", - "publisher_id", - ] - - for col in metadata_columns: - if col in filled_df.columns: - filled_df = filled_df.with_columns(pl.col(col).forward_fill()) - - return filled_df - def get_session_info( self, date_or_datetime: datetime | date ) -> dict[str, datetime | date | None]: diff --git a/src/ml4t/data/update_manager.py b/src/ml4t/data/update_manager.py index b1a7d53a..02295914 100644 --- a/src/ml4t/data/update_manager.py +++ b/src/ml4t/data/update_manager.py @@ -3,6 +3,7 @@ from __future__ import annotations import time +from collections.abc import Callable from dataclasses import dataclass from datetime import UTC, date, datetime, timedelta from enum import Enum @@ -13,9 +14,11 @@ from ml4t.data.core.schemas import align_frames_for_concat from ml4t.data.providers.base import BaseProvider +from ml4t.data.providers.protocols import TimestampRangeProvider from ml4t.data.sessions.assigner import SessionAssigner, _daily_session_labels from ml4t.data.storage.backend import StorageBackend, normalize_storage_metadata from ml4t.data.storage.metadata_tracker import MetadataTracker, UpdateRecord +from ml4t.data.utils.gaps import GapDetector as ObservationGapDetector from ml4t.specs import UNSET_CALENDAR, CalendarUnset, FeedSpec logger = structlog.get_logger() @@ -76,6 +79,33 @@ class UpdateStrategy(Enum): BACKFILL = "backfill" # Fill gaps in historical data +def _merge_backfill_rows( + existing: pl.DataFrame, recovered: pl.DataFrame, feed: FeedSpec +) -> tuple[pl.DataFrame, int, int]: + if recovered.is_empty(): + return existing, 0, 0 + identity = ["timestamp"] + if feed.entity_col is not None or "symbol" in existing.columns: + entity = feed.resolve(existing.columns, ("symbol",)).entity_col + stored_ids = existing[entity].drop_nulls().unique() + if len(stored_ids) and not recovered[entity].is_in(stored_ids.implode()).all(): + raise ValueError("Backfill cannot introduce a different instrument identity") + if len(stored_ids) > 1: + identity.append(entity) + existing, recovered = align_frames_for_concat( + existing, recovered, right_fill_values={"is_imputed": False} + ) + recovered = recovered.unique(subset=identity, keep="first", maintain_order=True) + observed = ObservationGapDetector._observed_rows(existing) + recovered = recovered.join(observed.select(identity), on=identity, how="anti") + if recovered.is_empty(): + return existing, 0, 0 + retained = existing.join(recovered.select(identity), on=identity, how="anti") + replaced = len(existing) - len(retained) + combined = pl.concat([retained, recovered]).sort(identity) + return combined, len(recovered) - replaced, replaced + + @dataclass class UpdateResult: """Result of an update operation.""" @@ -89,6 +119,7 @@ class UpdateResult: gaps_filled: int = 0 duration_seconds: float = 0.0 errors: list[str] | None = None + remaining_gaps: list[dict[str, Any]] | None = None def __post_init__(self) -> None: """Initialize errors list if None.""" @@ -140,6 +171,76 @@ def _label_frame(df: pl.DataFrame, feed: FeedSpec, calendar: str | None) -> pl.D labels = _daily_session_labels(df, SessionAssigner(calendar, feed_spec=feed)) return df.with_columns(labels.cast(pl.Datetime("us")).dt.replace_time_zone("UTC")) + @staticmethod + def _by_entity( + df: pl.DataFrame, + feed: FeedSpec, + detect: Callable[[pl.DataFrame], list[dict[str, Any]]], + *, + storage: StorageBackend | None = None, + key: str | None = None, + ) -> list[dict[str, Any]]: + if feed.entity_col is None and "symbol" not in df.columns: + return detect(df) + if storage is not None and key is not None and not storage.exists(key): + return detect(df) + entity = ( + feed.with_overrides(timestamp_col="timestamp") + .resolve(df.columns, ("symbol",)) + .entity_col + ) + if not isinstance(entity, str): + raise ValueError("a single entity column is required") + identity_columns = [entity] + (["is_imputed"] if "is_imputed" in df.columns else []) + identity_frame = ( + storage.read(key).select(identity_columns).collect() + if storage is not None and key is not None + else df.select(identity_columns) + ) + identifiers = ObservationGapDetector._observed_rows(identity_frame)[entity].unique() + if identifiers.null_count(): + raise ValueError("entity identifiers must not be null") + if len(identifiers) <= 1: + return detect(df) + return [ + {**gap, "entity": identifier} + for identifier in identifiers.sort().to_list() + for gap in detect(df.filter(pl.col(entity) == identifier)) + ] + + @staticmethod + def _select_gap_rows( + data: pl.DataFrame, gap: dict[str, Any], feed: FeedSpec, calendar: str + ) -> pl.DataFrame: + actual = ObservationGapDetector._observed_rows(data) + if "entity" in gap: + entity = feed.resolve(actual.columns, ("symbol",)).entity_col + actual = actual.filter(pl.col(entity) == gap["entity"]) + frequency = feed.data_frequency or "daily" + if frequency.lower() in {"daily", "1d", "1day"}: + schedule = SessionAssigner(calendar).calendar.schedule( + start_date=gap["start"].date(), end_date=gap["end"].date() + ) + labels = GapDetector._label_frame(actual, feed, calendar)["timestamp"].dt.date() + return actual.filter(labels.is_in([label.date() for label in schedule.index])) + actual = actual.filter( + (pl.col("timestamp") >= gap["start"]) & (pl.col("timestamp") <= gap["end"]) + ) + if not actual.is_empty(): + grid, timestamps = ObservationGapDetector()._intraday_grid( + actual, + "timestamp", + frequency, + calendar, + feed, + start=gap["start"], + end=gap["end"], + ) + if grid is not None: + for timestamp in timestamps: + grid.position(timestamp) + return actual + @staticmethod def _daily_ranges( df: pl.DataFrame, @@ -153,7 +254,7 @@ def _daily_ranges( assigner = SessionAssigner(calendar) observed = { value.date() if isinstance(value, datetime) else value - for value in df["timestamp"].to_list() + for value in ObservationGapDetector._observed_rows(df)["timestamp"].to_list() } schedule = assigner.calendar.schedule(start_date=start.date(), end_date=end.date()) gaps = [] @@ -183,38 +284,83 @@ def append_gap() -> None: def detect_gaps( self, df: pl.DataFrame, - frequency: str = "daily", + frequency: str | None = None, tolerance_days: int = 0, *, calendar: str | None | CalendarUnset = UNSET_CALENDAR, feed_spec: Any | None = None, + tolerance_bars: int = 0, + start_date: datetime | None = None, + end_date: datetime | None = None, ) -> list[dict[str, Any]]: """ Detect gaps in a DataFrame's timestamp column. Args: df: DataFrame with timestamp column - frequency: Data frequency (daily, hourly, etc.) + frequency: Inherit the feed frequency when omitted, otherwise daily. tolerance_days: Missing sessions tolerated for daily data; elapsed days otherwise. calendar: Per-call override of configured calendar and feed metadata. feed_spec: Per-call feed declaration. + tolerance_bars: Number of consecutive missing intraday bars to tolerate. + start_date: Inclusive requested coverage start; supply together with end_date. + end_date: Inclusive coverage end. Daily bounds are labels; intraday bounds are instants. Returns: List of gap dictionaries with start, end, and size """ + if (start_date is None) != (end_date is None): + raise ValueError("start_date and end_date must be supplied together") feed, selected = self._feed_calendar(calendar, feed_spec) + frequency = (feed.data_frequency or "daily") if frequency is None else frequency if selected is None: return [] - if df.is_empty() or "timestamp" not in df.columns: + if df.is_empty() and start_date is None: return [] - if frequency.lower() in {"daily", "1day"}: - df = self._label_frame(df, feed, selected) - return self._daily_ranges( + if feed.timestamp_col != "timestamp": + if feed.timestamp_col not in df.columns: + raise ValueError(f"declared timestamp column {feed.timestamp_col!r} is missing") + df = df.select( + pl.col(feed.timestamp_col).alias("timestamp"), + *[name for name in df.columns if name not in {"timestamp", feed.timestamp_col}], + ) + feed = feed.with_overrides(timestamp_col="timestamp") + if "timestamp" not in df.columns: + if not df.is_empty(): + raise ValueError("DataFrame must have 'timestamp' column") + df = df.with_columns(pl.Series("timestamp", [], dtype=pl.Datetime("us", "UTC"))) + if frequency.lower() in {"daily", "1day", "1d"}: + + def daily_ranges(part: pl.DataFrame) -> list[dict[str, Any]]: + labeled = self._label_frame(part, feed, selected) + return self._daily_ranges( + labeled, + _session_label( + start_date if start_date is not None else labeled["timestamp"].min() + ), + _session_label( + end_date if end_date is not None else labeled["timestamp"].max() + ), + selected, + tolerance_days, + ) + + return self._by_entity(df, feed, daily_ranges) + if ObservationGapDetector._intraday_interval(frequency) is not None: + if tolerance_days: + raise ValueError("use tolerance_bars, not tolerance_days, for intraday data") + return self._by_entity( df, - _session_label(df["timestamp"].min()), - _session_label(df["timestamp"].max()), - selected, - tolerance_days, + feed, + lambda part: self._intraday_ranges( + part, + frequency, + selected, + feed, + tolerance_bars=tolerance_bars, + start=_ensure_datetime(start_date) if start_date is not None else None, + end=_ensure_datetime(end_date) if end_date is not None else None, + ), ) if ( calendar is not UNSET_CALENDAR @@ -252,13 +398,52 @@ def detect_gaps( return gaps + @staticmethod + def _intraday_ranges( + df: pl.DataFrame, + frequency: str, + calendar: str, + feed: FeedSpec, + *, + start: datetime | None = None, + end: datetime | None = None, + tolerance_bars: int = 0, + ) -> list[dict[str, Any]]: + if ( + isinstance(tolerance_bars, bool) + or not isinstance(tolerance_bars, int) + or tolerance_bars < 0 + ): + raise ValueError("tolerance_bars must be a nonnegative integer") + grid, observed = ObservationGapDetector()._intraday_grid( + df, "timestamp", frequency, calendar, feed, start=start, end=end + ) + if grid is None: + return [] + positions = sorted({grid.position(timestamp) for timestamp in observed}) + first = grid.bound(grid.start) + stop = grid.bound(grid.end, inclusive=True) + gaps = [] + cursor = first + for position in [p for p in positions if first <= p < stop] + [stop]: + if position - cursor > tolerance_bars: + gaps.append( + { + "start": grid.timestamp(cursor), + "end": grid.timestamp(position - 1), + "size_bars": position - cursor, + } + ) + cursor = position + 1 + return gaps + def detect_gaps_in_storage( self, storage: StorageBackend, key: str, start_date: datetime, end_date: datetime, - frequency: str = "daily", + frequency: str | None = None, *, calendar: str | None | CalendarUnset = UNSET_CALENDAR, feed_spec: Any | None = None, @@ -271,22 +456,24 @@ def detect_gaps_in_storage( key: Storage key for the dataset start_date: Start of range to check end_date: End of range to check - frequency: Data frequency + frequency: Inherit the stored/feed frequency when omitted, otherwise daily. Returns: - List of gaps found in storage, with UTC-aware bounds + Read-only gap report with inclusive UTC-aware bounds. Daily size_days counts + sessions; intraday size_bars counts expected bars, including boundary absences. """ - if frequency.lower() in {"daily", "1day"}: + if feed_spec is None and self.feed_spec is None: + try: + feed_spec = normalize_storage_metadata(storage.get_metadata(key), key) + except KeyError: + pass + feed, selected = self._feed_calendar(calendar, feed_spec) + frequency = (feed.data_frequency or "daily") if frequency is None else frequency + if selected is None: + return [] + if frequency.lower() in {"daily", "1day", "1d"}: start_date = _session_label(start_date) end_date = _session_label(end_date) - if feed_spec is None and self.feed_spec is None: - try: - feed_spec = normalize_storage_metadata(storage.get_metadata(key), key) - except KeyError: - pass - feed, selected = self._feed_calendar(calendar, feed_spec) - if selected is None: - return [] # Session labels retain their civil date even when stored with a timezone. start_bound = start_date - timedelta(days=1) end_bound = datetime.combine(end_date.date(), datetime.min.time(), UTC) + timedelta( @@ -296,14 +483,33 @@ def detect_gaps_in_storage( df = storage.read(key, start_bound, end_bound).collect() except KeyError: df = pl.DataFrame(schema={"timestamp": pl.Datetime("us", "UTC")}) - return self._daily_ranges( - self._label_frame(df, feed, selected), start_date, end_date, selected + return self._by_entity( + df, + feed, + lambda part: self._daily_ranges( + self._label_frame(part, feed, selected), start_date, end_date, selected + ), + storage=storage, + key=key, ) start_date = _ensure_datetime(start_date) end_date = _ensure_datetime(end_date) - _, selected = self._feed_calendar(calendar, feed_spec) - if selected is None: - return [] + if start_date > end_date: + raise ValueError("start_date must not be after end_date") + if ObservationGapDetector._intraday_interval(frequency) is not None: + try: + df = storage.read(key, start_date, end_date + timedelta(microseconds=1)).collect() + except KeyError: + df = pl.DataFrame(schema={"timestamp": pl.Datetime("us", "UTC")}) + return self._by_entity( + df, + feed, + lambda part: self._intraday_ranges( + part, frequency, selected, feed, start=start_date, end=end_date + ), + storage=storage, + key=key, + ) if ( calendar is not UNSET_CALENDAR or self.calendar is not UNSET_CALENDAR @@ -374,8 +580,7 @@ def _get_expected_delta(self, frequency: str) -> timedelta: return timedelta(hours=1) if frequency == "weekly": return timedelta(weeks=1) - # Default to daily - return timedelta(days=1) + raise ValueError(f"unsupported frequency: {frequency!r}") class IncrementalUpdater: @@ -400,6 +605,9 @@ def determine_update_range( key: str, requested_start: datetime, requested_end: datetime, + *, + calendar: str | None | CalendarUnset = UNSET_CALENDAR, + feed_spec: Any | None = None, ) -> tuple[datetime, datetime, str]: """ Determine the actual range to update based on existing data. @@ -415,43 +623,43 @@ def determine_update_range( """ requested_start = _ensure_datetime(requested_start) requested_end = _ensure_datetime(requested_end) + if requested_start > requested_end: + raise ValueError("requested_start must not be after requested_end") try: # Check if data exists if not storage.exists(key): return requested_start, requested_end, "full" # Get existing data range - df = storage.read(key).select("timestamp").collect() - - if df.is_empty(): - return requested_start, requested_end, "full" - - last_timestamp = _ensure_datetime(df["timestamp"].max()) - # If requested end is before last data, no update needed - if requested_end <= last_timestamp: - logger.info( - f"Data for {key} already up to date", - last_timestamp=last_timestamp, - requested_end=requested_end, - ) - return requested_end, requested_end, "none" - - # Incremental update from day after last data - actual_start = last_timestamp + timedelta(days=1) - - logger.info( - f"Incremental update for {key}", - last_data=last_timestamp, - update_from=actual_start, - update_to=requested_end, - ) - - return actual_start, requested_end, "incremental" - + df = storage.read(key).collect() + metadata = normalize_storage_metadata(storage.get_metadata(key), key) except Exception as e: logger.warning(f"Error checking existing data: {e}") return requested_start, requested_end, "full" + observed = ObservationGapDetector._observed_rows(df) + if observed.is_empty(): + return requested_start, requested_end, "full" + feed, selected = self.gap_detector._feed_calendar( + calendar, metadata if feed_spec is None else feed_spec + ) + frequency = feed.data_frequency or "daily" + if selected is not None and frequency.lower() in {"daily", "1d", "1day"}: + observed = self.gap_detector._label_frame(observed, feed, selected) + last_timestamp = _ensure_datetime(observed["timestamp"].max()) + if requested_end <= last_timestamp: + return requested_end, requested_end, "none" + lower = max(last_timestamp, requested_start) + if selected is None: + # Without a schedule, overlap the last real row rather than invent a next bar. + return lower, requested_end, "incremental" + gaps = self.gap_detector.detect_gaps_in_storage( + storage, key, lower, requested_end, calendar=selected, feed_spec=feed + ) + if not gaps: + return requested_end, requested_end, "none" + return min(gap["start"] for gap in gaps), requested_end, "incremental" + def update_incremental( self, storage: StorageBackend, @@ -653,40 +861,36 @@ def apply_strategy( feed, selected_calendar = self.gap_detector._feed_calendar( UNSET_CALENDAR, feed_metadata ) - gaps_filled = 0 - - # Filter new data to only gap periods - gap_data = pl.DataFrame() - for gap in gaps: - expected = SessionAssigner(selected_calendar).calendar.schedule( - start_date=gap["start"].date(), end_date=gap["end"].date() - ) - gap_rows = new_data.filter( - self.gap_detector._label_frame(new_data, feed, selected_calendar)[ - "timestamp" - ] - .dt.date() - .is_in([label.date() for label in expected.index]) - ) - if not gap_rows.is_empty(): - gap_data = ( - pl.concat([gap_data, gap_rows]) if not gap_data.is_empty() else gap_rows - ) - gaps_filled += 1 + parts = [ + self.gap_detector._select_gap_rows(new_data, gap, feed, selected_calendar) + for gap in gaps + if selected_calendar is not None + ] + gap_data = pl.concat(parts) if parts else new_data.head(0) if not gap_data.is_empty(): - # Merge with existing data - combined = pl.concat([existing_df, gap_data]).sort("timestamp") - _rewrite_preserving_metadata(storage, key, combined) + combined, added, replaced = _merge_backfill_rows(existing_df, gap_data, feed) + if added or replaced: + _rewrite_preserving_metadata(storage, key, combined) + remaining = self.gap_detector.detect_gaps(combined, feed_spec=feed_metadata) return UpdateResult( success=True, update_type=strategy.value, - rows_added=len(gap_data), - rows_updated=0, + rows_added=added, + rows_updated=replaced, rows_before=len(existing_df), rows_after=len(combined), - gaps_filled=gaps_filled, + gaps_filled=sum( + not any( + missing.get("entity") == gap.get("entity") + and missing["start"] <= gap["end"] + and missing["end"] >= gap["start"] + for missing in remaining + ) + for gap in gaps + ), + remaining_gaps=remaining, ) return UpdateResult( success=True, @@ -696,6 +900,7 @@ def apply_strategy( rows_before=len(existing_df), rows_after=len(existing_df), gaps_filled=0, + remaining_gaps=gaps, ) storage.write(new_data, key) return UpdateResult( @@ -787,6 +992,8 @@ def identify_candidates( self, min_gap_days: int = 5, max_age_days: int = 30, + *, + min_gap_bars: int = 1, ) -> list[dict[str, Any]]: """ Identify datasets that need backfilling. @@ -794,6 +1001,7 @@ def identify_candidates( Args: min_gap_days: Minimum gap size to consider for backfill max_age_days: Maximum age of data to consider + min_gap_bars: Minimum expected-bar count for intraday datasets, not elapsed days. Returns: List of backfill candidates with gap information @@ -822,16 +1030,23 @@ def identify_candidates( gaps = self.gap_detector.detect_gaps(df, feed_spec=feed_metadata) # Filter significant gaps - significant_gaps = [g for g in gaps if g["size_days"] >= min_gap_days] + significant_gaps = [ + gap + for gap in gaps + if gap.get("size_bars", gap.get("size_days", 0)) + >= (min_gap_bars if "size_bars" in gap else min_gap_days) + ] if significant_gaps: - total_gap_days = sum(g["size_days"] for g in significant_gaps) + intraday = "size_bars" in significant_gaps[0] + size_key = "size_bars" if intraday else "size_days" + total_key = "total_gap_bars" if intraday else "total_gap_days" candidates.append( { "symbol": key, "gaps": significant_gaps, - "total_gap_days": total_gap_days, + total_key: sum(g[size_key] for g in significant_gaps), "last_update": last_update, "importance": "normal", # Could be enhanced with business logic } @@ -860,7 +1075,7 @@ def prioritize_tasks( score = 0 # Factor 1: Gap size (larger gaps = higher priority) - score += candidate["total_gap_days"] * 2 + score += candidate.get("total_gap_days", candidate.get("total_gap_bars", 0)) * 2 # Factor 2: Data staleness (older last update = higher priority) if "last_update" in candidate: @@ -887,97 +1102,179 @@ def execute_backfill( symbol: str, gaps: list[dict[str, Any]], provider: BaseProvider, + *, + source_symbol: str | None = None, + calendar: str | None | CalendarUnset = UNSET_CALENDAR, + feed_spec: Any | None = None, ) -> UpdateResult: - """ - Execute backfill operation for identified gaps. - - Args: - symbol: Symbol to backfill - gaps: List of gaps to fill - provider: Data provider to fetch from + """Download real observations for selected missing ranges in one stored series. - Returns: - UpdateResult with backfill details + The symbol argument is the storage key. Supply source_symbol separately for a + path-like key or a vendor alias. Intraday recovery requires the provider's + optional timestamp-range capability and a matching left-labeled feed grid. + Success describes processing, not completeness; inspect remaining_gaps. """ - total_rows_added = 0 - gaps_filled = 0 - errors = [] - start_time = time.time() - + started = time.time() + errors: list[str] = [] + rows_before = rows_after = added = replaced = filled = 0 + remaining = None try: - # Fetch data for each gap - all_gap_data = pl.DataFrame() - - for gap in gaps: - try: - logger.info( - f"Backfilling gap for {symbol}", - start=gap["start"], - end=gap["end"], - size_days=gap["size_days"], + if self.storage is None: + raise ValueError("Backfill requires a storage backend") + existing = ( + self.storage.read(symbol).collect() + if self.storage.exists(symbol) + else pl.DataFrame(schema={"timestamp": pl.Datetime("us", "UTC")}) + ) + rows_before = rows_after = len(existing) + if not gaps: + return UpdateResult( + True, "backfill", 0, 0, rows_before, rows_after, remaining_gaps=[] + ) + metadata = ( + normalize_storage_metadata(self.storage.get_metadata(symbol), symbol) or {} + if self.storage.exists(symbol) + else {} + ) + feed, selected = self.gap_detector._feed_calendar( + calendar, metadata if feed_spec is None else feed_spec + ) + if selected is None: + raise ValueError("calendar=None disables calendar-based recovery") + frequency = feed.data_frequency or "daily" + interval = ObservationGapDetector._intraday_interval(frequency) + if interval is not None: + if not isinstance(provider, TimestampRangeProvider): + raise NotImplementedError( + f"{provider.name} does not support exact timestamp-range downloads" ) - - # Fetch data from provider - gap_data = provider.fetch_ohlcv( + if feed.bar_label != "left": + raise NotImplementedError("Targeted OHLCV recovery requires left-labeled bars") + stored_provider = metadata.get("provider") + if stored_provider and str(stored_provider).lower() not in { + provider.name.lower(), + provider.__class__.__name__.lower(), + }: + raise ValueError("Selected provider does not match the stored dataset provider") + source_symbol = source_symbol or metadata.get("source_symbol") + if source_symbol is None: + if "/" in symbol: + raise ValueError("source_symbol is required separately from the storage key") + source_symbol = symbol + if not isinstance(source_symbol, str) or not source_symbol.strip(): + raise ValueError("source_symbol must identify one vendor instrument") + entity = None + entity_value = None + if feed.entity_col is not None or "symbol" in existing.columns: + entity = feed.resolve(existing.columns, ("symbol",)).entity_col + if not isinstance(entity, str): + raise ValueError("Targeted recovery requires one entity column") + source_rows = ObservationGapDetector._observed_rows(existing) + identifiers = existing[entity].drop_nulls().unique() + if source_rows[entity].null_count() or len(identifiers) > 1: + raise ValueError("Targeted recovery requires one instrument per storage key") + entity_value = identifiers[0] if len(identifiers) else source_symbol.upper() + + def inspect(ranges: list[dict[str, Any]]) -> list[dict[str, Any]]: + report: list[dict[str, Any]] = [] + for gap in ranges: + for missing in self.gap_detector.detect_gaps_in_storage( + self.storage, symbol, - gap["start"].strftime("%Y-%m-%d"), - gap["end"].strftime("%Y-%m-%d"), - "daily", - ) - - if not gap_data.is_empty(): - all_gap_data = ( - pl.concat([all_gap_data, gap_data]) - if not all_gap_data.is_empty() - else gap_data + gap["start"], + gap["end"], + frequency, + calendar=selected, + feed_spec=feed, + ): + if missing not in report: + report.append(missing) + return report + + fresh = inspect(gaps) + remaining = fresh + parts: list[pl.DataFrame] = [] + for gap in fresh: + try: + if interval is not None and isinstance(provider, TimestampRangeProvider): + data = provider.fetch_ohlcv_range( + source_symbol, gap["start"], gap["end"] + interval, frequency ) - gaps_filled += 1 - total_rows_added += len(gap_data) - - except Exception as e: - error_msg = f"Failed to fetch gap {gap['start']} - {gap['end']}: {e}" - errors.append(error_msg) - logger.warning(error_msg) - - # Apply backfill update if we got any data - if not all_gap_data.is_empty() and self.storage and self.tracker: - updater = IncrementalUpdater(strategy=UpdateStrategy.BACKFILL) - result = updater.update_incremental( - self.storage, - self.tracker, - symbol, - all_gap_data, - provider=provider.__class__.__name__, - strategy=UpdateStrategy.BACKFILL, + else: + data = provider.fetch_ohlcv( + source_symbol, + gap["start"].date().isoformat(), + gap["end"].date().isoformat(), + frequency, + ) + if not isinstance(data, pl.DataFrame): + raise TypeError("Provider must return a DataFrame") + if data.is_empty(): + continue + if "symbol" in data.columns: + identities = data["symbol"].unique().to_list() + if identities != [source_symbol.upper()]: + raise ValueError("Provider returned a different source instrument") + if "symbol" in existing.columns and not existing.is_empty(): + symbols = existing["symbol"].drop_nulls().unique() + target_symbol = ( + symbols[0] if len(symbols) == 1 else source_symbol.upper() + ) + data = data.with_columns(pl.lit(target_symbol).alias("symbol")) + if entity is not None and not existing.is_empty(): + data = data.with_columns(pl.lit(entity_value).alias(entity)) + recovered = self.gap_detector._select_gap_rows(data, gap, feed, selected) + if not recovered.is_empty(): + parts.append(recovered) + except Exception as error: + errors.append(f"Failed to fetch gap {gap['start']} - {gap['end']}: {error}") + + if parts: + recovered = pl.concat(parts, how="diagonal_relaxed") + combined, proposed_added, proposed_replaced = _merge_backfill_rows( + existing, recovered, feed ) - - result.gaps_filled = gaps_filled - return result - - return UpdateResult( - success=gaps_filled > 0, - update_type="backfill", - rows_added=total_rows_added, - rows_updated=0, - rows_before=0, - rows_after=0, - gaps_filled=gaps_filled, - duration_seconds=time.time() - start_time, - errors=errors if errors else None, - ) - - except Exception as e: - error_msg = f"Backfill failed for {symbol}: {e}" - errors.append(error_msg) - logger.error(error_msg) - - return UpdateResult( - success=False, - update_type="backfill", - rows_added=0, - rows_updated=0, - rows_before=0, - rows_after=0, - duration_seconds=time.time() - start_time, - errors=errors, - ) + if proposed_added or proposed_replaced: + remaining = None + _rewrite_preserving_metadata(self.storage, symbol, combined) + added, replaced = proposed_added, proposed_replaced + rows_after = len(combined) + if self.tracker is not None: + try: + self.tracker.update_metadata( + symbol, + UpdateRecord( + timestamp=datetime.now(), + update_type="backfill", + rows_before=rows_before, + rows_after=rows_after, + rows_added=added, + rows_updated=replaced, + start_date=_ensure_datetime(recovered["timestamp"].min()), + end_date=_ensure_datetime(recovered["timestamp"].max()), + provider=provider.name, + duration_seconds=time.time() - started, + errors=errors, + ), + rows_after, + _ensure_datetime(combined["timestamp"].min()), + _ensure_datetime(combined["timestamp"].max()), + ) + except Exception as error: + errors.append(f"Observations stored but update history failed: {error}") + remaining = inspect(gaps) + filled = sum(not inspect([gap]) for gap in fresh) + except Exception as error: + errors.append(f"Backfill failed for {symbol}: {error}") + return UpdateResult( + success=not errors, + update_type="backfill", + rows_added=added, + rows_updated=replaced, + rows_before=rows_before, + rows_after=rows_after, + gaps_filled=filled, + duration_seconds=time.time() - started, + errors=errors, + remaining_gaps=remaining, + ) diff --git a/src/ml4t/data/utils/gaps.py b/src/ml4t/data/utils/gaps.py index 9aa863ee..a0cefc56 100644 --- a/src/ml4t/data/utils/gaps.py +++ b/src/ml4t/data/utils/gaps.py @@ -5,7 +5,7 @@ import re from bisect import bisect_left, bisect_right from dataclasses import dataclass -from datetime import UTC, date, datetime, timedelta +from datetime import UTC, datetime, timedelta from typing import Any, ClassVar import polars as pl @@ -53,6 +53,66 @@ def __str__(self) -> str: ) +class _IntradayGrid: + def __init__( + self, + intervals: list[tuple[datetime, datetime, int, int]], + interval: timedelta, + right_labeled: bool, + include_partial: bool, + *, + start: datetime, + end: datetime, + ) -> None: + self.intervals = intervals + self.interval = interval + self.right_labeled = right_labeled + self.include_partial = include_partial + self.start = start + self.end = end + self.boundaries = [row[1] if right_labeled else row[0] for row in intervals] + self.offsets = [row[3] for row in intervals] + self.size = intervals[-1][3] + intervals[-1][2] if intervals else 0 + + def position(self, timestamp: datetime) -> int: + index = ( + bisect_left(self.boundaries, timestamp) + if self.right_labeled + else bisect_right(self.boundaries, timestamp) - 1 + ) + if index < 0 or index >= len(self.intervals): + raise ValueError("intraday observations contain off-grid timestamps") + opening, closing, count, offset = self.intervals[index] + if self.right_labeled and timestamp == closing: + if (closing - opening) % self.interval == timedelta(0) or self.include_partial: + return offset + count - 1 + raise ValueError("intraday observations contain off-grid partial-close timestamps") + position, remainder = divmod(timestamp - opening, self.interval) + if self.right_labeled: + position -= 1 + if remainder or timestamp < opening or timestamp >= closing or not 0 <= position < count: + raise ValueError("intraday observations contain off-grid timestamps") + return offset + position + + def timestamp(self, position: int) -> datetime: + if not 0 <= position < self.size: + raise IndexError("bar position outside the expected grid") + opening, closing, _, offset = self.intervals[bisect_right(self.offsets, position) - 1] + timestamp = opening + (position - offset + int(self.right_labeled)) * self.interval + return min(timestamp, closing) if self.right_labeled else timestamp + + def bound(self, timestamp: datetime, *, inclusive: bool = False) -> int: + left, right = 0, self.size + while left < right: + middle = (left + right) // 2 + candidate = self.timestamp(middle) + if candidate < timestamp or (inclusive and candidate == timestamp): + left = middle + 1 + else: + right = middle + return left + + class GapDetector: """Detect and analyze gaps in time series data.""" @@ -121,6 +181,7 @@ def detect_gaps( selected_calendar = feed.resolve_calendar(calendar=calendar) if selected_calendar is None: return [] + df = self._observed_rows(df) declared_interval = ( self._intraday_interval(str(feed.data_frequency)) if feed.data_frequency is not None @@ -189,6 +250,14 @@ def detect_gaps( return gaps + @staticmethod + def _observed_rows(df: pl.DataFrame) -> pl.DataFrame: + if "is_imputed" not in df.columns: + return df + if df.schema["is_imputed"] != pl.Boolean: + raise TypeError("is_imputed must be a Boolean placeholder marker") + return df.filter(~pl.col("is_imputed").fill_null(False)) + @staticmethod def _intraday_interval(frequency: str) -> timedelta | None: value = frequency.lower().strip().replace("hourly", "hour") @@ -200,14 +269,17 @@ def _intraday_interval(frequency: str) -> timedelta | None: raise ValueError("bar frequency must be positive") return timedelta(hours=count) if match[2].startswith("h") else timedelta(minutes=count) - def _detect_intraday_gaps( + def _intraday_grid( self, df: pl.DataFrame, timestamp_col: str, frequency: str, calendar: str, feed: FeedSpec, - ) -> list[DataGap]: + *, + start: datetime | None = None, + end: datetime | None = None, + ) -> tuple[_IntradayGrid | None, list[datetime]]: interval = self._intraday_interval(frequency) if interval is None: raise ValueError("unsupported intraday bar frequency") @@ -234,20 +306,33 @@ def _detect_intraday_gaps( raise TypeError(f"'{timestamp_col}' must contain Datetime values for an intraday grid") if df[timestamp_col].null_count(): raise ValueError(f"'{timestamp_col}' must not contain null timestamps") - if ( - dtype.time_unit == "ns" - and df.select((pl.col(timestamp_col).dt.nanosecond() % 1000 != 0).any()).item() - ): - raise ValueError("intraday observations contain off-grid sub-microsecond timestamps") if dtype.time_zone is None and feed.timezone is None: raise ValueError("naive intraday timestamps require a declared timezone") - if df.is_empty(): - return [] expr = pl.col(timestamp_col) if dtype.time_zone is None: expr = expr.dt.replace_time_zone(feed.timezone) - observed = df.select(expr.dt.convert_time_zone("UTC")).to_series().unique().sort().to_list() - start, end = observed[0], observed[-1] + columns = [expr.dt.convert_time_zone("UTC").alias(timestamp_col)] + if "is_imputed" in df.columns: + columns.append(pl.col("is_imputed")) + scoped = df.select(columns) + if start is not None: + scoped = scoped.filter(pl.col(timestamp_col) >= start) + if end is not None: + scoped = scoped.filter(pl.col(timestamp_col) <= end) + timestamps = scoped[timestamp_col].unique().sort().to_list() + if not timestamps and (start is None or end is None): + return None, [] + start = start if start is not None else timestamps[0] + end = end if end is not None else timestamps[-1] + if start > end: + raise ValueError("start_date must not be after end_date") + actual = self._observed_rows(scoped) + if ( + dtype.time_unit == "ns" + and actual.select((pl.col(timestamp_col).dt.nanosecond() % 1000 != 0).any()).item() + ): + raise ValueError("intraday observations contain off-grid sub-microsecond timestamps") + observed = actual[timestamp_col].unique().sort().to_list() schedule = _schedule_for_instants(assigner.calendar, start, end, padding_days=2) grid_intervals: list[tuple[datetime, datetime, int, int]] = [] total_bars = 0 @@ -263,37 +348,30 @@ def _detect_intraday_gaps( raise ValueError("calendar returned overlapping tradable intervals") grid_intervals.append((opening, closing, count, total_bars)) total_bars += count - right_labeled = feed.bar_label == BarLabel.RIGHT - boundaries = [row[1] if right_labeled else row[0] for row in grid_intervals] - - def bar_index(timestamp: datetime) -> int: - index = ( - bisect_left(boundaries, timestamp) - if right_labeled - else bisect_right(boundaries, timestamp) - 1 - ) - if index < 0 or index >= len(grid_intervals): - raise ValueError("intraday observations contain off-grid timestamps") - opening, closing, count, offset = grid_intervals[index] - if right_labeled and timestamp == closing: - if (closing - opening) % interval == timedelta(0) or ( - feed.partial_bar_policy == PartialBarPolicy.INCLUDE - ): - return offset + count - 1 - raise ValueError("intraday observations contain off-grid partial-close timestamps") - position, remainder = divmod(timestamp - opening, interval) - if right_labeled: - position -= 1 - if ( - remainder - or timestamp < opening - or timestamp >= closing - or not 0 <= position < count - ): - raise ValueError("intraday observations contain off-grid timestamps") - return offset + position - - positions = [bar_index(timestamp) for timestamp in observed] + return ( + _IntradayGrid( + grid_intervals, + interval, + feed.bar_label == BarLabel.RIGHT, + feed.partial_bar_policy == PartialBarPolicy.INCLUDE, + start=start, + end=end, + ), + observed, + ) + + def _detect_intraday_gaps( + self, + df: pl.DataFrame, + timestamp_col: str, + frequency: str, + calendar: str, + feed: FeedSpec, + ) -> list[DataGap]: + grid, observed = self._intraday_grid(df, timestamp_col, frequency, calendar, feed) + if grid is None: + return [] + positions = [grid.position(timestamp) for timestamp in observed] gaps = [] for current, following, left_index, right_index in zip( observed, observed[1:], positions, positions[1:] @@ -401,74 +479,14 @@ def fill_gaps( self, df: pl.DataFrame, gaps: list[DataGap], - method: str = "forward", + method: str = "none", timestamp_col: str = "timestamp", ) -> pl.DataFrame: - """ - Fill detected gaps in data. - - Args: - df: DataFrame with gaps - gaps: List of detected gaps - method: Fill method ('forward', 'backward', 'interpolate', 'zero') - timestamp_col: Name of timestamp column - - Returns: - DataFrame with gaps filled - """ + """Reject synthetic market values; an empty gap list remains a no-op.""" if not gaps: return df - - logger.info( - f"Filling {len(gaps)} gaps using {method} method", - total_periods=sum(g.missing_periods for g in gaps), + raise ValueError( + f"Synthetic gap imputation ({method!r}, {timestamp_col!r}) is unsupported; " + "inspect missing observations " + "and download available source data instead" ) - - # Create a complete time index - min_ts = df[timestamp_col].min() - max_ts = df[timestamp_col].max() - if not isinstance(min_ts, date) or not isinstance(max_ts, date): - raise TypeError(f"'{timestamp_col}' must contain non-null Date or Datetime values") - - if gaps: - frequency = gaps[0].frequency - interval = self._get_expected_interval(frequency) - - # Generate complete timestamp range - complete_timestamps = [] - current = min_ts - while current <= max_ts: - complete_timestamps.append(current) - current = current + interval - - # Create DataFrame with complete timestamps - complete_df = pl.DataFrame({timestamp_col: complete_timestamps}) - - # Cast to match original datetime precision to avoid join errors - original_dtype = df[timestamp_col].dtype - complete_df = complete_df.with_columns(pl.col(timestamp_col).cast(original_dtype)) - - # Join with original data - filled_df = complete_df.join(df, on=timestamp_col, how="left") - - # Fill missing values based on method - if method == "forward": - filled_df = filled_df.fill_null(strategy="forward") - elif method == "backward": - filled_df = filled_df.fill_null(strategy="backward") - elif method == "interpolate": - # Interpolate numeric columns - numeric_cols = [ - col - for col in filled_df.columns - if col != timestamp_col - and filled_df[col].dtype in [pl.Float32, pl.Float64, pl.Int32, pl.Int64] - ] - for col in numeric_cols: - filled_df = filled_df.with_columns(pl.col(col).interpolate()) - elif method == "zero": - filled_df = filled_df.fill_null(0) - - return filled_df - - return df diff --git a/src/ml4t/data/validation/cross_validation.py b/src/ml4t/data/validation/cross_validation.py index 53e014f2..60c80de7 100644 --- a/src/ml4t/data/validation/cross_validation.py +++ b/src/ml4t/data/validation/cross_validation.py @@ -9,6 +9,7 @@ import structlog from ml4t.data.validation.base import Severity, ValidationIssue, ValidationResult, Validator +from ml4t.specs import UNSET_CALENDAR, CalendarUnset, FeedSpec logger = structlog.get_logger() @@ -31,6 +32,9 @@ def __init__( volume_spike_threshold: float = 10.0, # 10x average volume price_gap_threshold: float = 0.1, # 10% price gap is_crypto: bool = False, # Crypto trades 24/7 + *, + calendar: str | None | CalendarUnset = UNSET_CALENDAR, + feed_spec: Any | None = None, ) -> None: """ Initialize cross-validator. @@ -51,13 +55,22 @@ def __init__( self.volume_spike_threshold = volume_spike_threshold self.price_gap_threshold = price_gap_threshold self.is_crypto = is_crypto + self.calendar = calendar + self.feed_spec = FeedSpec.from_any(feed_spec) def name(self) -> str: """Return validator name.""" return "CrossValidator" def validate( - self, df: pl.DataFrame, reference_df: pl.DataFrame | None = None, **kwargs: Any + self, + df: pl.DataFrame, + reference_df: pl.DataFrame | None = None, + *, + frequency: str | None = None, + calendar: str | None | CalendarUnset = UNSET_CALENDAR, + feed_spec: Any | None = None, + **kwargs: Any, ) -> ValidationResult: """ Validate DataFrame with cross-validation checks. @@ -85,11 +98,20 @@ def validate( if self.check_volume_spikes: self._check_volume_spikes(df, result) - if self.check_weekend_trading and not self.is_crypto: - self._check_weekend_trading(df, result) - - if self.check_market_hours and not self.is_crypto: - self._check_market_hours(df, result) + feed = self.feed_spec if feed_spec is None else FeedSpec.from_any(feed_spec) + if calendar is UNSET_CALENDAR: + calendar = self.calendar + if ( + calendar is UNSET_CALENDAR + and self.is_crypto + and ( + feed.calendar is UNSET_CALENDAR + or (feed.time_contract_version == 1 and feed.calendar is None) + ) + ): + calendar = "24/7" + if self.check_weekend_trading or self.check_market_hours: + self._check_sessions(df, result, calendar, feed, frequency) # Cross-reference validation if reference data provided if reference_df is not None: @@ -171,59 +193,71 @@ def _check_volume_spikes(self, df: pl.DataFrame, result: ValidationResult) -> No ) ) - def _check_weekend_trading(self, df: pl.DataFrame, result: ValidationResult) -> None: - """Check for weekend trading (Saturday/Sunday) for non-crypto assets.""" - # Add day of week column (Polars: 1 = Monday, 7 = Sunday) - df_with_dow = df.with_columns(pl.col("timestamp").dt.weekday().alias("day_of_week")) + def _check_sessions( + self, + df: pl.DataFrame, + result: ValidationResult, + calendar: str | None | CalendarUnset, + feed: FeedSpec, + frequency: str | None, + ) -> None: + from ml4t.data.sessions import SessionAssigner + from ml4t.data.sessions.assigner import _uses_session_labels - # Find weekend trading (Saturday = 6, Sunday = 7 in Polars) - weekend_trading = df_with_dow.filter(pl.col("day_of_week").is_in([6, 7])) + assigner = SessionAssigner(calendar=calendar, feed_spec=feed) + if assigner.calendar is None or df.is_empty() or "timestamp" not in df.columns: + return + frequency = frequency or feed.data_frequency + daily = frequency is not None and frequency.lower() in {"daily", "1day", "1d"} + frame = df.select("timestamp").with_row_index("_validation_row") + bar_frequency = "daily" if daily else "auto" if frequency is None else "intraday" + assigned = assigner.assign_sessions(frame, bar_frequency=bar_frequency) + outside = assigned.filter(pl.col("session_date").is_null()) + if outside.is_empty(): + return - if len(weekend_trading) > 0: + daily = _uses_session_labels(frame, feed, bar_frequency) + clock = pl.col("timestamp") + dtype = df.schema["timestamp"] + if isinstance(dtype, pl.Datetime): + if daily: + if dtype.time_zone is not None and feed.timezone is not None: + clock = clock.dt.convert_time_zone(feed.timezone) + else: + if dtype.time_zone is None: + clock = clock.dt.replace_time_zone(feed.timezone or "UTC") + clock = clock.dt.convert_time_zone(str(assigner.calendar.tz)) + weekend = outside.filter(clock.dt.weekday() >= 6) + if self.check_weekend_trading and not weekend.is_empty(): result.add_issue( ValidationIssue( severity=Severity.WARNING, check="weekend_trading", - message=f"Found {len(weekend_trading)} records on weekends (non-crypto asset)", + message=f"Found {len(weekend)} weekend records outside the selected schedule", details={ - "weekend_count": len(weekend_trading), + "weekend_count": len(weekend), "total_count": len(df), - "percentage": len(weekend_trading) / len(df) * 100, + "percentage": len(weekend) / len(df) * 100, }, - row_count=len(weekend_trading), - sample_rows=weekend_trading.with_row_index()["index"].to_list()[:10], + row_count=len(weekend), + sample_rows=weekend["_validation_row"].head(10).to_list(), ) ) - def _check_market_hours(self, df: pl.DataFrame, result: ValidationResult) -> None: - """Check for trades outside regular market hours (9:30 AM - 4:00 PM ET).""" - # Add hour column - df_with_hour = df.with_columns( - pl.col("timestamp").dt.hour().alias("hour"), - pl.col("timestamp").dt.minute().alias("minute"), - ) - - # Check for trades outside 9:30 AM - 4:00 PM (assuming UTC times need adjustment) - # This is a simplified check - in production, would need timezone handling - outside_hours = df_with_hour.filter( - (pl.col("hour") < 14) - | (pl.col("hour") >= 21) # Approximate ET in UTC - | ((pl.col("hour") == 14) & (pl.col("minute") < 30)) - ) - - if len(outside_hours) > 0: + if self.check_market_hours: result.add_issue( ValidationIssue( severity=Severity.INFO, check="market_hours", - message=f"Found {len(outside_hours)} records outside regular market hours", + message=f"Found {len(outside)} records outside the selected market schedule", details={ - "outside_hours_count": len(outside_hours), + "outside_hours_count": len(outside), + "weekend_count": len(weekend), "total_count": len(df), - "percentage": len(outside_hours) / len(df) * 100, + "percentage": len(outside) / len(df) * 100, }, - row_count=len(outside_hours), - sample_rows=outside_hours.with_row_index()["index"].to_list()[:10], + row_count=len(outside), + sample_rows=outside["_validation_row"].head(10).to_list(), ) ) diff --git a/src/ml4t/data/validation/rules.py b/src/ml4t/data/validation/rules.py index a5dccdb7..9ea950c9 100644 --- a/src/ml4t/data/validation/rules.py +++ b/src/ml4t/data/validation/rules.py @@ -4,13 +4,22 @@ from dataclasses import dataclass, field from pathlib import Path +from typing import Any import structlog import yaml -from pydantic import BaseModel, ConfigDict, Field +from pydantic import ( + BaseModel, + ConfigDict, + Field, + field_validator, + model_serializer, + model_validator, +) from ml4t.data.assets.asset_class import AssetClass from ml4t.data.validation.ohlcv import NegativePricePolicy +from ml4t.specs import UNSET_CALENDAR, CalendarUnset, FeedSpec logger = structlog.get_logger() @@ -47,12 +56,44 @@ class ValidationRuleConfig(BaseModel): check_volume_spikes: bool = Field(default=True, description="Check for volume spikes") check_weekend_trading: bool = Field(default=True, description="Check for weekend trading") check_market_hours: bool = Field(default=True, description="Check market hours") + calendar: str | None | CalendarUnset = UNSET_CALENDAR + time_contract_version: int = 2 # Asset class specific asset_class: AssetClass = Field(default=AssetClass.EQUITY, description="Asset class") model_config = ConfigDict(use_enum_values=True, extra="forbid") + @field_validator("calendar", mode="wrap") + @classmethod + def preserve_calendar_presence(cls, value: Any, handler: Any) -> Any: + return value if value is UNSET_CALENDAR else handler(value) + + @field_validator("time_contract_version", mode="before") + @classmethod + def validate_time_contract_version(cls, value: Any) -> int: + return FeedSpec(time_contract_version=value).time_contract_version + + @model_validator(mode="after") + def normalize_calendar(self) -> ValidationRuleConfig: + declaration = FeedSpec( + calendar=self.calendar, time_contract_version=self.time_contract_version + ) + object.__setattr__(self, "calendar", declaration.calendar) + return self + + @model_serializer(mode="wrap") + def serialize_calendar(self, handler: Any) -> dict[str, Any]: + data = handler(self) + declaration = FeedSpec( + calendar=self.calendar, time_contract_version=self.time_contract_version + ).to_dict() + if "calendar" in declaration and "calendar" in data: + data["calendar"] = declaration["calendar"] + else: + data.pop("calendar", None) + return data + @dataclass class ValidationRuleSet: @@ -152,8 +193,8 @@ def crypto_rules() -> ValidationRuleConfig: """Cryptocurrency validation rules.""" return ValidationRuleConfig( asset_class=AssetClass.CRYPTO, - check_weekend_trading=False, # Crypto trades 24/7 - check_market_hours=False, # No market hours for crypto + check_weekend_trading=True, + check_market_hours=True, max_return_threshold=0.5, # 50% - crypto is volatile staleness_threshold=3, # Less staleness tolerance volume_spike_threshold=20.0, # Higher spike threshold @@ -166,7 +207,7 @@ def forex_rules() -> ValidationRuleConfig: return ValidationRuleConfig( asset_class=AssetClass.FOREX, check_weekend_trading=True, # Limited weekend trading - check_market_hours=False, # 24/5 market + check_market_hours=True, max_return_threshold=0.1, # 10% - forex is less volatile staleness_threshold=10, volume_spike_threshold=5.0, diff --git a/tests/assets/test_validation.py b/tests/assets/test_validation.py index 66957a23..dae00872 100644 --- a/tests/assets/test_validation.py +++ b/tests/assets/test_validation.py @@ -66,7 +66,7 @@ def test_init_equity(self): assert validator.asset_info == info assert validator.asset_class == AssetClass.EQUITY - assert validator.calendar is None + assert validator.calendar == "NYSE" assert validator.session_validator is None def test_init_crypto_creates_calendar(self): @@ -370,7 +370,7 @@ def test_validate_saturday_data(self): results = validator.validate(df) assert "forex_specific" in results - assert any("saturday" in issue.lower() for issue in results["forex_specific"]) + assert any("weekend" in issue.lower() for issue in results["forex_specific"]) def test_validate_low_exchange_rate(self): """Test detecting unusually low exchange rates.""" diff --git a/tests/test_calendar_frame_bounds.py b/tests/test_calendar_frame_bounds.py new file mode 100644 index 00000000..b4e2866b --- /dev/null +++ b/tests/test_calendar_frame_bounds.py @@ -0,0 +1,210 @@ +"""Existing files need bounded reports without migration or a provider connection.""" + +from datetime import date, datetime + +import polars as pl +import pytest +from polars.testing import assert_frame_equal + +from ml4t.data.providers import SyntheticProvider +from ml4t.data.sessions import SessionCompleter +from ml4t.data.update_manager import GapDetector +from ml4t.data.utils.gaps import GapDetector as ObservationGapDetector +from ml4t.specs import FeedSpec + + +def instant(day, clock): + return datetime.fromisoformat(f"{day}T{clock}:00+00:00") + + +def feed(calendar="NYSE", **kwargs): + return FeedSpec( + time_contract_version=2, + calendar=calendar, + data_frequency="minute", + timestamp_semantics="event_time", + bar_grid_anchor="session_open", + bar_label="left", + partial_bar_policy="include", + **kwargs, + ) + + +def rows(day="2025-07-07"): + with SyntheticProvider(calendar_mode="continuous") as provider: + return provider.fetch_ohlcv("SPY", day, day, "minute") + + +def test_frame_report_ignores_off_session_observations_outside_requested_range(): + frame = rows().filter(pl.col("timestamp") != instant("2025-07-07", "13:31")) + original = frame.clone() + report = GapDetector(feed_spec=feed()).detect_gaps( + frame, + start_date=instant("2025-07-07", "13:30"), + end_date=instant("2025-07-07", "13:33"), + ) + assert report == [ + { + "start": instant("2025-07-07", "13:31"), + "end": instant("2025-07-07", "13:31"), + "size_bars": 1, + } + ] + assert_frame_equal(frame, original) + + +@pytest.mark.parametrize( + "calendar,day,start,end,count", + [ + ("NYSE", "2025-07-07", "13:30", "13:32", 3), + ("NYSE", "2025-07-12", "12:00", "12:02", 0), + ("24/7", "2025-07-12", "12:00", "12:02", 3), + ("CME Globex Crypto", "2026-09-05", "12:00", "12:02", 3), + ("CME Globex Crypto", "2026-10-17", "06:59", "09:00", 2), + (None, "2025-07-07", "13:30", "13:32", 0), + ], +) +def test_empty_file_interval_is_compared_with_authoritative_schedule( + calendar, day, start, end, count +): + empty = rows(day).head(0) + report = GapDetector(feed_spec=feed(calendar)).detect_gaps( + empty, + start_date=instant(day, start), + end_date=instant(day, end), + ) + assert sum(gap["size_bars"] for gap in report) == count + + +def test_frame_bounds_retain_a_wholly_missing_daily_session(): + with SyntheticProvider() as provider: + frame = provider.fetch_ohlcv("SPY", "2025-07-15", "2025-07-17", "daily") + frame = frame.filter(pl.col("timestamp").dt.date() != date(2025, 7, 16)) + report = GapDetector().detect_gaps( + frame, + start_date=instant("2025-07-16", "00:00"), + end_date=instant("2025-07-16", "23:59"), + ) + assert report == [ + { + "start": instant("2025-07-16", "00:00"), + "end": instant("2025-07-16", "00:00"), + "size_days": 1, + } + ] + + +def test_frame_report_honors_explicit_timestamp_column_without_changing_input(): + frame = rows().filter(pl.col("timestamp").dt.strftime("%H:%M").is_in(["13:31", "13:32"])) + frame = frame.rename({"timestamp": "time"}) + original = frame.clone() + report = GapDetector(feed_spec=feed(timestamp_col="time")).detect_gaps( + frame, + start_date=instant("2025-07-07", "13:30"), + end_date=instant("2025-07-07", "13:33"), + ) + assert sum(gap["size_bars"] for gap in report) == 2 + assert_frame_equal(frame, original) + + +def test_panel_instrument_absent_inside_bounds_is_not_omitted(): + complete = rows().filter( + pl.col("timestamp") + .dt.strftime("%H:%M") + .is_in( + [ + "13:29", + "13:30", + "13:31", + "13:32", + ] + ) + ) + spy = complete.filter(pl.col("timestamp").dt.minute() == 29) + qqq = complete.with_columns(pl.lit("QQQ").alias("symbol")) + report = GapDetector(feed_spec=feed(entity_col="symbol")).detect_gaps( + pl.concat([spy, qqq]), + start_date=instant("2025-07-07", "13:30"), + end_date=instant("2025-07-07", "13:32"), + ) + assert report == [ + { + "entity": "SPY", + "start": instant("2025-07-07", "13:30"), + "end": instant("2025-07-07", "13:32"), + "size_bars": 3, + } + ] + + +def test_a_single_bound_cannot_silently_choose_a_different_coverage_interval(): + with pytest.raises(ValueError, match="together|both"): + GapDetector(feed_spec=feed()).detect_gaps(rows(), start_date=instant("2025-07-07", "13:30")) + + +@pytest.mark.parametrize("mode", ["range", "observation"]) +def test_real_native_placeholder_is_not_a_recovered_source_observation(mode): + actual = rows().filter(pl.col("timestamp").dt.strftime("%H:%M").is_in(["13:30", "13:32"])) + expanded = SessionCompleter("NYSE").complete_sessions(actual) + assert expanded["is_imputed"].to_list() == [False, True, False] + before = expanded.clone() + if mode == "range": + report = GapDetector(feed_spec=feed()).detect_gaps(expanded) + assert sum(gap["size_bars"] for gap in report) == 1 + else: + report = ObservationGapDetector().detect_gaps(expanded, feed_spec=feed()) + assert sum(gap.missing_periods for gap in report) == 1 + assert_frame_equal(expanded, before) + + +def test_raw_parquet_uses_the_same_bounded_report_without_storage_migration(tmp_path): + path = tmp_path / "prices.parquet" + frame = rows().filter(pl.col("timestamp").dt.strftime("%H:%M").is_in(["13:31", "13:32"])) + frame.write_parquet(path) + original = path.read_bytes() + report = GapDetector(feed_spec=feed()).detect_gaps( + pl.read_parquet(path), + start_date=instant("2025-07-07", "13:30"), + end_date=instant("2025-07-07", "13:33"), + ) + assert sum(gap["size_bars"] for gap in report) == 2 + assert path.read_bytes() == original + + +def test_wholly_placeholder_interval_is_not_reported_as_complete(): + actual = rows().filter(pl.col("timestamp").dt.strftime("%H:%M").is_in(["13:30", "13:33"])) + expanded = SessionCompleter("NYSE").complete_sessions(actual) + placeholders = expanded.filter(pl.col("is_imputed")) + report = GapDetector(feed_spec=feed()).detect_gaps(placeholders) + assert report == [ + { + "start": instant("2025-07-07", "13:31"), + "end": instant("2025-07-07", "13:32"), + "size_bars": 2, + } + ] + + +def test_daily_known_placeholder_does_not_satisfy_a_session(): + with SyntheticProvider() as provider: + frame = provider.fetch_ohlcv("SPY", "2025-07-15", "2025-07-17", "daily") + frame = frame.with_columns( + (pl.col("timestamp").dt.date() == date(2025, 7, 16)).alias("is_imputed") + ) + report = GapDetector().detect_gaps(frame) + assert len(report) == 1 + assert report[0]["start"].date() == date(2025, 7, 16) + assert report[0]["size_days"] == 1 + + +def test_zero_volume_alone_is_not_treated_as_a_missing_observation(): + frame = rows().filter(pl.col("timestamp").dt.strftime("%H:%M").is_in(["13:30", "13:31"])) + frame = frame.with_columns(pl.lit(0.0).alias("volume")) + assert GapDetector(feed_spec=feed()).detect_gaps(frame) == [] + + +def test_invalid_placeholder_marker_cannot_silently_hide_source_rows(): + frame = rows().filter(pl.col("timestamp").dt.strftime("%H:%M").is_in(["13:30", "13:31"])) + frame = frame.with_columns(pl.lit("False").alias("is_imputed")) + with pytest.raises(TypeError, match="Boolean"): + GapDetector(feed_spec=feed()).detect_gaps(frame) diff --git a/tests/test_cli_batch.py b/tests/test_cli_batch.py index 3f9cd3e0..7340532b 100644 --- a/tests/test_cli_batch.py +++ b/tests/test_cli_batch.py @@ -65,6 +65,7 @@ def test_update_all_passes_storage_and_dataset_config() -> None: provider="yahoo", lookback_days=14, fill_gaps=False, + check_gaps=True, initial_start="2010-01-01", initial_end="2024-12-31", initial_load_days=365, @@ -106,7 +107,8 @@ def test_update_all_uses_initial_load_days_when_configured() -> None: asset_class="equity", provider="mock", lookback_days=7, - fill_gaps=True, + fill_gaps=None, + check_gaps=True, initial_start=None, initial_end=None, initial_load_days=3650, diff --git a/tests/test_crypto_assets.py b/tests/test_crypto_assets.py index 7e1d20d8..3a3af540 100644 --- a/tests/test_crypto_assets.py +++ b/tests/test_crypto_assets.py @@ -13,15 +13,25 @@ ) from ml4t.data.calendar.crypto import CryptoCalendar, CryptoSessionValidator from ml4t.data.providers import SyntheticProvider +from ml4t.specs import FeedSpec @pytest.fixture(params=["direct", "asset"]) def continuity_issues(request): def validate(frame, frequency): + feed = FeedSpec( + time_contract_version=2, + calendar="24/7", + data_frequency=frequency, + timestamp_semantics="session_label" if frequency == "daily" else "event_time", + bar_grid_anchor=None if frequency == "daily" else "session_open", + bar_label=None if frequency == "daily" else "left", + partial_bar_policy=None if frequency == "daily" else "exclude", + ) if request.param == "direct": - return CryptoSessionValidator().validate_continuity(frame, frequency) + return CryptoSessionValidator(feed_spec=feed).validate_continuity(frame, frequency) info = AssetInfo(symbol="BTC/USD", asset_class=AssetClass.CRYPTO) - return AssetValidator(info).validate(frame, frequency).get("continuity", []) + return AssetValidator(info, feed_spec=feed).validate(frame, frequency).get("continuity", []) return validate diff --git a/tests/test_data_manager_storage.py b/tests/test_data_manager_storage.py index da9cfae3..e1f17ca9 100644 --- a/tests/test_data_manager_storage.py +++ b/tests/test_data_manager_storage.py @@ -587,7 +587,7 @@ def test_update_with_gaps(self, manager, storage): key = manager.load("AAPL", "2024-01-01", "2024-01-05", provider="mock") # Update - should merge data - manager.update("AAPL", fill_gaps=True, provider="mock") + manager.update("AAPL", check_gaps=True, provider="mock") stored_df = storage.read(key).collect() # Should have data without duplicates diff --git a/tests/test_gaps.py b/tests/test_gaps.py index 61641d57..3d3fd858 100644 --- a/tests/test_gaps.py +++ b/tests/test_gaps.py @@ -188,8 +188,8 @@ def test_gap_summary(self) -> None: assert summary["largest_gap"].missing_periods == 3 assert summary["smallest_gap"].missing_periods == 1 - def test_fill_gaps_forward(self) -> None: - """Test forward fill of gaps.""" + def test_fill_gaps_rejects_synthetic_forward_values(self) -> None: + """A report of absence must not authorize invented market values.""" timestamps = [ datetime(2024, 1, 1, 14, 30, tzinfo=UTC), datetime(2024, 1, 2, 14, 30, tzinfo=UTC), @@ -206,17 +206,9 @@ def test_fill_gaps_forward(self) -> None: detector = GapDetector() gaps = detector.detect_gaps(df, frequency="daily") - filled_df = detector.fill_gaps(df, gaps, method="forward") - - # Should have 4 rows after filling - assert len(filled_df) == 4 - # Check that gap was filled with forward value - assert ( - filled_df.filter(pl.col("timestamp") == datetime(2024, 1, 3, 14, 30, tzinfo=UTC))[ - "value" - ][0] - == 101.0 - ) + with pytest.raises(ValueError, match="imputation"): + detector.fill_gaps(df, gaps, method="forward") + assert df["value"].to_list() == [100.0, 101.0, 103.0] def test_fill_gaps_rejects_all_null_timestamps(self) -> None: df = pl.DataFrame( @@ -233,10 +225,10 @@ def test_fill_gaps_rejects_all_null_timestamps(self) -> None: frequency="daily", ) - with pytest.raises(TypeError, match="non-null Date or Datetime"): + with pytest.raises(ValueError, match="imputation"): GapDetector().fill_gaps(df, [gap]) - def test_fill_daily_gaps_supports_date_columns(self) -> None: + def test_date_columns_do_not_enable_synthetic_gap_imputation(self) -> None: df = pl.DataFrame( { "timestamp": [date(2024, 1, 1), date(2024, 1, 3)], @@ -251,13 +243,9 @@ def test_fill_daily_gaps_supports_date_columns(self) -> None: frequency="daily", ) - filled = GapDetector().fill_gaps(df, [gap], method="forward") - - assert filled.get_column("timestamp").to_list() == [ - date(2024, 1, 1), - date(2024, 1, 2), - date(2024, 1, 3), - ] + with pytest.raises(ValueError, match="imputation"): + GapDetector().fill_gaps(df, [gap], method="forward") + assert df["timestamp"].to_list() == [date(2024, 1, 1), date(2024, 1, 3)] def test_empty_dataframe(self) -> None: """Test gap detection on empty DataFrame.""" diff --git a/tests/test_incremental_update_system.py b/tests/test_incremental_update_system.py index 094319a0..e0c4ec74 100644 --- a/tests/test_incremental_update_system.py +++ b/tests/test_incremental_update_system.py @@ -600,6 +600,7 @@ def test_execute_backfill(self): # Mock data provider mock_provider = MagicMock() + mock_provider.name = "mock" gap_data = pl.DataFrame( { "timestamp": pl.date_range( @@ -642,7 +643,7 @@ def test_execute_backfill(self): provider=mock_provider, ) - assert result.success is True + assert result.success is True, result.errors assert result.gaps_filled == 1 assert result.rows_added == 2 # Only 2 weekdays in Jan 6-9 gap @@ -728,7 +729,16 @@ def test_update_range_starts_after_stored_data(self): "value": range(existing_days), } ) - storage.write(df_existing, "test") + storage.write( + df_existing, + "test", + { + "calendar": "24/7", + "time_contract_version": 2, + "data_frequency": "daily", + "timestamp_semantics": "session_label", + }, + ) # Determine update range for full year request start, end, update_type = updater.determine_update_range( diff --git a/tests/test_ingestion_calendar_declarations.py b/tests/test_ingestion_calendar_declarations.py index 1c13c17a..6279ab00 100644 --- a/tests/test_ingestion_calendar_declarations.py +++ b/tests/test_ingestion_calendar_declarations.py @@ -93,7 +93,7 @@ def test_disabled_update_does_not_impute_an_old_provider_outage(manager): recent = provider_frame(str(recent_day - timedelta(days=3)), str(recent_day)) frame = pl.concat([historic, recent]).unique("timestamp").sort("timestamp") key = instance.import_data(frame, "SPY", "synthetic", calendar=None) - instance.update("SPY", provider="synthetic", fill_gaps=True) + instance.update("SPY", provider="synthetic", check_gaps=True) stored = storage.read(key).collect() assert stored.filter(pl.col("timestamp").dt.date() == date(2025, 7, 16)).is_empty() assert FeedSpec.from_mapping(instance.get_metadata_for_key(key)).resolve_calendar() is None @@ -165,7 +165,7 @@ def test_explicit_update_disable_overrides_stored_schedule_without_filling(manag recent = provider_frame(str(recent_day - timedelta(days=3)), str(recent_day)) frame = pl.concat([historic, recent]).unique("timestamp").sort("timestamp") key = instance.import_data(frame, "SPY", "synthetic", calendar="24/7") - instance.update("SPY", provider="synthetic", calendar=None, fill_gaps=True) + instance.update("SPY", provider="synthetic", calendar=None, check_gaps=True) assert ( storage.read(key) .collect() diff --git a/tests/test_preserve_market_absences.py b/tests/test_preserve_market_absences.py new file mode 100644 index 00000000..01ade4c6 --- /dev/null +++ b/tests/test_preserve_market_absences.py @@ -0,0 +1,161 @@ +"""Production update paths must never invent market observations or prices.""" + +from datetime import UTC, date, datetime + +import polars as pl +import pytest +from click.testing import CliRunner +from polars.testing import assert_frame_equal + +from ml4t.data import DataManager +from ml4t.data.cli import cli +from ml4t.data.config.models import DatasetConfig +from ml4t.data.providers import SyntheticProvider +from ml4t.data.sessions import SessionCompleter +from ml4t.data.storage.config import StorageConfig +from ml4t.data.storage.flat import FlatStorage +from ml4t.data.storage.hive import HiveStorage +from ml4t.data.utils.gaps import GapDetector + + +@pytest.mark.parametrize("backend", [FlatStorage, HiveStorage]) +@pytest.mark.parametrize("check", ["absent_timestamp", "observed_null"]) +def test_default_real_provider_update_preserves_historical_absence_and_observed_null( + tmp_path, monkeypatch, backend, check +): + class Clock(datetime): + @classmethod + def now(cls, tz=UTC): + return cls(2025, 7, 15, 23, tzinfo=tz) + + monkeypatch.setattr("ml4t.data.managers.storage_manager.datetime", Clock) + with SyntheticProvider(seed=42) as provider: + frame = provider.fetch_ohlcv("SPY", "2025-07-07", "2025-07-14", "daily") + observed = frame.filter(pl.col("timestamp").dt.date() != date(2025, 7, 9)).with_columns( + pl.when(pl.col("timestamp").dt.date() == date(2025, 7, 8)) + .then(None) + .otherwise(pl.col("close")) + .alias("close") + ) + storage = backend(StorageConfig(base_path=tmp_path)) + with DataManager( + storage=storage, providers={"synthetic": {"extra": {"seed": 42}}}, enable_validation=False + ) as manager: + key = manager.import_data(observed, "SPY", "synthetic") + manager.update("SPY", lookback_days=0) + actual = storage.read(key).collect() + if check == "absent_timestamp": + assert date(2025, 7, 9) not in actual["timestamp"].dt.date().to_list() + else: + assert ( + actual.filter(pl.col("timestamp").dt.date() == date(2025, 7, 8))["close"].null_count() + == 1 + ) + retained = actual.filter(pl.col("timestamp").dt.date() < date(2025, 7, 14)) + assert_frame_equal(retained, observed.filter(pl.col("timestamp").dt.date() < date(2025, 7, 14))) + assert ( + actual.filter(pl.col("timestamp").dt.date() == date(2025, 7, 8))["close"].null_count() == 1 + ) + + +@pytest.mark.parametrize("backend", [FlatStorage, HiveStorage]) +def test_explicit_legacy_imputation_is_rejected_before_initial_dataset_creation(tmp_path, backend): + storage = backend(StorageConfig(base_path=tmp_path)) + with DataManager(storage=storage, providers={"synthetic": {}}) as manager: + with pytest.raises(ValueError, match="[Ss]ynthetic|imputation"): + manager.update( + "SPY", + provider="synthetic", + fill_gaps=True, + initial_start="2025-07-07", + initial_end="2025-07-14", + ) + assert storage.list_keys() == [] + + +def test_explicit_grid_expansion_keeps_missing_market_values_null(): + with SyntheticProvider(calendar_mode="continuous") as provider: + frame = provider.fetch_ohlcv("SPY", "2025-07-07", "2025-07-07", "minute") + frame = frame.filter(pl.col("timestamp").dt.strftime("%H:%M").is_in(["13:30", "13:32"])) + result = SessionCompleter("NYSE").complete_sessions(frame) + missing = result.filter(pl.col("is_imputed")) + assert missing.height == 1 + for name in ["open", "high", "low", "close", "volume"]: + assert missing[name].null_count() == 1 + retained = result.filter(~pl.col("is_imputed")).select(frame.columns).cast(frame.schema) + assert_frame_equal(retained, frame) + + +def test_legacy_gap_imputation_cannot_create_market_values(): + with SyntheticProvider(seed=42) as provider: + frame = provider.fetch_ohlcv("SPY", "2025-07-07", "2025-07-14", "daily") + observed = frame.filter(pl.col("timestamp").dt.date() != date(2025, 7, 9)) + detector = GapDetector() + with pytest.raises(ValueError, match="[Ss]ynthetic|imputation"): + detector.fill_gaps(observed, detector.detect_gaps(observed)) + + +def test_configuration_never_enables_synthetic_filling_by_default(): + config = DatasetConfig(name="market", symbols=["SPY"], provider="synthetic") + assert config.fill_gaps is not True + + +def test_configuration_rejects_explicit_synthetic_imputation(): + with pytest.raises(ValueError, match="imputation"): + DatasetConfig(name="market", symbols=["SPY"], provider="synthetic", fill_gaps=True) + + +def test_real_cli_rejects_imputation_before_creating_a_dataset(tmp_path): + result = CliRunner().invoke( + cli, + [ + "update", + "--symbol", + "SPY", + "--provider", + "synthetic", + "--fill-gaps", + "--initial-start", + "2025-07-07", + "--initial-end", + "2025-07-14", + "--storage-path", + str(tmp_path), + ], + ) + assert result.exit_code != 0 + assert "imputation" in result.output + assert HiveStorage(StorageConfig(base_path=tmp_path)).list_keys() == [] + + +def test_real_cli_default_update_preserves_a_historical_missing_observation(tmp_path, monkeypatch): + class Clock(datetime): + @classmethod + def now(cls, tz=UTC): + return cls(2025, 7, 15, 23, tzinfo=tz) + + monkeypatch.setattr("ml4t.data.managers.storage_manager.datetime", Clock) + with SyntheticProvider(seed=42) as provider: + complete = provider.fetch_ohlcv("SPY", "2025-07-07", "2025-07-14", "daily") + observed = complete.filter(pl.col("timestamp").dt.date() != date(2025, 7, 9)) + storage = HiveStorage(StorageConfig(base_path=tmp_path)) + with DataManager(storage=storage, enable_validation=False) as manager: + key = manager.import_data(observed, "SPY", "synthetic") + result = CliRunner().invoke( + cli, + [ + "update", + "--symbol", + "SPY", + "--provider", + "synthetic", + "--lookback-days", + "0", + "--storage-path", + str(tmp_path), + ], + ) + assert result.exit_code == 0, result.output + actual = storage.read(key).collect() + assert date(2025, 7, 9) not in actual["timestamp"].dt.date().to_list() + assert storage.get_metadata(key)["custom"]["attributes"]["gaps_filled"] is False diff --git a/tests/test_sessions.py b/tests/test_sessions.py index 971736fb..ff6eb313 100644 --- a/tests/test_sessions.py +++ b/tests/test_sessions.py @@ -401,8 +401,8 @@ def test_explicit_half_open_range_is_clipped_exactly(self): assert result["timestamp"].min() == start assert result["timestamp"].max() == end - timedelta(minutes=1) - def test_generated_bar_uses_prior_close_and_is_marked(self): - """A generated OHLC bar is a marked prior-close bar.""" + def test_absent_minute_is_marked_and_keeps_unknown_prices_and_volume(self): + """A null placeholder must not be mistaken for a recovered market observation.""" from ml4t.data.sessions.completer import SessionCompleter completer = SessionCompleter("NYSE") @@ -423,15 +423,15 @@ def test_generated_bar_uses_prior_close_and_is_marked(self): assert result["is_imputed"].to_list() == [False, True] assert result.select("open", "high", "low", "close").row(1) == ( - 105.0, - 105.0, - 105.0, - 105.0, + None, + None, + None, + None, ) - assert result["volume"].to_list() == [1000.0, 0.0] + assert result["volume"].to_list() == [1000.0, None] - def test_second_session_open_uses_previous_close_and_metadata(self): - """A session-opening gap carries the prior close and symbol forward.""" + def test_second_session_open_does_not_acquire_previous_session_market_values(self): + """A session-opening absence must not copy a previous session's price.""" from ml4t.data.sessions.completer import SessionCompleter completer = SessionCompleter("NYSE") @@ -462,8 +462,8 @@ def test_second_session_open_uses_previous_close_and_metadata(self): closed="left", ) ) - assert opening_gap["close"].to_list() == [100.0, 100.0] - assert opening_gap["symbol"].to_list() == ["AAPL", "AAPL"] + assert opening_gap["close"].to_list() == [None, None] + assert opening_gap["symbol"].to_list() == [None, None] def test_rejects_observations_outside_the_minute_template(self): """Completion never silently replaces unmatched observations.""" @@ -475,7 +475,7 @@ def test_rejects_observations_outside_the_minute_template(self): with pytest.raises(ValueError, match="outside the completion template"): completer.complete_sessions(daily) - def test_forward_fill_uses_available_price_when_close_is_absent(self): + def test_missing_open_remains_unknown_when_close_is_absent(self): from ml4t.data.sessions.completer import SessionCompleter completer = SessionCompleter("NYSE") @@ -488,7 +488,7 @@ def test_forward_fill_uses_available_price_when_close_is_absent(self): end_date=start + timedelta(minutes=2), ) - assert result["open"].to_list() == [100.0, 100.0] + assert result["open"].to_list() == [100.0, None] def test_early_close_and_break_minutes_are_excluded(self): from ml4t.data.sessions.completer import SessionCompleter @@ -520,8 +520,8 @@ def test_early_close_and_break_minutes_are_excluded(self): ) ).height - def test_complete_sessions_fill_methods(self): - """Test different fill methods.""" + def test_complete_sessions_rejects_synthetic_price_methods(self): + """Only null grid expansion remains supported.""" from ml4t.data.sessions.completer import SessionCompleter completer = SessionCompleter("NYSE") @@ -540,20 +540,16 @@ def test_complete_sessions_fill_methods(self): } ) - # Test forward fill - result_forward = completer.complete_sessions(df, fill_method="forward") - assert len(result_forward) >= len(df) - - # Test backward fill - result_backward = completer.complete_sessions(df, fill_method="backward") - assert len(result_backward) >= len(df) + for method in ("forward", "backward"): + with pytest.raises(ValueError, match="imputation"): + completer.complete_sessions(df, fill_method=method) # Test no fill result_none = completer.complete_sessions(df, fill_method="none") - assert len(result_none) >= len(df) + assert result_none["close"].to_list() == [100.2, None, 101.2] - def test_complete_sessions_zero_volume(self): - """Test volume handling for filled rows.""" + def test_complete_sessions_rejects_invented_zero_volume(self): + """An absent observation has no known volume, not a known zero.""" from ml4t.data.sessions.completer import SessionCompleter completer = SessionCompleter("NYSE") @@ -569,13 +565,10 @@ def test_complete_sessions_zero_volume(self): } ) - # With zero_volume=True (default) - result = completer.complete_sessions(df, zero_volume=True) - # Result should have more rows (gaps filled) - assert len(result) >= len(df) - # Verify volume column exists and has no NaN values (filled with 0) - assert "volume" in result.columns - assert result["volume"].null_count() == 0 + with pytest.raises(ValueError, match="imputation"): + completer.complete_sessions(df, zero_volume=True) + result = completer.complete_sessions(df) + assert result["volume"].to_list() == [1000.0, None, 1100.0] def test_get_session_info_trading_day(self): """Test getting session info for a trading day.""" @@ -603,8 +596,8 @@ def test_get_session_info_non_trading_day(self): assert info["market_open"] is None assert info["market_close"] is None - def test_fill_missing_data_with_metadata_columns(self): - """Test that metadata columns are forward-filled.""" + def test_null_placeholder_does_not_claim_observed_symbol_metadata(self): + """Only observed rows carry source symbol values in the expanded grid.""" from ml4t.data.sessions.completer import SessionCompleter completer = SessionCompleter("NYSE") @@ -622,8 +615,7 @@ def test_fill_missing_data_with_metadata_columns(self): ) result = completer.complete_sessions(df) - # Symbol should be forward-filled - assert "symbol" in result.columns + assert result["symbol"].to_list() == ["AAPL", None, "AAPL"] class TestSessionIntegration: diff --git a/tests/test_stored_gap_report.py b/tests/test_stored_gap_report.py new file mode 100644 index 00000000..f0d18562 --- /dev/null +++ b/tests/test_stored_gap_report.py @@ -0,0 +1,409 @@ +"""Calendar comparison reads real stored producer data without altering it.""" + +from datetime import date, datetime +from pathlib import Path + +import polars as pl +import pytest +from polars.testing import assert_frame_equal + +from ml4t.data.providers import SyntheticProvider +from ml4t.data.storage.config import StorageConfig +from ml4t.data.storage.flat import FlatStorage +from ml4t.data.storage.hive import HiveStorage +from ml4t.data.update_manager import GapDetector, IncrementalUpdater +from ml4t.specs import FeedSpec + + +def declaration(calendar="NYSE", frequency="minute", **kwargs): + return FeedSpec( + time_contract_version=2, + calendar=calendar, + data_frequency=frequency, + timestamp_semantics=kwargs.pop("timestamp_semantics", "event_time"), + bar_grid_anchor="session_open", + bar_label=kwargs.pop("bar_label", "left"), + partial_bar_policy=kwargs.pop("partial_bar_policy", "include"), + **kwargs, + ) + + +def rows(start, end, clocks, frequency="minute"): + with SyntheticProvider(seed=42, calendar_mode="continuous") as provider: + frame = provider.fetch_ohlcv("SPY", start, end, frequency) + return frame.filter(pl.col("timestamp").dt.strftime("%H:%M").is_in(clocks)) + + +def instant(day, clock): + return datetime.fromisoformat(f"{day}T{clock}:00+00:00") + + +@pytest.fixture(params=[FlatStorage, HiveStorage]) +def storage(request, tmp_path): + return request.param(StorageConfig(base_path=tmp_path)) + + +def test_disk_report_finds_inclusive_leading_and_trailing_minute_absences(storage): + frame = rows("2025-07-07", "2025-07-07", ["13:31", "13:32"]) + key = "equities/minute/SPY" + storage.write(frame, key, declaration().to_dict()) + before = storage.get_metadata(key) + gaps = GapDetector().detect_gaps_in_storage( + storage, key, instant("2025-07-07", "13:30"), instant("2025-07-07", "13:33") + ) + assert gaps == [ + {"start": instant("2025-07-07", clock), "end": instant("2025-07-07", clock), "size_bars": 1} + for clock in ["13:30", "13:33"] + ] + assert storage.get_metadata(key) == before + assert_frame_equal(storage.read(key).collect(), frame) + + +def test_disk_report_applies_stored_continuous_calendar_and_hour_frequency(storage): + frame = rows("2025-07-11", "2025-07-14", ["00:00"], "hourly").filter( + pl.col("timestamp").dt.date().is_in([date(2025, 7, 11), date(2025, 7, 14)]) + ) + key = "crypto/hourly/SPY" + storage.write(frame, key, declaration("24/7", "hourly").to_dict()) + report = GapDetector().detect_gaps_in_storage( + storage, key, instant("2025-07-11", "00:00"), instant("2025-07-14", "00:00") + ) + assert report == [ + { + "start": instant("2025-07-11", "01:00"), + "end": instant("2025-07-13", "23:00"), + "size_bars": 71, + } + ] + assert ( + GapDetector().detect_gaps_in_storage( + storage, + key, + instant("2025-07-11", "00:00"), + instant("2025-07-14", "00:00"), + calendar=None, + ) + == [] + ) + + +def test_direct_report_does_not_invent_a_weekend_between_adjacent_nyse_bars(): + frame = rows("2025-07-11", "2025-07-14", ["19:59", "13:30"]).filter( + ( + (pl.col("timestamp").dt.date() == date(2025, 7, 11)) + & (pl.col("timestamp").dt.hour() == 19) + ) + | ( + (pl.col("timestamp").dt.date() == date(2025, 7, 14)) + & (pl.col("timestamp").dt.hour() == 13) + ) + ) + assert frame.height == 2 + assert GapDetector(feed_spec=declaration()).detect_gaps(frame) == [] + + +@pytest.mark.parametrize("calendar,expected", [("NYSE", 210), ("24/7", 2880), (None, 0)]) +def test_completely_absent_disk_dataset_uses_schedule_not_observed_dates( + storage, calendar, expected +): + report = GapDetector(feed_spec=declaration(calendar)).detect_gaps_in_storage( + storage, + "equities/minute/ABSENT", + instant("2025-07-03", "00:00"), + instant("2025-07-04", "23:59"), + frequency="minute", + ) + assert sum(gap["size_bars"] for gap in report) == expected + assert not storage.exists("equities/minute/ABSENT") + + +@pytest.mark.parametrize( + "day,clocks", + [ + ("2026-09-05", ["12:00", "12:02"]), + ("2026-05-29", ["21:30", "21:32"]), + ], +) +def test_cme_crypto_report_covers_the_following_business_trade_date(storage, day, clocks): + frame = rows(day, day, clocks) + key = "futures/minute/BTC" + storage.write(frame, key, declaration("CME Globex Crypto").to_dict()) + report = GapDetector().detect_gaps_in_storage( + storage, key, instant(day, clocks[0]), instant(day, clocks[1]), frequency="minute" + ) + middle = "12:01" if day == "2026-09-05" else "21:31" + assert report == [{"start": instant(day, middle), "end": instant(day, middle), "size_bars": 1}] + + +@pytest.mark.parametrize( + "calendar,day,clocks", + [ + ("HKEX", "2025-07-07", ["03:59", "05:00"]), + ("CME Globex Crypto", "2026-10-17", ["06:59", "09:00"]), + ], +) +def test_disk_report_excludes_selected_exchange_breaks_and_maintenance( + storage, calendar, day, clocks +): + frame = rows(day, day, clocks) + key = "futures/minute/SPY" + storage.write(frame, key, declaration(calendar).to_dict()) + assert ( + GapDetector().detect_gaps_in_storage( + storage, key, instant(day, clocks[0]), instant(day, clocks[1]), frequency="minute" + ) + == [] + ) + + +def test_calendar_report_needs_timestamps_not_ohlcv_values(storage): + frame = rows("2025-07-07", "2025-07-07", ["13:30", "13:32"]).select( + "timestamp", pl.col("close").alias("measurement") + ) + key = "measurements/minute/series" + storage.write(frame, key, declaration().to_dict()) + report = GapDetector().detect_gaps_in_storage( + storage, key, instant("2025-07-07", "13:30"), instant("2025-07-07", "13:32") + ) + assert sum(gap["size_bars"] for gap in report) == 1 + assert_frame_equal(storage.read(key).collect(), frame) + + +def test_disk_report_rejects_reversed_intraday_bounds_without_rewriting(storage): + key = "equities/minute/SPY" + frame = rows("2025-07-07", "2025-07-07", ["13:30", "13:32"]) + storage.write(frame, key, declaration().to_dict()) + before = storage.get_metadata(key) + with pytest.raises(ValueError, match="start_date"): + GapDetector().detect_gaps_in_storage( + storage, + key, + instant("2025-07-07", "13:32"), + instant("2025-07-07", "13:30"), + frequency="minute", + ) + assert storage.get_metadata(key) == before + assert_frame_equal(storage.read(key).collect(), frame) + + +@pytest.mark.parametrize( + "side,policy,expected", + [ + ("left", "include", 1), + ("left", "exclude", 0), + ("right", "include", 2), + ("right", "exclude", 1), + ], +) +def test_requested_bounds_respect_actual_partial_close_bar_labels(storage, side, policy, expected): + feed = declaration( + "NYSE", + "hourly", + bar_label=side, + partial_bar_policy=policy, + timestamp_semantics="bar_close" if side == "right" else "event_time", + ) + report = GapDetector(feed_spec=feed).detect_gaps_in_storage( + storage, + "equities/hourly/ABSENT", + instant("2025-07-03", "16:00"), + instant("2025-07-03", "17:00"), + ) + assert sum(gap["size_bars"] for gap in report) == expected + if report: + assert report[0]["end"] == instant( + "2025-07-03", "17:00" if side == "right" and policy == "include" else "16:30" + ) + + +def test_whole_session_missing_for_every_stored_asset_remains_missing(storage): + frame = rows("2025-07-15", "2025-07-17", ["13:30", "19:59"]).filter( + pl.col("timestamp").dt.date() != date(2025, 7, 16) + ) + for symbol in ["SPY", "QQQ"]: + key = f"equities/minute/{symbol}" + storage.write( + frame.with_columns(pl.lit(symbol).alias("symbol")), key, declaration().to_dict() + ) + report = GapDetector().detect_gaps_in_storage( + storage, key, instant("2025-07-16", "00:00"), instant("2025-07-16", "23:59") + ) + assert report == [ + { + "start": instant("2025-07-16", "13:30"), + "end": instant("2025-07-16", "19:59"), + "size_bars": 390, + } + ] + + +def test_requested_frequency_cannot_override_stored_intraday_grid(storage): + key = "equities/minute/SPY" + storage.write(rows("2025-07-07", "2025-07-07", ["13:30"]), key, declaration().to_dict()) + with pytest.raises(ValueError, match="frequency"): + GapDetector().detect_gaps_in_storage( + storage, + key, + instant("2025-07-07", "13:30"), + instant("2025-07-07", "14:30"), + frequency="hourly", + ) + + +def test_missing_intraday_grid_is_rejected_instead_of_assuming_elapsed_days(storage): + frame = rows("2025-07-07", "2025-07-07", ["13:30", "13:32"]) + key = "equities/minute/SPY" + storage.write(frame, key, {"frequency": "minute"}) + with pytest.raises(ValueError, match="declared.*grid"): + GapDetector().detect_gaps_in_storage( + storage, key, instant("2025-07-07", "13:30"), instant("2025-07-07", "13:32") + ) + + +def test_one_instrument_cannot_hide_another_instruments_missing_bar(storage): + complete = rows("2025-07-07", "2025-07-07", ["13:30", "13:31", "13:32"]) + spy = complete.filter(pl.col("timestamp").dt.minute() != 31) + qqq = complete.with_columns(pl.lit("QQQ").alias("symbol")) + panel = pl.concat([spy, qqq]) + key = "equities/minute/panel" + storage.write(panel, key, declaration(entity_col="symbol").to_dict()) + report = GapDetector().detect_gaps_in_storage( + storage, key, instant("2025-07-07", "13:30"), instant("2025-07-07", "13:32") + ) + assert report == [ + { + "entity": "SPY", + "start": instant("2025-07-07", "13:31"), + "end": instant("2025-07-07", "13:31"), + "size_bars": 1, + } + ] + assert_frame_equal(storage.read(key).collect(), panel) + + +def test_stored_instrument_absent_from_the_entire_query_is_still_inspected(storage): + complete = rows("2025-07-07", "2025-07-07", ["13:29", "13:30", "13:31", "13:32"]) + spy = complete.filter(pl.col("timestamp").dt.minute() == 29) + qqq = complete.with_columns(pl.lit("QQQ").alias("symbol")) + key = "equities/minute/panel" + storage.write(pl.concat([spy, qqq]), key, declaration(entity_col=["symbol"]).to_dict()) + report = GapDetector().detect_gaps_in_storage( + storage, key, instant("2025-07-07", "13:30"), instant("2025-07-07", "13:32") + ) + assert report == [ + { + "entity": "SPY", + "start": instant("2025-07-07", "13:30"), + "end": instant("2025-07-07", "13:32"), + "size_bars": 3, + } + ] + + +@pytest.mark.parametrize("frequency", ["", "unknown", "ticks"]) +def test_unknown_direct_frequency_cannot_silently_become_daily(frequency): + frame = rows("2025-07-07", "2025-07-07", ["13:30", "13:32"]) + with pytest.raises(ValueError, match="frequency"): + GapDetector().detect_gaps(frame, frequency=frequency) + + +def test_documented_disk_inspection_reports_absences_without_changing_observations( + tmp_path, monkeypatch +): + from ml4t.data.storage import create_storage + + monkeypatch.chdir(tmp_path) + storage = create_storage("./data", strategy="hive") + key = "crypto/minute/coinbase-btc-usd-spot" + frame = rows("2026-10-16", "2026-10-16", ["00:00", "00:02", "23:59"]) + storage.write(frame, key) + before = storage.get_metadata(key) + guide = Path(__file__).resolve().parents[1] / "docs/user-guide/incremental-updates.md" + section = guide.read_text().split("### Inspect Missing Observations on Disk", 1)[1] + code = section.split("```python\n", 1)[1].split("```", 1)[0] + namespace = {} + exec(compile(code, str(guide), "exec"), namespace) + assert sum(gap["size_bars"] for gap in namespace["report"]) == 1437 + assert storage.get_metadata(key) == before + assert_frame_equal(storage.read(key).collect(), frame) + frame.write_parquet("./prices.parquet") + original_file = Path("./prices.parquet").read_bytes() + raw_code = section.split("```python\n", 2)[2].split("```", 1)[0] + exec(compile(raw_code, str(guide), "exec"), namespace) + assert sum(gap["size_bars"] for gap in namespace["report"]) == 1437 + assert Path("./prices.parquet").read_bytes() == original_file + + +@pytest.mark.parametrize( + "calendar,day,clock,end,expected", + [ + ("24/7", "2026-09-05", "12:00", "2026-09-05T12:02", "2026-09-05T12:01"), + ("CME Globex Crypto", "2026-09-05", "12:00", "2026-09-05T12:02", "2026-09-05T12:01"), + ("NYSE", "2025-07-11", "19:59", "2025-07-14T13:31", "2025-07-14T13:30"), + ("NYSE", "2025-07-03", "16:59", "2025-07-07T13:31", "2025-07-07T13:30"), + ("NYSE", "2025-07-11", "19:59", "2025-07-13T20:00", None), + ], +) +def test_incremental_range_starts_at_next_scheduled_bar_not_a_day_later( + storage, calendar, day, clock, end, expected +): + frame = rows(day, day, [clock]) + storage.write(frame, "SPY", declaration(calendar).to_dict()) + before = storage.get_metadata("SPY") + end = datetime.fromisoformat(end + ":00+00:00") + expected = datetime.fromisoformat(expected + ":00+00:00") if expected else end + actual = IncrementalUpdater().determine_update_range(storage, "SPY", instant(day, clock), end) + assert actual == (expected, end, "none" if expected == end else "incremental") + assert storage.get_metadata("SPY") == before + assert_frame_equal(storage.read("SPY").collect(), frame) + + +def test_incremental_disabled_calendar_keeps_row_mode_without_skipping_a_day(storage): + frame = rows("2026-09-05", "2026-09-05", ["12:00"]) + storage.write(frame, "SPY", declaration(None).to_dict()) + start = instant("2026-09-05", "12:00") + end = instant("2026-09-05", "12:02") + assert IncrementalUpdater().determine_update_range(storage, "SPY", start, end) == ( + start, + end, + "incremental", + ) + + +@pytest.mark.parametrize("calendar,expected_day", [("NYSE", "2025-07-14"), ("24/7", "2025-07-12")]) +def test_incremental_daily_labels_inherit_the_stored_calendar(storage, calendar, expected_day): + frame = rows("2025-07-11", "2025-07-11", ["00:00"], "daily") + storage.write( + frame, + "SPY", + FeedSpec( + time_contract_version=2, + calendar=calendar, + data_frequency="daily", + timestamp_semantics="session_label", + ).to_dict(), + ) + end = instant("2025-07-14", "00:00") + assert IncrementalUpdater().determine_update_range( + storage, "SPY", instant("2025-07-11", "00:00"), end + ) == (instant(expected_day, "00:00"), end, "incremental") + + +def test_incremental_explicit_calendar_overrides_stored_schedule(storage): + frame = rows("2026-09-05", "2026-09-05", ["12:00"]) + storage.write(frame, "SPY", declaration("NYSE").to_dict()) + start = instant("2026-09-05", "12:00") + end = instant("2026-09-05", "12:02") + updater = IncrementalUpdater() + assert updater.determine_update_range(storage, "SPY", start, end, calendar="24/7") == ( + instant("2026-09-05", "12:01"), + end, + "incremental", + ) + assert updater.determine_update_range(storage, "SPY", start, end, calendar=None) == ( + start, + end, + "incremental", + ) + with pytest.raises(ValueError, match="requested_start"): + updater.determine_update_range(storage, "SPY", end, start) diff --git a/tests/test_targeted_databento_acquisition.py b/tests/test_targeted_databento_acquisition.py new file mode 100644 index 00000000..27ad6078 --- /dev/null +++ b/tests/test_targeted_databento_acquisition.py @@ -0,0 +1,476 @@ +"""Real provider/storage workflows with only the external Databento SDK stubbed.""" + +from datetime import UTC, datetime, timedelta +from pathlib import Path +from types import SimpleNamespace + +import polars as pl +import pytest +from polars.testing import assert_frame_equal + +pytest.importorskip("databento") + +from ml4t.data.providers import SyntheticProvider +from ml4t.data.providers.databento import DataBentoProvider +from ml4t.data.sessions import SessionCompleter +from ml4t.data.storage.config import StorageConfig +from ml4t.data.storage.flat import FlatStorage +from ml4t.data.storage.hive import HiveStorage +from ml4t.data.storage.metadata_tracker import MetadataTracker +from ml4t.data.update_manager import ( + BackfillManager, + GapDetector, + IncrementalUpdater, + UpdateStrategy, +) +from ml4t.specs import FeedSpec + + +def instant(clock): + return datetime.fromisoformat(f"2026-09-05T{clock}:00+00:00") + + +class RecordedStore: + def __init__(self, frame): + self.frame = frame + + def to_df(self): + return ( + self.frame.rename({"timestamp": "ts_event"}) + .drop("symbol") + .with_columns(pl.lit("BTCU6").alias("raw_symbol")) + .to_pandas() + ) + + +@pytest.fixture +def source(monkeypatch): + with SyntheticProvider(seed=42, calendar_mode="continuous") as producer: + frame = producer.fetch_ohlcv("BTC.c.0", "2026-09-05", "2026-09-05", "minute") + frame = frame.filter( + pl.col("timestamp").dt.strftime("%H:%M").is_in(["12:00", "12:01", "12:02"]) + ) + calls = [] + + def get_range(**kwargs): + calls.append(kwargs) + return RecordedStore(frame) + + client = SimpleNamespace(timeseries=SimpleNamespace(get_range=get_range)) + monkeypatch.setattr("ml4t.data.providers.databento.Historical", lambda _: client) + with DataBentoProvider(api_key="offline-unit-test-key") as provider: + yield provider, frame, calls + + +def test_databento_timestamp_request_is_not_rounded_or_shifted_to_session_dates(source): + provider, expected, calls = source + provider.adjust_session_dates = True + result = provider.fetch_ohlcv_range( + "BTC.c.0", instant("12:01"), instant("12:02"), "minute", stype_in="continuous" + ) + assert calls == [ + { + "dataset": "GLBX.MDP3", + "start": instant("12:01"), + "end": instant("12:02"), + "symbols": ["BTC.c.0"], + "schema": "ohlcv-1m", + "stype_in": "continuous", + } + ] + assert_frame_equal( + result.select(expected.columns), expected.filter(pl.col("timestamp") == instant("12:01")) + ) + assert result["contract"].to_list() == ["BTCU6"] + + +@pytest.mark.parametrize("backend", [FlatStorage, HiveStorage]) +def test_calendar_report_downloads_only_actual_cme_observation_and_preserves_stored_rows( + source, backend, tmp_path +): + provider, expected, calls = source + provider.stype_in = "continuous" + complete = provider._validate_ohlcv( + provider._transform_data(RecordedStore(expected), "BTC.c.0"), provider.name, "BTC.c.0" + ) + observed = complete.filter(pl.col("timestamp") != instant("12:01")) + storage = backend(StorageConfig(base_path=tmp_path)) + feed = FeedSpec( + time_contract_version=2, + calendar="CME Globex Crypto", + data_frequency="minute", + timestamp_semantics="event_time", + bar_grid_anchor="session_open", + bar_label="left", + partial_bar_policy="include", + ) + key = "BTC.c.0" + storage.write(observed, key, {**feed.to_dict(), "provider": "databento"}) + report = GapDetector().detect_gaps_in_storage(storage, key, instant("12:00"), instant("12:02")) + assert report == [{"start": instant("12:01"), "end": instant("12:01"), "size_bars": 1}] + result = BackfillManager(storage, MetadataTracker(tmp_path)).execute_backfill( + key, report, provider + ) + assert result.success, result.errors + assert result.rows_added == 1 + assert result.remaining_gaps == [] + assert len(calls) == 1 + assert calls[0]["schema"] == "ohlcv-1m" + assert calls[0]["stype_in"] == "continuous" + assert calls[0]["start"] == instant("12:01") + assert calls[0]["end"] > instant("12:01") + assert calls[0]["end"] <= instant("12:02") + actual = storage.read(key).collect() + assert_frame_equal(actual, complete) + assert_frame_equal(actual.filter(pl.col("timestamp") != instant("12:01")), observed) + assert ( + FeedSpec.from_any(storage.get_metadata(key)["custom"]).resolve_calendar() + == "CME Globex Crypto" + ) + + +@pytest.mark.parametrize( + "start,end", + [ + (datetime(2026, 9, 5, 12, 1), instant("12:02")), + (instant("12:01"), datetime(2026, 9, 5, 12, 2)), + (instant("12:01"), instant("12:01")), + (instant("12:02"), instant("12:01")), + ], +) +def test_invalid_timestamp_bounds_are_rejected_before_a_billable_request(source, start, end): + provider, _, calls = source + with pytest.raises(ValueError, match="aware|before|UTC|range"): + provider.fetch_ohlcv_range("BTC.c.0", start, end, "minute") + assert calls == [] + + +@pytest.mark.parametrize("frequency", ["ticks", "weekly", "nonsense"]) +def test_unsupported_targeted_schema_is_not_silently_downloaded_as_daily(source, frequency): + provider, _, calls = source + with pytest.raises(ValueError, match="frequency|OHLCV|supported"): + provider.fetch_ohlcv_range("BTC.c.0", instant("12:01"), instant("12:02"), frequency) + assert calls == [] + + +def test_targeted_single_series_cannot_expand_a_parent_into_unrelated_contracts(source): + provider, _, calls = source + with pytest.raises(ValueError, match="stype|single|parent"): + provider.fetch_ohlcv_range( + "BTC.FUT", instant("12:01"), instant("12:02"), "minute", stype_in="parent" + ) + assert calls == [] + + +def stored_source(source, backend, tmp_path, *, key="BTC.c.0", alias=None): + provider, expected, calls = source + provider.stype_in = "continuous" + complete = provider._validate_ohlcv( + provider._transform_data(RecordedStore(expected), "BTC.c.0"), provider.name, "BTC.c.0" + ) + if alias is not None: + complete = complete.with_columns(pl.lit(alias).alias("symbol")) + observed = complete.filter(pl.col("timestamp") != instant("12:01")) + storage = backend(StorageConfig(base_path=tmp_path)) + declaration = FeedSpec( + time_contract_version=2, + calendar="CME Globex Crypto", + data_frequency="minute", + timestamp_semantics="event_time", + bar_grid_anchor="session_open", + bar_label="left", + partial_bar_policy="include", + ) + storage.write(observed, key, {**declaration.to_dict(), "provider": "databento"}) + report = GapDetector().detect_gaps_in_storage(storage, key, instant("12:00"), instant("12:02")) + return provider, complete, observed, calls, storage, report + + +@pytest.mark.parametrize("backend", [FlatStorage, HiveStorage]) +def test_storage_key_is_not_mistaken_for_the_vendor_instrument(source, backend, tmp_path): + key = "futures/minute/BTC" + provider, complete, observed, calls, storage, report = stored_source( + source, backend, tmp_path, key=key, alias="BTC" + ) + result = BackfillManager(storage, MetadataTracker(tmp_path)).execute_backfill( + key, report, provider, source_symbol="BTC.c.0" + ) + assert result.success, result.errors + assert result.rows_added == 1 + assert result.remaining_gaps == [] + assert calls[0]["symbols"] == ["BTC.c.0"] + actual = storage.read(key).collect() + assert_frame_equal(actual.select(complete.columns), complete) + assert_frame_equal( + actual.filter(pl.col("timestamp") != instant("12:01")).select(observed.columns), observed + ) + assert actual.filter(pl.col("timestamp") == instant("12:01"))["contract"].item() == "BTCU6" + + +@pytest.mark.parametrize("backend", [FlatStorage, HiveStorage]) +def test_no_trade_response_keeps_the_actual_gap_and_does_not_rewrite_storage( + source, backend, tmp_path, monkeypatch +): + provider, _, observed, calls, storage, report = stored_source(source, backend, tmp_path) + original = storage.get_metadata("BTC.c.0") + + def no_trades(**kwargs): + calls.append(kwargs) + return RecordedStore(observed.head(0)) + + monkeypatch.setattr(provider.client.timeseries, "get_range", no_trades) + result = BackfillManager(storage, MetadataTracker(tmp_path)).execute_backfill( + "BTC.c.0", report, provider + ) + assert result.success, result.errors + assert result.rows_added == result.gaps_filled == 0 + assert result.remaining_gaps == report + assert len(calls) == 1 + assert storage.get_metadata("BTC.c.0") == original + assert_frame_equal(storage.read("BTC.c.0").collect(), observed) + + +@pytest.mark.parametrize("backend", [FlatStorage, HiveStorage]) +def test_a_stale_report_does_not_redownload_an_observation_already_recovered( + source, backend, tmp_path +): + provider, complete, _, calls, storage, report = stored_source(source, backend, tmp_path) + storage.write(complete, "BTC.c.0", preserve_metadata=True) + original = storage.get_metadata("BTC.c.0") + result = BackfillManager(storage, MetadataTracker(tmp_path)).execute_backfill( + "BTC.c.0", report, provider + ) + assert result.success, result.errors + assert result.rows_added == result.gaps_filled == 0 + assert result.remaining_gaps == [] + assert calls == [] + assert storage.get_metadata("BTC.c.0") == original + + +def test_timezone_aware_offsets_preserve_the_exact_requested_instants(source): + provider, _, calls = source + start = datetime.fromisoformat("2026-09-05T08:01:00-04:00") + provider.fetch_ohlcv_range("BTC.c.0", start, start + timedelta(minutes=1), "minute") + assert calls[0]["start"] == instant("12:01") + assert calls[0]["end"] == instant("12:02") + + +@pytest.mark.parametrize("backend", [FlatStorage, HiveStorage]) +def test_real_nullable_grid_placeholder_is_replaced_only_by_a_source_observation( + source, backend, tmp_path +): + provider, complete, observed, calls, storage, _ = stored_source(source, backend, tmp_path) + expanded = SessionCompleter("CME Globex Crypto").complete_sessions(observed) + assert expanded["is_imputed"].to_list() == [False, True, False] + storage.write(expanded, "BTC.c.0", preserve_metadata=True) + report = GapDetector().detect_gaps_in_storage( + storage, "BTC.c.0", instant("12:00"), instant("12:02") + ) + result = BackfillManager(storage, MetadataTracker(tmp_path)).execute_backfill( + "BTC.c.0", report, provider + ) + assert result.success, result.errors + assert result.rows_added == 0 + assert result.rows_updated == 1 + assert result.remaining_gaps == [] + actual = storage.read("BTC.c.0").collect() + assert_frame_equal(actual.select(complete.columns), complete) + assert not actual["is_imputed"].any() + assert len(calls) == 1 + + +@pytest.mark.parametrize("backend", [FlatStorage, HiveStorage]) +def test_generic_backfill_cannot_put_another_instrument_in_a_missing_bar(source, backend, tmp_path): + _, complete, observed, _, storage, _ = stored_source(source, backend, tmp_path) + wrong = complete.with_columns(pl.lit("MBT.C.0").alias("symbol")) + original = storage.get_metadata("BTC.c.0") + with pytest.raises(ValueError, match="instrument|entity|identity"): + IncrementalUpdater().apply_strategy(storage, "BTC.c.0", wrong, UpdateStrategy.BACKFILL) + assert_frame_equal(storage.read("BTC.c.0").collect(), observed) + assert storage.get_metadata("BTC.c.0") == original + + +@pytest.mark.parametrize("symbol", ["ALL_SYMBOLS", " ALL_SYMBOLS ", "ALL_SYMBOLS,", "BTCU6,MBTU6"]) +def test_targeted_single_series_cannot_expand_to_all_vendor_instruments(source, symbol): + provider, _, calls = source + with pytest.raises(ValueError, match="single|universe|instrument"): + provider.fetch_ohlcv_range(symbol, instant("12:01"), instant("12:02"), "minute") + assert calls == [] + + +@pytest.mark.parametrize("frequency", ["1min", "1minute"]) +def test_native_minute_alias_does_not_become_daily_or_lose_targeted_support(source, frequency): + provider, _, calls = source + frame = provider.fetch_ohlcv_range("BTC.c.0", instant("12:01"), instant("12:02"), frequency) + assert len(frame) == 1 + assert calls[0]["schema"] == "ohlcv-1m" + + +@pytest.mark.parametrize("backend", [FlatStorage, HiveStorage]) +def test_partial_source_response_reports_the_remaining_missing_bar(source, backend, tmp_path): + provider, complete, _, calls, storage, _ = stored_source(source, backend, tmp_path) + observed = complete.head(1) + storage.write(observed, "BTC.c.0", preserve_metadata=True) + report = GapDetector().detect_gaps_in_storage( + storage, "BTC.c.0", instant("12:00"), instant("12:03") + ) + result = BackfillManager(storage, MetadataTracker(tmp_path)).execute_backfill( + "BTC.c.0", report, provider + ) + assert result.success, result.errors + assert result.rows_added == 2 + assert result.gaps_filled == 0 + assert result.remaining_gaps == [ + {"start": instant("12:03"), "end": instant("12:03"), "size_bars": 1} + ] + assert len(calls) == 1 + assert_frame_equal(storage.read("BTC.c.0").collect(), complete) + + +@pytest.mark.parametrize("backend", [FlatStorage, HiveStorage]) +def test_leading_and_trailing_report_ranges_are_both_recovered(source, backend, tmp_path): + provider, complete, _, calls, storage, _ = stored_source(source, backend, tmp_path) + storage.write(complete.slice(1, 1), "BTC.c.0", preserve_metadata=True) + report = GapDetector().detect_gaps_in_storage( + storage, "BTC.c.0", instant("12:00"), instant("12:02") + ) + result = BackfillManager(storage, MetadataTracker(tmp_path)).execute_backfill( + "BTC.c.0", report, provider + ) + assert result.success, result.errors + assert result.rows_added == 2 + assert result.gaps_filled == 2 + assert result.remaining_gaps == [] + assert len(calls) == 2 + assert_frame_equal(storage.read("BTC.c.0").collect(), complete) + + +@pytest.mark.parametrize("backend", [FlatStorage, HiveStorage]) +def test_gap_recovery_does_not_replace_a_present_observation_containing_a_null( + source, backend, tmp_path +): + provider, _, observed, _, storage, report = stored_source(source, backend, tmp_path) + observed = observed.with_columns(pl.lit(None, dtype=pl.Float64).alias("close")) + storage.write(observed, "BTC.c.0", preserve_metadata=True) + result = BackfillManager(storage, MetadataTracker(tmp_path)).execute_backfill( + "BTC.c.0", report, provider + ) + assert result.success, result.errors + assert result.rows_added == 1 + assert_frame_equal( + storage.read("BTC.c.0").collect().filter(pl.col("timestamp") != instant("12:01")), + observed, + ) + + +@pytest.mark.parametrize("backend", [FlatStorage, HiveStorage]) +def test_intraday_candidate_scan_uses_bar_counts_instead_of_discarding_the_dataset( + source, backend, tmp_path +): + _, _, _, _, storage, report = stored_source(source, backend, tmp_path) + candidates = BackfillManager(storage, MetadataTracker(tmp_path)).identify_candidates( + max_age_days=10000, min_gap_bars=1 + ) + assert len(candidates) == 1 + assert candidates[0]["gaps"] == report + assert candidates[0]["total_gap_bars"] == 1 + assert "total_gap_days" not in candidates[0] + + +@pytest.mark.parametrize("backend", [FlatStorage, HiveStorage]) +def test_documented_targeted_download_uses_actual_report_and_storage( + source, backend, tmp_path, monkeypatch +): + key = "futures/minute/BTC" + _, complete, _, calls, storage, _ = stored_source( + source, backend, tmp_path, key=key, alias="BTC" + ) + monkeypatch.setenv("DATABENTO_API_KEY", "offline-unit-test-key") + guide = Path(__file__).resolve().parents[1] / "docs/user-guide/incremental-updates.md" + section = guide.read_text().split("### Download Real Observations for a Selected Report", 1)[1] + code = section.split("```python\n", 1)[1].split("```", 1)[0] + namespace = { + "storage": storage, + "feed": FeedSpec.from_any(storage.get_metadata(key)["custom"]), + "GapDetector": GapDetector, + "datetime": datetime, + "UTC": UTC, + } + exec(compile(code, str(guide), "exec"), namespace) + assert namespace["result"].rows_added == 1 + assert namespace["result"].remaining_gaps == [] + assert len(calls) == 1 + assert calls[0]["symbols"] == ["BTC.c.0"] + assert_frame_equal(storage.read(key).collect().select(complete.columns), complete) + + +def test_documented_native_timestamp_request_uses_exact_half_open_bounds(source, monkeypatch): + _, _, calls = source + monkeypatch.setenv("DATABENTO_API_KEY", "offline-unit-test-key") + guide = Path(__file__).resolve().parents[1] / "docs/providers/databento.md" + section = guide.read_text().split("## Exact Timestamp Ranges", 1)[1] + code = section.split("```python\n", 1)[1].split("```", 1)[0] + namespace = {"DataBentoProvider": DataBentoProvider} + exec(compile(code, str(guide), "exec"), namespace) + assert len(namespace["rows"]) == 1 + assert calls[0]["start"] == instant("12:01") + assert calls[0]["end"] == instant("12:02") + assert calls[0]["stype_in"] == "continuous" + + +@pytest.mark.parametrize("mode", ["unsupported_provider", "wrong_provider", "disabled"]) +def test_unsupported_recovery_fails_without_vendor_calls_or_storage_mutation( + source, tmp_path, mode +): + provider, _, observed, calls, storage, report = stored_source(source, FlatStorage, tmp_path) + kwargs = {} + if mode == "wrong_provider": + storage.write(observed, "BTC.c.0", {"provider": "coinmetrics"}, preserve_metadata=True) + elif mode == "disabled": + kwargs["calendar"] = None + before = storage.get_metadata("BTC.c.0") + with SyntheticProvider() as unsupported: + result = BackfillManager(storage, MetadataTracker(tmp_path)).execute_backfill( + "BTC.c.0", report, unsupported if mode == "unsupported_provider" else provider, **kwargs + ) + assert not result.success + assert result.errors + assert calls == [] + assert storage.get_metadata("BTC.c.0") == before + assert_frame_equal(storage.read("BTC.c.0").collect(), observed) + + +def test_history_write_failure_still_reports_the_observations_actually_published(source, tmp_path): + provider, complete, _, _, storage, report = stored_source(source, FlatStorage, tmp_path) + tracker = MetadataTracker(tmp_path) + tracker._get_metadata_path("BTC.c.0").mkdir() + result = BackfillManager(storage, tracker).execute_backfill("BTC.c.0", report, provider) + assert not result.success + assert any( + error.startswith("Observations stored but update history failed:") + for error in result.errors + ) + assert result.rows_added == 1 + assert result.rows_after == 3 + assert result.remaining_gaps == [] + assert_frame_equal(storage.read("BTC.c.0").collect(), complete) + + +def test_an_empty_actual_report_completes_without_download_or_storage_changes(source, tmp_path): + provider, complete, _, calls, storage, _ = stored_source(source, FlatStorage, tmp_path) + storage.write(complete, "BTC.c.0", preserve_metadata=True) + report = GapDetector().detect_gaps_in_storage( + storage, "BTC.c.0", instant("12:00"), instant("12:02") + ) + assert report == [] + before = storage.get_metadata("BTC.c.0") + result = BackfillManager(storage, MetadataTracker(tmp_path)).execute_backfill( + "BTC.c.0", report, provider + ) + assert result.success, result.errors + assert result.rows_added == result.gaps_filled == 0 + assert result.rows_before == result.rows_after == 3 + assert result.remaining_gaps == [] + assert calls == [] + assert storage.get_metadata("BTC.c.0") == before + assert_frame_equal(storage.read("BTC.c.0").collect(), complete) diff --git a/tests/test_update_manager.py b/tests/test_update_manager.py index 71826b07..16feb6a5 100644 --- a/tests/test_update_manager.py +++ b/tests/test_update_manager.py @@ -30,6 +30,7 @@ UpdateStrategy, _ensure_datetime, ) +from ml4t.specs import FeedSpec class TestUpdateStrategy: @@ -190,10 +191,10 @@ def test_detect_gaps_empty_dataframe(self, detector: GapDetector): assert len(gaps) == 0 def test_detect_gaps_missing_timestamp(self, detector: GapDetector): - """Missing timestamp column returns no gaps.""" + """A nonempty frame without timestamps cannot be certified as complete.""" df = pl.DataFrame({"close": [100, 101, 102]}) - gaps = detector.detect_gaps(df) - assert len(gaps) == 0 + with pytest.raises(ValueError, match="timestamp"): + detector.detect_gaps(df) def test_detect_gaps_with_tolerance(self, detector: GapDetector, data_with_gaps: pl.DataFrame): """Tolerance days affects gap detection.""" @@ -216,8 +217,21 @@ def test_detect_gaps_hourly(self, detector: GapDetector): } ) - gaps = detector.detect_gaps(df, frequency="hourly") - assert len(gaps) >= 1 + gaps = detector.detect_gaps( + df, + frequency="hourly", + feed_spec=FeedSpec( + time_contract_version=2, + calendar="24/7", + timezone="UTC", + data_frequency="hourly", + timestamp_semantics="event_time", + bar_grid_anchor="session_open", + bar_label="left", + partial_bar_policy="include", + ), + ) + assert [gap["size_bars"] for gap in gaps] == [5] def test_ensure_datetime_with_date(self): """Ensure date objects are converted to datetime.""" @@ -260,10 +274,10 @@ def test_expected_delta_weekly(self, detector: GapDetector): delta = detector._get_expected_delta("weekly") assert delta == timedelta(weeks=1) - def test_expected_delta_default(self, detector: GapDetector): - """Unknown frequency defaults to daily.""" - delta = detector._get_expected_delta("unknown") - assert delta == timedelta(days=1) + def test_expected_delta_rejects_unknown_frequency(self, detector: GapDetector): + """An unsupported frequency must not silently change the observation unit.""" + with pytest.raises(ValueError, match="frequency"): + detector._get_expected_delta("unknown") class TestGapDetectorStorage: @@ -784,12 +798,14 @@ def test_execute_backfill_no_gaps(self, manager: BackfillManager): result = manager.execute_backfill("TEST", [], mock_provider) - assert result.success is False # No gaps filled + assert result.success assert result.gaps_filled == 0 + mock_provider.fetch_ohlcv.assert_not_called() def test_execute_backfill_with_mocked_provider(self, manager: BackfillManager): """Execute backfill with mocked provider.""" mock_provider = MagicMock() + mock_provider.name = "mock" # Mock fetch_ohlcv to return data mock_data = pl.DataFrame( @@ -812,11 +828,12 @@ def test_execute_backfill_with_mocked_provider(self, manager: BackfillManager): } ] - _result = manager.execute_backfill("TEST", gaps, mock_provider) + result = manager.execute_backfill("TEST", gaps, mock_provider) # Provider should have been called mock_provider.fetch_ohlcv.assert_called() - assert _result is not None + assert result.success, result.errors + assert result.rows_added == 2 class TestBackfillManagerPrioritization: diff --git a/tests/test_validation.py b/tests/test_validation.py index e83f8fcd..f477dec1 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -410,7 +410,7 @@ def test_rule_presets(self): assert equity_rules.max_return_threshold == 0.2 crypto_rules = ValidationRulePresets.crypto_rules() - assert not crypto_rules.check_weekend_trading + assert crypto_rules.check_weekend_trading assert crypto_rules.max_return_threshold == 0.5 strict_rules = ValidationRulePresets.strict_rules() diff --git a/tests/test_validation_calendar_contract.py b/tests/test_validation_calendar_contract.py new file mode 100644 index 00000000..4f5ffc49 --- /dev/null +++ b/tests/test_validation_calendar_contract.py @@ -0,0 +1,280 @@ +"""Public provider/validator/storage workflows must use the declared schedule.""" + +from datetime import date + +import polars as pl +import pytest +from polars.testing import assert_frame_equal + +from ml4t.data import DataManager +from ml4t.data.assets import AssetClass, AssetInfo, AssetValidator +from ml4t.data.calendar.crypto import CryptoCalendar, CryptoSessionValidator +from ml4t.data.providers import SyntheticProvider +from ml4t.data.sessions import SessionAssigner +from ml4t.data.storage.config import StorageConfig +from ml4t.data.storage.flat import FlatStorage +from ml4t.data.storage.hive import HiveStorage +from ml4t.data.validation.cross_validation import CrossValidator +from ml4t.data.validation.rules import ValidationRuleConfig, ValidationRuleSet +from ml4t.specs import UNSET_CALENDAR, FeedSpec + + +def minute_observation(day: str, clock: str) -> pl.DataFrame: + with SyntheticProvider(calendar_mode="continuous") as provider: + frame = provider.fetch_ohlcv("SPY", day, day, "minute") + selected = frame.filter(pl.col("timestamp").dt.strftime("%H:%M") == clock) + assert selected.height == 1 + return selected + + +def temporal_issues(result): + return [issue for issue in result.issues if issue.check in {"weekend_trading", "market_hours"}] + + +def test_daily_provider_labels_are_not_after_hours_instants(): + with SyntheticProvider() as provider: + frame = provider.fetch_ohlcv("SPY", "2025-07-07", "2025-07-11", "daily") + before = frame.clone() + result = AssetValidator(AssetInfo(symbol="SPY", asset_class=AssetClass.EQUITY)).validate( + frame, "daily" + ) + assert not any("market hours" in issue for issue in result.get("equity_specific", [])) + assert_frame_equal(frame, before) + + +def test_summer_nyse_open_is_not_rejected_by_winter_utc_clock(): + frame = minute_observation("2025-07-07", "13:30") + assert temporal_issues(CrossValidator().validate(frame, frequency="minute")) == [] + + +@pytest.mark.parametrize("day,clock", [("2025-07-03", "17:01"), ("2025-07-04", "15:00")]) +def test_nyse_early_close_and_holiday_are_not_regular_market_hours(day, clock): + frame = minute_observation(day, clock) + result = CrossValidator().validate(frame, frequency="minute") + assert any(issue.check == "market_hours" for issue in temporal_issues(result)) + + +def test_disabled_cross_validator_does_not_reinstate_weekend_rules(): + frame = minute_observation("2025-07-12", "18:00") + before = frame.clone() + result = CrossValidator().validate(frame, frequency="minute", calendar=None) + assert temporal_issues(result) == [] + assert_frame_equal(frame, before) + + +def test_disabled_crypto_validator_does_not_reinstate_gap_calendar(): + with SyntheticProvider(calendar_mode="continuous") as provider: + frame = provider.fetch_ohlcv("BTCUSD", "2025-07-07", "2025-07-09", "daily") + frame = frame.filter(pl.col("timestamp").dt.day() != 8) + assert CryptoSessionValidator(calendar=None).validate_continuity(frame, "daily") == [] + + +@pytest.mark.parametrize("validator", ["cross", "asset"]) +def test_explicit_override_metadata_and_disabled_metadata_have_same_membership(validator): + frame = minute_observation("2025-07-12", "18:00") + feed = FeedSpec(calendar="24/7", timestamp_semantics="event_time", time_contract_version=2) + if validator == "cross": + instance = CrossValidator() + + def has_temporal(**kw): + return bool(temporal_issues(instance.validate(frame, **kw))) + else: + instance = AssetValidator(AssetInfo(symbol="SPY", asset_class=AssetClass.EQUITY)) + + def has_temporal(**kw): + return bool(instance.validate(frame, **kw).get("equity_specific")) + + assert not has_temporal(frequency="minute", feed_spec=feed) + assert has_temporal(frequency="minute", calendar="NYSE", feed_spec=feed) + assert not has_temporal(frequency="minute", calendar=None, feed_spec=feed) + assert not has_temporal( + frequency="minute", feed_spec=FeedSpec(calendar=None, time_contract_version=2) + ) + assert has_temporal(frequency="minute", feed_spec=FeedSpec()) + + +def test_disabled_temporal_validation_keeps_price_and_volume_checks(): + frame = minute_observation("2025-07-12", "18:00").with_columns( + pl.lit(-1.0).alias("volume"), pl.lit(-1.0).alias("close") + ) + result = AssetValidator(AssetInfo(symbol="SPY", asset_class=AssetClass.EQUITY)).validate( + frame, "minute", calendar=None + ) + assert any("non-positive" in item for item in result.get("equity_specific", [])) + assert result.get("data_quality") + zero_volume = frame.with_columns(pl.lit(0.0).alias("volume")) + assert CryptoSessionValidator(None).validate_volume_profile(zero_volume) == [ + "Found 1 periods with zero volume" + ] + + +def test_declared_daily_event_instant_overrides_frequency_and_date_label_heuristics(): + frame = minute_observation("2025-07-06", "23:30") + feed = FeedSpec(calendar="CME Globex Crypto", timestamp_semantics="event_time") + assert temporal_issues(CrossValidator(feed_spec=feed).validate(frame, frequency="daily")) == [] + + +def test_closed_instant_weekend_reporting_uses_selected_local_day_not_utc_date(): + frame = minute_observation("2025-07-12", "01:00") + result = CrossValidator().validate(frame, frequency="minute") + assert [issue.check for issue in temporal_issues(result)] == ["market_hours"] + + +def test_cross_validator_daily_feed_timezone_preserves_the_session_label(): + with SyntheticProvider() as provider: + frame = provider.fetch_ohlcv("SPY", "2025-07-07", "2025-07-07", "daily") + frame = frame.with_columns(pl.col("timestamp").dt.replace(hour=0)) + local = frame.with_columns(pl.col("timestamp").dt.replace_time_zone("Asia/Tokyo")) + utc = local.with_columns(pl.col("timestamp").dt.convert_time_zone("UTC")) + assert utc["timestamp"].dt.date().to_list() == [date(2025, 7, 6)] + feed = FeedSpec( + calendar="NYSE", + timezone="Asia/Tokyo", + data_frequency="daily", + timestamp_semantics="session_label", + ) + assert temporal_issues(CrossValidator(feed_spec=feed).validate(utc)) == [] + + +def test_explicit_legacy_crypto_object_retains_its_continuous_gap_contract(): + with SyntheticProvider(calendar_mode="continuous") as provider: + frame = provider.fetch_ohlcv("BTCUSD", "2025-07-11", "2025-07-14", "daily") + observed = frame.filter(pl.col("timestamp").dt.weekday() <= 5) + assert len(CryptoSessionValidator(CryptoCalendar()).validate_continuity(observed)) == 1 + assert CryptoSessionValidator().validate_continuity(observed) == [] + assert len(CryptoSessionValidator("24/7").validate_continuity(observed)) == 1 + + +def test_legacy_crypto_hint_cannot_override_explicit_or_declared_calendar(): + frame = minute_observation("2025-07-12", "18:00") + instance = CrossValidator(is_crypto=True) + assert temporal_issues(instance.validate(frame, frequency="minute")) == [] + assert temporal_issues(instance.validate(frame, frequency="minute", calendar="NYSE")) + assert temporal_issues( + instance.validate(frame, frequency="minute", feed_spec=FeedSpec(calendar="NYSE")) + ) + + +@pytest.mark.parametrize("storage_class", [FlatStorage, HiveStorage]) +@pytest.mark.parametrize("operation", ["load", "import"]) +@pytest.mark.parametrize("calendar", ["NYSE", "24/7", None]) +def test_real_storage_validation_receives_frequency_and_effective_feed( + tmp_path, monkeypatch, storage_class, operation, calendar +): + reports = [] + validate = CrossValidator.validate + + def record(self, frame, *args, **kwargs): + report = validate(self, frame, *args, **kwargs) + reports.append(report) + return report + + monkeypatch.setattr(CrossValidator, "validate", record) + storage = storage_class(StorageConfig(base_path=tmp_path)) + providers = {"synthetic": {"extra": {"calendar_mode": "continuous", "seed": 42}}} + feed = FeedSpec(calendar="24/7", timestamp_semantics="event_time", time_contract_version=2) + with DataManager(providers=providers, storage=storage) as manager: + if operation == "load": + manager.load( + "SPY", + "2025-07-12", + "2025-07-12", + provider="synthetic", + frequency="minute", + asset_class="crypto", + calendar=calendar, + feed_spec=feed, + ) + else: + frame = minute_observation("2025-07-12", "18:00") + manager.import_data( + frame, + "SPY", + provider="synthetic", + frequency="minute", + asset_class="crypto", + calendar=calendar, + feed_spec=feed, + ) + assert len(reports) == 1 + assert bool(temporal_issues(reports[0])) == (calendar == "NYSE") + + +@pytest.mark.parametrize("calendar", [UNSET_CALENDAR, None, "NYSE", "24/7"]) +def test_validation_rule_json_yaml_copy_preserves_omitted_disabled_and_named_calendars( + tmp_path, calendar +): + frame = minute_observation("2025-07-12", "18:00") + config = ValidationRuleConfig(calendar=calendar) + wire = config.model_dump(mode="json") + assert ("calendar" in wire) == (calendar is not UNSET_CALENDAR) + rules = ValidationRuleSet(name="calendar_rules", default_rule=config) + path = tmp_path / "rules.yaml" + rules.save(path) + restored = ValidationRuleSet.load(path).get_rules("SPY") + copies = [ + config.model_copy(), + ValidationRuleConfig.model_validate_json(config.model_dump_json()), + restored, + ] + for copy in copies: + assert copy.calendar is calendar or copy.calendar == calendar + issues = temporal_issues(CrossValidator(feed_spec=copy).validate(frame, frequency="minute")) + assert bool(issues) == (calendar is UNSET_CALENDAR or calendar == "NYSE") + assert not temporal_issues( + CrossValidator(feed_spec=copy).validate(frame, frequency="minute", calendar=None) + ) + + +def test_legacy_rule_null_calendar_still_inherits_default_instead_of_disabling(): + config = ValidationRuleConfig.model_validate({"calendar": None, "time_contract_version": 1}) + assert FeedSpec.from_any(config).resolve_calendar() == "NYSE" + frame = minute_observation("2025-07-12", "18:00") + assert temporal_issues(CrossValidator(feed_spec=config).validate(frame, frequency="minute")) + + +@pytest.mark.parametrize("frequency", ["daily", "1d", "1day"]) +def test_crypto_continuity_preserves_supported_daily_frequency_aliases(frequency): + with SyntheticProvider(calendar_mode="continuous") as provider: + frame = provider.fetch_ohlcv("BTCUSD", "2025-07-11", "2025-07-14", "daily") + observed = frame.filter(pl.col("timestamp").dt.weekday() <= 5) + assert len(CryptoSessionValidator("24/7").validate_continuity(observed, frequency)) == 1 + + +@pytest.mark.parametrize("calendar,last_minute", [("NYSE", "19:59"), ("HKEX", "03:59")]) +@pytest.mark.parametrize("offset_ns", [-1, 0, 1]) +def test_nanosecond_close_labels_respect_exact_session_and_break_boundaries( + calendar, last_minute, offset_ns +): + frame = minute_observation("2025-07-07", last_minute).with_columns( + pl.col("timestamp").cast(pl.Datetime("ns", "UTC")) + + pl.duration(minutes=1, nanoseconds=offset_ns) + ) + before = frame.clone() + feed = FeedSpec(calendar=calendar, timestamp_semantics="bar_close") + assigner = SessionAssigner(feed_spec=feed) + assigned = assigner.assign_sessions(frame, bar_frequency="intraday") + assert assigned["session_date"].to_list() == ([None] if offset_ns > 0 else [date(2025, 7, 7)]) + report = CrossValidator(feed_spec=feed).validate(frame, frequency="minute") + assert bool(temporal_issues(report)) == (offset_ns > 0) + if offset_ns > 0: + with pytest.raises(ValueError, match="outside a trading session"): + assigner.assign_sessions(frame, bar_frequency="intraday", outside_session="raise") + assert assigner.assign_sessions( + frame, bar_frequency="intraday", outside_session="drop" + ).is_empty() + assert_frame_equal(frame, before) + + +def test_nullable_nanosecond_instants_keep_original_precision_and_null_policy(): + frame = minute_observation("2025-07-07", "13:30").with_columns( + pl.col("timestamp").cast(pl.Datetime("ns", "UTC")) + pl.duration(nanoseconds=1) + ) + missing = frame.with_columns(pl.lit(None, dtype=pl.Datetime("ns", "UTC")).alias("timestamp")) + observed = pl.concat([frame, missing]) + before = observed.clone() + assigned = SessionAssigner( + feed_spec=FeedSpec(timestamp_semantics="event_time") + ).assign_sessions(observed) + assert assigned["session_date"].to_list() == [date(2025, 7, 7), None] + assert_frame_equal(assigned.select(observed.columns), before) diff --git a/tests/test_validation_rules.py b/tests/test_validation_rules.py index fb0e4d1c..975ef28b 100644 --- a/tests/test_validation_rules.py +++ b/tests/test_validation_rules.py @@ -238,8 +238,8 @@ def test_crypto_rules(self): rules = ValidationRulePresets.crypto_rules() assert rules.asset_class == AssetClass.CRYPTO - assert rules.check_weekend_trading is False # Crypto trades 24/7 - assert rules.check_market_hours is False + assert rules.check_weekend_trading is True + assert rules.check_market_hours is True assert rules.max_return_threshold == 0.5 # More volatile assert rules.staleness_threshold == 3 assert rules.volume_spike_threshold == 20.0 @@ -251,7 +251,7 @@ def test_forex_rules(self): assert rules.asset_class == AssetClass.FOREX assert rules.check_weekend_trading is True - assert rules.check_market_hours is False # 24/5 market + assert rules.check_market_hours is True assert rules.max_return_threshold == 0.1 assert rules.staleness_threshold == 10 assert rules.volume_spike_threshold == 5.0 @@ -376,7 +376,7 @@ def test_complete_workflow(self): # Test getting rules for different symbols btc_rules = rule_set.get_rules("BTCUSD") assert btc_rules.asset_class == AssetClass.CRYPTO - assert btc_rules.check_weekend_trading is False + assert btc_rules.check_weekend_trading is True eur_rules = rule_set.get_rules("EURUSD") assert eur_rules.asset_class == AssetClass.FOREX @@ -401,4 +401,4 @@ def test_complete_workflow(self): # Test that loaded rule set works the same loaded_btc_rules = loaded_rule_set.get_rules("BTCUSD") assert loaded_btc_rules.asset_class == AssetClass.CRYPTO - assert loaded_btc_rules.check_weekend_trading is False + assert loaded_btc_rules.check_weekend_trading is True diff --git a/tests/validation/test_rules.py b/tests/validation/test_rules.py index 0f9a2001..bbe9a710 100644 --- a/tests/validation/test_rules.py +++ b/tests/validation/test_rules.py @@ -253,8 +253,8 @@ def test_crypto_rules(self): config = ValidationRulePresets.crypto_rules() assert config.asset_class == AssetClass.CRYPTO - assert config.check_weekend_trading is False # 24/7 trading - assert config.check_market_hours is False # No market hours + assert config.check_weekend_trading is True + assert config.check_market_hours is True assert config.max_return_threshold == 0.5 # Higher volatility assert config.staleness_threshold == 3 assert config.volume_spike_threshold == 20.0 @@ -265,7 +265,7 @@ def test_forex_rules(self): assert config.asset_class == AssetClass.FOREX assert config.check_weekend_trading is True - assert config.check_market_hours is False # 24/5 market + assert config.check_market_hours is True assert config.max_return_threshold == 0.1 # Less volatile assert config.price_gap_threshold == 0.02