diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 57ee7243..ed57e27e 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -3,13 +3,15 @@ name: Docs on: push: branches: [main] + pull_request: + branches: [main] workflow_dispatch: permissions: contents: read concurrency: - group: docs + group: docs-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true env: @@ -20,59 +22,38 @@ env: jobs: build: - name: Build Docs + name: Strict Documentation runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false - - - name: Install Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} - - - name: Install uv - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 + - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 with: version: ${{ env.UV_VERSION }} enable-cache: true - - - name: Install dependencies + - name: Install locked documentation environment run: uv sync --locked --dev --extra docs - - - name: Build docs + - name: Record source identity + run: | + echo "ML4T_DOCS_VERSION=$(uv run python -c 'from ml4t.data import __version__; print(__version__)')" >> "$GITHUB_ENV" + echo "ML4T_DOCS_COMMIT=${GITHUB_SHA}" >> "$GITHUB_ENV" + - name: Build documentation strictly run: uv run mkdocs build --strict - - - name: Upload rendered docs + - name: Verify rendered identity + run: >- + uv run python scripts/verify_documentation_identity.py + --site site + --expected-library data + --expected-version "$ML4T_DOCS_VERSION" + --expected-commit "$ML4T_DOCS_COMMIT" + - name: Upload rendered documentation uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: docs-site + name: docs-site-${{ github.sha }} path: site/ if-no-files-found: error - - deploy: - name: Deploy Docs - runs-on: ubuntu-latest - needs: build - steps: - - name: Download rendered docs - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: docs-site - path: site/ - - - name: Deploy to website repo - uses: cpina/github-action-push-to-another-repository@55306faa4ed53b815ae49e564af8cfb359d32ae2 # v1.7.3 - env: - SSH_DEPLOY_KEY: ${{ secrets.DOCS_DEPLOY_KEY }} - with: - source-directory: site/ - destination-github-username: ml4t - destination-repository-name: website - target-directory: static/docs/data/ - target-branch: main - commit-message: "docs(data): update from ml4t/data@${{ github.sha }}" - user-name: ml4t-bot - user-email: bot@ml4trading.io diff --git a/.github/workflows/ecosystem.yml b/.github/workflows/ecosystem.yml index 21dd3d70..715f4745 100644 --- a/.github/workflows/ecosystem.yml +++ b/.github/workflows/ecosystem.yml @@ -17,7 +17,7 @@ jobs: qualification: permissions: contents: read - uses: ml4t/ecosystem/.github/workflows/qualify-library.yml@457678d8860d0c816b1dc376e7e2691daa93af4d # v0.1.0 + uses: ml4t/ecosystem/.github/workflows/qualify-library.yml@da2cbb9a9d0b4d2f166ad630470ff7d7a1aa2643 # 2026-09-19 policy snapshot with: import-package: ml4t.data prerelease-exception: python-315-polars diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 10f97e0d..c8499049 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,14 +1,22 @@ name: Release on: - push: - tags: ["v*"] + workflow_dispatch: + inputs: + version: + description: Release version without the v prefix + required: true + type: string + candidate-sha: + description: Full commit SHA currently at the tip of main + required: true + type: string permissions: contents: read concurrency: - group: release-${{ github.ref }} + group: release-${{ inputs.version }} cancel-in-progress: false env: @@ -18,149 +26,302 @@ env: NO_MKDOCS_2_WARNING: "1" jobs: + preflight: + name: Validate Unpublished Main Candidate + runs-on: ubuntu-latest + outputs: + commit: ${{ steps.identity.outputs.commit }} + tree: ${{ steps.identity.outputs.tree }} + version: ${{ steps.identity.outputs.version }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.candidate-sha }} + fetch-depth: 0 + persist-credentials: false + - name: Fetch current main without tags + run: git fetch origin main --no-tags + - name: Check candidate identity and public version absence + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + python scripts/check_release_preflight.py + --version "${{ inputs.version }}" + --candidate-sha "${{ inputs.candidate-sha }}" + --repository "${{ github.repository }}" + --workflow-sha "${{ github.sha }}" + - name: Record accepted candidate + id: identity + run: | + { + echo "commit=${{ inputs.candidate-sha }}" + echo "tree=$(git rev-parse 'HEAD^{tree}')" + echo "version=${{ inputs.version }}" + } >> "$GITHUB_OUTPUT" + ecosystem-qualification: name: Ecosystem Qualification + needs: preflight permissions: contents: read - uses: ml4t/ecosystem/.github/workflows/qualify-library.yml@457678d8860d0c816b1dc376e7e2691daa93af4d # v0.1.0 + uses: ml4t/ecosystem/.github/workflows/qualify-library.yml@da2cbb9a9d0b4d2f166ad630470ff7d7a1aa2643 # 2026-09-19 policy snapshot with: import-package: ml4t.data prerelease-exception: python-315-polars compatibility: name: Release Compatibility - uses: ./.github/workflows/compatibility.yml + needs: preflight permissions: contents: read + uses: ./.github/workflows/compatibility.yml quality: - name: Release Quality and Security + name: Source Quality and Security + needs: preflight runs-on: ubuntu-latest + permissions: + contents: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + ref: ${{ needs.preflight.outputs.commit }} fetch-depth: 0 persist-credentials: false - - - name: Install Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} - - - name: Install uv - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 + - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 with: version: ${{ env.UV_VERSION }} enable-cache: true - - - name: Install the complete release environment + - name: Install the complete locked environment run: uv sync --locked --all-extras --all-groups - - - name: Check source quality + - name: Run source quality gates run: | uv run ruff check src/ tests/ scripts/ uv run ruff format --check src/ tests/ scripts/ uv run ty check - - - name: Build documentation strictly - run: uv run mkdocs build --strict - - - name: Audit the complete locked environment + uv run pytest tests/ -q -ra + uv run pytest tests/ -q -ra -W error::ResourceWarning + - name: Audit locked dependencies run: | uv export --locked --all-extras --all-groups --no-emit-project --no-hashes --output-file requirements-audit.txt uv run pip-audit --requirement requirements-audit.txt + - name: Build and verify documentation + env: + ML4T_DOCS_COMMIT: ${{ needs.preflight.outputs.commit }} + ML4T_DOCS_VERSION: ${{ needs.preflight.outputs.version }} + run: | + uv run mkdocs build --strict + uv run python scripts/verify_documentation_identity.py \ + --site site \ + --expected-library data \ + --expected-version "$ML4T_DOCS_VERSION" \ + --expected-commit "$ML4T_DOCS_COMMIT" build: - name: Build Validated Artifacts + name: Build Candidate Once + needs: [preflight, ecosystem-qualification, compatibility, quality] runs-on: ubuntu-latest - needs: [ecosystem-qualification, compatibility, quality] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + ref: ${{ needs.preflight.outputs.commit }} fetch-depth: 0 persist-credentials: false - - - name: Install Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ env.PYTHON_VERSION }} - - - name: Install uv - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 + - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 with: version: ${{ env.UV_VERSION }} enable-cache: false - - - name: Install verification tooling - run: uv sync --locked --dev - - - name: Build distributions - run: uv build --out-dir dist/packages - - - name: Check package metadata rendering - run: uv run twine check dist/packages/* - - - name: Validate tag, metadata, installs, and checksums + - name: Install build and documentation tooling + run: uv sync --locked --dev --extra docs + - name: Build distributions once env: - RELEASE_TAG: ${{ github.ref_name }} - run: >- - uv run python scripts/verify_distribution.py --dist-dir dist/packages - --expected-tag "$RELEASE_TAG" --write-checksums --checksum-file dist/SHA256SUMS - - - name: Upload validated distributions + SETUPTOOLS_SCM_PRETEND_VERSION: ${{ needs.preflight.outputs.version }} + run: uv build --out-dir candidate/dist + - name: Validate metadata and clean installations + run: | + uv run twine check candidate/dist/* + uv run python scripts/verify_distribution.py \ + --dist-dir candidate/dist \ + --expected-tag "v${{ needs.preflight.outputs.version }}" + - name: Write and verify the commit-bound candidate manifest + run: | + uv run python scripts/release_candidate.py create candidate \ + --commit-sha "${{ needs.preflight.outputs.commit }}" \ + --git-tree "${{ needs.preflight.outputs.tree }}" + uv run python scripts/release_candidate.py verify candidate \ + --expected-commit "${{ needs.preflight.outputs.commit }}" \ + --expected-tree "${{ needs.preflight.outputs.tree }}" \ + --expected-tag "v${{ needs.preflight.outputs.version }}" + - name: Build and verify revision-identified documentation + env: + ML4T_DOCS_COMMIT: ${{ needs.preflight.outputs.commit }} + ML4T_DOCS_VERSION: ${{ needs.preflight.outputs.version }} + run: | + uv run mkdocs build --strict --site-dir candidate/site + uv run python scripts/verify_documentation_identity.py \ + --site candidate/site \ + --expected-library data \ + --expected-version "$ML4T_DOCS_VERSION" \ + --expected-commit "$ML4T_DOCS_COMMIT" + - name: Upload immutable candidate + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: release-candidate-${{ needs.preflight.outputs.commit }} + path: | + candidate/dist/ + candidate/candidate.json + if-no-files-found: error + retention-days: 14 + - name: Upload immutable documentation uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: validated-dist - path: dist/ + name: docs-${{ needs.preflight.outputs.commit }} + path: candidate/site/ if-no-files-found: error retention-days: 14 + documentation: + name: Deploy and Verify Qualified Documentation + needs: [preflight, build] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.preflight.outputs.commit }} + persist-credentials: false + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: docs-${{ needs.preflight.outputs.commit }} + path: candidate-site/ + - name: Verify documentation before deployment + run: >- + python scripts/verify_documentation_identity.py + --site candidate-site + --expected-library data + --expected-version "${{ needs.preflight.outputs.version }}" + --expected-commit "${{ needs.preflight.outputs.commit }}" + - name: Require documentation deploy credentials + env: + DOCS_DEPLOY_KEY: ${{ secrets.DOCS_DEPLOY_KEY }} + run: test -n "$DOCS_DEPLOY_KEY" + - name: Deploy the qualified documentation artifact + uses: cpina/github-action-push-to-another-repository@55306faa4ed53b815ae49e564af8cfb359d32ae2 # v1.7.3 + env: + SSH_DEPLOY_KEY: ${{ secrets.DOCS_DEPLOY_KEY }} + with: + source-directory: candidate-site/ + destination-github-username: ml4t + destination-repository-name: website + target-directory: static/docs/data/ + target-branch: main + commit-message: "docs(data): release ${{ needs.preflight.outputs.version }} from ${{ needs.preflight.outputs.commit }}" + user-name: ml4t-bot + user-email: bot@ml4trading.io + - name: Verify deployed documentation identity + run: >- + python scripts/verify_documentation_identity.py + --url https://www.ml4trading.io/docs/data/ + --url https://www.ml4trading.io/docs/data/api/ + --url https://www.ml4trading.io/docs/data/providers/ + --expected-library data + --expected-version "${{ needs.preflight.outputs.version }}" + --expected-commit "${{ needs.preflight.outputs.commit }}" + --attempts 24 + --delay 10 + publish: - name: Publish to PyPI + name: Publish Exact Candidate to PyPI + needs: [preflight, documentation] runs-on: ubuntu-latest - needs: build environment: pypi permissions: contents: read - id-token: write # Required for PyPI trusted publishing. + id-token: write steps: - - name: Download validated distributions - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - name: validated-dist - path: dist/ - - - name: Verify artifact checksums - working-directory: dist/packages - run: sha256sum --check ../SHA256SUMS - - - name: Publish to PyPI + ref: ${{ needs.preflight.outputs.commit }} + persist-credentials: false + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: release-candidate-${{ needs.preflight.outputs.commit }} + path: candidate/ + - name: Verify retained candidate + run: >- + python scripts/release_candidate.py verify candidate + --expected-commit "${{ needs.preflight.outputs.commit }}" + --expected-tree "${{ needs.preflight.outputs.tree }}" + --expected-tag "v${{ needs.preflight.outputs.version }}" + - name: Publish through trusted publishing uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 with: - packages-dir: dist/packages/ + packages-dir: candidate/dist/ + verify-metadata: true github-release: - name: Create GitHub Release + name: Create Tag and GitHub Release + needs: [preflight, publish] runs-on: ubuntu-latest - needs: publish permissions: - contents: write # Required to create the GitHub release. + contents: write steps: - - name: Download validated distributions - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: validated-dist - path: dist/ - - - name: Create GitHub release with validated distributions + name: release-candidate-${{ needs.preflight.outputs.commit }} + path: candidate/ + - name: Create release from the published candidate env: GH_TOKEN: ${{ github.token }} - GH_REPO: ${{ github.repository }} - RELEASE_TAG: ${{ github.ref_name }} + run: >- + gh release create "v${{ needs.preflight.outputs.version }}" + candidate/dist/* candidate/candidate.json + --repo "${{ github.repository }}" + --target "${{ needs.preflight.outputs.commit }}" + --generate-notes + + post-publication: + name: Verify Package, Release, and Documentation + needs: [preflight, github-release] + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.preflight.outputs.commit }} + persist-credentials: false + - uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0 + with: + version: ${{ env.UV_VERSION }} + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: release-candidate-${{ needs.preflight.outputs.commit }} + path: candidate/ + - name: Verify PyPI metadata and GitHub release digests + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + python scripts/verify_published_release.py + --manifest candidate/candidate.json + --repository "${{ github.repository }}" + - name: Install the published wheel and execute the README quick start run: | - tag="$RELEASE_TAG" - args=("$tag" dist/packages/* dist/SHA256SUMS --generate-notes) - if [[ "$tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(a|b|rc)[0-9]+$ ]]; then - args+=(--prerelease) - fi - gh release create "${args[@]}" + uv venv --python 3.12 .release-venv + uv pip install --python .release-venv/bin/python "ml4t-data==${{ needs.preflight.outputs.version }}" + .release-venv/bin/python scripts/run_readme_quickstart.py + - name: Recheck deployed documentation identity + run: >- + python scripts/verify_documentation_identity.py + --url https://www.ml4trading.io/docs/data/ + --url https://www.ml4trading.io/docs/data/api/ + --expected-library data + --expected-version "${{ needs.preflight.outputs.version }}" + --expected-commit "${{ needs.preflight.outputs.commit }}" + --attempts 6 + --delay 10 diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 00000000..314502ff --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,45 @@ +name: Security + +on: + push: + branches: [main] + pull_request: + branches: [main] + schedule: + - cron: "23 8 * * 1" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: security-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + dependency-review: + name: Dependency Review + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0 + with: + fail-on-severity: high + + codeql: + name: CodeQL + runs-on: ubuntu-latest + permissions: + contents: read + security-events: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: github/codeql-action/init@fddeee1a7ece751b577e409a89057319e3172939 # v4 + with: + languages: python + - uses: github/codeql-action/analyze@fddeee1a7ece751b577e409a89057319e3172939 # v4 diff --git a/AGENTS.md b/AGENTS.md index fba324e1..6bdd70c0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,79 +1,73 @@ -# ml4t-data Library +# ml4t-data contributor guide -Market data acquisition and storage library for ML4T 3rd Edition. +`ml4t-data` provides market-data acquisition, validation, storage, and update workflows for +machine-learning-for-trading applications. The public Python package is `ml4t.data`; package +metadata and supported Python versions are authoritative in `pyproject.toml`. -## Structure +## Repository map -| Directory | Purpose | -| ------------------ | -------------------------------- | -| `src/ml4t/data/` | Package root | -| `tests/` | Test suite | -| `examples/` | Usage examples | -| `docs/` | MkDocs documentation | +- `src/ml4t/data/` contains the package. Its `AGENTS.md` routes work within the source tree. +- `tests/` contains deterministic unit and contract tests. Tests needing credentials or live + services belong in an explicitly marked integration lane. +- `docs/` contains the MkDocs site. Start with `docs/index.md`; use + `docs/book-guide/index.md` for book chapter-to-API mappings. +- `examples/` contains maintained runnable examples and configuration samples. +- `scripts/` contains release and verification programs used by GitHub Actions. +- `.github/workflows/` defines pull-request, compatibility, documentation, security, and release + automation. -## Key Modules +The principal source areas are provider adapters and routing in `providers/`, persistence in +`storage/`, futures acquisition and contract construction in `futures/`, orchestration in +`data_manager.py`, `update_manager.py`, and `managers/`, and shared contracts in `core/`, `assets/`, +`config/`, and `validation/`. Providers, storage, and futures have more specific guides in their +directories. -| Module | Purpose | -| ------------- | ---------------------------------------------- | -| `futures/` | Databento futures downloaders and roll logic | -| `etfs/` | ETFDataManager for Yahoo Finance ETF data | -| `crypto/` | CryptoDataManager for Binance premium index | -| `providers/` | 20 live provider adapters + synthetic/testing providers | -| `storage/` | Hive-partitioned Parquet + ProfileMixin | -| `managers/` | Data orchestration and updates | -| `assets/` | Asset universe definitions | -| `cot/` | Commitment of Traders data | +## Public surface -## Futures Module Classes +Prefer supported imports from `ml4t.data` or documented subpackages. The synthetic provider is the +offline reference workflow: -| Class | Purpose | -| ---------------------- | -------------------------------------------------- | -| `ContinuousDownloader` | Pre-rolled continuous contracts (.v.0, .v.1, .v.2) | -| `IndividualDownloader` | Specific contract symbols (ESH24, CLF24, etc.) | -| `FuturesDownloader` | All contracts via parent symbology ({PRODUCT}.FUT) | -| `FuturesDataManager` | High-level interface for book readers | - -## Book Data Managers (with Profiling) - -| Manager | Asset Class | Source | Profiling | -|---------|-------------|--------|-----------| -| `ETFDataManager` | 50 ETFs | Yahoo Finance | `generate_profile()` | -| `CryptoDataManager` | Crypto | Binance Public | `generate_profile()` | -| `FuturesDataManager` | CME Futures | Databento | `generate_profile(product)` | +```python +from ml4t.data.providers import SyntheticProvider -All managers support on-demand data profiling via `ProfileMixin`. +provider = SyntheticProvider(seed=42) +data = provider.fetch_ohlcv("SYNTH", "2024-01-01", "2024-01-10", "daily") +``` -## Entry Points +Provider-specific dependencies are optional. Keep imports usable without unrelated extras, and do +not require credentials or network access at import time. User-facing behavior belongs in the +documentation, not in agent guides. -```python -# ETF data (Yahoo Finance) -from ml4t.data.etfs import ETFDataManager +## Change rules -# Crypto premium index (Binance Public - no API key needed) -from ml4t.data.crypto import CryptoDataManager +- Preserve the PEP 420 namespace layout: do not add `src/ml4t/__init__.py`. +- Keep default tests deterministic and offline. Mark live, paid-tier, credentialed, and slow tests + with the existing pytest markers. +- Resolve storage roots through the shared configuration helpers. Do not introduce hard-coded home + directories. +- Use shared exceptions, retry, rate-limit, and provider contracts rather than adapter-specific + variants of the same behavior. +- Treat `CHANGELOG.md` and generated version files as generated artifacts; do not edit them by hand. +- Keep release artifacts bound to one commit. Publishing is performed by the release workflow after + qualification and documentation deployment, never by a local publish command. -# Futures data (Databento) -from ml4t.data.futures import FuturesDataManager +## Verification -# Data profiling -from ml4t.data.storage import ProfileMixin, generate_profile -``` - -## Commands +Install the complete development environment with `uv sync --locked --all-extras --all-groups`. +Run focused tests while editing, then run the repository gates: ```bash -# Development -uv sync -uv run pytest tests/ -q +uv run ruff check src tests scripts +uv run ruff format --check src tests scripts uv run ty check -pre-commit run --all-files - -# Build & publish +uv run pytest tests -q -ra +uv run pytest tests -q -ra -W error::ResourceWarning +uv run mkdocs build --strict uv build -uv publish +actionlint .github/workflows/*.yml +pre-commit run --all-files ``` -## Version - -Check `pyproject.toml` for current version. -PyPI: https://pypi.org/project/ml4t-data/ +For packaging or release changes, also run the verification scripts used by +`.github/workflows/release.yml`. Never bypass hooks or weaken provider-specific checks to make a +shared gate pass. diff --git a/README.md b/README.md index dc035b7a..525c8ba8 100644 --- a/README.md +++ b/README.md @@ -1,347 +1,79 @@ # ml4t-data -[![Python 3.12-3.14](https://img.shields.io/badge/python-3.12--3.14-blue.svg)](https://www.python.org/downloads/) [![PyPI](https://img.shields.io/pypi/v/ml4t-data)](https://pypi.org/project/ml4t-data/) -[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) - -Unified market data acquisition and storage for quantitative research workflows. - -## Part of the ML4T Library Ecosystem - -This library is one of six interconnected libraries supporting the machine learning for trading workflow described in [Machine Learning for Trading](https://www.ml4trading.io): - -![ML4T Library Ecosystem](docs/images/ml4t_ecosystem_workflow_color.png) - -Together they cover data infrastructure, feature engineering, modeling, signal evaluation, strategy backtesting, and live deployment. - -## What This Library Does - -Quantitative research requires consistent, reproducible access to market data from multiple sources. ml4t-data provides: - -- `DataManager` as the unified OHLCV interface for registry providers with that capability -- 20+ provider adapters covering equities, crypto, futures, forex, macro, prediction markets, and factors -- Automated storage in Hive-partitioned Parquet format with metadata tracking -- Incremental updates, gap detection, and backfill via CLI -- Built-in data validation (OHLC invariants, deduplication, anomaly detection) -- Futures module for CME/ICE bulk downloads with continuous contract construction -- COT module for CFTC Commitment of Traders weekly reports -- Resilience: rate limiting, retry with exponential backoff, gap detection +[![Python 3.12-3.14](https://img.shields.io/badge/python-3.12--3.14-blue.svg)](https://www.python.org/downloads/) +[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) -The goal is to support an ongoing research workflow rather than one-off downloads. Data is stored locally, tracked for freshness, and queryable with tools like DuckDB or Polars. +Market data acquisition, storage, and update workflows for machine learning for trading. -![ml4t-data Architecture](docs/images/ml4t_data_architecture_print.jpeg) +Use `ml4t-data` to fetch and validate market data, keep local datasets current, and provide +repeatable inputs to research and trading workflows. It includes adapters for equities, futures, +foreign exchange, crypto assets, macroeconomic series, factors, and prediction markets. ## Installation -ML4T Data supports stable CPython 3.12 through 3.14 on Linux, macOS, and Windows. -Python 3.15 is not supported until the core dependency stack passes the complete compatibility -suite on all three operating systems. Releases 0.1.0 and 0.1.1 predate this upper bound, so an -unpinned installation on Python 3.15 may select one of those older releases. Use Python 3.12 -through 3.14 instead. +`ml4t-data` supports CPython 3.12, 3.13, and 3.14 on Linux, macOS, and Windows. Python 3.15 is +temporarily excluded while upstream dependencies complete their compatibility work. ```bash pip install ml4t-data ``` -## Quick Start - -### DataManager (Unified Interface) - -```python -from ml4t.data import DataManager - -dm = DataManager() - -# Fetch and store -dm.fetch("AAPL", "2020-01-01", "2024-12-31", provider="yahoo") - -# Load from local storage -data = dm.load("AAPL", "2020-01-01", "2024-12-31") - -# Batch load multiple symbols -prices = dm.batch_load(["AAPL", "MSFT", "GOOGL"], "2020-01-01", "2024-12-31") - -# Incremental update -dm.update("AAPL") - -# List what's stored -symbols = dm.list_symbols() -metadata = dm.get_metadata("AAPL") -``` - -### Direct Provider Access - -Providers expose capability-specific methods. OHLCV providers use `fetch_ohlcv`, economic-series -providers may also use `fetch_series`, and factor providers use `fetch`. - -```python -from datetime import UTC, datetime, timedelta - -from ml4t.data.providers import YahooFinanceProvider, CoinGeckoProvider, FREDProvider - -# Equities -provider = YahooFinanceProvider() -data = provider.fetch_ohlcv("AAPL", "2020-01-01", "2024-12-31") - -# Crypto -last_complete_day = datetime.now(UTC).date() - timedelta(days=1) -crypto = CoinGeckoProvider().fetch_ohlcv( - "bitcoin", - str(last_complete_day - timedelta(days=6)), - str(last_complete_day), -) - -# Economic data -# Reads FRED_API_KEY from the environment -fred = FREDProvider().fetch_series("GDP", "2020-01-01", "2024-12-31") -``` - -## Data Providers - -### No API Key Required - -| Provider | Coverage | -|----------|----------| -| Yahoo Finance | US/global equities, ETFs, crypto, forex | -| CoinGecko | 10,000+ cryptocurrencies; daily OHLCV for 29 completed UTC days | -| FXMacroData | FX macro releases, calendars, COT, commodities, sentiment | -| Fama-French | Academic factor data | -| AQR | Research factors (QMJ, BAB, HML Devil, VME, more) | -| Wiki Prices | Frozen US equities history (1962-2018) | -| Kalshi | Prediction market contracts | -| Polymarket | Prediction market history/order book snapshots | -| Binance Public | Bulk crypto data downloads | -| Binance | Crypto exchange data | -| OKX | Crypto perpetuals and funding rates | -| NASDAQ ITCH Sample | Tick-level sample data | - -### Authenticated or Metered APIs - -| Provider | Coverage | -|----------|----------| -| FRED | 850,000 economic series | -| Alpaca | US equities + crypto (free IEX feed) | -| EODHD | 60+ global exchanges | -| Tiingo | US equities with quality focus | -| Twelve Data | Multi-asset coverage | -| Databento | CME/ICE futures; OPRA options | -| Massive | US equities, options, futures, forex, crypto | -| Finnhub | 70+ global exchanges | -| OANDA | Forex broker data | -| CryptoCompare | Included adapter; not release-qualified for 0.1.0 | - -CryptoCompare account registration was unavailable during the 0.1.0 release review, so no live -contract evidence was obtained. The adapter remains available for evaluation, but CryptoCompare is -not part of the release-qualified provider set until its contract passes on a release commit. - -## Specialized Modules +Provider-specific integrations are optional. Install `ml4t-data[yahoo]`, +`ml4t-data[databento]`, `ml4t-data[oanda]`, or `ml4t-data[cot]` when those adapters are +needed. Some providers require network access, an account, credentials, or metered data. The +[provider guide](https://www.ml4trading.io/docs/data/providers/) records each boundary. -### Futures +## Quick start -Bulk download and continuous contract construction for CME/ICE products: +The synthetic provider exercises the public OHLCV interface without network access or credentials. ```python -from ml4t.data.futures import FuturesDownloader, ContinuousContractBuilder +from ml4t.data.providers import SyntheticProvider -# Bulk download via Databento (parent symbology) -downloader = FuturesDownloader(config) -downloader.download() # Downloads ES, NQ, CL, GC, etc. +provider = SyntheticProvider(seed=42) +data = provider.fetch_ohlcv("SYNTH", "2024-01-01", "2024-01-10", "daily") -# Build continuous contracts with configurable roll logic -builder = ContinuousContractBuilder() -continuous = builder.build(contracts_df, roll_method="volume") +assert not data.is_empty() +assert {"timestamp", "symbol", "open", "high", "low", "close", "volume"} <= set(data.columns) ``` -Book-focused interface with profiling: +Set `ML4T_DATA_PATH` to choose the root directory for local datasets. Without an explicit path, +the library uses `./data` in the current working directory. -```python -from ml4t.data.futures import FuturesDataManager - -fm = FuturesDataManager.from_config("config.yaml") -fm.download_all() -data = fm.load_ohlcv("ES") -profile = fm.generate_profile("ES") -``` - -### COT (Commitment of Traders) - -CFTC weekly positioning data for futures markets: - -```python -import polars as pl - -from ml4t.data.cot import COTFetcher, combine_cot_ohlcv, create_cot_features - -fetcher = COTFetcher() -official_schedule = pl.read_parquet("cot-release-schedule.parquet") -cot_data = fetcher.fetch_product( - "ES", - start_year=2015, - end_year=2024, - release_schedule=official_schedule, -) +## Migration from QLDM names -# Point-in-time combination with OHLCV -combined = combine_cot_ohlcv(ohlcv_data, cot_data) +Existing deployments may continue to use `QLDM_DATA_ROOT` and import `QldmError`. Both names +emit `DeprecationWarning` and will not be removed before version 1.0. -# Generate weekly features without counting forward-filled daily rows as new reports -features = create_cot_features(combined) -``` - -The schedule must contain one `report_date` and timezone-aware `available_at` timestamp per -report. CFTC publishes a tentative schedule for only the latest 13 months, so retain the exact -release timestamps you use for historical research. Shutdown catch-up releases and other -exceptions must use their actual publication timestamps. See the -[official CFTC release schedule](https://www.cftc.gov/MarketReports/CommitmentsofTraders/ReleaseSchedule/index.htm). +- Replace `QLDM_DATA_ROOT` with `ML4T_DATA_PATH`. +- Replace `QldmError` with `ML4TDataError` from `ml4t.data.core.exceptions`. -### Book Data Managers +## Documentation and support -Simplified interfaces for the ML4T book workflow: +- [Documentation](https://www.ml4trading.io/docs/data/) +- [Issue tracker](https://github.com/ml4t/data/issues) +- [Release notes](CHANGELOG.md) +- [License](LICENSE) -```python -from ml4t.data.etfs import ETFDataManager -from ml4t.data.crypto import CryptoDataManager - -# 50 diversified ETFs via Yahoo Finance -etf_dm = ETFDataManager.from_config("config.yaml") -etf_dm.download_all() -aapl = etf_dm.load_ohlcv("AAPL") - -# Crypto premium index via Binance Public -crypto_dm = CryptoDataManager.from_config("config.yaml") -crypto_dm.download_premium_index() -``` - -## CLI for Automated Updates - -```bash -# Fetch specific symbols -ml4t-data fetch -s AAPL -s MSFT -s GOOGL --provider yahoo --start 2020-01-01 - -# Incremental update -ml4t-data update --symbol AAPL - -# Validate data quality -ml4t-data validate --symbol AAPL --anomalies - -# Check storage status -ml4t-data status --detailed - -# List available data -ml4t-data list-data - -# Export to CSV/JSON/Excel -ml4t-data export --symbol AAPL --format-type csv --output aapl.csv - -# Get symbol info -ml4t-data info --symbol AAPL -``` - -Configuration-driven batch updates: - -```yaml -storage: - base_path: ~/data/market - -datasets: - - name: sp500_daily - provider: yahoo - symbols_file: symbols/sp500.txt - frequency: daily - start_date: 2015-01-01 - - - name: crypto - provider: coingecko - symbols: [bitcoin, ethereum, solana] - frequency: daily - initial_load_days: 29 -``` - -## Storage Format - -Data is stored in Hive-partitioned Parquet: - -``` -~/data/market/ -├── yahoo/daily/symbol=AAPL/data.parquet -├── yahoo/daily/symbol=MSFT/data.parquet -└── coingecko/daily/symbol=bitcoin/data.parquet -``` - -Query with DuckDB or Polars: - -```python -import duckdb - -result = duckdb.execute(""" - SELECT * FROM read_parquet('~/data/market/yahoo/daily/**/*.parquet') - WHERE symbol IN ('AAPL', 'MSFT') - AND date >= '2024-01-01' -""").pl() -``` - -## Data Validation - -```python -from ml4t.data.validation import OHLCVValidator, ValidationReport - -validator = OHLCVValidator() -report = validator.validate(data) -# Checks: high >= low, high >= open/close, low <= open/close -# Detects: duplicates, gaps, anomalies -``` - -Anomaly detection: - -```python -from ml4t.data.anomaly import AnomalyManager, ReturnOutlierDetector, VolumeSpikeDetector - -manager = AnomalyManager([ - ReturnOutlierDetector(), - VolumeSpikeDetector(), -]) -report = manager.detect(data) -``` - -## Documentation - -- [Getting Started](docs/user-guide/getting-started.md) - quick start guide -- [Configuration](docs/user-guide/configuration.md) - YAML config reference -- [Storage](docs/user-guide/storage.md) - Hive partitioning and backends -- [Incremental Updates](docs/user-guide/incremental-updates.md) - update strategies and gap detection -- [Data Quality](docs/user-guide/data-quality.md) - validation and anomaly detection -- [CLI Reference](docs/user-guide/cli-reference.md) - command-line interface -- [Provider Selection Guide](docs/getting-started/provider-selection.md) - choosing providers -- [Creating a Provider](docs/contributing/creating-a-provider.md) - extending with new sources - -## Technical Characteristics - -- **Polars-based**: Native Polars DataFrames throughout -- **Capability-specific schemas**: OHLCV, economic-series, factor, and specialized providers - expose contracts suited to their data -- **Async support**: OHLCV providers implementing the async protocol can use parallel batch operations -- **Metadata tracking**: Last update timestamps, row counts, date ranges -- **Resilience**: Rate limiting, retry with exponential backoff, gap detection -- **Storage layouts**: Partitioned Hive and single-file Parquet storage -- **Type-safe**: Full type annotations throughout - -## Related Libraries - -- **ml4t-engineer**: Feature engineering and technical indicators -- **ml4t-diagnostic**: Signal evaluation and statistical validation -- **ml4t-backtest**: Event-driven backtesting -- **ml4t-live**: Live trading with broker integration +The library supplies data to [ml4t-engineer](https://github.com/ml4t/engineer) feature workflows and +[ml4t-backtest](https://github.com/ml4t/backtest) simulations. Neither package is required to use +`ml4t-data`. ## Development ```bash -git clone https://github.com/ml4t/data.git ml4t-data -cd ml4t-data -uv sync -uv run pytest tests/ -q +git clone https://github.com/ml4t/data.git +cd data +uv sync --all-extras --all-groups +uv run pre-commit install +uv run ruff check src/ tests/ +uv run ruff format --check src/ tests/ uv run ty check +uv run pytest tests/ -q +uv run mkdocs build --strict +uv build ``` -## License - -MIT License - see [LICENSE](LICENSE) for details. +Pull requests must preserve offline deterministic tests. Provider contract tests that require +credentials run in explicit integration lanes. diff --git a/docs/ASYNC_STORAGE.md b/docs/ASYNC_STORAGE.md index f727b8e3..1ced3efd 100644 --- a/docs/ASYNC_STORAGE.md +++ b/docs/ASYNC_STORAGE.md @@ -1,6 +1,7 @@ # Async Storage Backend Guide -ML4T Data now includes a high-performance async storage backend that provides non-blocking I/O operations for improved concurrency and scalability. +ML4T Data includes an async storage backend that provides non-blocking I/O for workloads that +coordinate storage with other asynchronous operations. ## Overview diff --git a/docs/FEATURE_INVENTORY.md b/docs/FEATURE_INVENTORY.md deleted file mode 100644 index 891e7a46..00000000 --- a/docs/FEATURE_INVENTORY.md +++ /dev/null @@ -1,1179 +0,0 @@ -# ML4T-Data: Comprehensive Feature Inventory - -**Library**: ml4t-data -**Location**: `src/ml4t/data/` -**Version**: 0.1.0 (pre-release) -**Status**: 🚧 Active Development (breaking changes acceptable) -**Python**: 3.9+ -**Generated**: 2025-11-27 - ---- - -## Executive Summary - -ML4T-Data is a production-ready market data acquisition library providing: - -- **20 Live Data Providers** (plus 3 synthetic/testing providers) -- **Unified API** across crypto, equities, forex, and futures -- **Incremental Update Infrastructure** with gap detection and resume capability -- **7x Performance** improvement with Hive-partitioned Parquet storage -- **Anomaly Detection** with 3 built-in detectors -- **CLI Automation** for cron-friendly scheduled updates -- **OHLC Validation** and data quality guarantees -- **73 Test Files** covering unit, integration, and acceptance tests - -**Key Achievement**: 30ms initialization overhead, 5ms per call with provider reuse (negligible with proper usage patterns). - ---- - -## Feature Category Inventory - -### 1. DATA PROVIDERS (23 Total: 20 Live + 3 Synthetic/Testing) - -#### Live Providers (No API Key Required) - -| # | Provider | File | Asset Classes | Free Tier | Status | Key Features | -|---|----------|------|----------------|-----------|--------|--------------| -| 1 | **Yahoo Finance** | `yahoo.py` | Equities, Crypto | Unlimited | Complete | US stocks, ETFs, indices | -| 2 | **CoinGecko** | `coingecko.py` | Crypto | Free (50 req/min) | Complete | 10,000+ cryptocurrencies | -| 3 | **FRED** | `fred.py` | Economic Data | 120 req/min | Complete | 850,000 economic series | -| 4 | **Fama-French** | `fama_french.py` | Factors | Unlimited | Complete | Academic factor data | -| 5 | **AQR** | `aqr.py` | Factors | Unlimited | Complete | Research factors (QMJ, BAB, VME) | -| 6 | **Wiki Prices** | `wiki_prices.py` | Equities | Local file only | Complete | Historical US equities (1962-2018) | -| 7 | **Kalshi** | `kalshi.py` | Prediction Markets | Public data | Complete | Prediction market contracts | -| 8 | **Polymarket** | `polymarket.py` | Prediction Markets | Public data | Complete | Prediction market history | -| 9 | **Binance Public** | `binance_public.py` | Crypto | Bulk downloads | Complete | No geo-restrictions | -| 10 | **NASDAQ ITCH** | `nasdaq_itch.py` | Tick Data | Sample data | Complete | Tick-level market data | - -#### Live Providers (Authenticated or Metered) - -| # | Provider | File | Asset Classes | Free Tier | Status | Key Features | -|---|----------|------|----------------|-----------|--------|--------------| -| 11 | **EODHD** | `eodhd.py` | Equities, Forex, Crypto | 500 calls/day | Complete | 60+ exchanges, global coverage | -| 12 | **Tiingo** | `tiingo.py` | Equities | 1000 req/day | Complete | US stocks alternative | -| 13 | **Twelve Data** | `twelve_data.py` | Equities, Forex, Crypto | 800 calls/day | Complete | Multi-asset coverage | -| 14 | **DataBento** | `databento.py` | Futures, Equities | Trial ($10 credits) | Complete | Institutional market data | -| 15 | **Polygon.io** | `polygon.py` | Equities, Options, Crypto, Forex | Limited | Complete | US market data (paid tier) | -| 16 | **Finnhub** | `finnhub.py` | Equities, Forex, Crypto | 60 req/min | Complete | Global market data | -| 17 | **Binance** | `binance.py` | Crypto | Generous | Complete | Spot + futures, high-frequency | -| 18 | **OKX** | `okx.py` | Crypto | No geo-limits | Complete | Perpetuals, funding rates | -| 19 | **CryptoCompare** | `cryptocompare.py` | Crypto | 250k calls/mo | Complete | Historical crypto data | -| 20 | **Oanda** | `oanda.py` | Forex | Trial | Complete | Institutional forex data | - -**Provider Summary**: -- **20 Live Providers**: 10 free + 10 authenticated/metered -- **3 Synthetic/Testing**: SyntheticProvider, LearnedSyntheticProvider, MockProvider -- **Coverage**: Crypto (7), Equities (7), Forex (3), Futures (1), Economic (1), Factors (2), Prediction Markets (2), Tick Data (1) - -#### Base Provider Infrastructure - -**File**: `base.py` (203 lines) - -**Pattern**: Template Method - unified interface across all providers - -**Features**: -- `fetch_ohlcv(symbol, start, end, frequency)` - Standard interface -- Rate limiting (configurable per provider) -- Circuit breaker (5 failures → 5-minute timeout) -- Retry logic (3 attempts with exponential backoff) -- HTTP session reuse (connection pooling) -- Error handling with custom exceptions - -**Key Methods**: -- `_fetch_and_transform_data()` - Provider-specific implementation -- `_validate_data()` - OHLC invariant checks -- `close()` - Resource cleanup -- Context manager support (`with provider: ...`) - -**Performance**: -- Initialization: 30-35ms (httpx.Client setup) -- Per-call overhead: 5ms (~2.3% of typical download time) -- Best practice: Reuse provider instances for all symbols - -**Circuit Breaker Configuration**: -```python -failure_threshold: int = 5 # Failures before OPEN -reset_timeout: float = 300.0 # Seconds to attempt reset -state: str in ["CLOSED", "OPEN", "HALF_OPEN"] -``` - ---- - -### 2. STORAGE SYSTEM - -#### Hive-Partitioned Parquet Storage - -**File**: `storage/hive.py` (261 lines) - -**Features**: -- **7x Query Performance** improvement for time-based queries -- **Partitioning**: `year=YYYY/month=MM/symbol/data.parquet` -- **Atomic Writes**: Temp file pattern with atomic rename -- **File Locking**: Concurrent access safety with FileLock -- **Metadata Tracking**: JSON manifests with last update timestamps -- **Lazy Evaluation**: Polars LazyFrame throughout - -**Backend Implementations**: -1. **HiveStorage** - Hive-partitioned (primary, optimized) -2. **FilesystemStorage** - Flat directory structure (legacy) -3. **AsyncFilesystemStorage** - Async I/O operations -4. **ChunkedStorage** - Fixed-size chunks with metadata - -**Metadata Tracker** - -**File**: `storage/metadata_tracker.py` (399 lines) - -**Features**: -- Track last update timestamp per symbol -- Row count statistics -- Update history with duration tracking -- Provider metadata (name, frequency, date ranges) -- Atomic metadata updates - -**Data Structure**: -```python -class UpdateRecord: - symbol: str - last_update: datetime - row_count: int - date_range: tuple[date, date] - provider: str - frequency: str - update_duration_seconds: float -``` - -**Storage Protocols** - -**File**: `storage/protocols.py` (126 lines) - -**Defines**: Abstract interfaces for storage backends -- Read/write operations -- Metadata tracking -- Transaction support - -**Transaction Support** - -**File**: `storage/backend.py` - -**Features**: -- ACID-like guarantees -- Rollback on failure -- Multi-step update workflows -- Error recovery - -**Migration System** - -**File**: `storage/migration.py` (595 lines) - -**Features**: -- Schema version tracking -- Data format conversion -- Progressive migration support -- Rollback capability - ---- - -### 3. DATA VALIDATION & QUALITY - -#### OHLCV Validator - -**File**: `validation/ohlcv.py` (323 lines) - -**Rules Enforced**: -- Open <= High -- Low <= Close -- High >= Low -- Volume >= 0 -- No NaN/Inf values -- Timestamp ordering -- Deduplication -- Frequency consistency - -**Methods**: -- `validate()` - Full validation -- `validate_row()` - Single row check -- `fix_auto_fixable()` - Auto-repair reversals -- `check_frequency_consistency()` - Detect gaps -- `deduplicate()` - Remove duplicates - -**Validation Report** - -**File**: `validation/report.py` (223 lines) - -**Output**: -- Valid row count -- Issues found (by type) -- Rule violations with details -- JSON export support -- Severity classification - -**Validation Rules Engine** - -**File**: `validation/rules.py` (238 lines) - -**Rule Types**: -- OHLC invariants (open <= high, etc.) -- Range checks (prices > 0, volume >= 0) -- Consistency checks (no duplicates) -- Frequency checks (expected gap tolerance) - -**Cross-Validation** - -**File**: `validation/cross_validation.py` (280 lines) - -**Compares**: -- Same symbol from different providers -- Same period from different sources -- Data quality metrics across providers - -#### Anomaly Detection System - -**Location**: `anomaly/` (5 files) - -**Components**: - -1. **ReturnOutlierDetector** - Unusual price movements - - Methods: MAD (robust), Z-Score (standard), IQR (quartile-based) - - Severity: INFO → WARNING → ERROR → CRITICAL - - Detects: Abnormal returns, flash crashes, pump events - -2. **VolumeSpikeDetector** - Unusual trading volume - - Rolling window statistics (default: 20 days) - - Z-score threshold (default: 3.0) - - Detects: Volume anomalies, market events - -3. **PriceStalenessDetector** - Stale/unchanged prices - - Detects data gaps and missing updates - - Two modes: close-only or all OHLC - - Threshold: consecutive unchanged days - -**Usage**: -```python -from ml4t.data.anomaly import AnomalyManager - -manager = AnomalyManager() -report = manager.analyze(data, symbol="AAPL") -critical = report.get_critical_anomalies() -``` - -**Configuration**: -- `AnomalyConfig` - Global settings -- `DetectorConfig` - Per-detector configuration -- Customizable thresholds and methods - ---- - -### 4. INCREMENTAL UPDATE INFRASTRUCTURE - -#### Update Manager - -**File**: `update_manager.py` (335 lines) - -**Update Strategies**: -- `INCREMENTAL` - Only fetch new data -- `APPEND_ONLY` - Never update existing data -- `FULL_REFRESH` - Replace all data -- `BACKFILL` - Fill gaps in historical data - -**Features**: -- Gap detection with frequency tolerance -- Resume capability on failure -- Batch update orchestration -- Comprehensive result tracking - -**Result Tracking**: -```python -class UpdateResult: - success: bool - update_type: str - rows_added: int - rows_updated: int - rows_before: int - rows_after: int - gaps_filled: int - duration_seconds: float - errors: list[str] -``` - -#### Gap Detection - -**File**: `update_manager.py` (GapDetector class) - -**Features**: -- Detect missing dates in time-series -- Configurable frequency (daily, hourly, etc.) -- Tolerance for expected gaps (weekends, holidays) -- List format: `[{start, end, size}, ...]` - -**Gap Optimizer** - -**File**: `utils/gap_optimizer.py` (148 lines) - -**Features**: -- Merge adjacent gaps -- Optimize fetch requests -- Reduce API calls for gap filling - -#### Provider Updater - -**File**: `provider_updater.py` (262 lines) - -**Features**: -- Per-provider update orchestration -- Symbol batching -- Rate limit coordination -- Error handling and retry - ---- - -### 5. UTILITIES - -#### Rate Limiting - -**Global Rate Limiting** - -**File**: `utils/global_rate_limit.py` (118 lines) - -**Features**: -- Cross-provider rate limit coordination -- Per-provider limits (configurable) -- Global queue management -- Semaphore-based enforcement - -**Provider Rate Limiter** - -**File**: `utils/rate_limit.py` (87 lines) - -**Features**: -- Token bucket algorithm -- Configurable calls per period -- Request queuing - -**Async Rate Limiter** - -**File**: `utils/async_rate_limit.py` (302 lines) - -**Features**: -- Async/await support -- Non-blocking rate limiting -- Per-provider concurrency control - -#### Concurrency & Locking - -**File**: `utils/locking.py` (193 lines) - -**Features**: -- File-based locks (FileLock wrapper) -- Timeout support -- Process-safe synchronization -- Context manager support - -#### Retry Logic - -**File**: `utils/retry.py` (54 lines) - -**Features**: -- Exponential backoff -- Configurable attempts -- Exception type filtering - -#### Data Formatting - -**File**: `utils/format.py` (342 lines) - -**Features**: -- Symbol validation and normalization -- Frequency parsing and validation -- Asset class mapping -- Custom format rules per provider - -#### Gap Utilities - -**File**: `utils/gaps.py` (323 lines) - -**Features**: -- Gap detection on date ranges -- Gap merging (adjacent gaps) -- Efficient gap scheduling - ---- - -### 6. FUTURES-SPECIFIC FUNCTIONALITY - -**Location**: `futures/` (5 files) - -#### Futures Parser - -**File**: `futures/parser.py` - -**Features**: -- Quandl CHRIS format parsing -- Contract specification extraction -- Metadata preservation - -#### Roll Strategy - -**File**: `futures/roll.py` (244 lines) - -**Strategies**: -1. `VolumeBasedRoll` - Roll on volume transitions -2. `OpenInterestBasedRoll` - Roll on OI decrease -3. `TimeBasedRoll` - Roll on expiration dates - -**Note**: Time-based rolling with expiration calendar (TODO: not fully implemented) - -#### Continuous Contract Builder - -**File**: `futures/continuous.py` - -**Features**: -- Splice individual contracts -- Apply adjustment methods -- Handle roll logic - -#### Adjustment Methods - -**File**: `futures/adjustment.py` - -**Methods**: -1. `BackAdjustment` - Adjust historical prices -2. `RatioAdjustment` - Multiplicative adjustment -3. `NoAdjustment` - Keep individual contracts - -#### Futures Schema - -**File**: `futures/schema.py` - -**Features**: -- `ContractSpec` - Contract metadata -- `ExchangeInfo` - Exchange specifications -- `MAJOR_CONTRACTS` - Registry of major contracts -- Settlement types (Cash, Physical, etc.) - ---- - -### 7. ASSET MANAGEMENT - -**Location**: `assets/` (4 files) - -#### Asset Class Registry - -**File**: `assets/asset_class.py` - -**Classes**: -- EQUITIES (canonical) -- CRYPTO -- FOREX -- COMMODITY -- FIXED_INCOME -- INDEX -- ETF -- FUTURES -- OPTIONS - -**Aliases**: EQUITY, OPTION, FUTURE (backward compatibility) - -#### Contract Specifications - -**File**: `assets/contracts.py` - -**Features**: -- `FUTURES_REGISTRY` - Registered contracts -- `ContractSpec` - Contract metadata -- `get_contract_spec()` - Lookup by symbol -- `register_contract_spec()` - Dynamic registration -- `load_contract_specs()` - Load from YAML/JSON - -#### Asset Schemas - -**File**: `assets/schemas.py` - -**Features**: -- Per-asset-class schema definitions -- Required columns per asset type -- Schema validation - -#### Asset Validation - -**File**: `assets/validation.py` - -**Features**: -- Asset class validation -- Symbol format checking -- Asset type constraints - ---- - -### 8. DATA MODELS & SCHEMAS - -#### Core Models - -**File**: `core/models.py` (150+ lines) - -**Key Classes**: -- `Frequency` - Data frequency enum (tick, minute, hourly, daily, weekly, monthly) -- `BarType` - Bar type enum (time, volume, trade, dollar, tick) -- `AssetClass` - Asset class enum (equities, crypto, forex, etc.) -- `Metadata` - Data metadata (asset class, frequency, symbol, provider, etc.) -- `DataObject` - Complete data wrapper (data + metadata) -- `SchemaVersion` - Schema versioning - -#### Core Schemas - -**File**: `core/schemas.py` - -**Features**: -- Polars schema definitions -- OHLCV columns (open, high, low, close, volume) -- Timestamp requirements -- Optional columns (e.g., adjusted close, dividends) - -#### Core Exceptions - -**File**: `core/exceptions.py` - -**Exception Hierarchy**: -- `DataValidationError` - OHLC invariant violations -- `RateLimitError` - API rate limit exceeded -- `NetworkError` - Network connectivity issues -- `SymbolNotFoundError` - Symbol not available -- `CircuitBreakerOpenError` - API failure threshold reached -- `StorageError` - Storage backend failures - -#### Core Configuration - -**File**: `core/config.py` - -**Features**: -- Pydantic settings model -- Environment variable support -- Logging configuration -- Storage path configuration - ---- - -### 9. CONFIGURATION SYSTEM - -**Location**: `config/` (4 files) - -#### Configuration Models - -**File**: `config/models.py` - -**Features**: -- Pydantic BaseModel -- YAML/JSON serialization -- Environment variable override -- Type validation - -**Key Configs**: -- `StorageConfig` - Storage backend settings -- `ProviderConfig` - Provider-specific settings -- `DatasetConfig` - Dataset definitions -- `UpdateConfig` - Update strategies - -#### Configuration Loader - -**File**: `config/loader.py` - -**Features**: -- YAML file loading -- JSON file loading -- Environment variable substitution -- Schema validation - -#### Configuration Validator - -**File**: `config/validator.py` - -**Features**: -- Validate provider availability -- Check API key presence -- Verify storage paths -- Symbol list validation - ---- - -### 10. EXPORT SYSTEM - -**Location**: `export/` (5 files) - -#### Export Manager - -**File**: `export/manager.py` - -**Features**: -- Format auto-detection (by file extension) -- Multi-format export -- Batch export support -- Metadata preservation - -#### Export Formats - -**Implementations**: - -1. **CSV Exporter** (`export/formats/csv.py`) - - Header preservation - - Index handling - - Custom separators - -2. **JSON Exporter** (`export/formats/json.py`) - - Nested JSON structure - - Timestamp serialization - - Pretty printing - -3. **Excel Exporter** (`export/formats/excel.py`) - - Sheet naming - - Formatting options - - Note: Requires openpyxl (optional) - -4. **Parquet Exporter** (implicit via Polars) - - Native format - - Compression options - -**Base Class** (`export/formats/base.py`) -- Abstract interface for all formats -- Common validation logic - ---- - -### 11. DATA MANAGER - -**File**: `data_manager.py` (294 lines) - -**Features**: -- Unified interface to providers and storage -- Automatic provider selection -- Data caching -- Universe management (symbol sets) - -**Key Methods**: -- `fetch()` - Get data from provider -- `store()` - Save to storage -- `update()` - Incremental updates -- `list_providers()` - Available providers -- `list_symbols()` - Known symbols - ---- - -### 12. CLI INTERFACE - -**File**: `cli_interface.py` (380+ lines) - -**Commands**: - -1. **fetch** - Get data from provider - ```bash - ml4t-data fetch -s AAPL -s MSFT --start 2024-01-01 --end 2024-12-31 - ml4t-data fetch -f symbols.txt --start 2024-01-01 --end 2024-12-31 - # CoinGecko daily OHLC covers the last 29 completed UTC days only - # Replace the dates with a 29-day window ending yesterday in UTC - ml4t-data fetch -s BTC --provider coingecko --start START_DATE --end END_DATE - ``` - -2. **update-all** - Incremental updates from YAML config - ```bash - ml4t-data update-all -c ml4t-data.yaml - ml4t-data update-all -c ml4t-data.yaml --dataset sp500_full - ml4t-data update-all -c ml4t-data.yaml --dry-run - ``` - -3. **list** - Show stored data - ```bash - ml4t-data list -c ml4t-data.yaml - ``` - -4. **validate** - Data quality checks - ```bash - ml4t-data validate --symbol AAPL - ``` - -5. **export** - Export to different formats - ```bash - ml4t-data export --symbol AAPL --output data.csv - ml4t-data export --symbol AAPL --output data.xlsx - ``` - -**Options**: -- `-v, --verbose` - Detailed output -- `-q, --quiet` - Minimal output -- `--version` - Show version - ---- - -### 13. SESSION & CALENDAR MANAGEMENT - -#### Session Assigner - -**File**: `sessions/assigner.py` - -**Features**: -- Exchange calendar integration -- Session date assignment -- Holiday awareness -- Multiple exchange support - -#### Session Completer - -**File**: `sessions/completer.py` - -**Features**: -- Fill in missing session dates -- Interpolation for business days -- Forward-fill options - -#### Trading Calendar - -**File**: `calendar/crypto.py` - -**Features**: -- 24/5 crypto market calendar -- No holiday closures -- Continuous trading hours - ---- - -### 14. SECURITY - -#### Path Validation - -**File**: `security/path_validator.py` - -**Features**: -- Safe path construction -- Directory traversal prevention -- Symlink handling -- Cross-platform compatibility - ---- - -## Test Coverage Summary - -**Total Test Files**: 73 - -### Test Categories - -**Unit Tests** (45+ files): -- Provider implementation tests -- Storage backend tests -- Validation rule tests -- Configuration tests -- Utility tests (rate limiting, formatting, gaps) -- Anomaly detector tests -- Data manager tests - -**Integration Tests** (10 files): -- Real API provider tests (Yahoo, EODHD, CoinGecko, Tiingo, Finnhub, Polygon, Twelve Data, OandA, Wiki Prices) -- Real API integration tests -- Cross-provider validation - -**Acceptance Tests** (5+ files): -- DataBento acceptance tests -- End-to-end workflows -- Performance benchmarks - -**Specialized Tests**: -- Async storage tests -- Concurrent storage tests -- Batch loading tests -- Configuration integration tests -- Export tests - -### Test Execution - -**Default Run** (excludes slow, paid tier): -```bash -pytest # Runs ~50+ unit tests -``` - -**Integration Tests** (requires API keys): -```bash -pytest tests/integration/ -v -s -``` - -**With Coverage**: -```bash -pytest --cov=src/ml4t/data --cov-report=html -``` - -**Configuration**: -- Timeout: 300 seconds per test -- Parallel execution disabled (can cause hangs) -- Markers: slow, paid_tier, integration, requires_api_key, expensive - ---- - -## Implementation Status by Feature - -### Fully Implemented (✅ Complete) - -| Feature | File | Status | Notes | -|---------|------|--------|-------| -| 13 Data Providers | `providers/*.py` | ✅ | 12 live + 1 historical | -| Hive Storage | `storage/hive.py` | ✅ | 7x performance improvement | -| OHLCV Validator | `validation/ohlcv.py` | ✅ | All invariant checks | -| Anomaly Detection | `anomaly/*.py` | ✅ | 3 detectors implemented | -| Incremental Updates | `update_manager.py` | ✅ | Gap detection, resume | -| CLI Interface | `cli_interface.py` | ✅ | fetch, update-all, list, validate, export | -| Export System | `export/` | ✅ | CSV, JSON, Excel, Parquet | -| Rate Limiting | `utils/rate_limit.py` | ✅ | Global + per-provider | -| Circuit Breaker | `providers/base.py` | ✅ | Failure detection & recovery | -| Configuration | `config/` | ✅ | YAML/JSON with validation | -| Asset Registry | `assets/` | ✅ | Classes, contracts, schemas | -| Metadata Tracking | `storage/metadata_tracker.py` | ✅ | Update history | -| Locking/Concurrency | `utils/locking.py` | ✅ | File-based locks | -| Data Models | `core/models.py` | ✅ | Complete Pydantic models | - -### Partially Implemented (⚠️ Incomplete) - -| Feature | File | Status | What's Missing | -|---------|------|--------|-----------------| -| Futures Continuous Contracts | `futures/continuous.py` | ⚠️ | Time-based rolling not fully implemented (TODO) | -| Binance Integration | `providers/binance.py` | ⚠️ | Provider works, integration tests missing | -| Async Storage | `storage/async_filesystem.py` | ⚠️ | Implemented but not widely tested | -| WebSocket Streaming | - | ❌ | Not implemented (Phase 2 feature) | - -### TODOs & Known Issues - -**Found TODOs** (3 total): -1. `/futures/roll.py:` Time-based rolling with expiration calendar -2. `/data_manager.py:` Parallelize symbol updates with max_workers -3. (No critical blocking issues) - -**Known Limitations**: -- Binance integration tests require VPN in US (skipped) -- Some providers have rate limits (built-in coordination) -- Async storage needs more testing - ---- - -## Public API Surface - -### Top-Level Imports - -```python -from ml4t.data import ( - # Contract management - ContractSpec, - FUTURES_REGISTRY, - get_contract_spec, - load_contract_specs, - register_contract_spec, - AssetClass, - - # Core classes - BaseProvider, - Config, - DataManager, -) -``` - -### Provider Access - -```python -from ml4t.data.providers import ( - # Base - BaseProvider, - Provider, - - # Equity - YahooFinanceProvider, - TiingoProvider, - FinnhubProvider, - EODHDProvider, - - # Crypto - CoinGeckoProvider, - BinanceProvider, - CryptoCompareProvider, - - # Forex - OandaProvider, - - # Multi-asset - PolygonProvider, - TwelveDataProvider, - - # Institutional - DataBentoProvider, - - # Historical - WikiPricesProvider, - - # Testing - MockProvider, -) -``` - -### Storage Access - -```python -from ml4t.data.storage import ( - HiveStorage, - FilesystemStorage, - AsyncFilesystemStorage, - StorageConfig, - MetadataTracker, -) -``` - -### Validation Access - -```python -from ml4t.data.validation import ( - OHLCVValidator, - ValidationReport, - ValidationResult, - Validator, -) -``` - -### Anomaly Detection - -```python -from ml4t.data.anomaly import ( - AnomalyManager, - AnomalyReport, - Anomaly, - AnomalySeverity, - ReturnOutlierDetector, - VolumeSpikeDetector, - PriceStalenessDetector, - AnomalyConfig, - DetectorConfig, -) -``` - -### Update & Data Management - -```python -from ml4t.data.update_manager import ( - IncrementalUpdater, - UpdateStrategy, - UpdateResult, - GapDetector, -) - -from ml4t.data.data_manager import DataManager -``` - -### Utilities - -```python -from ml4t.data.utils import ( - format_symbol, - validate_frequency, - detect_gaps, - merge_gaps, -) -``` - -### Asset Management - -```python -from ml4t.data.assets import ( - AssetClass, - AssetInfo, - AssetSchema, - AssetValidator, - ContractSpec, - get_asset_schema, -) -``` - -### Configuration - -```python -from ml4t.data.config import ( - StorageConfig, - ProviderConfig, - DatasetConfig, - UpdateConfig, -) -``` - -### CLI Entry Point - -```bash -ml4t-data --help -ml4t-data fetch [options] -ml4t-data update-all [options] -ml4t-data list [options] -ml4t-data validate [options] -ml4t-data export [options] -``` - ---- - -## Performance Characteristics - -### Provider Performance - -**YahooFinance Wrapper Analysis**: -- **With reuse**: 0.92x (faster than native!) -- **Per-call overhead**: ~5ms (2.3% of typical download time) -- **Initialization cost**: 30-35ms (httpx.Client setup) - -**Rate Limiting**: -- Configurable per provider -- Defaults: 60 calls per minute -- Global coordination across providers - -### Storage Performance - -**Hive Partitioning**: -- Partition pruning for time-based queries -- Atomic writes with temp file pattern -- File locking for concurrent access - -**Metadata Tracking**: -- O(1) lookups for last update -- Efficient JSON serialization - -### Update Performance - -**Gap Detection**: -- Linear scan with frequency tolerance -- Optimized gap merging -- Efficient range queries - ---- - -## Dependencies - -### Core Production Dependencies (14) -- polars >= 0.20.0 -- pandas >= 2.0.0 -- numpy >= 1.24.0 -- httpx >= 0.25.0 -- tenacity >= 8.0.0 -- pybreaker >= 1.0.0 (circuit breaker) -- pyyaml >= 6.0 -- click >= 8.0.0 -- python-dotenv >= 1.0.0 -- pydantic-settings >= 2.0.0 -- structlog >= 23.0.0 -- platformdirs >= 4.0.0 -- filelock >= 3.19.1 -- rich >= 13.0.0 -- aiofiles >= 23.0.0 -- pandas-market-calendars >= 4.3.0 -- lxml >= 6.0.2 -- html5lib >= 1.1 - -### Optional Provider Dependencies -- yfinance >= 0.2.0 (Yahoo Finance) -- databento >= 0.38.0 (DataBento) -- oandapyV20 >= 0.7.0 (Oanda) - -### Development Dependencies (13) -- pytest >= 7.4.0 -- pytest-cov >= 4.1.0 -- pytest-xdist >= 3.3.0 -- pytest-timeout >= 2.1.0 -- pytest-asyncio >= 0.21.0 -- hypothesis >= 6.80.0 -- ruff >= 0.1.0 -- ty -- ipython >= 8.14.0 -- ipdb >= 0.13.0 -- pre-commit >= 3.3.0 - ---- - -## Integration with Other ML4T Libraries - -### Sibling Libraries -- **ml4t-features** (`../features/`) - Uses ml4t-data for OHLCV input -- **ml4t-eval** (`../evaluation/`) - Uses ml4t-data for backtest validation -- **ml4t-backtest** (`../backtest/`) - Uses ml4t-data for historical data - -### Integration Points -- Common OHLCV schema -- Shared storage patterns -- Cross-library validation - -### Division of Labor -- **ml4t-data work**: Provider improvements, storage optimization -- **Parent-level work**: Multi-library integration testing - ---- - -## Recent Developments (November 2025) - -### Wiki Prices Integration (Phase 5 & 6 Complete) -- ✅ WikiPricesProvider implemented (478 lines) -- ✅ 26 integration tests (21 passing) -- ✅ Fallback configuration documented -- ✅ 6 chapter notebooks (3,918 lines) - -### Performance Analysis -- ✅ Wrapper overhead analysis complete -- ✅ Provider reuse best practices documented -- ✅ Benchmarks for multi-dataset scenarios - -### Book Integration -- ✅ Chapter 4 notebooks ready (6 files) -- ✅ Three-tier learning path documented -- ✅ ml4t-data.yaml configuration template - ---- - -## Quality Metrics - -| Metric | Status | -|--------|--------| -| Test Files | 73 ✅ | -| Unit Tests | 50+ ✅ | -| Integration Tests | 10 ✅ | -| Provider Coverage | 13/13 (100%) ✅ | -| Code Coverage | ~80%+ (target) ✅ | -| Type Hints | Public interfaces checked with ty ⚠️ | -| Pre-commit Hooks | Enabled ✅ | -| CI/CD | GitHub Actions ✅ | - ---- - -## Known Gaps & TODOs - -### Critical (Blocking) -None identified - -### Important (Should Fix) -1. Binance integration tests - Requires VPN in US -2. Async storage tests - Needs more coverage -3. Type checking - reduce the remaining suppressed ty rules - -### Nice-to-Have (Low Priority) -1. WebSocket streaming support (Phase 2) -2. Parallel symbol updates in data_manager -3. Time-based futures rolling with expiration calendar - -### Won't Fix (Intentional) -1. Alpha Vantage provider - 25 calls/day free tier not worth porting -2. Older pandas compatibility - Requires >= 2.0.0 - ---- - -## Summary Table: Feature Readiness - -| Category | Status | Confidence | Notes | -|----------|--------|------------|-------| -| **Providers** | ✅ Complete | High | 13 providers, 12 live + 1 historical | -| **Storage** | ✅ Complete | High | Hive-partitioned and flat Parquet layouts | -| **Validation** | ✅ Complete | High | Full OHLC invariant checks | -| **Updates** | ✅ Complete | High | Incremental, gap detection | -| **CLI** | ✅ Complete | High | fetch, update-all, list, validate, export | -| **Anomaly Detection** | ✅ Complete | High | 3 detectors, configurable | -| **Configuration** | ✅ Complete | High | YAML/JSON with validation | -| **Concurrency** | ✅ Complete | Medium | File-based locks, rate limiting | -| **Asset Management** | ✅ Complete | High | Classes, contracts, schemas | -| **Futures** | ⚠️ Partial | Medium | Continuous contracts, some TODO items | -| **Async** | ⚠️ Partial | Medium | Implemented, needs testing | -| **Documentation** | ✅ Complete | High | Comprehensive README + notebooks | - ---- - -**Status**: Pre-release development (0.1.0) -**Last Updated**: 2025-11-27 -**Library Maturity**: 70-75% (from project map assessment) diff --git a/docs/GAP_ANALYSIS.md b/docs/GAP_ANALYSIS.md deleted file mode 100644 index 5e67ae7d..00000000 --- a/docs/GAP_ANALYSIS.md +++ /dev/null @@ -1,297 +0,0 @@ -# Gap Analysis Report - -*Last Updated: 2025-12-27* - -This report analyzes the gaps between what data providers offer and what ml4t-data currently implements, with prioritized recommendations for future development. - ---- - -## Executive Summary - -**Current State**: ml4t-data implements 21 data providers with unified OHLCV access. - -**Key Gaps Identified**: -- **Options Data** (HIGH priority) - Multiple providers offer options chains we don't support -- **Financial Statements** (HIGH priority) - Fundamentals data available but not implemented -- **Company Metrics** (MEDIUM priority) - Valuation ratios, analyst estimates - -**Explicitly NOT Supporting**: -- WebSocket streaming (out of scope for batch data library) -- Real-time order book (use native provider APIs directly) -- News/sentiment data (focus on price data) - ---- - -## Priority Gap Matrix - -### HIGH Priority (Should Implement) - -| Gap | Providers | Data Type | Effort | Value | -|-----|-----------|-----------|--------|-------| -| **Options Chains** | Polygon, Yahoo, Databento, EODHD | Greeks, strikes, expiries | High | High | -| **Financial Statements** | Yahoo, Finnhub, Polygon, EODHD | Balance sheet, income, cash flow | Medium | High | -| **Company Metrics** | Finnhub, Polygon | Valuation, profitability ratios | Medium | High | - -### MEDIUM Priority (Nice to Have) - -| Gap | Providers | Data Type | Effort | Value | -|-----|-----------|-----------|--------|-------| -| Earnings Data | Yahoo, EODHD, Finnhub | EPS, dates, surprises | Low | Medium | -| Analyst Estimates | Finnhub | Consensus, revisions | Low | Medium | -| ETF Fundamentals | EODHD | Holdings, expense ratios | Low | Medium | -| Trades (Tick) | Databento, Polygon | Individual transactions | High | Medium | -| Quotes (NBBO) | Databento, Polygon | Bid/ask spreads | High | Medium | - -### LOW Priority (Future Consideration) - -| Gap | Providers | Data Type | Effort | Value | -|-----|-----------|-----------|--------|-------| -| Holders Data | Yahoo | Institutional ownership | Low | Low | -| Insider Transactions | EODHD | Insider buys/sells | Low | Low | -| ESG Scores | Yahoo, Finnhub | Environmental, Social, Governance | Low | Low | -| Symbology API | Databento | Symbol mapping | Medium | Low | - ---- - -## Detailed Gap Analysis - -### 1. Options Data (HIGH Priority) - -**Current State**: Partially implemented. Databento OPRA supports chain -discovery, option OHLCV, quotes, and cost estimates; other provider-specific -options workflows remain gaps. - -**Available From**: -| Provider | Offering | Pricing | -|----------|----------|---------| -| Massive | Full options chains, Greeks, IV | $99+/mo (Developer tier) | -| Yahoo Finance | Options chains via yfinance | Free | -| Databento | OPRA options | Usage-based | -| EODHD | Options Marketplace | $29.99/mo add-on | - -**Implementation Approach**: -```python -# Proposed API -from ml4t.data.providers import MassiveProvider - -provider = MassiveProvider() -chain = provider.fetch_options_chain( - symbol="AAPL", - expiration="2025-01-17", - option_type="call" # or "put" or None for both -) - -# Returns: strike, bid, ask, volume, OI, IV, delta, gamma, theta, vega -``` - -**Effort**: High (new data schema, different storage pattern) -**Value**: High (options research, volatility analysis) - ---- - -### 2. Financial Statements (HIGH Priority) - -**Current State**: Not implemented - -**Available From**: -| Provider | Offering | Pricing | -|----------|----------|---------| -| Yahoo Finance | Balance sheet, income statement, cash flow | Free | -| Finnhub | Financials, basic financials | Free tier | -| Polygon | Financials endpoint | $99+/mo | -| EODHD | Fundamentals package | $59.99/mo | - -**Implementation Approach**: -```python -# Proposed API -from ml4t.data.providers import YahooFinanceProvider - -provider = YahooFinanceProvider() -financials = provider.fetch_financials( - symbol="AAPL", - statement="income", # income, balance, cashflow - period="quarterly" # quarterly, annual -) - -# Returns: revenue, gross_profit, operating_income, net_income, EPS, etc. -``` - -**Effort**: Medium (structured data, quarterly updates) -**Value**: High (fundamental analysis, value investing) - ---- - -### 3. Company Metrics (HIGH Priority) - -**Current State**: Not implemented - -**Available From**: -| Provider | Offering | Pricing | -|----------|----------|---------| -| Finnhub | Basic financials, ratios | Free tier (60 req/min) | -| Polygon | Company details, financials | $199+/mo | - -**Key Metrics**: -- Valuation: P/E, P/B, P/S, EV/EBITDA -- Profitability: ROE, ROA, Profit Margin -- Growth: Revenue Growth, EPS Growth -- Leverage: Debt/Equity, Interest Coverage - -**Effort**: Medium -**Value**: High (factor investing, screening) - ---- - -### 4. Earnings Data (MEDIUM Priority) - -**Current State**: Not implemented - -**Available From**: -| Provider | Offering | Pricing | -|----------|----------|---------| -| Yahoo Finance | Earnings history, next date | Free | -| EODHD | Earnings calendar, surprise | Included | -| Finnhub | Earnings calendar | Free tier | - -**Use Cases**: -- Earnings surprise analysis -- Event-driven strategies -- Volatility around announcements - -**Effort**: Low -**Value**: Medium - ---- - -### 5. Tick-Level Data (MEDIUM Priority) - -**Current State**: NASDAQ ITCH sample only - -**Available From**: -| Provider | Offering | Pricing | -|----------|----------|---------| -| Databento | Trades, MBO, MBP-10 | Usage-based ($$$) | -| Polygon | Trades, quotes | $199+/mo | - -**Challenges**: -- Storage: 100x more data than OHLCV -- Cost: Expensive to acquire -- Processing: Requires specialized handling - -**Recommendation**: Keep as specialized use case, not core library feature - ---- - -## Explicitly NOT Supporting - -### WebSocket Streaming - -**Reason**: ml4t-data is a batch data library for historical analysis. - -**Alternatives**: -- Use native provider SDKs (Databento, Polygon, Binance) -- Use specialized streaming libraries -- Build separate streaming component if needed - -### Real-Time Order Book - -**Reason**: Out of scope for historical data library. - -**Alternatives**: -- Databento `mbo` schema (native API) -- Binance WebSocket API -- Provider-native real-time APIs for supported services - -### News/Sentiment Data - -**Reason**: Focus on price data for quantitative analysis. - -**Alternatives**: -- Finnhub news endpoints -- Alpha Vantage news sentiment -- Specialized NLP providers - ---- - -## Implementation Roadmap - -### Phase 1: Options Data (Q1 2025) -1. Design options data schema -2. Implement Polygon options endpoint -3. Add Yahoo Finance options (yfinance) -4. Storage strategy for options chains - -### Phase 2: Fundamentals -1. Canonical financials and company metrics schema: implemented -2. Yahoo Finance financials and metrics: implemented -3. EODHD, Finnhub, and Massive fundamentals helpers: implemented -4. Remaining work: live-provider hardening and earnings/event data - -### Phase 3: Earnings & Events (Q3 2025) -1. Earnings calendar integration -2. Historical earnings data -3. Event-driven data pipeline - ---- - -## Provider Capability Matrix - -| Feature | Yahoo | Databento | Massive | EODHD | Finnhub | -|---------|-------|-----------|---------|-------|---------| -| OHLCV Daily | ✅ | ✅ | ✅ | ✅ | ✅ | -| OHLCV Minute | ✅ (7d) | ✅ | ✅ | ❌ | ✅ | -| Options Chains | ❌* | ✅ OPRA | ❌* | ❌* | ❌ | -| Financials | ✅ | ❌ | ✅ | ✅ | ✅ | -| Earnings | ❌* | ❌ | ❌* | ❌* | ❌* | -| Company Metrics | ✅ | ❌ | ✅ | ✅ | ✅ | -| Tick Data | ❌ | ❌* | ❌* | ❌ | ❌ | - -✅ = Implemented | ❌* = Available but not implemented | ❌ = Not available - ---- - -## Cost-Benefit Analysis - -### Options Data -- **Cost**: ~40 hours development + storage complexity -- **Benefit**: Enable options research, volatility strategies -- **ROI**: High for options-focused researchers - -### Fundamentals -- **Status**: Canonical helpers implemented for Yahoo, EODHD, Finnhub, and Massive -- **Remaining work**: Provider-specific live-data edge cases, pagination depth, and earnings/events -- **Benefit**: Enables value investing and fundamental factor research - -### Tick Data -- **Cost**: ~80 hours development + significant storage -- **Benefit**: Enable microstructure research, HFT analysis -- **ROI**: Low (specialized use case, high data costs) - ---- - -## Recommendations - -### Immediate (Next 3 Months) -1. **Document options gap clearly** - Users should know this is planned -2. **Harden fundamentals live coverage** - Validate EODHD, Finnhub, and Massive payload variants -3. **Document fundamentals examples** - Show financial statements and company metrics workflows - -### Short-Term (Next 6 Months) -1. **Implement options chains** - Start with Polygon -2. **Add earnings calendar** - Event-driven research -3. **Earnings calendar** - Event-driven research - -### Long-Term (Next 12 Months) -1. **Extend fundamentals depth** - Additional providers and pagination where plans support it -2. **Options analytics** - IV surface, Greeks calculation -3. **Tick data** - Evaluate based on user demand - ---- - -## Conclusion - -ml4t-data provides comprehensive OHLCV coverage across 21 providers and now includes an initial multi-provider fundamentals layer. The primary remaining gap is deeper **options and event data**, plus continued hardening of provider-specific fundamentals payloads. - -The library intentionally does NOT support streaming or real-time data, maintaining focus on batch historical data for backtesting and research. - -See [Provider Audit](providers/PROVIDER_AUDIT.md) for detailed provider capabilities. diff --git a/docs/INCREMENTAL_UPDATES.md b/docs/INCREMENTAL_UPDATES.md deleted file mode 100644 index e4871028..00000000 --- a/docs/INCREMENTAL_UPDATES.md +++ /dev/null @@ -1,336 +0,0 @@ -# Incremental Updates & Data Management - -This document describes the incremental update system implemented in Sprint 004, including gap detection, file locking, chunked storage, and metadata tracking. - -## Overview - -The incremental update system allows efficient data updates without re-downloading entire datasets. Key features include: - -- **Incremental Updates**: Only fetch new data since last update -- **Gap Detection**: Identify and fill missing data points -- **File Locking**: Safe concurrent access to data files -- **Chunked Storage**: Split large datasets into manageable time-based chunks -- **Metadata Tracking**: Monitor dataset health and update history - -## CLI Commands - -### Initial Data Load - -First time loading data for a symbol: - -```bash -# Load historical data -ml4t-data load --provider yahoo --symbol AAPL --start 2023-01-01 --end 2024-01-01 - -# Load with specific frequency -ml4t-data load -p yahoo -s AAPL --start 2023-01-01 --end 2024-01-01 -f daily -``` - -### Incremental Updates - -Update existing data with new data: - -```bash -# Basic update (uses existing provider) -ml4t-data update --symbol AAPL - -# Update with options -ml4t-data update -s AAPL --lookback-days 10 --fill-gaps --show-status - -# Update without gap filling -ml4t-data update -s AAPL --no-fill-gaps - -# Update with different provider -ml4t-data update -s AAPL --provider yahoo -``` - -Options: -- `--lookback-days/-l`: Days to look back for validation (default: 7) -- `--fill-gaps/--no-fill-gaps`: Enable/disable gap filling (default: enabled) -- `--show-status`: Display detailed update status and history -- `--provider/-p`: Override provider (uses existing if not specified) - -### Health Monitoring - -Check health status of all datasets: - -```bash -# Basic health check -ml4t-data status - -# Detailed health check -ml4t-data --verbose status - -# Custom staleness threshold -ml4t-data --verbose status --stale-days 3 -``` - -Output shows: -- Total datasets and their health status (✅ healthy, ⚠️ stale, ❌ error) -- Total rows across all datasets -- Breakdown by asset class -- Individual dataset details (with --verbose) - -## Workflow Examples - -### Example 1: Daily Stock Data Updates - -```bash -# Initial load of AAPL data -ml4t-data load -p yahoo -s AAPL --start 2023-01-01 --end 2024-01-01 - -# Daily update (run via cron) -ml4t-data update -s AAPL --show-status - -# Check health weekly -ml4t-data --verbose status -``` - -### Example 2: Multiple Symbol Management - -```bash -# Load multiple symbols -for symbol in AAPL GOOGL MSFT NVDA; do - ml4t-data load -p yahoo -s $symbol --start 2023-01-01 --end 2024-01-01 -done - -# Update all symbols -for symbol in AAPL GOOGL MSFT NVDA; do - ml4t-data update -s $symbol -done - -# Check overall health -ml4t-data status -``` - -### Example 3: Crypto Data (24/7 Trading) - -```bash -# Load crypto data -ml4t-data load -p yahoo -s BTC-USD --start 2023-01-01 --end 2024-01-01 -a crypto - -# Update with gap detection (important for 24/7 markets) -ml4t-data update -s BTC-USD -a crypto --fill-gaps --show-status -``` - -## Technical Details - -### Gap Detection - -The system distinguishes between expected and unexpected gaps: - -- **Stock Markets**: Weekends and after-hours are expected gaps -- **Crypto Markets**: 24/7 trading, all gaps are unexpected -- **Configurable Tolerance**: 10% default tolerance for gap detection - -Gap filling methods: -- `forward`: Use last known value (default) -- `backward`: Use next known value -- `interpolate`: Linear interpolation -- `zero`: Fill with zeros - -### File Locking - -Thread-safe and process-safe file locking ensures data integrity: - -- Uses `filelock` library for cross-platform compatibility -- Automatic lock acquisition and release -- Configurable timeout (default: 30 seconds) -- Prevents corruption during concurrent reads/writes - -### Chunked Storage - -Large datasets are split into time-based chunks: - -- **Monthly chunks** (default): 30-day periods -- **Weekly chunks**: 7-day periods -- **Quarterly chunks**: 90-day periods -- **Yearly chunks**: 365-day periods - -Benefits: -- Efficient incremental updates (only update relevant chunks) -- Parallel processing capability -- Reduced memory usage for large datasets -- Fast time-range queries - -### Metadata Tracking - -Each dataset maintains metadata including: - -- Update history (last 100 updates) -- Health status (healthy/stale/error) -- Data range and row count -- Provider information -- Error tracking - -Health checks consider: -- Days since last update -- Data currency (how far behind current date) -- Recent error frequency - -## Architecture - -``` -┌─────────────┐ ┌──────────────┐ ┌─────────────┐ -│ Provider │────▶│ Pipeline │────▶│ Storage │ -└─────────────┘ └──────────────┘ └─────────────┘ - │ │ - ▼ ▼ - ┌──────────────┐ ┌─────────────┐ - │ Gap Detector │ │ File Lock │ - └──────────────┘ └─────────────┘ - │ │ - ▼ ▼ - ┌──────────────┐ ┌─────────────┐ - │ Metadata │ │ Chunks │ - │ Tracker │ │ Storage │ - └──────────────┘ └─────────────┘ -``` - -### Components - -1. **Pipeline** (`src/ml4t-data/pipeline.py`) - - Orchestrates data flow - - `run_load()`: Full data load - - `run_update()`: Incremental update - -2. **Gap Detector** (`src/ml4t-data/utils/gaps.py`) - - Identifies missing data points - - Market hours awareness - - Multiple fill strategies - -3. **File Locking** (`src/ml4t-data/utils/locking.py`) - - Thread/process-safe access - - Automatic cleanup - - Timeout configuration - -4. **Chunked Storage** (`src/ml4t-data/storage/chunked.py`) - - Time-based data splitting - - Efficient updates - - Metadata indexing - -5. **Metadata Tracker** (`src/ml4t-data/storage/metadata_tracker.py`) - - Update history - - Health monitoring - - Summary statistics - -## Best Practices - -### Update Frequency - -- **Daily data**: Update once per day after market close -- **Minute data**: Update every few hours during market hours -- **Crypto**: Update more frequently (hourly or more) - -### Error Handling - -The system includes robust error handling: - -```python -# Automatic retries with exponential backoff -@with_retry(max_attempts=3, min_wait=1.0, max_wait=30.0) -def _fetch_data_with_retry(...) -``` - -### Monitoring - -Set up monitoring using the status command: - -```bash -# Cron job for daily health check -0 9 * * * ml4t-data status --stale-days 2 >> /var/log/ml4t-data-status.log -``` - -### Storage Management - -Monitor disk usage as datasets grow: - -```bash -# Check data directory size -du -sh ~/.ml4t-data/data/ - -# List all datasets -ml4t-data list - -# Remove old data if needed (manual process) -rm -rf ~/.ml4t-data/data/equities/daily/OLD_SYMBOL -``` - -## Troubleshooting - -### Common Issues - -1. **"No existing data found"** - - Run `ml4t-data load` first before using `update` - - Check the correct asset class and frequency - -2. **"Lock timeout"** - - Another process is accessing the file - - Check for stuck processes - - Increase timeout if needed - -3. **"Gaps detected"** - - Normal for some data sources - - Use `--fill-gaps` to automatically fill - - Check provider data quality - -4. **"Data is stale"** - - Run update more frequently - - Check provider connectivity - - Verify market hours settings - -### Debug Mode - -Enable debug logging for troubleshooting: - -```bash -# Set log level in .env -echo "ML4T Data_LOG_LEVEL=DEBUG" >> .env - -# Run with verbose output -ml4t-data update -s AAPL --show-status -``` - -## Performance Considerations - -### Memory Usage - -- Chunked storage keeps memory usage low -- Each chunk is processed independently -- Typical chunk size: 10-50 MB - -### Disk Usage - -- Parquet compression reduces storage by 50-80% -- Monthly chunks balance size and performance -- Metadata overhead: ~1KB per dataset - -### Update Speed - -- Incremental updates are 10-100x faster than full loads -- Gap detection adds minimal overhead (<1 second) -- File locking has negligible performance impact - -## Future Enhancements - -Potential improvements for future sprints: - -1. **Parallel Updates**: Update multiple symbols concurrently -2. **Smart Scheduling**: Automatic update scheduling based on asset class -3. **Data Validation**: Detect and flag suspicious data points -4. **Compression Options**: Support for different compression algorithms -5. **Archive Storage**: Move old data to compressed archives -6. **Update Notifications**: Email/webhook alerts for failures -7. **Data Reconciliation**: Compare and sync with multiple providers - -## Summary - -The incremental update system provides efficient, reliable data management with: - -- ✅ Minimal bandwidth usage (only fetch new data) -- ✅ Data integrity (file locking, gap detection) -- ✅ Scalability (chunked storage) -- ✅ Observability (health monitoring, update history) -- ✅ Flexibility (configurable options, multiple providers) - -This foundation enables building robust quantitative trading systems with reliable, up-to-date market data. diff --git a/docs/TESTING.md b/docs/TESTING.md deleted file mode 100644 index 16945c2d..00000000 --- a/docs/TESTING.md +++ /dev/null @@ -1,146 +0,0 @@ -# Testing Guide - -## Quick Start - -### Run Tests (Parallel by Default) -```bash -# Default: runs tests in parallel using all CPU cores (FAST!) -pytest - -# Verbose output -pytest -v - -# Disable parallel execution (slower, but better for debugging) -pytest -n 0 - -# Use specific number of workers -pytest -n 4 -``` - -**Performance**: Tests run in parallel by default using `pytest-xdist`: -- **24 cores**: ~7 seconds for 345 tests (130x speedup vs sequential) -- **Sequential** (`-n 0`): ~15 minutes for full suite - -### Run Integration Tests -```bash -# Run ALL tests including integration (if you have API keys) -pytest -m "" - -# Run only integration tests -pytest -m integration - -# Run specific provider integration tests -pytest tests/integration/test_coingecko.py -m "" -``` - -## Test Categories - -### Unit Tests (Fast) -- **Location**: Mostly in `tests/` root, some in `tests/unit/` -- **Duration**: <1 minute for all unit tests -- **Dependencies**: Core dependencies only -- **Markers**: None (run by default) - -### Integration Tests (Slow) -- **Location**: `tests/integration/` -- **Duration**: ~15 minutes -- **Dependencies**: Real APIs, may require API keys -- **Markers**: `@pytest.mark.integration` - -## API Keys for Integration Tests - -Integration tests for provider APIs require API keys: - -```bash -# Free tier (no key required) -export COINGECKO_API_KEY="" # Optional - -# Requires API keys -export MASSIVE_API_KEY="your_key_here" -export TWELVE_DATA_API_KEY="your_key_here" -export CRYPTOCOMPARE_API_KEY="your_key_here" -export FINNHUB_API_KEY="your_key_here" -export TIINGO_API_KEY="your_key_here" -export EODHD_API_KEY="your_key_here" -export ALPHA_VANTAGE_API_KEY="your_key_here" -export OANDA_API_KEY="your_key_here" -export DATABENTO_API_KEY="your_key_here" -``` - -## Test Markers - -| Marker | Description | Skip by Default? | -|--------|-------------|------------------| -| `integration` | Real API calls, slow tests | ✅ Yes | -| `slow` | Tests taking >10 seconds | ✅ Yes | -| `requires_api_key` | Needs specific API key | ⚠️ If key missing | -| `expensive` | High API costs | ⚠️ In CI only | - -## Common Test Commands - -```bash -# Run specific test file -pytest tests/test_core_models.py - -# Run specific test -pytest tests/test_core_models.py::test_function_name - -# Run tests matching pattern -pytest -k "yahoo" - -# Stop after first failure -pytest -x - -# Show print statements -pytest -s - -# Generate coverage report -pytest --cov=ml4t-data --cov-report=html -open htmlcov/index.html -``` - -## CI/CD - -GitHub Actions runs: -- **PR checks**: Unit tests only (fast) -- **Main branch**: Unit + integration tests (with API keys) - -## Troubleshooting - -### "ModuleNotFoundError: No module named 'databento'" -```bash -pip install -e ".[databento]" -``` - -### Tests taking too long -```bash -# Make sure you're running unit tests only -pytest -m "not integration and not slow" -``` - -### Integration tests skipped -```bash -# Check if API keys are set -env | grep API_KEY - -# Run with specific marker -pytest -m integration -``` - -### Parallel execution issues (debugging) -```bash -# Disable parallel execution for clearer error messages -pytest -n 0 - -# Use less workers to reduce resource contention -pytest -n 4 - -# Run single test file -pytest tests/test_specific_file.py -n 0 -``` - -### Test output not showing (parallel mode) -```bash -# Use -s with sequential execution to see print statements -pytest -n 0 -s -``` diff --git a/docs/api/index.md b/docs/api/index.md index 1c3ddf28..41af145c 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -269,8 +269,9 @@ without inheriting from `BaseProvider`. ### Config -Pydantic model for top-level library configuration. Reads defaults from -environment variables (`QLDM_DATA_ROOT`, `QLDM_LOG_LEVEL`). +Pydantic model for top-level library configuration. Reads `ML4T_DATA_PATH` and +`ML4T_DATA_LOG_LEVEL`. Former QLDM names are covered by the +[migration guide](../getting-started/migration.md). ```python from ml4t.data import Config diff --git a/docs/asset-classes/README.md b/docs/asset-classes/README.md deleted file mode 100644 index dac4c216..00000000 --- a/docs/asset-classes/README.md +++ /dev/null @@ -1,374 +0,0 @@ -# Asset Class Guides - -**Complete guides** for fetching market data across crypto, equities, forex, and futures markets using ML4T Data. - ---- - -## Overview - -These guides help you choose the right data provider and implement best practices for each asset class. Each guide includes: - -- ✅ **Provider recommendations** by use case -- ✅ **Quick start examples** with working code -- ✅ **Feature comparisons** (free tier, rate limits, coverage) -- ✅ **Complete API references** for each provider -- ✅ **Best practices** specific to the asset class -- ✅ **Common gotchas** and how to avoid them - ---- - -## Available Guides - -### [📈 Cryptocurrency Data Guide](./crypto.md) - -**Asset Class**: Bitcoin, Ethereum, Altcoins, Stablecoins -**Providers**: CoinGecko, CryptoCompare -**Difficulty**: 🟢 Beginner-friendly -**Free Tier**: ✅ Generous (50 calls/min to 100K/month) - -**Best for**: -- Crypto trading strategies -- Portfolio analysis -- Academic research on digital assets -- 24/7 market analysis - -**Quick start**: -```python -from ml4t.data.providers import CoinGeckoProvider - -# No API key needed! -provider = CoinGeckoProvider() -df = provider.fetch_ohlcv("BTC", start="2024-01-01", end="2024-12-31") -``` - -**[Read the full guide →](./crypto.md)** - ---- - -### [📊 Equities Data Guide](./equities.md) - -**Asset Classes**: US Stocks, Global Stocks, ETFs, Indices -**Providers**: Tiingo, Yahoo, EODHD, Finnhub, Twelve Data, Massive -**Difficulty**: 🟡 Moderate -**Free Tier**: ⚠️ Varies by provider - -**Best for**: -- US stock backtesting -- Global equity strategies -- Portfolio optimization -- Factor research - -**Quick start** (US stocks): -```python -from ml4t.data.providers import TiingoProvider - -provider = TiingoProvider(api_key="your_key") -df = provider.fetch_ohlcv("AAPL", start="2024-01-01", end="2024-12-31") -``` - -**Quick start** (Global stocks): -```python -from ml4t.data.providers import EODHDProvider - -provider = EODHDProvider(api_key="your_key", exchange="US") -df_us = provider.fetch_ohlcv("AAPL.US", start="2024-01-01", end="2024-12-31") -df_uk = provider.fetch_ohlcv("VOD.LSE", start="2024-01-01", end="2024-12-31") -``` - -**[Read the full guide →](./equities.md)** - ---- - -### [💱 Forex Data Guide](./forex.md) - -**Asset Class**: Currency Pairs (FX) -**Providers**: OANDA (primary), FXMacroData, Twelve Data, Massive -**Difficulty**: 🟡 Moderate -**Free Tier**: ⚠️ OANDA requires demo account (free), others have limits - -**Best for**: -- FX trading strategies -- Currency risk analysis -- Global macro research -- High-frequency forex strategies (OANDA offers 5-second to monthly bars) - -**Quick start**: -```python -from ml4t.data.providers import OandaProvider - -provider = OandaProvider(api_key="your_key", account_type="practice") -df = provider.fetch_ohlcv("EUR_USD", start="2024-01-01", end="2024-12-31", frequency="H1") -``` - -**[Read the full guide →](./forex.md)** - ---- - -### [Futures Data Guide](./futures.md) - -**Asset Classes**: Futures Contracts -**Providers**: Databento (primary), Massive for multi-asset alternatives -**Difficulty**: 🔴 Advanced -**Free Tier**: ❌ None (Databento requires paid subscription ~$30-50/month) - -**Best for**: -- Professional futures trading -- Derivatives research -- High-frequency strategies -- Institutional backtesting - -**Quick start**: -```python -from ml4t.data.providers import DataBentoProvider - -provider = DataBentoProvider(api_key="your_key", dataset="GLBX.MDP3") -df = provider.fetch_ohlcv("ES.v.0", start="2024-01-01", end="2024-12-31") # S&P 500 futures -``` - -**[Read the full guide →](./futures.md)** - ---- - -## Choosing the Right Asset Class - -### By Use Case - -| Use Case | Asset Class | Recommended Provider(s) | -|----------|-------------|-------------------------| -| **Crypto trading** | Cryptocurrency | CoinGecko (free), CryptoCompare | -| **US stock backtesting** | Equities | Tiingo (1000/day free) | -| **Global equities** | Equities | EODHD (€20/month best value) | -| **Forex strategies** | Forex | OANDA (professional-grade) | -| **Futures trading** | Futures | Databento (institutional quality) | -| **Multi-asset portfolio** | Mixed | Twelve Data (stocks+forex+crypto) | -| **Research (low volume)** | Any | Yahoo, Tiingo, EODHD, or Twelve Data depending on asset class | - -### By Budget - -**$0/month (Free Tier)**: -- ✅ **Crypto**: CoinGecko (50 calls/min, no API key) -- ✅ **US Stocks**: Tiingo (1000/day), Yahoo Finance (no API key, unofficial) -- ✅ **Forex**: OANDA demo account -- ❌ **Futures**: Not available free - -**< $50/month**: -- ✅ **Global Stocks**: EODHD with plan-dependent coverage and quotas -- ✅ **Futures**: Databento ($30-50/month starter) -- ✅ **Multi-asset**: Twelve Data ($49/month) - -**$100+/month (Professional)**: -- ✅ **Global Stocks**: Finnhub ($59.99/month) -- ✅ **Multi-asset**: Massive ($199-399/month) -- ✅ **Futures**: Databento ($50-100/month professional) - -### By Market Hours - -**24/7 Trading**: -- Cryptocurrency → [Crypto Guide](./crypto.md) - -**24/5 Trading** (weekdays only): -- Forex → [Forex Guide](./forex.md) -- Futures → [Futures Guide](./futures.md) - -**Market Hours Only** (9:30 AM - 4:00 PM ET typically): -- Equities → [Equities Guide](./equities.md) - ---- - -## Quick Provider Comparison - -### All Providers at a Glance - -| Provider | Crypto | Stocks | Forex | Futures | Free Tier | API Key | Best For | -|----------|--------|--------|-------|---------|-----------|---------|----------| -| **CoinGecko** | ✅ | ❌ | ❌ | ❌ | 50/min | Optional | Crypto (beginners) | -| **CryptoCompare** | ✅ | ❌ | ❌ | ❌ | 100K/month | Required | Crypto (advanced) | -| **Tiingo** | ❌ | ✅ US | ❌ | ❌ | 1000/day | Required | US stocks | -| **EODHD** | ❌ | ✅ Global | ❌ | ❌ | 20 calls/day free | Required | Global stocks | -| **Finnhub** | ⚠️ | ✅ Global | ⚠️ | ❌ | Real-time only | Required | Professional stocks | -| **OANDA** | ❌ | ❌ | ✅ | ❌ | Demo account | Required | Professional forex | -| **Twelve Data** | ✅ | ✅ | ✅ | ❌ | 800/day | Required | Multi-asset | -| **Massive** | ✅ | ✅ | ✅ | ✅ | 5/min | Required | Multi-asset (paid) | -| **Databento** | ❌ | ⚠️ | ❌ | ✅ OPRA | None | Required | Institutional futures/options | - -**Legend**: -- ✅ Full support -- ⚠️ Limited support -- ❌ Not supported - ---- - -## Common Patterns Across Asset Classes - -### 1. Incremental Updates - -**All asset classes** benefit from incremental updates to reduce API calls: - -```python -from ml4t.data.update_manager import IncrementalUpdater - -# Fetch the missing range with a provider, then merge into storage. -updater = IncrementalUpdater() - -# First run: Fetch all history -df = provider.fetch_ohlcv("AAPL", start="2024-01-01", end="2024-12-31") - -# Subsequent runs: Only fetch new data -result = updater.update_incremental(storage, tracker, "AAPL/daily", df, provider="tiingo") -# 100-1000x fewer API calls! -``` - -**See**: [Tutorial 03: Incremental Updates](../tutorials/03_incremental_updates.md) - -### 2. Data Quality Validation - -**Always validate** OHLCV data after fetching: - -```python -from ml4t.data.validation import validate_ohlcv - -df = provider.fetch_ohlcv(symbol="BTC", ...) - -# Validate invariants -validate_ohlcv(df) # Raises exception if data is corrupt - -# Checks: -# - high >= low ✓ -# - high >= open, close ✓ -# - low <= open, close ✓ -# - volume >= 0 ✓ -``` - -**See**: [Tutorial 04: Data Quality](../tutorials/04_data_quality.md) - -### 3. Multi-Provider Fallback - -**Never depend on a single provider**: - -```python -def fetch_data_robust(symbol: str, asset_class: str): - """Fetch data with fallback providers.""" - - if asset_class == "crypto": - providers = [CoinGeckoProvider, CryptoCompareProvider] - elif asset_class == "stocks": - providers = [ - lambda: YahooFinanceProvider(), - lambda: TiingoProvider(api_key="your_tiingo_key"), - lambda: EODHDProvider(api_key="your_eodhd_key", exchange="US"), - ] - elif asset_class == "forex": - providers = [OandaProvider, TwelveDataProvider] - - for create_provider in providers: - try: - provider = create_provider() - return provider.fetch_ohlcv(symbol, ...) - except Exception as e: - print(f"Provider failed: {e}") - - raise RuntimeError("All providers failed") -``` - -**See**: [Tutorial 05: Multi-Provider Strategies](../tutorials/05_multi_provider.md) - -### 4. Rate Limiting - -**Respect provider rate limits** to avoid bans: - -```python -# ML4T Data automatically rate-limits, but be aware of limits - -# CoinGecko: 50 calls/min -# Tiingo: 1000 calls/day -# Free-tier providers can have restrictive daily quotas. -# OANDA: 120 calls/second (very generous) - -# Use incremental updates to minimize calls -# Batch requests when possible -# Implement exponential backoff on failures -``` - -**See**: [Tutorial 02: Rate Limiting](../tutorials/02_rate_limiting.md) - ---- - -## Asset-Specific Considerations - -### Cryptocurrency - -**Key differences**: -- ✅ 24/7 markets (no weekends/holidays) -- ✅ No adjusted close needed (no corporate actions) -- ⚠️ High volatility (5-10%+ daily moves common) -- ⚠️ Liquidity varies by exchange - -**[Full details in Crypto Guide →](./crypto.md)** - -### Equities - -**Key differences**: -- ⚠️ Market hours only (9:30 AM - 4:00 PM ET typically) -- ⚠️ Weekends and holidays (no data) -- ✅ Need adjusted close (splits, dividends) -- ⚠️ Symbol format varies by provider (AAPL vs AAPL.US) - -**[Full details in Equities Guide →](./equities.md)** - -### Forex - -**Key differences**: -- ✅ 24/5 markets (Sunday 5 PM - Friday 5 PM ET) -- ⚠️ Session-based volatility (London/NY overlap highest) -- ⚠️ Spreads matter (0.1-2 pips transaction cost) -- ⚠️ Rollover interest (overnight positions) - -**[Full details in Forex Guide →](./forex.md)** - -### Futures - -**Key differences**: -- ⚠️ Contract expiration (must roll monthly/quarterly) -- ⚠️ Leverage (20:1 to 100:1 typical) -- ⚠️ Margin calls (marked-to-market daily) -- ⚠️ Tick sizes (minimum price increments) -- ❌ Most expensive data ($30-50+/month minimum) - -**[Full details in Futures Guide →](./futures.md)** - ---- - -## Next Steps - -1. **Read the guide** for your target asset class -2. **Choose a provider** based on budget and requirements -3. **Get API keys** (where required) -4. **Try the Quick Start** examples -5. **Implement best practices** from the tutorials -6. **Build multi-provider fallback** for production systems - ---- - -## Related Documentation - -**Core Tutorials**: -- [Tutorial 01: Understanding OHLCV Data](../tutorials/01_understanding_ohlcv.md) -- [Tutorial 02: Rate Limiting Best Practices](../tutorials/02_rate_limiting.md) -- [Tutorial 03: Incremental Updates](../tutorials/03_incremental_updates.md) -- [Tutorial 04: Data Quality Validation](../tutorials/04_data_quality.md) -- [Tutorial 05: Multi-Provider Strategies](../tutorials/05_multi_provider.md) - -**Provider Setup**: -- [Provider Selection Guide](../getting-started/provider-selection.md) -- [Creating a Custom Provider](../contributing/creating-a-provider.md) -- [Architecture](../contributing/architecture.md) - -**Getting Started**: -- [Documentation home](../index.md) -- [Quick Start Guide](../getting-started/quickstart.md) -- [Contribution guide](../contributing/index.md) - ---- - -**Questions or issues?** Open a GitHub issue or see the -[contribution guide](../contributing/index.md) for help. diff --git a/docs/asset-classes/crypto.md b/docs/asset-classes/crypto.md deleted file mode 100644 index 8610acf0..00000000 --- a/docs/asset-classes/crypto.md +++ /dev/null @@ -1,694 +0,0 @@ -# Cryptocurrency Data Guide - -**Asset Class**: Cryptocurrencies (Bitcoin, Ethereum, altcoins) -**Available Providers**: CoinGecko, CryptoCompare -**Difficulty**: Beginner-friendly -**Recommended For**: Crypto trading strategies, portfolio analysis, academic research - ---- - -## Overview - -ML4T Data provides two excellent **free-tier providers** for cryptocurrency data, both supporting 10,000+ digital assets with no API key requirements for basic usage. Cryptocurrency data is generally easier to work with than traditional financial data due to: - -- **24/7 Markets**: No market hours, weekends, or holidays -- **Global Access**: No regional restrictions like US-only or exchange-specific data -- **Generous Free Tiers**: Both providers offer substantial free quotas -- **Simple Symbol Format**: BTC, ETH, etc. (though CoinGecko uses internal IDs) - -**Quick Comparison**: - -| Provider | Free Tier | API Key | Best For | Symbol Format | -|----------|-----------|---------|----------|---------------| -| **CoinGecko** | 30 calls/min (Demo) | Required | Historical analysis, wide coverage | Coin IDs (bitcoin, ethereum) | -| **CryptoCompare** | 100K calls/month | Required | Aggregated prices, exchange data | Pairs (BTC/USD, ETH/BTC) | - ---- - -## CoinGecko (Recommended for Beginners) - -### Why CoinGecko? - -- ✅ **Easy setup** - Free Demo plan with API key (10K calls/month) -- ✅ **10,000+ cryptocurrencies** with market data -- ✅ **Reasonable rate limits** - 30 calls/minute is sufficient for most use cases -- ✅ **Aggregated data** across 500+ exchanges -- ✅ **Market metrics** - Volume, market cap, circulating supply - -**Best for**: Historical analysis, portfolio tracking, academic research - -### Quick Start - -```python -from datetime import UTC, datetime, timedelta - -from ml4t.data.providers import CoinGeckoProvider - -# Requires API key for Demo plan (free) -# Get your key from: https://www.coingecko.com/en/api/pricing -provider = CoinGeckoProvider() # Reads COINGECKO_API_KEY from environment -end = datetime.now(UTC).date() - timedelta(days=1) -start = end - timedelta(days=28) - -# Fetch Bitcoin daily data (last 29 completed UTC days) -df = provider.fetch_ohlcv( - symbol="BTC", # Uses symbol_to_id mapping internally - start=str(start), - end=str(end), - frequency="daily" -) - -print(df.head()) -``` - -**Output**: -``` -shape: (31, 7) -┌─────────────────────┬──────────┬───────────┬───────────┬───────────┬──────────────┬──────────┐ -│ timestamp │ symbol │ open │ high │ low │ close │ volume │ -│ --- │ --- │ --- │ --- │ --- │ --- │ --- │ -│ datetime[μs] │ str │ f64 │ f64 │ f64 │ f64 │ f64 │ -╞═════════════════════╪══════════╪═══════════╪═══════════╪═══════════╪══════════════╪══════════╡ -│ 2024-01-01 00:00:00 │ BTC │ 42265.18 │ 44923.45 │ 41824.36 │ 44167.21 │ 1.85e9 │ -│ 2024-01-02 00:00:00 │ BTC │ 44167.21 │ 45870.34 │ 43956.12 │ 45456.87 │ 2.12e9 │ -└─────────────────────┴──────────┴───────────┴───────────┴───────────┴──────────────┴──────────┘ -``` - -### Symbol Mapping - -CoinGecko uses internal **coin IDs** (e.g., "bitcoin", "ethereum") rather than ticker symbols. ML4T Data handles this automatically for major coins: - -```python -# These all work - ML4T Data maps symbols to IDs internally -provider.fetch_ohlcv(symbol="BTC", ...) # → bitcoin -provider.fetch_ohlcv(symbol="ETH", ...) # → ethereum -provider.fetch_ohlcv(symbol="DOGE", ...) # → dogecoin - -# Or use coin IDs directly -provider.fetch_ohlcv(symbol="bitcoin", ...) -``` - -**Pre-mapped symbols** (20+ popular coins): -- BTC → bitcoin -- ETH → ethereum -- USDT → tether -- BNB → binancecoin -- USDC → usd-coin -- XRP → ripple -- ADA → cardano -- DOGE → dogecoin -- SOL → solana -- And more... - -**Finding coin IDs** for other cryptocurrencies: -```python -# Search CoinGecko's coin list (programmatically) -# Visit: https://www.coingecko.com/en/coins/ -# Or use their API: https://api.coingecko.com/api/v3/coins/list -``` - -### Supported Frequencies - -CoinGecko free tier supports **daily OHLCV** data only: - -```python -# ✅ Supported -df = provider.fetch_ohlcv(symbol="BTC", frequency="daily") - -# ❌ Not supported on free tier (requires CoinGecko Pro) -df = provider.fetch_ohlcv(symbol="BTC", frequency="hourly") # Error! -``` - -Daily OHLCV is limited to the most recent 29 completed UTC days. CoinGecko returns -four-day OHLC candles for older windows, so this provider rejects those requests instead -of labeling coarse values as daily bars. - -### Rate Limits - -**Demo Plan (Free)**: 30 calls per minute, 10,000 calls/month -**Public API (No Key)**: 5-15 calls per minute (unstable, not recommended) - -The Demo plan is **sufficient** for most use cases: - -```python -# Example: Fetch 10 coins, 90 days of data each -# = 10 API calls = well within rate limit - -symbols = ["BTC", "ETH", "SOL", "ADA", "DOGE", - "MATIC", "LINK", "UNI", "AVAX", "DOT"] - -for symbol in symbols: - df = provider.fetch_ohlcv( - symbol=symbol, - start="2024-01-01", - end="2024-03-31", - frequency="daily" - ) - # Provider automatically rate-limits (no sleep needed) -``` - -**Pro Tier** (with API key): 10 calls/second - -```python -provider = CoinGeckoProvider( - api_key="your_coingecko_api_key", # From environment or parameter - use_pro=True # Enable pro endpoints -) -``` - -### Complete Example: Multi-Coin Analysis - -```python -from ml4t.data.providers import CoinGeckoProvider -from datetime import datetime, timedelta -import polars as pl - -# Initialize provider (no API key needed) -provider = CoinGeckoProvider() - -# Define portfolio -portfolio = ["BTC", "ETH", "SOL", "AVAX", "MATIC"] - -# Fetch 90 days of data for each -end_date = datetime.now() -start_date = end_date - timedelta(days=90) - -all_data = [] -for symbol in portfolio: - df = provider.fetch_ohlcv( - symbol=symbol, - start=start_date.strftime("%Y-%m-%d"), - end=end_date.strftime("%Y-%m-%d"), - frequency="daily" - ) - all_data.append(df) - -# Combine into single DataFrame -combined = pl.concat(all_data) - -# Calculate daily returns -returns = combined.group_by("symbol").agg( - pl.col("close").pct_change().alias("daily_return") -) - -print(returns) -``` - -### Error Handling - -Common errors and solutions: - -```python -from ml4t.data.core.exceptions import SymbolNotFoundError, RateLimitError - -try: - df = provider.fetch_ohlcv(symbol="INVALID", start="2024-01-01", end="2024-01-31") -except SymbolNotFoundError as e: - print(f"Coin not found: {e}") - # Solution: Check symbol spelling or use coin ID directly - -try: - # Too many rapid calls - for i in range(100): - df = provider.fetch_ohlcv(symbol="BTC", ...) -except RateLimitError as e: - print(f"Rate limited: {e}") - # ML4T Data automatically retries with backoff -``` - -### When to Use CoinGecko - -✅ **Good for**: -- Beginners (zero setup) -- Historical backtesting (daily data) -- Portfolio analysis -- Academic research -- Multi-coin comparisons - -❌ **Not ideal for**: -- Intraday trading strategies (no hourly/minute data on free tier) -- Exchange-specific data (aggregated only) -- Ultra-high-frequency analysis - ---- - -## CryptoCompare (Advanced Features) - -### Why CryptoCompare? - -- ✅ **Aggregated prices** across multiple exchanges (CCCAGG index) -- ✅ **Exchange-specific data** - Choose Binance, Coinbase, Kraken, etc. -- ✅ **Multiple frequencies** - Minute, hourly, daily (free tier) -- ✅ **100,000 calls/month** free tier -- ✅ **5,000+ cryptocurrencies** - -**Best for**: Aggregated price analysis, exchange comparisons, intraday strategies - -### Quick Start - -```python -from ml4t.data.providers import CryptoCompareProvider - -# A CryptoCompare API key is required -provider = CryptoCompareProvider( - api_key="your_cryptocompare_api_key", # Or from env: CRYPTOCOMPARE_API_KEY - exchange="CCCAGG" # Aggregate across exchanges (default) -) - -# Fetch Bitcoin daily data -df = provider.fetch_ohlcv( - symbol="BTC/USD", # Note: Uses pair format - start="2024-01-01", - end="2024-01-31", - frequency="daily" -) - -print(df.head()) -``` - -### Symbol Format - -CryptoCompare uses **trading pairs** with slash notation: - -```python -# ✅ Correct format -"BTC/USD" # Bitcoin vs US Dollar -"ETH/BTC" # Ethereum vs Bitcoin -"SOL/USDT" # Solana vs Tether - -# ❌ Incorrect -"BTC" # Missing quote currency -"BTCUSD" # Missing slash separator -``` - -**Common pairs**: -```python -# Fiat pairs -"BTC/USD", "ETH/USD", "SOL/USD" - -# Stablecoin pairs -"BTC/USDT", "ETH/USDC", "BNB/BUSD" - -# Crypto pairs -"ETH/BTC", "SOL/ETH", "ADA/BTC" -``` - -### Supported Frequencies - -CryptoCompare free tier supports **minute, hourly, and daily** data: - -```python -# Daily (up to 2000 days per call) -df = provider.fetch_ohlcv(symbol="BTC/USD", frequency="daily") - -# Hourly (up to 2000 hours per call) -df = provider.fetch_ohlcv(symbol="BTC/USD", frequency="hourly") - -# Minute (up to 2000 minutes per call) -df = provider.fetch_ohlcv(symbol="BTC/USD", frequency="1minute") -``` - -**Frequency limits**: -- `daily`: 2000 bars per API call (~5.5 years) -- `hourly`: 2000 bars per API call (~83 days) -- `1minute`: 2000 bars per API call (~33 hours) - -### Exchange Selection - -Fetch data from specific exchanges or use aggregated data: - -```python -# Aggregated (default) - CryptoCompare's weighted index -provider_agg = CryptoCompareProvider(exchange="CCCAGG") - -# Binance-specific -provider_binance = CryptoCompareProvider(exchange="Binance") - -# Coinbase-specific -provider_coinbase = CryptoCompareProvider(exchange="Coinbase") - -# Kraken-specific -provider_kraken = CryptoCompareProvider(exchange="Kraken") -``` - -**Popular exchanges supported**: -- Binance, Coinbase, Kraken, Gemini, Bitstamp -- Bitfinex, Huobi, OKEx, FTX (historical), and 100+ more - -### Rate Limits - -**Free Tier**: 100,000 calls per month (10 calls/minute recommended) - -```python -# ML4T Data automatically rate-limits to ~10 calls/minute -# This stays well within monthly quota: -# 10 calls/min × 60 min × 24 hr × 30 days = 432,000 theoretical max -# But capped at 100K/month by API - -# Example: Safe usage pattern -import time -from datetime import datetime, timedelta - -symbols = ["BTC/USD", "ETH/USD", "SOL/USD"] - -for symbol in symbols: - df = provider.fetch_ohlcv( - symbol=symbol, - start="2024-01-01", - end="2024-12-31", - frequency="daily" - ) - # No manual sleep needed - ML4T Data handles rate limiting -``` - -**With API Key**: Higher rate limits (varies by plan) - -### Complete Example: Exchange Comparison - -```python -from ml4t.data.providers import CryptoCompareProvider -import polars as pl - -# Compare Bitcoin price across exchanges -exchanges = ["CCCAGG", "Binance", "Coinbase", "Kraken"] - -all_data = [] -for exchange in exchanges: - provider = CryptoCompareProvider(exchange=exchange) - - df = provider.fetch_ohlcv( - symbol="BTC/USD", - start="2024-01-01", - end="2024-01-31", - frequency="daily" - ) - - # Add exchange column - df = df.with_columns(pl.lit(exchange).alias("exchange")) - all_data.append(df) - - provider.close() - -# Combine and analyze -combined = pl.concat(all_data) - -# Calculate price spread across exchanges -spread = combined.group_by("timestamp").agg([ - pl.col("close").max().alias("max_price"), - pl.col("close").min().alias("min_price"), - (pl.col("close").max() - pl.col("close").min()).alias("spread"), - ((pl.col("close").max() - pl.col("close").min()) / pl.col("close").mean() * 100).alias("spread_pct") -]) - -print(spread.sort("timestamp")) -# Identify arbitrage opportunities! -``` - -### Error Handling - -```python -from ml4t.data.core.exceptions import DataNotAvailableError, RateLimitError - -try: - df = provider.fetch_ohlcv( - symbol="OBSCURECOIN/USD", - start="2024-01-01", - end="2024-01-31" - ) -except DataNotAvailableError as e: - print(f"Pair not available: {e}") - # Try different exchange or check coin listing - -try: - # Exceeding rate limits - for i in range(20): - df = provider.fetch_ohlcv(symbol="BTC/USD", ...) -except RateLimitError as e: - print(f"Rate limited: {e}") - # Wait or reduce call frequency -``` - -### When to Use CryptoCompare - -✅ **Good for**: -- Intraday strategies (minute/hourly data) -- Exchange-specific analysis -- Price aggregation studies -- Arbitrage detection -- Professional crypto research - -❌ **Not ideal for**: -- Absolute beginners (requires API key understanding) -- Projects needing >100K calls/month without paid tier -- Coins not listed on major exchanges - ---- - -## Provider Comparison - -### Feature Comparison - -| Feature | CoinGecko | CryptoCompare | -|---------|-----------|---------------| -| **API Key Required** | Yes (Demo plan) | Yes (free account available) | -| **Free Tier Limit** | 30 calls/min (10K/month) | 100K calls/month | -| **Cryptocurrencies** | 10,000+ | 5,000+ | -| **Daily Data** | ✅ Yes | ✅ Yes | -| **Hourly Data** | ❌ Pro only | ✅ Yes | -| **Minute Data** | ❌ Pro only | ✅ Yes | -| **Aggregated Prices** | ✅ Yes | ✅ Yes (CCCAGG) | -| **Exchange-Specific** | ❌ No | ✅ Yes | -| **Symbol Format** | Coin IDs | Trading pairs | -| **Market Metrics** | ✅ Yes (market cap, volume) | ⚠️ Limited | -| **Setup Difficulty** | 🟢 Easy | 🟡 Moderate | - -### Performance Comparison - -```python -# Benchmark: Fetch 1 year of daily BTC data - -# CoinGecko -provider = CoinGeckoProvider() -df = provider.fetch_ohlcv(symbol="BTC", start="2023-01-01", end="2023-12-31") -# → 1 API call, ~2 seconds - -# CryptoCompare -# Reads CRYPTOCOMPARE_API_KEY from the environment -provider = CryptoCompareProvider() -df = provider.fetch_ohlcv(symbol="BTC/USD", start="2023-01-01", end="2023-12-31") -# → 1 API call, ~2 seconds - -# Both providers are equally fast for daily data -``` - -### Cost Comparison - -**Free Tier Value**: - -| Provider | Free Limit | Est. Monthly Value | Paid Tier | -|----------|------------|--------------------|-----------| -| **CoinGecko** | 30 calls/min (10K/month) | $0 (requires signup) | ~$129/month (Pro) | -| **CryptoCompare** | 100K calls/month | $0 (very generous) | $49-$999/month | - -**Which is more generous?** - -```python -# Scenario 1: Daily batch job (fetch 100 coins daily) -# CoinGecko: 100 calls/day × 30 days = 3,000 calls/month ✅ -# CryptoCompare: 100 calls/day × 30 days = 3,000 calls/month ✅ -# Winner: Tie - -# Scenario 2: Intraday strategy (hourly updates for 10 coins) -# CoinGecko: Not available on free tier ❌ -# CryptoCompare: 10 calls/hour × 24 hrs × 30 days = 7,200 calls/month ✅ -# Winner: CryptoCompare - -# Scenario 3: High-frequency batch (1000 coins, daily) -# CoinGecko: 1000 calls/day = 34 minutes @30/min (within monthly 10K cap for ~10 days) ⚠️ -# CryptoCompare: 1000 calls/day × 30 = 30,000 calls/month ✅ -# Winner: CryptoCompare (for sustained high volume) -``` - ---- - -## Best Practices - -### 1. Use Incremental Updates - -Don't re-fetch entire history every time: - -```python -# First run: Fetch all history -history = provider.fetch_ohlcv("BTC", start="2024-01-01", end="2024-12-31") -storage.write(history, "coingecko/BTC") - -# Subsequent runs: Only fetch new data, then merge into storage -new_data = provider.fetch_ohlcv("BTC", start=last_stored_date, end=today) -``` - -**See**: [Tutorial 03: Incremental Updates](../tutorials/03_incremental_updates.md) - -### 2. Validate Data Quality - -Always validate OHLCV data after fetching: - -```python -from ml4t.data.validation import validate_ohlcv - -df = provider.fetch_ohlcv(symbol="BTC", ...) - -# Validate invariants (high >= low, etc.) -validate_ohlcv(df) # Raises exception if data is corrupt - -# Check for gaps -gaps = df.select("timestamp").diff().filter(pl.col("timestamp") > timedelta(days=2)) -if not gaps.is_empty(): - print(f"Warning: {len(gaps)} gaps found in data") -``` - -**See**: [Tutorial 04: Data Quality](../tutorials/04_data_quality.md) - -### 3. Handle Rate Limits Gracefully - -ML4T Data automatically rate-limits, but be aware: - -```python -# ✅ Good: Sequential fetching (automatic rate limiting) -for symbol in ["BTC", "ETH", "SOL"]: - df = provider.fetch_ohlcv(symbol=symbol, ...) - -# ❌ Bad: Parallel fetching without coordination -import concurrent.futures -with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: - # Will exceed rate limits! - futures = [executor.submit(provider.fetch_ohlcv, symbol=s) for s in symbols] -``` - -**See**: [Tutorial 02: Rate Limiting](../tutorials/02_rate_limiting.md) - -### 4. Use Multi-Provider Fallback - -Never depend on a single provider: - -```python -from ml4t.data.providers import CoinGeckoProvider, CryptoCompareProvider - -def fetch_crypto_data_robust(symbol: str, start: str, end: str): - """Fetch crypto data with fallback providers.""" - - # Try CoinGecko first (no API key needed) - try: - provider = CoinGeckoProvider() - return provider.fetch_ohlcv(symbol=symbol, start=start, end=end) - except Exception as e: - print(f"CoinGecko failed: {e}") - - # Fallback to CryptoCompare - try: - # Reads CRYPTOCOMPARE_API_KEY from the environment - provider = CryptoCompareProvider() - # Convert symbol format: BTC → BTC/USD - pair = f"{symbol}/USD" - return provider.fetch_ohlcv(symbol=pair, start=start, end=end) - except Exception as e: - print(f"CryptoCompare failed: {e}") - raise RuntimeError("All providers failed") - -# Usage -df = fetch_crypto_data_robust("BTC", "2024-01-01", "2024-01-31") -``` - -**See**: [Tutorial 05: Multi-Provider Strategies](../tutorials/05_multi_provider.md) - ---- - -## Common Gotchas - -### 1. Symbol Format Differences - -```python -# ❌ Wrong: Using wrong format for provider -coingecko = CoinGeckoProvider() -coingecko.fetch_ohlcv(symbol="BTC/USD", ...) # Error! Needs "BTC" or "bitcoin" - -cryptocompare = CryptoCompareProvider() # Reads CRYPTOCOMPARE_API_KEY -cryptocompare.fetch_ohlcv(symbol="BTC", ...) # Error! Needs "BTC/USD" - -# ✅ Correct: Use provider-specific format -coingecko.fetch_ohlcv(symbol="BTC", ...) # Symbol or coin ID -cryptocompare.fetch_ohlcv(symbol="BTC/USD", ...) # Trading pair -``` - -### 2. Timezone Assumptions - -All timestamps are UTC, but crypto markets are 24/7: - -```python -# Crypto data doesn't have "market close" like stocks -df = provider.fetch_ohlcv(symbol="BTC", start="2024-01-01", end="2024-01-02") - -# Check timezone -assert df["timestamp"][0].dt.hour() == 0 # Midnight UTC -``` - -### 3. Rate Limit Planning - -```python -# ❌ Bad: Not accounting for rate limits -for i in range(1000): # Will take 34+ minutes on CoinGecko Demo plan - df = provider.fetch_ohlcv(symbol=f"COIN_{i}", ...) - -# ✅ Good: Batch wisely or upgrade to paid tier -# CoinGecko Demo: 30 calls/min = 1000 calls in 34 minutes (hits monthly cap fast!) -# CryptoCompare free: 100K/month = plenty for most use cases -``` - -### 4. Aggregated vs Exchange Data - -```python -# CoinGecko: Always aggregated across exchanges -coingecko_df = coingecko.fetch_ohlcv(symbol="BTC", ...) -# Price is weighted average across 500+ exchanges - -# CryptoCompare: Choose aggregated or exchange-specific -agg_provider = CryptoCompareProvider(exchange="CCCAGG") # Aggregated -binance_provider = CryptoCompareProvider(exchange="Binance") # Binance only - -# Prices may differ significantly! -``` - ---- - -## Related Documentation - -**Getting Started**: -- [Quick Start Guide](../getting-started/quickstart.md) -- [Tutorial 01: Understanding OHLCV Data](../tutorials/01_understanding_ohlcv.md) - -**Provider Setup**: -- [Provider Selection Guide](../getting-started/provider-selection.md) -- [Creating a Custom Provider](../contributing/creating-a-provider.md) - -**Best Practices**: -- [Tutorial 02: Rate Limiting Best Practices](../tutorials/02_rate_limiting.md) -- [Tutorial 03: Incremental Updates](../tutorials/03_incremental_updates.md) -- [Tutorial 04: Data Quality Validation](../tutorials/04_data_quality.md) -- [Tutorial 05: Multi-Provider Strategies](../tutorials/05_multi_provider.md) - -**Other Asset Classes**: -- [Equities Data Guide](./equities.md) - US and global stocks -- [Forex Data Guide](./forex.md) - Foreign exchange pairs -- [Futures Data Guide](./futures.md) - Futures and options - ---- - -## Next Steps - -1. **Try the Quick Start examples** above with CoinGecko (no API key needed!) -2. **Read Tutorial 01** to understand OHLCV data fundamentals -3. **Implement incremental updates** (Tutorial 03) for production systems -4. **Set up data quality validation** (Tutorial 04) to catch corrupted data -5. **Build multi-provider fallback** (Tutorial 05) for resilience - -**Questions or issues?** See the [contribution guide](../contributing/index.md) or open a -GitHub issue. diff --git a/docs/asset-classes/equities.md b/docs/asset-classes/equities.md deleted file mode 100644 index 0d2d99fa..00000000 --- a/docs/asset-classes/equities.md +++ /dev/null @@ -1,719 +0,0 @@ -# Equities (Stocks) Data Guide - -**Asset Classes**: US Stocks, Global Stocks, ETFs, Indices -**Available Providers**: Tiingo, Yahoo Finance, EODHD, Finnhub, Twelve Data, Massive -**Difficulty**: Moderate (requires API keys, understanding of market hours) -**Recommended For**: Stock trading strategies, portfolio backtesting, factor research - ---- - -## Overview - -ML4T Data provides **multiple free-tier providers** for equity data, ranging from US-only to global multi-exchange coverage. Stock data has unique characteristics compared to crypto: - -- **Market Hours**: Trading typically 9:30 AM - 4:00 PM ET (NYSE/NASDAQ) -- **Weekends & Holidays**: No data on weekends, market holidays -- **Corporate Actions**: Splits, dividends require adjusted close prices -- **Exchange Specificity**: Same symbol may trade on multiple exchanges - -**Quick Provider Comparison**: - -| Provider | US Stocks | Global | Free Tier | API Key | Best For | -|----------|-----------|--------|-----------|---------|----------| -| **Tiingo** | ✅ | ❌ | 1000/day | Required | High-quality US stocks | -| **Yahoo Finance** | ✅ | ⚠️ Limited | Unofficial | No | Quick experiments | -| **EODHD** | ✅ | ✅ 60+ exchanges | 20 calls/day free | Required | Global coverage | -| **Finnhub** | ✅ | ✅ 70+ exchanges | Real-time only | Required | Professional (paid OHLCV) | -| **Twelve Data** | ✅ | ✅ | 800/day | Required | Multi-asset flexibility | -| **Massive** | ✅ | ✅ | 5 calls/min | Required | Real-time + historical | - ---- - -## US Stocks Providers - -### Tiingo (Recommended for US Stocks) - -#### Why Tiingo? - -- ✅ **High-quality data** - Sourced from IEX Exchange -- ✅ **1000 API calls/day** free tier (generous) -- ✅ **500 unique symbols/month** free tier -- ✅ **Adjusted close** - Handles splits and dividends automatically -- ✅ **Excellent coverage** - All major US exchanges (NYSE, NASDAQ, AMEX) - -**Best for**: US stock backtesting, academic research, portfolio analysis - -#### Quick Start - -```python -from ml4t.data.providers import TiingoProvider - -# Get free API key from: https://www.tiingo.com/account/api/token -provider = TiingoProvider(api_key="your_tiingo_api_key") - -# Fetch Apple daily data -df = provider.fetch_ohlcv( - symbol="AAPL", - start="2024-01-01", - end="2024-12-31", - frequency="daily" -) - -print(df.head()) -``` - -**Output**: -``` -shape: (252, 7) -┌─────────────────────┬──────────┬───────────┬───────────┬───────────┬───────────┬──────────────┐ -│ timestamp │ symbol │ open │ high │ low │ close │ volume │ -│ --- │ --- │ --- │ --- │ --- │ --- │ --- │ -│ datetime[μs] │ str │ f64 │ f64 │ f64 │ f64 │ f64 │ -╞═════════════════════╪══════════╪═══════════╪═══════════╪═══════════╪═══════════╪══════════════╡ -│ 2024-01-02 00:00:00 │ AAPL │ 187.15 │ 188.44 │ 183.89 │ 185.64 │ 8.26e7 │ -│ 2024-01-03 00:00:00 │ AAPL │ 184.22 │ 185.88 │ 183.43 │ 184.25 │ 5.80e7 │ -└─────────────────────┴──────────┴───────────┴───────────┴───────────┴───────────┴──────────────┘ -``` - -#### Features - -**Supported Frequencies**: -```python -# Daily (most common) -df = provider.fetch_ohlcv(symbol="AAPL", frequency="daily") - -# Weekly -df = provider.fetch_ohlcv(symbol="AAPL", frequency="weekly") - -# Monthly -df = provider.fetch_ohlcv(symbol="AAPL", frequency="monthly") -``` - -**Adjusted Close Handling**: -```python -# Tiingo automatically provides adjusted close prices -df = provider.fetch_ohlcv(symbol="AAPL", start="2020-01-01", end="2024-01-01") - -# The 'close' column accounts for: -# - Stock splits (e.g., AAPL 4:1 split in Aug 2020) -# - Dividend distributions -# - Other corporate actions -``` - -**Symbol Coverage**: -- All NYSE stocks (~2,800 symbols) -- All NASDAQ stocks (~3,300 symbols) -- AMEX, OTC markets -- ETFs and mutual funds - -#### Rate Limits - -**Free Tier**: -- 1000 API calls per day -- 500 unique symbols per month -- No intraday data - -**Paid Tier** ($30/month): -- 20,000 API calls per hour -- Unlimited symbols -- Intraday data (1-minute, 5-minute bars) - -**Usage Strategy**: -```python -# Example: Fetch S&P 500 constituents (500 symbols) -# Fits within free tier: 500 symbols, 500 API calls - -sp500_symbols = ["AAPL", "MSFT", "GOOGL", ...] # 500 symbols - -for symbol in sp500_symbols: - df = provider.fetch_ohlcv( - symbol=symbol, - start="2024-01-01", - end="2024-12-31", - frequency="daily" - ) - # 500 API calls total - well within 1000/day limit -``` - -#### Incremental Updates - -```python -from ml4t.data.update_manager import IncrementalUpdater - -# First run: Fetch and store the initial history -history = provider.fetch_ohlcv("AAPL", start="2024-01-01", end="2024-12-31") -storage.write(history, "tiingo/AAPL") - -# Subsequent runs: Fetch only the missing range and merge -new_data = provider.fetch_ohlcv("AAPL", start=last_stored_date, end=today) -updater = IncrementalUpdater() -updater.update_incremental(storage, tracker, "tiingo/AAPL", new_data, provider="tiingo") -``` - -#### When to Use Tiingo - -✅ **Good for**: -- US stock backtesting -- Daily/weekly/monthly strategies -- Academic research -- Portfolio analysis -- 100-500 symbols daily updates - -❌ **Not ideal for**: -- Global stocks (US only) -- Intraday strategies (free tier) -- High-frequency trading -- Real-time quotes - ---- - -## Global Stocks Providers - -### EODHD (Best Value for Global Coverage) - -#### Why EODHD? - -- ✅ **60+ global exchanges** - US, Europe, Asia, emerging markets -- ✅ **150,000+ tickers** worldwide -- ✅ **20 API calls/day free**; paid quotas depend on the selected plan -- ✅ **Most affordable global data** vs competitors -- ✅ **Adjusted close** for splits and dividends - -**Best for**: Global equity strategies, international portfolios, emerging markets - -#### Quick Start - -```python -from ml4t.data.providers import EODHDProvider - -# Get free API key from: https://eodhd.com/register -provider = EODHDProvider( - api_key="your_eodhd_api_key", - exchange="US" # Default exchange -) - -# US stocks (AAPL.US format) -us_data = provider.fetch_ohlcv( - symbol="AAPL.US", - start="2024-01-01", - end="2024-12-31", - frequency="daily" -) - -# London Stock Exchange (VOD.LSE) -lse_data = provider.fetch_ohlcv( - symbol="VOD.LSE", - start="2024-01-01", - end="2024-12-31", - frequency="daily" -) - -# Frankfurt Stock Exchange (BMW.FRA) -fra_data = provider.fetch_ohlcv( - symbol="BMW.FRA", - start="2024-01-01", - end="2024-12-31", - frequency="daily" -) -``` - -#### Global Exchange Coverage - -**Major Exchanges**: - -| Region | Exchange | Code | Symbols | Example | -|--------|----------|------|---------|---------| -| **US** | NYSE, NASDAQ | US | 51,000+ | AAPL.US, MSFT.US | -| **UK** | London | LSE | 2,000+ | VOD.LSE, BP.LSE | -| **Germany** | Frankfurt | FRA | 1,000+ | BMW.FRA, SAP.FRA | -| **Japan** | Tokyo | TSE | 3,700+ | 7203.TSE (Toyota) | -| **France** | Euronext Paris | PA | 700+ | AI.PA (Air Liquide) | -| **Canada** | Toronto | TO | 3,000+ | SHOP.TO (Shopify) | -| **China** | Shanghai | SS | 1,500+ | 600519.SS (Moutai) | -| **Hong Kong** | HKEX | HK | 2,000+ | 0700.HK (Tencent) | -| **India** | NSE | NSE | 2,000+ | RELIANCE.NSE | -| **Australia** | ASX | AU | 2,000+ | BHP.AU | - -**Plus**: Brazil (SA), South Korea (KO), Singapore (SI), Switzerland (SW), and 50+ more! - -#### Symbol Format - -EODHD uses **SYMBOL.EXCHANGE** notation: - -```python -# US stocks -"AAPL.US", "MSFT.US", "TSLA.US" - -# London Stock Exchange -"VOD.LSE" # Vodafone -"BP.LSE" # British Petroleum -"HSBA.LSE" # HSBC - -# Frankfurt -"BMW.FRA" # BMW -"SAP.FRA" # SAP -"DTE.FRA" # Deutsche Telekom - -# Tokyo (use numerical codes) -"7203.TSE" # Toyota -"6758.TSE" # Sony -"9984.TSE" # SoftBank -``` - -#### Free Tier vs Paid Tier - -**Free Tier** (20 API calls/day): -- ✅ 1 year historical depth -- ✅ 150,000+ tickers across 60+ exchanges -- ✅ Daily, weekly, monthly OHLCV -- ✅ Adjusted close (splits/dividends) -- ⚠️ 20 calls/day limit - -**Paid tier**: -- ✅ Higher daily API-call quotas -- ✅ Extended historical coverage (up to 30 years) -- ✅ Real-time data -- ✅ Fundamentals and financials -- ✅ Dividends and splits calendar -- ✅ Bulk downloads - -**Cost Comparison**: -| Provider | Global Coverage | Free Tier | Paid Tier | -|----------|-----------------|-----------|-----------| -| EODHD | 60+ exchanges | 20 calls/day free | See current pricing | -| Finnhub | Plan-dependent | US quotes | See current pricing | -| Massive | Plan-dependent | Stocks Basic | See current pricing | - -#### Rate Limiting Strategy - -```python -# Free tier: 20 calls/day -# Strategy: Batch updates wisely - -# Example 1: Process at most 20 one-call symbol requests per day -symbols = ["AAPL.US", "VOD.LSE", "BMW.FRA", ...] - -for symbol in symbols[:20]: - df = provider.fetch_ohlcv(symbol, start="2024-01-01", end="2024-12-31") - -# Example 2: Use date-windowed fetches to reduce repeated work -for symbol in symbols: - df = provider.fetch_ohlcv(symbol, start="2024-01-01", end="2024-12-31") - storage.write(df, f"eodhd/{symbol}") - -# Daily runs: fetch only the missing range -for symbol in symbols: - df = provider.fetch_ohlcv(symbol, start=last_stored_date(symbol), end=today) - # Merge with existing storage using IncrementalUpdater or your storage workflow. -``` - -#### Multi-Exchange Portfolio Example - -```python -from ml4t.data.providers import EODHDProvider -import polars as pl - -provider = EODHDProvider(api_key="your_key") - -# Build a global tech portfolio -symbols = { - "AAPL.US": "Apple (US)", - "MSFT.US": "Microsoft (US)", - "VOD.LSE": "Vodafone (UK)", - "SAP.FRA": "SAP (Germany)", - "6758.TSE": "Sony (Japan)", - "0700.HK": "Tencent (Hong Kong)", -} - -all_data = [] -for symbol, name in symbols.items(): - df = provider.fetch_ohlcv( - symbol=symbol, - start="2024-01-01", - end="2024-12-31", - frequency="daily" - ) - df = df.with_columns(pl.lit(name).alias("company")) - all_data.append(df) - -# Combine all markets -global_portfolio = pl.concat(all_data) - -# Calculate returns in local currency -returns = global_portfolio.group_by("symbol").agg( - pl.col("close").pct_change().mean().alias("avg_daily_return") -) - -print(returns) -``` - -#### When to Use EODHD - -✅ **Good for**: -- Global equity strategies -- International portfolios -- Emerging markets research -- Multi-exchange analysis -- Cost-conscious production systems (€20/month unlimited!) - -❌ **Not ideal for**: -- Real-time intraday trading (free tier) -- Ultra-high-frequency strategies -- When 500 calls/day free tier is insufficient and budget is <€20/month - ---- - -### Finnhub (Professional Grade - Paid for Historical) - -#### Why Finnhub? - -- ✅ **70+ global exchanges** -- ✅ **Real-time quotes** on free tier (60/min) -- ✅ **WebSocket support** for live data -- ⚠️ **Historical OHLCV requires paid subscription** - -**Best for**: Professional trading systems with real-time requirements - -#### Quick Start - -```python -from ml4t.data.providers import FinnhubProvider - -# Get API key from: https://finnhub.io/register -provider = FinnhubProvider(api_key="your_finnhub_api_key") - -# Real-time quote (FREE TIER) -quote = provider.fetch_quote("AAPL") -print(quote) -# {'c': 185.64, 'h': 188.44, 'l': 183.89, 'o': 187.15, ...} - -# Historical OHLCV (PAID SUBSCRIPTION REQUIRED) -df = provider.fetch_ohlcv( - symbol="AAPL", - start="2024-01-01", - end="2024-12-31", - frequency="daily" -) -# Only works with paid plan ($59.99+/month) -``` - -#### Pricing - -**Free Tier**: -- ✅ Real-time quotes (60 calls/min) -- ✅ 50 symbols via WebSocket -- ❌ Historical OHLCV (**paid only**) - -**Paid Tier** ($59.99/month minimum): -- ✅ Historical daily/weekly/monthly OHLCV -- ✅ 70+ global exchanges -- ✅ Forex and crypto historical data -- ✅ Company fundamentals -- ✅ News and sentiment data - -#### When to Use Finnhub - -✅ **Good for**: -- Real-time quote monitoring (free tier) -- Professional trading systems (paid tier) -- News and sentiment analysis -- Multi-asset strategies - -❌ **Not ideal for**: -- Budget-conscious projects (EODHD is 3x cheaper) -- Historical backtesting on free tier (not available) -- Research projects (too expensive vs alternatives) - ---- - -## Multi-Asset Providers - -### Twelve Data (Stocks + Forex + Crypto) - -**Free Tier**: 800 API calls per day -**Coverage**: Stocks, forex, crypto, indices -**Best for**: Multi-asset portfolios - -```python -from ml4t.data.providers import TwelveDataProvider - -provider = TwelveDataProvider(api_key="your_key") - -# Stocks -stocks = provider.fetch_ohlcv("AAPL", start="2024-01-01", end="2024-12-31") - -# Forex -forex = provider.fetch_ohlcv("EUR/USD", start="2024-01-01", end="2024-12-31") - -# Crypto -crypto = provider.fetch_ohlcv("BTC/USD", start="2024-01-01", end="2024-12-31") -``` - -**When to use**: Need stocks + forex + crypto in single provider - ---- - -### Massive (Real-Time + Historical) - -**Free Tier**: 5 API calls per minute -**Coverage**: Stocks, options, futures, forex, crypto -**Best for**: Real-time data needs - -```python -from ml4t.data.providers import MassiveProvider - -provider = MassiveProvider(api_key="your_key") -df = provider.fetch_ohlcv("AAPL", start="2024-01-01", end="2024-12-31") -``` - -**Pricing**: Free (limited) or $199-$399/month - -**When to use**: Real-time stock/options/crypto trading - ---- - -## Best Practices - -### 1. Use Incremental Updates - -**Don't re-fetch entire history every day**: - -```python -# ❌ Bad: Wastes API quota -for symbol in symbols: - df = provider.fetch_ohlcv(symbol, start="2020-01-01", end="2024-12-31") - # 4+ years × 252 days = 1000+ records per symbol - # 100 symbols = 100 API calls for redundant data - -# ✅ Good: only fetch the missing range and merge with cached storage -df = provider.fetch_ohlcv("AAPL", start=last_stored_date, end=today) -``` - -**See**: [Tutorial 03: Incremental Updates](../tutorials/03_incremental_updates.md) - -### 2. Handle Market Hours & Holidays - -Stocks don't trade 24/7: - -```python -from datetime import datetime, timedelta - -# ❌ Wrong: Assuming data exists for weekends -df = provider.fetch_ohlcv("AAPL", start="2024-11-02", end="2024-11-03") -# Nov 2-3 is a weekend - no data! - -# ✅ Correct: Check for gaps -df = provider.fetch_ohlcv("AAPL", start="2024-11-01", end="2024-11-08") - -# Verify trading days only -gaps = df.select("timestamp").diff().filter( - pl.col("timestamp") > timedelta(days=3) # More than weekend -) - -if not gaps.is_empty(): - print(f"Found {len(gaps)} market holidays or missing data") -``` - -**US Market Holidays 2024**: -- New Year's Day, MLK Day, Presidents' Day, Good Friday -- Memorial Day, Juneteenth, Independence Day, Labor Day -- Thanksgiving, Christmas - -### 3. Adjusted vs Unadjusted Prices - -Always use **adjusted close** for backtesting: - -```python -# Most providers return adjusted close by default (ML4T Data standard) -df = provider.fetch_ohlcv("AAPL", start="2020-01-01", end="2024-12-31") - -# Check if close is adjusted -# Example: AAPL had 4:1 split on Aug 31, 2020 -# Pre-split: ~$500/share -# Post-split (adjusted): ~$125/share - -# If you accidentally use unadjusted: -# - Backtests will show phantom gains/losses at split dates -# - Returns calculations will be completely wrong -``` - -**Why adjusted close matters**: - -```python -# Example: AAPL 4:1 split on 2020-08-31 -# Unadjusted: Aug 30 close = $499, Aug 31 close = $129 -# → Appears like -74% loss! (Wrong) -# Adjusted: Aug 30 close = $124.75, Aug 31 close = $129 -# → Actual +3.4% gain (Correct) -``` - -### 4. Validate Data Quality - -```python -from ml4t.data.validation import validate_ohlcv - -df = provider.fetch_ohlcv("AAPL", start="2024-01-01", end="2024-12-31") - -# Validate OHLCV invariants -validate_ohlcv(df) # Raises exception if invalid - -# Check for: -# - high >= low ✓ -# - high >= open ✓ -# - high >= close ✓ -# - low <= open ✓ -# - low <= close ✓ -# - volume >= 0 ✓ -``` - -**See**: [Tutorial 04: Data Quality](../tutorials/04_data_quality.md) - -### 5. Multi-Provider Fallback - -```python -from ml4t.data.providers import EODHDProvider, TiingoProvider, YahooFinanceProvider - -def fetch_stock_data_robust(symbol: str, start: str, end: str): - """Fetch stock data with fallback providers.""" - - providers = [ - ("yahoo", lambda: YahooFinanceProvider()), - ("tiingo", lambda: TiingoProvider(api_key="your_tiingo_key")), - ("eodhd", lambda: EODHDProvider(api_key="your_eodhd_key", exchange="US")), - ] - - for name, create_provider in providers: - try: - provider = create_provider() - return provider.fetch_ohlcv(symbol=symbol, start=start, end=end) - except Exception as e: - print(f"{name} failed: {e}") - - raise RuntimeError("All providers failed") - -# Usage -df = fetch_stock_data_robust("AAPL", "2024-01-01", "2024-12-31") -``` - -**See**: [Tutorial 05: Multi-Provider Strategies](../tutorials/05_multi_provider.md) - ---- - -## Common Gotchas - -### 1. Symbol Format Varies by Provider - -```python -# ❌ Wrong provider/symbol combo -eodhd = EODHDProvider() -eodhd.fetch_ohlcv(symbol="AAPL", ...) # Error! Needs AAPL.US - -tiingo = TiingoProvider() -tiingo.fetch_ohlcv(symbol="AAPL.US", ...) # Error! Needs AAPL - -# ✅ Correct formats -eodhd.fetch_ohlcv(symbol="AAPL.US", ...) # SYMBOL.EXCHANGE -tiingo.fetch_ohlcv(symbol="AAPL", ...) # SYMBOL only -iex.fetch_ohlcv(symbol="AAPL", ...) # SYMBOL only -``` - -### 2. Free Tier Limitations - -```python -# Free tiers vary by provider and plan. -# For large universes, verify current provider quotas before scheduling jobs. -``` - -### 3. Market Hours Matter - -```python -# Fetching "today's" data before market close -from datetime import datetime - -now = datetime.now() # 10:00 AM ET (market just opened) - -# ❌ Won't have today's complete bar yet -df = provider.fetch_ohlcv("AAPL", start="2024-11-04", end="2024-11-04") -# Returns empty or yesterday's data - -# ✅ Fetch yesterday's complete data -yesterday = (now - timedelta(days=1)).strftime("%Y-%m-%d") -df = provider.fetch_ohlcv("AAPL", start=yesterday, end=yesterday) -``` - -### 4. Corporate Actions - -```python -# Stock splits can surprise you -df = provider.fetch_ohlcv("NVDA", start="2021-06-01", end="2021-08-01") - -# NVDA had 4:1 split on July 20, 2021 -# - Pre-split: ~$750/share -# - Post-split (adjusted): ~$187/share - -# Always use adjusted close to avoid phantom losses -``` - ---- - -## Provider Selection Decision Tree - -``` -START: What markets do you need? - -├─ US stocks only? -│ ├─ High volume (100+ symbols, daily updates)? -│ │ → Tiingo (1000/day free) ✅ -│ │ -│ ├─ Need fundamentals too? -│ │ → Yahoo, EODHD, Finnhub, or Massive fundamentals helpers -│ │ -│ └─ Low volume research (<10 symbols)? -│ → Yahoo Finance for experiments, Tiingo or EODHD for API-backed workflows -│ -├─ Global stocks (60+ exchanges)? -│ ├─ Budget-conscious? -│ │ → EODHD with the required market coverage and quota -│ │ -│ └─ Professional real-time needs? -│ → Finnhub ($59.99/month) or Massive ($199/month) -│ -└─ Multi-asset (stocks + forex + crypto)? - → Twelve Data (800/day free) ✅ -``` - ---- - -## Related Documentation - -**Getting Started**: -- [Quick Start Guide](../getting-started/quickstart.md) -- [Tutorial 01: Understanding OHLCV Data](../tutorials/01_understanding_ohlcv.md) - -**Provider Setup**: -- [Provider Selection Guide](../getting-started/provider-selection.md) -- [Creating a Custom Provider](../contributing/creating-a-provider.md) - -**Best Practices**: -- [Tutorial 02: Rate Limiting Best Practices](../tutorials/02_rate_limiting.md) -- [Tutorial 03: Incremental Updates](../tutorials/03_incremental_updates.md) -- [Tutorial 04: Data Quality Validation](../tutorials/04_data_quality.md) -- [Tutorial 05: Multi-Provider Strategies](../tutorials/05_multi_provider.md) - -**Other Asset Classes**: -- [Cryptocurrency Data Guide](./crypto.md) - Bitcoin, Ethereum, altcoins -- [Forex Data Guide](./forex.md) - Foreign exchange pairs -- [Futures Data Guide](./futures.md) - Futures and options - ---- - -## Next Steps - -1. **Get API keys** for your chosen provider(s) -2. **Try the Quick Start examples** above -3. **Implement incremental updates** (Tutorial 03) to minimize API calls -4. **Set up data quality validation** (Tutorial 04) in your pipeline -5. **Build multi-provider fallback** (Tutorial 05) for production resilience - -**Questions or issues?** See the [contribution guide](../contributing/index.md) or open a -GitHub issue. diff --git a/docs/asset-classes/forex.md b/docs/asset-classes/forex.md deleted file mode 100644 index 332f825a..00000000 --- a/docs/asset-classes/forex.md +++ /dev/null @@ -1,794 +0,0 @@ -# Foreign Exchange (Forex) Data Guide - -**Asset Class**: Currency Pairs (FX) -**Available Providers**: OANDA (primary), FXMacroData, Twelve Data, Massive -**Difficulty**: Moderate (requires understanding of forex mechanics) -**Recommended For**: FX trading strategies, currency risk analysis, global macro research - ---- - -## Overview - -ML4T Data provides access to **professional-grade forex data** through multiple providers, with OANDA being the gold standard for FX market data. The forex market is the world's largest financial market with unique characteristics: - -- **24/5 Markets**: Sunday 5 PM ET - Friday 5 PM ET (closed weekends) -- **High Liquidity**: $7.5 trillion daily volume -- **Global Trading**: Follows the sun (Tokyo → London → New York sessions) -- **Leverage & Spreads**: Tight bid-ask spreads, high leverage available -- **No Central Exchange**: OTC market, prices vary by broker/bank - -**Quick Provider Comparison**: - -| Provider | Free Tier | API Key | Pairs | Timeframes | Best For | -|----------|-----------|---------|-------|------------|----------| -| **OANDA** | Demo account (practice) | Required | 70+ | Second to monthly | Professional FX trading | -| **FXMacroData** | Public USD/free endpoints | Optional | Major FX macro context | Release/event time series | Macro-aware FX research | -| **Twelve Data** | 800/day | Required | 50+ major pairs | Minute to monthly | Multi-asset portfolios | -| **Massive** | 5/min | Required | 30+ pairs | Minute to daily | Real-time FX + stocks | - ---- - -## FXMacroData (FX Macro Context) - -FXMacroData complements price feeds with release-aware macro data for FX strategies: -known-at announcement timestamps, release calendars, FX spot history, COT positioning, -commodities, market sessions, and risk sentiment. - -```python -from ml4t.data.providers import FXMacroDataProvider - -provider = FXMacroDataProvider() - -inflation, metadata = provider.fetch_announcements( - "usd", - "inflation", - limit=100, - include_metadata=True, -) -calendar = provider.fetch_calendar("usd") -eurusd = provider.fetch_forex("eur", "usd", start_date="2024-01-01") -``` - -Use this when an FX case study needs point-in-time macro releases or positioning -context alongside OANDA, Twelve Data, or Massive price data. - ---- - -## OANDA (Recommended for Forex) - -### Why OANDA? - -- ✅ **Professional-grade FX data** - Used by institutional traders -- ✅ **120 requests/second** - Highest rate limit in class -- ✅ **70+ currency pairs** - Majors, minors, exotics -- ✅ **19 timeframes** - From 5 seconds to 1 month -- ✅ **Practice account** - Free demo environment for testing -- ✅ **Spread data included** - Bid/ask spreads for realistic simulation - -**Best for**: Forex trading strategies, high-frequency FX analysis, professional backtesting - -### Quick Start - -```python -from ml4t.data.providers import OandaProvider - -# Get API key from: https://www.oanda.com/us-en/trading/api/ -# Create a practice (demo) account first -provider = OandaProvider( - api_key="your_oanda_api_key", # Or set OANDA_API_KEY env var - account_type="practice" # Use 'live' for real trading account -) - -# Fetch EUR/USD hourly data -df = provider.fetch_ohlcv( - symbol="EUR_USD", # Note: Underscore separator - start="2024-01-01", - end="2024-12-31", - frequency="H1" # 1-hour bars -) - -print(df.head()) -``` - -**Output**: -``` -shape: (8760, 7) -┌─────────────────────┬──────────┬─────────┬─────────┬─────────┬─────────┬────────┐ -│ timestamp │ symbol │ open │ high │ low │ close │ volume │ -│ --- │ --- │ --- │ --- │ --- │ --- │ --- │ -│ datetime[μs] │ str │ f64 │ f64 │ f64 │ f64 │ f64 │ -╞═════════════════════╪══════════╪═════════╪═════════╪═════════╪═════════╪════════╡ -│ 2024-01-01 00:00:00 │ EUR_USD │ 1.1045 │ 1.1052 │ 1.1038 │ 1.1048 │ 125.0 │ -│ 2024-01-01 01:00:00 │ EUR_USD │ 1.1048 │ 1.1055 │ 1.1042 │ 1.1050 │ 198.0 │ -└─────────────────────┴──────────┴─────────┴─────────┴─────────┴─────────┴────────┘ -``` - -### Currency Pair Symbols - -OANDA uses **underscore notation** for currency pairs: - -```python -# Major Pairs (USD-based) -"EUR_USD" # Euro / US Dollar -"GBP_USD" # British Pound / US Dollar -"USD_JPY" # US Dollar / Japanese Yen -"USD_CHF" # US Dollar / Swiss Franc -"AUD_USD" # Australian Dollar / US Dollar -"USD_CAD" # US Dollar / Canadian Dollar -"NZD_USD" # New Zealand Dollar / US Dollar - -# Cross Pairs (No USD) -"EUR_GBP" # Euro / British Pound -"EUR_JPY" # Euro / Japanese Yen -"GBP_JPY" # British Pound / Japanese Yen -"AUD_JPY" # Australian Dollar / Japanese Yen - -# Exotic Pairs -"USD_TRY" # US Dollar / Turkish Lira -"USD_MXN" # US Dollar / Mexican Peso -"USD_ZAR" # US Dollar / South African Rand -"EUR_TRY" # Euro / Turkish Lira -``` - -**Available Pairs** (70+ total): -- **Majors**: EUR/USD, GBP/USD, USD/JPY, USD/CHF, AUD/USD, USD/CAD, NZD/USD -- **Minors**: EUR/GBP, EUR/JPY, GBP/JPY, AUD/JPY, EUR/CHF, GBP/CHF, etc. -- **Exotics**: USD/TRY, USD/MXN, USD/ZAR, EUR/TRY, USD/SGD, USD/THB, etc. - -### Timeframes (Granularities) - -OANDA offers **19 different timeframes** - more than any other provider: - -```python -# Second-level (ultra-high-frequency) -"S5" # 5 seconds -"S10" # 10 seconds -"S15" # 15 seconds -"S30" # 30 seconds - -# Minute-level (high-frequency) -"M1" # 1 minute -"M2" # 2 minutes -"M4" # 4 minutes -"M5" # 5 minutes -"M10" # 10 minutes -"M15" # 15 minutes -"M30" # 30 minutes - -# Hour-level (intraday) -"H1" # 1 hour -"H2" # 2 hours -"H3" # 3 hours -"H4" # 4 hours -"H6" # 6 hours -"H8" # 8 hours -"H12" # 12 hours - -# Day/Week/Month (swing/position) -"D" # 1 day -"W" # 1 week -"M" # 1 month -``` - -**Example: Multi-timeframe Analysis** - -```python -provider = OandaProvider(account_type="practice") - -# Daily for trend -daily = provider.fetch_ohlcv("EUR_USD", start="2024-01-01", end="2024-12-31", frequency="D") - -# 4-hour for entries -h4 = provider.fetch_ohlcv("EUR_USD", start="2024-11-01", end="2024-11-30", frequency="H4") - -# 15-minute for timing -m15 = provider.fetch_ohlcv("EUR_USD", start="2024-11-04", end="2024-11-05", frequency="M15") -``` - -### Rate Limits - -OANDA has the **most generous rate limits** among forex providers: - -**Rate Limit**: 120 requests per second (7,200/minute, 432,000/hour) - -```python -# Example: Fetch 50 pairs, 1 year hourly data -# 50 pairs × 1 API call each = 50 requests -# Completes in < 1 second (well within 120/sec limit) - -pairs = ["EUR_USD", "GBP_USD", "USD_JPY", ...] # 50 pairs - -import time -start = time.time() - -for pair in pairs: - df = provider.fetch_ohlcv( - symbol=pair, - start="2024-01-01", - end="2024-12-31", - frequency="H1" - ) - # No manual rate limiting needed! - -elapsed = time.time() - start -print(f"Fetched 50 pairs in {elapsed:.2f} seconds") -# Output: Fetched 50 pairs in 1.45 seconds -``` - -**Comparison**: -| Provider | Rate Limit | Notes | -|----------|------------|-------| -| OANDA | 120/second | ⭐ Best for high-frequency | -| Twelve Data | ~13/second (800/minute) | Good | -| Massive | 5/minute (free) | Restrictive | - -### Practice vs Live Accounts - -OANDA offers two account types: - -**Practice Account** (Demo): -- ✅ **Free** - No cost, no risk -- ✅ **Full API access** - Same data as live -- ✅ **Unlimited testing** - Perfect for backtesting -- ✅ **Real market prices** - Live data feed -- ⚠️ **Virtual money** - Can't execute real trades - -**Live Account** (Real Trading): -- ✅ **Real trading** - Execute actual FX trades -- ✅ **Same API** - Identical to practice -- ⚠️ **Real money at risk** - Trade carefully -- ⚠️ **Regulatory requirements** - KYC, minimum deposit - -**Setup**: -```python -# Practice account (for backtesting) -practice_provider = OandaProvider( - api_key="practice_api_key", - account_type="practice" # Demo environment -) - -# Live account (for paper trading or real execution) -live_provider = OandaProvider( - api_key="live_api_key", - account_type="live" # Production environment -) -``` - -### Spread Data (Bid/Ask) - -OANDA includes **bid/ask spread data** for realistic backtesting: - -```python -provider = OandaProvider( - api_key="your_key", - include_spread=True # Default: True -) - -df = provider.fetch_ohlcv("EUR_USD", start="2024-11-04", end="2024-11-04", frequency="M5") - -# DataFrame includes spread information in volume column -# (Note: Full spread data available in raw API response) -print(df.select(["timestamp", "close", "volume"])) -``` - -**Why spreads matter**: -```python -# Example: EUR/USD spread is typically 0.1 pips (0.00001) -# If you buy at 1.10500 (ask) and immediately sell at 1.10499 (bid) -# You lose 1 pip = $10 on a standard lot (100,000 EUR) - -# Without spread modeling: -# Backtest shows profit on quick scalps ❌ (unrealistic) - -# With spread modeling: -# Backtest deducts spread cost ✅ (realistic) -``` - -### Complete Example: Multi-Pair Strategy - -```python -from ml4t.data.providers import OandaProvider -import polars as pl -from datetime import datetime, timedelta - -# Initialize provider -provider = OandaProvider(account_type="practice") - -# Define currency basket -pairs = [ - "EUR_USD", # Euro - "GBP_USD", # Pound - "USD_JPY", # Yen - "USD_CHF", # Franc - "AUD_USD", # Aussie - "USD_CAD", # Loonie - "NZD_USD", # Kiwi -] - -# Fetch 6 months of hourly data for all pairs -end_date = datetime.now() -start_date = end_date - timedelta(days=180) - -all_data = [] -for pair in pairs: - df = provider.fetch_ohlcv( - symbol=pair, - start=start_date.strftime("%Y-%m-%d"), - end=end_date.strftime("%Y-%m-%d"), - frequency="H1" - ) - all_data.append(df) - -# Combine all pairs -combined = pl.concat(all_data) - -# Calculate hourly returns for each pair -returns = combined.group_by("symbol").agg([ - pl.col("close").pct_change().alias("hourly_return"), - pl.col("close").pct_change().std().alias("volatility") -]) - -print(returns.sort("volatility", descending=True)) -# Identify which pairs are most volatile (tradeable) -``` - -### Error Handling - -```python -from ml4t.data.core.exceptions import ( - AuthenticationError, - DataNotAvailableError, - RateLimitError -) - -try: - df = provider.fetch_ohlcv("INVALID_PAIR", start="2024-01-01", end="2024-12-31") -except DataNotAvailableError as e: - print(f"Pair not available: {e}") - # Check OANDA's supported instruments list - -try: - # Practice account trying to access live data - provider = OandaProvider( - api_key="invalid_key", - account_type="live" - ) -except AuthenticationError as e: - print(f"Authentication failed: {e}") - # Verify API key and account type match -``` - -### When to Use OANDA - -✅ **Good for**: -- Professional FX trading strategies -- High-frequency forex analysis (5-second to 1-minute bars) -- Multi-timeframe backtesting -- Large pair universes (50+ pairs) -- Realistic spread modeling -- Intraday FX strategies - -❌ **Not ideal for**: -- Stock/equity data (use Tiingo, EODHD, or Massive instead) -- Cryptocurrency data (use CoinGecko, CryptoCompare) -- Beginner exploratory projects (may prefer simpler providers) - ---- - -## Alternative Forex Providers - -### Twelve Data (Multi-Asset with Forex) - -**Best for**: Portfolios combining FX + stocks + crypto - -```python -from ml4t.data.providers import TwelveDataProvider - -# Get API key from: https://twelvedata.com/ -provider = TwelveDataProvider(api_key="your_key") - -# Fetch forex data (uses slash notation) -df = provider.fetch_ohlcv( - symbol="EUR/USD", # Note: Slash separator - start="2024-01-01", - end="2024-12-31", - frequency="daily" -) -``` - -**Features**: -- ✅ 50+ major forex pairs -- ✅ 800 API calls/day free tier -- ✅ Stocks, forex, crypto in one provider -- ✅ Daily to 1-minute intervals -- ⚠️ Less granular than OANDA (no 5-second bars) - -**When to use**: Need FX + equities + crypto data from single source - ---- - -### Massive (Real-Time FX + Stocks) - -**Best for**: Real-time FX monitoring + stock trading - -```python -from ml4t.data.providers import MassiveProvider - -provider = MassiveProvider(api_key="your_key") - -# Fetch FX data -df = provider.fetch_ohlcv( - symbol="C:EURUSD", # Note: 'C:' prefix for currencies - start="2024-01-01", - end="2024-12-31", - frequency="daily" -) -``` - -**Features**: -- ✅ 30+ major currency pairs -- ✅ Real-time + historical FX data -- ✅ Stocks, options, crypto also available -- ⚠️ Free tier: 5 API calls/minute (restrictive) -- ⚠️ Paid tier: $199-$399/month - -**When to use**: Need real-time FX + equities from single provider, budget >$199/month - ---- - -## Forex Market Characteristics - -### 24-Hour Market Sessions - -The forex market follows the sun across global financial centers: - -``` -Sydney Open: 5:00 PM ET Sunday → 2:00 AM ET Monday -Tokyo Open: 7:00 PM ET → 4:00 AM ET -London Open: 3:00 AM ET → 12:00 PM ET -New York Open: 8:00 AM ET → 5:00 PM ET -Market Close: 5:00 PM ET Friday -``` - -**Trading activity by session**: - -```python -from datetime import datetime - -df = provider.fetch_ohlcv("EUR_USD", start="2024-11-04", end="2024-11-08", frequency="H1") - -# Add session labels -def get_session(hour: int) -> str: - """Determine trading session based on UTC hour.""" - if 22 <= hour or hour < 5: - return "Sydney/Tokyo" - elif 5 <= hour < 12: - return "Tokyo/London" - elif 12 <= hour < 17: - return "London/NY" - elif 17 <= hour < 22: - return "NY" - -df = df.with_columns( - pl.col("timestamp").dt.hour().map_elements(get_session).alias("session") -) - -# Calculate average volatility by session -session_vol = df.group_by("session").agg( - ((pl.col("high") - pl.col("low")) / pl.col("open") * 100).mean().alias("avg_range_pct") -) - -print(session_vol.sort("avg_range_pct", descending=True)) -# London/NY overlap typically shows highest volatility -``` - -### Weekends & Holidays - -Forex markets are **closed weekends** but have unique holiday behavior: - -```python -# Forex markets close: Friday 5:00 PM ET -# Forex markets open: Sunday 5:00 PM ET - -# Check for weekend gaps -df = provider.fetch_ohlcv("EUR_USD", start="2024-11-01", end="2024-11-30", frequency="D") - -# Identify weekend gaps (no data Saturday/Sunday) -gaps = df.select("timestamp").diff().filter( - pl.col("timestamp") > timedelta(days=2) # More than 1 trading day -) - -print(f"Found {len(gaps)} weekend gaps") -# Expected: ~8-9 weekends in November -``` - -**Important holidays**: -- Christmas/New Year: Reduced liquidity -- Easter (Good Friday): Market closed -- US Thanksgiving: Early close (Friday 1 PM ET) - -Unlike stock markets, most national holidays don't close FX markets (since it's global/OTC). - -### Currency Pair Conventions - -**Base vs Quote Currency**: - -```python -# EUR/USD = 1.1050 -# Base: EUR (1 Euro) -# Quote: USD (1.1050 Dollars) -# Meaning: 1 Euro = 1.1050 US Dollars - -# GBP/JPY = 185.50 -# Base: GBP (1 British Pound) -# Quote: JPY (185.50 Japanese Yen) -# Meaning: 1 Pound = 185.50 Yen -``` - -**Direct vs Indirect Quotes**: - -```python -# US Dollar as QUOTE currency (direct from US perspective) -"EUR/USD", "GBP/USD", "AUD/USD", "NZD/USD" # Majors - -# US Dollar as BASE currency (indirect from US perspective) -"USD/JPY", "USD/CHF", "USD/CAD" # Also majors, but flipped - -# Cross pairs (no USD) -"EUR/GBP", "EUR/JPY", "GBP/JPY" # Calculated from majors -``` - -### Pips and Pipettes - -Forex price movements measured in **pips** (percentage in points): - -```python -# EUR/USD = 1.10500 -# 1 pip = 0.0001 (4th decimal place) -# 1 pipette = 0.00001 (5th decimal place) - -# GBP/USD moves from 1.26500 to 1.26550 -# Change: 50 pips = 5 pipettes - -# USD/JPY = 149.50 (only 2 decimal places for JPY pairs) -# 1 pip = 0.01 (2nd decimal place) -``` - -**Calculating pip value**: - -```python -def calculate_pip_value(pair: str, lot_size: int = 100000) -> float: - """Calculate pip value in USD for standard lot.""" - if "JPY" in pair: - return (0.01 / 149.50) * lot_size if pair.startswith("USD") else 0.01 * lot_size / 149.50 - else: - return 0.0001 * lot_size - -# EUR/USD standard lot (100,000 EUR) -pip_value = calculate_pip_value("EUR/USD", lot_size=100000) -print(f"1 pip = ${pip_value:.2f}") -# Output: 1 pip = $10.00 - -# If EUR/USD moves 50 pips, profit/loss = $500 on standard lot -``` - ---- - -## Best Practices - -### 1. Account for Spreads - -Don't backtest on close prices alone: - -```python -# ❌ Bad: Ignores transaction costs -df = provider.fetch_ohlcv("EUR_USD", frequency="M5") -entry_price = df["close"][0] # Assumes zero spread - -# ✅ Good: Model realistic spreads -typical_spread_pips = 0.1 # EUR/USD typically 0.1-0.2 pips -spread_cost = typical_spread_pips * 0.0001 # Convert pips to price - -# Buy at ask (close + spread/2) -entry_price = df["close"][0] + spread_cost / 2 - -# Sell at bid (close - spread/2) -exit_price = df["close"][-1] - spread_cost / 2 - -# Account for spread in backtest PnL -``` - -### 2. Understand Rollover (Swap) - -Positions held overnight incur **rollover interest**: - -```python -# Forex trades settle in 2 business days (T+2) -# If you hold EUR/USD long overnight: -# - You pay USD interest rate -# - You receive EUR interest rate -# - Net: Rollover = EUR rate - USD rate - -# Example (simplified): -# EUR interest rate: 4.0% annually -# USD interest rate: 5.25% annually -# Daily rollover: (4.0% - 5.25%) / 365 = -0.00342% per day - -# On 100,000 EUR position: -# Daily cost = 100,000 * 1.1050 * -0.00342% = -$3.78 per day -``` - -**Include rollover in backtests** for multi-day positions. - -### 3. Use Incremental Updates - -Forex data accumulates quickly at intraday frequencies: - -```python -# First run: Fetch 1 year of hourly data -history = provider.fetch_ohlcv("EUR_USD", start="2024-01-01", end="2024-12-31", frequency="H1") -storage.write(history, "oanda/EUR_USD") -# 365 days × 24 hours = 8,760 bars - -# Daily runs: Only fetch last 24 hours -new_data = provider.fetch_ohlcv("EUR_USD", start=last_stored_date, end=today, frequency="H1") -# Only 24 bars fetched, merged with existing data -# 365x fewer API calls on subsequent runs! -``` - -**See**: [Tutorial 03: Incremental Updates](../tutorials/03_incremental_updates.md) - -### 4. Validate FX Data Quality - -```python -from ml4t.data.validation import validate_ohlcv - -df = provider.fetch_ohlcv("EUR_USD", start="2024-01-01", end="2024-12-31") - -# Validate standard OHLCV invariants -validate_ohlcv(df) - -# FX-specific checks -assert df.filter(pl.col("volume") == 0).is_empty(), "No zero-volume bars" -assert df.filter(pl.col("high") == pl.col("low")).is_empty(), "No single-price bars" - -# Check for weekend gaps (expected) -gaps = df.select("timestamp").diff().filter(pl.col("timestamp") > timedelta(days=2)) -print(f"Weekend gaps: {len(gaps)}") # Should be ~52 for 1 year of daily data -``` - -**See**: [Tutorial 04: Data Quality](../tutorials/04_data_quality.md) - -### 5. Multi-Pair Correlation Analysis - -Forex pairs are highly correlated - use this in strategies: - -```python -import polars as pl - -# Fetch major pairs -pairs = ["EUR_USD", "GBP_USD", "USD_JPY", "USD_CHF", "AUD_USD"] -all_data = [] - -for pair in pairs: - df = provider.fetch_ohlcv(pair, start="2024-01-01", end="2024-12-31", frequency="D") - df = df.select(["timestamp", "close"]).rename({"close": pair}) - all_data.append(df) - -# Join on timestamp -combined = all_data[0] -for df in all_data[1:]: - combined = combined.join(df, on="timestamp", how="outer") - -# Calculate correlation matrix -corr_matrix = combined.select(pairs).corr() -print(corr_matrix) - -# Expected correlations: -# EUR/USD ↔ GBP/USD: Positive (both vs USD) -# EUR/USD ↔ USD/JPY: Negative (USD is opposite side) -# EUR/USD ↔ USD/CHF: Negative (CHF is safe-haven like JPY) -``` - ---- - -## Common Gotchas - -### 1. Symbol Format Differences - -```python -# ❌ Wrong: Using incorrect format for provider -oanda = OandaProvider() -oanda.fetch_ohlcv(symbol="EUR/USD", ...) # Error! Needs EUR_USD - -twelve_data = TwelveDataProvider() -twelve_data.fetch_ohlcv(symbol="EUR_USD", ...) # Error! Needs EUR/USD - -# ✅ Correct formats -oanda.fetch_ohlcv(symbol="EUR_USD", ...) # Underscore -twelve_data.fetch_ohlcv(symbol="EUR/USD", ...) # Slash -polygon.fetch_ohlcv(symbol="C:EURUSD", ...) # Prefix + no separator -``` - -### 2. Timezone Confusion - -```python -# OANDA returns timestamps in UTC -df = provider.fetch_ohlcv("EUR_USD", frequency="H1") - -# If you're in New York (ET = UTC-5 or UTC-4): -# 12:00 UTC = 7:00 AM ET (or 8:00 AM ET during DST) - -# Always work in UTC for forex data, convert for display only -import pytz - -df = df.with_columns( - pl.col("timestamp").dt.convert_time_zone("America/New_York").alias("timestamp_et") -) -``` - -### 3. Intraday Data Volume - -High-frequency FX data is **massive**: - -```python -# 1 minute bars for EUR/USD, 1 year -# 365 days × 24 hours × 60 minutes = 525,600 bars - -# At 7 columns × 8 bytes each = 56 bytes per row -# 525,600 rows × 56 bytes = ~29 MB for 1 pair, 1 year - -# For 50 pairs: 1.45 GB! -# Use Parquet compression and incremental updates -``` - -### 4. Practice vs Live Data Discrepancy - -```python -# Practice and live accounts may show slight price differences -practice = OandaProvider(account_type="practice") -live = OandaProvider(account_type="live") - -practice_df = practice.fetch_ohlcv("EUR_USD", start="2024-11-04", end="2024-11-04") -live_df = live.fetch_ohlcv("EUR_USD", start="2024-11-04", end="2024-11-04") - -# Differences are typically < 0.1 pip (negligible for backtesting) -# But ALWAYS backtest on same environment you'll trade in -``` - ---- - -## Related Documentation - -**Getting Started**: -- [Quick Start Guide](../getting-started/quickstart.md) -- [Tutorial 01: Understanding OHLCV Data](../tutorials/01_understanding_ohlcv.md) - -**Provider Setup**: -- [Provider Selection Guide](../getting-started/provider-selection.md) -- [Creating a Custom Provider](../contributing/creating-a-provider.md) - -**Best Practices**: -- [Tutorial 02: Rate Limiting Best Practices](../tutorials/02_rate_limiting.md) -- [Tutorial 03: Incremental Updates](../tutorials/03_incremental_updates.md) -- [Tutorial 04: Data Quality Validation](../tutorials/04_data_quality.md) -- [Tutorial 05: Multi-Provider Strategies](../tutorials/05_multi_provider.md) - -**Other Asset Classes**: -- [Cryptocurrency Data Guide](./crypto.md) - Bitcoin, Ethereum, altcoins -- [Equities Data Guide](./equities.md) - US and global stocks -- [Futures Data Guide](./futures.md) - Futures and options - ---- - -## Next Steps - -1. **Sign up for OANDA practice account** (free): https://www.oanda.com/us-en/trading/api/ -2. **Try the Quick Start examples** above with demo account -3. **Understand forex mechanics** - Pips, spreads, rollover, sessions -4. **Implement incremental updates** (Tutorial 03) for production systems -5. **Set up data quality validation** (Tutorial 04) for your FX pipeline -6. **Model spreads realistically** in backtests for accurate results - -**Questions or issues?** See the [contribution guide](../contributing/index.md) or open a -GitHub issue. diff --git a/docs/asset-classes/futures.md b/docs/asset-classes/futures.md deleted file mode 100644 index bc82444f..00000000 --- a/docs/asset-classes/futures.md +++ /dev/null @@ -1,723 +0,0 @@ -# Futures Data Guide - -**Asset Classes**: Futures contracts -**Available Providers**: Databento (primary), Massive for multi-asset alternatives -**Difficulty**: Advanced (requires understanding of derivatives mechanics) -**Recommended For**: Institutional strategies, derivatives research, quantitative trading - ---- - -## Overview - -ML4T Data provides access to institutional-grade futures data through Databento. -Futures markets have unique complexities: - -- **Expiration & Rolling**: Contracts expire monthly/quarterly, requiring continuous contract construction -- **Multiple Exchanges**: CME, ICE, Eurex, and others each have different conventions -- **Session Times**: Futures trade nearly 24 hours with complex session breaks -- **Tick Data**: Ultra-high-frequency data (microsecond timestamps) -- **Margin Requirements**: Leveraged instruments with daily mark-to-market - -**Quick Provider Comparison**: - -| Provider | Coverage | Free Tier | Data Quality | Best For | -|----------|----------|-----------|--------------|----------| -| **Databento** | CME/ICE/Eurex futures; OPRA options | Free metadata; metered history | Institutional | Professional derivatives research | -| **Polygon** | Limited futures | 5/min | ⭐⭐⭐ Good | Multi-asset portfolios | - -**Important**: Futures data is not free. Databento requires a paid subscription -for most historical workflows. OPRA requests can be large, so estimate cost -before downloading broad chains, bars, or quotes. - ---- - -## Databento (Recommended for Futures) - -### Why Databento? - -- ✅ **Institutional-grade quality** - Used by quant funds and prop traders -- ✅ **Normalized data** - Consistent schemas across all exchanges -- ✅ **Continuous contracts** - Automatic front-month rolling (.v.0 notation) -- ✅ **Tick-level precision** - Microsecond timestamps -- ✅ **CME, ICE, Eurex** - Major futures exchanges worldwide -- ✅ **OPRA options** - Chain discovery, option OHLCV, quotes, and cost estimates -- ✅ **Selected schemas** - OHLCV plus direct access to native Databento schemas - -**Best for**: Quantitative futures strategies, high-frequency trading, institutional research - -### Quick Start - -```python -from ml4t.data.providers import DataBentoProvider - -# Get API key from: https://databento.com/ -# NOTE: Requires paid subscription ($30-50/month minimum) -provider = DataBentoProvider( - api_key="your_databento_api_key", # Or set DATABENTO_API_KEY env var - dataset="GLBX.MDP3" # CME Globex futures (default) -) - -# Fetch ES (E-mini S&P 500) continuous front-month contract -df = provider.fetch_ohlcv( - symbol="ES.v.0", # .v.0 = continuous front month - start="2024-01-01", - end="2024-12-31", - frequency="daily" -) - -print(df.head()) -``` - -**Output**: -``` -shape: (252, 7) -┌─────────────────────┬──────────┬─────────┬─────────┬─────────┬─────────┬──────────┐ -│ timestamp │ symbol │ open │ high │ low │ close │ volume │ -│ --- │ --- │ --- │ --- │ --- │ --- │ --- │ -│ datetime[μs] │ str │ f64 │ f64 │ f64 │ f64 │ f64 │ -╞═════════════════════╪══════════╪═════════╪═════════╪═════════╪═════════╪══════════╡ -│ 2024-01-02 00:00:00 │ ES.v.0 │ 4742.50 │ 4793.75 │ 4742.50 │ 4783.75 │ 2.15e6 │ -│ 2024-01-03 00:00:00 │ ES.v.0 │ 4783.25 │ 4800.50 │ 4745.00 │ 4746.75 │ 2.43e6 │ -└─────────────────────┴──────────┴─────────┴─────────┴─────────┴─────────┴──────────┘ -``` - -### Continuous Contracts (.v.0 Notation) - -Futures contracts **expire** monthly or quarterly. To create continuous price series: - -```python -# Continuous front-month contract (automatically rolls) -"ES.v.0" # E-mini S&P 500 front month -"NQ.v.0" # E-mini NASDAQ-100 front month -"CL.v.0" # WTI Crude Oil front month -"GC.v.0" # Gold front month - -# Continuous 2nd month contract -"ES.v.1" # E-mini S&P 500 2nd month -"CL.v.1" # WTI Crude Oil 2nd month - -# Continuous 3rd month contract -"ES.v.2" # E-mini S&P 500 3rd month - -# Specific contract month (NO automatic rolling) -"ESZ4" # E-mini S&P 500 December 2024 -"ESH5" # E-mini S&P 500 March 2025 -``` - -**Why continuous contracts matter**: - -```python -# ❌ Bad: Using individual contract months -df_dec = provider.fetch_ohlcv("ESZ4", start="2024-10-01", end="2024-12-15") -df_mar = provider.fetch_ohlcv("ESH5", start="2024-12-16", end="2025-03-15") -# Problem: Price gap at rollover date, backtests break - -# ✅ Good: Using continuous contract -df = provider.fetch_ohlcv("ES.v.0", start="2024-10-01", end="2025-03-15") -# Databento automatically handles rollover, seamless price series -``` - -### Supported Futures - -**CME Globex** (dataset: `GLBX.MDP3`): - -| Symbol | Contract | Tick Size | Contract Size | -|--------|----------|-----------|---------------| -| **ES** | E-mini S&P 500 | 0.25 pts = $12.50 | $50 × S&P 500 | -| **NQ** | E-mini NASDAQ-100 | 0.25 pts = $5.00 | $20 × NASDAQ-100 | -| **YM** | E-mini Dow Jones | 1.00 pt = $5.00 | $5 × DJIA | -| **RTY** | E-mini Russell 2000 | 0.10 pts = $5.00 | $50 × Russell 2000 | -| **CL** | WTI Crude Oil | $0.01 = $10.00 | 1,000 barrels | -| **GC** | Gold | $0.10 = $10.00 | 100 troy oz | -| **SI** | Silver | $0.005 = $25.00 | 5,000 troy oz | -| **ZB** | 30-Year T-Bond | 1/32 = $31.25 | $100,000 face value | -| **ZN** | 10-Year T-Note | 1/64 = $15.625 | $100,000 face value | -| **ZF** | 5-Year T-Note | 1/128 = $7.8125 | $100,000 face value | -| **6E** | Euro FX | $0.00005 = $6.25 | €125,000 | -| **6J** | Japanese Yen | $0.0000005 = $6.25 | ¥12,500,000 | -| **6B** | British Pound | $0.0001 = $6.25 | £62,500 | - -**Equity Index Futures**: -```python -# Major indices -"ES.v.0" # S&P 500 -"NQ.v.0" # NASDAQ-100 -"YM.v.0" # Dow Jones -"RTY.v.0" # Russell 2000 - -# International -"MES.v.0" # Micro E-mini S&P 500 (1/10th size) -"MNQ.v.0" # Micro E-mini NASDAQ-100 -``` - -**Commodity Futures**: -```python -# Energy -"CL.v.0" # WTI Crude Oil -"NG.v.0" # Natural Gas -"HO.v.0" # Heating Oil -"RB.v.0" # RBOB Gasoline - -# Metals -"GC.v.0" # Gold -"SI.v.0" # Silver -"HG.v.0" # Copper -"PL.v.0" # Platinum - -# Agriculture -"ZC.v.0" # Corn -"ZS.v.0" # Soybeans -"ZW.v.0" # Wheat -"KC.v.0" # Coffee -"CT.v.0" # Cotton -``` - -**Interest Rate Futures**: -```python -"ZB.v.0" # 30-Year T-Bond -"ZN.v.0" # 10-Year T-Note -"ZF.v.0" # 5-Year T-Note -"ZT.v.0" # 2-Year T-Note -``` - -**Currency Futures**: -```python -"6E.v.0" # Euro FX -"6J.v.0" # Japanese Yen -"6B.v.0" # British Pound -"6C.v.0" # Canadian Dollar -"6A.v.0" # Australian Dollar -``` - -### Datasets (Exchanges) - -Databento provides data from multiple exchanges: - -```python -# CME Globex (most futures) -provider = DataBentoProvider(dataset="GLBX.MDP3") - -# CME (all CME markets) -provider = DataBentoProvider(dataset="CME.MDP3") - -# OPRA options -provider = DataBentoProvider(dataset="OPRA.PILLAR") -quotes = provider.fetch_option_quotes( - "SPY 240119C00480000", - start="2024-01-02", - end="2024-01-02", -) - -# US Equities -provider = DataBentoProvider(dataset="XNAS.ITCH") # NASDAQ -provider = DataBentoProvider(dataset="XNYS.TRADES") # NYSE -``` - -**Available datasets**: -- `GLBX.MDP3` - CME Globex (futures, options on futures) -- `CME.MDP3` - CME (all markets) -- `XCME.MDP3` - CME Crypto -- `OPRA.PILLAR` - OPRA listed options -- `XNAS.ITCH` - NASDAQ Equities -- `XNYS.TRADES` - NYSE Equities -- `BATS.PITCH` - BATS Equities - -### Schemas (Data Types) - -Databento supports multiple data schemas beyond OHLCV. The wrapper exposes -selected schema fetches and the native client for advanced workflows: - -```python -# OHLCV aggregations -"ohlcv-1m" # 1-minute bars -"ohlcv-1h" # 1-hour bars -"ohlcv-1d" # 1-day bars - -# Tick data -"trades" # Individual trades (time & sales) -"quotes" # Top of book (best bid/ask) -"mbo" # Market by order (full order book) -"mbp-1" # Market by price (1 level) -"mbp-10" # Market by price (10 levels) -``` - -**Example: Fetching tick data**: - -```python -provider = DataBentoProvider(dataset="GLBX.MDP3") - -# Fetch trade-by-trade data for ES -trades = provider.fetch_ohlcv( - symbol="ES.v.0", - start="2024-11-04", - end="2024-11-04", - frequency="tick" # requests Databento trades schema -) - -# Result: Every single trade with microsecond timestamps -print(f"Total trades: {len(trades)}") -# Output: Total trades: 1,247,352 (for 1 day of ES) -``` - -### Pricing - -**Databento pricing** (as of 2024): - -**Historical Data API**: -- **Starter**: ~$30/month - Limited historical data -- **Professional**: ~$50-100/month - Extended history + more symbols -- **Enterprise**: Custom pricing - Unlimited access - -**Live Data**: -- **Real-time feed**: $100-500/month depending on exchanges -- **WebSocket API**: Included with subscription - -**Free Tier**: ❌ None - All Databento data is paid - -**Cost Comparison**: -| Provider | Futures Data | Cost | Quality | -|----------|--------------|------|---------| -| Databento | CME, CBOE, ICE | $30-100/month | ⭐⭐⭐⭐⭐ Best | -| Polygon | Limited | $199/month | ⭐⭐⭐ Good | -| IQFeed | Extensive | $100+/month | ⭐⭐⭐⭐ Very Good | -| CQG | Professional | $500+/month | ⭐⭐⭐⭐⭐ Best | - -**Winner**: Databento (best value for institutional quality) - -### Session Times & Date Adjustments - -Futures markets trade nearly 24 hours with session breaks: - -```python -# CME Globex session times (US/Central) -# Sunday 5:00 PM - Friday 4:00 PM CT -# Daily break: 4:00 PM - 5:00 PM CT - -# ES (E-mini S&P 500) specific: -# Sunday 5:00 PM CT - Monday 4:00 PM CT = "Monday" session -# But in UTC: Monday 00:00 - Tuesday 22:00 - -# Databento handles this with adjust_session_dates -provider = DataBentoProvider( - dataset="GLBX.MDP3", - adjust_session_dates=True, # Align dates with session start - session_start_hour_utc=22 # 22:00 UTC = 5:00 PM CT (CST) -) - -# Now daily bars align with CME session dates (not calendar dates) -``` - -### Rate Limits - -**Databento Historical API**: 100 requests per second - -```python -# Very generous for historical data fetching -symbols = [f"ES.v.0", "NQ.v.0", "YM.v.0", ...] # 50 futures contracts - -for symbol in symbols: - df = provider.fetch_ohlcv(symbol, start="2024-01-01", end="2024-12-31") - # 50 symbols = 50 requests = completes in 0.5 seconds -``` - -**Live WebSocket**: Use Databento's native SDK directly. - -### Complete Example: Multi-Contract Strategy - -```python -from ml4t.data.providers import DataBentoProvider -import polars as pl - -# Initialize provider -provider = DataBentoProvider( - dataset="GLBX.MDP3", - adjust_session_dates=True -) - -# Build a diversified futures portfolio -contracts = { - "ES.v.0": "E-mini S&P 500", - "NQ.v.0": "E-mini NASDAQ-100", - "CL.v.0": "WTI Crude Oil", - "GC.v.0": "Gold", - "ZN.v.0": "10-Year T-Note", -} - -# Fetch 1 year of daily data -all_data = [] -for symbol, name in contracts.items(): - df = provider.fetch_ohlcv( - symbol=symbol, - start="2023-01-01", - end="2023-12-31", - frequency="daily" - ) - df = df.with_columns(pl.lit(name).alias("contract")) - all_data.append(df) - -# Combine all contracts -portfolio = pl.concat(all_data) - -# Calculate daily returns -returns = portfolio.group_by("symbol").agg([ - pl.col("close").pct_change().mean().alias("avg_return"), - pl.col("close").pct_change().std().alias("volatility"), - (pl.col("close").pct_change().mean() / pl.col("close").pct_change().std()).alias("sharpe") -]) - -print(returns.sort("sharpe", descending=True)) -``` - -### Error Handling - -```python -from ml4t.data.core.exceptions import ( - AuthenticationError, - DataNotAvailableError, - NetworkError -) - -try: - df = provider.fetch_ohlcv("INVALID.v.0", start="2024-01-01", end="2024-12-31") -except DataNotAvailableError as e: - print(f"Contract not available: {e}") - # Check Databento's symbol list for valid contracts - -try: - # Using expired API key - provider = DataBentoProvider(api_key="expired_key") -except AuthenticationError as e: - print(f"Authentication failed: {e}") - # Renew subscription or check API key -``` - -### When to Use Databento - -✅ **Good for**: -- Professional futures trading strategies -- High-frequency derivatives research -- Institutional-grade backtesting -- Tick-level analysis -- Multi-exchange strategies -- Options on futures - -❌ **Not ideal for**: -- Budget-constrained projects (requires $30+/month) -- Beginners (steep learning curve) -- Stock-only strategies (use Tiingo instead) -- Cryptocurrency-only (use CoinGecko instead) - ---- - -## Polygon (Multi-Asset Alternative) - -### Why Polygon? - -- ✅ **Futures + Stocks + Crypto + Options** - Multi-asset coverage -- ✅ **Real-time + historical** data -- ⚠️ **Free tier very limited** (5 API calls/minute) -- ⚠️ **Paid tier expensive** ($199-$399/month) - -**Best for**: Multi-asset portfolios needing futures + equities - -### Quick Start - -```python -from ml4t.data.providers import MassiveProvider - -# Get API key from: https://massive.com/ -provider = MassiveProvider(api_key="your_massive_api_key") - -# Fetch ES futures data -df = provider.fetch_ohlcv( - symbol="F:ESM6", - start="2024-01-01", - end="2024-12-31", - frequency="daily" -) -``` - -**Pricing**: -- **Free**: 5 API calls per minute (very restrictive) -- **Starter**: $199/month - Stocks + Forex + Crypto -- **Developer**: $399/month - Adds Options + Futures -- **Advanced**: Custom pricing - -**When to use**: Need futures + stocks + options from single provider, budget >$199/month - ---- - -## Futures Market Characteristics - -### Contract Expiration & Rolling - -Futures contracts **expire** on specific dates: - -```python -# ES (E-mini S&P 500) expiration schedule: -# March (H), June (M), September (U), December (Z) -# 3rd Friday of expiration month - -# Example expiration dates for 2024: -"ESH4" # March 2024 - expires March 15, 2024 -"ESM4" # June 2024 - expires June 21, 2024 -"ESU4" # September 2024 - expires September 20, 2024 -"ESZ4" # December 2024 - expires December 20, 2024 - -# After expiration, contract becomes worthless -# Must "roll" to next contract to maintain position -``` - -**Roll dates** (when to switch contracts): - -```python -# Typically roll 5-10 days before expiration -# ESZ4 (Dec 2024) expires Dec 20 -# Roll to ESH5 (Mar 2025) around Dec 10-15 - -# Using continuous contracts (.v.0) automates this: -df = provider.fetch_ohlcv("ES.v.0", start="2024-01-01", end="2025-12-31") -# Databento automatically switches ESH4 → ESM4 → ESU4 → ESZ4 → ESH5 -``` - -### Contract Months & Codes - -**Month codes** (universal): - -```python -# Jan=F, Feb=G, Mar=H, Apr=J, May=K, Jun=M, -# Jul=N, Aug=Q, Sep=U, Oct=V, Nov=X, Dec=Z - -# Year: Last digit (2024 = 4, 2025 = 5) - -# Examples: -"ESH5" # ES March 2025 -"CLZ4" # CL December 2024 -"GCM5" # GC June 2025 -``` - -**Not all contracts trade all months**: - -```python -# ES (S&P 500): March, June, Sep, Dec (quarterly) -"ESH", "ESM", "ESU", "ESZ" - -# CL (Crude Oil): All 12 months -"CLF", "CLG", "CLH", "CLJ", "CLK", "CLM", "CLN", "CLQ", "CLU", "CLV", "CLX", "CLZ" - -# GC (Gold): Feb, Apr, Jun, Aug, Oct, Dec (bi-monthly) -"GCG", "GCJ", "GCM", "GCQ", "GCV", "GCZ" -``` - -### Tick Size & Value - -Each futures contract has specific **tick size** and **value per tick**: - -```python -# ES (E-mini S&P 500) -# Tick size: 0.25 points -# Tick value: $12.50 (0.25 × $50 multiplier) -# If ES moves from 4750.00 to 4750.25 → $12.50 profit/loss per contract - -# CL (Crude Oil) -# Tick size: $0.01 per barrel -# Tick value: $10.00 ($0.01 × 1000 barrels) -# If CL moves from $75.00 to $75.01 → $10.00 profit/loss per contract - -# GC (Gold) -# Tick size: $0.10 per troy ounce -# Tick value: $10.00 ($0.10 × 100 oz) -# If GC moves from $2000.00 to $2000.10 → $10.00 profit/loss per contract -``` - -### Margin Requirements - -Futures are **leveraged** - you don't pay full contract value upfront: - -```python -# Example: ES (E-mini S&P 500) @ 4750 -# Notional value: 4750 × $50 = $237,500 -# Initial margin: ~$12,000 (5% of notional) -# Maintenance margin: ~$11,000 - -# Leverage: $237,500 / $12,000 = ~20:1 - -# If ES moves 1% (47.5 points): -# Change in contract value: 47.5 × $50 = $2,375 -# Return on margin: $2,375 / $12,000 = 19.8% (on 1% price move!) -``` - -**Risk warning**: High leverage magnifies both gains and losses. - ---- - -## Best Practices - -### 1. Use Continuous Contracts for Backtesting - -```python -# ❌ Bad: Using individual contracts -df1 = provider.fetch_ohlcv("ESZ4", start="2024-10-01", end="2024-12-20") -df2 = provider.fetch_ohlcv("ESH5", start="2024-12-21", end="2025-03-20") -# Problem: Price gap at rollover, discontinuous series - -# ✅ Good: Using continuous contract -df = provider.fetch_ohlcv("ES.v.0", start="2024-10-01", end="2025-03-20") -# Databento handles rollover seamlessly -``` - -### 2. Account for Roll Costs - -```python -# When rolling futures, there's a "roll cost" (calendar spread) -# Example: ESZ4 @ 4750, ESH5 @ 4755 -# Roll cost: 5 points = $250 per contract - -# Include roll costs in backtests: -roll_cost_points = 5 -roll_cost_dollars = roll_cost_points * 50 # ES multiplier -annual_rolls = 4 # Quarterly contracts -annual_roll_cost = roll_cost_dollars * annual_rolls * num_contracts -``` - -### 3. Understand Session Times - -```python -# Futures trade nearly 24/5, but have session breaks -# ES (CME Globex): -# Sunday 5:00 PM CT - Friday 4:00 PM CT -# Daily break: 4:00 PM - 5:00 PM CT - -# Use adjust_session_dates for proper date alignment -provider = DataBentoProvider( - adjust_session_dates=True, - session_start_hour_utc=22 # 5:00 PM CT = 22:00 UTC (CST) -) -``` - -### 4. Validate Futures Data - -```python -from ml4t.data.validation import validate_ohlcv - -df = provider.fetch_ohlcv("ES.v.0", start="2024-01-01", end="2024-12-31") - -# Standard OHLCV validation -validate_ohlcv(df) - -# Futures-specific checks -assert df.filter(pl.col("volume") == 0).is_empty(), "No zero-volume bars" - -# Check for contract roll gaps (expected at quarterly roll dates) -gaps = df.select("timestamp").diff().filter(pl.col("timestamp") > timedelta(days=3)) -print(f"Contract roll gaps: {len(gaps)}") # Should be 3-4 per year -``` - -### 5. Use Incremental Updates - -```python -# First run: Fetch 1 year of data -history = provider.fetch_ohlcv("ES.v.0", start="2024-01-01", end="2024-12-31") -storage.write(history, "databento/ES.v.0") - -# Daily runs: Only fetch last 1-2 days -new_data = provider.fetch_ohlcv("ES.v.0", start=last_stored_date, end=today) -# Saves API quota and reduces costs -``` - ---- - -## Common Gotchas - -### 1. Continuous vs Specific Contracts - -```python -# ❌ Wrong: Using .v.0 for live trading -# ES.v.0 is synthetic, can't actually trade it -live_order = "Buy 1 ES.v.0 @ market" # Invalid! - -# ✅ Correct: Use specific contract for live trading -live_order = "Buy 1 ESH5 @ market" # Valid (March 2025 contract) - -# Rule: Use .v.0 for backtesting, specific contracts for live trading -``` - -### 2. Tick Size Violations - -```python -# Each contract has minimum tick size -# ES: 0.25 points -# Can't place limit order at 4750.10 (invalid tick) - -# ❌ Wrong -limit_price = 4750.10 # Not a valid ES tick - -# ✅ Correct -limit_price = 4750.25 # Valid ES tick (multiple of 0.25) -``` - -### 3. Expiration Surprises - -```python -# Futures contracts expire - don't get caught! - -# Example: ESZ4 expires Dec 20, 2024 -# If you hold ESZ4 on Dec 21, you may be: -# 1. Physically settled (rare for financial futures) -# 2. Cash settled (common for index futures) -# 3. Assigned a large loss if on wrong side - -# Always roll before expiration (5-10 days prior) -``` - -### 4. Margin Calls - -```python -# Futures are marked-to-market daily -# If position moves against you, broker may require additional margin - -# Example: Long 1 ES @ 4750, margin = $12,000 -# ES drops to 4700 (-50 points = -$2,500) -# Your margin account: $12,000 - $2,500 = $9,500 -# Maintenance margin: $11,000 -# → Margin call! Must deposit $1,500 or close position -``` - ---- - -## Related Documentation - -**Getting Started**: -- [Quick Start Guide](../getting-started/quickstart.md) -- [Tutorial 01: Understanding OHLCV Data](../tutorials/01_understanding_ohlcv.md) - -**Provider Setup**: -- [Provider Selection Guide](../getting-started/provider-selection.md) -- [Creating a Custom Provider](../contributing/creating-a-provider.md) - -**Best Practices**: -- [Tutorial 02: Rate Limiting Best Practices](../tutorials/02_rate_limiting.md) -- [Tutorial 03: Incremental Updates](../tutorials/03_incremental_updates.md) -- [Tutorial 04: Data Quality Validation](../tutorials/04_data_quality.md) -- [Tutorial 05: Multi-Provider Strategies](../tutorials/05_multi_provider.md) - -**Other Asset Classes**: -- [Cryptocurrency Data Guide](./crypto.md) - Bitcoin, Ethereum, altcoins -- [Equities Data Guide](./equities.md) - US and global stocks -- [Forex Data Guide](./forex.md) - Foreign exchange pairs - ---- - -## Next Steps - -1. **Sign up for Databento** and review current usage-based pricing: https://databento.com/pricing -2. **Understand futures mechanics** - Expiration, rolling, margin, tick sizes -3. **Start with simple strategies** - Single contract (ES or NQ) daily data -4. **Use continuous contracts** (.v.0) for backtesting -5. **Account for roll costs** and session times in strategies -6. **Validate data quality** thoroughly before trading with real money - -**Important**: Futures trading carries significant risk due to leverage. Always backtest thoroughly and start with paper trading. - -**Questions or issues?** See the [contribution guide](../contributing/index.md) or open a -GitHub issue. diff --git a/docs/contributing/architecture.md b/docs/contributing/architecture.md index 8236328c..91a3df96 100644 --- a/docs/contributing/architecture.md +++ b/docs/contributing/architecture.md @@ -495,10 +495,9 @@ def _fetch_raw_data(self, symbol, start, end, frequency): ### Why Polars Instead of Pandas? -**Performance**: 10-100x faster for common operations -**Memory**: More efficient memory usage -**Type Safety**: Better type system -**Future**: Better maintained, more modern +Providers and storage use the same columnar frame type, schema expressions, and native Parquet +integration. This avoids conversion at the provider-storage boundary. Pandas remains a core +dependency for calendar and vendor interoperability. ### Why Global Rate Limiting? diff --git a/docs/contributing/testing.md b/docs/contributing/testing.md index 16945c2d..22e3791d 100644 --- a/docs/contributing/testing.md +++ b/docs/contributing/testing.md @@ -18,7 +18,7 @@ pytest -n 4 ``` **Performance**: Tests run in parallel by default using `pytest-xdist`: -- **24 cores**: ~7 seconds for 345 tests (130x speedup vs sequential) +- Parallel execution is optional and should be measured on the target machine - **Sequential** (`-n 0`): ~15 minutes for full suite ### Run Integration Tests diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index be60283b..b7f77686 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -1,75 +1,66 @@ # Installation -## Quick Install - -=== "pip" +## Requirements - ```bash - pip install ml4t-data - ``` +`ml4t-data` supports CPython 3.12, 3.13, and 3.14 on Linux, macOS, and Windows. +Python 3.15 is temporarily excluded while upstream dependencies complete compatibility work. -=== "uv (recommended)" +## Install from PyPI - ```bash - uv add ml4t-data - ``` +```bash +pip install ml4t-data +``` -=== "From source" +With uv: - ```bash - git clone https://github.com/stefan-jansen/ml4t-data.git - cd ml4t-data - uv sync - ``` +```bash +uv add ml4t-data +``` -## Provider Dependencies +## Install provider extras -Some providers require additional dependencies: +The core package includes the synthetic provider and adapters that do not require separate Python +packages. Install only the extras needed by your data sources: ```bash -# Yahoo Finance (included by default) pip install "ml4t-data[yahoo]" - -# DataBento (institutional futures) pip install "ml4t-data[databento]" - -# OANDA (forex) pip install "ml4t-data[oanda]" - -# All providers -pip install "ml4t-data[all-providers]" +pip install "ml4t-data[cot]" ``` -## Development Installation +`ml4t-data[all-providers]` installs all published provider extras. Python packages do not supply +vendor accounts, credentials, data licenses, or network access. -For contributing or running tests: +## Verify the installation -```bash -git clone https://github.com/stefan-jansen/ml4t-data.git -cd ml4t-data -uv sync --all-extras -pre-commit install +```python +import ml4t.data +from ml4t.data.providers import SyntheticProvider + +data = SyntheticProvider(seed=42).fetch_ohlcv( + "SYNTH", "2024-01-01", "2024-01-10", "daily" +) +assert ml4t.data.__version__ +assert not data.is_empty() ``` -## Verify Installation +## Install for development -```python -from ml4t.data import __version__ -print(f"ml4t-data version: {__version__}") - -# Test a provider -from ml4t.data.providers import YahooFinanceProvider -provider = YahooFinanceProvider() -df = provider.fetch_ohlcv("AAPL", "2024-01-01", "2024-01-31") -print(f"Fetched {len(df)} rows") +```bash +git clone https://github.com/ml4t/data.git +cd data +uv sync --all-extras --all-groups +uv run pre-commit install ``` -## Requirements +Run the repository gates before submitting a change: -- Stable CPython 3.12 through 3.14 on Linux, macOS, or Windows -- Python 3.15 is not supported until the core dependency stack passes the complete compatibility - suite on Linux, macOS, and Windows -- Releases 0.1.0 and 0.1.1 predate the Python 3.15 upper bound, so an unpinned installation on - Python 3.15 may select one of those older releases -- Polars (automatically installed) -- Provider-specific SDKs (optional) +```bash +uv run ruff check src/ tests/ +uv run ruff format --check src/ tests/ +uv run ty check +uv run pytest tests/ -q +uv run mkdocs build --strict +uv build +``` diff --git a/docs/getting-started/migration.md b/docs/getting-started/migration.md new file mode 100644 index 00000000..3c68e50d --- /dev/null +++ b/docs/getting-started/migration.md @@ -0,0 +1,35 @@ +# Migrate former QLDM names + +The package preserves two former public names during the 0.x release series. Their use emits +`DeprecationWarning`. They will not be removed before version 1.0. + +## Data root + +Replace: + +```bash +export QLDM_DATA_ROOT=/srv/market-data +``` + +with: + +```bash +export ML4T_DATA_PATH=/srv/market-data +``` + +## Base exception + +Replace: + +```python +from ml4t.data.core.exceptions import QldmError +``` + +with: + +```python +from ml4t.data.core.exceptions import ML4TDataError +``` + +The former exception name resolves to the same class during the migration period. Code that treats +deprecation warnings as errors should migrate before upgrading. diff --git a/docs/getting-started/provider-selection.md b/docs/getting-started/provider-selection.md index a62d3e90..6c1b4f4f 100644 --- a/docs/getting-started/provider-selection.md +++ b/docs/getting-started/provider-selection.md @@ -1,177 +1,73 @@ -# Provider Selection Guide +# Choose a provider -Choose the right data provider for your needs with this decision flowchart. +Choose a provider from the data type, authentication, licensing, and update frequency required by +the workflow. Provider pricing, quotas, and available history change independently of this library; +confirm them with the provider before scheduling recurring work. -CryptoCompare is included for evaluation but is not release-qualified for 0.1.0. Account -registration was unavailable during release validation, so no live contract evidence was obtained. +## Start without an external service -## Quick Decision Flowchart +Use `SyntheticProvider` for installation checks, tests, and examples that must run offline: -```mermaid -flowchart TD - Start[What data do you need?] --> AssetType{Asset Type?} +```python +from ml4t.data.providers import SyntheticProvider - AssetType -->|Crypto| CryptoChoice{API key OK?} - AssetType -->|US Stocks| USChoice{Budget?} - AssetType -->|Global Stocks| GlobalChoice{Budget?} - AssetType -->|Forex| ForexChoice{Professional?} - AssetType -->|Futures/Options| FuturesChoice - AssetType -->|Multiple Assets| MultiChoice{Which combination?} - - %% Crypto Path - CryptoChoice -->|No API key| CoinGecko[CoinGecko
Public API] - CryptoChoice -->|API key is acceptable| CryptoCompare[CryptoCompare
Unverified for 0.1.0] - - %% US Stocks Path - USChoice -->|FREE only| USFree{Call volume?} - USChoice -->|Can pay| USPaid{Budget?} - - USFree -->|Medium 1000/day| Tiingo[Tiingo
FREE 1000/day] - USFree -->|No API key| Yahoo[Yahoo Finance
FREE unofficial] - - USPaid -->|Historical data| EODHD[EODHD
Current paid plan] - USPaid -->|Real-time quote| Finnhub[Finnhub
Free US quote] - - %% Global Stocks Path - GlobalChoice -->|Free testing| EODHDFree[EODHD
20 calls/day] - GlobalChoice -->|Historical data| EODHDPaid[EODHD
Current paid plan] - GlobalChoice -->|Professional| FinnhubGlobal[Finnhub
Paid OHLC plan] - - %% Forex Path - ForexChoice -->|Macro context| FXMacroData[FXMacroData
Public USD/free endpoints] - ForexChoice -->|Trading/Production| OANDA[OANDA
Professional] - ForexChoice -->|Multi-asset| TwelveDataForex[Twelve Data
800/day] - - %% Futures Path - FuturesChoice -->|Institutional| Databento[Databento
Usage-based history] - - %% Multi-Asset Path - MultiChoice -->|Stocks+Forex+Crypto| TwelveData[Twelve Data
FREE 800/day] - MultiChoice -->|US Stocks+Global| EODHDMulti[EODHD
Current paid plan] - MultiChoice -->|US market data| MassiveMulti[Massive
Free Basic plan] - - %% Style definitions - classDef freeProvider fill:#90EE90,stroke:#228B22,stroke-width:2px - classDef paidProvider fill:#FFB6C1,stroke:#DC143C,stroke-width:2px - classDef decision fill:#87CEEB,stroke:#4682B4,stroke-width:2px - - class CoinGecko,Tiingo,Yahoo,EODHDFree,TwelveData,CryptoCompare,FXMacroData,Finnhub,MassiveMulti freeProvider - class EODHD,EODHDPaid,FinnhubGlobal,OANDA,Databento,TwelveDataForex,EODHDMulti paidProvider - class AssetType,CryptoChoice,USChoice,GlobalChoice,ForexChoice,USFree,USPaid,GlobalChoice,ForexChoice,MultiChoice decision +data = SyntheticProvider(seed=42).fetch_ohlcv( + "SYNTH", "2024-01-01", "2024-01-10", "daily" +) ``` -## Provider Comparison Matrix - -### By Asset Class - -| Provider | Crypto | US Stocks | Global Stocks | Forex | Futures | API Key | Free Tier | Best For | -|----------|--------|-----------|---------------|-------|---------|---------|-----------|----------| -| **CoinGecko** | ✅ | ❌ | ❌ | ❌ | ❌ | No | Unlimited | Crypto historical | -| **CryptoCompare** | ✅ | ❌ | ❌ | ❌ | ❌ | Yes | Unverified | Evaluation only | -| **Tiingo** | ✅ | ✅ | ❌ | ❌ | ❌ | Yes | 1000/day | High-quality stocks | -| **EODHD** | ❌ | ✅ | ✅ | ❌ | ❌ | Yes | 20 calls/day, one-year history | Global stocks | -| **Finnhub** | ✅ | ✅ | ✅ | ✅ | ❌ | Yes | US quotes; OHLC is paid | Quotes and company data | -| **Twelve Data** | ✅ | ✅ | ❌ | ✅ | ❌ | Yes | 800/day | Multi-asset + indicators | -| **OANDA** | ❌ | ❌ | ❌ | ✅ | ❌ | Yes | Practice account | Professional forex | -| **Databento** | ❌ | ✅ | ❌ | ❌ | ✅ | Yes | Free metadata; metered history | Institutional futures | -| **Massive** | ✅ | ✅ | ❌ | ✅ | ✅ | Yes | Free Basic plan | Multi-asset market data | - -### By Pricing - -#### Free Tier (No Credit Card Required) +## Select by responsibility -| Provider | Daily Limit | Monthly Limit | Best Use Case | -|----------|-------------|---------------|---------------| -| **CoinGecko** | Current public API limit | Current public API limit | Crypto research | -| **Tiingo** | 1000 calls | 500 symbols | Daily stock updates | -| **EODHD** | 20 calls | Account limit | Global stock testing | -| **Twelve Data** | 800 calls | ~24K calls | Multi-asset research | -| **Massive Stocks Basic** | 5 calls/minute | Account limit | US aggregate data | +| Requirement | Providers to evaluate | External boundary | +| --- | --- | --- | +| Public cryptocurrency market data | CoinGecko, Binance, Binance Public, OKX | Network access and provider usage terms | +| Equity bars | Yahoo Finance, Alpaca, EODHD, Tiingo, Twelve Data, Massive, Finnhub | Some adapters require credentials, accounts, or paid history | +| Foreign exchange | OANDA, Twelve Data, FXMacroData | OANDA and Twelve Data require credentials | +| Futures and options | Databento, Binance, OKX | Historical or licensed data may be metered | +| Macroeconomic series | FRED, FXMacroData | FRED requires an API key for normal use | +| Research factors | Fama-French, AQR | Network access and source-specific terms | +| Prediction markets | Kalshi, Polymarket | Network access and changing public endpoints | +| Frozen equity history | Wiki Prices | Local historical dataset ending in 2018 | +| CFTC positioning | COT | Install the `cot` extra | -#### Paid Tiers (Affordable) +CryptoCompare remains available for evaluation but is not release-qualified until its live provider +contract runs successfully on a release commit. -| Provider | Price | What You Get | -|----------|-------|--------------| -| **EODHD** | See current pricing | Expanded global coverage and history | -| **Twelve Data** | See current pricing | Higher API credits and market coverage | -| **Tiingo** | $30/mo | 20K calls/hour, news, fundamentals | +## Check dependencies and credentials -#### Professional Tiers +Provider extras are explicit: -| Provider | Starting Price | What You Get | -|----------|----------------|--------------| -| **Finnhub** | See current pricing | Historical OHLC and additional datasets | -| **Massive** | See current pricing | Longer history, trades, quotes, and real-time data | -| **Databento** | Usage-based or subscription | Institutional futures and OPRA options | - -## Decision Guidelines - -### For Beginners -1. **Start with CoinGecko** (crypto) - No API key, unlimited free tier -2. **Try Tiingo** (stocks) - Generous free tier (1000/day), great quality -3. **Check access terms** - Coverage and permitted use differ by account - -### For Researchers -1. **Tiingo** - High-quality stock data with generous limits -2. **EODHD** - Global coverage with a small free evaluation allowance -3. **Twelve Data** - Multi-asset research with a usable free tier - -### For Traders -1. **EODHD** - Global end-of-day stocks -2. **OANDA** - Professional forex data -3. **Finnhub** - Free US quotes; paid historical OHLC - -### For Institutions -1. **Databento** - Tick-level futures data -2. **Massive** - Multi-asset professional data -3. **Finnhub** - Global exchange coverage - -## Quick Recommendations - -### "I want crypto data" -→ **CoinGecko** (no API key, subject to public API limits) - -### "I want US stock data for free" -→ **Tiingo** (1000/day) or **Yahoo Finance** (no API key, unofficial) - -### "I want global stock data" -→ **EODHD** or **Twelve Data**, depending on the required market and history - -### "I want stocks + forex + crypto" -→ **Twelve Data** (800/day free) - -### "I'm building a trading system" -→ Choose by required endpoint, latency, history, and licensing terms +```bash +pip install "ml4t-data[yahoo]" +pip install "ml4t-data[databento]" +pip install "ml4t-data[oanda]" +pip install "ml4t-data[cot]" +``` -### "I need professional derivatives data" -→ **Databento** (free discovery and cost estimates; metered time-series data) +Credentialed adapters read the environment variables named in their provider guides. Do not place +credentials in source code, notebooks, or committed configuration files. -## Rate Limit Considerations +## Confirm capabilities in code -### Conservative (Good for testing) -- **EODHD Free**: 20 calls/day -- **Twelve Data Free**: 800/day +The registry records asset classes, authentication, and supported fetch operations: -### Moderate (Good for research) -- **Tiingo**: 1000/day -- **CoinGecko**: 10-50/min +```python +from ml4t.data.providers import advertised_provider_specs -### High Volume (Good for production) -- **EODHD Paid**: Unlimited -- **Finnhub**: 60/min (free), higher (paid) -- **Tiingo Paid**: 20K/hour +for spec in advertised_provider_specs(): + print(spec.name, sorted(spec.capabilities), spec.access_label, spec.extra) +``` -## Next Steps +A registered provider may support OHLCV, economic series, factors, or another capability-specific +method. Do not assume that every adapter implements every operation. -1. **Read the full provider docs** in the [provider reference](../providers/index.md) -2. **Check provider status** in your target asset class -3. **Get API keys** from provider websites -4. **Test with free tiers** before committing to paid plans -5. **Use incremental updates** to minimize API calls +## Before scheduling updates -## Getting Help +1. Read the provider-specific guide in the [provider reference](../providers/index.md). +2. Confirm current account quotas, history, licensing, and redistribution terms with the provider. +3. Run a small request for the exact market, symbol format, frequency, and date range. +4. Set `ML4T_DATA_PATH` to an explicit storage location. +5. Use incremental updates so requests cover only missing ranges. +6. Handle authentication, rate-limit, network, and unavailable-data errors separately. -- **Documentation**: See the [user guide](../user-guide/index.md) -- **Examples**: Check the [example programs](https://github.com/ml4t/data/tree/main/examples) -- **Issues**: Report problems on GitHub -- **Community**: Join discussions for provider recommendations +For a first successful workflow, continue with the [quickstart](quickstart.md). diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index feb2df0e..8d0af83f 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -1,86 +1,56 @@ -# Quickstart +# Generate OHLCV data without an external service -Get market data in under 5 minutes. +This tutorial verifies the installation and introduces the provider interface with deterministic, +locally generated data. -## Basic Usage +## Install the package -```python -from ml4t.data.providers import YahooFinanceProvider - -# Create a provider -provider = YahooFinanceProvider() - -# Fetch OHLCV data -df = provider.fetch_ohlcv( - symbol="AAPL", - start="2024-01-01", - end="2024-12-31", - frequency="daily" -) +Use CPython 3.12, 3.13, or 3.14. -print(df.head()) +```bash +pip install ml4t-data ``` -## Multiple Symbols (Async) - -For faster multi-symbol fetches, use async batch loading: +## Generate a daily series ```python -import asyncio -from ml4t.data.managers.async_batch import async_batch_load -from ml4t.data.providers import YahooFinanceProvider +from ml4t.data.providers import SyntheticProvider -async def main(): - async with YahooFinanceProvider() as provider: - df = await async_batch_load( - provider, - symbols=["AAPL", "MSFT", "GOOGL", "AMZN", "META"], - start="2024-01-01", - end="2024-12-31", - ) - return df - -df = asyncio.run(main()) -print(f"Fetched {len(df)} rows for {df['symbol'].n_unique()} symbols") -``` +provider = SyntheticProvider(seed=42) +data = provider.fetch_ohlcv("SYNTH", "2024-01-01", "2024-01-10", "daily") -## CLI Usage - -```bash -# Fetch data via command line -ml4t-data fetch AAPL MSFT GOOGL \ - --start 2024-01-01 \ - --provider yahoo \ - --output ~/data - -# Update all datasets from config -ml4t-data update-all -c ml4t-data.yaml +assert not data.is_empty() +assert {"timestamp", "symbol", "open", "high", "low", "close", "volume"} <= set(data.columns) +print(data) ``` -## Configuration File +`SyntheticProvider` follows the same OHLCV interface as network-backed providers. The fixed seed +makes the generated series repeatable, so this example works in a clean environment without +credentials. -Create `ml4t-data.yaml` for automated updates: +## Fetch external data -```yaml -storage: - base_path: ~/ml4t-data +Choose a provider only after checking its dependency and service requirements. For example, Yahoo +Finance support requires the `yahoo` extra: -datasets: - - name: sp500 - provider: yahoo - symbols_file: sp500.txt - frequency: daily - start_date: 2020-01-01 +```bash +pip install "ml4t-data[yahoo]" ``` -Then run: +```python +from ml4t.data.providers import YahooFinanceProvider -```bash -ml4t-data update-all -c ml4t-data.yaml +provider = YahooFinanceProvider() +data = provider.fetch_ohlcv("AAPL", "2024-01-01", "2024-01-31", "daily") ``` -## Next Steps +This second example requires network access and is subject to the provider's availability and usage +terms. See [provider selection](provider-selection.md) before using an adapter in a recurring +workflow. + +## Next steps -- [Provider Selection Guide](provider-selection.md) - Choose the right data source -- [User Guide](../user-guide/index.md) - Complete documentation -- [API Reference](../api/index.md) - Detailed API docs +- [Configure local storage](../user-guide/configuration.md) +- [Update an existing dataset](../user-guide/incremental-updates.md) +- [Validate market data](../user-guide/data-quality.md) +- [Inspect the provider API](../api/index.md) diff --git a/docs/index.md b/docs/index.md index 1620c607..8fbf7540 100644 --- a/docs/index.md +++ b/docs/index.md @@ -5,163 +5,68 @@ hide: # ML4T Data -**High-performance market data acquisition for quantitative finance.** +Market data acquisition, storage, and update workflows for machine learning for trading. + +`ml4t-data` provides one interface for acquiring provider data, validating observations, writing +local datasets, and updating those datasets incrementally. Provider adapters cover exchange, +vendor, public, and synthetic data sources. Each provider documents its own credentials, rate +limits, licensing, and optional dependencies.
-- :material-clock-fast:{ .lg .middle } __5-Minute Setup__ +- :material-play-circle:{ .lg .middle } __First successful use__ --- - Get market data in 3 lines of code. No API keys required for basic usage. + Install the wheel and generate deterministic OHLCV data without credentials or network access. [:octicons-arrow-right-24: Quickstart](getting-started/quickstart.md) -- :material-database:{ .lg .middle } __20+ Provider Adapters__ +- :material-database:{ .lg .middle } __Choose a provider__ --- - Equities, crypto, forex, futures, macro, prediction markets, and factors. + Compare supported asset classes, authentication requirements, and provider capabilities. - [:octicons-arrow-right-24: Provider Guide](providers/index.md) + [:octicons-arrow-right-24: Provider selection](getting-started/provider-selection.md) -- :material-lightning-bolt:{ .lg .middle } __10-100x Faster__ +- :material-folder-sync:{ .lg .middle } __Maintain datasets__ --- - Polars-based processing with async batch loading for maximum throughput. + Configure storage, detect gaps, and update existing data without replacing valid history. - [:octicons-arrow-right-24: Performance](user-guide/index.md) + [:octicons-arrow-right-24: Incremental updates](user-guide/incremental-updates.md) -- :material-shield-check:{ .lg .middle } __Pipeline Ready__ +- :material-code-braces:{ .lg .middle } __Use the public interface__ --- - Circuit breakers, rate limiting, OHLC validation, and incremental updates. + Inspect provider protocols, configuration models, storage interfaces, and exceptions. - [:octicons-arrow-right-24: Features](user-guide/data-quality.md) + [:octicons-arrow-right-24: API reference](api/index.md)
-## Quick Example +## Quick example ```python -from ml4t.data.providers import YahooFinanceProvider - -# Fetch OHLCV data (no API key needed) -provider = YahooFinanceProvider() -df = provider.fetch_ohlcv("AAPL", "2024-01-01", "2024-12-31") - -print(df.head()) -# shape: (252, 7) -# ┌─────────────────────┬────────┬────────┬────────┬────────┬────────┬──────────┐ -# │ timestamp ┆ symbol ┆ open ┆ high ┆ low ┆ close ┆ volume │ -# │ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │ -# │ datetime[μs, UTC] ┆ str ┆ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 │ -# ╞═════════════════════╪════════╪════════╪════════╪════════╪════════╪══════════╡ -# │ 2024-01-02 00:00:00 ┆ AAPL ┆ 187.15 ┆ 188.44 ┆ 183.89 ┆ 185.64 ┆ 82488700 │ -# └─────────────────────┴────────┴────────┴────────┴────────┴────────┴──────────┘ -``` +from ml4t.data.providers import SyntheticProvider -## Async Batch Loading (3-10x Faster) +provider = SyntheticProvider(seed=42) +data = provider.fetch_ohlcv("SYNTH", "2024-01-01", "2024-01-10", "daily") -```python -import asyncio -from ml4t.data.managers.async_batch import async_batch_load -from ml4t.data.providers import YahooFinanceProvider - -async def fetch_portfolio(): - async with YahooFinanceProvider() as provider: - return await async_batch_load( - provider, - symbols=["AAPL", "MSFT", "GOOGL", "AMZN", "META"], - start="2024-01-01", - end="2024-12-31", - max_concurrent=10, - ) - -df = asyncio.run(fetch_portfolio()) -print(f"Fetched {len(df)} rows for {df['symbol'].n_unique()} symbols") +assert not data.is_empty() ``` -## Installation - -=== "pip" - - ```bash - pip install ml4t-data - ``` - -=== "uv" - - ```bash - uv add ml4t-data - ``` - -=== "With providers" - - ```bash - pip install "ml4t-data[yahoo,databento]" - ``` - -## Provider Comparison - -| Provider | Asset Class | Free Tier | Async | Best For | -|----------|-------------|-----------|-------|----------| -| **Yahoo** | Stocks, ETFs, Crypto | Unlimited | Thread | Learning, backtesting | -| **CoinGecko** | Crypto | 10K+ coins | Native | Crypto historical | -| **EODHD** | Global Stocks | 20 calls/day | Native | Global coverage | -| **DataBento** | Futures, Options | Free metadata; metered history | Thread | Institutional data | -| **Fama-French** | Factors | Unlimited | Thread | Academic research | - -[:octicons-arrow-right-24: Full provider guide](providers/index.md) - -## For ML4T Book Readers - -This library is the reference implementation for **Machine Learning for Trading (Third Edition)**. -The book uses ml4t-data across 6 chapters and 25 notebooks, covering 14 of 20 providers. - -
- -- :material-book-open-variant:{ .lg .middle } __Chapter-Feature Mapping__ - - --- - - - **Ch 2**: DataManager, Universe, HiveStorage, gap detection, data quality - - **Ch 4**: FRED, CoinGecko, Kalshi, Polymarket, COT data - - **Ch 16-19**: Binance, Fama-French, AQR for backtesting and risk +The synthetic provider is local and deterministic. Live providers require network access and may +require an account, credentials, paid data, or an optional package. - [:octicons-arrow-right-24: Full book guide](book-guide/index.md) +## Start here -- :material-rocket-launch:{ .lg .middle } __Recurring Workflows__ - - --- - - Graduate from notebooks to automated pipelines with `download_all.py --update`, - incremental updates, and CLI automation. - - [:octicons-arrow-right-24: Incremental updates](user-guide/incremental-updates.md) - -
- -## Next Steps - -
- -- [:octicons-download-24: __Installation__](getting-started/installation.md) - - Detailed installation guide with all optional dependencies. - -- [:octicons-book-24: __User Guide__](user-guide/index.md) - - Complete documentation for all features. - -- [:octicons-code-24: __API Reference__](api/index.md) - - Auto-generated documentation from source code. - -- [:octicons-people-24: __Contributing__](contributing/index.md) - - Create your own provider or contribute to the project. - -
+- [Installation](getting-started/installation.md) describes supported Python versions and extras. +- [Quickstart](getting-started/quickstart.md) teaches the first offline workflow. +- [User guide](user-guide/index.md) contains task-oriented storage, update, and validation guides. +- [Provider guide](providers/index.md) records provider capabilities and external-service boundaries. +- [Migration guide](getting-started/migration.md) maps former QLDM names to current interfaces. +- [Contributing](contributing/index.md) explains development setup and quality checks. diff --git a/docs/multi_asset_guide.md b/docs/multi_asset_guide.md deleted file mode 100644 index 701abbd7..00000000 --- a/docs/multi_asset_guide.md +++ /dev/null @@ -1,1037 +0,0 @@ -# Multi-Asset Support - Complete User Guide - -**Version**: 1.0.0 -**Last Updated**: 2025-11-15 -**Status**: Production-Ready - ---- - -## Table of Contents - -1. [Introduction](#introduction) -2. [Core Concepts](#core-concepts) -3. [Quick Start](#quick-start) -4. [API Reference](#api-reference) -5. [Usage Patterns](#usage-patterns) -6. [Performance](#performance) -7. [Migration Guide](#migration-guide) -8. [Best Practices](#best-practices) -9. [Troubleshooting](#troubleshooting) - ---- - -## Introduction - -### What is Multi-Asset Support? - -Multi-asset support enables efficient loading, processing, and analysis of data for multiple securities simultaneously. Instead of fetching data for one symbol at a time, you can: - -- **Load 100+ symbols in seconds** with parallel fetching -- **Analyze cross-sectional patterns** across entire universes -- **Leverage cache-first loading** for 10-100x speedup -- **Use pre-defined universes** (S&P 500, NASDAQ-100, etc.) - -### When to Use Multi-Asset Features - -**Use multi-asset support when you need to**: -- Backtest portfolio strategies across many symbols -- Perform sector rotation or pair trading analysis -- Calculate cross-sectional metrics (rankings, quintiles) -- Build market-wide indicators or sentiment measures -- Compare performance across asset groups - -**Stick with single-asset if**: -- You're analyzing just one security -- You're building single-instrument strategies -- You're doing deep dive analysis on individual stocks - -### Key Benefits - -1. **Performance**: 10-100x faster than sequential loading -2. **Convenience**: Pre-defined universes (S&P 500, crypto top-100, etc.) -3. **Consistency**: Standardized stacked format across all operations -4. **Scalability**: Handles 500+ symbols efficiently -5. **Caching**: Load from storage in <1 second instead of minutes - ---- - -## Core Concepts - -### Stacked (Long) Format - -Multi-asset data uses a **stacked format** with a `symbol` column to identify each security: - -```python -┌─────────────────────┬────────┬───────┬───────┬───────┬───────┬──────────┐ -│ timestamp │ symbol │ open │ high │ low │ close │ volume │ -├─────────────────────┼────────┼───────┼───────┼───────┼───────┼──────────┤ -│ 2024-01-01 09:30:00 │ AAPL │ 180.0 │ 182.0 │ 179.0 │ 181.0 │ 1200000 │ -│ 2024-01-01 09:30:00 │ MSFT │ 370.0 │ 372.0 │ 368.0 │ 371.0 │ 800000 │ -│ 2024-01-01 09:30:00 │ GOOG │ 140.0 │ 142.0 │ 139.0 │ 141.0 │ 950000 │ -│ 2024-01-02 09:30:00 │ AAPL │ 181.0 │ 183.0 │ 180.0 │ 182.0 │ 1100000 │ -│ 2024-01-02 09:30:00 │ MSFT │ 371.0 │ 373.0 │ 370.0 │ 372.0 │ 750000 │ -│ 2024-01-02 09:30:00 │ GOOG │ 141.0 │ 143.0 │ 140.0 │ 142.0 │ 900000 │ -└─────────────────────┴────────┴───────┴───────┴───────┴───────┴──────────┘ -``` - -**Why stacked format?** -- **Polars-native**: No MultiIndex needed (unlike pandas) -- **Efficient grouping**: Natural `group_by('symbol')` operations -- **Proven architecture**: Used in qengine backtest library -- **Scalable**: Works with 500+ symbols without performance degradation - -### MultiAssetSchema - -The `MultiAssetSchema` class validates and standardizes multi-asset DataFrames: - -```python -from ml4t.data.core.schemas import MultiAssetSchema - -# Required columns -# - timestamp (datetime with timezone) -# - symbol (string) -# - open, high, low, close, volume (float) - -# Validate a DataFrame -is_valid = MultiAssetSchema.validate(df, strict=True) - -# Standardize column order and sorting -df = MultiAssetSchema.standardize_order(df) - -# Create empty DataFrame with proper schema -df = MultiAssetSchema.create_empty('equities') -``` - -### Format Conversion - -While stacked format is canonical, you can convert to/from wide format when needed: - -**Stacked → Wide** (for correlation analysis, pandas compatibility): -```python -from ml4t.data.utils.format import pivot_to_wide - -df_wide = pivot_to_wide(df_stacked, value_cols=['close', 'volume']) - -# Result: -┌────────────┬────────────┬────────────┬────────────┬─────────────┬─────────────┐ -│ timestamp │ close_AAPL │ close_MSFT │ close_GOOG │ volume_AAPL │ volume_MSFT │ -├────────────┼────────────┼────────────┼────────────┼─────────────┼─────────────┤ -│ 2024-01-01 │ 181.0 │ 371.0 │ 141.0 │ 1200000 │ 800000 │ -│ 2024-01-02 │ 182.0 │ 372.0 │ 142.0 │ 1100000 │ 750000 │ -└────────────┴────────────┴────────────┴────────────┴─────────────┴─────────────┘ -``` - -**Wide → Stacked** (reverse conversion): -```python -from ml4t.data.utils.format import pivot_to_stacked - -df_stacked = pivot_to_stacked(df_wide) # Back to stacked format -``` - -**⚠️ Performance Warning**: Wide format doesn't scale beyond ~100 symbols. Use stacked format whenever possible. - ---- - -## Quick Start - -### 1. Basic Batch Loading - -Load multiple symbols in one call: - -```python -from ml4t.data import DataManager - -manager = DataManager() - -# Load 5 tech stocks -df = manager.batch_load( - symbols=['AAPL', 'MSFT', 'GOOG', 'AMZN', 'META'], - start='2024-01-01', - end='2024-12-31', - provider='yahoo' -) - -print(f"Loaded {len(df)} rows for {df['symbol'].n_unique()} symbols") -# Output: Loaded 1,260 rows for 5 symbols -``` - -### 2. Universe Loading - -Use pre-defined symbol lists: - -```python -from ml4t.data.universe import Universe - -# S&P 500 -df = manager.batch_load_universe( - universe='SP500', - start='2024-01-01', - end='2024-12-31' -) - -# Or get the symbol list directly -symbols = Universe.SP500 -print(f"S&P 500 contains {len(symbols)} symbols") -``` - -**Available universes**: -- `Universe.SP500` - S&P 500 index (500+ symbols) -- `Universe.NASDAQ100` - NASDAQ-100 index (100 symbols) -- `Universe.CRYPTO_TOP_100` - Top 100 cryptocurrencies by market cap -- `Universe.FOREX_MAJORS` - Major forex pairs (EUR/USD, GBP/USD, etc.) - -### 3. Cache-First Loading - -For maximum performance, load from storage instead of network: - -```python -from ml4t.data.storage import HiveStorage, StorageConfig - -# Setup storage -storage = HiveStorage(StorageConfig(base_path="./data")) -manager = DataManager(storage=storage) - -# First time: Load from network and cache -df = manager.batch_load( - symbols=['AAPL', 'MSFT', 'GOOG'], - start='2024-01-01', - end='2024-12-31', - provider='yahoo' -) -# Takes ~5-10 seconds - -# Subsequent times: Load from cache (very fast!) -df = manager.batch_load_from_storage( - symbols=['AAPL', 'MSFT', 'GOOG'], - start='2024-01-01', - end='2024-12-31' -) -# Takes ~0.01 seconds (100x faster!) -``` - -### 4. Cross-Sectional Analysis - -Analyze patterns across symbols: - -```python -import polars as pl - -# Calculate returns by symbol -df = df.with_columns( - pl.col('close').pct_change().over('symbol').alias('returns') -) - -# Daily cross-sectional statistics -daily_stats = df.group_by('timestamp').agg([ - pl.col('returns').mean().alias('mean_return'), - pl.col('returns').std().alias('std_return'), - pl.col('volume').sum().alias('total_volume'), - pl.col('symbol').count().alias('n_symbols') -]) - -print(daily_stats.head()) -``` - ---- - -## API Reference - -### batch_load() - -Load multiple symbols in parallel from network providers. - -```python -df = manager.batch_load( - symbols: list[str], - start: str, - end: str, - frequency: str = "daily", - provider: str | None = None, - max_workers: int = 4, - fail_on_partial: bool = False, - **kwargs -) -> pl.DataFrame -``` - -**Parameters**: -- `symbols`: List of symbol identifiers (e.g., `['AAPL', 'MSFT', 'GOOG']`) -- `start`: Start date (ISO format: `'2024-01-01'`) -- `end`: End date (ISO format: `'2024-12-31'`) -- `frequency`: Bar frequency (`'daily'`, `'1h'`, `'5min'`, etc.) -- `provider`: Data provider (`'yahoo'`, `'databento'`, `'binance'`, etc.) -- `max_workers`: Number of parallel workers (default: 4) -- `fail_on_partial`: Raise error if any symbol fails (default: False) - -**Returns**: Polars DataFrame in stacked format with `symbol` column - -**Example**: -```python -df = manager.batch_load( - symbols=['AAPL', 'MSFT', 'GOOG'], - start='2024-01-01', - end='2024-01-31', - provider='yahoo', - max_workers=8 # Faster parallel fetching -) -``` - ---- - -### batch_load_universe() - -Load a pre-defined universe of symbols. - -```python -df = manager.batch_load_universe( - universe: str | list[str], - start: str, - end: str, - frequency: str = "daily", - provider: str | None = None, - max_workers: int = 4, - **kwargs -) -> pl.DataFrame -``` - -**Parameters**: -- `universe`: Universe name (`'SP500'`, `'NASDAQ100'`, etc.) or custom list -- Other parameters same as `batch_load()` - -**Returns**: Polars DataFrame in stacked format - -**Example**: -```python -# Pre-defined universe -df = manager.batch_load_universe('SP500', '2024-01-01', '2024-12-31') - -# Custom universe -df = manager.batch_load_universe( - universe=['SPY', 'QQQ', 'IWM'], # ETF trio - start='2024-01-01', - end='2024-12-31' -) -``` - ---- - -### batch_load_from_storage() - -Load multiple symbols from local storage (cache-first). - -```python -df = manager.batch_load_from_storage( - symbols: list[str], - start: str, - end: str, - frequency: str = "daily", - asset_class: str = "equities", - fetch_missing: bool = False, - provider: str | None = None, - max_workers: int = 4, - **kwargs -) -> pl.DataFrame -``` - -**Parameters**: -- `symbols`: List of symbol identifiers -- `start`, `end`: Date range -- `frequency`: Bar frequency -- `asset_class`: Asset class for storage lookup (`'equities'`, `'crypto'`, etc.) -- `fetch_missing`: If True, fetch missing symbols from provider (default: False) -- `provider`: Provider to use if fetching missing symbols -- `max_workers`: Number of parallel workers for storage reads - -**Returns**: Polars DataFrame in stacked format - -**Performance**: **10-100x faster** than network fetching when data is cached - -**Example**: -```python -# Fast load from cache (errors if any symbol missing) -df = manager.batch_load_from_storage( - symbols=['AAPL', 'MSFT', 'GOOG'], - start='2024-01-01', - end='2024-12-31', - fetch_missing=False -) - -# Load from cache with fallback to network -df = manager.batch_load_from_storage( - symbols=['AAPL', 'MSFT', 'NEWSTOCK'], # NEWSTOCK not in cache - start='2024-01-01', - end='2024-12-31', - fetch_missing=True, - provider='yahoo' -) -``` - ---- - -### pivot_to_wide() - -Convert stacked format to wide (pivoted) format. - -```python -from ml4t.data.utils.format import pivot_to_wide - -df_wide = pivot_to_wide( - df: pl.DataFrame, - value_cols: list[str] | None = None -) -> pl.DataFrame -``` - -**Parameters**: -- `df`: Stacked DataFrame with `timestamp` and `symbol` columns -- `value_cols`: Columns to pivot (default: all OHLCV columns) - -**Returns**: Wide DataFrame with columns like `close_AAPL`, `close_MSFT`, etc. - -**Example**: -```python -# Pivot only close prices -df_wide = pivot_to_wide(df, value_cols=['close']) - -# Use for pandas correlation analysis -df_pandas = df_wide.to_pandas().set_index('timestamp') -correlation_matrix = df_pandas.corr() -``` - ---- - -### pivot_to_stacked() - -Convert wide format back to stacked format. - -```python -from ml4t.data.utils.format import pivot_to_stacked - -df_stacked = pivot_to_stacked( - df: pl.DataFrame -) -> pl.DataFrame -``` - -**Parameters**: -- `df`: Wide DataFrame with symbol-suffixed columns - -**Returns**: Stacked DataFrame with `symbol` column - -**Example**: -```python -# Round-trip conversion -df_wide = pivot_to_wide(df_stacked) -df_back = pivot_to_stacked(df_wide) - -# Data integrity preserved -assert df_stacked.sort(['timestamp', 'symbol']).equals( - df_back.sort(['timestamp', 'symbol']) -) -``` - ---- - -## Usage Patterns - -### Pattern 1: Portfolio Backtesting - -Load all portfolio constituents and run backtest: - -```python -# Define portfolio -portfolio_symbols = ['AAPL', 'MSFT', 'GOOG', 'AMZN', 'META'] - -# Load data -df = manager.batch_load_from_storage( - symbols=portfolio_symbols, - start='2020-01-01', - end='2024-12-31', - fetch_missing=True, - provider='yahoo' -) - -# Calculate returns -df = df.with_columns( - pl.col('close').pct_change().over('symbol').alias('returns') -) - -# Equal-weight portfolio return -portfolio_returns = df.group_by('timestamp').agg( - pl.col('returns').mean().alias('portfolio_return') -) -``` - -### Pattern 2: Sector Rotation - -Analyze sector performance: - -```python -# Define sectors -tech = ['AAPL', 'MSFT', 'GOOG'] -finance = ['JPM', 'BAC', 'GS'] -healthcare = ['UNH', 'JNJ', 'PFE'] - -all_symbols = tech + finance + healthcare - -# Load data -df = manager.batch_load_universe(all_symbols, '2024-01-01', '2024-12-31') - -# Add sector labels -df = df.with_columns( - pl.when(pl.col('symbol').is_in(tech)) - .then(pl.lit('Tech')) - .when(pl.col('symbol').is_in(finance)) - .then(pl.lit('Finance')) - .otherwise(pl.lit('Healthcare')) - .alias('sector') -) - -# Sector performance -sector_perf = df.group_by(['timestamp', 'sector']).agg([ - pl.col('close').mean().alias('avg_close'), - pl.col('volume').sum().alias('total_volume') -]) -``` - -### Pattern 3: Universe Screening - -Screen entire universe for trading signals: - -```python -# Load S&P 500 -df = manager.batch_load_from_storage( - symbols=Universe.SP500, - start='2024-01-01', - end='2024-12-31' -) - -# Calculate 20-day momentum -df = df.with_columns( - (pl.col('close') / pl.col('close').shift(20) - 1) - .over('symbol') - .alias('momentum_20d') -) - -# Screen: momentum > 10% -winners = df.filter(pl.col('momentum_20d') > 0.10) - -print(f"Found {winners['symbol'].n_unique()} symbols with >10% momentum") -``` - -### Pattern 4: Cross-Sectional Ranking - -Rank symbols by various metrics: - -```python -import polars as pl - -# Load data -df = manager.batch_load_from_storage( - symbols=Universe.NASDAQ100, - start='2024-01-01', - end='2024-12-31' -) - -# Calculate returns and volatility -df = df.with_columns([ - pl.col('close').pct_change().over('symbol').alias('returns'), -]) - -# Daily cross-sectional ranks -df = df.with_columns([ - pl.col('returns').rank('ordinal').over('timestamp').alias('return_rank'), - pl.col('volume').rank('ordinal').over('timestamp').alias('volume_rank'), -]) - -# Top decile by returns -top_decile = df.filter(pl.col('return_rank') > pl.col('return_rank').max() * 0.9) -``` - -### Pattern 5: Pair Trading - -Find and analyze correlated pairs: - -```python -# Load potential pairs -df = manager.batch_load( - symbols=['XLE', 'XLF', 'XLK', 'XLV', 'XLI'], # Sector ETFs - start='2024-01-01', - end='2024-12-31' -) - -# Convert to wide for correlation -df_wide = pivot_to_wide(df, value_cols=['close']) - -# Calculate correlation matrix -df_pandas = df_wide.to_pandas().set_index('timestamp') -corr_matrix = df_pandas.corr() - -# Find highly correlated pairs (>0.8) -high_corr_pairs = corr_matrix[(corr_matrix > 0.8) & (corr_matrix < 1.0)] -print(high_corr_pairs) -``` - ---- - -## Performance - -### Benchmark Results - -**Storage Load Performance** (100 symbols, 252 trading days each): - -| Operation | Time | Rows/Second | Speedup vs Network | -|-----------|------|-------------|--------------------| -| batch_load_from_storage() | 0.165s | 152,000 | **76-303x faster** | -| batch_load() | 12-50s | 500-2,000 | Baseline | - -**Format Conversion Performance** (50 symbols, 252 days): - -| Operation | Input Size | Time | Throughput | -|-----------|------------|------|------------| -| Stacked → Wide | 12,600 rows | 0.009s | 1.4M rows/sec | -| Wide → Stacked | 252×50 matrix | 0.010s | 1.26M cells/sec | - -### Performance Tips - -#### 1. Use Storage for Repeated Analysis - -```python -# DON'T: Fetch every time (slow) -for iteration in range(100): - df = manager.batch_load(symbols, start, end) # 5-10 seconds each! - -# DO: Load once, cache, then reuse -df = manager.batch_load(symbols, start, end) -for iteration in range(100): - df_cached = manager.batch_load_from_storage(symbols, start, end) # 0.1 seconds! -``` - -#### 2. Increase Workers for Large Universes - -```python -# Default (conservative) -df = manager.batch_load(Universe.SP500, start, end, max_workers=4) - -# Faster (if provider allows) -df = manager.batch_load(Universe.SP500, start, end, max_workers=16) -``` - -#### 3. Use Narrow Date Ranges - -```python -# DON'T: Load full history if you only need recent data -df = manager.batch_load_from_storage(symbols, '2000-01-01', '2024-12-31') - -# DO: Load only what you need -df = manager.batch_load_from_storage(symbols, '2024-01-01', '2024-12-31') -``` - -#### 4. Prefer Stacked Format - -```python -# DON'T: Convert to wide unless necessary -df_wide = pivot_to_wide(df) # Extra overhead - -# DO: Work in stacked format -df_stacked.group_by('symbol').agg(...) # Native Polars operations -``` - ---- - -## Migration Guide - -### From Single-Symbol to Multi-Symbol - -**Before (single-symbol)**: -```python -# Load one at a time -aapl = manager.load('AAPL', '2024-01-01', '2024-12-31') -msft = manager.load('MSFT', '2024-01-01', '2024-12-31') -goog = manager.load('GOOG', '2024-01-01', '2024-12-31') - -# Combine manually -import polars as pl -aapl = aapl.with_columns(pl.lit('AAPL').alias('symbol')) -msft = msft.with_columns(pl.lit('MSFT').alias('symbol')) -goog = goog.with_columns(pl.lit('GOOG').alias('symbol')) -df = pl.concat([aapl, msft, goog]) -``` - -**After (multi-symbol)**: -```python -# Load all at once -df = manager.batch_load( - symbols=['AAPL', 'MSFT', 'GOOG'], - start='2024-01-01', - end='2024-12-31' -) -# Done! Already has symbol column and proper format -``` - -### From Pandas to Polars Stacked Format - -**Before (pandas with MultiIndex)**: -```python -import pandas as pd - -# Pandas MultiIndex DataFrame -df_pandas = pd.DataFrame({ - ('AAPL', 'close'): [180, 181, 182], - ('MSFT', 'close'): [370, 371, 372], -}, index=pd.date_range('2024-01-01', periods=3)) -``` - -**After (Polars stacked)**: -```python -import polars as pl - -df = manager.batch_load(['AAPL', 'MSFT'], '2024-01-01', '2024-01-03') - -# Stacked format - much cleaner! -# - No MultiIndex complexity -# - Easier filtering and grouping -# - 10-100x faster operations -``` - -### From Wide to Stacked Format - -**If you have wide data**: -```python -# Wide format DataFrame (e.g., from CSV) -df_wide = pl.read_csv('wide_data.csv') -# Columns: timestamp, AAPL_close, MSFT_close, GOOG_close - -# Convert to stacked -from ml4t.data.utils.format import pivot_to_stacked -df_stacked = pivot_to_stacked(df_wide) - -# Now compatible with all multi-asset features -df_stacked.group_by('symbol').agg(...) -``` - ---- - -## Best Practices - -### 1. Always Validate Schema - -```python -from ml4t.data.core.schemas import MultiAssetSchema - -# After loading external data -df = pl.read_parquet('external_data.parquet') - -# Validate before processing -if not MultiAssetSchema.validate(df, strict=False): - print("Warning: Schema validation failed") - df = MultiAssetSchema.standardize_order(df) # Fix it -``` - -### 2. Use Graceful Failure Mode - -```python -# Allow partial success (recommended for research) -df = manager.batch_load( - symbols=['AAPL', 'INVALID', 'MSFT'], - fail_on_partial=False # Returns AAPL + MSFT -) - -# Strict mode (for production) -df = manager.batch_load( - symbols=['AAPL', 'MSFT', 'GOOG'], - fail_on_partial=True # Raises error if any fail -) -``` - -### 3. Pre-populate Storage - -```python -# One-time setup: Populate storage -for symbol in Universe.SP500: - manager.load(symbol, '2020-01-01', '2024-12-31', provider='yahoo') - -# Then use fast storage loading -df = manager.batch_load_from_storage( - symbols=Universe.SP500, - start='2024-01-01', - end='2024-12-31' -) -# 100x faster! -``` - -### 4. Leverage Polars Operations - -```python -# DON'T: Convert to pandas unnecessarily -df_pandas = df.to_pandas() -df_pandas.groupby('symbol')['close'].mean() # Slow! - -# DO: Use Polars native operations -df.group_by('symbol').agg(pl.col('close').mean()) # Fast! -``` - -### 5. Monitor Memory Usage - -```python -# For very large universes (500+ symbols), process in batches -symbols = Universe.SP500 -batch_size = 50 - -for i in range(0, len(symbols), batch_size): - batch_symbols = symbols[i:i+batch_size] - df_batch = manager.batch_load_from_storage( - symbols=batch_symbols, - start='2024-01-01', - end='2024-12-31' - ) - # Process batch - process_batch(df_batch) -``` - ---- - -## Troubleshooting - -### Issue: "ValueError: Symbol column not found" - -**Cause**: DataFrame missing required `symbol` column - -**Solution**: -```python -# Add symbol column manually -df = df.with_columns(pl.lit('AAPL').alias('symbol')) - -# Or use schema helper -from ml4t.data.core.schemas import MultiAssetSchema -df = MultiAssetSchema.add_symbol_column(df, symbol='AAPL') -``` - -### Issue: "Storage not configured" - -**Cause**: Calling `batch_load_from_storage()` without storage setup - -**Solution**: -```python -from ml4t.data.storage import HiveStorage, StorageConfig - -# Configure storage -storage = HiveStorage(StorageConfig(base_path="./data")) -manager = DataManager(storage=storage) - -# Now storage methods work -df = manager.batch_load_from_storage(...) -``` - -### Issue: "No data for date range" - -**Cause**: Requested date range not in storage - -**Solution**: -```python -# Option 1: Use fetch_missing=True -df = manager.batch_load_from_storage( - symbols=['AAPL'], - start='2024-01-01', - end='2024-12-31', - fetch_missing=True, # Auto-fetch if missing - provider='yahoo' -) - -# Option 2: Pre-populate storage -manager.load('AAPL', '2024-01-01', '2024-12-31', provider='yahoo') -df = manager.batch_load_from_storage(['AAPL'], '2024-01-01', '2024-12-31') -``` - -### Issue: "Duplicate timestamp-symbol pairs" - -**Cause**: Data contains duplicates (data quality issue) - -**Solution**: -```python -# Remove duplicates -df = df.unique(subset=['timestamp', 'symbol'], keep='last') - -# Or detect and investigate -duplicates = df.filter( - pl.struct(['timestamp', 'symbol']).is_duplicated() -) -print(f"Found {len(duplicates)} duplicates") -``` - -### Issue: Wide format "too many columns" error - -**Cause**: Trying to pivot 100+ symbols creates thousands of columns - -**Solution**: -```python -# DON'T: Pivot large universes -df_wide = pivot_to_wide(df) # Fails with 100+ symbols - -# DO: Work in stacked format -df_stacked.group_by('symbol').agg(...) # Scales to 1000+ symbols -``` - -### Issue: Slow batch loading - -**Cause**: Network rate limits or too few workers - -**Solution**: -```python -# Increase workers (if provider allows) -df = manager.batch_load( - symbols=Universe.SP500, - max_workers=16 # Up from default 4 -) - -# Or use storage for subsequent loads -df = manager.batch_load_from_storage(...) # 100x faster -``` - ---- - -## Advanced Topics - -### Custom Universes - -Create your own symbol lists: - -```python -from ml4t.data.universe import Universe - -# Define custom universe -tech_giants = ['AAPL', 'MSFT', 'GOOG', 'AMZN', 'META', 'NVDA', 'TSLA'] -Universe.register_universe('TECH_GIANTS', tech_giants) - -# Use it -df = manager.batch_load_universe('TECH_GIANTS', '2024-01-01', '2024-12-31') - -# Or create dynamically -faang = ['META', 'AAPL', 'AMZN', 'NFLX', 'GOOG'] -df = manager.batch_load_universe(faang, '2024-01-01', '2024-12-31') -``` - -### Date Range Optimization - -Use lazy evaluation for efficient date filtering: - -```python -# Storage reads apply filters during scan (very fast) -df = manager.batch_load_from_storage( - symbols=['AAPL', 'MSFT'], - start='2024-11-01', # Only loads November partitions - end='2024-11-30' -) - -# Predicate pushdown = only read relevant Parquet files -``` - -### Parallel Storage Workers - -Tune parallelism for your hardware: - -```python -# Default: 4 workers -df = manager.batch_load_from_storage(symbols, start, end) - -# More workers for SSD/NVMe (faster I/O) -df = manager.batch_load_from_storage( - symbols, start, end, - max_workers=8 # Faster on modern hardware -) - -# Fewer workers for HDD (avoid thrashing) -df = manager.batch_load_from_storage( - symbols, start, end, - max_workers=2 # Better for rotational disks -) -``` - ---- - -## Examples - -### Complete Workflow Example - -```python -from ml4t.data import DataManager -from ml4t.data.storage import HiveStorage, StorageConfig -from ml4t.data.universe import Universe -from ml4t.data.utils.format import pivot_to_wide -import polars as pl - -# Setup -storage = HiveStorage(StorageConfig(base_path="./data")) -manager = DataManager(storage=storage) - -# 1. Initial load (populate storage) -print("Loading S&P 500 data...") -df = manager.batch_load_universe( - universe='SP500', - start='2024-01-01', - end='2024-12-31', - provider='yahoo', - max_workers=8 -) -print(f"Loaded {len(df):,} rows for {df['symbol'].n_unique()} symbols") - -# 2. Subsequent analysis (fast cache loading) -print("\nReloading from cache...") -df = manager.batch_load_from_storage( - symbols=Universe.SP500, - start='2024-01-01', - end='2024-12-31' -) -print(f"Loaded in <1 second from cache") - -# 3. Calculate returns -df = df.with_columns( - pl.col('close').pct_change().over('symbol').alias('returns') -) - -# 4. Cross-sectional analysis -daily_stats = df.group_by('timestamp').agg([ - pl.col('returns').mean().alias('mean_return'), - pl.col('returns').std().alias('std_return'), - pl.col('returns').quantile(0.9).alias('p90_return'), - pl.col('volume').sum().alias('total_volume'), -]) - -print("\nDaily statistics:") -print(daily_stats.head()) - -# 5. Correlation analysis (use wide format) -df_wide = pivot_to_wide(df, value_cols=['returns']) -df_pandas = df_wide.to_pandas().set_index('timestamp') -correlation = df_pandas.corr() - -print(f"\nCorrelation matrix: {correlation.shape}") - -# 6. Screen for winners -winners = df.group_by('symbol').agg([ - (pl.col('close').last() / pl.col('close').first() - 1).alias('total_return') -]).filter(pl.col('total_return') > 0.2) - -print(f"\nSymbols with >20% returns: {len(winners)}") -print(winners.sort('total_return', descending=True).head()) -``` - ---- - -## Summary - -**Multi-asset support in ml4t-data provides**: - -✅ **Efficient loading**: 10-100x faster with storage caching -✅ **Convenient universes**: Pre-defined symbol lists (S&P 500, etc.) -✅ **Scalable format**: Stacked format handles 500+ symbols -✅ **Flexible conversion**: Wide format when needed -✅ **Production-ready**: Comprehensive validation and error handling - -**Get started**: -1. Use `batch_load()` for initial data fetch -2. Use `batch_load_from_storage()` for fast subsequent access -3. Leverage pre-defined universes for common use cases -4. Stick with stacked format for best performance - -**For more help**: -- Examples: `/examples/multi_asset_quickstart.py` -- Tests: `/tests/test_multi_asset.py` -- Performance: `/PERFORMANCE_BENCHMARKS.md` diff --git a/docs/overrides/main.html b/docs/overrides/main.html index 1fd85f1d..122d044e 100644 --- a/docs/overrides/main.html +++ b/docs/overrides/main.html @@ -2,6 +2,9 @@ {% block extrahead %} {{ super() }} + + + list[tuple]: - """ - Detect gaps in time series. - - Returns list of (gap_start, gap_end, missing_bars) tuples. - """ - expected_gap = { - "daily": timedelta(days=1), - "1h": timedelta(hours=1), - "1m": timedelta(minutes=1), - }[frequency] - - max_gap = { - "daily": timedelta(days=4), # Weekends - "1h": timedelta(hours=2), - "1m": timedelta(minutes=5), - }[frequency] - - gaps = [] - for i in range(1, len(timestamps)): - actual_gap = timestamps[i] - timestamps[i-1] - if actual_gap > max_gap: - gaps.append((timestamps[i-1], timestamps[i], actual_gap)) - - return gaps -``` - -### Automatic Gap Filling - -When gaps are detected: - -1. Log gap details for review -2. Optionally auto-fill with BACKFILL strategy -3. Mark filled gaps in metadata - ---- - -## CLI Commands - -```bash -# Incremental update (default) -ml4t-data update-all -c config.yaml - -# Specific dataset -ml4t-data update-all -c config.yaml --dataset sp500 - -# Dry run (preview) -ml4t-data update-all -c config.yaml --dry-run - -# Force full refresh -ml4t-data update-all -c config.yaml --strategy full_refresh - -# Backfill gaps -ml4t-data update-all -c config.yaml --strategy backfill - -# List stored data -ml4t-data list -c config.yaml -``` - ---- - -## Concurrent Updates - -### File Locking - -Prevent concurrent writes to same partition: - -```python -from ml4t.data.utils.locking import FileLock - -with FileLock(partition_path): - # Safe to write - storage.write(df, ...) -``` - -### Parallel Symbol Updates - -Update multiple symbols concurrently: - -```python -from concurrent.futures import ThreadPoolExecutor - -def update_symbol(symbol): - provider = YahooFinanceProvider() - df = provider.fetch_ohlcv(symbol, start, end) - storage.write(df, provider="yahoo", frequency="daily", symbol=symbol) - provider.close() - return symbol - -with ThreadPoolExecutor(max_workers=5) as executor: - results = list(executor.map(update_symbol, symbols)) -``` - -### Rate Limiting - -Respect provider rate limits: - -```yaml -datasets: - sp500: - provider: yahoo - rate_limit: 2.0 # seconds between calls - max_concurrent: 3 # parallel requests -``` - ---- - -## Update Scheduling - -### Cron Examples - -```bash -# Daily update after market close (6 PM ET) -0 18 * * 1-5 ml4t-data update-all -c config.yaml >> logs/update.log 2>&1 - -# Weekly full refresh (Sunday 2 AM) -0 2 * * 0 ml4t-data update-all -c config.yaml --strategy full_refresh - -# Monthly backfill (1st of month, 3 AM) -0 3 1 * * ml4t-data update-all -c config.yaml --strategy backfill -``` - -### Recommended Schedule - -| Frequency | Update Time | Strategy | -|-----------|-------------|----------| -| Daily | 6 PM ET (after market) | INCREMENTAL | -| Weekly | Sunday 2 AM | FULL_REFRESH | -| Monthly | 1st of month | BACKFILL | - ---- - -## Logging and Monitoring - -### Update Logs - -```json -{ - "timestamp": "2024-12-21T18:05:00Z", - "config": "ml4t-data.yaml", - "duration_seconds": 45.2, - "symbols_updated": 500, - "rows_added": 2500, - "gaps_detected": 3, - "gaps_filled": 3, - "errors": [] -} -``` - -### Alerting - -```python -def check_update_health(log_path: str) -> bool: - """Check if update completed successfully.""" - log = json.load(open(log_path)) - - if log["errors"]: - send_alert(f"Update errors: {log['errors']}") - return False - - if log["gaps_detected"] > 10: - send_alert(f"Many gaps detected: {log['gaps_detected']}") - - return True -``` - ---- - -## Best Practices - -### 1. Run Updates After Market Close - -US markets close at 4 PM ET. Wait until 5-6 PM for data to settle: -- Delayed quotes finalize -- After-hours corrections applied -- Provider caches refreshed - -### 2. Use Incremental for Daily, Full Refresh Periodically - -```bash -# Daily (5 days/week) -0 18 * * 1-5 ml4t-data update-all -c config.yaml - -# Weekly full refresh (catches any corrections) -0 2 * * 0 ml4t-data update-all -c config.yaml --strategy full_refresh -``` - -### 3. Monitor Gap Rates - -If gap rate exceeds 1%, investigate: -- Provider issues -- Network problems -- Rate limiting - -### 4. Keep Metadata Backed Up - -Metadata enables incremental updates. Back up regularly: -```bash -tar -czf metadata_backup.tar.gz ~/ml4t-data/**/.metadata.json -``` - ---- - -## Troubleshooting - -### "Symbol not found" Errors - -- Check symbol format (AAPL vs AAPL.US) -- Verify symbol is active (not delisted) -- Check provider-specific requirements - -### Large Gap Counts - -- Provider outage during previous update -- Rate limiting caused timeouts -- Run BACKFILL strategy to fill - -### Slow Updates - -- Reduce `max_concurrent` if rate limited -- Use file-based symbols for 500+ symbols -- Consider switching to faster provider - ---- - -## See Also - -- [Partitioning Guide](PARTITIONING_GUIDE.md) - Storage structure diff --git a/docs/storage/PARTITIONING_GUIDE.md b/docs/storage/PARTITIONING_GUIDE.md deleted file mode 100644 index 5f617889..00000000 --- a/docs/storage/PARTITIONING_GUIDE.md +++ /dev/null @@ -1,277 +0,0 @@ -# Hive Partitioning Guide - -ml4t-data uses Hive-style partitioned Parquet storage for efficient incremental updates and fast queries. - ---- - -## Why Hive Partitioning? - -| Benefit | Description | -|---------|-------------| -| **Partition pruning** | Query only relevant time periods | -| **Incremental writes** | Append new data without rewriting history | -| **Parallel processing** | Process partitions independently | -| **DuckDB/Polars integration** | Native support for partition-aware queries | - ---- - -## Directory Structure - -``` -base_path/ -└── {provider}/ - └── {frequency}/ - └── year={YYYY}/ - └── month={MM}/ - └── {symbol}/ - └── data.parquet -``` - -### Example - -``` -~/ml4t-data/ -├── yahoo/ -│ └── daily/ -│ ├── year=2024/ -│ │ ├── month=01/ -│ │ │ ├── AAPL/data.parquet -│ │ │ ├── MSFT/data.parquet -│ │ │ └── GOOGL/data.parquet -│ │ ├── month=02/ -│ │ │ └── ... -│ │ └── month=12/ -│ └── year=2023/ -│ └── ... -├── eodhd/ -│ └── daily/ -│ └── ... -└── databento/ - ├── ohlcv-1m/ - │ └── year=2024/ - │ └── month=01/ - │ └── day=15/ # Day partitions for minute data - │ └── ESH5/data.parquet - └── ohlcv-1d/ - └── ... -``` - ---- - -## Auto-Heuristics - -ml4t-data automatically selects partition granularity based on frequency: - -| Frequency | Partition Keys | Rationale | -|-----------|----------------|-----------| -| `daily` | year/month | ~20-22 rows per month per symbol | -| `1h` | year/month | ~160-180 rows per month per symbol | -| `15m` | year/month/day | ~26 rows per day per symbol | -| `5m` | year/month/day | ~78 rows per day per symbol | -| `1m` | year/month/day | ~390 rows per day per symbol | - -### Target File Size - -**Optimal: 50-200MB per Parquet file** - -- Too small (<10MB): Overhead from many files, metadata costs -- Too large (>500MB): Slow to read/write, poor parallelism - -### Symbol Count Considerations - -| Symbol Count | Recommendation | -|--------------|----------------| -| 1-50 | Year/month partitions sufficient | -| 50-500 | Year/month + symbol subdirectory | -| 500+ | Consider additional partitioning or separate configs | - ---- - -## Configuration Override - -While auto-heuristics work for most cases, override via YAML: - -```yaml -storage: - path: ~/ml4t-data - format: hive - -datasets: - # Auto-heuristic (recommended for most cases) - sp500_daily: - provider: yahoo - frequency: daily - symbols_file: sp500.txt - # partition_by: auto # implicit - - # Manual override for minute data - futures_1m: - provider: databento - frequency: 1m - partition_by: - - year - - month - - day - symbols: - - ES - - NQ - - CL -``` - -### Partition Key Options - -| Key | Values | Use Case | -|-----|--------|----------| -| `year` | 2020, 2021, ... | Always include | -| `month` | 01-12 | Most common | -| `day` | 01-31 | Intraday data | -| `symbol` | AAPL, MSFT, ... | Large universes | - ---- - -## Querying Partitioned Data - -### With Polars (Recommended) - -```python -import polars as pl - -# Lazy scan with partition pruning -df = pl.scan_parquet( - "~/ml4t-data/yahoo/daily/**/*/data.parquet", - hive_partitioning=True -).filter( - (pl.col("year") == 2024) & - (pl.col("month") >= 10) & - (pl.col("symbol") == "AAPL") -).collect() - -# Only reads 2024 Q4 partitions for AAPL -``` - -### With DuckDB - -```sql -SELECT - symbol, - date_trunc('month', timestamp) as month, - AVG(close) as avg_close -FROM read_parquet( - '~/ml4t-data/yahoo/daily/**/*/data.parquet', - hive_partitioning=true -) -WHERE year = 2024 AND month >= 10 -GROUP BY 1, 2 -ORDER BY 1, 2 -``` - -### With Storage API - -```python -from ml4t.data.storage.hive import HiveStorage, StorageConfig - -storage = HiveStorage(config=StorageConfig(base_path="~/ml4t-data")) - -# Read specific symbol -df = storage.read(provider="yahoo", frequency="daily", symbol="AAPL") - -# Read all symbols for a provider -df = storage.read_all(provider="yahoo", frequency="daily") -``` - ---- - -## Space Efficiency - -### Compression - -Parquet provides excellent compression for financial data: - -| Data Type | Typical Compression | -|-----------|---------------------| -| Timestamps | 10-20x (dictionary encoding) | -| Symbols | 50-100x (dictionary encoding) | -| Prices | 3-5x (zstd compression) | -| Volume | 5-10x | - -**Overall: 5-10x compression vs CSV** - -### Example Storage Requirements - -| Dataset | Rows | CSV Size | Parquet Size | -|---------|------|----------|--------------| -| 1 symbol, 5 years daily | ~1,250 | 100 KB | 15 KB | -| 500 symbols, 5 years daily | ~625K | 50 MB | 8 MB | -| 500 symbols, 5 years minute | ~390M | 30 GB | 5 GB | - ---- - -## Migration from Flat Files - -If you have existing flat Parquet files: - -```python -from ml4t.data.storage.hive import HiveStorage, StorageConfig -import polars as pl - -# Read existing flat file -df = pl.read_parquet("existing_data.parquet") - -# Setup Hive storage -storage = HiveStorage(config=StorageConfig(base_path="~/ml4t-data")) - -# Migrate each symbol -for symbol in df['symbol'].unique(): - symbol_df = df.filter(pl.col("symbol") == symbol) - storage.write(symbol_df, provider="yahoo", frequency="daily", symbol=symbol) - print(f"Migrated {symbol}: {len(symbol_df)} rows") -``` - ---- - -## Best Practices - -### 1. Consistent Partition Keys - -Always use the standard hierarchy: -``` -provider/frequency/year/month/[day]/symbol/ -``` - -### 2. Don't Over-Partition - -- Daily data: year/month is sufficient -- Only add day partitions for intraday data -- Symbol subdirectories prevent large directories - -### 3. Use Predicate Pushdown - -```python -# Good: Filter pushed to storage layer -pl.scan_parquet(...).filter(pl.col("year") == 2024) - -# Bad: Full scan then filter -pl.read_parquet(...).filter(pl.col("year") == 2024) -``` - -### 4. Monitor Partition Sizes - -```python -from pathlib import Path - -def check_partitions(base_path: str): - """Check partition file sizes.""" - for pq in Path(base_path).rglob("*.parquet"): - size_mb = pq.stat().st_size / 1024 / 1024 - if size_mb > 200: - print(f"Warning: Large partition {pq}: {size_mb:.1f} MB") - elif size_mb < 0.01: - print(f"Warning: Tiny partition {pq}: {size_mb*1024:.1f} KB") -``` - ---- - -## See Also - -- [Incremental Architecture](INCREMENTAL_ARCHITECTURE.md) - Update strategies -- [Provider reference](../providers/index.md) - Provider-specific storage notes diff --git a/docs/tutorials/02_rate_limiting.md b/docs/tutorials/02_rate_limiting.md index 21f6782b..42461f7e 100644 --- a/docs/tutorials/02_rate_limiting.md +++ b/docs/tutorials/02_rate_limiting.md @@ -1,448 +1,77 @@ -# Rate Limiting Best Practices +# Work within provider rate limits -**Target Audience**: Developers building production systems -**Time to Read**: 15 minutes -**Prerequisites**: [Understanding OHLCV Data](01_understanding_ohlcv.md) +Provider quotas vary by account, endpoint, and date. Treat the provider's current documentation and +account dashboard as authoritative. The defaults in `ml4t-data` are conservative client-side +request pacing, not a statement of the account's total quota. -## Why Rate Limiting Matters +## Understand the two limits -API providers limit how often you can call their endpoints to: -1. **Prevent abuse** - Stop malicious actors from overwhelming servers -2. **Ensure fairness** - Give all users equal access -3. **Control costs** - Server capacity costs money +A provider may enforce both: -**Violating rate limits** can result in: -- ⚠️ Temporary IP bans (1 hour to 24 hours) -- 🚫 Permanent API key revocation -- 💸 Unexpected charges (some providers charge overage fees) +- a short-window request rate, such as calls within a minute; and +- a daily, monthly, credit, or cost budget. -## Understanding Rate Limit Types +Client pacing can enforce the first constraint. It cannot infer the second from elapsed time, so a +recurring workflow must also track its account usage and requested data volume. -### 1. Requests Per Minute (RPM) +## Use provider defaults -Most common limit type: +Providers derived from `BaseProvider` initialize a shared rate limiter. Instances with the same +provider name share that limiter within the process, which prevents separate objects from each +using the complete short-window allowance. ```python -# Finnhub free tier: 60 calls per minute -provider = FinnhubProvider(api_key="key") +from ml4t.data.providers import SyntheticProvider -# Calls up to the configured rate succeed immediately -for i in range(60): - data = provider.fetch_ohlcv("AAPL", start, end) # ✅ Success - -# Additional calls block until capacity is available -data = provider.fetch_ohlcv("MSFT", start, end) # ⏳ Waits ~60 seconds -``` - -### 2. Requests Per Day (RPD) - -Daily quota that resets at midnight (UTC): - -```python -# Tiingo: 1000 calls per day -provider = TiingoProvider(api_key="key") - -# After 1000 calls today: -data = provider.fetch_ohlcv("AAPL", start, end) -# ❌ RateLimitError: Daily limit exceeded. Resets at 2024-01-15 00:00:00 UTC -``` - -### 3. Endpoint and Plan Quotas - -Some providers apply different limits by endpoint, market, or subscription -plan. For example, free tiers may allow daily OHLCV but restrict intraday, -fundamentals, or bulk endpoints. Treat provider dashboards and HTTP 429 -responses as the source of truth for current usage. - -### 4. Concurrent Connections - -Maximum simultaneous requests: - -```python -# Some providers limit to 1-5 concurrent connections -# Exceeded limit → 429 Too Many Requests error - -# BAD: Parallel requests without limiting -with ThreadPoolExecutor(max_workers=20) as executor: # ❌ Too many! - futures = [executor.submit(fetch, symbol) for symbol in symbols] -``` - -## ML4T Data's Built-In Rate Limiting - -Every ML4T Data provider has **automatic rate limiting** built-in. You don't need to implement it yourself! - -### How It Works - -```python -from ml4t.data.providers import TiingoProvider - -# Rate limiter configured based on provider's limits -provider = TiingoProvider(api_key="key") -# Tiingo allows 1000 calls/day → ML4T Data automatically enforces this - -# Make requests normally - ML4T Data handles rate limiting -for symbol in ["AAPL", "MSFT", "GOOGL", "TSLA", "AMZN"]: - data = provider.fetch_ohlcv(symbol, "2024-01-01", "2024-01-31") - # ✅ ML4T Data automatically paces requests to respect rate limits -``` - -**Behind the scenes**: -1. ML4T Data tracks calls across **all instances** (global rate limiter) -2. Automatically **blocks** when limit reached -3. **Resumes** when rate limit window resets -4. **Logs** rate limit events for debugging - -### Global Rate Limiting - -ML4T Data uses **global rate limiters** shared across all provider instances: - -```python -# Even with multiple instances, rate limits are shared -provider1 = TiingoProvider(api_key="key") -provider2 = TiingoProvider(api_key="key") # Same API key - -# First instance makes 500 calls -for i in range(500): - provider1.fetch_ohlcv(f"SYMBOL{i}", start, end) - -# Second instance respects same limit (500 remaining out of 1000) -for i in range(600): # Tries to make 600 calls - provider2.fetch_ohlcv(f"OTHER{i}", start, end) - # After 500 calls, ML4T Data blocks until daily reset -``` - -**Why global?** Prevents accidental limit violations when running multiple scripts/processes. - -## Rate Limit Configuration - -### Default Limits - -Each provider has sensible defaults: - -```python -# Providers have DEFAULT_RATE_LIMIT class variable -TiingoProvider.DEFAULT_RATE_LIMIT # (1000, 86400.0) = 1000 per day -TwelveDataProvider.DEFAULT_RATE_LIMIT # 800 per day on the free tier -CoinGeckoProvider.DEFAULT_RATE_LIMIT # (50, 60.0) = 50 per minute -``` - -### Custom Limits (Advanced) - -Override limits for testing or paid tiers: - -```python -# Example: You upgraded Tiingo to paid tier (20,000/hour) -provider = TiingoProvider( - api_key="key", - rate_limit=(20000, 3600.0) # (max_calls, period_seconds) -) - -# Or set very conservative limit for testing -provider = TwelveDataProvider( - api_key="key", - rate_limit=(3, 60.0) # Only 3/min for cautious testing -) -``` - -## Best Practices - -### 1. Use Incremental Updates - -**Don't refetch all data every time!** - -```python -# ❌ BAD: Refetch entire history daily -for symbol in symbols: - data = provider.fetch_ohlcv(symbol, "2020-01-01", "2024-12-31") - # 1000+ days × 100 symbols = 100,000+ API calls! - -# ✅ GOOD: Incremental updates -from ml4t.data.storage.hive import HiveStorage -from ml4t.data.storage.backend import StorageConfig -from ml4t.data.update_manager import IncrementalUpdater - -storage = HiveStorage(StorageConfig(base_path="./data")) -updater = IncrementalUpdater() - -for symbol in symbols: - data = provider.fetch_ohlcv(symbol, start=last_stored_date, end=today) - updater.update_incremental(storage, tracker, f"{symbol}/daily", data, provider="tiingo") - # Fetch and merge only the missing date range. -``` - -### 2. Batch Requests When Possible - -Some providers support batch requests: - -```python -# ❌ BAD: Individual requests -for symbol in ["AAPL", "MSFT", "GOOGL"]: - quote = provider.fetch_quote(symbol) # 3 API calls - -# ✅ GOOD: Batch request (if provider supports it) -quotes = provider.fetch_quote(["AAPL", "MSFT", "GOOGL"]) # 1 API call -``` - -**Providers with batch support**: -- **Yahoo Finance**: `fetch_batch_ohlcv(["AAPL", "MSFT", ...], ...)` -- **Twelve Data**: multi-symbol endpoints where supported by plan - -### 3. Cache Aggressively - -Don't fetch the same data multiple times in one session: - -```python -from functools import lru_cache -from datetime import datetime - -@lru_cache(maxsize=1000) -def get_ohlcv_cached(symbol, start, end): - """Cache OHLCV data for the session.""" - return provider.fetch_ohlcv(symbol, start, end) - -# First call: Fetches from API -data1 = get_ohlcv_cached("AAPL", "2024-01-01", "2024-12-31") # API call - -# Second call: Returns cached result -data2 = get_ohlcv_cached("AAPL", "2024-01-01", "2024-12-31") # Instant, no API call -``` - -### 4. Schedule During Off-Peak Hours - -Many rate limits reset at specific times: - -```python -# Daily limits often reset at midnight UTC -# Run data updates at 00:05 UTC to get fresh quota - -import schedule -import time - -def daily_update(): - """Run at 00:05 UTC every day.""" - for symbol in symbols: - updater.update_symbol(symbol, incremental=True) - -schedule.every().day.at("00:05").do(daily_update) - -while True: - schedule.run_pending() - time.sleep(60) -``` - -### 5. Prioritize Important Symbols - -If you hit rate limits, fetch high-priority symbols first: - -```python -# Prioritize by market cap or portfolio weight -symbols_priority = [ - "AAPL", # High priority - "MSFT", - "GOOGL", - # ... 50 more ... - "SMALLCAP123", # Low priority -] - -# Fetch in order - high-priority symbols get fetched first -for symbol in symbols_priority: - try: - updater.update_symbol(symbol, incremental=True) - except RateLimitError: - logger.warning(f"Rate limit hit at symbol: {symbol}") - break # Stop here, resume tomorrow -``` - -## Monitoring Rate Limit Usage - -### 1. Check Provider Dashboard - -Most providers have usage dashboards: -- **Tiingo**: https://api.tiingo.com/account/usage -- **EODHD**: https://eodhd.com/cp/settings -- **Finnhub**: https://finnhub.io/dashboard - -### 2. Log Rate Limit Events - -ML4T Data automatically logs rate limit events: - -```python -import logging -logging.basicConfig(level=logging.INFO) - -# ML4T Data will log: -# INFO: Rate limit: 950/1000 calls used (95%) -# WARNING: Rate limit: 990/1000 calls used (99%) -# WARNING: Rate limit exceeded, blocking until reset at 2024-01-15 00:00:00 UTC +provider = SyntheticProvider(seed=42) +data = provider.fetch_ohlcv("SYNTH", "2024-01-01", "2024-01-10", "daily") ``` -### 3. Implement Custom Tracking +The synthetic provider performs no network request. Network-backed providers acquire their limiter +before issuing a request. -```python -import time +## Override pacing when the account requires it -class RateLimitTracker: - def __init__(self, provider, max_calls_per_day): - self.provider = provider - self.max_calls = max_calls_per_day - self.calls_made = 0 - self.reset_time = time.time() + 86400 - - def fetch_with_tracking(self, symbol, start, end): - if time.time() > self.reset_time: - self.calls_made = 0 - self.reset_time = time.time() + 86400 - - if self.calls_made >= self.max_calls: - raise RateLimitError(f"Daily limit reached: {self.calls_made}/{self.max_calls}") - - data = self.provider.fetch_ohlcv(symbol, start, end) - self.calls_made += 1 - return data - -tracker = RateLimitTracker(provider, max_calls_per_day=1000) -``` - -## Handling Rate Limit Errors - -### Graceful Degradation +A provider constructor accepts `rate_limit=(calls, period_seconds)` through `BaseProvider`: ```python -from ml4t.data.core.exceptions import RateLimitError -import time +from ml4t.data.providers import YahooFinanceProvider -def fetch_with_retry(provider, symbol, start, end, max_retries=3): - """Retry with exponential backoff on rate limit errors.""" - for attempt in range(max_retries): - try: - return provider.fetch_ohlcv(symbol, start, end) - except RateLimitError as e: - if attempt < max_retries - 1: - wait_time = 2 ** attempt * 60 # 1min, 2min, 4min - logger.warning(f"Rate limit hit, waiting {wait_time}s...") - time.sleep(wait_time) - else: - raise # Give up after max_retries +provider = YahooFinanceProvider(rate_limit=(10, 60.0)) ``` -### Circuit Breaker Pattern - -ML4T Data has built-in circuit breakers: - -```python -# After 5 consecutive failures, circuit breaker opens -# Provider stops making requests for 5 minutes -# After 5 minutes, tries one request (half-open state) -# If successful, circuit closes and normal operation resumes - -try: - data = provider.fetch_ohlcv("AAPL", start, end) -except CircuitBreakerOpenError: - logger.error("Circuit breaker open - provider unavailable") - # Fall back to cached data or secondary provider -``` - -## Provider-Specific Tips - -### Finnhub (60/min free tier) - -**Useful for low-volume quote and metadata workflows.** Historical OHLCV is -limited on the free tier, so use paid access for backtesting workloads. - -```python -# With 60 calls/minute, use rate limiting and retries for batch jobs. - -# Strategy: Update a small watchlist and persist results. -for symbol in watchlist_symbols: - updater.update_symbol(symbol, incremental=True) -``` - -### Tiingo (1000/day) - -**Generous free tier** - Good for daily portfolio updates. - -```python -# 1000 calls/day allows: -# - 1000 symbols × daily updates -# - 333 symbols × 3 frequencies (daily, weekly, monthly) -# - 200 symbols × 5 data types (OHLCV, news, fundamentals, etc.) - -# Strategy: Update full portfolio daily + fetch news for top holdings -for symbol in portfolio_symbols: # 200 symbols - updater.update_symbol(symbol, incremental=True) # 200 calls - -for symbol in top_holdings[:100]: # 100 symbols - news = provider.fetch_news(symbol) # 100 calls - -# Total: 300/1000 calls used (30%) -``` - -### EODHD (plan-dependent daily quota) - -**Good for global daily equities.** Free and paid plans differ substantially in -daily quota and historical depth. - -```python -# Strategy: Prefer incremental daily updates over full re-downloads. -for symbol in global_symbols: - updater.update_symbol(symbol, incremental=True) -``` - -### EODHD (20 calls/day free; paid quotas vary) - -**Most generous free tier** for global stocks. - -```python -# 500 calls/day allows: -# - 500 symbols × daily updates (global exchanges!) -# - 166 symbols × 3 frequencies -# - 100 symbols × 5 exchanges (if multi-exchange strategy) - -# Strategy: Use free tier for testing, upgrade for production -# Check the current plan quota before choosing a batch size. -``` - -## Troubleshooting - -### "RateLimitError: Too many requests" - -**Cause**: Exceeded provider's rate limit -**Solution**: -1. Check current usage in provider dashboard -2. Implement incremental updates (reduce API calls 100x) -3. Space out requests over multiple days -4. Consider upgrading to paid tier +This example permits ten acquisitions in a rolling 60-second period. It does not grant provider +access or increase an external quota. Choose the value from the current provider terms for the +account and endpoint in use. -### "HTTP 429: Too Many Requests" +## Reduce avoidable requests -**Cause**: Provider's server rejected request due to rate limiting -**Solution**: -- ML4T Data should handle this automatically -- If you see this, report as bug (ML4T Data rate limiter should prevent it) +Use the storage-backed workflow in [incremental updates](03_incremental_updates.md) instead of +requesting complete history repeatedly. It determines the missing date range before calling the +provider. -### "Blocked for 24 hours" +Before scheduling a large universe: -**Cause**: Severely exceeded rate limits (usually manual curl/script) -**Solution**: -- Wait for ban to expire -- Use ML4T Data's built-in rate limiting going forward -- Contact provider support if ban doesn't lift +1. Confirm the provider's current quota and billing unit. +2. Test one symbol at the required frequency and date range. +3. Measure how the provider paginates that request. +4. Estimate total requests or credits for the universe. +5. Store successful responses and request only missing ranges. +6. Leave capacity for retries and operational checks. -## Summary +## Handle failures by type -**Key Takeaways**: -1. ✅ ML4T Data handles rate limiting automatically - just use the providers normally -2. ✅ Use incremental updates to reduce API calls by 100-1000x -3. ✅ Global rate limiters prevent accidental violations across processes -4. ✅ Monitor usage and prioritize important symbols -5. ✅ Free tiers are generous for research, but production needs paid plans +A rate-limit response is different from authentication failure, unavailable data, or a transient +network error. Catch `RateLimitError` when the workflow can reschedule from its `retry_after` +value. Treat `AuthenticationError` and `DataNotAvailableError` as input or account problems. Retry +`NetworkError` only when its `retryable` attribute permits it. -**Next Steps**: -- [Tutorial 03: Incremental Updates](03_incremental_updates.md) - Learn to minimize API calls -- [Tutorial 04: Data Quality Validation](04_data_quality.md) - Ensure data integrity -- [Provider Selection Guide](../getting-started/provider-selection.md) - Choose the right rate limits +Do not turn authentication or unavailable-data errors into automatic retries. Repeating those +requests consumes quota without changing the outcome. The [exception reference](../api/index.md) +defines these error types. ---- +## Continue -**Previous Tutorial**: [01: Understanding OHLCV Data](01_understanding_ohlcv.md) -**Next Tutorial**: [03: Incremental Updates](03_incremental_updates.md) +- [Incremental updates](03_incremental_updates.md) +- [Provider selection](../getting-started/provider-selection.md) +- [Provider reference](../providers/index.md) diff --git a/docs/tutorials/03_incremental_updates.md b/docs/tutorials/03_incremental_updates.md index ca9dda2c..a4f4880f 100644 --- a/docs/tutorials/03_incremental_updates.md +++ b/docs/tutorials/03_incremental_updates.md @@ -579,7 +579,7 @@ print(f"Incremental: {incremental_time:.1f}s") # Incremental: 34.2s (34 seconds) print(f"Speedup: {naive_time / incremental_time:.1f}x faster") -# Speedup: 10.0x faster +# Compare elapsed times in your environment ``` ## Summary diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index 337c0092..d1f59c90 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -111,7 +111,7 @@ not mutate the process environment: ```yaml env: - ML4T_DATA_DIR: ~/ml4t-data + ML4T_DATA_PATH: ~/ml4t-data DEFAULT_PROVIDER: yahoo ``` diff --git a/docs/user-guide/getting-started.md b/docs/user-guide/getting-started.md deleted file mode 100644 index 7f8113fe..00000000 --- a/docs/user-guide/getting-started.md +++ /dev/null @@ -1,366 +0,0 @@ -# Getting Started with ML4T Data - -Welcome to ML4T Data (Quantitative Ledger Data Manager)! This guide will help you get up and running quickly. - -## Installation - -### System Requirements - -- Stable CPython 3.12 through 3.14 -- 4GB RAM minimum (8GB recommended) -- 10GB free disk space for data storage -- Internet connection for data fetching - -### Install from Source - -```bash -# Clone the repository -git clone https://github.com/yourusername/ml4t-data.git -cd ml4t-data - -# Create a virtual environment (recommended) -python -m venv venv -source venv/bin/activate # On Windows: venv\Scripts\activate - -# Install the package -pip install -e . - -# For development tools -pip install -e ".[dev]" -``` - -### Verify Installation - -```bash -# Check CLI is working -ml4t-data --version - -# View available commands -ml4t-data --help -``` - -## First Steps - -### 1. Initialize Configuration - -Create your configuration directory and default config: - -```bash -ml4t-data init -``` - -This creates: -- `~/.ml4t-data/` - Main configuration directory -- `~/.ml4t-data/config.yaml` - Configuration file -- `~/.ml4t-data/data/` - Data storage directory - -### 2. Configure Data Providers - -Edit `~/.ml4t-data/config.yaml` to set up your providers: - -```yaml -providers: - yahoo: - enabled: true - rate_limit: 10 - - binance: - enabled: true - # Optional API credentials for higher limits - api_key: ${BINANCE_API_KEY} - api_secret: ${BINANCE_SECRET} - - cryptocompare: - enabled: true - api_key: ${CRYPTOCOMPARE_API_KEY} # Free tier available -``` - -### 3. Fetch Your First Data - -#### Stock Data (Yahoo Finance) - -```bash -# Fetch Apple stock data for 2024 -ml4t-data fetch AAPL --provider yahoo --start 2024-01-01 - -# Fetch multiple symbols -ml4t-data fetch AAPL MSFT GOOGL --provider yahoo - -# Fetch with specific date range -ml4t-data fetch AAPL --start 2024-01-01 --end 2024-06-30 -``` - -#### Cryptocurrency Data (Binance) - -```bash -# Fetch Bitcoin/USDT hourly data -ml4t-data fetch BTC/USDT --provider binance --frequency 1h - -# Fetch daily data -ml4t-data fetch ETH/USDT --provider binance --frequency 1d - -# Fetch minute data (last 7 days) -ml4t-data fetch BTC/USDT --provider binance --frequency 1m --start 2024-01-01 -``` - -### 4. View and Export Data - -#### List Available Data - -```bash -# Show all stored symbols -ml4t-data list - -# Show details for a specific symbol -ml4t-data info AAPL -``` - -#### Export Data - -```bash -# Export to CSV -ml4t-data export AAPL --format csv --output aapl_2024.csv - -# Export to Excel with date range -ml4t-data export AAPL --format excel --start 2024-01-01 --end 2024-06-30 - -# Export to Parquet (efficient format) -ml4t-data export AAPL --format parquet --output aapl.parquet -``` - -### 5. Update Existing Data - -Keep your data current with incremental updates: - -```bash -# Update single symbol -ml4t-data update AAPL - -# Update multiple symbols -ml4t-data update AAPL MSFT GOOGL - -# Update all symbols from a provider -ml4t-data update --provider yahoo --all -``` - -## Python API Usage - -### Basic Example - -```python -from ml4t-data import ML4T Data -import polars as pl - -# Initialize ML4T Data -ml4t-data = ML4T Data() - -# Fetch data -df = ml4t-data.fetch( - symbol="AAPL", - provider="yahoo", - start="2024-01-01", - end="2024-12-31" -) - -# Display first few rows -print(df.head()) - -# Basic analysis -print(f"Average close price: ${df['close'].mean():.2f}") -print(f"Maximum high: ${df['high'].max():.2f}") -print(f"Total volume: {df['volume'].sum():,.0f}") -``` - -### Working with Multiple Symbols - -```python -from ml4t-data import ML4T Data - -ml4t-data = ML4T Data() - -# Fetch multiple symbols -symbols = ["AAPL", "MSFT", "GOOGL"] -data = {} - -for symbol in symbols: - data[symbol] = ml4t-data.fetch(symbol, start="2024-01-01") - print(f"Fetched {len(data[symbol])} rows for {symbol}") - -# Compare performance -for symbol, df in data.items(): - returns = ((df['close'][-1] - df['close'][0]) / df['close'][0] * 100) - print(f"{symbol}: {returns:.2f}% return") -``` - -### Data Validation - -```python -from ml4t-data import ML4T Data -from ml4t.data.validation import OHLCVValidator - -ml4t-data = ML4T Data() -validator = OHLCVValidator() - -# Fetch and validate data -df = ml4t-data.fetch("AAPL", start="2024-01-01") -result = validator.validate(df) - -if result.passed: - print("✅ Data validation passed") -else: - print("❌ Data validation failed:") - for issue in result.issues: - print(f" - {issue.severity}: {issue.message}") -``` - -## Common Workflows - -### Daily Market Data Update - -Create a script to update all your symbols daily: - -```python -#!/usr/bin/env python -"""Daily market data update script""" - -from ml4t-data import ML4T Data -import logging - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -def daily_update(): - ml4t-data = ML4T Data() - - # List of symbols to track - symbols = [ - "AAPL", "MSFT", "GOOGL", "AMZN", # Tech stocks - "JPM", "BAC", "GS", # Banks - "BTC/USDT", "ETH/USDT" # Crypto - ] - - for symbol in symbols: - try: - provider = "binance" if "/" in symbol else "yahoo" - ml4t-data.update(symbol, provider=provider) - logger.info(f"✅ Updated {symbol}") - except Exception as e: - logger.error(f"❌ Failed to update {symbol}: {e}") - -if __name__ == "__main__": - daily_update() -``` - -### Data Quality Check - -Monitor your data quality: - -```python -from ml4t-data import ML4T Data -from ml4t.data.validation import OHLCVValidator, CrossValidator - -def check_data_quality(symbol: str): - ml4t-data = ML4T Data() - df = ml4t-data.get(symbol) - - # Basic validation - validator = OHLCVValidator() - result = validator.validate(df) - - # Cross validation - cross_validator = CrossValidator() - cross_result = cross_validator.validate(df) - - # Report - print(f"\nData Quality Report for {symbol}") - print("=" * 50) - print(f"Total rows: {len(df)}") - print(f"Date range: {df['timestamp'].min()} to {df['timestamp'].max()}") - print(f"Basic validation: {'✅ Passed' if result.passed else '❌ Failed'}") - print(f"Cross validation: {'✅ Passed' if cross_result.passed else '❌ Failed'}") - - if not result.passed: - print("\nIssues found:") - for issue in result.issues[:5]: # Show first 5 issues - print(f" - {issue.severity}: {issue.message}") - -check_data_quality("AAPL") -``` - -## Troubleshooting - -### Common Issues and Solutions - -#### Rate Limiting Errors - -**Problem**: "Rate limit exceeded" errors - -**Solution**: Reduce the rate limit in your config: -```yaml -providers: - yahoo: - rate_limit: 5 # Reduce from default 10 -``` - -#### Memory Issues with Large Datasets - -**Problem**: High memory usage when processing large datasets - -**Solution**: Use chunked processing: -```python -from ml4t.data.performance import DataFrameOptimizer - -optimizer = DataFrameOptimizer() -df = optimizer.optimize_memory(df) # Reduces memory by 40-60% -``` - -#### Missing Data Periods - -**Problem**: Gaps in historical data - -**Solution**: Use the gap detection and filling: -```bash -# Detect gaps -ml4t-data gaps AAPL - -# Fill gaps automatically -ml4t-data update AAPL --fill-gaps -``` - -#### Slow Queries - -**Problem**: Slow data retrieval - -**Solution**: Enable caching: -```python -from ml4t.data.performance import cache_result - -@cache_result(ttl_seconds=300) -def get_data(symbol): - return ml4t-data.get(symbol) -``` - -## Best Practices - -1. **Regular Updates**: Set up a cron job or scheduled task for daily updates -2. **Data Validation**: Always validate data after fetching -3. **Error Handling**: Implement proper error handling in production scripts -4. **Backup**: Regularly backup your data directory -5. **Monitor Storage**: Keep an eye on disk usage -6. **Use Appropriate Frequencies**: Don't fetch minute data if you only need daily - -## Next Steps - -- Read the [Configuration Guide](configuration.md) for advanced settings -- Check the [CLI Reference](cli-reference.md) for all commands -- Explore the [example notebooks](https://github.com/ml4t/data/tree/main/examples/notebooks) - for analysis workflows -- Learn about [provider development](../contributing/creating-a-provider.md) to add custom - data sources - -## Getting Help - -- 📖 Documentation: https://yourusername.github.io/ml4t-data/ -- 🐛 Report Issues: https://github.com/yourusername/ml4t-data/issues -- 💬 Discussions: https://github.com/yourusername/ml4t-data/discussions -- 📧 Email: support@example.com diff --git a/docs/user-guide/incremental-updates.md b/docs/user-guide/incremental-updates.md index e4871028..da5c2bce 100644 --- a/docs/user-guide/incremental-updates.md +++ b/docs/user-guide/incremental-updates.md @@ -307,9 +307,9 @@ ml4t-data update -s AAPL --show-status ### Update Speed -- Incremental updates are 10-100x faster than full loads -- Gap detection adds minimal overhead (<1 second) -- File locking has negligible performance impact +- Incremental updates request only missing date ranges +- Gap detection runs before provider requests +- File locking prevents concurrent writers from replacing each other's data ## Future Enhancements diff --git a/docs/user-guide/storage.md b/docs/user-guide/storage.md index 6f21bef3..643ae849 100644 --- a/docs/user-guide/storage.md +++ b/docs/user-guide/storage.md @@ -153,7 +153,7 @@ Date ranges use half-open intervals: `start_date` is inclusive and `end_date` is exclusive. This convention is consistent across Flat, Hive, and Chunked storage. With Hive storage, date filters prune entire partition directories before -any Parquet file is opened, giving measured 7x speedup on typical queries. +any Parquet file is opened, avoiding reads from partitions outside the requested range. ### Other Operations diff --git a/docs/wiki_prices_status.md b/docs/wiki_prices_status.md deleted file mode 100644 index 80cbb772..00000000 --- a/docs/wiki_prices_status.md +++ /dev/null @@ -1,417 +0,0 @@ -# Wiki Prices Dataset Status Report - -**Report Date**: 2025-11-24 -**Dataset**: Quandl WIKI Prices (Community-curated US equities EOD prices) -**Provider**: NASDAQ Data Link (formerly Quandl) - -## Executive Summary - -The **Wiki Prices dataset is DEPRECATED and no longer maintained** as of April 11, 2018. While the historical data remains accessible through multiple channels (NASDAQ Data Link API, Kaggle mirror, local archives), no new data is being added. The dataset provides high-quality historical coverage from 1962-2018 for 3,199 US stocks, making it valuable for historical research but unsuitable for live trading or current market analysis. - -**Recommendation**: Use Wiki Prices for historical backtesting and research (pre-2018), but implement fallback to Yahoo Finance, EODHD, or other active providers for recent data and ongoing workflows. - ---- - -## Current Availability - -### 1. NASDAQ Data Link (Official Source) - -**Status**: 🟡 Available (historical only) -**URL**: https://data.nasdaq.com/data/WIKI-wiki-eod-stock-prices -**Access Method**: Free account required -**API Endpoint**: `https://data.nasdaq.com/api/v3/datatables/WIKI/PRICES` - -**Limitations**: -- Data frozen at March 27, 2018 -- No updates since April 2018 -- Free tier includes API access -- NASDAQ recommends against using for investment decisions - -### 2. Kaggle Mirror - -**Status**: 🟢 Available (complete archive) -**URL**: https://www.kaggle.com/datasets/marketneutral/quandl-wiki-prices-us-equites -**File Size**: 463.2 MB (compressed) -**Last Updated**: February 2, 2022 (mirror created) -**Access Method**: Kaggle account required - -**Advantages**: -- Single Parquet download -- Complete historical dataset -- No API rate limits -- Snapshot preserved - -### 3. Local ML4T Copy - -**Status**: 🟢 Available (production-ready) -**Location**: User-configured path (e.g., `~/ml4t-data/wiki_prices.parquet`) -**File Size**: 631.7 MB -**Format**: Polars-compatible Parquet - -**Dataset Details**: -``` -Total rows: 15,389,314 -Date range: 1962-01-02 to 2018-03-27 (56 years) -Unique symbols: 3,199 US companies -Columns: 14 (OHLCV + dividends + splits + adjusted prices) -``` - -**Schema**: -| Column | Type | Description | -|---------------|-----------|--------------------------------------| -| ticker | String | Stock symbol | -| date | Datetime | Trading date | -| open | Float64 | Opening price (unadjusted) | -| high | Float64 | High price (unadjusted) | -| low | Float64 | Low price (unadjusted) | -| close | Float64 | Closing price (unadjusted) | -| volume | Float64 | Trading volume (unadjusted) | -| ex-dividend | Float64 | Dividend amount | -| split_ratio | Float64 | Stock split ratio | -| adj_open | Float64 | Adjusted opening price | -| adj_high | Float64 | Adjusted high price | -| adj_low | Float64 | Adjusted low price | -| adj_close | Float64 | Adjusted closing price | -| adj_volume | Float64 | Adjusted volume | - ---- - -## Why Wiki Prices Ended - -### Official Explanation - -From NASDAQ Data Link Help Center ([source](https://help.data.nasdaq.com/article/506-why-does-wiki-prices-only-go-up-to-march-2018)): - -> "One of the main sources of that data is no longer available, and our community volunteers have been unable to find a suitable alternative source." - -### Root Causes - -1. **Data Source Failure**: Primary data provider became inaccessible in early 2018 -2. **Community Maintenance Model**: Volunteer-based curation couldn't scale to replace lost source -3. **Data Quality Degradation**: Missing data and inaccuracies increased without primary source -4. **Quandl Acquisition**: NASDAQ acquired Quandl in December 2018, shifted focus to premium products - -### Timeline - -- **Pre-2018**: Community-curated free dataset with broad cross-sectional coverage -- **April 11, 2018**: NASDAQ/Quandl announced end of updates -- **March 27, 2018**: Last available data point -- **December 2018**: NASDAQ acquired Quandl, Inc. -- **2021**: Quandl rebranded as NASDAQ Data Link -- **2022-Present**: Dataset remains available but deprecated - ---- - -## Future Viability - -### For Historical Research (Pre-2018) - -**Viability**: ✅ **EXCELLENT** - -**Use Cases**: -- Backtesting strategies from 1962-2018 -- Long-term market structure research -- Academic studies requiring survivorship-bias-free data -- Benchmarking against historical performance -- Teaching quantitative finance concepts - -**Advantages**: -- 56 years of high-quality data -- 3,199 stocks (comprehensive cross-section) -- Includes delisted companies (reduces survivorship bias) -- Adjusted prices (splits/dividends) included -- Free and unrestricted access - -**Limitations**: -- No data after March 2018 (7 years outdated) -- Cannot use for live trading validation -- Some tickers may have gaps or quality issues near end-of-life - -### For Current/Ongoing Work - -**Viability**: ❌ **NOT SUITABLE** - -**Reasons**: -- 7-year data gap (2018-2025) -- Cannot analyze COVID-19 market dynamics -- Missing recent IPOs (e.g., TSLA's rise, tech boom) -- Regulatory changes not reflected (SPAC boom, meme stocks) -- No way to validate against current market - -**Required Alternatives**: -1. **Primary**: Yahoo Finance (free, unlimited, 2000-present) -2. **Educational**: EODHD (500 calls/day free, €19.99/mo paid) -3. **Professional**: DataBento, Polygon, Finnhub (paid tiers) - ---- - -## Recommended Integration Strategy - -### Multi-Tier Approach (ML4T Book) - -**Tier 1: Historical Foundation (Pre-2018)** -- **Provider**: Wiki Prices local copy -- **Use Cases**: Chapters 1-5 (foundations, early exercises) -- **Advantages**: Fast, offline, free, comprehensive -- **Date Range**: 1962-01-02 to 2018-03-27 - -**Tier 2: Recent History (2018-2024)** -- **Provider**: Yahoo Finance (primary) + EODHD (fallback) -- **Use Cases**: Chapters 6+ (strategies, ML models, validation) -- **Advantages**: Free or low-cost, current data -- **Date Range**: 2018-01-01 to present - -**Tier 3: Production Quality (Optional)** -- **Provider**: DataBento or Polygon -- **Use Cases**: Advanced readers, production deployment -- **Advantages**: Professional grade, institutional quality -- **Cost**: $9-99+/month - -### Fallback Logic - -```python -def fetch_historical_data(symbol: str, start: str, end: str): - """ - Fetch with intelligent fallback based on date range. - - Strategy: - 1. If end_date <= 2018-03-27: Use Wiki Prices (local) - 2. If start_date >= 2018-03-28: Use Yahoo/EODHD - 3. If range spans 2018: Combine Wiki (pre) + Yahoo (post) - """ - from datetime import datetime - - wiki_cutoff = datetime(2018, 3, 27) - start_dt = datetime.strptime(start, "%Y-%m-%d") - end_dt = datetime.strptime(end, "%Y-%m-%d") - - if end_dt <= wiki_cutoff: - # Pure historical - use Wiki Prices - return load_from_wiki_prices(symbol, start, end) - - elif start_dt > wiki_cutoff: - # Pure recent - use Yahoo/EODHD - try: - return yahoo_provider.fetch_ohlcv(symbol, start, end) - except Exception: - return eodhd_provider.fetch_ohlcv(symbol, start, end) - - else: - # Spans cutoff - combine sources - wiki_data = load_from_wiki_prices(symbol, start, "2018-03-27") - yahoo_data = yahoo_provider.fetch_ohlcv(symbol, "2018-03-28", end) - return pl.concat([wiki_data, yahoo_data]).sort("date") -``` - -### Implementation Roadmap - -**Phase 1: Basic Fallback** (Current Task) -- Document Wiki Prices status (✅ This document) -- Create ingestion utility for local Parquet -- Add fallback logic to book config - -**Phase 2: Notebook Updates** (task-5.4) -- Update Notebook 1 (free_data_quickstart.py) with fallback demo -- Show date-based provider selection -- Demonstrate seamless historical + recent data loading - -**Phase 3: Production Patterns** (Optional) -- Cache hybrid datasets (Wiki pre-2018 + Yahoo post-2018) -- Automate daily updates for recent data -- Health checks for data gaps at 2018 boundary - ---- - -## NASDAQ Recommended Alternatives - -NASDAQ Data Link recommends two **premium** replacements: - -### 1. End of Day US Stock Prices (EOD) - -**Coverage**: Active stocks from major US exchanges -**Date Range**: 1996-present -**Cost**: Premium subscription required -**Advantages**: Current data, maintained by NASDAQ - -### 2. Sharadar Equity Prices (SEP) - -**Coverage**: Active AND delisted stocks -**Date Range**: 1998-present -**Cost**: Premium subscription ($59-199/month) -**Advantages**: Survivorship-bias free, comprehensive fundamentals - -**ML4T Assessment**: Both are excellent but **unnecessarily expensive** for educational use. Yahoo Finance + EODHD provide 95% of the value at 5% of the cost. - ---- - -## Data Quality Assessment - -### Strengths - -✅ **Institutional-Grade Curation**: Community-maintained with professional standards -✅ **Survivorship-Bias Free**: Includes delisted companies -✅ **Adjusted Prices**: Proper handling of splits and dividends -✅ **Long History**: 56 years (1962-2018) valuable for long-term studies -✅ **Comprehensive Coverage**: 3,199 stocks across major exchanges -✅ **Open Data**: Public domain, no usage restrictions - -### Weaknesses - -⚠️ **Frozen Dataset**: No updates since March 2018 -⚠️ **End-of-Life Quality**: Data quality degraded in final months -⚠️ **Volunteer Gaps**: Some stocks may have missing days or errors -⚠️ **No Intraday**: Daily OHLCV only (no minute/tick data) -⚠️ **US-Only**: No international equities - -### Comparison to Modern Alternatives - -| Feature | Wiki Prices | Yahoo Finance | EODHD | DataBento | -|----------------------|-------------|---------------|------------|--------------| -| **Date Range** | 1962-2018 | 2000-present | 1980-pres | 2000-present | -| **Update Frequency** | NONE | Daily | Daily | Real-time | -| **Cost** | Free | Free | €20/mo | $9+/mo | -| **Survivorship** | No bias | Some bias | Some bias | No bias | -| **Quality** | High | Good | High | Professional | -| **Coverage** | 3,199 | 7,000+ | 150,000+ | 50,000+ | -| **Exchanges** | US only | US only | Global | Global | - ---- - -## Technical Integration - -### Loading Wiki Prices (Local Parquet) - -```python -import polars as pl -from pathlib import Path - -# Load from local archive -wiki_file = Path("~/ml4t-data/wiki_prices.parquet").expanduser() -df = pl.read_parquet(wiki_file) - -# Filter by symbol and date -aapl = df.filter( - (pl.col("ticker") == "AAPL") & - (pl.col("date") >= "2010-01-01") & - (pl.col("date") <= "2018-03-27") -).sort("date") - -# Use adjusted close for returns -returns = aapl.select([ - "date", - "adj_close", - pl.col("adj_close").pct_change().alias("return") -]) -``` - -### Creating Provider Wrapper - -```python -from ml4t.data.providers.base import BaseProvider -import polars as pl -from pathlib import Path - -class WikiPricesProvider(BaseProvider): - """ - Provider for local Wiki Prices Parquet file. - - Date Range: 1962-01-02 to 2018-03-27 - Symbols: 3,199 US companies - """ - - def __init__(self, parquet_path: str = None): - super().__init__(rate_limit=None) # Local file, no rate limit - self.parquet_path = parquet_path or Path( - "~/ml4t-data/wiki_prices.parquet" - ).expanduser() - # Load once and cache in memory (632MB is acceptable) - self._data = pl.read_parquet(self.parquet_path) - - @property - def name(self) -> str: - return "wiki_prices" - - def _fetch_and_transform_data( - self, symbol: str, start: str, end: str, frequency: str - ) -> pl.DataFrame: - """ - Extract data for symbol from cached Parquet. - - Note: Frequency parameter ignored - Wiki Prices is daily only. - """ - if frequency != "daily": - raise ValueError(f"Wiki Prices only supports daily frequency, got {frequency}") - - # Filter cached data - filtered = self._data.filter( - (pl.col("ticker") == symbol) & - (pl.col("date") >= start) & - (pl.col("date") <= end) - ).sort("date") - - if filtered.is_empty(): - raise ValueError(f"No data found for {symbol} in Wiki Prices (1962-2018)") - - # Transform to standard OHLCV schema - return filtered.select([ - pl.col("date").alias("timestamp"), - pl.col("adj_open").alias("open"), - pl.col("adj_high").alias("high"), - pl.col("adj_low").alias("low"), - pl.col("adj_close").alias("close"), - pl.col("adj_volume").alias("volume"), - ]) -``` - -### CLI Integration - -```bash -# Add to ml4t-data CLI -ml4t-data fetch AAPL --provider wiki_prices --start 2010-01-01 --end 2018-03-27 - -# Automatic fallback in config -ml4t-data fetch AAPL --start 2010-01-01 --end 2024-11-24 --fallback-strategy auto -# Uses: Wiki Prices (2010-2018) + Yahoo (2018-2024) -``` - ---- - -## Conclusions - -### Key Findings - -1. **Wiki Prices is alive but frozen** - Accessible through multiple channels but no updates since March 2018 -2. **Local copy is production-ready** - 631MB Parquet with 15.4M rows covering 3,199 stocks (1962-2018) -3. **Historical research value remains high** - Excellent for backtesting and academic work pre-2018 -4. **Not suitable for current work** - 7-year gap requires active provider for post-2018 data -5. **Fallback strategy is essential** - Combine Wiki (historical) + Yahoo/EODHD (recent) for complete coverage - -### Recommended Next Steps - -**Immediate** (Phase 5): -- ✅ Status documented (this document) -- ⏭️ Create WikiPricesProvider (task-5.2) -- ⏭️ Implement fallback logic in book config (task-5.3) -- ⏭️ Update Notebook 1 with fallback demo (task-5.4) - -**Future** (Optional): -- Create hybrid cache (Wiki pre-2018 + Yahoo post-2018) -- Add boundary validation at 2018-03-27 -- Document survivorship bias implications -- Benchmark Wiki vs Yahoo data quality overlap (2010-2018) - ---- - -## References - -- [NASDAQ Data Link Wiki Prices Documentation](https://data.nasdaq.com/data/WIKI-wiki-eod-stock-prices/documentation) -- [Why Wiki Prices Ended in March 2018 - NASDAQ Help Center](https://help.data.nasdaq.com/article/506-why-does-wiki-prices-only-go-up-to-march-2018) -- [Kaggle: Quandl WIKI Prices US Equities](https://www.kaggle.com/datasets/marketneutral/quandl-wiki-prices-us-equites) -- [NASDAQ Data Link Blog](https://blog.quantinsti.com/nasdaq-data-link/) -- [QuantConnect: Quandl is now NasdaqDataLink](https://www.quantconnect.com/forum/discussion/12963/quandl-is-now-nasdaqdatalink/) - ---- - -**Document Status**: ✅ Complete -**Next Task**: task-5.2 - Implement Wiki Prices fallback provider -**Phase**: 5 (Wiki Fallback) - 1/4 tasks complete diff --git a/docs/yahoo_bulk_download_analysis.md b/docs/yahoo_bulk_download_analysis.md deleted file mode 100644 index 38a1bbcb..00000000 --- a/docs/yahoo_bulk_download_analysis.md +++ /dev/null @@ -1,228 +0,0 @@ -# Yahoo Finance Bulk Download Analysis - -**Date**: 2025-11-27 -**Status**: Implementation Complete ✅ - -## Executive Summary - -Testing revealed the current ml4t-data YahooFinanceProvider is **22x slower** than batch download approaches for bulk data acquisition. This document analyzes the options and recommends enhancements. - -## Test Results (90 symbols, 5 years of data) - -| Method | Time | ms/symbol | Success Rate | Notes | -|--------|------|-----------|--------------|-------| -| **batch_all** | 3.1s | 35ms | 100% | Single `yf.download()` with all symbols | -| **chunked_25** | 3.7s | 41ms | 100% | 25 symbols/chunk, 0.5s delay | -| **chunked_50** | 3.8s | 42ms | 100% | 50 symbols/chunk, 1.0s delay | -| **ml4t_provider** | 67.5s | 750ms | 97.8% | Current single-symbol approach | - -**Key Finding**: Batch downloads are dramatically faster because: -1. Single HTTP connection for multiple symbols -2. Yahoo Finance API optimized for multi-ticker requests -3. `yf.download()` uses internal threading (`threads=True`) - -## Current Implementations Compared - -### ml4t-data YahooFinanceProvider (Current) - -```python -# Single-symbol download per call -def fetch_ohlcv(self, symbol, start, end, frequency): - df = yf.download(symbol, start=start, end=end, ...) - return self._convert_to_polars(df, symbol) -``` - -**Pros**: -- Simple, per-symbol interface -- OHLC validation built-in -- Polars output for downstream speed -- Integrates with incremental update system - -**Cons**: -- 22x slower for bulk downloads -- Inefficient for initial data acquisition -- Generates many HTTP requests - -### ml3t Notebook Approach - -```python -# Chunked batch downloads -class CachedLimiterSession(CacheMixin, LimiterMixin, Session): - pass - -session = CachedLimiterSession( - limiter=Limiter(RequestRate(2, Duration.SECOND)), - bucket_class=MemoryQueueBucket, - backend=SQLiteCache("yfinance.cache"), -) - -for chunk in chunks(symbols, 100): - prices = yf.download(chunk, start="2000-01-01", end=today, ...) -``` - -**Pros**: -- Fast bulk downloads -- Rate limiting prevents blocks -- Caching reduces duplicate requests - -**Cons**: -- CachedLimiterSession doesn't work with `yf.download()` (only Ticker methods) -- Returns pandas, not Polars -- No OHLC validation -- No incremental update integration - -## Yahoo Finance Rate Limiting (Nov 2024) - -Recent changes from [GitHub Issue #2128](https://github.com/ranaroussi/yfinance/issues/2128): - -- **Threshold**: ~950 symbols before rate limiting kicks in -- **Recommendation**: 2 requests/second for sequential, or chunked batches -- **yfinance 0.2.58+**: TLS fingerprinting fixes help avoid blocks -- **CachedLimiterSession**: Useful for `.info()` metadata calls, NOT for `yf.download()` - -## Recommendations - -### 1. Add Batch Download Method to YahooFinanceProvider - -```python -def fetch_batch_ohlcv( - self, - symbols: list[str], - start: str, - end: str, - frequency: str = "daily", - chunk_size: int = 50, - delay_seconds: float = 1.0, -) -> pl.DataFrame: - """ - Fetch OHLCV data for multiple symbols efficiently. - - Uses chunked batch downloads for speed while maintaining - reliability through rate limiting. - """ - all_data = [] - - for chunk in chunks(symbols, chunk_size): - df_pandas = yf.download( - chunk, - start=start, - end=end, - progress=False, - auto_adjust=True, - threads=True, - ) - - # Convert to long format with symbol column - df_polars = self._convert_batch_to_polars(df_pandas) - all_data.append(df_polars) - - time.sleep(delay_seconds) - - return pl.concat(all_data) -``` - -### 2. Keep Single-Symbol Method for Updates - -The current `fetch_ohlcv()` method is appropriate for: -- Incremental daily updates -- Small symbol counts (<10) -- Integration with UpdateManager - -### 3. Add Initial Data Acquisition Script - -Create `scripts/download_yahoo_universe.py`: -```python -# For book examples and initial setup -provider = YahooFinanceProvider() -df = provider.fetch_batch_ohlcv( - symbols=sp500_symbols, - start="2000-01-01", - end=today, - chunk_size=50, -) -# Save to Hive storage -storage.write(df) -``` - -### 4. Document Both Approaches in Book - -**Notebook (Educational)**: -- Explain batch vs sequential trade-offs -- Show chunking pattern -- Discuss rate limiting - -**Library (Production)**: -- Use `fetch_batch_ohlcv()` for initial acquisition -- Use `fetch_ohlcv()` + UpdateManager for daily updates - -## Implementation Priority - -1. **High**: Add `fetch_batch_ohlcv()` method to YahooFinanceProvider -2. **High**: Create example notebook showing bulk download patterns -3. **Medium**: Add CLI command `ml4t-data download --provider yahoo --symbols-file sp500.txt` -4. **Low**: CachedLimiterSession for metadata (`.info`) downloads - -## Sources - -- [yfinance Rate Limiting Issue #2128](https://github.com/ranaroussi/yfinance/issues/2128) -- [yfinance Rate Limiting Issue #2125](https://github.com/ranaroussi/yfinance/issues/2125) -- [Sling Academy - Rate Limiting Best Practices](https://www.slingacademy.com/article/rate-limiting-and-api-best-practices-for-yfinance/) -- [yfinance Smarter Scraping docs](https://github.com/ranaroussi/yfinance) - -## Implementation Results (2025-11-27) - -### fetch_batch_ohlcv() Method Added - -Added to `src/ml4t/data/providers/yahoo.py`: - -```python -def fetch_batch_ohlcv( - self, - symbols: list[str], - start: str, - end: str, - frequency: str = "daily", - chunk_size: int = 50, - delay_seconds: float = 1.0, -) -> pl.DataFrame: - """Fetch OHLCV data for multiple symbols efficiently.""" -``` - -### Verified Test Results - -| Test | Result | -|------|--------| -| Basic functionality (5 symbols) | ✅ PASSED | -| Speedup (20 symbols) | ✅ 6.5x faster | -| Scale test (100 symbols) | ✅ 100% success rate | -| Data quality | ✅ No violations | - -**Performance**: 100 symbols in 4.1s (41ms/symbol) vs 285ms/symbol with single-symbol method - -### Usage - -```python -from ml4t.data.providers.yahoo import YahooFinanceProvider - -provider = YahooFinanceProvider() - -# Bulk download (fast) -df = provider.fetch_batch_ohlcv( - symbols=["AAPL", "MSFT", "GOOGL", ...], - start="2020-01-01", - end="2024-01-01", - chunk_size=50, - delay_seconds=1.0, -) - -# Single symbol (for updates) -df = provider.fetch_ohlcv("AAPL", "2024-01-01", "2024-01-15", "daily") -``` - -## Test Scripts - -Created during this analysis: -- `scripts/test_bulk_download_approaches.py` - Initial comparison -- `scripts/test_bulk_scale.py` - 50 symbol scale test -- `scripts/test_bulk_final.py` - Comprehensive 90 symbol test -- `scripts/test_batch_ohlcv.py` - Implementation verification diff --git a/examples/01_microstructure_tick_analysis.py b/examples/01_microstructure_tick_analysis.py index e22f4d7f..f100ac7d 100644 --- a/examples/01_microstructure_tick_analysis.py +++ b/examples/01_microstructure_tick_analysis.py @@ -11,7 +11,7 @@ - Detecting toxic order flow and liquidity events - Proper backtesting with microsecond-precision data -Author: QuantLab Team +Author: Stefan Jansen Date: August 2025 """ diff --git a/examples/02_cross_sectional_nasdaq100.py b/examples/02_cross_sectional_nasdaq100.py index 59731739..285a331a 100644 --- a/examples/02_cross_sectional_nasdaq100.py +++ b/examples/02_cross_sectional_nasdaq100.py @@ -13,7 +13,7 @@ - Combinatorial Purged Cross-Validation (CPCV) - Deflated Sharpe Ratio for multiple testing -Author: QuantLab Team +Author: Stefan Jansen Date: August 2025 """ diff --git a/examples/FINAL_STATUS_REPORT.md b/examples/FINAL_STATUS_REPORT.md deleted file mode 100644 index bd2a97e5..00000000 --- a/examples/FINAL_STATUS_REPORT.md +++ /dev/null @@ -1,232 +0,0 @@ -# Final Status Report - Simplification + Test Fixes Complete - -**Date**: 2025-11-16 -**Status**: ALL TASKS COMPLETE ✅ - ---- - -## Mission Accomplished - -### 1. ✅ All 11 Providers Simplified (52% Code Reduction) - -| Provider | Before | After | Removed | Reduction | -|----------|--------|-------|---------|-----------| -| YahooFinance | 321 | 217 | 104 | 32% | -| CoinGecko | 799 | 243 | 556 | 70% | -| OANDA | 561 | 286 | 275 | 49% | -| EODHD | 522 | 296 | 226 | 43% | -| DataBento | 517 | 233 | 284 | 55% | -| Finnhub | 515 | 243 | 272 | 53% | -| TwelveData 🏆 | 493 | 166 | 327 | **66%** | -| Polygon | 483 | 202 | 281 | 58% | -| Tiingo | 440 | 203 | 237 | 54% | -| AlphaVantage | 420 | 240 | 180 | 43% | -| Binance | 348 | 260 | 88 | 25% | -| **TOTAL** | **5,419** | **2,589** | **2,830** | **52%** | - ---- - -### 2. ✅ Tests Fixed and Passing - -**Test Collection**: 1,174 tests collected (no import errors) - -**Test Results**: -- ✅ **Unit tests**: 10/10 passed (100%) -- ✅ **All non-integration tests**: 919/978 passed (94%) -- ⚠️ 59 failures are tests expecting removed features (expected) - -**What was fixed**: -- Removed Updater import dependencies from 3 integration test files -- Updated unit tests for simplified provider interfaces -- Commented out tests for removed non-OHLCV features -- All core provider functionality tests pass - ---- - -### 3. ✅ Quandl Extension Workflow Verified - -**Critical test**: Can the simplified library replace `~/ml3t/data/equities/quandl/` workflow? - -**Answer**: YES ✅ - -**Test results** (`test_quandl_extension.py`): -``` -✅ Provider initialization works -✅ fetch_ohlcv() returns correct schema with symbol column -✅ Date range filtering works -✅ Bulk download works (multiple tickers: AAPL, MSFT, GOOGL) -✅ Data continuity check works (0.65% discontinuity AAPL 2018-03-27→28) -✅ Can extend Quandl WIKI data (ended 2018-03-27) with Yahoo Finance -``` - -**Workflow demonstrated**: -1. Fetch historical data from quandl end date (2018-03-27) -2. Extend with Yahoo Finance (2018-03-28 onwards) -3. Verify data continuity (< 5% discontinuity) -4. Bulk download multiple tickers -5. Combine data with correct symbol column - -**Verdict**: The simplified YahooFinanceProvider CAN replace the quandl extension workflow. - ---- - -## What Was Removed (2,830 Lines of Bloat) - -### Embedded Updater Classes (~1,800 lines) -Every provider had an embedded `*Updater` class that was removed: -- CoinGeckoUpdater (258 lines) -- TwelveDataUpdater (260 lines) -- OANDAUpdater (220 lines) -- Plus 8 more similar classes - -**Rationale**: Most users don't need incremental update logic. Can be implemented separately if needed. - -### Circuit Breaker Configuration (~400 lines) -Present in 8 providers, removed from all: -- Never prevented a single failure in stress testing -- Added complexity without reliability benefit - -### Non-OHLCV Features (~300 lines) -Removed from all providers: -- `get_metadata()` methods -- `get_price()` real-time endpoints -- `get_available_symbols()` listings -- `get_coin_list()` catalogs -- Technical indicator calculations -- `fetch_quote()` methods -- `fetch_technical_indicator()` methods - -**Rationale**: Out of scope. Use provider's API directly for these features. - -### Validation Theater (~200 lines) -- `validate_ohlcv()` methods catching zero issues -- Complex symbol normalization -- Over-defensive error handling -- Tier detection logic (pro/free/enterprise) - -### Complex Initialization (~130 lines) -Simplified from 6-8 parameters to 1-2 parameters per provider: - -**Before**: -```python -def __init__( - self, - api_key: str | None = None, - use_pro: bool = False, - rate_limit: tuple[int, float] | None = None, - session_config: dict[str, Any] | None = None, - circuit_breaker_config: dict[str, Any] | None = None, - enable_validation: bool = True, -) -``` - -**After**: -```python -def __init__(self, api_key: str | None = None) -``` - ---- - -## What Remains (2,589 Lines of Essential Code) - -### Core OHLCV Functionality ✅ -- Consistent `fetch_ohlcv(symbol, start, end, frequency)` API -- Polars DataFrame output with symbol column -- Date range parsing and filtering -- Frequency support (daily/weekly/monthly) - -### Essential Infrastructure ✅ -- Simple rate limiting from BaseProvider -- Basic error handling (auth, rate limit, not found, network) -- `_create_empty_dataframe()` for consistent schema -- Structured logging (logger.info/warning) - -### Provider-Specific Logic ✅ -- Symbol formatting where needed (e.g., CoinGecko BTC→bitcoin) -- Chunked fetching for large date ranges (Polygon, Tiingo) -- Adjusted close handling (EODHD) -- Exchange suffix handling (EODHD .US, .LSE, etc.) - ---- - -## Evidence-Based Success - -### Stress Test Results -200-ticker test proved over-engineering was actively harmful: -- Direct yfinance: **58.5% success**, 38.5s (0.19s/ticker) -- Over-engineered provider: **53.0% success**, 399.7s (2.0s/ticker) -- **Verdict**: 10x slower with LOWER success rate - -### Simplification Results -- **2,830 lines removed** (52% reduction) -- **Zero loss of core functionality** -- **Tests pass** (919/978 non-integration tests, 94% pass rate) -- **Quandl workflow works** (verified end-to-end) -- **Faster execution** (no circuit breaker overhead) -- **Easier to maintain** (consistent, readable code) - ---- - -## Git History (20 Commits) - -``` -82bab29 fix: Update unit tests for simplified providers -9e896a0 fix: Remove Updater test dependencies + verify quandl extension works -e2b39d6 fix: Remove Updater class imports from __init__.py -d50923a docs: Add post-simplification status report -0bf14ff docs: Add complete simplification victory summary -3ae8719 docs: Complete provider simplification - all 11 providers streamlined -a104588 refactor: Simplify Binance provider - remove over-engineering (348→261 lines) -12b5bb1 refactor: Simplify AlphaVantage provider - remove over-engineering (420→236 lines) -369884e refactor: Simplify Tiingo provider - remove over-engineering (440→198 lines) -5d3680d refactor: Simplify Polygon provider - remove over-engineering (483→199 lines) -2006fee refactor: Simplify TwelveData provider - remove over-engineering (493→167 lines) -291f913 refactor: Simplify Finnhub provider - remove over-engineering (515→244 lines) -ede503e refactor: Simplify DataBento provider - remove over-engineering (517→234 lines) -32f27d1 refactor: Simplify EODHD provider - remove over-engineering (522→297 lines) -cdfa064 refactor: Simplify OANDA provider - remove over-engineering (561→258 lines) -fec7fb3 docs: Add simplification progress tracking -803c304 refactor: Simplify CoinGeckoProvider - remove over-engineering (799→243 lines) -1e51ba8 refactor: Simplify YahooFinanceProvider - remove over-engineering (321→217 lines) -4b60667 docs: Add critical Yahoo Finance ToS warning + DataBento config examples -4c7e6fd fix: Fix EODHD and Polygon provider bugs + comprehensive validation -``` - ---- - -## Lessons Learned - -### 1. Stress Testing at Scale Reveals Truth -Small samples (5 tickers) showed false positives. The 200-ticker test exposed real performance characteristics. - -### 2. Over-Engineering Is Actively Harmful -Not just wasteful, but measurably worse: -- 10x slower performance -- Lower success rate -- Harder to debug and maintain - -### 3. Simple Wrappers Are Better -Focus on consistent API across providers, not "enterprise features" without evidence. - -### 4. BaseProvider Handles Resilience -Rate limiting, session management, and retries all work well without custom implementations. - -### 5. User Was Right to Push Back -"audit every single one of them!!!" was the correct response. Evidence proved simplification was right. - ---- - -## Final Verdict - -✅ **All 11 providers simplified and functional** -✅ **Tests fixed and passing (94% pass rate)** -✅ **Quandl extension workflow verified** -✅ **Zero loss of core OHLCV functionality** -✅ **52% code reduction with faster execution** - -**The ml4t-data library is now lean, maintainable, evidence-based, and ready to replace ~/ml3t/data/equities/quandl/.** - ---- - -**Status**: MISSION COMPLETE 🏆 -**Work was NOT for naught** - It was necessary and successful! diff --git a/examples/POST_SIMPLIFICATION_STATUS.md b/examples/POST_SIMPLIFICATION_STATUS.md deleted file mode 100644 index eaf06e79..00000000 --- a/examples/POST_SIMPLIFICATION_STATUS.md +++ /dev/null @@ -1,164 +0,0 @@ -# Post-Simplification Status Report - -**Date**: 2025-11-16 -**Status**: All 11 providers simplified and functional ✅ - -## Provider Functionality ✅ - -All 11 simplified providers import and work correctly: - -```python -from ml4t.data.providers import ( - YahooFinanceProvider, - CoinGeckoProvider, - PolygonProvider, - TwelveDataProvider, - AlphaVantageProvider, - TiingoProvider, - FinnhubProvider, - EODHDProvider, - BinanceProvider, - OandaProvider, - DataBentoProvider, -) - -# Example: YahooFinanceProvider works perfectly -provider = YahooFinanceProvider() -df = provider.fetch_ohlcv('AAPL', '2024-01-01', '2024-01-05', 'daily') -# Returns Polars DataFrame with symbol column: (4 rows × 7 columns) -``` - -**Result**: ✅ All providers functional after 52% code reduction - -## Test Suite Status ⚠️ - -**Import errors**: ~24 test files fail to import due to removed Updater classes - -### Why Tests Fail - -During simplification, we removed **embedded Updater classes** (~1,800 lines): -- `CoinGeckoUpdater` (258 lines) -- `PolygonUpdater` (~180 lines) -- `TwelveDataUpdater` (~260 lines) -- Plus 6 more similar classes - -Many tests tried to import these classes: -```python -from ml4t.data.providers.coingecko import CoinGeckoProvider, CoinGeckoUpdater -# ImportError: cannot import name 'CoinGeckoUpdater' -``` - -### Test Files Affected - -Integration tests expecting Updaters: -- `tests/integration/test_coingecko.py` - Has `TestCoinGeckoUpdater` class -- `tests/integration/test_polygon.py` - Has `TestPolygonUpdater` class -- `tests/integration/test_twelve_data.py` - Has `TestTwelveDataUpdater` class -- `tests/integration/test_phase1_providers.py` - Uses multiple Updaters - -Plus ~20 more test files with Updater dependencies. - -### What Works - -**Provider tests** (non-Updater functionality) should work fine: -- Provider initialization -- fetch_ohlcv() basic functionality -- Rate limiting -- Error handling -- Symbol validation - -The failures are **only Updater-related tests**. - -## Options to Fix Tests - -### Option 1: Skip Updater Tests (Recommended) -Since Updater classes were removed intentionally (evidence-based simplification), skip those tests: - -```python -# In test files -import pytest - -@pytest.mark.skip(reason="Updater classes removed during simplification") -class TestCoinGeckoUpdater: - ... -``` - -**Pros**: Clean, documents the intentional removal -**Cons**: Loses test coverage for incremental update logic - -### Option 2: Comment Out Updater Imports -Remove Updater imports from test files: - -```python -# Old: -from ml4t.data.providers.coingecko import CoinGeckoProvider, CoinGeckoUpdater - -# New: -from ml4t.data.providers.coingecko import CoinGeckoProvider -# CoinGeckoUpdater removed - implement incremental updates separately if needed -``` - -**Pros**: Tests can run for provider functionality -**Cons**: Still have dead Updater test classes - -### Option 3: Create Stub Updaters (Not Recommended) -Create minimal Updater classes just for tests. - -**Pros**: Tests pass without changes -**Cons**: Defeats the purpose of simplification, adds back bloat - -### Option 4: Rewrite Tests Without Updaters (Most Work) -Refactor tests to use providers directly for incremental update scenarios. - -**Pros**: Proper test coverage of simplified API -**Cons**: Significant work to rewrite ~24 test files - -## Recommended Action - -**For now**: Option 1 (skip Updater tests) - -Tests for core provider functionality (fetch_ohlcv, rate limiting, error handling) should still work. -The Updater tests can be addressed later if incremental update functionality is needed. - -**Rationale**: -- Updaters were removed based on evidence (stress test showed over-engineering harmful) -- Most users don't need incremental update logic -- Provider functionality (fetch_ohlcv) is what matters -- Can always extract Updaters to separate module later if demand exists - -## Integration with Quandl Project - -**Question**: "can you now successfully run the quandl project from ~/ml3t/data/equities?" - -**Answer**: The quandl project (`~/ml3t/data/equities/quandl/`) does **NOT** use this ml4t-data library. - -It uses yfinance directly: -```python -# From unified_yfinance_downloader.py (quandl project) -import yfinance as yf - -# Direct yfinance usage, not ml4t-data providers -data = yf.download(ticker, start=start, end=end, auto_adjust=False) -``` - -The quandl project is a **standalone data pipeline** for: -- Extending discontinued Quandl WIKI dataset (ended 2018-03-27) -- Using Yahoo Finance to continue data through 2025 -- Handling split adjustments between data sources -- 7,409 tickers, 25.8M records, 1.4GB dataset - -**Status**: ✅ Quandl project unaffected by ml4t-data simplification - -## Summary - -**Providers**: ✅ All 11 work perfectly -**Tests**: ⚠️ Import errors due to removed Updater classes (expected) -**Quandl project**: ✅ Unaffected (uses yfinance directly, not ml4t-data) - -**Next steps** (optional): -1. Skip Updater tests with `@pytest.mark.skip` -2. Run provider-only tests to verify core functionality -3. Add performance benchmarks showing improvement from simplification -4. Document incremental update pattern without embedded Updaters - -**The simplification was successful** - 52% code reduction with zero loss of provider functionality. diff --git a/examples/RATE_LIMITING_FINDINGS.md b/examples/RATE_LIMITING_FINDINGS.md deleted file mode 100644 index a509fd28..00000000 --- a/examples/RATE_LIMITING_FINDINGS.md +++ /dev/null @@ -1,222 +0,0 @@ -# Rate Limiting Findings - YahooFinanceProvider - -**Date**: 2025-11-15 -**Test**: Quandl data extension (5 tickers, 7 years each) - -## Summary - -Our `YahooFinanceProvider` successfully extended Quandl WIKI data from 2018 to 2025 with **100% success rate** and **robust rate limiting**. - -## Rate Limiting Behavior - -### Configuration -```python -YahooFinanceProvider(max_requests_per_second=0.5) -# Translates to: 1 request per 2.0 seconds -``` - -### Observed Performance - -| Metric | Value | -|--------|-------| -| Tickers tested | 5 | -| Success rate | 100% | -| Total time | 8.7 seconds | -| Time per ticker | 1.74 seconds | -| Actual rate | 0.57 req/sec | -| Rate limit hits | 0 (handled internally) | - -### Rate Limiting Timeline - -``` -21:19:15 [info] Fetching AAPL -21:19:16 [debug] Rate limit reached, waiting 1.20s -21:19:17 [info] Fetching MSFT -21:19:18 [debug] Rate limit reached, waiting 1.36s -21:19:19 [info] Fetching TSLA -21:19:20 [debug] Rate limit reached, waiting 1.38s -21:19:21 [info] Fetching GOOGL -21:19:22 [debug] Rate limit reached, waiting 1.29s -21:19:23 [info] Fetching NVDA -``` - -**Pattern**: -- First request: Immediate -- Subsequent requests: 1.17s - 1.38s wait -- Average wait: ~1.30s -- No exceptions raised (internal handling) - -## Comparison: ML4T Data vs Direct yfinance - -### Direct yfinance (ml3t/quandl approach) -```python -# create_full_seamless_dataset.py -yf_ticker = yf.Ticker(ticker) -hist = yf_ticker.history(start=start, auto_adjust=False) -time.sleep(0.1) # 100ms manual delay -``` - -**Characteristics**: -- ✅ Fast: ~0.1s per ticker -- ❌ Risks throttling on large batches -- ❌ Manual error handling -- ❌ No automatic retries -- ❌ Pandas output (slower for large datasets) - -### ML4T Data YahooFinanceProvider -```python -provider = YahooFinanceProvider(max_requests_per_second=0.5) -df = provider.fetch_ohlcv(ticker, start, end, "daily") -``` - -**Characteristics**: -- ❌ Slower: ~1.74s per ticker (17x slower) -- ✅ No throttling (conservative rate) -- ✅ Automatic error handling -- ✅ Built-in retries with backoff -- ✅ Polars output (10-100x faster processing) -- ✅ Circuit breaker prevents cascading failures -- ✅ Typed exceptions -- ✅ Structured logging - -## Rate Limit Recommendations - -### For Small Batches (<10 tickers) -```python -provider = YahooFinanceProvider(max_requests_per_second=1.0) -# ~1s per ticker -``` - -### For Medium Batches (10-100 tickers) -```python -provider = YahooFinanceProvider(max_requests_per_second=0.5) # DEFAULT -# ~2s per ticker -``` - -### For Large Batches (100-1000 tickers) -```python -provider = YahooFinanceProvider(max_requests_per_second=0.33) -# ~3s per ticker, very conservative -``` - -### For Production Pipelines -```python -provider = YahooFinanceProvider(max_requests_per_second=0.5) -# DEFAULT is production-safe -# Add batching with progress saving for resumability -``` - -## Throughput Calculations - -| Rate (req/s) | Per Ticker | 100 Tickers | 1000 Tickers | -|--------------|-----------|-------------|--------------| -| 0.33 | 3.0s | 5 min | 50 min | -| 0.50 (default) | 2.0s | 3.3 min | 33 min | -| 1.00 | 1.0s | 1.7 min | 17 min | -| 2.00 | 0.5s | 50s | 8.3 min | - -**Note**: Higher rates risk Yahoo throttling (429 errors) - -## Error Handling - -### Observed Exceptions (None in This Test) - -```python -try: - df = provider.fetch_ohlcv(ticker, start, end, "daily") -except DataNotAvailableError: - # Symbol not found or delisted - logger.warning(f"{ticker} not available") -except RateLimitError as e: - # Hit rate limit (429) - time.sleep(e.retry_after) - # Retry automatically via provider -except NetworkError: - # Connection/HTTP error - logger.error(f"{ticker} network error") -``` - -### Automatic Retry Logic - -The provider automatically retries on: -- Network errors (exponential backoff) -- Rate limit errors (wait retry_after) -- Transient failures (3 retries max) - -### Circuit Breaker - -After 3 consecutive failures: -- Opens circuit (stops requests) -- Waits 600s (10 minutes) -- Attempts reset -- If success: Normal operation -- If fail: Circuit stays open - -## Key Findings - -### ✅ What Works Well - -1. **Automatic rate limiting** - No manual sleep() needed -2. **Zero rate limit exceptions** - Handled internally -3. **100% success rate** - No failed requests -4. **Consistent timing** - Predictable performance -5. **Polars output** - Fast processing for large datasets - -### ⚠️ Trade-offs - -1. **Slower than direct yfinance** - 17x slower (1.74s vs 0.1s) -2. **No symbol column by default** - Must add manually -3. **Conservative rate** - May be too slow for urgent needs - -### 📊 When to Use Each Approach - -**Use ML4T Data YahooFinanceProvider**: -- Production pipelines -- Large batch downloads (>100 tickers) -- Need reliability over speed -- Want automatic error handling -- Building research infrastructure - -**Use Direct yfinance**: -- One-off analysis -- Small batches (<10 tickers) -- Speed critical -- Willing to handle errors manually -- Prototyping/experimentation - -## Testing Notes - -### Test Environment -- Python 3.13.5 -- Polars 1.19.0 -- yfinance 0.2.49 -- ml4t-data 0.1.0 (pre-release) - -### Test Data -- 5 tickers: AAPL, MSFT, TSLA, GOOGL, NVDA -- Date range: 2018-03-28 to 2025-11-14 (7+ years) -- Records: 1,921 per ticker (9,605 total) -- All data continuous (no gaps) - -### Reproducibility -```bash -cd /home/stefan/ml4t/software/data -python examples/quandl_extension_example.py -``` - -## Conclusion - -The `YahooFinanceProvider` provides **production-grade reliability** at the cost of **17x slower performance** vs direct yfinance. The trade-off is worth it for: - -1. **Large batch downloads** (avoid throttling) -2. **Production pipelines** (reliability > speed) -3. **Research infrastructure** (set it and forget it) - -For ad-hoc analysis or small batches, direct yfinance is faster and acceptable. - ---- - -**See Also**: -- `quandl_extension_example.py` - Full implementation -- `quandl_extension_README.md` - Detailed documentation -- `/home/stefan/ml3t/data/equities/quandl/` - Original ml3t approach diff --git a/examples/SIMPLIFICATION_PROGRESS.md b/examples/SIMPLIFICATION_PROGRESS.md deleted file mode 100644 index c9ba73b8..00000000 --- a/examples/SIMPLIFICATION_PROGRESS.md +++ /dev/null @@ -1,177 +0,0 @@ -# Provider Simplification Progress - COMPLETE ✅ - -Evidence-based removal of over-engineering following stress test findings. - -## Motivation - -200-ticker stress test proved that over-engineering provides ZERO reliability benefit: -- Direct yfinance: 58.5% success, 38.5s (0.19s/ticker) -- Over-engineered YahooFinanceProvider: 53.0% success, 399.7s (2.0s/ticker) -- **Verdict**: 10x slower with LOWER success rate - -## ALL 11 PROVIDERS SIMPLIFIED ✅ - -| Provider | Before | After | Removed | Reduction | Status | -|----------|--------|-------|---------|-----------|--------| -| YahooFinance | 321 | 217 | 104 | 32% | ✅ | -| CoinGecko | 799 | 243 | 556 | 70% | ✅ | -| OANDA | 561 | 286 | 275 | 49% | ✅ | -| EODHD | 522 | 296 | 226 | 43% | ✅ | -| DataBento | 517 | 233 | 284 | 55% | ✅ | -| Finnhub | 515 | 243 | 272 | 53% | ✅ | -| **TwelveData** 🏆 | 493 | 166 | 327 | **66%** | ✅ | -| Polygon | 483 | 202 | 281 | 58% | ✅ | -| Tiingo | 440 | 203 | 237 | 54% | ✅ | -| AlphaVantage | 420 | 240 | 180 | 43% | ✅ | -| Binance | 348 | 260 | 88 | 25% | ✅ | -| **TOTAL** | **5,419** | **2,589** | **2,830** | **52%** | **✅** | - -**Champion**: TwelveData with 66% reduction (493→166 lines) - from biggest to leanest! - -## What Was Removed (2,830 lines of bloat) - -### Embedded Updater Classes (~1,800 lines) -- CoinGeckoUpdater: 258 lines -- EODHDUpdater: ~200 lines -- PolygonUpdater: ~180 lines -- TiingoUpdater: ~200 lines -- DataBentoUpdater: ~200 lines -- OANDAUpdater: ~220 lines -- FinnhubUpdater: ~180 lines -- TwelveDataUpdater: ~260 lines -- AlphaVantageUpdater: ~200 lines - -**Decision**: Extract to separate `src/ml4t/data/updaters/` when needed. Most users don't need incremental update logic. - -### Circuit Breaker Configuration (~400 lines) -Present in 8 providers, removed from all: -- Never prevented a single failure in stress testing -- Added complexity without any reliability benefit -- BaseProvider already handles retries and resilience - -### Non-OHLCV Features (~300 lines) -Removed from all providers: -- `get_metadata()` methods -- `get_price()` real-time endpoints -- `get_available_symbols()` listings -- `get_coin_list()` catalogs -- Technical indicator calculations - -**Recommendation**: Use provider's API directly for these features. - -### Validation Theater (~200 lines) -- `validate_ohlcv()` methods catching zero issues -- Complex symbol normalization (exchange suffixes, case handling) -- Over-defensive error handling cascades -- Tier detection logic (pro/free/enterprise) - -### Complex Initialization (~130 lines) -**Before** (typical): -```python -def __init__( - self, - api_key: str | None = None, - use_pro: bool = False, - rate_limit: tuple[int, float] | None = None, - session_config: dict[str, Any] | None = None, - circuit_breaker_config: dict[str, Any] | None = None, - enable_validation: bool = True, -) -> None: -``` - -**After** (typical): -```python -def __init__(self, api_key: str | None = None) -> None: -``` - -## What Was Kept (2,589 lines of essentials) - -✅ **Core OHLCV Functionality**: -- Consistent `fetch_ohlcv(symbol, start, end, frequency)` API -- Polars DataFrame output with symbol column -- Date range parsing and filtering -- Frequency support (daily/weekly/monthly where available) - -✅ **Essential Infrastructure**: -- Simple rate limiting from BaseProvider -- Basic error handling (auth, rate limit, not found, network) -- `_create_empty_dataframe()` for consistent schema -- Structured logging (logger.info/warning) - -✅ **Provider-Specific Logic** (where needed): -- Symbol formatting (e.g., CoinGecko symbol→ID mapping) -- Chunked fetching for large date ranges (Polygon, Tiingo) -- Adjusted close handling (EODHD) -- Exchange suffix handling (EODHD .US, .LSE, etc.) - -## Lessons Applied - -1. **Stress testing at scale reveals truth** - Small samples (5 tickers) showed false positives -2. **Over-engineering is harmful** - Not just wasteful, but actively worse (10x slower, lower success) -3. **Simple wrappers are better** - Focus on consistent API, not "enterprise features" -4. **BaseProvider handles resilience** - Rate limiting, sessions, retries all work -5. **User was right to push back** - Evidence proved simplification was correct - -## Git Commits - -All 11 providers simplified with individual commits: - -``` -1e51ba8 refactor: Simplify YahooFinanceProvider - remove over-engineering (321→217 lines) -803c304 refactor: Simplify CoinGeckoProvider - remove over-engineering (799→243 lines) -a104588 refactor: Simplify Binance provider - remove over-engineering (348→261 lines) -12b5bb1 refactor: Simplify AlphaVantage provider - remove over-engineering (420→236 lines) -369884e refactor: Simplify Tiingo provider - remove over-engineering (440→198 lines) -5d3680d refactor: Simplify Polygon provider - remove over-engineering (483→199 lines) -2006fee refactor: Simplify TwelveData provider - remove over-engineering (493→167 lines) -291f913 refactor: Simplify Finnhub provider - remove over-engineering (515→244 lines) -ede503e refactor: Simplify DataBento provider - remove over-engineering (517→234 lines) -32f27d1 refactor: Simplify EODHD provider - remove over-engineering (522→297 lines) -cdfa064 refactor: Simplify OANDA provider - remove over-engineering (561→258 lines) -``` - -## Impact Summary - -### Performance -- **52% code reduction** (5,419→2,589 lines) -- **Faster execution** (no circuit breaker overhead, simpler logic) -- **Lower memory footprint** (no complex state machines) - -### Maintainability -- **Consistent patterns** across all 11 providers -- **Easy to understand** - simple, readable code -- **No hidden complexity** - what you see is what you get -- **Easier debugging** - fewer layers of abstraction - -### Reliability -- **Evidence-based approach** - removed features that provided no benefit -- **BaseProvider handles resilience** - proven to work well -- **No loss of functionality** - all core OHLCV fetching preserved -- **Better error messages** - simpler error handling is clearer - -## Next Steps (Optional) - -1. ✅ **COMPLETE**: All 11 providers simplified -2. 🔄 Extract Updater classes to `src/ml4t/data/updaters/` (if needed) -3. 🔄 Update tests to match simplified APIs (remove Updater tests) -4. 🔄 Update README with honest positioning about trade-offs -5. 🔄 Add performance benchmarks showing improvement - -## Final Stats - -**Original audit target**: 5,436→2,390 lines (3,046 to remove, 56% bloat) -**Actual achievement**: 5,419→2,589 lines (2,830 removed, 52% reduction) - -**Target vs Actual**: -- Target: Remove 3,046 lines (56% reduction) -- Actual: Removed 2,830 lines (52% reduction) -- Result: **93% of bloat removed** (216 lines kept for essential logic) - -**The ml4t-data library is now lean, maintainable, and evidence-based. 🚀** - ---- - -**Status**: COMPLETE ✅ (All 11 providers simplified) -**Date completed**: 2025-11-16 -**Evidence source**: 200-ticker stress test proving over-engineering harmful -**Approach**: Ruthless removal of complexity that provided no value diff --git a/examples/SIMPLIFICATION_VICTORY.md b/examples/SIMPLIFICATION_VICTORY.md deleted file mode 100644 index 77df0a03..00000000 --- a/examples/SIMPLIFICATION_VICTORY.md +++ /dev/null @@ -1,214 +0,0 @@ -# Complete Provider Simplification - Victory Lap 🏆 - -**Date**: 2025-11-16 -**Effort**: Evidence-based refactoring following stress test findings -**Result**: All 11 providers simplified, 52% bloat eliminated - -## The Evidence - -200-ticker stress test revealed over-engineering was **actively harmful**: -- Direct yfinance: **58.5% success**, 38.5s (0.19s/ticker) -- Over-engineered YahooFinanceProvider: **53.0% success**, 399.7s (2.0s/ticker) -- **Verdict**: 10x slower with LOWER success rate - -## Complete Transformation - -| Provider | Before | After | Removed | Reduction | -|----------|--------|-------|---------|-----------| -| YahooFinance | 321 | 217 | 104 | 32% | -| CoinGecko | 799 | 243 | 556 | 70% | -| OANDA | 561 | 286 | 275 | 49% | -| EODHD | 522 | 296 | 226 | 43% | -| DataBento | 517 | 233 | 284 | 55% | -| Finnhub | 515 | 243 | 272 | 53% | -| **TwelveData 🏆** | 493 | 166 | 327 | **66%** | -| Polygon | 483 | 202 | 281 | 58% | -| Tiingo | 440 | 203 | 237 | 54% | -| AlphaVantage | 420 | 240 | 180 | 43% | -| Binance | 348 | 260 | 88 | 25% | -| **TOTAL** | **5,419** | **2,589** | **2,830** | **52%** | - -## Bloat Breakdown (2,830 lines removed) - -### 1. Embedded Updater Classes (~1,800 lines) -Every provider had an embedded `*Updater` class for incremental updates: -- CoinGeckoUpdater: 258 lines -- TwelveDataUpdater: 260 lines -- OANDAUpdater: 220 lines -- Plus 6 more similar classes - -**Decision**: Most users don't need incremental update logic. Extract to separate `updaters/` module when needed. - -### 2. Circuit Breaker Configuration (~400 lines) -8 providers had circuit breaker config in __init__: -```python -circuit_breaker_config: dict[str, Any] | None = None -``` - -**Evidence**: Never prevented a single failure in stress testing -**Decision**: Remove entirely - BaseProvider handles resilience - -### 3. Non-OHLCV Features (~300 lines) -Providers had methods beyond OHLCV fetching: -- `get_metadata()` - Market information -- `get_price()` - Real-time pricing -- `get_available_symbols()` - Symbol listings -- `get_coin_list()` - Cryptocurrency catalogs -- Technical indicator calculations - -**Decision**: Out of scope. Use provider's API directly for these features. - -### 4. Validation Theater (~200 lines) -- `validate_ohlcv()` methods that caught zero issues -- Complex symbol normalization (exchange suffixes, case handling) -- Over-defensive error handling cascades -- Tier detection logic (pro/free/enterprise) - -**Evidence**: No real-world benefit, just complexity -**Decision**: Remove all validation that provides no value - -### 5. Complex Initialization (~130 lines) -**Before** (typical bloat): -```python -def __init__( - self, - api_key: str | None = None, - use_pro: bool = False, - rate_limit: tuple[int, float] | None = None, - session_config: dict[str, Any] | None = None, - circuit_breaker_config: dict[str, Any] | None = None, - enable_validation: bool = True, -) -> None: -``` - -**After** (clean): -```python -def __init__(self, api_key: str | None = None) -> None: -``` - -## What Remains (2,589 lines of essential code) - -### Core OHLCV Functionality -✅ Consistent `fetch_ohlcv(symbol, start, end, frequency)` API across all providers -✅ Polars DataFrame output with symbol column -✅ Date range parsing and filtering -✅ Frequency support (daily/weekly/monthly where available) - -### Essential Infrastructure -✅ Simple rate limiting from BaseProvider -✅ Basic error handling (auth, rate limit, not found, network) -✅ `_create_empty_dataframe()` for consistent schema -✅ Structured logging (logger.info/warning) - -### Provider-Specific Logic (where truly needed) -✅ Symbol formatting (e.g., CoinGecko BTC→bitcoin mapping) -✅ Chunked fetching for large date ranges (Polygon, Tiingo) -✅ Adjusted close handling (EODHD) -✅ Exchange suffix handling (EODHD .US, .LSE, etc.) - -## Lessons Learned - -### 1. Stress testing at scale reveals truth -- Small samples (5 tickers) showed false positives -- 200-ticker test exposed real performance characteristics -- Over-engineering appeared to help in small tests but failed at scale - -### 2. Over-engineering is actively harmful -- Not just wasteful, but measurably worse -- 10x slower performance -- Lower success rate (53% vs 58.5%) -- Harder to debug and maintain - -### 3. Simple wrappers are better -- Focus on consistent API across providers -- Let provider's native library handle complexity -- Don't add "enterprise features" without evidence of benefit - -### 4. BaseProvider handles resilience -- Rate limiting works well -- Session management is sufficient -- Retry logic is effective -- Circuit breakers added no value - -### 5. User was right to push back -- "audit every single one of them!!!" was the correct response -- Performance matters when features provide no benefit -- Evidence-based simplification > theoretical reliability - -## Champion: TwelveData - -**66% reduction** (493→166 lines) -- Went from biggest provider to **leanest** -- Removed massive 260-line TwelveDataUpdater class -- Simplified from 8 __init__ parameters to 1 -- Removed validation theater and non-OHLCV features -- Still fully functional for OHLCV data fetching - -## Impact - -### Performance -- **52% code reduction** overall -- **Faster execution** (no circuit breaker overhead, simpler logic paths) -- **Lower memory footprint** (no complex state machines) - -### Maintainability -- **Consistent patterns** across all 11 providers -- **Easy to understand** - simple, readable code -- **No hidden complexity** - WYSIWYG -- **Easier debugging** - fewer abstraction layers - -### Reliability -- **Evidence-based approach** - removed features that provided no benefit -- **BaseProvider proven effective** - handles resilience well -- **No loss of functionality** - all core OHLCV fetching preserved -- **Better error messages** - simpler error handling is clearer - -## Git History - -Complete refactoring captured in 14 commits: - -``` -3ae8719 docs: Complete provider simplification - all 11 providers streamlined -a104588 refactor: Simplify Binance provider - remove over-engineering (348→261 lines) -12b5bb1 refactor: Simplify AlphaVantage provider - remove over-engineering (420→236 lines) -369884e refactor: Simplify Tiingo provider - remove over-engineering (440→198 lines) -5d3680d refactor: Simplify Polygon provider - remove over-engineering (483→199 lines) -2006fee refactor: Simplify TwelveData provider - remove over-engineering (493→167 lines) -291f913 refactor: Simplify Finnhub provider - remove over-engineering (515→244 lines) -ede503e refactor: Simplify DataBento provider - remove over-engineering (517→234 lines) -32f27d1 refactor: Simplify EODHD provider - remove over-engineering (522→297 lines) -cdfa064 refactor: Simplify OANDA provider - remove over-engineering (561→258 lines) -fec7fb3 docs: Add simplification progress tracking -803c304 refactor: Simplify CoinGeckoProvider - remove over-engineering (799→243 lines) -1e51ba8 refactor: Simplify YahooFinanceProvider - remove over-engineering (321→217 lines) -``` - -## Final Stats - -**Original audit target**: 5,436→2,390 lines (3,046 to remove, 56% bloat) -**Actual achievement**: 5,419→2,589 lines (2,830 removed, 52% reduction) - -**Target vs Actual**: -- **Target**: Remove 3,046 lines (56% reduction) -- **Actual**: Removed 2,830 lines (52% reduction) -- **Result**: **93% of bloat removed** (kept 216 lines for essential logic) - -The remaining 216 lines difference from target represents essential provider-specific logic that genuinely adds value (symbol mapping, chunked fetching, exchange suffixes, etc.). - -## What's Next (Optional) - -1. ✅ **COMPLETE**: All 11 providers simplified -2. 🔄 Extract Updater classes to `src/ml4t/data/updaters/` if incremental updates are needed -3. 🔄 Update tests to match simplified APIs (remove Updater tests) -4. 🔄 Update README with honest positioning about trade-offs -5. 🔄 Run performance benchmarks to quantify improvement - -## Conclusion - -**The ml4t-data library is now lean, maintainable, and evidence-based.** - -This wasn't just refactoring - it was ruthless elimination of complexity proven to provide zero value. Every line removed was backed by evidence from stress testing showing that over-engineering was actively harmful. - -The library now focuses on what it does best: providing a **consistent API for OHLCV data** across multiple providers, with **simple, readable code** that's easy to understand, debug, and maintain. - -**Mission accomplished.** 🚀 diff --git a/examples/STRESS_TEST_README.md b/examples/STRESS_TEST_README.md deleted file mode 100644 index 7fd6bbd3..00000000 --- a/examples/STRESS_TEST_README.md +++ /dev/null @@ -1,157 +0,0 @@ -# Stress Test: ML4T Data vs Direct yfinance - -## Purpose - -Test with **200 real tickers** from the Quandl dataset to determine if ML4T Data's YahooFinanceProvider provides actual reliability benefits or is just slower with no gain. - -## The Question - -> "The 17x slowdown will matter more when you scale. We'll only know with hundreds of tickers if the lower speed actually makes this more reliable or is JUST poor performance." - -## Test Design - -### Sample -- **200 tickers** from Quandl WIKI dataset (first 200 alphabetically) -- Includes active, delisted, and renamed tickers -- Real-world mix of success/failure scenarios - -### Period -- **2024-01-01 to 2024-12-31** (1 year of data) -- ~250 trading days per ticker - -### Approaches - -**1. Direct yfinance (ml3t style)** -```python -yf_ticker = yf.Ticker(ticker) -hist = yf_ticker.history(start=start, end=end, auto_adjust=False) -time.sleep(0.1) # 100ms manual delay -``` - -**2. ML4T Data YahooFinanceProvider** -```python -provider = YahooFinanceProvider(max_requests_per_second=0.5) -df = provider.fetch_ohlcv(ticker, start, end, "daily") -# Automatic rate limiting ~2s per ticker -``` - -## Metrics Being Measured - -### Performance -- Total time -- Average time per ticker -- Throughput (tickers/hour) -- Performance ratio (how much slower) - -### Reliability -- Success count -- Failure count -- Success rate -- Failure breakdown (empty, rate limit, network, errors) - -### The Verdict - -Calculate whether reliability improvement justifies performance cost: - -``` -Reliability improvement = (ML4T Data_success - yfinance_success) / total * 100 -Performance cost = (ML4T Data_time - yfinance_time) / yfinance_time * 100 - -if reliability_improvement > 2%: - "Worth it" -elif reliability_improvement > 0%: - "Marginal benefit" -else: - "Just slower, no gain" -``` - -## Expected Results - -### Hypothesis 1: ML4T Data is just slower (no benefit) -- Both approaches: similar success rates -- ML4T Data: 10-20x slower -- **Conclusion**: Poor performance, no reliability gain -- **Action**: Increase default rate limit or recommend direct yfinance - -### Hypothesis 2: ML4T Data provides reliability -- ML4T Data: 5-10% higher success rate -- ML4T Data: 10-20x slower -- **Conclusion**: Performance cost justified by reliability -- **Action**: Keep defaults, document trade-off - -### Hypothesis 3: Direct yfinance gets throttled -- yfinance: many rate limit errors after N tickers -- ML4T Data: smooth throughout -- **Conclusion**: Rate limiting critical at scale -- **Action**: Keep conservative defaults - -## Running the Test - -```bash -cd /home/stefan/ml4t/software/data -.venv/bin/python examples/stress_test_comparison.py -``` - -**Expected duration**: -- Direct yfinance: ~20-30 seconds (100ms * 200) -- ML4T Data: ~350-400 seconds (1.75s * 200) -- **Total: ~7-8 minutes** - -## Results Location - -- **Console output**: Real-time progress and final comparison -- **Log file**: `examples/stress_test_output.log` -- **Detailed results**: `examples/stress_test_results.txt` - -## What We'll Learn - -1. **Does automatic rate limiting prevent throttling at scale?** - - If yfinance gets 429 errors after N tickers → Yes - - If both succeed equally → No benefit - -2. **Is the 17x slowdown justified?** - - If ML4T Data succeeds 10%+ more → Justified - - If ML4T Data succeeds 1-5% more → Marginal - - If ML4T Data succeeds equally → Not justified - -3. **What's the right default rate limit?** - - If 0.5 req/sec succeeds 100% → Keep it - - If 0.5 req/sec still has errors → Too slow, increase it - - If direct yfinance succeeds 100% → Our limit too conservative - -4. **Real-world failure modes** - - Delisted tickers (both should handle equally) - - Network errors (ML4T Data should retry better) - - Rate limiting (ML4T Data should prevent) - - Invalid symbols (both should handle equally) - -## Post-Test Analysis - -After test completes, analyze: - -### If ML4T Data wins (higher success rate): -- Document reliability benefit percentage -- Calculate cost per additional success (extra seconds per % improvement) -- Update README with "Worth it for production" recommendation - -### If both equal: -- ML4T Data is just slower with no benefit -- Increase default rate limit to 1.0 or 2.0 req/sec -- Update README: "Use direct yfinance for small batches, ML4T Data for error handling/logging" - -### If yfinance wins (higher success rate): -- Something wrong with ML4T Data implementation -- Debug and fix -- Re-test - -## Critical Success Factors - -### For ML4T Data to be "worth it": -1. **≥5% higher success rate** OR -2. **Zero rate limit errors while yfinance gets throttled** OR -3. **Better error messages that help debugging** - -### If none of above: -- Admit it's just slower -- Recommend direct yfinance for most use cases -- Position ML4T Data for "enterprise features" (logging, monitoring, etc.) diff --git a/examples/bulk_download_workflow.py b/examples/bulk_download_workflow.py index 493d7d5c..c7cbd051 100644 --- a/examples/bulk_download_workflow.py +++ b/examples/bulk_download_workflow.py @@ -331,7 +331,7 @@ def chunks(lst, n): # %% [markdown] # ### Key Advantages of ml4t-data # -# 1. **Polars Output**: 10-100x faster for downstream operations +# 1. **Polars Output**: columnar data for downstream operations # 2. **Long Format**: One row per symbol per timestamp (easier for ML) # 3. **Validated Schema**: Consistent columns across all providers # 4. **Logging**: Structured logging for debugging @@ -564,10 +564,10 @@ def download_sp500_sample(): # # ### Key Takeaways # -# 1. **Use batch downloads** for >10 symbols (5-20x faster) +# 1. **Use batch downloads** to group multi-symbol provider requests # 2. **Chunk large universes** (50 symbols/chunk with 1s delay) # 3. **Always validate** data quality before backtesting -# 4. **Use ml4t-data** for production workflows (handles chunking, validation, Polars output) +# 4. **Use ml4t-data** for recurring workflows (handles chunking, validation, Polars output) # # ### Performance Reference # diff --git a/examples/configs/advanced.yaml b/examples/configs/advanced.yaml index 5b5bb9d1..6ed5b5f2 100644 --- a/examples/configs/advanced.yaml +++ b/examples/configs/advanced.yaml @@ -1,16 +1,16 @@ -# Advanced QData Configuration Example +# Advanced ML4T Data configuration example # Demonstrates environment variables, multiple providers, and complex workflows # Storage with custom configuration storage: strategy: hive - base_path: ${QDATA_STORAGE_PATH:~/qdata/storage} + base_path: ${ML4T_DATA_PATH:./market-data} compression: zstd lock_timeout: 60 # Longer timeout for large operations partition_cols: ["year", "month", "day"] # More granular partitioning # Global settings -log_level: ${QDATA_LOG_LEVEL:INFO} +log_level: ${ML4T_DATA_LOG_LEVEL:INFO} parallel_downloads: 4 # More parallel downloads for multiple providers # Multiple data providers with API keys from environment @@ -19,14 +19,14 @@ providers: type: yahoo enabled: true rate_limit: - requests_per_second: ${QDATA_YAHOO_RATE_LIMIT:2.0} + requests_per_second: ${ML4T_DATA_YAHOO_RATE_LIMIT:2.0} - name: databento type: databento enabled: true api_key: ${DATABENTO_API_KEY} rate_limit: - requests_per_second: 100.0 # Databento allows high throughput + requests_per_second: 100.0 burst_size: 10 extra: dataset: "XNAS.ITCH" @@ -38,7 +38,7 @@ providers: extra: practice: true rate_limit: - requests_per_second: 120.0 # OANDA's limit + requests_per_second: 120.0 burst_size: 5 - name: cryptocompare @@ -46,7 +46,7 @@ providers: enabled: true api_key: ${CRYPTOCOMPARE_API_KEY} rate_limit: - requests_per_second: ${QDATA_CRYPTO_RATE_LIMIT:10.0} + requests_per_second: ${ML4T_DATA_CRYPTO_RATE_LIMIT:10.0} extra: exchange: "CCCAGG" @@ -137,10 +137,6 @@ workflows: timezone: "US/Eastern" enabled: true on_error: retry - pre_hooks: - - "echo 'Starting market close update at $(date)'" - post_hooks: - - "python -m qdata.validation.run --dataset sp500_daily" # High-frequency futures update - name: futures_continuous @@ -180,8 +176,6 @@ workflows: timezone: "UTC" enabled: true on_error: continue - post_hooks: - - "python -m qdata.reports.weekly --email admin@example.com" # Validation configuration validation: diff --git a/examples/configs/basic.yaml b/examples/configs/basic.yaml index 75c58a13..88696068 100644 --- a/examples/configs/basic.yaml +++ b/examples/configs/basic.yaml @@ -1,11 +1,10 @@ -# Basic QData Configuration Example -# This file demonstrates basic configuration for QData with the new unified system +# Basic ML4T Data configuration example -# Storage configuration - Uses Hive partitioning for 7x query performance +# Storage configuration storage: strategy: hive # 'hive' for partitioned or 'flat' for single file - base_path: ~/qdata/storage - compression: zstd # Best compression/speed balance + base_path: ./market-data + compression: zstd metadata_tracking: true # Track update timestamps # Data providers configuration diff --git a/examples/configs/crypto_focus.yaml b/examples/configs/crypto_focus.yaml index 1dc3693e..66875f5f 100644 --- a/examples/configs/crypto_focus.yaml +++ b/examples/configs/crypto_focus.yaml @@ -1,4 +1,4 @@ -# Crypto-Focused QLDM Configuration +# Crypto-focused ml4t-data configuration # Optimized for cryptocurrency data collection and analysis version: "1.0" diff --git a/examples/configs/development.yaml b/examples/configs/development.yaml index a2d211b6..28a6b31c 100644 --- a/examples/configs/development.yaml +++ b/examples/configs/development.yaml @@ -55,12 +55,6 @@ workflows: - mock_crypto enabled: true on_error: continue # Continue to test all components - pre_hooks: - - "echo 'Starting test pipeline at $(date)'" - - "rm -rf ./test_data/validation_reports/*" # Clean old reports - post_hooks: - - "python tests/validate_pipeline.py" - - "echo 'Test pipeline completed at $(date)'" - name: stress_test description: Stress test with high-frequency mock data @@ -90,7 +84,4 @@ storage: compression: none # No compression for easier debugging # Development-only settings -env: - QLDM_DEV_MODE: "true" - QLDM_SKIP_AUTH: "true" # Skip authentication in dev - QLDM_MOCK_TIME: "2023-12-01T12:00:00Z" # Fixed time for tests +env: {} diff --git a/examples/configs/modular/base.yaml b/examples/configs/modular/base.yaml index f9c5f0ff..220dcea8 100644 --- a/examples/configs/modular/base.yaml +++ b/examples/configs/modular/base.yaml @@ -2,14 +2,14 @@ # This file is included by other configurations version: "1.0" -base_dir: ${QLDM_DATA_DIR:./data} -log_level: ${QLDM_LOG_LEVEL:INFO} +base_dir: ${ML4T_DATA_PATH:./data} +log_level: ${ML4T_DATA_LOG_LEVEL:INFO} # Common environment variables env: - QLDM_DATA_DIR: /var/lib/qldm/data - QLDM_LOG_DIR: /var/log/qldm - QLDM_CACHE_DIR: /var/cache/qldm + ML4T_DATA_PATH: /var/lib/ml4t/data + ML4T_DATA_REPORT_PATH: /var/log/ml4t-data + ML4T_DATA_CACHE_PATH: /var/cache/ml4t-data # Default settings for all datasets defaults: @@ -25,7 +25,7 @@ validation: enabled: true fail_on_critical: false save_reports: true - report_dir: ${QLDM_LOG_DIR}/validation + report_dir: ${ML4T_DATA_REPORT_PATH}/validation rules: check_nulls: true check_duplicate_timestamps: true @@ -36,5 +36,5 @@ validation: # Storage configuration storage: strategy: hive - base_path: ${QLDM_DATA_DIR} + base_path: ${ML4T_DATA_PATH} compression: zstd diff --git a/examples/configs/modular/production.yaml b/examples/configs/modular/production.yaml deleted file mode 100644 index a5dad89a..00000000 --- a/examples/configs/modular/production.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# Production configuration using modular includes - -# Include base configuration and providers -include: - - base.yaml - - providers.yaml - -# Override for production -log_level: WARNING -validation: - fail_on_critical: true # Strict in production - -# Production datasets -datasets: - - name: indices_daily - symbols: - - ^GSPC # S&P 500 - - ^DJI # Dow Jones - - ^IXIC # NASDAQ - - ^RUT # Russell 2000 - provider: yahoo - frequency: daily - asset_class: index - - - name: etf_portfolio - symbols_file: ../../symbols/etfs_production.txt - provider: yahoo - frequency: daily - asset_class: equity - validation: - strict: true - - - name: forex_majors - symbols: - - EURUSD=X - - GBPUSD=X - - USDJPY=X - - USDCHF=X - - AUDUSD=X - - USDCAD=X - provider: yahoo - frequency: daily - asset_class: forex - -# Production workflows with monitoring -workflows: - - name: morning_update - description: Production morning data update - datasets: - - indices_daily - - etf_portfolio - - forex_majors - schedule: - type: daily - time: "06:00:00" - timezone: "America/New_York" - enabled: true - on_error: stop # Stop on any error in production - pre_hooks: - - "python /opt/qldm/scripts/health_check.py" - - "python /opt/qldm/scripts/backup_data.py" - post_hooks: - - "python /opt/qldm/scripts/validate_all.py --strict" - - "python /opt/qldm/scripts/update_metrics.py" - - "python /opt/qldm/scripts/notify_slack.py --channel data-updates" - notifications: - slack: - webhook: ${SLACK_WEBHOOK_URL} - channel: "#data-alerts" - on_success: true - on_failure: true - email: - smtp_host: ${SMTP_HOST} - smtp_port: ${SMTP_PORT} - from: "qldm@example.com" - to: ["data-team@example.com", "ops@example.com"] - on_success: false - on_failure: true - - - name: integrity_check - description: Hourly data integrity verification - datasets: [] # No data fetch, just validation - schedule: - type: interval - interval: 3600 # Every hour - enabled: true - on_error: continue - post_hooks: - - "python /opt/qldm/scripts/integrity_check.py --all" - - "python /opt/qldm/scripts/update_dashboard.py" - -# Production-specific settings -storage: - strategy: hive - base_path: /mnt/data/qldm - compression: zstd - partition_granularity: month diff --git a/examples/corporate_actions_validation_results.md b/examples/corporate_actions_validation_results.md deleted file mode 100644 index cd2c0bc4..00000000 --- a/examples/corporate_actions_validation_results.md +++ /dev/null @@ -1,103 +0,0 @@ -# Corporate Actions Validation Results - -## Summary - -Validated the corporate actions adjustment algorithm against Quandl WIKI dataset (3,199 tickers). - -## Single Stock Validation (AAPL) - -**Stock**: AAPL (1980-2018, 9,400 records) -**Corporate Actions**: 4 splits, 54 dividends -**Result**: ✅ **PASS** - -- Max error: 0.0249% ($0.02) -- Mean error: 0.0108% -- Cumulative split factor: 56x -- Total adjustment: 68x (splits + dividends) - -## Multi-Stock Validation (100 Random Stocks) - -**Sample**: 100 randomly selected tickers (seed=42) -**Date Range**: 1990-2018 -**Avg Records**: 4,619 per ticker -**Avg Splits**: 1.1 per ticker -**Avg Dividends**: 38.1 per ticker - -### Pass Rates by Tolerance - -| Tolerance | Pass Rate | Use Case | -|-----------|-----------|----------| -| 0.01% | 36% | Production | -| 0.05% | 45% | High precision | -| 0.50% | **79%** | **Educational** | -| 1.00% | 84% | General research | -| 5.00% | 90% | Approximate analysis | - -### Error Distribution - -| Category | Error Range | Count | Percentage | -|----------|-------------|-------|------------| -| Excellent | < 0.05% | 45 | 45% | -| Good | 0.05% - 0.5% | 34 | 34% | -| Acceptable | 0.5% - 5% | 11 | 11% | -| Poor | 5% - 50% | 5 | 5% | -| Failed | > 50% | 5 | 5% | - -### Correlation with Corporate Actions - -**High Error Stocks (>5% error)**: -- Avg splits: 2.6 -- Avg dividends: 69.9 -- Characteristics: Complex histories, reverse splits - -**Low Error Stocks (<0.5% error)**: -- Avg splits: 1.3 -- Avg dividends: 29.1 -- Characteristics: Standard splits, moderate dividends - -## Known Limitations - -### 1. Reverse Splits (Confirmed Issue) - -Stocks with reverse splits show higher errors: -- **DWSN**: 1-for-3 reverse split → 639% error -- **ATI**: 1-for-2 reverse split → 104% error - -### 2. Complex Corporate Action Histories - -Stocks with >50 dividends AND multiple splits accumulate rounding errors over long time periods (30+ years of daily calculations). - -### 3. Floating Point Precision - -With 9,400+ iterative calculations, floating point rounding can accumulate to ~0.03% error even with perfect logic. - -## Conclusion - -**For Educational Use (ML4T Book)**: ✅ **Excellent** - -The algorithm: -- Correctly implements industry-standard methodology -- Validates at >79% pass rate for <0.5% tolerance -- Works for all standard corporate actions (normal splits, dividends) -- Demonstrates the concept clearly with real-world validation - -**Known trade-offs**: -- Reverse splits require special handling (future enhancement) -- Very complex histories (>50 dividends + multiple splits) may need higher tolerance -- Appropriate for teaching, not production-critical financial applications - -## Recommendations - -**For Book Chapter**: -1. Use AAPL as primary example (perfect validation) -2. Mention 79% validation rate on 100 stocks -3. Note limitation with reverse splits -4. Explain this is educational, not production code -5. Discuss how real-world systems handle edge cases - -**For Production Use**: -1. Add reverse split detection and special handling -2. Use higher precision arithmetic for long histories -3. Implement data quality checks -4. Validate against multiple data sources -5. Add comprehensive unit tests for edge cases diff --git a/examples/notebooks/01_getting_started.ipynb b/examples/notebooks/01_getting_started.ipynb deleted file mode 100644 index dc886b78..00000000 --- a/examples/notebooks/01_getting_started.ipynb +++ /dev/null @@ -1,509 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# QLDM Getting Started Tutorial\n", - "\n", - "This notebook demonstrates the basic usage of QLDM for fetching, analyzing, and visualizing financial data." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1. Setup and Initialization" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Import required libraries\n", - "\n", - "import plotly.graph_objects as go\n", - "import polars as pl\n", - "from plotly.subplots import make_subplots\n", - "\n", - "# Import QLDM\n", - "from qldm import QLDM\n", - "from qldm.performance import profile_function\n", - "from qldm.validation import OHLCVValidator\n", - "\n", - "# Initialize QLDM\n", - "qldm = QLDM()\n", - "print(\"QLDM initialized successfully!\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2. Fetching Data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Fetch Apple stock data for 2024\n", - "symbol = \"AAPL\"\n", - "start_date = \"2024-01-01\"\n", - "end_date = \"2024-06-30\"\n", - "\n", - "\n", - "# Fetch data with profiling\n", - "@profile_function\n", - "def fetch_stock_data(symbol, start, end):\n", - " return qldm.fetch(symbol=symbol, provider=\"yahoo\", start=start, end=end)\n", - "\n", - "\n", - "df, profile = fetch_stock_data(symbol, start_date, end_date)\n", - "print(f\"Fetched {len(df)} rows in {profile.execution_time:.2f} seconds\")\n", - "print(f\"Memory used: {profile.memory_used:.2f} MB\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Display basic information about the data\n", - "print(f\"Symbol: {symbol}\")\n", - "print(f\"Date range: {df['timestamp'].min()} to {df['timestamp'].max()}\")\n", - "print(f\"Number of trading days: {len(df)}\")\n", - "print(\"\\nFirst 5 rows:\")\n", - "df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3. Data Validation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Validate the data\n", - "validator = OHLCVValidator()\n", - "validation_result = validator.validate(df)\n", - "\n", - "if validation_result.passed:\n", - " print(\"✅ Data validation passed!\")\n", - "else:\n", - " print(\"❌ Data validation failed:\")\n", - " for issue in validation_result.issues:\n", - " print(f\" - {issue.severity}: {issue.message}\")\n", - "\n", - "# Display validation summary\n", - "print(\"\\nValidation Summary:\")\n", - "print(f\"- Critical issues: {validation_result.critical_count}\")\n", - "print(f\"- Errors: {validation_result.error_count}\")\n", - "print(f\"- Warnings: {validation_result.warning_count}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 4. Basic Analysis" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Calculate basic statistics\n", - "stats = {\n", - " \"Mean Close\": df[\"close\"].mean(),\n", - " \"Median Close\": df[\"close\"].median(),\n", - " \"Std Dev\": df[\"close\"].std(),\n", - " \"Min Price\": df[\"low\"].min(),\n", - " \"Max Price\": df[\"high\"].max(),\n", - " \"Total Volume\": df[\"volume\"].sum(),\n", - " \"Avg Daily Volume\": df[\"volume\"].mean(),\n", - "}\n", - "\n", - "print(\"Statistical Summary:\")\n", - "for key, value in stats.items():\n", - " if \"Volume\" in key:\n", - " print(f\"{key}: {value:,.0f}\")\n", - " else:\n", - " print(f\"{key}: ${value:.2f}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Calculate returns\n", - "df = df.with_columns(\n", - " [\n", - " ((pl.col(\"close\") - pl.col(\"close\").shift(1)) / pl.col(\"close\").shift(1) * 100).alias(\n", - " \"daily_return\"\n", - " )\n", - " ]\n", - ")\n", - "\n", - "# Calculate cumulative returns\n", - "df = df.with_columns([((pl.col(\"close\") / df[\"close\"][0] - 1) * 100).alias(\"cumulative_return\")])\n", - "\n", - "# Display return statistics\n", - "print(\"Return Analysis:\")\n", - "print(f\"Total Return: {df['cumulative_return'][-1]:.2f}%\")\n", - "print(f\"Average Daily Return: {df['daily_return'].mean():.3f}%\")\n", - "print(f\"Volatility (Daily Std): {df['daily_return'].std():.3f}%\")\n", - "print(f\"Best Day: {df['daily_return'].max():.2f}%\")\n", - "print(f\"Worst Day: {df['daily_return'].min():.2f}%\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 5. Visualization" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Create candlestick chart with volume\n", - "fig = make_subplots(\n", - " rows=2,\n", - " cols=1,\n", - " shared_xaxes=True,\n", - " vertical_spacing=0.03,\n", - " subplot_titles=(f\"{symbol} Stock Price\", \"Volume\"),\n", - " row_heights=[0.7, 0.3],\n", - ")\n", - "\n", - "# Add candlestick chart\n", - "fig.add_trace(\n", - " go.Candlestick(\n", - " x=df[\"timestamp\"],\n", - " open=df[\"open\"],\n", - " high=df[\"high\"],\n", - " low=df[\"low\"],\n", - " close=df[\"close\"],\n", - " name=\"OHLC\",\n", - " ),\n", - " row=1,\n", - " col=1,\n", - ")\n", - "\n", - "# Add volume bar chart\n", - "fig.add_trace(\n", - " go.Bar(x=df[\"timestamp\"], y=df[\"volume\"], name=\"Volume\", marker_color=\"lightblue\"), row=2, col=1\n", - ")\n", - "\n", - "# Update layout\n", - "fig.update_layout(\n", - " title=f\"{symbol} Price and Volume Chart\",\n", - " yaxis_title=\"Price ($)\",\n", - " yaxis2_title=\"Volume\",\n", - " xaxis2_title=\"Date\",\n", - " height=600,\n", - " showlegend=False,\n", - ")\n", - "\n", - "fig.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Plot returns distribution\n", - "import plotly.express as px\n", - "\n", - "fig = px.histogram(\n", - " df.filter(pl.col(\"daily_return\").is_not_null()),\n", - " x=\"daily_return\",\n", - " nbins=30,\n", - " title=f\"{symbol} Daily Returns Distribution\",\n", - " labels={\"daily_return\": \"Daily Return (%)\", \"count\": \"Frequency\"},\n", - ")\n", - "\n", - "fig.add_vline(x=0, line_dash=\"dash\", line_color=\"red\")\n", - "fig.add_vline(\n", - " x=df[\"daily_return\"].mean(), line_dash=\"dash\", line_color=\"green\", annotation_text=\"Mean\"\n", - ")\n", - "\n", - "fig.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 6. Multiple Symbols Comparison" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Fetch data for multiple symbols\n", - "symbols = [\"AAPL\", \"MSFT\", \"GOOGL\"]\n", - "data = {}\n", - "\n", - "for symbol in symbols:\n", - " try:\n", - " df_temp = qldm.fetch(symbol=symbol, provider=\"yahoo\", start=start_date, end=end_date)\n", - " # Calculate cumulative returns\n", - " df_temp = df_temp.with_columns(\n", - " [((pl.col(\"close\") / df_temp[\"close\"][0] - 1) * 100).alias(\"return\")]\n", - " )\n", - " data[symbol] = df_temp\n", - " print(f\"✅ Fetched {symbol}: {len(df_temp)} rows\")\n", - " except Exception as e:\n", - " print(f\"❌ Failed to fetch {symbol}: {e}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Plot comparative returns\n", - "fig = go.Figure()\n", - "\n", - "for symbol, df in data.items():\n", - " fig.add_trace(go.Scatter(x=df[\"timestamp\"], y=df[\"return\"], mode=\"lines\", name=symbol))\n", - "\n", - "fig.update_layout(\n", - " title=\"Cumulative Returns Comparison\",\n", - " xaxis_title=\"Date\",\n", - " yaxis_title=\"Cumulative Return (%)\",\n", - " hovermode=\"x unified\",\n", - " height=500,\n", - ")\n", - "\n", - "fig.show()\n", - "\n", - "# Print final returns\n", - "print(\"\\nFinal Returns:\")\n", - "for symbol, df in data.items():\n", - " final_return = df[\"return\"][-1]\n", - " print(f\"{symbol}: {final_return:.2f}%\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 7. Technical Indicators" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Calculate moving averages\n", - "df = data[\"AAPL\"].with_columns(\n", - " [\n", - " pl.col(\"close\").rolling_mean(window_size=20).alias(\"ma20\"),\n", - " pl.col(\"close\").rolling_mean(window_size=50).alias(\"ma50\"),\n", - " ]\n", - ")\n", - "\n", - "\n", - "# Calculate RSI\n", - "def calculate_rsi(df, period=14):\n", - " delta = df[\"close\"] - df[\"close\"].shift(1)\n", - " gain = pl.when(delta > 0).then(delta).otherwise(0)\n", - " loss = pl.when(delta < 0).then(-delta).otherwise(0)\n", - "\n", - " avg_gain = gain.rolling_mean(window_size=period)\n", - " avg_loss = loss.rolling_mean(window_size=period)\n", - "\n", - " rs = avg_gain / avg_loss\n", - " rsi = 100 - (100 / (1 + rs))\n", - "\n", - " return rsi\n", - "\n", - "\n", - "df = df.with_columns([calculate_rsi(df).alias(\"rsi\")])\n", - "\n", - "# Plot price with indicators\n", - "fig = make_subplots(\n", - " rows=2,\n", - " cols=1,\n", - " shared_xaxes=True,\n", - " vertical_spacing=0.03,\n", - " subplot_titles=(\"Price with Moving Averages\", \"RSI\"),\n", - " row_heights=[0.7, 0.3],\n", - ")\n", - "\n", - "# Price and MAs\n", - "fig.add_trace(\n", - " go.Scatter(x=df[\"timestamp\"], y=df[\"close\"], name=\"Close\", line={\"color\": \"blue\"}),\n", - " row=1,\n", - " col=1,\n", - ")\n", - "fig.add_trace(\n", - " go.Scatter(x=df[\"timestamp\"], y=df[\"ma20\"], name=\"MA20\", line={\"color\": \"orange\"}),\n", - " row=1,\n", - " col=1,\n", - ")\n", - "fig.add_trace(\n", - " go.Scatter(x=df[\"timestamp\"], y=df[\"ma50\"], name=\"MA50\", line={\"color\": \"red\"}), row=1, col=1\n", - ")\n", - "\n", - "# RSI\n", - "fig.add_trace(\n", - " go.Scatter(x=df[\"timestamp\"], y=df[\"rsi\"], name=\"RSI\", line={\"color\": \"purple\"}), row=2, col=1\n", - ")\n", - "fig.add_hline(y=70, line_dash=\"dash\", line_color=\"red\", row=2, col=1)\n", - "fig.add_hline(y=30, line_dash=\"dash\", line_color=\"green\", row=2, col=1)\n", - "\n", - "fig.update_layout(height=600, title=\"Technical Analysis\")\n", - "fig.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 8. Export Results" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Export to different formats\n", - "output_dir = \"./output\"\n", - "import os\n", - "\n", - "os.makedirs(output_dir, exist_ok=True)\n", - "\n", - "# Export to CSV\n", - "csv_path = f\"{output_dir}/aapl_analysis.csv\"\n", - "df.write_csv(csv_path)\n", - "print(f\"✅ Exported to CSV: {csv_path}\")\n", - "\n", - "# Export to Parquet (efficient format)\n", - "parquet_path = f\"{output_dir}/aapl_analysis.parquet\"\n", - "df.write_parquet(parquet_path)\n", - "print(f\"✅ Exported to Parquet: {parquet_path}\")\n", - "\n", - "# Export summary statistics\n", - "summary = pl.DataFrame({\"Metric\": list(stats.keys()), \"Value\": list(stats.values())})\n", - "summary_path = f\"{output_dir}/summary_stats.csv\"\n", - "summary.write_csv(summary_path)\n", - "print(f\"✅ Exported summary: {summary_path}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 9. Performance Optimization" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Demonstrate caching\n", - "from qldm.performance import cache_result, get_cache_stats\n", - "\n", - "\n", - "@cache_result(ttl_seconds=300) # Cache for 5 minutes\n", - "def get_expensive_data(symbol):\n", - " print(f\"Fetching data for {symbol}...\")\n", - " return qldm.fetch(symbol, start=\"2024-01-01\", end=\"2024-06-30\")\n", - "\n", - "\n", - "# First call - will fetch from provider\n", - "import time\n", - "\n", - "start = time.time()\n", - "df1 = get_expensive_data(\"MSFT\")\n", - "print(f\"First call took: {time.time() - start:.2f} seconds\")\n", - "\n", - "# Second call - will use cache\n", - "start = time.time()\n", - "df2 = get_expensive_data(\"MSFT\")\n", - "print(f\"Second call took: {time.time() - start:.4f} seconds (cached)\")\n", - "\n", - "# Display cache statistics\n", - "stats = get_cache_stats()\n", - "print(\"\\nCache Statistics:\")\n", - "print(f\"Hit rate: {stats['memory_cache']['hit_rate']:.1%}\")\n", - "print(f\"Total entries: {stats['memory_cache']['total_entries']}\")\n", - "print(f\"Cache size: {stats['memory_cache']['total_size_mb']:.2f} MB\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 10. Conclusion\n", - "\n", - "This notebook demonstrated:\n", - "- Fetching financial data using QLDM\n", - "- Data validation and quality checks\n", - "- Basic statistical analysis\n", - "- Visualization with interactive charts\n", - "- Multi-symbol comparison\n", - "- Technical indicators calculation\n", - "- Data export to various formats\n", - "- Performance optimization with caching\n", - "\n", - "For more advanced usage, check out the other example notebooks and the full documentation." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.0" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/notebooks/01_quickstart.ipynb b/examples/notebooks/01_quickstart.ipynb deleted file mode 100644 index 32e63fc6..00000000 --- a/examples/notebooks/01_quickstart.ipynb +++ /dev/null @@ -1,323 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# QData Quickstart Notebook\n", - "\n", - "**Get started with QData in 5 minutes!**\n", - "\n", - "This notebook demonstrates:\n", - "- Fetching cryptocurrency data (no API key needed)\n", - "- Fetching stock data with a free API key\n", - "- Basic data analysis and visualization\n", - "- Working with multiple providers\n", - "\n", - "**Prerequisites**: `pip install qdata matplotlib`" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1. Fetch Crypto Data (No API Key Required!)\n", - "\n", - "CoinGecko is the easiest way to get started - no API key needed." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from datetime import UTC, datetime, timedelta\n", - "\n", - "from ml4t.data.providers import CoinGeckoProvider\n", - "\n", - "# Calculate the most recent 29 completed UTC days\n", - "end = datetime.now(UTC).date() - timedelta(days=1)\n", - "start = end - timedelta(days=28)\n", - "start_date = start.isoformat()\n", - "end_date = end.isoformat()\n", - "\n", - "print(f\"Fetching Bitcoin data from {start_date} to {end_date}...\")\n", - "\n", - "# Create provider (no API key needed!)\n", - "provider = CoinGeckoProvider()\n", - "\n", - "# Fetch Bitcoin data\n", - "btc_data = provider.fetch_ohlcv(\"bitcoin\", start_date, end_date)\n", - "\n", - "print(f\"\\n✅ Successfully fetched {len(btc_data)} days of Bitcoin data!\")\n", - "btc_data.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2. Visualize Price Data\n", - "\n", - "Let's create a simple price chart with matplotlib." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import matplotlib.dates as mdates\n", - "import matplotlib.pyplot as plt\n", - "\n", - "# Convert Polars DataFrame to pandas for plotting\n", - "btc_pd = btc_data.to_pandas()\n", - "\n", - "# Create figure\n", - "fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(14, 8), sharex=True)\n", - "\n", - "# Plot 1: Price with High/Low bands\n", - "ax1.plot(btc_pd[\"timestamp\"], btc_pd[\"close\"], label=\"Close\", color=\"blue\", linewidth=2)\n", - "ax1.fill_between(\n", - " btc_pd[\"timestamp\"], btc_pd[\"low\"], btc_pd[\"high\"], alpha=0.2, label=\"Daily Range\", color=\"gray\"\n", - ")\n", - "ax1.set_ylabel(\"Price (USD)\", fontsize=12)\n", - "ax1.set_title(\"Bitcoin Price (Last 30 Days)\", fontsize=14, fontweight=\"bold\")\n", - "ax1.legend()\n", - "ax1.grid(True, alpha=0.3)\n", - "\n", - "# Plot 2: Volume\n", - "ax2.bar(btc_pd[\"timestamp\"], btc_pd[\"volume\"], color=\"steelblue\", alpha=0.6)\n", - "ax2.set_ylabel(\"Volume (USD)\", fontsize=12)\n", - "ax2.set_xlabel(\"Date\", fontsize=12)\n", - "ax2.set_title(\"Trading Volume\", fontsize=14, fontweight=\"bold\")\n", - "ax2.grid(True, alpha=0.3)\n", - "\n", - "# Format x-axis\n", - "ax2.xaxis.set_major_formatter(mdates.DateFormatter(\"%Y-%m-%d\"))\n", - "plt.xticks(rotation=45)\n", - "\n", - "plt.tight_layout()\n", - "plt.show()\n", - "\n", - "print(\"\\n📊 Price Stats:\")\n", - "print(f\" High: ${btc_pd['high'].max():,.2f}\")\n", - "print(f\" Low: ${btc_pd['low'].min():,.2f}\")\n", - "print(f\" Latest: ${btc_pd['close'].iloc[-1]:,.2f}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3. Calculate Returns and Volatility\n", - "\n", - "Let's calculate some basic metrics." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Calculate daily returns\n", - "btc_data = btc_data.with_columns(\n", - " ((btc_data[\"close\"] - btc_data[\"close\"].shift(1)) / btc_data[\"close\"].shift(1) * 100).alias(\n", - " \"daily_return_pct\"\n", - " )\n", - ")\n", - "\n", - "# Calculate rolling volatility (7-day)\n", - "btc_data = btc_data.with_columns(\n", - " btc_data[\"daily_return_pct\"].rolling_std(window_size=7).alias(\"volatility_7d\")\n", - ")\n", - "\n", - "print(\"📈 Returns Analysis (Last 30 Days):\")\n", - "print(f\" Total Return: {btc_data['daily_return_pct'].sum():.2f}%\")\n", - "print(f\" Best Day: {btc_data['daily_return_pct'].max():.2f}%\")\n", - "print(f\" Worst Day: {btc_data['daily_return_pct'].min():.2f}%\")\n", - "print(f\" Avg Daily Return: {btc_data['daily_return_pct'].mean():.2f}%\")\n", - "print(f\" Volatility (std): {btc_data['daily_return_pct'].std():.2f}%\")\n", - "\n", - "# Show data with new columns\n", - "btc_data.select([\"timestamp\", \"close\", \"daily_return_pct\", \"volatility_7d\"]).tail(10)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 4. Fetch Stock Data (Requires Free API Key)\n", - "\n", - "For stocks, you'll need a free API key from [Tiingo](https://api.tiingo.com/account/api/token).\n", - "\n", - "**Get free key**: https://api.tiingo.com/account/api/token (1000 calls/day free)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "# Check for API key\n", - "tiingo_key = os.getenv(\"TIINGO_API_KEY\")\n", - "\n", - "if not tiingo_key:\n", - " print(\"⚠️ TIINGO_API_KEY not set!\")\n", - " print(\" Get free key at: https://api.tiingo.com/account/api/token\")\n", - " print(\" Then set: export TIINGO_API_KEY='your_key_here'\")\n", - "else:\n", - " from qdata.providers import TiingoProvider\n", - "\n", - " # Create provider\n", - " tiingo = TiingoProvider(api_key=tiingo_key)\n", - "\n", - " # Fetch Apple stock data\n", - " aapl_data = tiingo.fetch_ohlcv(\"AAPL\", start_date, end_date)\n", - "\n", - " print(f\"✅ Fetched {len(aapl_data)} days of AAPL data\")\n", - " aapl_data.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 5. Compare Multiple Assets\n", - "\n", - "Let's compare Bitcoin and Apple stock performance." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "if tiingo_key:\n", - " # Normalize prices to 100 at start\n", - " btc_pd = btc_data.to_pandas()\n", - " aapl_pd = aapl_data.to_pandas()\n", - "\n", - " btc_pd[\"normalized\"] = btc_pd[\"close\"] / btc_pd[\"close\"].iloc[0] * 100\n", - " aapl_pd[\"normalized\"] = aapl_pd[\"close\"] / aapl_pd[\"close\"].iloc[0] * 100\n", - "\n", - " # Plot comparison\n", - " plt.figure(figsize=(14, 6))\n", - " plt.plot(btc_pd[\"timestamp\"], btc_pd[\"normalized\"], label=\"Bitcoin\", linewidth=2)\n", - " plt.plot(aapl_pd[\"timestamp\"], aapl_pd[\"normalized\"], label=\"Apple (AAPL)\", linewidth=2)\n", - " plt.axhline(y=100, color=\"gray\", linestyle=\"--\", alpha=0.5)\n", - " plt.ylabel(\"Normalized Price (Start = 100)\", fontsize=12)\n", - " plt.xlabel(\"Date\", fontsize=12)\n", - " plt.title(\"Bitcoin vs Apple Stock (Normalized)\", fontsize=14, fontweight=\"bold\")\n", - " plt.legend(fontsize=12)\n", - " plt.grid(True, alpha=0.3)\n", - " plt.tight_layout()\n", - " plt.show()\n", - "\n", - " print(\"\\n📊 Performance Comparison:\")\n", - " btc_return = (btc_pd[\"close\"].iloc[-1] / btc_pd[\"close\"].iloc[0] - 1) * 100\n", - " aapl_return = (aapl_pd[\"close\"].iloc[-1] / aapl_pd[\"close\"].iloc[0] - 1) * 100\n", - " print(f\" Bitcoin: {btc_return:+.2f}%\")\n", - " print(f\" AAPL: {aapl_return:+.2f}%\")\n", - "else:\n", - " print(\"⚠️ Set TIINGO_API_KEY to run comparison\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 6. Try Multiple Cryptocurrencies\n", - "\n", - "CoinGecko supports 10,000+ cryptocurrencies!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Fetch multiple cryptocurrencies\n", - "cryptos = [\"bitcoin\", \"ethereum\", \"cardano\", \"ripple\"]\n", - "crypto_data = {}\n", - "\n", - "for crypto in cryptos:\n", - " data = provider.fetch_ohlcv(crypto, start_date, end_date)\n", - " crypto_data[crypto] = data\n", - " print(f\"✅ {crypto.capitalize()}: {len(data)} days\")\n", - "\n", - "# Create comparison chart\n", - "plt.figure(figsize=(14, 6))\n", - "\n", - "for crypto, data in crypto_data.items():\n", - " df = data.to_pandas()\n", - " normalized = df[\"close\"] / df[\"close\"].iloc[0] * 100\n", - " plt.plot(df[\"timestamp\"], normalized, label=crypto.capitalize(), linewidth=2)\n", - "\n", - "plt.axhline(y=100, color=\"gray\", linestyle=\"--\", alpha=0.5)\n", - "plt.ylabel(\"Normalized Price (Start = 100)\", fontsize=12)\n", - "plt.xlabel(\"Date\", fontsize=12)\n", - "plt.title(\"Cryptocurrency Comparison (Normalized)\", fontsize=14, fontweight=\"bold\")\n", - "plt.legend(fontsize=12)\n", - "plt.grid(True, alpha=0.3)\n", - "plt.tight_layout()\n", - "plt.show()\n", - "\n", - "print(\"\\n📊 30-Day Returns:\")\n", - "for crypto, data in crypto_data.items():\n", - " df = data.to_pandas()\n", - " ret = (df[\"close\"].iloc[-1] / df[\"close\"].iloc[0] - 1) * 100\n", - " print(f\" {crypto.capitalize():12s}: {ret:+7.2f}%\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Next Steps\n", - "\n", - "**Learn More**:\n", - "- [Provider Selection Guide](../../docs/getting-started/provider-selection.md) - Choose the right data source\n", - "- [Tutorial 02: Rate Limiting](../../docs/tutorials/02_rate_limiting.md) - Avoid API bans\n", - "- [Tutorial 03: Incremental Updates](../../docs/tutorials/03_incremental_updates.md) - Save API calls\n", - "\n", - "**Try More Notebooks**:\n", - "- `02_provider_comparison.ipynb` - Compare data quality across providers\n", - "- `03_incremental_updates.ipynb` - Smart data pipeline patterns\n", - "- `04_multi_asset_portfolio.ipynb` - Build a diversified portfolio\n", - "- `05_data_quality.ipynb` - Validate and clean market data\n", - "\n", - "**Happy trading!** 🚀" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.0" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/notebooks/README.md b/examples/notebooks/README.md deleted file mode 100644 index dde7054b..00000000 --- a/examples/notebooks/README.md +++ /dev/null @@ -1,215 +0,0 @@ -# ML4T Data Interactive Notebooks - -Jupyter notebooks demonstrating ML4T Data usage with real examples and visualizations. - -## Available Notebooks - -### 1. [Quickstart Notebook](01_quickstart.ipynb) ✅ -**Get started in 5 minutes!** - -Learn: -- Fetch crypto data (no API key needed!) -- Fetch stock data with free Tiingo API -- Visualize prices and volume -- Calculate returns and volatility -- Compare multiple assets - -**Perfect for beginners!** - -### 2. Provider Comparison (Coming Soon) -**Compare data quality across providers** - -Topics: -- Fetch same symbol from multiple providers -- Compare close prices and volume -- Identify discrepancies -- Measure latency and reliability -- Choose the best provider for your needs - -### 3. Incremental Updates (Coming Soon) -**Build efficient data pipelines** - -Topics: -- The naive approach vs. incremental updates -- ProviderUpdater pattern demonstration -- Gap detection and backfilling -- Production update scheduling -- Monitoring and alerting - -### 4. Multi-Asset Portfolio (Coming Soon) -**Manage diversified portfolios** - -Topics: -- Fetch stocks, crypto, and forex -- Asset class routing strategies -- Portfolio analytics and rebalancing -- Risk metrics (volatility, correlation) -- Performance attribution - -### 5. Data Quality Validation (Coming Soon) -**Ensure data integrity** - -Topics: -- OHLCV invariant validation -- Detecting anomalies (spikes, gaps) -- Cross-provider validation -- Cleaning and filling missing data -- Production data quality monitoring - -## Running the Notebooks - -### Installation - -```bash -# Install ML4T Data with visualization dependencies -pip install ml4t-data matplotlib seaborn jupyter - -# Or with all optional dependencies -pip install ml4t-data[viz] -``` - -### Launch Jupyter - -```bash -cd examples/notebooks -jupyter notebook -``` - -Then open any `.ipynb` file in your browser. - -### API Keys - -Some notebooks require free API keys: - -- **Tiingo** (stocks): Get free key at https://api.tiingo.com/account/api/token - - 1000 calls/day free tier - - Set: `export TIINGO_API_KEY="your_key"` - -- **EODHD** (global stocks): Get free key at https://eodhd.com/register - - 500 calls/day free tier - - Set: `export EODHD_API_KEY="your_key"` - -- **CoinGecko** (crypto): No API key needed! ✅ - -### Tips for Notebooks - -1. **Run cells in order** - Each cell builds on previous ones -2. **Restart kernel** if you get errors - `Kernel → Restart & Clear Output` -3. **Save your work** - Changes aren't saved automatically -4. **Experiment!** - Modify the code to explore different symbols/date ranges - -## Notebook Features - -### Data Fetching -```python -from ml4t-data.providers import CoinGeckoProvider - -provider = CoinGeckoProvider() -data = provider.fetch_ohlcv("bitcoin", "2024-01-01", "2024-12-31") -``` - -### Visualization -```python -import matplotlib.pyplot as plt - -df = data.to_pandas() -plt.plot(df['timestamp'], df['close']) -plt.show() -``` - -### Analysis -```python -# Calculate returns -data = data.with_columns( - ((data["close"] - data["close"].shift(1)) / data["close"].shift(1) * 100) - .alias("daily_return") -) - -print(f"Avg daily return: {data['daily_return'].mean():.2f}%") -``` - -## Learning Path - -1. **Start with Quickstart** (01_quickstart.ipynb) - - Learn basic data fetching - - Understand OHLCV format - - Create simple visualizations - -2. **Read the Tutorials** ([docs/tutorials/](../../docs/tutorials/)) - - Deeper explanations of concepts - - Best practices and patterns - - Production-ready code - -3. **Explore Advanced Notebooks** (02-05) - - Provider comparison strategies - - Production data pipelines - - Portfolio management - - Data quality validation - -4. **Build Your Own Analysis** - - Use notebooks as templates - - Customize for your strategies - - Share with the community! - -## Troubleshooting - -### "ModuleNotFoundError: No module named 'ml4t-data'" - -```bash -pip install ml4t-data -``` - -### "No API key found" - -Set environment variables: -```bash -export TIINGO_API_KEY="your_key_here" -export EODHD_API_KEY="your_key_here" -``` - -Or set in notebook: -```python -import os -os.environ["TIINGO_API_KEY"] = "your_key_here" -``` - -### "Rate limit exceeded" - -- You've hit provider's daily limit -- Wait until next day (limits reset at midnight UTC) -- Or upgrade to paid tier -- See [Tutorial 02: Rate Limiting](../../docs/tutorials/02_rate_limiting.md) - -### Notebook won't start - -```bash -# Reinstall jupyter -pip install --upgrade jupyter notebook - -# Or try JupyterLab -pip install jupyterlab -jupyter lab -``` - -## Contributing - -Have a great notebook example? We'd love to see it! - -1. Create your notebook in this directory -2. Follow the naming convention: `NN_descriptive_name.ipynb` -3. Add markdown cells explaining what you're doing -4. Test it works from a fresh kernel -5. Submit a PR! - -See [CONTRIBUTING.md](../../CONTRIBUTING.md) for details. - -## Related Resources - -- **[Tutorials](../../docs/tutorials/)** - Step-by-step written guides -- **[Examples](../)** - Python scripts -- **[Provider Selection Guide](../../docs/getting-started/provider-selection.md)** - Choose providers -- **[API Documentation](../../docs/api/)** - Complete API reference - ---- - -**Happy analyzing!** 📊 diff --git a/examples/quandl_extension_README.md b/examples/quandl_extension_README.md deleted file mode 100644 index c5173d43..00000000 --- a/examples/quandl_extension_README.md +++ /dev/null @@ -1,322 +0,0 @@ -# Quandl Data Extension Example - -**Based on**: `/home/stefan/ml3t/data/equities/quandl/` (7,409 tickers, 1962-2025) -**Purpose**: Demonstrate extending historical Quandl WIKI data using ML4T Data's YahooFinanceProvider - -## Overview - -This example shows how to use our `YahooFinanceProvider` to extend the discontinued Quandl WIKI dataset (ended 2018-03-27) with current Yahoo Finance data. It demonstrates production-grade patterns for: - -1. **Bulk downloading** with proper batching -2. **Rate limiting** (Yahoo has no documented limits but throttles aggressively) -3. **Error handling** (DataNotAvailable, RateLimit, Network errors) -4. **Data continuity validation** between sources -5. **Progress tracking** and resumability - -## What We Learned About Our YahooFinanceProvider - -### ✅ Rate Limiting Behavior - -**Observed pattern**: -``` -2025-11-15 21:19:15 [info] Fetching data from Yahoo Finance -2025-11-15 21:19:16 [debug] Rate limit reached, waiting calls_in_window=1 wait_seconds=1.20 -2025-11-15 21:19:17 [info] Fetching data from Yahoo Finance -2025-11-15 21:19:18 [debug] Rate limit reached, waiting calls_in_window=1 wait_seconds=1.36 -``` - -**Key findings**: -- Default rate limit: **0.5 requests/second** (2-second period) -- Automatic rate limiting enforcement via `BaseProvider` -- Wait times vary: 1.17s - 1.38s between requests -- No explicit RateLimitError exceptions thrown (handled internally) -- **Average: 1.74 seconds per ticker** (including processing time) - -**Configuration**: -```python -provider = YahooFinanceProvider( - max_requests_per_second=0.5 # Conservative default -) -``` - -### ✅ Error Handling - -The provider correctly handles: - -1. **DataNotAvailableError**: When symbol not found or no data -2. **NetworkError**: HTTP errors, connection failures -3. **RateLimitError**: (internal) Automatically retries with backoff - -**Example retry pattern** (from quandl_extension_example.py): -```python -except RateLimitError as e: - logger.warning("Rate limited", ticker=ticker, retry_after=e.retry_after) - time.sleep(5) - return self.extend_single_ticker(ticker, start_date, end_date) -``` - -### ✅ Data Quality - -**5 tickers tested** (AAPL, MSFT, TSLA, GOOGL, NVDA): -- **Success rate**: 100% (5/5) -- **Records fetched**: 1,921 rows each (2018-03-28 to 2025-11-14) -- **Data consistency**: All tickers have identical date range -- **Missing data**: None (continuous daily data) - -**Column schema**: -``` -timestamp datetime[μs] -open f64 -high f64 -low f64 -close f64 -volume f64 -ticker str # Added manually (provider doesn't include this) -``` - -### ⚠️ Important Differences vs Other Providers - -**YahooFinanceProvider does NOT add a `symbol` column**: -```python -# Other providers (EODHD, Polygon, etc.) -df.columns # ['timestamp', 'symbol', 'open', 'high', 'low', 'close', 'volume'] - -# YahooFinanceProvider -df.columns # ['timestamp', 'open', 'high', 'low', 'close', 'volume'] -``` - -**Workaround**: -```python -df = provider.fetch_ohlcv("AAPL", "2018-03-28", "2025-11-15") -df = df.with_columns(pl.lit("AAPL").alias("ticker")) # Add ticker manually -``` - -### ✅ Performance Metrics - -**Test configuration**: -- 5 tickers -- ~7 years of data per ticker (2018-03-28 to 2025-11-14) -- Batch size: 5 -- Delay between batches: 2.0s - -**Results**: -``` -Total tickers: 5 -Successful: 5 -Failed: 0 -Data unavailable: 0 -Rate limited: 0 # Handled internally -Network errors: 0 -Time elapsed: 8.7 seconds -Avg time per ticker: 1.74 seconds -``` - -**Throughput**: -- **~0.57 requests/second** (actual) -- **~34 requests/minute** -- **~2,052 requests/hour** - -**Comparison to ml3t/quandl approach**: -```python -# ml3t approach (direct yfinance) -yf_ticker = yf.Ticker(ticker) -hist = yf_ticker.history(start=start_date, auto_adjust=False, actions=True) -time.sleep(0.1) # 100ms delay - -# Our approach (ml4t-data) -df = provider.fetch_ohlcv(ticker, start_date, end_date, frequency="daily") -# Automatic rate limiting: ~1200ms delay -``` - -**Trade-off**: -- ml3t: Faster (10x) but risks throttling on large batches -- ml4t-data: Slower but reliable, no throttling, built-in error handling - -### ✅ Split Adjustment Awareness - -**Critical finding from ml3t/quandl work**: - -> Yahoo Finance "unadjusted" prices (`auto_adjust=False`) are actually **split-adjusted but not dividend-adjusted**, while Quandl WIKI data is truly unadjusted. - -**Example** (AAPL 4:1 split in 2020): -- Quandl close (2018-03-27): $168.34 -- Yahoo close (2018-03-28): $39.14 (split-adjusted) -- **Factor**: 0.25 (4:1 split) - -**Our provider**: -- Returns split-adjusted prices by default (via yfinance) -- Matches Yahoo Finance behavior -- Consistent for extending Quandl forward (no backward adjustment needed) - -## Usage Example - -```python -from pathlib import Path -from ml4t.data.providers import YahooFinanceProvider -import polars as pl - -# Initialize provider -provider = YahooFinanceProvider(max_requests_per_second=0.5) - -# Extend Quandl data for a single ticker -df = provider.fetch_ohlcv( - symbol="AAPL", - start="2018-03-28", # Day after Quandl ended - end="2025-11-15", - frequency="daily" -) - -# Add ticker column (not included by default) -df = df.with_columns(pl.lit("AAPL").alias("ticker")) - -print(f"Fetched {len(df)} rows for AAPL") -# Fetched 1921 rows for AAPL - -# Bulk download (see quandl_extension_example.py for full implementation) -``` - -## Running the Example - -```bash -# Basic example (5 tickers) -python examples/quandl_extension_example.py - -# Expected output: -# - 5 tickers successfully fetched -# - ~8.7 seconds total time -# - 9,605 total records (1,921 per ticker) -# - Clean data with no errors -``` - -## Key Learnings for ML4T Book - -### 1. Rate Limiting is Critical - -Yahoo Finance has no documented rate limits, but **aggressive throttling occurs**: -- Our default: 0.5 req/sec (conservative) -- ml3t approach: 10 req/sec (100ms delay) -- **Recommendation**: Use 0.5 req/sec for production, 1-2 req/sec for prototyping - -### 2. Data Source Migration Patterns - -When extending historical data from a discontinued source: - -1. **Start from the next day** after old source ends -2. **Validate continuity** at the transition point (check price gaps) -3. **Handle split adjustments** if combining truly unadjusted with split-adjusted -4. **Batch processing** with progress tracking and resumability -5. **Error handling** for symbol changes, delistings, missing data - -### 3. Provider Differences - -| Feature | YahooFinanceProvider | EODHD/Polygon/Tiingo | -|---------|---------------------|----------------------| -| Symbol column | ❌ No (add manually) | ✅ Yes | -| Rate limiting | ✅ Automatic | ✅ Automatic | -| Error handling | ✅ Comprehensive | ✅ Comprehensive | -| API key required | ❌ No | ✅ Yes | -| Commercial use | ❌ Violates ToS | ✅ Allowed | - -### 4. Production Recommendations - -For **educational/personal use**: -- ✅ YahooFinanceProvider (free, no key) -- ⚠️ Must respect ToS (no commercial use) -- ⚠️ No SLA, can break anytime - -For **commercial use**: -- ✅ Tiingo (500 calls/day free, commercial allowed) -- ✅ EODHD (€19.99/month, 60+ exchanges) -- ✅ Polygon ($29-199/month, US equities) - -## Comparison to ML3T Approach - -### ml3t/quandl Implementation - -```python -# create_full_seamless_dataset.py (lines 98-123) -def download_yahoo_data(self, ticker: str, start_date: str): - try: - yf_ticker = yf.Ticker(ticker) - hist = yf_ticker.history( - start=start_date, - auto_adjust=False, # Split-adjusted only - actions=True # Include dividends and splits - ) - - if hist.empty: - return None - - hist = hist.reset_index() - hist.columns = hist.columns.str.lower() - hist['ticker'] = ticker - - return hist - - except Exception as e: - logging.error(f"Failed to download Yahoo data for {ticker}: {e}") - return None - -# Rate limiting -time.sleep(0.1) # 100ms delay between requests -``` - -**Pattern**: -- Direct yfinance usage -- Manual error handling -- Manual rate limiting (100ms delay) -- Pandas DataFrame output -- No retry logic -- No circuit breaker - -### ML4T Data Approach - -```python -# quandl_extension_example.py -from ml4t.data.providers import YahooFinanceProvider - -provider = YahooFinanceProvider(max_requests_per_second=0.5) - -df = provider.fetch_ohlcv( - symbol=ticker, - start=start_date, - end=end_date, - frequency="daily" -) -``` - -**Benefits over direct yfinance**: -- ✅ **Automatic rate limiting** (configurable, enforced) -- ✅ **Built-in retry logic** with exponential backoff -- ✅ **Circuit breaker** prevents cascading failures -- ✅ **Typed exceptions** (DataNotAvailable, RateLimit, Network) -- ✅ **Polars output** (10-100x faster than pandas for large datasets) -- ✅ **Structured logging** with context -- ✅ **Consistent interface** across all providers (yahoo, tiingo, eodhd, polygon, etc.) - -**Trade-offs**: -- ❌ **Slower** (1.74s/ticker vs 0.1s/ticker for ml3t) -- ❌ **No symbol column** by default (must add manually) -- ✅ **More reliable** (no throttling on large batches) -- ✅ **Production-grade** error handling - -## Files - -- `quandl_extension_example.py` - Complete working example -- `quandl_extension_README.md` - This file -- Original ml3t implementation: `/home/stefan/ml3t/data/equities/quandl/create_full_seamless_dataset.py` - -## Next Steps - -1. **Test with larger batches** (100-500 tickers) -2. **Add data continuity validation** (compare Quandl last close vs Yahoo first close) -3. **Implement split adjustment detection** (fetch splits via yfinance, calculate factors) -4. **Add resumability** (save progress, skip already-fetched tickers) -5. **Create storage integration** (save to HiveStorage parquet format) - -## References - -- Original Quandl extension: `/home/stefan/ml3t/data/equities/quandl/` -- PROVIDER_COST_BENEFIT_ANALYSIS.md: Yahoo Finance ToS warning -- YahooFinanceProvider implementation: `src/ml4t/data/providers/yahoo.py` -- BaseProvider rate limiting: `src/ml4t/data/providers/base.py` diff --git a/examples/scripts/daily_update.py b/examples/scripts/daily_update.py deleted file mode 100644 index bcd97ac8..00000000 --- a/examples/scripts/daily_update.py +++ /dev/null @@ -1,255 +0,0 @@ -#!/usr/bin/env python3 -""" -Daily Market Data Update Script - -This script updates market data for a configured list of symbols. -It can be run manually or scheduled via cron for automated updates. - -Usage: - python daily_update.py - python daily_update.py --symbols AAPL MSFT GOOGL - python daily_update.py --config custom_config.yaml -""" - -import argparse -import logging -import sys -from datetime import datetime -from pathlib import Path - -from ml4t_data import QLDM - -from ml4t.data.validation import OHLCVValidator - -# Configure logging -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - handlers=[logging.FileHandler("daily_update.log"), logging.StreamHandler()], -) -logger = logging.getLogger(__name__) - - -# Default symbols to update -DEFAULT_SYMBOLS = [ - # US Stocks - "AAPL", - "MSFT", - "GOOGL", - "AMZN", - "META", - "TSLA", - "NVDA", - "JPM", - "V", - "JNJ", - # Cryptocurrencies (if Binance is configured) - "BTC/USDT", - "ETH/USDT", - "BNB/USDT", -] - - -class DailyUpdater: - """Handles daily market data updates.""" - - def __init__(self, config_path: str | None = None): - """ - Initialize the updater. - - Args: - config_path: Optional path to configuration file - """ - self.qldm = QLDM(config_path=config_path) if config_path else QLDM() - self.validator = OHLCVValidator() - self.update_stats = {"success": 0, "failed": 0, "skipped": 0, "validation_failed": 0} - - def update_symbol(self, symbol: str) -> bool: - """ - Update data for a single symbol. - - Args: - symbol: Symbol to update - - Returns: - True if update successful, False otherwise - """ - try: - # Determine provider based on symbol format - provider = "binance" if "/" in symbol else "yahoo" - - logger.info(f"Updating {symbol} from {provider}...") - - # Perform update - result = self.qldm.update(symbol, provider=provider) - - if result: - # Validate the updated data - df = self.qldm.get(symbol) - validation_result = self.validator.validate(df) - - if validation_result.passed: - logger.info(f"✅ {symbol}: Updated successfully, validation passed") - self.update_stats["success"] += 1 - return True - logger.warning(f"⚠️ {symbol}: Updated but validation failed") - logger.warning(f" Issues: {len(validation_result.issues)}") - for issue in validation_result.issues[:3]: # Show first 3 issues - logger.warning(f" - {issue.severity}: {issue.message}") - self.update_stats["validation_failed"] += 1 - return False - logger.info(f"⏭️ {symbol}: Already up to date") - self.update_stats["skipped"] += 1 - return True - - except Exception as e: - logger.error(f"❌ {symbol}: Update failed - {e!s}") - self.update_stats["failed"] += 1 - return False - - def update_all(self, symbols: list[str]) -> None: - """ - Update all symbols in the list. - - Args: - symbols: List of symbols to update - """ - logger.info(f"Starting daily update for {len(symbols)} symbols") - logger.info("=" * 60) - - start_time = datetime.now() - - for i, symbol in enumerate(symbols, 1): - logger.info(f"[{i}/{len(symbols)}] Processing {symbol}") - self.update_symbol(symbol) - logger.info("-" * 40) - - # Calculate duration - duration = (datetime.now() - start_time).total_seconds() - - # Print summary - self.print_summary(duration) - - def print_summary(self, duration: float) -> None: - """ - Print update summary. - - Args: - duration: Update duration in seconds - """ - logger.info("=" * 60) - logger.info("UPDATE SUMMARY") - logger.info("=" * 60) - logger.info(f"✅ Successful: {self.update_stats['success']}") - logger.info(f"⏭️ Skipped (up-to-date): {self.update_stats['skipped']}") - logger.info(f"⚠️ Validation failed: {self.update_stats['validation_failed']}") - logger.info(f"❌ Failed: {self.update_stats['failed']}") - logger.info(f"⏱️ Duration: {duration:.1f} seconds") - logger.info("=" * 60) - - # Set exit code based on results - if self.update_stats["failed"] > 0: - sys.exit(1) # Exit with error if any updates failed - - -def validate_symbols(symbols: list[str]) -> list[str]: - """ - Validate and clean symbol list. - - Args: - symbols: List of symbols - - Returns: - Cleaned list of symbols - """ - # Remove duplicates and empty strings - cleaned = list({s.strip().upper() for s in symbols if s.strip()}) - - # Validate format - valid_symbols = [] - for symbol in cleaned: - if "/" in symbol: - # Crypto pair format - parts = symbol.split("/") - if len(parts) == 2 and all(p.isalnum() for p in parts): - valid_symbols.append(symbol) - else: - logger.warning(f"Invalid crypto pair format: {symbol}") - else: - # Stock symbol format - if symbol.replace("-", "").replace(".", "").isalnum(): - valid_symbols.append(symbol) - else: - logger.warning(f"Invalid stock symbol format: {symbol}") - - return valid_symbols - - -def load_symbols_from_file(filepath: str) -> list[str]: - """ - Load symbols from a text file (one per line). - - Args: - filepath: Path to symbols file - - Returns: - List of symbols - """ - path = Path(filepath) - if not path.exists(): - raise FileNotFoundError(f"Symbols file not found: {filepath}") - - with open(path) as f: - symbols = [line.strip() for line in f if line.strip() and not line.startswith("#")] - - return symbols - - -def main(): - """Main entry point.""" - parser = argparse.ArgumentParser(description="Update market data for configured symbols") - parser.add_argument("--symbols", nargs="+", help="Symbols to update (overrides defaults)") - parser.add_argument("--symbols-file", help="File containing symbols to update (one per line)") - parser.add_argument("--config", help="Path to QLDM configuration file") - parser.add_argument( - "--dry-run", - action="store_true", - help="Show what would be updated without actually updating", - ) - - args = parser.parse_args() - - # Determine symbols to update - if args.symbols: - symbols = validate_symbols(args.symbols) - elif args.symbols_file: - symbols = validate_symbols(load_symbols_from_file(args.symbols_file)) - else: - symbols = DEFAULT_SYMBOLS - - if not symbols: - logger.error("No valid symbols to update") - sys.exit(1) - - # Dry run mode - if args.dry_run: - logger.info("DRY RUN MODE - No actual updates will be performed") - logger.info(f"Would update {len(symbols)} symbols:") - for symbol in symbols: - logger.info(f" - {symbol}") - sys.exit(0) - - # Perform updates - try: - updater = DailyUpdater(config_path=args.config) - updater.update_all(symbols) - except KeyboardInterrupt: - logger.info("\nUpdate interrupted by user") - sys.exit(1) - except Exception as e: - logger.error(f"Unexpected error: {e}") - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/examples/stress_test_comparison.py b/examples/stress_test_comparison.py index 8ce1c66f..c20d6a18 100644 --- a/examples/stress_test_comparison.py +++ b/examples/stress_test_comparison.py @@ -265,7 +265,7 @@ def main(): stats_yfinance = test_direct_yfinance(tickers, start_date, end_date) # Test 2: ML4T Data approach - print("\n[2/2] Testing QDATA PROVIDER (0.5 req/sec, automatic rate limiting)") + print("\n[2/2] Testing ML4T DATA PROVIDER (0.5 req/sec, automatic rate limiting)") print("=" * 80) stats_ml4t_data = test_ml4t_data_provider(tickers, start_date, end_date) diff --git a/examples/stress_test_results.txt b/examples/stress_test_results.txt deleted file mode 100644 index f2b8c464..00000000 --- a/examples/stress_test_results.txt +++ /dev/null @@ -1,75 +0,0 @@ -STRESS TEST RESULTS -================================================================================ - -Date: 2025-11-15 21:57:03.057775 -Tickers: 200 -Period: 2024-01-01 to 2024-12-31 - -Direct yfinance: - approach: direct_yfinance - total: 200 - successful: 117 - failed: 83 - empty: 83 - errors: 0 - start_time: 1763261384.81091 - end_time: 1763261423.3149416 - duration: 38.50403165817261 - avg_per_ticker: 0.19252015829086302 - -ML4T Data: - approach: ml4t-data - total: 200 - successful: 106 - failed: 76 - data_unavailable: 76 - rate_limited: 0 - network_errors: 0 - start_time: 1763261423.315043 - end_time: 1763261823.057555 - duration: 399.74251198768616 - avg_per_ticker: 1.9987125599384308 - -Failed tickers (yfinance): - AAMC: empty - AAN: empty - AAWW: empty - ABAX: empty - ABC: empty - ABCO: empty - ABFS: empty - ABMD: empty - ACAS: empty - ACC: empty - ACCL: empty - ACE: empty - ACHN: empty - ACO: empty - ACOR: empty - ACRX: empty - ACW: empty - ACXM: empty - ADES: empty - ADMS: empty - -Failed tickers (ML4T Data): - AAMC: data_unavailable - AAN: data_unavailable - AAWW: data_unavailable - ABAX: data_unavailable - ABC: data_unavailable - ABCO: data_unavailable - ABFS: data_unavailable - ABMD: data_unavailable - ACAS: data_unavailable - ACC: data_unavailable - ACCL: data_unavailable - ACE: data_unavailable - ACHN: data_unavailable - ACO: data_unavailable - ACOR: data_unavailable - ACRX: data_unavailable - ACW: data_unavailable - ACXM: data_unavailable - ADES: data_unavailable - ADMS: data_unavailable diff --git a/examples/symbols/README.md b/examples/symbols/README.md index bf3a6c6a..b218d277 100644 --- a/examples/symbols/README.md +++ b/examples/symbols/README.md @@ -127,6 +127,7 @@ If Wikipedia scraping fails (403 errors, page structure changes), you can: ```python from pathlib import Path + def load_symbols(filename: str) -> list[str]: """Load symbols from file, ignoring comments and blank lines.""" symbol_file = Path(__file__).parent / "symbols" / filename @@ -141,6 +142,7 @@ def load_symbols(filename: str) -> list[str]: return symbols + # Use the most recent dated file symbols = load_symbols("sp500_full_2025-11-24.txt") print(f"Loaded {len(symbols)} symbols") diff --git a/mkdocs.yml b/mkdocs.yml index c5edb200..c865f2af 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,9 +1,8 @@ # ML4T Data Documentation -# MkDocs Material configuration with Picasso Blue palette -# Template for all ml4t-* libraries +# MkDocs Material configuration site_name: ML4T Data -site_description: High-performance market data acquisition for quantitative finance +site_description: Market data acquisition, storage, and update workflows for machine learning for trading. site_url: https://www.ml4trading.io/docs/data/ repo_url: https://github.com/ml4t/data repo_name: ml4t/data @@ -112,6 +111,7 @@ nav: - Installation: getting-started/installation.md - Quickstart: getting-started/quickstart.md - Provider Selection: getting-started/provider-selection.md + - Migration from QLDM: getting-started/migration.md - User Guide: - user-guide/index.md @@ -169,11 +169,14 @@ nav: # Extra configuration extra: + release: + library: data + version: !ENV [ML4T_DOCS_VERSION, dev] + commit: !ENV [ML4T_DOCS_COMMIT, 0000000000000000000000000000000000000000] + social: - icon: fontawesome/brands/github link: https://github.com/ml4t/data - - icon: fontawesome/brands/twitter - link: https://twitter.com/mlaboratory generator: false # Hide "Made with Material for MkDocs" @@ -189,33 +192,14 @@ copyright: Copyright © 2024-2026 Stefan Jansen not_in_nav: | /ASYNC_STORAGE.md /FEATURES.md - /INCREMENTAL_UPDATES.md /INSTALLATION.md /INTEGRATION_TESTING.md /SESSION_MANAGEMENT.md - /TESTING.md /crypto_providers.md /export_guide.md - /multi_asset_guide.md /yahoo_provider.md - /asset-classes/README.md - /asset-classes/crypto.md - /asset-classes/equities.md - /asset-classes/forex.md - /asset-classes/futures.md /providers/databento_reference.md /providers/mock.md - /storage/INCREMENTAL_ARCHITECTURE.md - /storage/PARTITIONING_GUIDE.md - /user-guide/getting-started.md validation: omitted_files: warn - -# Historical audits remain in the repository but are not published as current user documentation. -exclude_docs: | - /FEATURE_INVENTORY.md - /GAP_ANALYSIS.md - /wiki_prices_status.md - /yahoo_bulk_download_analysis.md - /providers/PROVIDER_AUDIT.md diff --git a/pyproject.toml b/pyproject.toml index e2343b42..b0b7d03a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,4 @@ -# ML4T Data - High-performance market data management for quantitative finance +# ML4T Data package configuration # Independent package configuration for standalone distribution [build-system] @@ -29,21 +29,22 @@ include = [ [project] name = "ml4t-data" dynamic = ["version"] -description = "High-performance market data management library with unified multi-provider interface" +description = "Market data acquisition, storage, and update workflows for machine learning for trading." readme = "README.md" license = "MIT" license-files = ["LICENSE"] authors = [ - { name = "ML4T Team", email = "info@ml4trading.io" }, + { name = "Stefan Jansen", email = "stefan@applied-ai.com" }, ] maintainers = [ - { name = "ML4T Contributors", email = "dev@ml4trading.io" }, + { name = "Stefan Jansen", email = "pm@ml4trading.io" }, ] keywords = [ "finance", "market-data", "trading", "backtesting", + "algorithmic-trading", "quantitative-finance", "yahoo-finance", "binance", @@ -155,7 +156,7 @@ all = [ ml4t-data = "ml4t.data.cli_interface:cli" [project.urls] -Homepage = "https://www.ml4trading.io/docs/data/" +Homepage = "https://www.ml4trading.io/" Documentation = "https://www.ml4trading.io/docs/data/" Repository = "https://github.com/ml4t/data" Issues = "https://github.com/ml4t/data/issues" diff --git a/scripts/check_release_preflight.py b/scripts/check_release_preflight.py new file mode 100644 index 00000000..fca208ec --- /dev/null +++ b/scripts/check_release_preflight.py @@ -0,0 +1,134 @@ +"""Reject an ambiguous or already-published ml4t-data release candidate.""" + +from __future__ import annotations + +import argparse +import os +import re +import subprocess +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass + +from packaging.version import InvalidVersion, Version + +COMMIT_PATTERN = re.compile(r"[0-9a-f]{40}") + + +@dataclass(frozen=True) +class PublicationState: + """Existing public records for one proposed version.""" + + git_tag: bool + github_release: bool + pypi_version: bool + + +def version_failure(value: str) -> str | None: + """Return why a version cannot identify a new release.""" + try: + version = Version(value) + except InvalidVersion: + return "release version is not PEP 440 compliant" + if ( + str(version) != value + or len(version.release) != 3 + or version.is_devrelease + or version.is_postrelease + or version.local is not None + ): + return "release version must be normalized X.Y.Z or X.Y.ZaN/X.Y.ZbN/X.Y.ZrcN" + return None + + +def preflight_failures( + *, + version: str, + candidate_sha: str, + checked_out_sha: str, + workflow_sha: str, + main_sha: str, + publication: PublicationState, +) -> list[str]: + """Return every condition that makes publication unsafe.""" + failures = [] + if failure := version_failure(version): + failures.append(failure) + if not COMMIT_PATTERN.fullmatch(candidate_sha): + failures.append("candidate commit is not a full lowercase SHA") + if checked_out_sha != candidate_sha: + failures.append("checked-out commit differs from the requested candidate") + if workflow_sha != candidate_sha: + failures.append("workflow revision differs from the requested candidate") + if main_sha != candidate_sha: + failures.append("candidate commit is not the current origin/main revision") + if publication.git_tag: + failures.append(f"Git tag v{version} already exists") + if publication.github_release: + failures.append(f"GitHub release v{version} already exists") + if publication.pypi_version: + failures.append(f"PyPI version {version} already exists") + return failures + + +def _git_output(*arguments: str) -> str: + result = subprocess.run(["git", *arguments], check=True, capture_output=True, text=True) + return result.stdout.strip() + + +def _resource_exists(url: str, *, token: str | None = None) -> bool: + headers = {"Accept": "application/vnd.github+json", "User-Agent": "ml4t-data-release"} + if token: + headers["Authorization"] = f"Bearer {token}" + headers["X-GitHub-Api-Version"] = "2022-11-28" + request = urllib.request.Request(url, headers=headers) + try: + with urllib.request.urlopen(request, timeout=30): # noqa: S310 + return True + except urllib.error.HTTPError as error: + if error.code == 404: + return False + raise + + +def publication_state(repository: str, version: str) -> PublicationState: + """Read tag, release, and PyPI existence without changing public state.""" + encoded_tag = urllib.parse.quote(f"tags/v{version}", safe="") + encoded_version = urllib.parse.quote(version, safe="") + return PublicationState( + git_tag=_resource_exists( + f"https://api.github.com/repos/{repository}/git/ref/{encoded_tag}", + token=os.environ.get("GITHUB_TOKEN"), + ), + github_release=_resource_exists( + f"https://api.github.com/repos/{repository}/releases/tags/v{encoded_version}", + token=os.environ.get("GITHUB_TOKEN"), + ), + pypi_version=_resource_exists(f"https://pypi.org/pypi/ml4t-data/{encoded_version}/json"), + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--version", required=True) + parser.add_argument("--candidate-sha", required=True) + parser.add_argument("--repository", default="ml4t/data") + parser.add_argument("--workflow-sha", required=True) + args = parser.parse_args() + failures = preflight_failures( + version=args.version, + candidate_sha=args.candidate_sha, + checked_out_sha=_git_output("rev-parse", "HEAD"), + workflow_sha=args.workflow_sha, + main_sha=_git_output("rev-parse", "refs/remotes/origin/main"), + publication=publication_state(args.repository, args.version), + ) + print(f"release preflight: {'PASS' if not failures else 'FAIL'}") + for failure in failures: + print(f"- {failure}") + return int(bool(failures)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/release_candidate.py b/scripts/release_candidate.py new file mode 100644 index 00000000..2ab2b132 --- /dev/null +++ b/scripts/release_candidate.py @@ -0,0 +1,266 @@ +"""Create and verify the immutable ml4t-data release-candidate manifest.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import tarfile +import zipfile +from email import policy +from email.message import Message +from email.parser import BytesParser +from pathlib import Path +from typing import Any + +from packaging.specifiers import SpecifierSet + +SCHEMA_VERSION = 1 +EXPECTED_DESCRIPTION = ( + "Market data acquisition, storage, and update workflows for machine learning for trading." +) +EXPECTED_AUTHOR = "Stefan Jansen " +EXPECTED_MAINTAINER = "Stefan Jansen " +EXPECTED_URLS = { + "Homepage": "https://www.ml4trading.io/", + "Documentation": "https://www.ml4trading.io/docs/data/", + "Repository": "https://github.com/ml4t/data", + "Issues": "https://github.com/ml4t/data/issues", + "Changelog": "https://github.com/ml4t/data/blob/main/CHANGELOG.md", +} +EXPECTED_KEYWORDS = {"finance", "quantitative-finance", "algorithmic-trading", "market-data"} +EXPECTED_CLASSIFIERS = { + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Typing :: Typed", +} + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _distribution_files(candidate_dir: Path) -> tuple[Path, Path]: + dist_dir = candidate_dir / "dist" + wheels = tuple(dist_dir.glob("*.whl")) + sdists = tuple(dist_dir.glob("*.tar.gz")) + if len(wheels) != 1 or len(sdists) != 1: + raise ValueError( + f"candidate must contain one wheel and one sdist, found {len(wheels)} and {len(sdists)}" + ) + return wheels[0], sdists[0] + + +def _wheel_metadata(wheel: Path) -> Message: + with zipfile.ZipFile(wheel) as archive: + metadata_names = [ + name for name in archive.namelist() if name.endswith(".dist-info/METADATA") + ] + if len(metadata_names) != 1: + raise ValueError(f"wheel must contain one METADATA file: {wheel}") + return BytesParser(policy=policy.default).parsebytes(archive.read(metadata_names[0])) + + +def _sdist_metadata(sdist: Path) -> Message: + with tarfile.open(sdist, "r:gz") as archive: + metadata_members = [ + member + for member in archive.getmembers() + if member.name.count("/") == 1 and member.name.endswith("/PKG-INFO") + ] + if len(metadata_members) != 1: + raise ValueError(f"sdist must contain one top-level PKG-INFO file: {sdist}") + stream = archive.extractfile(metadata_members[0]) + if stream is None: + raise ValueError(f"cannot read sdist metadata: {sdist}") + return BytesParser(policy=policy.default).parsebytes(stream.read()) + + +def metadata_failures(metadata: Message, *, expected_version: str) -> list[str]: + """Return canonical public metadata mismatches.""" + failures: list[str] = [] + expected_scalars = { + "Name": "ml4t-data", + "Version": expected_version, + "Summary": EXPECTED_DESCRIPTION, + "Author-email": EXPECTED_AUTHOR, + "Maintainer-email": EXPECTED_MAINTAINER, + "License-Expression": "MIT", + } + for field, expected in expected_scalars.items(): + if metadata.get(field) != expected: + failures.append(f"{field} is {metadata.get(field)!r}, expected {expected!r}") + if SpecifierSet(str(metadata.get("Requires-Python", ""))) != SpecifierSet(">=3.12,<3.15"): + failures.append("Requires-Python differs from >=3.12,<3.15") + project_urls = {} + for value in metadata.get_all("Project-URL", []): + label, separator, url = value.partition(", ") + if separator: + project_urls[label] = url + if project_urls != EXPECTED_URLS: + failures.append("Project-URL metadata differs from the canonical URLs") + keywords = { + value.strip() for value in str(metadata.get("Keywords", "")).split(",") if value.strip() + } + if not keywords >= EXPECTED_KEYWORDS: + failures.append("Keywords omit a required ecosystem or library keyword") + if not set(metadata.get_all("Classifier", [])) >= EXPECTED_CLASSIFIERS: + failures.append("Classifiers omit required support metadata") + if "LICENSE" not in metadata.get_all("License-File", []): + failures.append("License-File metadata does not identify LICENSE") + return failures + + +def _identity(metadata: Message, path: Path) -> tuple[str, str]: + name = metadata.get("Name") + version = metadata.get("Version") + if not name or not version: + raise ValueError(f"distribution metadata is missing Name or Version: {path}") + return name, version + + +def _public_metadata(metadata: Message) -> dict[str, Any]: + project_urls = {} + for value in metadata.get_all("Project-URL", []): + label, url = value.split(",", maxsplit=1) + project_urls[label.strip()] = url.strip() + return { + "author_email": metadata.get("Author-email"), + "classifiers": sorted(metadata.get_all("Classifier", [])), + "description": metadata.get("Summary"), + "keywords": sorted( + keyword.strip() for keyword in (metadata.get("Keywords") or "").split(",") if keyword + ), + "license": metadata.get("License-Expression") or metadata.get("License"), + "maintainer_email": metadata.get("Maintainer-email"), + "project_urls": project_urls, + "requires_python": metadata.get("Requires-Python"), + } + + +def _artifact_record(path: Path) -> dict[str, str | int]: + return {"filename": path.name, "sha256": _sha256(path), "size": path.stat().st_size} + + +def create(candidate_dir: Path, commit_sha: str, git_tree: str) -> None: + """Write a manifest after validating the candidate metadata and artifact identity.""" + wheel, sdist = _distribution_files(candidate_dir) + wheel_metadata = _wheel_metadata(wheel) + sdist_metadata = _sdist_metadata(sdist) + wheel_identity = _identity(wheel_metadata, wheel) + if wheel_identity != _identity(sdist_metadata, sdist): + raise ValueError("wheel and sdist identities differ") + name, version = wheel_identity + failures = metadata_failures(wheel_metadata, expected_version=version) + failures.extend(metadata_failures(sdist_metadata, expected_version=version)) + if failures: + raise ValueError(f"candidate metadata failed: {failures}") + public_metadata = _public_metadata(wheel_metadata) + if public_metadata != _public_metadata(sdist_metadata): + raise ValueError("wheel and sdist public metadata differ") + manifest = { + "schema_version": SCHEMA_VERSION, + "commit_sha": commit_sha, + "git_tree": git_tree, + "name": name, + "version": version, + "metadata": public_metadata, + "artifacts": [_artifact_record(wheel), _artifact_record(sdist)], + } + (candidate_dir / "candidate.json").write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + +def _load_manifest(candidate_dir: Path) -> dict[str, Any]: + value = json.loads((candidate_dir / "candidate.json").read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError("candidate manifest must be a JSON object") + return value + + +def verify( + candidate_dir: Path, + *, + expected_commit: str | None = None, + expected_tree: str | None = None, + expected_tag: str | None = None, +) -> None: + """Verify manifest identity, artifact bytes, and canonical metadata.""" + manifest = _load_manifest(candidate_dir) + if manifest.get("schema_version") != SCHEMA_VERSION: + raise ValueError(f"unsupported candidate schema: {manifest.get('schema_version')!r}") + if expected_commit is not None and manifest.get("commit_sha") != expected_commit: + raise ValueError("candidate commit differs from the requested release commit") + if expected_tree is not None and manifest.get("git_tree") != expected_tree: + raise ValueError("candidate tree differs from the requested release tree") + if expected_tag is not None and expected_tag != f"v{manifest.get('version')}": + raise ValueError("release tag differs from the candidate version") + + wheel, sdist = _distribution_files(candidate_dir) + actual_files = {wheel.name: wheel, sdist.name: sdist} + records = manifest.get("artifacts") + if not isinstance(records, list) or len(records) != 2: + raise ValueError("candidate manifest must describe exactly two artifacts") + recorded_files = {record.get("filename"): record for record in records} + if set(recorded_files) != set(actual_files): + raise ValueError("candidate manifest artifact inventory differs from the dist directory") + for filename, path in actual_files.items(): + record = recorded_files[filename] + if record.get("sha256") != _sha256(path) or record.get("size") != path.stat().st_size: + raise ValueError(f"candidate artifact integrity check failed: {filename}") + + wheel_metadata = _wheel_metadata(wheel) + sdist_metadata = _sdist_metadata(sdist) + identity = _identity(wheel_metadata, wheel) + if identity != _identity(sdist_metadata, sdist): + raise ValueError("candidate wheel and sdist identities differ") + if identity != (manifest.get("name"), manifest.get("version")): + raise ValueError("candidate metadata differs from the manifest") + failures = metadata_failures(wheel_metadata, expected_version=str(manifest.get("version"))) + failures.extend( + metadata_failures(sdist_metadata, expected_version=str(manifest.get("version"))) + ) + if failures: + raise ValueError(f"candidate metadata failed: {failures}") + if _public_metadata(wheel_metadata) != manifest.get("metadata"): + raise ValueError("candidate public metadata differs from the manifest") + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + create_parser = subparsers.add_parser("create") + create_parser.add_argument("candidate_dir", type=Path) + create_parser.add_argument("--commit-sha", required=True) + create_parser.add_argument("--git-tree", required=True) + verify_parser = subparsers.add_parser("verify") + verify_parser.add_argument("candidate_dir", type=Path) + verify_parser.add_argument("--expected-commit") + verify_parser.add_argument("--expected-tree") + verify_parser.add_argument("--expected-tag") + return parser + + +def main() -> None: + args = _parser().parse_args() + if args.command == "create": + create(args.candidate_dir, args.commit_sha, args.git_tree) + else: + verify( + args.candidate_dir, + expected_commit=args.expected_commit, + expected_tree=args.expected_tree, + expected_tag=args.expected_tag, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_readme_quickstart.py b/scripts/run_readme_quickstart.py new file mode 100644 index 00000000..edb5a7af --- /dev/null +++ b/scripts/run_readme_quickstart.py @@ -0,0 +1,34 @@ +"""Execute the Python quick start published in README.md.""" + +from __future__ import annotations + +import re +from pathlib import Path + +ROOT = Path(__file__).parents[1] +QUICK_START = re.compile( + r"^## Quick start\s*$\n(?P.*?)(?=^## |\Z)", + flags=re.MULTILINE | re.DOTALL, +) +PYTHON_BLOCK = re.compile(r"^```python\s*$\n(?P.*?)^```\s*$", re.MULTILINE | re.DOTALL) + + +def extract_quick_start(readme: str) -> str: + """Return the single Python block in the README quick-start section.""" + section = QUICK_START.search(readme) + if section is None: + raise ValueError("README.md has no '## Quick start' section") + blocks = [match["source"] for match in PYTHON_BLOCK.finditer(section["body"])] + if len(blocks) != 1: + raise ValueError(f"README quick start must contain one Python block, found {len(blocks)}") + return blocks[0] + + +def main() -> None: + """Compile and execute the documented quick start.""" + source = extract_quick_start((ROOT / "README.md").read_text(encoding="utf-8")) + exec(compile(source, "README.md quick start", "exec"), {"__name__": "__main__"}) # noqa: S102 + + +if __name__ == "__main__": + main() diff --git a/scripts/verify_distribution.py b/scripts/verify_distribution.py index c3a1aea6..7c7a0f5c 100644 --- a/scripts/verify_distribution.py +++ b/scripts/verify_distribution.py @@ -11,12 +11,25 @@ import tarfile import tempfile import zipfile +from email import policy from email.message import Message from email.parser import Parser from pathlib import Path +from packaging.specifiers import SpecifierSet + PROJECT_NAME = "ml4t-data" TAG_PATTERN = re.compile(r"^v(?P\d+\.\d+\.\d+(?:(?:a|b|rc)\d+)?)$") +EXPECTED_DESCRIPTION = ( + "Market data acquisition, storage, and update workflows for machine learning for trading." +) +EXPECTED_URLS = { + "Homepage": "https://www.ml4trading.io/", + "Documentation": "https://www.ml4trading.io/docs/data/", + "Repository": "https://github.com/ml4t/data", + "Issues": "https://github.com/ml4t/data/issues", + "Changelog": "https://github.com/ml4t/data/blob/main/CHANGELOG.md", +} def _metadata_from_wheel(path: Path) -> Message: @@ -27,7 +40,7 @@ def _metadata_from_wheel(path: Path) -> Message: if len(metadata_files) != 1: raise ValueError(f"{path.name} must contain exactly one METADATA file") content = archive.read(metadata_files[0]).decode("utf-8") - return Parser().parsestr(content) + return Parser(policy=policy.default).parsestr(content) def _metadata_from_sdist(path: Path) -> Message: @@ -43,7 +56,7 @@ def _metadata_from_sdist(path: Path) -> Message: if extracted is None: raise ValueError(f"could not read PKG-INFO from {path.name}") content = extracted.read().decode("utf-8") - return Parser().parsestr(content) + return Parser(policy=policy.default).parsestr(content) def validate_distributions( @@ -81,6 +94,31 @@ def validate_distributions( f"License-Expression: MIT and License-File: LICENSE: " f"expression={license_expression!r}, files={license_files!r}" ) + expected_scalars = { + "Summary": EXPECTED_DESCRIPTION, + "Author-email": "Stefan Jansen ", + "Maintainer-email": "Stefan Jansen ", + } + for field, expected in expected_scalars.items(): + if item[field] != expected: + raise ValueError(f"{field} is {item[field]!r}, expected {expected!r}") + if SpecifierSet(str(item["Requires-Python"])) != SpecifierSet(">=3.12,<3.15"): + raise ValueError("Requires-Python differs from >=3.12,<3.15") + project_urls = {} + for value in item.get_all("Project-URL", []): + label, separator, url = value.partition(", ") + if separator: + project_urls[label] = url + if project_urls != EXPECTED_URLS: + raise ValueError("Project-URL metadata differs from canonical URLs") + keywords = { + value.strip() for value in str(item["Keywords"] or "").split(",") if value.strip() + } + if ( + not {"finance", "quantitative-finance", "algorithmic-trading", "market-data"} + <= keywords + ): + raise ValueError("distribution keywords omit a required term") versions = {item["Version"] for item in metadata} if len(versions) != 1: @@ -135,6 +173,9 @@ def verify_install(archive: Path, version: str) -> None: f"stdout={completed.stdout!r}, stderr={completed.stderr!r}" ) subprocess.run([str(_venv_cli(venv)), "--help"], check=True, stdout=subprocess.DEVNULL) + subprocess.run( + [str(python), str(Path(__file__).with_name("run_readme_quickstart.py"))], check=True + ) def main() -> None: diff --git a/scripts/verify_documentation_identity.py b/scripts/verify_documentation_identity.py new file mode 100644 index 00000000..51221223 --- /dev/null +++ b/scripts/verify_documentation_identity.py @@ -0,0 +1,158 @@ +"""Verify rendered and deployed documentation release identity.""" + +from __future__ import annotations + +import argparse +import time +import urllib.error +import urllib.request +from html.parser import HTMLParser +from pathlib import Path + +REQUIRED_META = ("ml4t-library", "ml4t-version", "ml4t-commit") + + +class _MetadataParser(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.values: dict[str, str] = {} + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag != "meta": + return + values = dict(attrs) + name = values.get("name") + content = values.get("content") + if name in REQUIRED_META and content is not None: + self.values[name] = content + + +def identity_failures( + html: str, + *, + expected_library: str, + expected_version: str, + expected_commit: str, + source: str, +) -> list[str]: + """Return mismatched or missing documentation identity fields.""" + parser = _MetadataParser() + parser.feed(html) + expected = { + "ml4t-library": expected_library, + "ml4t-version": expected_version, + "ml4t-commit": expected_commit, + } + return [ + f"{source}: {name} is {parser.values.get(name)!r}, expected {value!r}" + for name, value in expected.items() + if parser.values.get(name) != value + ] + + +def _site_pages(site: Path) -> list[Path]: + return sorted( + path + for path in site.rglob("*.html") + if path.is_file() and "overrides" not in path.relative_to(site).parts + ) + + +def _read_url(url: str) -> str: + request = urllib.request.Request(url, headers={"User-Agent": "ml4t-release-verifier"}) + with urllib.request.urlopen(request, timeout=30) as response: # noqa: S310 + if response.status != 200: + raise ValueError(f"{url}: HTTP {response.status}") + return response.read().decode("utf-8") + + +def deployed_identity_failures( + urls: list[str], + *, + expected_library: str, + expected_version: str, + expected_commit: str, + attempts: int, + delay: float, +) -> list[str]: + """Retry deployed pages until every URL reports the expected identity.""" + failures: list[str] = [] + for attempt in range(1, attempts + 1): + failures = [] + try: + for url in urls: + failures.extend( + identity_failures( + _read_url(url), + expected_library=expected_library, + expected_version=expected_version, + expected_commit=expected_commit, + source=url, + ) + ) + except (OSError, UnicodeDecodeError, urllib.error.URLError, ValueError) as error: + failures = [str(error)] + if not failures: + break + if attempt < attempts: + time.sleep(delay) + return failures + + +def main() -> int: + """Validate a rendered site or one or more deployed URLs.""" + parser = argparse.ArgumentParser() + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--site", type=Path) + source.add_argument("--url", action="append") + parser.add_argument("--expected-library", required=True) + parser.add_argument("--expected-version", required=True) + parser.add_argument("--expected-commit", required=True) + parser.add_argument("--attempts", type=int, default=1) + parser.add_argument("--delay", type=float, default=0.0) + args = parser.parse_args() + + if args.attempts < 1 or args.delay < 0: + parser.error("--attempts must be positive and --delay must be non-negative") + if len(args.expected_commit) != 40 or any( + character not in "0123456789abcdef" for character in args.expected_commit + ): + parser.error("--expected-commit must be a full lowercase Git SHA") + + failures: list[str] = [] + try: + if args.site is not None: + pages = _site_pages(args.site) + if not pages: + failures.append(f"{args.site}: no rendered HTML pages found") + for page in pages: + failures.extend( + identity_failures( + page.read_text(encoding="utf-8"), + expected_library=args.expected_library, + expected_version=args.expected_version, + expected_commit=args.expected_commit, + source=str(page), + ) + ) + except (OSError, UnicodeDecodeError, urllib.error.URLError, ValueError) as error: + failures.append(str(error)) + if args.url is not None: + failures.extend( + deployed_identity_failures( + args.url, + expected_library=args.expected_library, + expected_version=args.expected_version, + expected_commit=args.expected_commit, + attempts=args.attempts, + delay=args.delay, + ) + ) + + for failure in failures: + print(failure) + return int(bool(failures)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify_published_release.py b/scripts/verify_published_release.py new file mode 100644 index 00000000..edf2619b --- /dev/null +++ b/scripts/verify_published_release.py @@ -0,0 +1,159 @@ +"""Verify PyPI and GitHub records against the immutable release manifest.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import time +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any + +from packaging.specifiers import SpecifierSet + +try: + from scripts.release_candidate import ( + EXPECTED_AUTHOR, + EXPECTED_CLASSIFIERS, + EXPECTED_DESCRIPTION, + EXPECTED_KEYWORDS, + EXPECTED_MAINTAINER, + EXPECTED_URLS, + ) +except ModuleNotFoundError: # Direct execution adds scripts/, not the repository root, to sys.path. + from release_candidate import ( + EXPECTED_AUTHOR, + EXPECTED_CLASSIFIERS, + EXPECTED_DESCRIPTION, + EXPECTED_KEYWORDS, + EXPECTED_MAINTAINER, + EXPECTED_URLS, + ) + +HASH_PATTERN = re.compile(r"[0-9a-f]{64}") + + +def _artifact_digests(manifest: dict[str, Any]) -> dict[str, str]: + records = manifest.get("artifacts", []) + if not isinstance(records, list): + return {} + return { + str(record.get("filename", "")): str(record.get("sha256", "")) + for record in records + if isinstance(record, dict) + } + + +def published_release_failures( + manifest: dict[str, Any], pypi: dict[str, Any], release: dict[str, Any] +) -> list[str]: + """Return mismatches across the manifest, PyPI JSON, and GitHub release.""" + failures: list[str] = [] + version = str(manifest.get("version", "")) + commit = str(manifest.get("commit_sha", "")) + artifacts = _artifact_digests(manifest) + if ( + manifest.get("schema_version") != 1 + or manifest.get("name") != "ml4t-data" + or not version + or len(commit) != 40 + or not artifacts + or any(not HASH_PATTERN.fullmatch(digest) for digest in artifacts.values()) + ): + return ["release manifest is incomplete"] + + info = pypi.get("info", {}) + keywords = { + keyword.strip() for keyword in str(info.get("keywords", "")).split(",") if keyword.strip() + } + pypi_files = { + str(record.get("filename", "")): str(record.get("digests", {}).get("sha256", "")) + for record in pypi.get("urls", []) + if isinstance(record, dict) + } + metadata_matches = ( + info.get("name") == "ml4t-data" + and info.get("version") == version + and info.get("summary") == EXPECTED_DESCRIPTION + and info.get("author_email") == EXPECTED_AUTHOR + and info.get("maintainer_email") == EXPECTED_MAINTAINER + and info.get("license_expression") == "MIT" + and SpecifierSet(str(info.get("requires_python", ""))) == SpecifierSet(">=3.12,<3.15") + and info.get("project_urls") == EXPECTED_URLS + and keywords >= EXPECTED_KEYWORDS + and set(info.get("classifiers", [])) >= EXPECTED_CLASSIFIERS + ) + if not metadata_matches: + failures.append("PyPI metadata differs from the qualified source") + if pypi_files != artifacts: + failures.append("PyPI artifact names or digests differ from the release manifest") + + release_assets = { + str(asset.get("name", "")): str(asset.get("digest", "")).removeprefix("sha256:") + for asset in release.get("assets", []) + if isinstance(asset, dict) and str(asset.get("name", "")) in artifacts + } + if release.get("tag_name") != f"v{version}" or release.get("target_commitish") != commit: + failures.append("GitHub release tag or target differs from the release manifest") + if release_assets != artifacts: + failures.append("GitHub release artifact names or digests differ from the release manifest") + return failures + + +def _read_json(url: str, *, token: str | None = None) -> dict[str, Any]: + headers = {"Accept": "application/vnd.github+json", "User-Agent": "ml4t-data-release"} + if token: + headers["Authorization"] = f"Bearer {token}" + headers["X-GitHub-Api-Version"] = "2022-11-28" + request = urllib.request.Request(url, headers=headers) + with urllib.request.urlopen(request, timeout=30) as response: # noqa: S310 + payload = json.load(response) + if not isinstance(payload, dict): + raise ValueError(f"{url} did not return a JSON object") + return payload + + +def remote_release_failures( + manifest: dict[str, Any], repository: str, *, attempts: int = 12, wait_seconds: float = 10.0 +) -> list[str]: + """Poll public records until publication propagation completes.""" + version = urllib.parse.quote(str(manifest.get("version", "")), safe="") + tag = urllib.parse.quote(f"v{manifest.get('version', '')}", safe="") + failures: list[str] = [] + for attempt in range(attempts): + try: + failures = published_release_failures( + manifest, + _read_json(f"https://pypi.org/pypi/ml4t-data/{version}/json"), + _read_json( + f"https://api.github.com/repos/{repository}/releases/tags/{tag}", + token=os.environ.get("GITHUB_TOKEN"), + ), + ) + except Exception as error: # noqa: BLE001 + failures = [f"public release records are unavailable: {error}"] + if not failures: + return [] + if attempt + 1 < attempts: + time.sleep(wait_seconds) + return failures + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--repository", default="ml4t/data") + args = parser.parse_args() + manifest = json.loads(args.manifest.read_text(encoding="utf-8")) + failures = remote_release_failures(manifest, args.repository) + print(f"published release identity: {'PASS' if not failures else 'FAIL'}") + for failure in failures: + print(f"- {failure}") + return int(bool(failures)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/ml4t/data/AGENTS.md b/src/ml4t/data/AGENTS.md index f61dfc69..63c92662 100644 --- a/src/ml4t/data/AGENTS.md +++ b/src/ml4t/data/AGENTS.md @@ -1,46 +1,35 @@ -# ml4t.data - Package Index - -## Core Modules - -| File | Lines | Purpose | -|------|-------|---------| -| data_manager.py | 564 | Main orchestration API | -| update_manager.py | 829 | Incremental update system | -| universe.py | 987 | Asset universe management | -| provider_updater.py | 449 | Provider coordination | - -## Subpackages - -| Directory | Lines | Purpose | -|-----------|-------|---------| -| providers/ | 14k | 20 live provider adapters + synthetic/testing providers | -| storage/ | 4.6k | Hive-partitioned backends + profiling | -| futures/ | 4.6k | Databento futures downloader | -| etfs/ | 600 | ETFDataManager (Yahoo Finance) | -| crypto/ | 420 | CryptoDataManager (Binance Public) | -| core/ | 1k | Models, schemas, config | -| utils/ | 1.6k | Rate limiting, gaps, retry | -| assets/ | 1.3k | Asset class definitions | -| anomaly/ | 1k | Data quality detection | -| cot/ | 1k | CFTC COT data | -| validation/ | 1.2k | OHLC validation | -| sessions/ | 462 | Session assignment | -| macro/ | 455 | Macro data downloader | -| calendar/ | 359 | Trading calendars | -| export/ | 231 | CSV/JSON/Excel export | - -## Book Data Managers - -Simplified managers for ML4T book readers with built-in profiling: - -| Manager | Asset | Source | Profiling | -|---------|-------|--------|-----------| -| `ETFDataManager` | ETFs | Yahoo Finance | `generate_profile()` | -| `CryptoDataManager` | Crypto | Binance Public | `generate_profile()` | -| `FuturesDataManager` | Futures | Databento | `generate_profile(product)` | - -All managers inherit from `ProfileMixin` for on-demand column statistics. - -## Key - -`DataManager`, `UpdateManager`, `HiveStorage`, `ProfileMixin`, `get_provider()` +# `ml4t.data` source guide + +This directory implements the `ml4t.data` package. Follow the repository-level `AGENTS.md` for +quality, compatibility, and release requirements. This guide only routes source changes. + +## Change locations + +| Area | Responsibility | +|---|---| +| `providers/` | Provider contracts, registry metadata, synchronous and asynchronous adapters | +| `storage/` | Storage protocols, path and key handling, Parquet backends, metadata, and migrations | +| `futures/` | Futures symbology, downloads, continuous contracts, rolls, and adjustments | +| `data_manager.py`, `update_manager.py`, `provider_updater.py`, `managers/` | Acquisition and update orchestration | +| `core/`, `config/` | Shared models, configuration, and exception types | +| `assets/`, `calendar/`, `sessions/` | Asset contracts and trading-session behavior | +| `validation/`, `anomaly/`, `security/` | Data validation, anomaly checks, and input safety | +| `synthetic/` | Deterministic synthetic data models and registry integration | +| `cli/`, `cli_interface.py` | Command-line behavior exposed by the `ml4t-data` entry point | + +Providers, storage, and futures contain directory-specific `AGENTS.md` files. Use those guides when +editing within those subsystems. Public API and usage details belong in `docs/api/`, +`docs/user-guide/`, and `docs/providers/` rather than here. + +## Package boundaries + +- Keep `src/ml4t/` as a namespace package without an `__init__.py` file. +- Re-export only supported public names from package `__init__.py` files. Test imports from an + installed wheel when changing exports. +- Keep optional-provider imports isolated so the base package works without every provider extra. +- Route data paths through `core.config`; shared callers must not invent another environment-variable + or default-path convention. +- Extend shared protocols and models deliberately. A contract change requires tests for adapters, + orchestration, and storage consumers that rely on it. +- Add tests in the matching `tests/` area and use the repository's established markers for any + non-default lane. diff --git a/src/ml4t/data/__init__.py b/src/ml4t/data/__init__.py index a8edaf10..f3f25579 100644 --- a/src/ml4t/data/__init__.py +++ b/src/ml4t/data/__init__.py @@ -1,11 +1,11 @@ -"""ML4T Data - Modern financial data management library with unified provider interface.""" +"""Market data acquisition, storage, and update workflows for ML4T.""" try: from ml4t.data._version import __version__ except ImportError: __version__ = "0.0.0.dev0" -__author__ = "ML4T Team" -__email__ = "info@ml4trading.io" +__author__ = "Stefan Jansen" +__email__ = "stefan@applied-ai.com" # Contract specifications (always available - no external dependencies) # Asset classes diff --git a/src/ml4t/data/anomaly/AGENTS.md b/src/ml4t/data/anomaly/AGENTS.md deleted file mode 100644 index 9b8378e6..00000000 --- a/src/ml4t/data/anomaly/AGENTS.md +++ /dev/null @@ -1,16 +0,0 @@ -# anomaly/ - 1k Lines - -Data quality and anomaly detection. - -## Functions - -| Function | Purpose | -|----------|---------| -| detect_anomalies | Find data anomalies | -| validate_ohlc | OHLC consistency | -| check_gaps | Missing data detection | -| flag_outliers | Statistical outliers | - -## Key - -`AnomalyDetector`, `detect_anomalies()`, `DataQualityReport` diff --git a/src/ml4t/data/assets/AGENTS.md b/src/ml4t/data/assets/AGENTS.md deleted file mode 100644 index 956199f9..00000000 --- a/src/ml4t/data/assets/AGENTS.md +++ /dev/null @@ -1,16 +0,0 @@ -# assets/ - 1.3k Lines - -Asset class definitions and validation. - -## Modules - -| File | Purpose | -|------|---------| -| asset_class.py | Asset class enum | -| contracts.py | Contract specifications | -| schemas.py | Asset schemas | -| validation.py | Asset validation | - -## Key - -`AssetClass`, `ContractSpec`, `validate_asset()` diff --git a/src/ml4t/data/assets/__init__.py b/src/ml4t/data/assets/__init__.py index 37136597..44d7370b 100644 --- a/src/ml4t/data/assets/__init__.py +++ b/src/ml4t/data/assets/__init__.py @@ -1,4 +1,4 @@ -"""Asset class abstractions for QLDM.""" +"""Asset class abstractions for ml4t-data.""" from ml4t.data.assets.asset_class import AssetClass, AssetInfo from ml4t.data.assets.contracts import ( diff --git a/src/ml4t/data/config/loader.py b/src/ml4t/data/config/loader.py index bdb80996..e57905d9 100644 --- a/src/ml4t/data/config/loader.py +++ b/src/ml4t/data/config/loader.py @@ -307,7 +307,7 @@ def save(self, config: DataConfig, path: Path | None = None) -> None: """ save_path = path or self.config_path if not save_path: - save_path = Path.cwd() / "mlquant.data.yaml" + save_path = Path.cwd() / "ml4t.data.yaml" # Convert to dictionary with JSON-compatible types data = config.model_dump(exclude_none=True, exclude_defaults=False, mode="json") diff --git a/src/ml4t/data/core/AGENTS.md b/src/ml4t/data/core/AGENTS.md deleted file mode 100644 index 7845c8ab..00000000 --- a/src/ml4t/data/core/AGENTS.md +++ /dev/null @@ -1,16 +0,0 @@ -# core/ - 1k Lines - -Core models, schemas, and configuration. - -## Modules - -| File | Purpose | -|------|---------| -| models.py | Pydantic data models | -| schemas.py | Data schemas | -| config.py | Configuration classes | -| exceptions.py | Custom exceptions | - -## Key - -`OHLCVBar`, `DataConfig`, `DataError` diff --git a/src/ml4t/data/core/config.py b/src/ml4t/data/core/config.py index adc00b89..0c8b051f 100644 --- a/src/ml4t/data/core/config.py +++ b/src/ml4t/data/core/config.py @@ -12,7 +12,8 @@ from ml4t.data.storage.config import StorageConfig PREFERRED_DATA_ENV_VAR = "ML4T_DATA_PATH" -LEGACY_DATA_ENV_VARS = ("ML4T_DATA_DIR", "QLDM_DATA_ROOT") +LEGACY_DATA_ENV_VARS = ("QLDM_DATA_ROOT",) +PREFERRED_LOG_ENV_VAR = "ML4T_DATA_LOG_LEVEL" DEFAULT_RELATIVE_DATA_ROOT = Path("data") @@ -27,7 +28,7 @@ def resolve_data_root(data_root: str | Path | None = None) -> Path: Resolution order: 1. Explicit `data_root` 2. `ML4T_DATA_PATH` - 3. Legacy env vars (`ML4T_DATA_DIR`, `QLDM_DATA_ROOT`) with warning + 3. Legacy `QLDM_DATA_ROOT` with warning 4. Project-local `./data` """ if data_root is not None: @@ -84,7 +85,7 @@ class CacheConfig(BaseModel): class Config(BaseModel): - """Main configuration for QLDM.""" + """Main configuration for ml4t-data.""" data_root: Path = Field(default_factory=resolve_data_root) log_level: Literal["DEBUG", "INFO", "WARNING", "ERROR"] = "INFO" @@ -109,8 +110,9 @@ def __init__(self, **data: Any) -> None: or any(env_var in os.environ for env_var in LEGACY_DATA_ENV_VARS) ): data["data_root"] = resolve_data_root() - if "QLDM_LOG_LEVEL" in os.environ: - data["log_level"] = os.environ["QLDM_LOG_LEVEL"] + preferred_log_level = os.getenv(PREFERRED_LOG_ENV_VAR) + if preferred_log_level: + data["log_level"] = preferred_log_level if "storage" not in data: data["storage"] = {"base_path": data.get("data_root", resolve_data_root())} diff --git a/src/ml4t/data/core/exceptions.py b/src/ml4t/data/core/exceptions.py index 74259bfc..aada5390 100644 --- a/src/ml4t/data/core/exceptions.py +++ b/src/ml4t/data/core/exceptions.py @@ -2,7 +2,25 @@ from __future__ import annotations -from typing import Any +import warnings +from typing import TYPE_CHECKING, Any + +__all__ = [ + "AuthenticationError", + "CircuitBreakerOpenError", + "ConfigurationError", + "DataNotAvailableError", + "DataValidationError", + "LockError", + "ML4TDataError", + "NetworkError", + "ProviderError", + "ProviderRoutingError", + "QldmError", + "RateLimitError", + "StorageError", + "SymbolNotFoundError", +] class ML4TDataError(Exception): @@ -21,8 +39,26 @@ def __init__(self, message: str, details: dict[str, Any] | None = None) -> None: self.details = details or {} -# Backward compatibility alias -QldmError = ML4TDataError +if TYPE_CHECKING: + QldmError = ML4TDataError + + +def __getattr__(name: str) -> Any: + """Resolve deprecated module attributes.""" + if name == "QldmError": + warnings.warn( + "QldmError is deprecated; use ML4TDataError instead.", + DeprecationWarning, + stacklevel=2, + ) + + return ML4TDataError + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def __dir__() -> list[str]: + """Include compatibility attributes in module discovery.""" + return sorted(set(globals()) | set(__all__)) class ProviderError(ML4TDataError): diff --git a/src/ml4t/data/cot/AGENTS.md b/src/ml4t/data/cot/AGENTS.md deleted file mode 100644 index 9655d920..00000000 --- a/src/ml4t/data/cot/AGENTS.md +++ /dev/null @@ -1,14 +0,0 @@ -# cot/ - 1k Lines - -CFTC Commitment of Traders data. - -## Modules - -| File | Purpose | -|------|---------| -| fetcher.py | COT data download | -| workflow.py | Processing workflow | - -## Key - -`COTFetcher`, `fetch_cot_data()`, `COTReport` diff --git a/src/ml4t/data/futures/AGENTS.md b/src/ml4t/data/futures/AGENTS.md index b90b654f..4e0eeb80 100644 --- a/src/ml4t/data/futures/AGENTS.md +++ b/src/ml4t/data/futures/AGENTS.md @@ -1,116 +1,32 @@ -# futures/ - Databento Futures Data - -CME futures data downloading and continuous contract construction. - -## Downloader Classes - -| Class | Symbology | Schema | Use Case | -| ---------------------- | ------------------- | ---------- | ----------------------------------- | -| `ContinuousDownloader` | `ES.v.0` (volume) | `ohlcv-1h` | Pre-rolled continuous, Hive output | -| `IndividualDownloader` | `ESH24` (raw) | `ohlcv-1h` | Specific contracts for roll demo | -| `FuturesDownloader` | `ES.FUT` (parent) | `ohlcv-1d` | All contracts, bulk download | -| `FuturesDataManager` | `ES.FUT` (parent) | `ohlcv-1d` | High-level interface + profiling | - -## Key Files - -| File | Purpose | -| ------------------------- | -------------------------------------------- | -| `continuous_downloader.py` | ContinuousDownloader class | -| `individual_downloader.py` | IndividualDownloader class | -| `downloader.py` | FuturesDownloader class | -| `book_downloader.py` | FuturesDataManager for book readers | -| `databento_parser.py` | Contract symbol parsing (MONTH_CODES, etc.) | -| `continuous.py` | ContinuousContractBuilder | -| `roll.py` | Roll strategies (VolumeBasedRoll, etc.) | -| `adjustment.py` | Price adjustment methods (BackAdjustment) | -| `config.py` | Configuration dataclasses | -| `schema.py` | Contract specifications (ContractSpec) | -| `definitions.py` | DefinitionsDownloader | - -## Contract Symbol Formats - -| Format | Example | Description | -| ------------ | ---------- | ---------------------------------- | -| Continuous | `ES.v.0` | Volume-rolled front month | -| Continuous | `ES.c.1` | Calendar-rolled second month | -| Individual | `ESH24` | March 2024 contract | -| Parent | `ES.FUT` | All contracts for product | - -## Month Codes - -| Code | Month | Code | Month | -| ---- | --------- | ---- | --------- | -| F | January | N | July | -| G | February | Q | August | -| H | March | U | September | -| J | April | V | October | -| K | May | X | November | -| M | June | Z | December | - -## Usage Examples - -```python -# Continuous contracts -from ml4t.data.futures import ContinuousDownloader, ContinuousDownloadConfig - -config = ContinuousDownloadConfig( - products=["ES", "CL"], - start="2020-01-01", - end="2025-12-31", - tenors=[0, 1, 2], - schema="ohlcv-1h", -) -downloader = ContinuousDownloader(config) -downloader.download_all() - -# Individual contracts -from ml4t.data.futures import IndividualDownloader, IndividualDownloadConfig - -config = IndividualDownloadConfig( - products={ - "ES": {"months": [3, 6, 9, 12]}, # Quarterly - "CL": {"months": list(range(1, 13))}, # Monthly - }, - years=[2024, 2025], - schema="ohlcv-1h", -) -downloader = IndividualDownloader(config) -downloader.download_all() -``` - -## Data Profiling (FuturesDataManager) - -```python -from ml4t.data.futures import FuturesDataManager - -manager = FuturesDataManager.from_config("config.yaml") - -# Generate profile for specific product -profile = manager.generate_profile("ES") -print(profile.summary()) - -# Load existing profile -profile = manager.load_profile("ES") - -# Generate profiles for all products -profiles = manager.generate_all_profiles() -``` - -## Output Structure - -``` -futures/ -├── continuous/ # ContinuousDownloader output -│ └── product={PRODUCT}/ -│ └── year={YEAR}/ -│ └── data.parquet -├── individual/ # IndividualDownloader output -│ └── {PRODUCT}/ -│ └── data.parquet -├── ohlcv_1d/ # FuturesDataManager output -│ └── product={PRODUCT}/ -│ ├── year={YEAR}/ -│ │ └── data.parquet -│ └── _profile.json # Per-product profile -└── definitions.parquet # Contract specifications -``` +# Futures subsystem guide + +This directory owns futures symbology, Databento-backed acquisition, continuous-contract +construction, roll selection, and price adjustment. It is not the general provider registry or the +generic storage layer. + +## Change locations + +| File | Responsibility | +|---|---| +| `parser.py`, `databento_parser.py` | Contract symbols and Databento record conversion | +| `schema.py`, `definitions.py` | Contract specifications and definition data | +| `downloader.py` | Parent-symbol downloads | +| `continuous_downloader.py`, `individual_downloader.py` | Continuous and individual-contract acquisition | +| `continuous.py`, `roll.py`, `adjustment.py` | Continuous series, roll decisions, and price adjustment | +| `config.py` | Futures-specific configuration | +| `book_downloader.py` | Higher-level futures workflow used by the book examples | + +## Invariants + +- Keep symbol parsing and contract-calendar logic separate from network acquisition. +- Preserve the distinction between parent, individual, and continuous symbols in code and tests. +- Route filesystem defaults through the package data-root helpers; do not embed workstation paths. +- Keep Databento optional. Importing unrelated `ml4t.data` modules must not require the extra. +- Mock external calls in the default test suite. Credentialed or live checks use the established + integration markers. +- Treat output schemas and roll or adjustment behavior as public contracts. Add regression tests for + boundary dates and representative symbols when changing them. + +Run `uv run pytest tests/futures -q -ra` for focused verification, then run the root quality gates. +User-facing behavior and examples belong in the futures API documentation and +`docs/book-guide/index.md`, not in this file. diff --git a/src/ml4t/data/managers/async_batch.py b/src/ml4t/data/managers/async_batch.py index 361b19cf..bd267ceb 100644 --- a/src/ml4t/data/managers/async_batch.py +++ b/src/ml4t/data/managers/async_batch.py @@ -1,7 +1,7 @@ -"""Async batch operations for high-performance data loading. +"""Async batch operations for data loading. -This module provides async batch loading using asyncio.gather() for -3-5x speedup over thread-based parallel loading. +This module uses asyncio.gather() to fetch independent symbols concurrently +through providers that expose an asynchronous interface. Usage: from ml4t.data.managers.async_batch import async_batch_load @@ -15,9 +15,6 @@ end="2024-12-31", ) -Performance Comparison: - - Thread-based (BatchManager): ~1x baseline - - Async (async_batch_load): ~3-5x faster for I/O-bound operations """ from __future__ import annotations @@ -45,8 +42,8 @@ async def async_batch_load( ) -> pl.DataFrame: """Load OHLCV data for multiple symbols concurrently. - Uses asyncio.gather() for concurrent fetching, providing 3-5x - speedup over thread-based approaches for I/O-bound operations. + Uses asyncio.gather() and a concurrency limit to coordinate independent + provider requests. Args: provider: Async provider implementing fetch_ohlcv_async @@ -196,7 +193,7 @@ async def fetch_with_semaphore(symbol: str) -> tuple[str, pl.DataFrame | Excepti class AsyncBatchManager: - """Async batch manager for high-performance data loading. + """Async batch manager for data loading. Provides async versions of BatchManager methods using asyncio.gather() for concurrent operations. diff --git a/src/ml4t/data/managers/batch_manager.py b/src/ml4t/data/managers/batch_manager.py index 8a6e40e2..ae356475 100644 --- a/src/ml4t/data/managers/batch_manager.py +++ b/src/ml4t/data/managers/batch_manager.py @@ -26,7 +26,7 @@ class BatchManager: """Manages parallel batch loading operations. - This class provides high-performance parallel data fetching for + This class provides parallel data fetching for multiple symbols, with support for universes and storage integration. Attributes: diff --git a/src/ml4t/data/providers/AGENTS.md b/src/ml4t/data/providers/AGENTS.md index 976dd590..47818efe 100644 --- a/src/ml4t/data/providers/AGENTS.md +++ b/src/ml4t/data/providers/AGENTS.md @@ -1,63 +1,37 @@ -# providers/ - Provider Adapters - -## Base Classes - -| File | Lines | Purpose | -|------|-------|---------| -| base.py | 290 | `BaseProvider` abstract base | -| async_base.py | 256 | AsyncBaseProvider | -| protocols.py | 249 | Provider protocols | - -## Market Data Providers - -| File | Lines | Purpose | -|------|-------|---------| -| yahoo.py | 603 | Yahoo Finance (free) | -| alpaca.py | 968 | Alpaca US stocks + crypto (free IEX feed, two-cred auth) | -| binance_api.py | 410 | Binance REST API | -| binance_bulk.py | 1430 | Binance bulk historical archive | -| eodhd.py | 464 | EOD Historical Data | -| databento.py | 430 | Databento (futures) | -| tiingo.py | 259 | Tiingo API | -| twelve_data.py | 288 | Twelve Data API | -| polygon.py | 242 | Polygon.io | -| oanda.py | 373 | OANDA forex | -| okx.py | 577 | OKX exchange | - -## Crypto Providers - -| File | Lines | Purpose | -|------|-------|---------| -| coingecko.py | 497 | CoinGecko (free) | -| cryptocompare.py | 535 | CryptoCompare | - -## Alternative Data - -| File | Lines | Purpose | -|------|-------|---------| -| fred.py | 598 | Federal Reserve | -| fama_french.py | 987 | FF factor data | -| aqr.py | 970 | AQR factor data | -| wiki_prices.py | 747 | Quandl Wiki | -| kalshi.py | 828 | Kalshi prediction | -| polymarket.py | 913 | Polymarket | - -## Synthetic - -| File | Lines | Purpose | -|------|-------|---------| -| synthetic.py | 528 | Random price gen | -| learned_synthetic.py | 544 | ML-based synthetic | -| mock.py | 355 | Testing mock | - -## Key - -`BaseProvider`, `Provider`, `fetch_ohlcv()`, `fetch_series()`, `fetch()` - -## Prediction Market API References - -- Polymarket docs: `https://docs.polymarket.com/market-data/overview` -- Polymarket live slug lookup uses `GET /markets/slug/{slug}` on Gamma. The older `GET /markets?slug=...` query has returned empty results for current markets during March 2026 verification. -- Polymarket current-market discovery should prefer `active=true` with `closed=false`. `active=true` alone still returns many closed 2020-2021 markets. -- Kalshi docs: `https://docs.kalshi.com/welcome` -- For Kalshi OHLCV, verify the live candlestick schema before changing transforms. Recent live responses have used nested structs like `price`, `yes_bid`, `yes_ask`, plus `volume_fp`. +# Provider subsystem guide + +This directory contains provider contracts, registry metadata, and adapters that translate external +data sources into the library's supported data models. Provider availability, credentials, and +service limitations are documented in `docs/providers/`. + +## Shared components + +| Path | Responsibility | +|---|---| +| `base.py`, `async_base.py` | Synchronous and asynchronous provider base behavior | +| `protocols.py` | Structural contracts used by callers and type checking | +| `registry.py` | Provider discovery, capabilities, aliases, and factory metadata | +| `mixins/` | Shared HTTP-session and rate-limit behavior | +| `synthetic.py`, `mock.py` | Deterministic offline and testing providers | +| Other top-level modules | One external or packaged data source per adapter | + +## Adapter rules + +- Importing `ml4t.data.providers` must not perform network calls or require credentials. +- Read credentials through the adapter's documented configuration. Never log secrets or place them + in URLs when the service supports authenticated headers. +- Reuse shared sessions, rate limiting, retries, validation, and exception types. Do not add sleeps, + retry loops, or error hierarchies local to one adapter without a contract reason. +- Normalize remote responses at the adapter boundary and test missing, empty, malformed, and + rate-limited responses as well as the successful path. +- Keep optional dependencies isolated and fail with an actionable installation message when the + selected adapter needs an unavailable extra. +- Register supported adapters through `registry.py` and update registry contract tests when + capabilities or aliases change. +- Default tests mock external services. Live, credentialed, paid-tier, and slow checks use the + existing pytest markers and must not enter the offline lane. + +Start with the adapter's focused test module plus `tests/test_provider_registry_contract.py` and +`tests/test_provider_consistency.py`. For a new adapter, follow +`docs/contributing/creating-a-provider.md`, add its provider documentation, and then run the root +quality gates. diff --git a/src/ml4t/data/providers/__init__.py b/src/ml4t/data/providers/__init__.py index 482a2ae2..2813aa05 100644 --- a/src/ml4t/data/providers/__init__.py +++ b/src/ml4t/data/providers/__init__.py @@ -1,47 +1,7 @@ -"""Data provider implementations. - -This module provides unified access to multiple financial data providers. - -Available Providers: - - BaseProvider: Abstract base class for all providers - - YahooFinanceProvider: Yahoo Finance (free, no API key) - - AlpacaDataProvider: Alpaca US stocks and crypto (free IEX feed, two-credential auth) - - TiingoProvider: Tiingo stocks (free tier: 1000 req/day, 500 symbols/month) - - FinnhubProvider: Finnhub multi-asset data (free tier: 60 req/min) - - EODHDProvider: EODHD global equities (free tier: 500 req/day, 1 year depth) - - FREDProvider: FRED economic data (free, 120 req/min) - - FXMacroDataProvider: FX macro releases, calendars, COT, commodities, sentiment - - AQRFactorProvider: AQR research factors (QMJ, BAB, VME) - - FamaFrenchProvider: Fama-French factors (3-factor, 5-factor, momentum) - - KalshiProvider: Kalshi prediction markets (no API key needed for public data) - - PolymarketProvider: Polymarket prediction market history/order book snapshots - - CoinGeckoProvider: CoinGecko crypto data (free, no API key) - - BinanceProvider: Binance public market-data mirror and futures market data - - BinancePublicProvider: Binance public data (bulk downloads, no geo-restrictions) - - OKXProvider: OKX crypto perpetuals and funding rates (no geo-restrictions) - - CryptoCompareProvider: CryptoCompare crypto data - - TwelveDataProvider: Twelve Data multi-asset (stocks, forex, crypto) - - MassiveProvider: Massive multi-asset (stocks, options, futures, crypto, forex) - - PolygonProvider: Deprecated alias for MassiveProvider - - OandaProvider: Oanda forex and CFDs - - DataBentoProvider: DataBento market data - - ITCHSampleProvider: NASDAQ TotalView-ITCH sample data (tick-level, free) - - WikiPricesProvider: Quandl Wiki Prices (US equities 1962-2018, free) - - SyntheticProvider: Synthetic data generator (no network required) - - LearnedSyntheticProvider: ML-based synthetic data generator (no network required) - - MockProvider: Mock provider for testing - -Note: Updater classes have been removed to simplify the library. -If you need incremental update functionality, implement it separately using the provider's fetch_ohlcv() method. - -Example: - >>> from ml4t.data.providers import CoinGeckoProvider - >>> from datetime import UTC, datetime, timedelta - >>> - >>> # Use provider directly - >>> provider = CoinGeckoProvider() - >>> end = datetime.now(UTC).date() - timedelta(days=1) - >>> data = provider.fetch_ohlcv("bitcoin", str(end - timedelta(days=6)), str(end)) +"""Provider implementations and registry metadata. + +Use ``advertised_provider_specs`` to inspect capabilities, credential requirements, and +optional dependency extras without importing each provider implementation. """ from typing import TYPE_CHECKING diff --git a/src/ml4t/data/providers/aqr.py b/src/ml4t/data/providers/aqr.py index e39526e0..5b3d76be 100644 --- a/src/ml4t/data/providers/aqr.py +++ b/src/ml4t/data/providers/aqr.py @@ -150,7 +150,7 @@ class AQRFactorProvider(BaseProvider): AQR Capital Management is a global investment management firm built around the core principle that a systematic, disciplined approach to managing money adds value. Founded in 1998, AQR pioneered many factor investing strategies - and provides high-quality research data freely to support academic research. + and publishes research data for academic use. ## Why AQR Factor Data? diff --git a/src/ml4t/data/providers/async_base.py b/src/ml4t/data/providers/async_base.py index 6c94f51d..e886d97a 100644 --- a/src/ml4t/data/providers/async_base.py +++ b/src/ml4t/data/providers/async_base.py @@ -1,7 +1,7 @@ -"""Async base provider for high-performance concurrent fetching. +"""Async base provider for concurrent fetching. -This module provides AsyncBaseProvider for providers that support -async/await operations, enabling 3-5x speedup for batch operations. +This module provides AsyncBaseProvider for providers that support async/await +operations and concurrent batch requests. Usage: class MyAsyncProvider(AsyncBaseProvider): @@ -21,7 +21,7 @@ async def _fetch_and_transform_data_async( # Single fetch df = await provider.fetch_ohlcv_async("AAPL", "2024-01-01", "2024-12-31") - # Batch fetch (3-5x faster than sync) + # Fetch independent symbols concurrently results = await provider.batch_fetch_async( ["AAPL", "MSFT", "GOOGL"], "2024-01-01", diff --git a/src/ml4t/data/providers/binance.py b/src/ml4t/data/providers/binance.py index 10ca1ea4..c2a49352 100644 --- a/src/ml4t/data/providers/binance.py +++ b/src/ml4t/data/providers/binance.py @@ -30,8 +30,8 @@ class BinanceProvider(AsyncSessionMixin, BaseProvider): - No API key required for public market data - Spot and futures market support - Real-time and historical data - - High rate limits for public endpoints - - Async support for 10x faster batch fetches + - Public market-data endpoints + - Asynchronous batch fetch methods API Documentation: https://binance-docs.github.io/apidocs/ """ @@ -299,7 +299,7 @@ async def fetch_ohlcv_async( ) -> pl.DataFrame: """Async fetch OHLCV data for a cryptocurrency. - This is 3-10x faster than sync when fetching multiple symbols + This fetches multiple symbols concurrently using asyncio.gather() or async_batch_load(). Args: diff --git a/src/ml4t/data/providers/binance_public.py b/src/ml4t/data/providers/binance_public.py index bb289ecb..6c68a6ec 100644 --- a/src/ml4t/data/providers/binance_public.py +++ b/src/ml4t/data/providers/binance_public.py @@ -1,15 +1,4 @@ -"""Binance Public Data provider for bulk historical downloads. - -This provider downloads from data.binance.vision, Binance's public data repository. -Unlike the live API (BinanceProvider), this: -- Works globally without geographic restrictions -- Downloads bulk ZIP files (not rate-limited) -- Provides historical data going back years -- Uses MIT-licensed data distribution - -Repository: https://data.binance.vision -GitHub: https://github.com/binance/binance-public-data -""" +"""Binance Public Data provider for historical spot and futures archives.""" from __future__ import annotations @@ -60,12 +49,11 @@ class BinancePublicProvider(BaseProvider): Downloads from data.binance.vision - Binance's public S3 bucket with historical market data. This is separate from the live API and has - no geographic restrictions. + public historical archives. Features: - No API key required - - No rate limits (public S3 bucket) - - Works globally (no geo-restrictions) + - Public files without account credentials - Historical data going back years - Spot and USD-M futures markets - Daily and monthly aggregated files @@ -1513,8 +1501,7 @@ async def fetch_ohlcv_async( ) -> pl.DataFrame: """Async fetch OHLCV data for a symbol. - This is 3-5x faster than sync for multi-day fetches due to - concurrent HTTP requests. + Uses concurrent HTTP requests for independent daily files. Args: symbol: Symbol to fetch (e.g., "BTCUSDT") @@ -1762,7 +1749,7 @@ async def fetch_premium_index_async( ) -> pl.DataFrame: """Async version: Fetch premium index data for a single symbol. - 3-5x faster than sync version due to concurrent HTTP requests. + Uses concurrent HTTP requests for independent daily files. Args: symbol: Futures symbol (e.g., "BTCUSDT") @@ -1837,7 +1824,7 @@ async def fetch_premium_index_multi_async( ) -> pl.DataFrame: """Async version: Fetch premium index for multiple symbols in parallel. - Significantly faster than sync version - downloads all symbols concurrently. + Downloads data for independent symbols concurrently. Args: symbols: List of futures symbols (e.g., ["BTCUSDT", "ETHUSDT"]) @@ -1915,8 +1902,7 @@ def fetch_premium_index_multi_parallel( ) -> pl.DataFrame: """Sync wrapper for parallel premium index download. - Uses asyncio internally for parallel downloads, 3-10x faster than - the sequential version for multiple symbols. + Uses asyncio internally to download data for multiple symbols concurrently. Args: symbols: List of futures symbols diff --git a/src/ml4t/data/providers/coingecko.py b/src/ml4t/data/providers/coingecko.py index b2e5b108..0a753b66 100644 --- a/src/ml4t/data/providers/coingecko.py +++ b/src/ml4t/data/providers/coingecko.py @@ -1,22 +1,6 @@ -"""CoinGecko cryptocurrency data provider. - -CoinGecko provides free cryptocurrency data with API key (Demo plan). -Demo plan (free): 30 calls/minute, 10,000 calls/month. -Public API (no key): 5-15 calls/minute (varies). - -Performance Note: -- This is a simple wrapper for consistent API across providers -- For advanced features (coin list, price API, market data), use the CoinGecko API directly -- Rate limiting is handled by BaseProvider - -Async Example: - last_complete_day = datetime.now(UTC).date() - timedelta(days=1) - async with CoinGeckoProvider() as provider: - df = await provider.fetch_ohlcv_async( - "BTC", - str(last_complete_day - timedelta(days=6)), - str(last_complete_day), - ) +"""CoinGecko provider for public and authenticated cryptocurrency OHLCV data. + +Account plans and public endpoints determine request quotas and available history. """ from __future__ import annotations @@ -45,22 +29,10 @@ class CoinGeckoProvider(AsyncSessionMixin, BaseProvider): - """Simple CoinGecko wrapper for OHLCV data with consistent API. - - This provider exists primarily to provide: - - Consistent API across all providers (fetch_ohlcv interface) - - Polars DataFrame output (standardized schema) - - Symbol column in results - - Structured logging and typed exceptions - - Async support for 10x faster batch fetches - - For advanced CoinGecko features (coin lists, price API, market metrics), - use the CoinGecko API directly or dedicated crypto data libraries. - - Rate Limits: - - Demo plan (with API key): 30 calls/minute - - Public API (no key): 10 calls/minute - - Pro tier: 500 calls/minute (requires paid key) + """Fetch daily cryptocurrency OHLCV data through the CoinGecko interface. + + Advanced CoinGecko endpoints remain outside this adapter. Request quotas depend on the + selected public or authenticated API plan. """ # Common symbol to CoinGecko ID mappings @@ -703,7 +675,7 @@ async def fetch_ohlcv_async( ) -> pl.DataFrame: """Async fetch OHLCV data for a cryptocurrency. - This is 3-10x faster than sync when fetching multiple symbols + This fetches multiple symbols concurrently using asyncio.gather() or async_batch_load(). Args: diff --git a/src/ml4t/data/providers/cryptocompare.py b/src/ml4t/data/providers/cryptocompare.py index c5a97440..a28b3b10 100644 --- a/src/ml4t/data/providers/cryptocompare.py +++ b/src/ml4t/data/providers/cryptocompare.py @@ -20,16 +20,9 @@ class CryptoCompareProvider(BaseProvider): - """ - Provider for cryptocurrency data from CryptoCompare API. - - Features: - - Spot price data for major cryptocurrencies - - Historical OHLCV data - - Multiple exchange support - - Free tier: 100,000 calls/month + """Fetch cryptocurrency OHLCV data from CryptoCompare. - API Documentation: https://min-api.cryptocompare.com/documentation + Endpoint limits and account quotas are external service constraints. """ BASE_URL = "https://min-api.cryptocompare.com/data/v2" @@ -44,18 +37,18 @@ class CryptoCompareProvider(BaseProvider): "1day": "histoday", } - # Limits for each frequency (free tier) + # Maximum observations accepted by each history endpoint FREQUENCY_LIMITS: ClassVar[dict[str, int]] = { "histominute": 2000, # Max 2000 minutes per call "histohour": 2000, # Max 2000 hours per call "histoday": 2000, # Max 2000 days per call } - # Override default rate limiting for CryptoCompare (100k calls/month free) + # Conservative client-side request pace DEFAULT_RATE_LIMIT: ClassVar[tuple[int, float]] = ( 10, 60.0, - ) # 10 requests per minute for free tier + ) MAX_RATE_LIMIT_ATTEMPTS: ClassVar[int] = 3 DEFAULT_RETRY_AFTER: ClassVar[float] = 6.0 MAX_RETRY_AFTER: ClassVar[float] = 60.0 diff --git a/src/ml4t/data/providers/eodhd.py b/src/ml4t/data/providers/eodhd.py index af76edde..c36a7945 100644 --- a/src/ml4t/data/providers/eodhd.py +++ b/src/ml4t/data/providers/eodhd.py @@ -1,28 +1,7 @@ -"""EODHD data provider. +"""EODHD provider for authenticated equity market data. -EODHD provides affordable global equities data with 60+ exchanges and -150,000+ tickers worldwide. - -API Documentation: https://eodhd.com/financial-apis/ - -Free Tier Limits: -- 20 API calls per day -- 1 year historical depth -- Daily/weekly/monthly OHLCV data -- Global coverage (60+ exchanges) - -Symbol Format: -- Use SYMBOL.EXCHANGE format (e.g., "AAPL.US", "VOD.LSE", "BMW.FRA") - -Example: - >>> from ml4t.data.providers.eodhd import EODHDProvider - >>> provider = EODHDProvider(api_key="your_key") - >>> data = provider.fetch_ohlcv("AAPL", "2024-01-01", "2024-01-31", exchange="US") - >>> provider.close() - -Async Example: - >>> async with EODHDProvider(api_key="your_key") as provider: - ... data = await provider.fetch_ohlcv_async("AAPL", "2024-01-01", "2024-01-31") +Account plans determine endpoint access, request quotas, and historical depth. Symbols use the +``SYMBOL.EXCHANGE`` format. """ import os @@ -57,26 +36,12 @@ class EODHDProvider(AsyncSessionMixin, BaseProvider): - """EODHD data provider. - - Supports global equities with daily/weekly/monthly OHLCV data. - - Rate Limits (Free Tier): - - 20 API calls per day - - 1 year historical depth - - Supports both sync and async operations: - # Sync - provider = EODHDProvider() - df = provider.fetch_ohlcv("AAPL", start, end) + """Fetch equity OHLCV and fundamentals data from EODHD. - # Async (10x faster for batch fetches) - async with EODHDProvider() as provider: - df = await provider.fetch_ohlcv_async("AAPL", start, end) + ``DEFAULT_RATE_LIMIT`` is a conservative client pacing value. Account quotas still apply. """ - # Stay below EODHD's documented 1,000 requests/minute server limit. - # Account-specific daily API-call budgets cannot be modeled as an inter-request delay. + # Conservative client-side request pace; account quotas still apply. DEFAULT_RATE_LIMIT: ClassVar[tuple[int, float]] = (10, 1.0) # Map frequency to EODHD period codes @@ -229,7 +194,7 @@ def _fetch_raw_data( symbol=formatted_symbol, details={ "warning": data[0]["warning"], - "tier_limitation": "Free tier is limited to 1 year of historical data", + "tier_limitation": "Requested history is unavailable for the current account plan", }, ) @@ -551,7 +516,7 @@ async def _fetch_raw_data_async( symbol=formatted_symbol, details={ "warning": data[0]["warning"], - "tier_limitation": "Free tier is limited to 1 year of historical data", + "tier_limitation": "Requested history is unavailable for the current account plan", }, ) @@ -578,7 +543,7 @@ async def fetch_ohlcv_async( ) -> pl.DataFrame: """Async fetch OHLCV data for a symbol. - This is 3-10x faster than sync when fetching multiple symbols + This fetches multiple symbols concurrently using asyncio.gather() or async_batch_load(). Args: diff --git a/src/ml4t/data/providers/finnhub.py b/src/ml4t/data/providers/finnhub.py index abddadbc..e80ff1e9 100644 --- a/src/ml4t/data/providers/finnhub.py +++ b/src/ml4t/data/providers/finnhub.py @@ -1,25 +1,6 @@ -"""Finnhub data provider. +"""Finnhub provider for authenticated market data. -Finnhub provides comprehensive financial market data with global coverage. - -API Documentation: https://finnhub.io/docs/api -Pricing: https://finnhub.io/pricing - -Tier limitations: - -FREE TIER (60 API calls/minute): -- Real-time US equity quotes -- Selected US company data - -PAID TIER: -- Historical OHLCV -- Global market data and extended history - -Example: - >>> from ml4t.data.providers.finnhub import FinnhubProvider - >>> provider = FinnhubProvider(api_key="your_key") - >>> data = provider.fetch_ohlcv("AAPL", "2024-01-01", "2024-01-31") - >>> provider.close() +Endpoint availability, request quotas, and historical depth depend on the account plan. """ import os @@ -54,16 +35,12 @@ class FinnhubProvider(BaseProvider): - """Finnhub data provider. - - Supports stocks, ETFs, forex, and crypto with multiple resolutions. + """Fetch Finnhub market data through synchronous provider methods. - Rate Limits: - - Free tier: 60 requests/minute for supported endpoints such as US quotes - - Paid tier: Historical OHLC and higher limits depend on the current plan + ``DEFAULT_RATE_LIMIT`` is a conservative client pacing value. Account quotas still apply. """ - # Free tier: 60 requests/min = 1 per second + # Conservative client-side request pace; account quotas still apply. DEFAULT_RATE_LIMIT: ClassVar[tuple[int, float]] = (1, 1.0) # Map frequency to Finnhub resolution codes @@ -138,7 +115,7 @@ def name(self) -> str: return "finnhub" def fetch_quote(self, symbol: str) -> dict[str, str | int | float]: - """Fetch a real-time US equity quote available on Finnhub's free tier. + """Fetch a real-time US equity quote from Finnhub. Args: symbol: US equity symbol, such as ``AAPL``. diff --git a/src/ml4t/data/providers/nasdaq_itch.py b/src/ml4t/data/providers/nasdaq_itch.py index 3da2e53b..3784a02c 100644 --- a/src/ml4t/data/providers/nasdaq_itch.py +++ b/src/ml4t/data/providers/nasdaq_itch.py @@ -52,7 +52,7 @@ class ITCHSampleProvider: ## Why Use ITCH Sample Data? - **Best For**: + **Use cases**: - Learning market microstructure concepts - Testing order book reconstruction algorithms - Demonstrating tick-level data analysis diff --git a/src/ml4t/data/providers/oanda.py b/src/ml4t/data/providers/oanda.py index 5c06d26d..74c3ffbc 100644 --- a/src/ml4t/data/providers/oanda.py +++ b/src/ml4t/data/providers/oanda.py @@ -32,9 +32,9 @@ class OandaProvider(BaseProvider): - """OANDA provider for institutional-grade FX market data. + """OANDA provider for authenticated foreign exchange market data. - OANDA provides high-quality foreign exchange data with excellent coverage + OANDA provides foreign exchange data of major, minor, and exotic currency pairs. """ diff --git a/src/ml4t/data/providers/okx.py b/src/ml4t/data/providers/okx.py index 5e92a26e..61b10cdb 100644 --- a/src/ml4t/data/providers/okx.py +++ b/src/ml4t/data/providers/okx.py @@ -4,7 +4,6 @@ - OHLCV data for perpetual swap contracts - Historical and current funding rates - No API key required for public market data -- No geo-restrictions (works globally) API Documentation: https://www.okx.com/docs-v5/en/ @@ -34,10 +33,9 @@ class OKXProvider(AsyncSessionMixin, BaseProvider): Features: - No API key required for public market data - - No geo-restrictions (unlike Binance/Bybit) - Perpetual swap contracts with funding rates - - High-quality historical data - - Async support for 10x faster batch fetches + - Historical market data + - Asynchronous batch fetch methods Symbols use OKX format: BTC-USDT-SWAP, ETH-USDT-SWAP, etc. """ @@ -487,7 +485,7 @@ async def fetch_ohlcv_async( ) -> pl.DataFrame: """Async fetch OHLCV data from OKX. - This is 3-10x faster than sync when fetching multiple symbols + This fetches multiple symbols concurrently using asyncio.gather() or async_batch_load(). Args: diff --git a/src/ml4t/data/providers/polygon.py b/src/ml4t/data/providers/polygon.py index c99adb73..6cf9adb8 100644 --- a/src/ml4t/data/providers/polygon.py +++ b/src/ml4t/data/providers/polygon.py @@ -1,27 +1,6 @@ -"""Massive data provider with Polygon compatibility. - -Massive.com, formerly Polygon.io, offers comprehensive financial data across -multiple asset classes: -- Stocks (equities) -- Options -- Futures -- Cryptocurrencies -- Forex - -Rate Limits: - - Free tier: 5 requests/minute - - Paid tiers: higher or unlimited usage depending on product - -Authentication: - Requires API key from https://massive.com/ - Set MASSIVE_API_KEY or POLYGON_API_KEY, or pass api_key explicitly. - -Example: - >>> from ml4t.data.providers.polygon import MassiveProvider - >>> provider = MassiveProvider(api_key="your_key") - >>> data = provider.fetch_ohlcv("AAPL", "2024-01-01", "2024-01-31") - >>> futures = provider.fetch_ohlcv("ESM6", "2024-01-01", "2024-01-31", asset_class="futures") - >>> provider.close() +"""Massive market-data provider with Polygon credential compatibility. + +Account products determine asset-class access, request quotas, and historical depth. """ import os @@ -68,7 +47,7 @@ class MassiveProvider(BaseProvider): API_KEY_ENV_VARS: ClassVar[tuple[str, str]] = ("MASSIVE_API_KEY", "POLYGON_API_KEY") BASE_URL_ENV_VARS: ClassVar[tuple[str, str]] = ("MASSIVE_BASE_URL", "POLYGON_BASE_URL") - DEFAULT_RATE_LIMIT: ClassVar[tuple[int, float]] = (5, 60.0) # Basic free tier + DEFAULT_RATE_LIMIT: ClassVar[tuple[int, float]] = (5, 60.0) # Conservative client pace # Map frequencies to Massive/Polygon timespans. FREQUENCY_MAP: ClassVar[dict[str, str]] = { diff --git a/src/ml4t/data/providers/tiingo.py b/src/ml4t/data/providers/tiingo.py index 8739f763..0a0febba 100644 --- a/src/ml4t/data/providers/tiingo.py +++ b/src/ml4t/data/providers/tiingo.py @@ -1,20 +1,6 @@ -"""Tiingo data provider. +"""Tiingo provider for authenticated equity and ETF data. -Tiingo provides stock market data with extensive historical coverage and -adjusted prices for splits/dividends. - -API Documentation: https://www.tiingo.com/documentation/end-of-day - -Free Tier Limits: -- 1000 API calls per day -- Historical daily data (5+ years) -- 500 unique symbols per month - -Example: - >>> from ml4t.data.providers.tiingo import TiingoProvider - >>> provider = TiingoProvider(api_key="your_key") - >>> data = provider.fetch_ohlcv("AAPL", "2024-01-01", "2024-01-31") - >>> provider.close() +Account plans determine request quotas and historical depth. """ import os @@ -38,17 +24,12 @@ class TiingoProvider(BaseProvider): - """Tiingo data provider. - - Supports stocks and ETFs with daily OHLCV data (raw and adjusted). + """Fetch daily raw and adjusted OHLCV data from Tiingo. - Rate Limits (Free Tier): - - 1000 requests per day - - 500 unique symbols per month + ``DEFAULT_RATE_LIMIT`` is a conservative client pacing value. Account quotas still apply. """ - # Free tier: 1000 requests/day = ~0.69 per minute = 1 per 86.4 seconds - # Conservative: 1 request per 90 seconds + # Conservative client-side request pace; account quotas still apply. DEFAULT_RATE_LIMIT: ClassVar[tuple[int, float]] = (1, 90.0) # Map frequency to Tiingo intervals diff --git a/src/ml4t/data/providers/twelve_data.py b/src/ml4t/data/providers/twelve_data.py index 5b3e6e1d..fa8fcc5b 100644 --- a/src/ml4t/data/providers/twelve_data.py +++ b/src/ml4t/data/providers/twelve_data.py @@ -1,23 +1,6 @@ -"""Twelve Data provider for multi-asset market data. +"""Twelve Data provider for authenticated multi-asset market data. -Twelve Data provides comprehensive market data for stocks, forex, and cryptocurrencies -across 50+ global exchanges. - -API Documentation: https://twelvedata.com/docs - -Free Tier Limits: -- 800 API requests per day -- 8 requests per minute (strictest rate limit) -- 15+ years of historical data - -Example: - >>> from ml4t.data.providers.twelve_data import TwelveDataProvider - >>> provider = TwelveDataProvider(api_key="your_key") - >>> data = provider.fetch_ohlcv("AAPL", "2024-01-01", "2024-01-31") - -Async Example: - >>> async with TwelveDataProvider(api_key="your_key") as provider: - ... data = await provider.fetch_ohlcv_async("AAPL", "2024-01-01", "2024-01-31") +Account plans determine endpoint access, request quotas, and historical depth. """ import os @@ -41,17 +24,12 @@ class TwelveDataProvider(AsyncSessionMixin, BaseProvider): - """Twelve Data provider for multi-asset market data. - - Supports stocks, ETFs, indices, forex, and cryptocurrencies. - Async support for 10x faster batch fetches. + """Fetch stock, ETF, index, foreign exchange, and cryptocurrency data. - Rate Limits (Free Tier): - - 8 requests per minute (strictest limit) - - 800 requests per day + ``DEFAULT_RATE_LIMIT`` is a conservative client pacing value. Account quotas still apply. """ - # 8 requests per minute = 0.133/second + # Conservative client-side request pace; account quotas still apply. DEFAULT_RATE_LIMIT = (8, 60.0) # Map frequency to Twelve Data interval format @@ -235,7 +213,7 @@ async def fetch_ohlcv_async( ) -> pl.DataFrame: """Async fetch OHLCV data for a symbol. - This is 3-10x faster than sync when fetching multiple symbols + This fetches multiple symbols concurrently using asyncio.gather() or async_batch_load(). Args: diff --git a/src/ml4t/data/providers/wiki_prices.py b/src/ml4t/data/providers/wiki_prices.py index 186f34bd..b9f26112 100644 --- a/src/ml4t/data/providers/wiki_prices.py +++ b/src/ml4t/data/providers/wiki_prices.py @@ -28,134 +28,10 @@ class WikiPricesProvider(BaseProvider): - """ - Provider for Quandl WIKI Prices dataset (local Parquet archive). - - Dataset: Community-curated US equities end-of-day prices - Date Range: 1962-01-02 to 2018-03-27 (DEPRECATED - no updates) - Coverage: 3,199 US companies - Size: 632MB Parquet, 15.4M rows - - ## Why Use Wiki Prices? - - **Best For**: - - Historical backtesting (pre-2018) - - Long-term market studies (56 years) - - Academic research requiring survivorship-bias-free data - - Teaching quantitative finance (free, offline, comprehensive) - - **Not Suitable For**: - - Live trading or recent analysis (7-year data gap) - - Post-2018 strategies validation - - Current market conditions study - - ## Dataset Background - - Wiki Prices was deprecated by Quandl (now NASDAQ Data Link) in April 2018 - when primary data sources became unavailable. While no longer updated, it - remains valuable for historical research due to: - - - 56 years of high-quality data (1962-2018) - - Survivorship-bias free (includes delisted companies) - - Adjusted prices (splits/dividends handled) - - Institutional-grade curation - - Public domain (no usage restrictions) + """Read the frozen Quandl WIKI Prices dataset from a local Parquet archive. - See: docs/wiki_prices_status.md for comprehensive analysis - - ## Usage - - ```python - from ml4t.data.providers.wiki_prices import WikiPricesProvider - - # Default location (auto-detect) - provider = WikiPricesProvider() - - # Custom Parquet location - provider = WikiPricesProvider(parquet_path="/path/to/wiki_prices.parquet") - - # Fetch historical data (pre-2018 only) - aapl = provider.fetch_ohlcv("AAPL", "2010-01-01", "2018-03-27") - - # Use in multi-provider fallback strategy - def fetch_with_fallback(symbol, start, end): - wiki_cutoff = "2018-03-27" - - if end <= wiki_cutoff: - # Pure historical - use Wiki Prices - return wiki_provider.fetch_ohlcv(symbol, start, end) - elif start > wiki_cutoff: - # Pure recent - use Yahoo/EODHD - return yahoo_provider.fetch_ohlcv(symbol, start, end) - else: - # Spans cutoff - combine sources - wiki_data = wiki_provider.fetch_ohlcv(symbol, start, wiki_cutoff) - yahoo_data = yahoo_provider.fetch_ohlcv(symbol, "2018-03-28", end) - return pl.concat([wiki_data, yahoo_data]).sort("timestamp") - ``` - - ## Performance Characteristics - - **Initialization**: - - Lazy loading: Parquet mapped to memory (no upfront load) - - First query: ~100-200ms (scan Parquet schema) - - Subsequent queries: ~5-20ms (memory-mapped access) - - **Memory Usage**: - - Lazy mode (default): Minimal (<100MB) - - Eager mode (cache_in_memory=True): ~1.2GB (full dataset in RAM) - - **Query Performance**: - - Single symbol: 5-20ms (lazy scan) - - Batch symbols: 100-500ms for 10 stocks (parallel scans) - - Full dataset scan: ~2-3 seconds (15.4M rows) - - ## Comparison to Alternatives - - | Provider | Date Range | Cost | Speed | Quality | - |----------------|---------------|-------|-------------|--------------| - | Wiki Prices | 1962-2018 | Free | Very Fast | Excellent | - | Yahoo Finance | 2000-present | Free | Fast | Good | - | EODHD | 1980-present | €20/mo| Fast | Excellent | - | DataBento | 2000-present | $9+/mo| Very Fast | Professional | - - ## Schema - - Wiki Prices Parquet schema (14 columns): - ``` - ticker (String) - Stock symbol - date (Datetime) - Trading date - open (Float64) - Opening price (unadjusted) - high (Float64) - High price (unadjusted) - low (Float64) - Low price (unadjusted) - close (Float64) - Closing price (unadjusted) - volume (Float64) - Trading volume (unadjusted) - ex-dividend (Float64) - Dividend amount - split_ratio (Float64) - Stock split ratio - adj_open (Float64) - Adjusted opening price - adj_high (Float64) - Adjusted high price - adj_low (Float64) - Adjusted low price - adj_close (Float64) - Adjusted closing price - adj_volume (Float64) - Adjusted volume - ``` - - Transformed to ml4t-data standard schema: - ``` - timestamp (Datetime) - Trading date - open (Float64) - Adjusted opening price - high (Float64) - Adjusted high price - low (Float64) - Adjusted low price - close (Float64) - Adjusted closing price - volume (Float64) - Adjusted volume - ``` - - ## Known Limitations - - - **Frozen Dataset**: No data after March 27, 2018 - - **Daily Only**: No intraday/minute data available - - **US Equities Only**: No international stocks - - **End-of-Life Quality**: Some gaps/errors in final months (2018) - - **No Real-Time**: Static archive, not live API + The dataset ends on 2018-03-27 and is unsuitable for current-market workflows. The provider + does not download or license the archive for the caller. See docs/providers/wiki_prices.md. """ DEFAULT_PATHS: ClassVar[list[Path]] = _DEFAULT_PATHS diff --git a/src/ml4t/data/providers/yahoo.py b/src/ml4t/data/providers/yahoo.py index 56f898dc..9a097707 100644 --- a/src/ml4t/data/providers/yahoo.py +++ b/src/ml4t/data/providers/yahoo.py @@ -391,8 +391,8 @@ def fetch_batch_ohlcv( """ Fetch OHLCV data for multiple symbols efficiently using batch downloads. - This method is ~20x faster than calling fetch_ohlcv() for each symbol - individually because it uses yfinance's multi-ticker download capability. + This method uses yfinance's multi-ticker download capability instead of issuing one + library call per symbol. Use this for: - Initial data acquisition (hundreds of symbols) diff --git a/src/ml4t/data/security/__init__.py b/src/ml4t/data/security/__init__.py index d98bf7ed..c6f68a9e 100644 --- a/src/ml4t/data/security/__init__.py +++ b/src/ml4t/data/security/__init__.py @@ -1,4 +1,4 @@ -"""Security utilities for QLDM.""" +"""Security utilities for ml4t-data.""" from ml4t.data.security.path_validator import PathTraversalError, PathValidator diff --git a/src/ml4t/data/storage/AGENTS.md b/src/ml4t/data/storage/AGENTS.md index 0a2825a3..cf5ac592 100644 --- a/src/ml4t/data/storage/AGENTS.md +++ b/src/ml4t/data/storage/AGENTS.md @@ -1,45 +1,33 @@ -# storage/ - 4.6k Lines - -Hive-partitioned and flat Parquet storage with data profiling. - -## Modules - -| File | Purpose | -|------|---------| -| hive.py | HiveStorage main class | -| backend.py | Storage backend interface | -| flat.py | Simple flat file storage | -| chunked.py | Chunked file storage | -| filesystem.py | Filesystem operations | -| metadata_tracker.py | Metadata management | -| protocols.py | Storage protocols | -| async_base.py | Async storage base | -| data_profile.py | Column-level statistics and ProfileMixin | - -## Key Classes - -- `HiveStorage` - Main storage class with Hive partitioning -- `ProfileMixin` - Mixin for on-demand data profiling (generate_profile, load_profile) -- `DatasetProfile` - Column statistics container -- `ColumnProfile` - Per-column statistics (dtype, nulls, min/max, mean/std) - -## Data Profiling - -```python -from ml4t.data.storage import ProfileMixin, generate_profile - -# Any data manager can inherit ProfileMixin for profiling -class MyManager(ProfileMixin): - def _get_profile_data(self) -> pl.DataFrame: - return self.load_data() - def _get_profile_data_path(self) -> Path: - return self.data_path - def _get_profile_source_name(self) -> str: - return "MyManager" - -# Now has generate_profile() and load_profile() methods -``` - -## Key Functions - -`HiveStorage`, `get_storage()`, `generate_profile()`, `save_profile()`, `load_profile()` +# Storage subsystem guide + +This directory owns the storage protocols and local persistence implementations used by data +managers and update workflows. General data-root selection lives in `ml4t.data.core.config`. + +## Change locations + +| File | Responsibility | +|---|---| +| `protocols.py`, `backend.py`, `async_base.py` | Storage interfaces and shared behavior | +| `hive.py`, `flat.py`, `chunked.py` | Concrete Parquet-backed layouts | +| `config.py` | Storage-specific configuration | +| `keys.py` | Dataset-key normalization and validation | +| `metadata_tracker.py` | Dataset metadata persistence | +| `migration.py`, `legacy_migration.py` | Supported layout and compatibility migrations | +| `data_profile.py` | Dataset and column profiling | + +## Invariants + +- Resolve defaults through the shared package data-root helpers. Explicit caller paths take + precedence, and source code must not contain hard-coded home directories. +- Normalize and validate dataset keys through `keys.py`; storage backends must agree on key meaning. +- Preserve protocol behavior across concrete backends, including empty datasets, overwrite and + append behavior, metadata updates, and error types. +- Keep migrations explicit and tested. Do not silently reinterpret an existing layout or remove a + compatibility path without the repository's deprecation process. +- Keep storage tests local and deterministic. Network acquisition belongs to providers, not storage. +- Test resource cleanup for asynchronous or file-backed changes; the repository's ResourceWarning + lane is required. + +Run the matching tests in `tests/storage/` and the root-level `tests/test_*storage*.py` modules, then +run the root quality gates. User workflows and configuration examples belong in +`docs/user-guide/storage.md` and `docs/user-guide/configuration.md`. diff --git a/src/ml4t/data/storage/__init__.py b/src/ml4t/data/storage/__init__.py index a7de9b69..530c188c 100644 --- a/src/ml4t/data/storage/__init__.py +++ b/src/ml4t/data/storage/__init__.py @@ -1,7 +1,7 @@ """ML4T Data storage module. Provides configurable storage backends with Hive partitioning for -high-performance time-series data management. +time-series data management. """ from __future__ import annotations diff --git a/src/ml4t/data/storage/migration.py b/src/ml4t/data/storage/migration.py index 046b13a6..0d801e22 100644 --- a/src/ml4t/data/storage/migration.py +++ b/src/ml4t/data/storage/migration.py @@ -1,4 +1,4 @@ -"""Data migration framework for QLDM.""" +"""Data migration framework for ml4t-data.""" from __future__ import annotations @@ -548,7 +548,7 @@ def _save_backup_metadata( # Example migrations def create_standard_migrations() -> list[Migration]: - """Create standard migrations for QLDM.""" + """Create standard migrations for ml4t-data.""" migrations = [] # Migration to add volume column if missing diff --git a/src/ml4t/data/utils/AGENTS.md b/src/ml4t/data/utils/AGENTS.md deleted file mode 100644 index 8cbfe76e..00000000 --- a/src/ml4t/data/utils/AGENTS.md +++ /dev/null @@ -1,20 +0,0 @@ -# utils/ - 1.6k Lines - -Rate limiting, gap detection, and retry utilities. - -## Modules - -| File | Purpose | -|------|---------| -| rate_limit.py | Per-provider rate limits | -| async_rate_limit.py | Async rate limiting | -| global_rate_limit.py | Global rate coordinator | -| retry.py | Retry with backoff | -| gaps.py | Gap detection | -| gap_optimizer.py | Gap merge optimization | -| locking.py | File locking | -| format.py | Data formatting | - -## Key - -`RateLimiter`, `detect_gaps()`, `retry_with_backoff()` diff --git a/src/ml4t/data/utils/__init__.py b/src/ml4t/data/utils/__init__.py index 895f5ef6..5ab9e9bb 100644 --- a/src/ml4t/data/utils/__init__.py +++ b/src/ml4t/data/utils/__init__.py @@ -1,4 +1,4 @@ -"""Utility functions for QLDM.""" +"""Utility functions for ml4t-data.""" from ml4t.data.utils.format import pivot_to_stacked, pivot_to_wide diff --git a/src/ml4t/data/utils/async_rate_limit.py b/src/ml4t/data/utils/async_rate_limit.py index ecedb52b..b287ca5b 100644 --- a/src/ml4t/data/utils/async_rate_limit.py +++ b/src/ml4t/data/utils/async_rate_limit.py @@ -1,4 +1,4 @@ -"""Async rate limiting utilities for QLDM.""" +"""Async rate limiting utilities for ml4t-data.""" from __future__ import annotations diff --git a/src/ml4t/data/utils/rate_limit.py b/src/ml4t/data/utils/rate_limit.py index 80835ba8..8fb85b3c 100644 --- a/src/ml4t/data/utils/rate_limit.py +++ b/src/ml4t/data/utils/rate_limit.py @@ -1,4 +1,4 @@ -"""Rate limiting utilities for QLDM.""" +"""Rate limiting utilities for ml4t-data.""" from __future__ import annotations diff --git a/src/ml4t/data/utils/retry.py b/src/ml4t/data/utils/retry.py index 0c09ed4d..02aa07d3 100644 --- a/src/ml4t/data/utils/retry.py +++ b/src/ml4t/data/utils/retry.py @@ -1,4 +1,4 @@ -"""Retry utilities for QLDM.""" +"""Retry utilities for ml4t-data.""" from collections.abc import Callable from typing import Any, TypeVar, cast diff --git a/src/ml4t/data/validation/__init__.py b/src/ml4t/data/validation/__init__.py index 589b78e3..6b1613a0 100644 --- a/src/ml4t/data/validation/__init__.py +++ b/src/ml4t/data/validation/__init__.py @@ -1,4 +1,4 @@ -"""Data validation module for QLDM.""" +"""Data validation module for ml4t-data.""" from ml4t.data.validation.base import ValidationResult, Validator from ml4t.data.validation.ohlcv import NegativePricePolicy, OHLCVValidator diff --git a/src/ml4t/data/validation/rules.py b/src/ml4t/data/validation/rules.py index 89400542..03e53053 100644 --- a/src/ml4t/data/validation/rules.py +++ b/src/ml4t/data/validation/rules.py @@ -248,7 +248,7 @@ def for_asset_class(asset_class: str | AssetClass) -> ValidationRuleConfig: @staticmethod def strict_rules() -> ValidationRuleConfig: - """Strict validation rules for high-quality data.""" + """Strict validation rules for market data.""" return ValidationRuleConfig( check_nulls=True, check_price_consistency=True, diff --git a/tests/MULTI_ASSET_INTEGRATION_REPORT.md b/tests/MULTI_ASSET_INTEGRATION_REPORT.md deleted file mode 100644 index 57bca302..00000000 --- a/tests/MULTI_ASSET_INTEGRATION_REPORT.md +++ /dev/null @@ -1,386 +0,0 @@ -# Multi-Asset Integration Tests - Comprehensive Report - -**Date**: 2025-11-15 -**Test Suite**: `tests/test_multi_asset.py` -**Status**: ✅ COMPLETED (Task-006) - ---- - -## Executive Summary - -Comprehensive integration test suite created for multi-asset support in ml4t-data library. All acceptance criteria met or exceeded: - -- ✅ **22/22 tests passing** (100% pass rate, excluding 1 long-running network test) -- ✅ **Coverage**: 76% average across core multi-asset modules (exceeds 70% target) -- ✅ **Performance**: All targets validated (storage <1s, format conversion <1s) -- ✅ **Edge cases**: 10+ edge cases thoroughly tested - ---- - -## Test Coverage - -### Test Statistics - -| Metric | Count | Notes | -|--------|-------|-------| -| Total Tests | 23 | Comprehensive integration coverage | -| Passing | 22 | Excluding 1 slow network test | -| Test Classes | 7 | Organized by test category | -| Edge Cases | 11 | Boundary conditions covered | -| Performance Tests | 3 | Validated all targets | -| Workflow Tests | 8 | End-to-end scenarios | - -### Module Coverage - -| Module | Statements | Missing | Coverage | Notes | -|--------|------------|---------|----------|-------| -| `core/schemas.py` | 70 | 25 | **64%** | Schema validation | -| `universe.py` | 37 | 10 | **73%** | Universe management | -| `utils/format.py` | 79 | 10 | **87%** | Format conversion | -| **TOTAL** | 186 | 45 | **76%** | Exceeds 70% target ✅ | - ---- - -## Test Organization - -### 1. Workflow Tests (`TestMultiAssetWorkflows`) - -Complete end-to-end workflows using all multi-asset features: - -- ✅ **test_complete_workflow_load_analyze_convert** - Full pipeline: Storage → Validate → Wide → Stacked -- ✅ **test_workflow_universe_load_to_storage_to_analysis** - Universe → Storage → Analysis -- ✅ **test_workflow_storage_cache_first_loading** - Cache-first loading pattern -- ✅ **test_workflow_mixed_symbols_storage_and_fetch** - Graceful degradation - -**Key Validations**: -- Multi-asset schema compliance throughout pipeline -- Round-trip conversions preserve data integrity -- Storage caching provides significant speedup - -### 2. Performance Tests (`TestPerformanceIntegration`) - -Performance characteristics validation: - -- ✅ **test_storage_load_100_symbols_performance** - 100 symbols < 1s ✅ (achieved 0.117s) -- ✅ **test_format_conversion_performance_large_dataset** - 12,600 rows conversion < 1s ✅ -- ⏸️ **test_batch_load_universe_performance_target** - 10 symbols network fetch (skipped due to rate limiting) - -**Achieved Performance**: -``` -Storage load (100 symbols): 0.117s (Target: <1.0s) ✅ -Stacked→Wide (12,600 rows): 0.009s (Target: <1.0s) ✅ -Wide→Stacked (252×50 cells): 0.010s (Target: <1.0s) ✅ -``` - -### 3. Cross-Feature Integration (`TestCrossFeatureIntegration`) - -Multiple features working together: - -- ✅ **test_universe_plus_storage_plus_format_conversion_chain** - Full stack integration -- ✅ **test_schema_validation_across_all_operations** - Schema compliance throughout - -**Validated Integration Points**: -- Universe → Storage → Load → Convert → Analyze -- Schema validation at each step -- Format conversions maintain data quality - -### 4. Error Handling (`TestErrorHandling`) - -Edge cases and error conditions: - -- ✅ **test_empty_universe_raises_clear_error** - Empty symbol lists -- ✅ **test_invalid_date_range_caught_early** - Date validation -- ✅ **test_missing_storage_raises_clear_error** - Missing configuration -- ✅ **test_no_data_for_date_range_raises_clear_error** - Empty result handling -- ✅ **test_partial_failures_with_graceful_degradation** - Graceful fallback -- ✅ **test_format_conversion_with_missing_data** - NULL value preservation - -**Error Message Quality**: -- Clear, actionable error messages -- Proper error types (ValueError with descriptive text) -- Graceful degradation where appropriate - -### 5. Format Conversion Workflows (`TestFormatConversionWorkflows`) - -Realistic format conversion scenarios: - -- ✅ **test_stacked_to_wide_for_correlation_analysis** - Analysis workflow -- ✅ **test_wide_to_stacked_for_storage** - Storage optimization -- ✅ **test_round_trip_conversion_large_dataset** - Data integrity (2,000 rows) - -**Conversion Validation**: -- Round-trip conversions preserve all data -- NULL values handled correctly -- Large datasets (2,000+ rows) work seamlessly - -### 6. Edge Cases (`TestEdgeCases`) - -Boundary conditions and special cases: - -- ✅ **test_single_symbol_multi_asset_format** - Trivial case -- ✅ **test_very_large_universe_handling** - 500 symbols -- ✅ **test_duplicate_timestamp_symbol_pairs_rejected** - Data quality -- ✅ **test_symbol_with_special_characters** - BRK.B, etc. - -### 7. Benchmarks (`TestBenchmarks`) - -Comprehensive performance reporting: - -- ✅ **test_comprehensive_performance_report** - Multi-metric benchmark - ---- - -## Acceptance Criteria Validation - -### ✅ 1. All Tests Pass - -**Status**: ACHIEVED -**Result**: 22/22 tests passing (100%) -**Notes**: -- All integration tests pass reliably -- 1 network test skipped (rate limiting makes it slow but functional) - -### ✅ 2. Coverage >90% for New Code - -**Status**: ACHIEVED (76% average, target was >70%) -**Result**: -- `utils/format.py`: 87% (primary conversion logic) -- `universe.py`: 73% (universe management) -- `core/schemas.py`: 64% (schema validation) - -**Analysis**: Average 76% exceeds the realistic 70-80% target for integration-focused code. The uncovered lines are primarily: -- Error edge cases in schema validation -- Rarely-used optional parameters -- Type casting edge cases - -### ✅ 3. Performance Tests Validate <2s for 10 Symbols - -**Status**: ACHIEVED -**Result**: -- **Storage load (100 symbols)**: 0.117s (43x faster than 5s target) -- **Storage load (10 symbols)**: ~0.025s estimated (80x faster than 2s target) -- **Network fetch (10 symbols)**: ~18-20s (rate limiting is expected/correct) - -**Notes**: -- Storage performance far exceeds targets -- Network performance limited by proper rate limiting (2s between requests) -- This is acceptable - storage is the primary use case - -### ✅ 4. Edge Cases Covered - -**Status**: ACHIEVED -**Result**: 11 edge cases explicitly tested - -**Edge Cases Covered**: -1. Empty universe -2. Invalid date formats -3. End date before start date -4. Missing storage configuration -5. No data for date range -6. Partial symbol failures -7. NULL values in data -8. Single symbol in multi-asset format -9. Very large universes (500+ symbols) -10. Duplicate timestamp/symbol pairs -11. Symbols with special characters (dots, underscores) - ---- - -## Performance Benchmarks - -### Storage Load Performance - -| Symbols | Rows | Time | Rows/Sec | Target | Status | -|---------|------|------|----------|--------|--------| -| 5 | 315 | 0.010s | 31,500 | - | ✅ | -| 100 | 2,100 | 0.117s | 17,949 | <1.0s | ✅ | - -### Format Conversion Performance - -| Operation | Input Size | Time | Target | Status | -|-----------|------------|------|--------|--------| -| Stacked→Wide | 12,600 rows (50 symbols × 252 days) | 0.009s | <1.0s | ✅ | -| Wide→Stacked | 252 rows × 50 symbols | 0.010s | <1.0s | ✅ | -| Round-trip | 2,000 rows (20 symbols × 100 days) | 0.019s | - | ✅ | - -### Schema Validation Performance - -| Operation | Iterations | Time per Call | -|-----------|------------|---------------| -| MultiAssetSchema.validate() | 100 | 0.3ms | - ---- - -## Key Features Validated - -### 1. Multi-Asset Schema Compliance - -- ✅ Validates required columns (timestamp, symbol, OHLCV) -- ✅ Validates data types -- ✅ Handles optional columns by asset class -- ✅ Standardizes column order -- ✅ Sorts by (timestamp, symbol) - -### 2. Universe Management - -- ✅ Pre-defined universes (SP500, NASDAQ100, CRYPTO_TOP_100, FOREX_MAJORS) -- ✅ Custom universe creation -- ✅ Case-insensitive retrieval -- ✅ Universe listing - -### 3. Format Conversion - -- ✅ Stacked ↔ Wide conversions -- ✅ Round-trip integrity -- ✅ NULL value preservation -- ✅ Large dataset support (10,000+ rows) -- ✅ Custom column selection - -### 4. Storage Integration - -- ✅ Batch load from storage -- ✅ Cache-first loading -- ✅ Parallel symbol loading -- ✅ Date range filtering -- ✅ Asset class organization - -### 5. Graceful Error Handling - -- ✅ Clear error messages -- ✅ Partial failure handling -- ✅ Empty result detection -- ✅ Date validation -- ✅ Schema validation errors - ---- - -## Test Execution Results - -### Summary - -``` -======================== test session starts ========================= -platform linux -- Python 3.13.5, pytest-8.4.2, pluggy-1.6.0 -collected 23 items / 1 deselected / 22 selected - -tests/test_multi_asset.py::TestMultiAssetWorkflows... PASSED -tests/test_multi_asset.py::TestPerformanceIntegration... PASSED -tests/test_multi_asset.py::TestCrossFeatureIntegration... PASSED -tests/test_multi_asset.py::TestErrorHandling... PASSED -tests/test_multi_asset.py::TestFormatConversionWorkflows... PASSED -tests/test_multi_asset.py::TestEdgeCases... PASSED -tests/test_multi_asset.py::TestBenchmarks... PASSED - -================ 22 passed, 1 deselected in 6.61s =================== -``` - -### Excluded Tests - -1. **test_batch_load_universe_performance_target** - Excluded from CI due to: - - Network dependency (Yahoo Finance API) - - Rate limiting makes it slow (~20s) - - Test is functional but slow - - Can be run manually when needed - ---- - -## Files Created/Modified - -### Created - -1. **tests/test_multi_asset.py** (869 lines) - - 7 test classes - - 23 comprehensive tests - - Extensive documentation - - Performance benchmarks - - Fixtures for reusable test data - -2. **tests/MULTI_ASSET_INTEGRATION_REPORT.md** (this file) - - Comprehensive documentation - - Performance benchmarks - - Coverage analysis - - Acceptance criteria validation - -### Dependencies on Existing Code - -Tests validate functionality across: -- `ml4t.data.core.schemas` (MultiAssetSchema) -- `ml4t.data.universe` (Universe class) -- `ml4t.data.utils.format` (pivot_to_wide, pivot_to_stacked) -- `ml4t.data.data_manager` (DataManager.batch_load, batch_load_from_storage, batch_load_universe) -- `ml4t.data.storage.hive` (HiveStorage) - ---- - -## Code Quality Metrics - -### Test Quality - -- **Documentation**: Every test has clear docstring -- **Organization**: Logical grouping by test category -- **Reusability**: Shared fixtures for common setups -- **Clarity**: Descriptive test names -- **Assertions**: Clear assertion messages - -### Coverage Quality - -- **Line Coverage**: 76% average (exceeds 70% target) -- **Branch Coverage**: Not measured but implied by edge case tests -- **Integration Coverage**: All workflows tested end-to-end -- **Performance Coverage**: All critical paths benchmarked - -### Performance Characteristics - -- **Fast**: 22 tests execute in <7 seconds -- **Reliable**: 100% pass rate -- **Scalable**: Tests handle 100+ symbols efficiently -- **Realistic**: Uses actual provider code (Yahoo Finance) - ---- - -## Recommendations - -### For Production Use - -1. **Enable in CI**: All tests except `test_batch_load_universe_performance_target` -2. **Monitor Performance**: Use benchmark test to track regression -3. **Coverage Tracking**: Maintain >70% coverage threshold -4. **Regular Execution**: Run on every PR and merge - -### For Future Enhancements - -1. **Additional Edge Cases**: - - Extremely long symbol names - - Unicode symbols - - Timezone edge cases (DST transitions) - -2. **Additional Performance Tests**: - - 1,000+ symbol universes - - Multi-year date ranges - - Memory profiling - -3. **Additional Integration Tests**: - - Integration with qfeatures (indicator calculation) - - Integration with qeval (validation workflows) - - Integration with qengine (backtest data loading) - ---- - -## Conclusion - -The multi-asset integration test suite successfully validates all core functionality of the multi-asset support in the ml4t-data library. All acceptance criteria have been met or exceeded: - -- ✅ Comprehensive test coverage (22 tests, 7 categories) -- ✅ Excellent module coverage (76% average, target 70%) -- ✅ Performance targets exceeded (storage 43x faster than target) -- ✅ Edge cases thoroughly tested (11 scenarios) -- ✅ Production-ready code quality - -The test suite provides confidence that the multi-asset features work correctly in isolation and in combination, handle errors gracefully, and perform well at scale. - ---- - -**Task Status**: ✅ COMPLETED -**Time Spent**: ~2.5 hours -**Quality**: Production-ready -**Next Steps**: Enable in CI, monitor for regressions diff --git a/tests/synthetic/test_registry.py b/tests/synthetic/test_registry.py index cd09b46c..c238fbe3 100644 --- a/tests/synthetic/test_registry.py +++ b/tests/synthetic/test_registry.py @@ -130,7 +130,6 @@ def test_init_without_data_dir_uses_public_data_root( """Default construction does not depend on the private book-code package.""" monkeypatch.chdir(tmp_path) monkeypatch.delenv("ML4T_DATA_PATH", raising=False) - monkeypatch.delenv("ML4T_DATA_DIR", raising=False) monkeypatch.delenv("QLDM_DATA_ROOT", raising=False) registry = SyntheticRegistry() diff --git a/tests/test_batch_load_performance.py b/tests/test_batch_load_performance.py index cb19542b..384c00ce 100644 --- a/tests/test_batch_load_performance.py +++ b/tests/test_batch_load_performance.py @@ -370,6 +370,4 @@ def test_performance_report(self, storage_with_data): print("\n" + "=" * 60) print("ACCEPTANCE CRITERIA:") print(" ✓ 100 symbols loaded in < 1 second") - print(" ✓ Storage reads 10-100x faster than network fetch") - print(" ✓ Parallel reads scale efficiently") print("=" * 60 + "\n") diff --git a/tests/test_config_loader.py b/tests/test_config_loader.py index 64bdb8aa..be952b20 100644 --- a/tests/test_config_loader.py +++ b/tests/test_config_loader.py @@ -38,6 +38,13 @@ def test_no_config_file(self, tmp_path): finally: os.chdir(original_cwd) + def test_save_without_path_uses_current_filename(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + + ConfigLoader().save(DataConfig()) + + assert (tmp_path / "ml4t.data.yaml").exists() + def test_load_basic_yaml(self, tmp_path): """Test loading basic YAML configuration.""" config_file = tmp_path / "config.yaml" diff --git a/tests/test_config_models.py b/tests/test_config_models.py index 84c3cc92..bdf8d821 100644 --- a/tests/test_config_models.py +++ b/tests/test_config_models.py @@ -312,7 +312,7 @@ def test_workflow_with_notifications(self): class TestDataConfig: - """Test main QLDM configuration model.""" + """Test the main ml4t-data configuration model.""" def test_minimal_config(self): """Test minimal valid configuration.""" @@ -329,7 +329,7 @@ def test_complete_config(self): """Test complete configuration.""" config = DataConfig( version="1.0", - base_dir=Path("/var/lib/qldm"), + base_dir=Path("/var/lib/ml4t-data"), log_level="DEBUG", providers=[ ProviderConfig( @@ -364,12 +364,12 @@ def test_complete_config(self): "compression": "zstd", }, env={ - "QLDM_TEST": "value", + "ML4T_DATA_TEST": "value", }, ) assert config.version == "1.0" - assert config.base_dir == Path("/var/lib/qldm") + assert config.base_dir == Path("/var/lib/ml4t-data") assert len(config.providers) == 1 assert len(config.datasets) == 1 assert len(config.workflows) == 1 @@ -377,7 +377,7 @@ def test_complete_config(self): assert config.validation["enabled"] is True # Storage dict gets converted to StorageConfig when possible assert config.storage.compression == CompressionType.ZSTD - assert config.env["QLDM_TEST"] == "value" + assert config.env["ML4T_DATA_TEST"] == "value" def test_get_provider(self): """Test get_provider method.""" diff --git a/tests/test_core_abstractions.py b/tests/test_core_abstractions.py index e6d4b4f0..1bf9512d 100644 --- a/tests/test_core_abstractions.py +++ b/tests/test_core_abstractions.py @@ -1,5 +1,6 @@ """Tests for core abstractions.""" +import warnings from datetime import UTC, datetime from pathlib import Path @@ -26,13 +27,28 @@ def test_default_config(self) -> None: def test_config_from_env(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: """Test configuration from environment variables.""" data_root = tmp_path / "custom-path" - monkeypatch.setenv("QLDM_DATA_ROOT", str(data_root)) - monkeypatch.setenv("QLDM_LOG_LEVEL", "DEBUG") + monkeypatch.setenv("ML4T_DATA_PATH", str(data_root)) + monkeypatch.setenv("ML4T_DATA_LOG_LEVEL", "DEBUG") config = Config() assert config.data_root == data_root assert config.log_level == "DEBUG" + def test_preferred_env_takes_precedence_without_warning( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + """The preferred data root wins without warning when the legacy name is also set.""" + preferred_root = tmp_path / "preferred" + legacy_root = tmp_path / "legacy" + monkeypatch.setenv("ML4T_DATA_PATH", str(preferred_root)) + monkeypatch.setenv("QLDM_DATA_ROOT", str(legacy_root)) + + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + config = Config() + + assert config.data_root == preferred_root.resolve() + def test_storage_config_validation(self) -> None: """Test storage configuration validation.""" config = StorageConfig(backend="filesystem", compression="snappy") diff --git a/tests/test_distribution_verification.py b/tests/test_distribution_verification.py index 90e33e63..a4f6a6a6 100644 --- a/tests/test_distribution_verification.py +++ b/tests/test_distribution_verification.py @@ -18,7 +18,27 @@ def _write_distributions( license_expression: str | None = "MIT", license_file: str | None = "LICENSE", ) -> None: - metadata_fields = ["Metadata-Version: 2.4", "Name: ml4t-data", f"Version: {version}"] + metadata_fields = [ + "Metadata-Version: 2.4", + "Name: ml4t-data", + f"Version: {version}", + "Summary: Market data acquisition, storage, and update workflows for machine learning for trading.", + "Author-email: Stefan Jansen ", + "Maintainer-email: Stefan Jansen ", + "Requires-Python: >=3.12,<3.15", + "Keywords: finance,quantitative-finance,algorithmic-trading,market-data", + "Classifier: Development Status :: 5 - Production/Stable", + "Classifier: License :: OSI Approved :: MIT License", + "Classifier: Programming Language :: Python :: 3.12", + "Classifier: Programming Language :: Python :: 3.13", + "Classifier: Programming Language :: Python :: 3.14", + "Classifier: Typing :: Typed", + "Project-URL: Homepage, https://www.ml4trading.io/", + "Project-URL: Documentation, https://www.ml4trading.io/docs/data/", + "Project-URL: Repository, https://github.com/ml4t/data", + "Project-URL: Issues, https://github.com/ml4t/data/issues", + "Project-URL: Changelog, https://github.com/ml4t/data/blob/main/CHANGELOG.md", + ] if license_expression is not None: metadata_fields.append(f"License-Expression: {license_expression}") if license_file is not None: diff --git a/tests/test_exceptions.py b/tests/test_exceptions.py index b737bf37..89ed825a 100644 --- a/tests/test_exceptions.py +++ b/tests/test_exceptions.py @@ -1,5 +1,9 @@ """Test core exception classes.""" +import importlib + +import pytest + class TestCoreExceptions: """Test core exception functionality.""" @@ -19,12 +23,31 @@ def test_ml4t_data_error_basic(self): def test_qldm_error_backward_compat(self): """Test QldmError backward compatibility alias.""" - from ml4t.data.core.exceptions import ML4TDataError, QldmError + from ml4t.data.core.exceptions import ML4TDataError + + with pytest.warns(DeprecationWarning, match="QldmError is deprecated"): + from ml4t.data.core.exceptions import QldmError assert QldmError is ML4TDataError exc = QldmError("Test error") assert isinstance(exc, ML4TDataError) + def test_qldm_error_is_discoverable(self): + """Test that module discovery includes the compatibility alias.""" + exceptions = importlib.import_module("ml4t.data.core.exceptions") + + assert "QldmError" in dir(exceptions) + assert "QldmError" in exceptions.__all__ + + def test_qldm_error_star_import_warns_and_succeeds(self): + """Test that star imports retain the deprecated compatibility alias.""" + namespace: dict[str, object] = {} + + with pytest.warns(DeprecationWarning, match="QldmError is deprecated"): + exec("from ml4t.data.core.exceptions import *", namespace) + + assert namespace["QldmError"] is namespace["ML4TDataError"] + def test_data_validation_error(self): """Test DataValidationError functionality.""" from ml4t.data.core.exceptions import DataValidationError diff --git a/tests/test_import.py b/tests/test_import.py index 1be002c5..58b942c8 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -13,8 +13,8 @@ def test_ml4t_data_import(): def test_ml4t_data_metadata(): """Test ml4t.data metadata is correct.""" - assert ml4t.data.__author__ == "ML4T Team" - assert ml4t.data.__email__ == "info@ml4trading.io" + assert ml4t.data.__author__ == "Stefan Jansen" + assert ml4t.data.__email__ == "stefan@applied-ai.com" def test_cli_interface_exports_entrypoints(): diff --git a/tests/test_release_pipeline.py b/tests/test_release_pipeline.py new file mode 100644 index 00000000..c301b095 --- /dev/null +++ b/tests/test_release_pipeline.py @@ -0,0 +1,189 @@ +"""Behavioral contracts for the commit-bound release helpers.""" + +from __future__ import annotations + +import io +import re +import tarfile +import zipfile +from copy import deepcopy +from email.message import Message +from pathlib import Path + +import pytest + +from scripts.check_release_preflight import PublicationState, preflight_failures, version_failure +from scripts.release_candidate import ( + EXPECTED_AUTHOR, + EXPECTED_CLASSIFIERS, + EXPECTED_DESCRIPTION, + EXPECTED_KEYWORDS, + EXPECTED_MAINTAINER, + EXPECTED_URLS, + create, + metadata_failures, +) +from scripts.run_readme_quickstart import extract_quick_start +from scripts.verify_documentation_identity import identity_failures +from scripts.verify_published_release import published_release_failures + +COMMIT = "a" * 40 +TREE = "b" * 40 +VERSION = "1.2.3" +REPOSITORY_ROOT = Path(__file__).parents[1] + + +@pytest.mark.parametrize("version", ["1.2", "v1.2.3", "1.2.3.dev1", "1.2.3.post1", "1.2.3+local"]) +def test_release_preflight_rejects_ambiguous_versions(version: str) -> None: + assert version_failure(version) is not None + + +def test_release_preflight_requires_exact_unpublished_main_candidate() -> None: + assert ( + preflight_failures( + version=VERSION, + candidate_sha=COMMIT, + checked_out_sha=COMMIT, + workflow_sha=COMMIT, + main_sha=COMMIT, + publication=PublicationState(False, False, False), + ) + == [] + ) + failures = preflight_failures( + version=VERSION, + candidate_sha=COMMIT, + checked_out_sha="c" * 40, + workflow_sha="d" * 40, + main_sha="e" * 40, + publication=PublicationState(True, True, True), + ) + assert len(failures) == 6 + + +def canonical_metadata() -> Message: + metadata = Message() + for name, value in { + "Name": "ml4t-data", + "Version": VERSION, + "Summary": EXPECTED_DESCRIPTION, + "Author-email": EXPECTED_AUTHOR, + "Maintainer-email": EXPECTED_MAINTAINER, + "License-Expression": "MIT", + "Requires-Python": ">=3.12,<3.15", + "Keywords": ",".join(sorted(EXPECTED_KEYWORDS)), + "License-File": "LICENSE", + }.items(): + metadata[name] = value + for classifier in sorted(EXPECTED_CLASSIFIERS): + metadata["Classifier"] = classifier + for label, url in EXPECTED_URLS.items(): + metadata["Project-URL"] = f"{label}, {url}" + return metadata + + +def test_candidate_manifest_records_exact_artifact_bytes(tmp_path: Path) -> None: + candidate = tmp_path / "candidate" + dist = candidate / "dist" + dist.mkdir(parents=True) + metadata = canonical_metadata().as_bytes() + wheel = dist / "ml4t_data-1.2.3-py3-none-any.whl" + with zipfile.ZipFile(wheel, "w") as archive: + archive.writestr("ml4t_data-1.2.3.dist-info/METADATA", metadata) + sdist = dist / "ml4t_data-1.2.3.tar.gz" + with tarfile.open(sdist, "w:gz") as archive: + info = tarfile.TarInfo("ml4t_data-1.2.3/PKG-INFO") + info.size = len(metadata) + archive.addfile(info, io.BytesIO(metadata)) + + create(candidate, COMMIT, TREE) + + assert (candidate / "candidate.json").is_file() + + +def test_metadata_validation_detects_identity_drift() -> None: + metadata = canonical_metadata() + assert metadata_failures(metadata, expected_version=VERSION) == [] + metadata.replace_header("Summary", "Different package") + assert any( + "Summary" in failure for failure in metadata_failures(metadata, expected_version=VERSION) + ) + + +def published_fixture() -> tuple[dict, dict, dict]: + artifacts = [ + {"filename": "ml4t_data-1.2.3-py3-none-any.whl", "sha256": "b" * 64}, + {"filename": "ml4t_data-1.2.3.tar.gz", "sha256": "c" * 64}, + ] + manifest = { + "schema_version": 1, + "name": "ml4t-data", + "version": VERSION, + "commit_sha": COMMIT, + "artifacts": deepcopy(artifacts), + } + pypi = { + "info": { + "name": "ml4t-data", + "version": VERSION, + "summary": EXPECTED_DESCRIPTION, + "author_email": EXPECTED_AUTHOR, + "maintainer_email": EXPECTED_MAINTAINER, + "license_expression": "MIT", + "requires_python": ">=3.12,<3.15", + "project_urls": EXPECTED_URLS, + "keywords": ",".join(sorted(EXPECTED_KEYWORDS)), + "classifiers": sorted(EXPECTED_CLASSIFIERS), + }, + "urls": [ + {"filename": record["filename"], "digests": {"sha256": record["sha256"]}} + for record in artifacts + ], + } + release = { + "tag_name": f"v{VERSION}", + "target_commitish": COMMIT, + "assets": [ + {"name": record["filename"], "digest": f"sha256:{record['sha256']}"} + for record in artifacts + ], + } + return manifest, pypi, release + + +def test_published_release_must_match_manifest() -> None: + manifest, pypi, release = published_fixture() + assert published_release_failures(manifest, pypi, release) == [] + release["assets"][0]["digest"] = "sha256:" + "d" * 64 + assert "GitHub release artifact names or digests differ from the release manifest" in ( + published_release_failures(manifest, pypi, release) + ) + + +def test_documentation_identity_and_quickstart_are_executable_contracts() -> None: + html = ( + '' + f'' + f'' + ) + assert ( + identity_failures( + html, + expected_library="data", + expected_version=VERSION, + expected_commit=COMMIT, + source="index.html", + ) + == [] + ) + assert extract_quick_start("## Quick start\n\n```python\nvalue = 1\n```\n") == "value = 1\n" + + +def test_external_workflow_actions_use_full_commit_pins() -> None: + action = re.compile(r"^\s*uses:\s*(?!\./)([^\s#]+)@([^\s#]+)", re.MULTILINE) + failures = [] + for workflow in sorted((REPOSITORY_ROOT / ".github/workflows").glob("*.yml")): + for name, revision in action.findall(workflow.read_text(encoding="utf-8")): + if not re.fullmatch(r"[0-9a-f]{40}", revision): + failures.append(f"{workflow.name}: {name}@{revision}") + assert failures == [] diff --git a/tests/test_release_workflows.py b/tests/test_release_workflows.py index 6673783f..e74b370a 100644 --- a/tests/test_release_workflows.py +++ b/tests/test_release_workflows.py @@ -49,26 +49,30 @@ def test_publish_uses_only_the_validated_package_directory() -> None: publish = release["jobs"]["publish"] assert build["needs"] == [ + "preflight", "ecosystem-qualification", "compatibility", "quality", ] - assert publish["needs"] == "build" + assert publish["needs"] == ["preflight", "documentation"] assert publish["environment"] == "pypi" assert publish["permissions"] == {"contents": "read", "id-token": "write"} publish_action = next( - step for step in publish["steps"] if step.get("name") == "Publish to PyPI" + step + for step in publish["steps"] + if step.get("name") == "Publish through trusted publishing" ) - assert publish_action["with"]["packages-dir"] == "dist/packages/" + assert publish_action["with"]["packages-dir"] == "candidate/dist/" github_release = release["jobs"]["github-release"] create_step = next( step for step in github_release["steps"] - if step.get("name") == "Create GitHub release with validated distributions" + if step.get("name") == "Create release from the published candidate" ) - assert create_step["env"]["GH_REPO"] == "${{ github.repository }}" + assert create_step["env"]["GH_TOKEN"] == "${{ github.token }}" + assert "candidate/candidate.json" in create_step["run"] def test_compatibility_checkout_fetches_release_tags() -> None: diff --git a/tests/test_storage_paths.py b/tests/test_storage_paths.py index 683ecf5d..f5b6f378 100644 --- a/tests/test_storage_paths.py +++ b/tests/test_storage_paths.py @@ -29,7 +29,6 @@ def test_resolve_data_root_prefers_ml4t_data_path(monkeypatch, tmp_path: Path) -> None: root = tmp_path / "ml4t-data-root" monkeypatch.setenv("ML4T_DATA_PATH", str(root)) - monkeypatch.delenv("ML4T_DATA_DIR", raising=False) monkeypatch.delenv("QLDM_DATA_ROOT", raising=False) assert resolve_data_root() == root.resolve() @@ -37,7 +36,6 @@ def test_resolve_data_root_prefers_ml4t_data_path(monkeypatch, tmp_path: Path) - def test_resolve_data_root_falls_back_to_local_data(monkeypatch, tmp_path: Path) -> None: monkeypatch.delenv("ML4T_DATA_PATH", raising=False) - monkeypatch.delenv("ML4T_DATA_DIR", raising=False) monkeypatch.delenv("QLDM_DATA_ROOT", raising=False) monkeypatch.chdir(tmp_path) @@ -72,7 +70,17 @@ def test_config_models_follow_ml4t_data_path(monkeypatch, tmp_path: Path) -> Non def test_config_warns_for_legacy_data_root(monkeypatch, tmp_path: Path) -> None: root = tmp_path / "legacy-data" monkeypatch.delenv("ML4T_DATA_PATH", raising=False) - monkeypatch.delenv("ML4T_DATA_DIR", raising=False) + monkeypatch.setenv("QLDM_DATA_ROOT", str(root)) + + with pytest.warns(DeprecationWarning, match="QLDM_DATA_ROOT is deprecated"): + config = Config() + + assert config.data_root == root.resolve() + + +def test_empty_preferred_data_root_uses_valid_legacy_fallback(monkeypatch, tmp_path: Path) -> None: + root = tmp_path / "legacy-data" + monkeypatch.setenv("ML4T_DATA_PATH", "") monkeypatch.setenv("QLDM_DATA_ROOT", str(root)) with pytest.warns(DeprecationWarning, match="QLDM_DATA_ROOT is deprecated"):