diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml new file mode 100644 index 0000000..b5e8fc6 --- /dev/null +++ b/.github/workflows/nightly.yaml @@ -0,0 +1,58 @@ +name: nightly + +on: + schedule: + - cron: '0 3 * * *' # Every night at 03:00 UTC + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + api-drift-check: + name: Ensembl Live API Drift Check + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python 3.14 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.14' + + - name: Set Poetry Version + id: set-poetry-version + run: echo "POETRY_VERSION=$(cat .poetry-version)" >> $GITHUB_OUTPUT + + - name: Install Poetry + id: install-poetry-version + uses: snok/install-poetry@a783c322200f0519c7926aa6faa857c4e23e9263 # v1.4.2 + with: + version: ${{ steps.set-poetry-version.outputs.POETRY_VERSION }} + virtualenvs-create: true + virtualenvs-in-project: true + virtualenvs-path: .venv + installer-parallel: true + + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .venv + key: venv-${{ runner.os }}-3.14-nightly-${{ hashFiles('**/poetry.lock') }} + + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: poetry install --no-interaction --no-root + + - name: Install project + run: poetry install --no-interaction + + - name: Run all tests against live Ensembl API + run: poetry run pytest -v tests/ diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index d046c82..fafe6ea 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -3,27 +3,87 @@ name: pull_request on: pull_request: paths: - - 'pyensemblrest/**.py' - - 'tests/**.py' + - 'pyensemblrest/**' + - 'tests/**' - 'pyproject.toml' - 'poetry.lock' - '.github/workflows/pull_request.yaml' +# Automatically cancel redundant in-progress workflow runs for new commits on the same PR branch +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + jobs: - test: - name: Run Tests + lint: + name: Code Quality & Type Check runs-on: ubuntu-latest - environment: dev + timeout-minutes: 10 + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python 3.14 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.14' + + - name: Set Poetry Version + id: set-poetry-version + run: echo "POETRY_VERSION=$(cat .poetry-version)" >> $GITHUB_OUTPUT + + - name: Install Poetry + id: install-poetry-version + uses: snok/install-poetry@a783c322200f0519c7926aa6faa857c4e23e9263 # v1.4.2 + with: + version: ${{ steps.set-poetry-version.outputs.POETRY_VERSION }} + virtualenvs-create: true + virtualenvs-in-project: true + virtualenvs-path: .venv + installer-parallel: true + + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .venv + key: venv-${{ runner.os }}-3.14-lint-${{ hashFiles('**/poetry.lock') }} + + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: poetry install --no-interaction --no-root + + - name: Install project + run: poetry install --no-interaction + + - name: Run Ruff Linter and Formatter Check + run: | + poetry run ruff check . + poetry run ruff format --check . + + - name: Run Mypy Type Check + run: poetry run mypy . --no-incremental + + - name: Run Poetry Config Check + run: poetry check + + unit-test: + name: Unit Tests (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + timeout-minutes: 10 strategy: + fail-fast: false matrix: python-version: ['3.10', '3.11', '3.12', '3.13', '3.14'] - timeout-minutes: 60 steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} @@ -33,7 +93,7 @@ jobs: - name: Install Poetry id: install-poetry-version - uses: snok/install-poetry@v1 + uses: snok/install-poetry@a783c322200f0519c7926aa6faa857c4e23e9263 # v1.4.2 with: version: ${{ steps.set-poetry-version.outputs.POETRY_VERSION }} virtualenvs-create: true @@ -43,7 +103,7 @@ jobs: - name: Load cached venv id: cached-poetry-dependencies - uses: actions/cache@v5 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} @@ -55,14 +115,57 @@ jobs: - name: Install project run: poetry install --no-interaction - - name: Run pre-commit hooks, excluding unit tests - run: SKIP=unit-test poetry run pre-commit run --all-files - - - name: Run unit tests with pytest and code coverage with pytest-cov - run: poetry run pytest -v --cov=pyensemblrest --cov-report lcov:./tests/lcov.info + - name: Run offline unit tests with coverage + run: poetry run pytest -v -m "not live" --cov=pyensemblrest --cov-report lcov:./tests/lcov.info tests/ - name: Upload Coverage Results - uses: coverallsapp/github-action@master + if: matrix.python-version == '3.14' + uses: coverallsapp/github-action@8d6379e14d29928660c4ba802d8e85393440b329 # v2.3.8 with: github-token: ${{ secrets.GITHUB_TOKEN }} path-to-lcov: ./tests/lcov.info + + live-test: + name: Live Ensembl API Smoke Test + needs: [unit-test] + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python 3.14 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.14' + + - name: Set Poetry Version + id: set-poetry-version + run: echo "POETRY_VERSION=$(cat .poetry-version)" >> $GITHUB_OUTPUT + + - name: Install Poetry + id: install-poetry-version + uses: snok/install-poetry@a783c322200f0519c7926aa6faa857c4e23e9263 # v1.4.2 + with: + version: ${{ steps.set-poetry-version.outputs.POETRY_VERSION }} + virtualenvs-create: true + virtualenvs-in-project: true + virtualenvs-path: .venv + installer-parallel: true + + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .venv + key: venv-${{ runner.os }}-3.14-live-${{ hashFiles('**/poetry.lock') }} + + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: poetry install --no-interaction --no-root + + - name: Install project + run: poetry install --no-interaction + + - name: Run Live Ensembl REST API tests (single runner to respect rate limits) + run: poetry run pytest -v -m "live" tests/ diff --git a/.github/workflows/push_tag.yaml b/.github/workflows/push_tag.yaml index 695e0f6..cb45d32 100644 --- a/.github/workflows/push_tag.yaml +++ b/.github/workflows/push_tag.yaml @@ -5,6 +5,10 @@ on: tags: - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + permissions: contents: read @@ -12,20 +16,21 @@ jobs: release: name: Release to GitHub runs-on: ubuntu-latest + timeout-minutes: 15 environment: dev permissions: contents: write discussions: write steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-tags: true - ref: ${{ github.ref_name}} + ref: ${{ github.ref_name }} - name: Set up Python id: setup-python - uses: actions/setup-python@v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ">=3.10 <3.15" @@ -35,7 +40,7 @@ jobs: - name: Install Poetry id: install-poetry-version - uses: snok/install-poetry@v1 + uses: snok/install-poetry@a783c322200f0519c7926aa6faa857c4e23e9263 # v1.4.2 with: version: ${{ steps.set-poetry-version.outputs.POETRY_VERSION }} virtualenvs-create: true @@ -49,7 +54,7 @@ jobs: - name: Load cached venv id: cached-poetry-dependencies - uses: actions/cache@v5 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} @@ -65,38 +70,40 @@ jobs: run: poetry build - name: Upload artifacts - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: release-dists path: dist/ - name: Create release - uses: softprops/action-gh-release@v3 + uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 if: startsWith(github.ref, 'refs/tags/') with: files: dist/* generate_release_notes: true draft: false prerelease: false + publish: needs: - release - permissions: - id-token: write - name: Publish to PyPI runs-on: ubuntu-latest + timeout-minutes: 15 environment: name: dev + permissions: + contents: read + id-token: write steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set Poetry Version id: set-poetry-version run: echo "POETRY_VERSION=$(cat .poetry-version)" >> $GITHUB_OUTPUT - name: Install Poetry - uses: snok/install-poetry@v1 + uses: snok/install-poetry@a783c322200f0519c7926aa6faa857c4e23e9263 # v1.4.2 with: version: ${{ steps.set-poetry-version.outputs.POETRY_VERSION }} @@ -104,7 +111,7 @@ jobs: run: poetry self add poetry-dynamic-versioning[plugin] - name: Retrieve release distributions - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: release-dists path: dist diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..34a4d95 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,131 @@ +# AGENTS.md + +This file provides guidance for AI agents when working with code in this repository. + +## Project Overview + +pyEnsemblRest is a Python client library for the Ensembl REST API. It provides a dynamic, Pythonic interface to all Ensembl REST endpoints without requiring explicit method definitions. The library handles rate limiting (15 requests/second), retry logic, and various response formats. + +## Development Commands + +### Setup +```bash +# Install dependencies +make install +# Or with Poetry directly +poetry install --sync +``` + +### Testing +```bash +# Run unit tests (excludes live API tests) +make unit-test +# Or +poetry run pytest -v -m "not live" + +# Run all tests with coverage (used in CI) +make ci-test +# Or +poetry run pytest -v --cov=pyensemblrest --cov-report lcov:./tests/lcov.info tests/ + +# Upload coverage to Coveralls +make coverage +``` + +### Code Quality +```bash +# Type checking +make type-check +# Or +poetry run mypy . --no-incremental + +# Linting +make lint +# Or +poetry run ruff check . --fix + +# Formatting +make format +# Or +poetry run ruff format . + +# Run all pre-commit hooks (except tests) +SKIP=unit-test poetry run pre-commit run --all-files +``` + +### Dependency Management +```bash +# Update lock file +make freeze +# Or +poetry lock +``` + +## Architecture + +### Dynamic Method Registration + +The library uses a dynamic method registration system rather than hard-coded methods. All REST API endpoints are defined in `pyensemblrest/ensembl_config.py` in the `ensembl_api_table` dictionary. Each entry specifies: +- `url`: URL template with `{{parameter}}` placeholders +- `method`: HTTP method (GET or POST) +- `content_type`: Default content type +- `post_parameters`: List of parameters passed in POST body (for POST methods) +- `doc`: Documentation string + +When `EnsemblRest` is instantiated, `__add_methods()` dynamically adds methods to the instance based on `ensembl_api_table`. This allows the library to support all Ensembl REST endpoints without explicit method definitions. + +### Request Flow + +1. User calls a dynamic method (e.g., `ensRest.getSequenceById(id='ENSG00000157764')`) +2. `call_api_func()` validates mandatory parameters from URL template +3. URL is constructed by replacing `{{parameter}}` placeholders +4. `__get_response()` handles rate limiting and executes the request +5. `parseResponse()` processes the response and handles retries if needed + +### Rate Limiting and Retry Logic + +The library implements automatic rate limiting (15 requests/second) in `__get_response()`: +- Tracks request count and timing +- Sleeps if rate limit would be exceeded +- Reads rate limit headers from Ensembl API responses + +Retry logic in `__retry_request()` handles: +- Ensembl known errors (defined in `ensembl_known_errors`) +- HTTP 500 errors (Ensembl sometimes returns 500 on valid requests) +- Timeouts +- Up to `max_attempts` (default 5) with exponential backoff + +### Exception Hierarchy + +- `EnsemblRestError`: Base exception for all REST API errors +- `EnsemblRestRateLimitError`: Raised on HTTP 429 (rate limit hit) +- `EnsemblRestServiceUnavailable`: Raised on connection errors + +### Key Files + +- `pyensemblrest/ensemblrest.py`: Main `EnsemblRest` class with dynamic method registration +- `pyensemblrest/ensembl_config.py`: API endpoint definitions, HTTP status codes, configuration +- `pyensemblrest/exceptions.py`: Custom exception classes +- `pyensemblrest/__init__.py`: Package exports and version metadata + +## Testing Notes + +The test suite uses pytest markers: +- `@pytest.mark.live`: Tests that call the live Ensembl REST API +- By default, `make unit-test` excludes live tests with `-m "not live"` +- CI runs all tests including live API calls + +## Version Management + +This project uses `poetry-dynamic-versioning` to automatically set the version from git tags. The version in `pyproject.toml` is set to `0.0.0` as a placeholder and is replaced at build time with the git tag version. + +## Type Checking + +mypy is configured with strict mode in `pyproject.toml`. The `tests/` directory and `examples.py` are excluded from type checking. + +## Code Style + +The project uses Ruff for both linting and formatting with: +- Line length: 88 characters +- Selected rules: E4, E7, E9, F, I (import sorting) +- Configured in `[tool.ruff]` section of `pyproject.toml` diff --git a/CLAUDE.md b/CLAUDE.md index aecbc58..83f00ce 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,131 +1,3 @@ # CLAUDE.md -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## Project Overview - -pyEnsemblRest is a Python client library for the Ensembl REST API. It provides a dynamic, Pythonic interface to all Ensembl REST endpoints without requiring explicit method definitions. The library handles rate limiting (15 requests/second), retry logic, and various response formats. - -## Development Commands - -### Setup -```bash -# Install dependencies -make install -# Or with Poetry directly -poetry install --sync -``` - -### Testing -```bash -# Run unit tests (excludes live API tests) -make unit-test -# Or -poetry run pytest -v -m "not live" - -# Run all tests with coverage (used in CI) -make ci-test -# Or -poetry run pytest -v --cov=pyensemblrest --cov-report lcov:./tests/lcov.info tests/ - -# Upload coverage to Coveralls -make coverage -``` - -### Code Quality -```bash -# Type checking -make type-check -# Or -poetry run mypy . --no-incremental - -# Linting -make lint -# Or -poetry run ruff check . --fix - -# Formatting -make format -# Or -poetry run ruff format . - -# Run all pre-commit hooks (except tests) -SKIP=unit-test poetry run pre-commit run --all-files -``` - -### Dependency Management -```bash -# Update lock file -make freeze -# Or -poetry lock -``` - -## Architecture - -### Dynamic Method Registration - -The library uses a dynamic method registration system rather than hard-coded methods. All REST API endpoints are defined in `pyensemblrest/ensembl_config.py` in the `ensembl_api_table` dictionary. Each entry specifies: -- `url`: URL template with `{{parameter}}` placeholders -- `method`: HTTP method (GET or POST) -- `content_type`: Default content type -- `post_parameters`: List of parameters passed in POST body (for POST methods) -- `doc`: Documentation string - -When `EnsemblRest` is instantiated, `__add_methods()` dynamically adds methods to the instance based on `ensembl_api_table`. This allows the library to support all Ensembl REST endpoints without explicit method definitions. - -### Request Flow - -1. User calls a dynamic method (e.g., `ensRest.getSequenceById(id='ENSG00000157764')`) -2. `call_api_func()` validates mandatory parameters from URL template -3. URL is constructed by replacing `{{parameter}}` placeholders -4. `__get_response()` handles rate limiting and executes the request -5. `parseResponse()` processes the response and handles retries if needed - -### Rate Limiting and Retry Logic - -The library implements automatic rate limiting (15 requests/second) in `__get_response()`: -- Tracks request count and timing -- Sleeps if rate limit would be exceeded -- Reads rate limit headers from Ensembl API responses - -Retry logic in `__retry_request()` handles: -- Ensembl known errors (defined in `ensembl_known_errors`) -- HTTP 500 errors (Ensembl sometimes returns 500 on valid requests) -- Timeouts -- Up to `max_attempts` (default 5) with exponential backoff - -### Exception Hierarchy - -- `EnsemblRestError`: Base exception for all REST API errors -- `EnsemblRestRateLimitError`: Raised on HTTP 429 (rate limit hit) -- `EnsemblRestServiceUnavailable`: Raised on connection errors - -### Key Files - -- `pyensemblrest/ensemblrest.py`: Main `EnsemblRest` class with dynamic method registration -- `pyensemblrest/ensembl_config.py`: API endpoint definitions, HTTP status codes, configuration -- `pyensemblrest/exceptions.py`: Custom exception classes -- `pyensemblrest/__init__.py`: Package exports and version metadata - -## Testing Notes - -The test suite uses pytest markers: -- `@pytest.mark.live`: Tests that call the live Ensembl REST API -- By default, `make unit-test` excludes live tests with `-m "not live"` -- CI runs all tests including live API calls - -## Version Management - -This project uses `poetry-dynamic-versioning` to automatically set the version from git tags. The version in `pyproject.toml` is set to `0.0.0` as a placeholder and is replaced at build time with the git tag version. - -## Type Checking - -mypy is configured with strict mode in `pyproject.toml`. The `tests/` directory and `examples.py` are excluded from type checking. - -## Code Style - -The project uses Ruff for both linting and formatting with: -- Line length: 88 characters -- Selected rules: E4, E7, E9, F, I (import sorting) -- Configured in `[tool.ruff]` section of `pyproject.toml` +See [AGENTS.md](AGENTS.md) for development guidelines, commands, and repository architecture. diff --git a/README.md b/README.md index 0389356..a658d20 100644 --- a/README.md +++ b/README.md @@ -527,11 +527,11 @@ print( variantType="DUP", ) ) -print(ensRest.getGA4GHFeaturesById(id="ENST00000408937.7")) +print(ensRest.getGA4GHFeaturesById(id="ENST00000408937.8")) ensRest.timeout = 180 print( ensRest.searchGA4GHFeatures( - parentId="ENST00000408937.7", + parentId="ENST00000408937.8", featureSetId="", featureTypes=["cds"], end=220023, diff --git a/examples.py b/examples.py index aba8ae8..5c514d0 100644 --- a/examples.py +++ b/examples.py @@ -250,11 +250,11 @@ variantType="DUP", ) ) -print(ensRest.getGA4GHFeaturesById(id="ENST00000408937.7")) +print(ensRest.getGA4GHFeaturesById(id="ENST00000408937.8")) ensRest.timeout = 180 print( ensRest.searchGA4GHFeatures( - parentId="ENST00000408937.7", + parentId="ENST00000408937.8", featureSetId="", featureTypes=["cds"], end=220023, diff --git a/poetry.lock b/poetry.lock index 949bbad..9eea363 100644 --- a/poetry.lock +++ b/poetry.lock @@ -6,7 +6,7 @@ version = "0.0.4" description = "Document parameters, class attributes, return types, and variables inline, with Annotated." optional = false python-versions = ">=3.8" -groups = ["main"] +groups = ["test"] files = [ {file = "annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320"}, {file = "annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4"}, @@ -18,7 +18,7 @@ version = "2026.5.20" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" -groups = ["main"] +groups = ["main", "test"] files = [ {file = "certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897"}, {file = "certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d"}, @@ -30,7 +30,7 @@ version = "3.5.0" description = "Validate configuration and produce human readable error messages." optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["dev"] files = [ {file = "cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0"}, {file = "cfgv-3.5.0.tar.gz", hash = "sha256:d5b1034354820651caa73ede66a6294d6e95c1b00acc5e9b098e917404669132"}, @@ -42,7 +42,7 @@ version = "3.4.7" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7" -groups = ["main"] +groups = ["main", "test"] files = [ {file = "charset_normalizer-3.4.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d"}, {file = "charset_normalizer-3.4.7-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e17b8d5d6a8c47c85e68ca8379def1303fd360c3e22093a807cd34a71cd082b8"}, @@ -181,12 +181,12 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -groups = ["main", "test"] +groups = ["test"] +markers = "sys_platform == \"win32\" or platform_system == \"Windows\"" files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -markers = {main = "platform_system == \"Windows\"", test = "sys_platform == \"win32\""} [[package]] name = "coverage" @@ -194,7 +194,7 @@ version = "7.14.1" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.10" -groups = ["main", "test"] +groups = ["test"] files = [ {file = "coverage-7.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3e3680291c4a1d0dadfa84a2c459576a4af5133abb617905714339a0c73138cf"}, {file = "coverage-7.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a5274669f37f2343635a347b91a60777621341ab3378e9c6ac9335eee704bddf"}, @@ -316,7 +316,7 @@ version = "4.1.0" description = "Show coverage stats online via coveralls.io" optional = false python-versions = "<4.0,>=3.10" -groups = ["main"] +groups = ["test"] files = [ {file = "coveralls-4.1.0-py3-none-any.whl", hash = "sha256:bfacfda2d443c24fc90d67035027cec15015fff2dbd036427e8bf8f4953dda2e"}, {file = "coveralls-4.1.0.tar.gz", hash = "sha256:dab364025ba80cbb95ce56c6fc62cd9172d7fd637060ea235dde99d9b46a4494"}, @@ -336,7 +336,7 @@ version = "0.4.0" description = "Distribution utilities" optional = false python-versions = "*" -groups = ["main"] +groups = ["dev"] files = [ {file = "distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16"}, {file = "distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d"}, @@ -367,7 +367,7 @@ version = "3.29.0" description = "A platform independent file lock." optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["dev"] files = [ {file = "filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258"}, {file = "filelock-3.29.0.tar.gz", hash = "sha256:69974355e960702e789734cb4871f884ea6fe50bd8404051a3530bc07809cf90"}, @@ -379,7 +379,7 @@ version = "2.6.19" description = "File identification library for Python" optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["dev"] files = [ {file = "identify-2.6.19-py2.py3-none-any.whl", hash = "sha256:20e6a87f786f768c092a721ad107fc9df0eb89347be9396cadf3f4abbd1fb78a"}, {file = "identify-2.6.19.tar.gz", hash = "sha256:6be5020c38fcb07da56c53733538a3081ea5aa70d36a156f83044bfbf9173842"}, @@ -394,7 +394,7 @@ version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" -groups = ["main"] +groups = ["main", "test"] files = [ {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, @@ -421,7 +421,7 @@ version = "0.11.0" description = "Mypyc runtime library" optional = false python-versions = ">=3.9" -groups = ["main"] +groups = ["dev"] markers = "platform_python_implementation != \"PyPy\"" files = [ {file = "librt-0.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6e94ebfcfa2d5e9926d6c3b9aa4617ffc42a845b4321fb84021b872358c82a0f"}, @@ -522,7 +522,7 @@ version = "4.2.0" description = "Python port of markdown-it. Markdown parsing, done right!" optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["test"] files = [ {file = "markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a"}, {file = "markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49"}, @@ -546,7 +546,7 @@ version = "0.1.2" description = "Markdown URL utilities" optional = false python-versions = ">=3.7" -groups = ["main"] +groups = ["test"] files = [ {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, @@ -558,7 +558,7 @@ version = "1.20.2" description = "Optional static typing for Python" optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["dev"] files = [ {file = "mypy-1.20.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cf5a4db6dca263010e2c7bff081c89383c72d187ba2cf4c44759aac970e2f0c4"}, {file = "mypy-1.20.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7b0e817b518bff7facd7f85ea05b643ad8bdcce684cf29784987b0a7c8e1f997"}, @@ -627,7 +627,7 @@ version = "1.1.0" description = "Type system extensions for programs checked with the mypy type checker." optional = false python-versions = ">=3.8" -groups = ["main"] +groups = ["dev"] files = [ {file = "mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505"}, {file = "mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558"}, @@ -639,7 +639,7 @@ version = "1.10.0" description = "Node.js virtual environment builder" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -groups = ["main"] +groups = ["dev"] files = [ {file = "nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827"}, {file = "nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb"}, @@ -663,7 +663,7 @@ version = "1.1.1" description = "Utility library for gitignore style pattern matching of file paths." optional = false python-versions = ">=3.9" -groups = ["main"] +groups = ["dev"] files = [ {file = "pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189"}, {file = "pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a"}, @@ -680,7 +680,7 @@ version = "4.10.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["dev"] files = [ {file = "platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a"}, {file = "platformdirs-4.10.0.tar.gz", hash = "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7"}, @@ -708,7 +708,7 @@ version = "4.6.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["dev"] files = [ {file = "pre_commit-4.6.0-py2.py3-none-any.whl", hash = "sha256:e2cf246f7299edcabcf15f9b0571fdce06058527f0a06535068a86d38089f29b"}, {file = "pre_commit-4.6.0.tar.gz", hash = "sha256:718d2208cef53fdc38206e40524a6d4d9576d103eb16f0fec11c875e7716e9d9"}, @@ -727,7 +727,7 @@ version = "2.20.0" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.9" -groups = ["main", "test"] +groups = ["test"] files = [ {file = "pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176"}, {file = "pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f"}, @@ -802,7 +802,7 @@ version = "1.4.0" description = "Python interpreter discovery" optional = false python-versions = ">=3.8" -groups = ["main"] +groups = ["dev"] files = [ {file = "python_discovery-1.4.0-py3-none-any.whl", hash = "sha256:26ed78d703e234879a66244c7d4114563fb13ec5cd30a2d1357e5fb4850782da"}, {file = "python_discovery-1.4.0.tar.gz", hash = "sha256:eb8bc7daad3c226c147e45bb4e970a1feb1bf4048ee178e6db59e197b8010ce3"}, @@ -822,7 +822,7 @@ version = "6.0.3" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" -groups = ["main"] +groups = ["dev", "test"] files = [ {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, @@ -905,7 +905,7 @@ version = "2.34.2" description = "Python HTTP for Humans." optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["main", "test"] files = [ {file = "requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0"}, {file = "requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed"}, @@ -921,13 +921,33 @@ urllib3 = ">=1.26,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<8)"] +[[package]] +name = "responses" +version = "0.25.8" +description = "A utility library for mocking out the `requests` Python library." +optional = false +python-versions = ">=3.8" +groups = ["test"] +files = [ + {file = "responses-0.25.8-py3-none-any.whl", hash = "sha256:0c710af92def29c8352ceadff0c3fe340ace27cf5af1bbe46fb71275bcd2831c"}, + {file = "responses-0.25.8.tar.gz", hash = "sha256:9374d047a575c8f781b94454db5cab590b6029505f488d12899ddb10a4af1cf4"}, +] + +[package.dependencies] +pyyaml = "*" +requests = ">=2.30.0,<3.0" +urllib3 = ">=1.25.10,<3.0" + +[package.extras] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli ; python_version < \"3.11\"", "tomli-w", "types-PyYAML", "types-requests"] + [[package]] name = "rich" version = "15.0.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false python-versions = ">=3.9.0" -groups = ["main"] +groups = ["test"] files = [ {file = "rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb"}, {file = "rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36"}, @@ -946,7 +966,7 @@ version = "0.14.14" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" -groups = ["main"] +groups = ["dev"] files = [ {file = "ruff-0.14.14-py3-none-linux_armv6l.whl", hash = "sha256:7cfe36b56e8489dee8fbc777c61959f60ec0f1f11817e8f2415f429552846aed"}, {file = "ruff-0.14.14-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6006a0082336e7920b9573ef8a7f52eec837add1265cc74e04ea8a4368cd704c"}, @@ -975,7 +995,7 @@ version = "1.5.4" description = "Tool to Detect Surrounding Shell" optional = false python-versions = ">=3.7" -groups = ["main"] +groups = ["test"] files = [ {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, @@ -987,8 +1007,7 @@ version = "2.4.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" -groups = ["main", "test"] -markers = "python_full_version <= \"3.11.0a6\"" +groups = ["dev", "test"] files = [ {file = "tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30"}, {file = "tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a"}, @@ -1038,6 +1057,7 @@ files = [ {file = "tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe"}, {file = "tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f"}, ] +markers = {dev = "python_version == \"3.10\"", test = "python_full_version <= \"3.11.0a6\""} [[package]] name = "typer" @@ -1045,7 +1065,7 @@ version = "0.26.4" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["test"] files = [ {file = "typer-0.26.4-py3-none-any.whl", hash = "sha256:11bfd7b43557137e373c2b10f6967a555f9678a61ed72c808968b011d95534d6"}, {file = "typer-0.26.4.tar.gz", hash = "sha256:25b128964de66c5ea36d5ac82adc579e5e113509b17469edf9f5a4a1864ff2a9"}, @@ -1063,7 +1083,7 @@ version = "2.33.0.20260518" description = "Typing stubs for requests" optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["dev"] files = [ {file = "types_requests-2.33.0.20260518-py3-none-any.whl", hash = "sha256:626d697d1adaaff76e2044dc8c5c051d8f21abc157bdfe204a75558076fe0bf0"}, {file = "types_requests-2.33.0.20260518.tar.gz", hash = "sha256:df7bd3bfe0ca8402dfb841e7d9be714bb5578203283d66d7dc4ef69343449a5e"}, @@ -1078,7 +1098,7 @@ version = "4.15.0" description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" -groups = ["main", "test"] +groups = ["dev", "test"] files = [ {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, @@ -1091,7 +1111,7 @@ version = "2.7.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["main", "dev", "test"] files = [ {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, @@ -1109,7 +1129,7 @@ version = "21.4.1" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.8" -groups = ["main"] +groups = ["dev"] files = [ {file = "virtualenv-21.4.1-py3-none-any.whl", hash = "sha256:caf4ff72d1b4039057f41d8e8466e859513d67c0400d9c6b62c02c9d1ebc3e12"}, {file = "virtualenv-21.4.1.tar.gz", hash = "sha256:2ca543c713b72840ceffd94e9bdedfbd09a661defa1f7f69e5429ad4059442e2"}, @@ -1125,4 +1145,4 @@ typing-extensions = {version = ">=4.13.2", markers = "python_version < \"3.11\"" [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.15" -content-hash = "abc2c75fa8bba868f4691183b8edff7b25194b5242c26db8ace65f3108496e52" +content-hash = "80b4ad5798a64c316704bad0f15e0d588c82939391036e1d31844e4ae08fdf59" diff --git a/pyensemblrest/__init__.py b/pyensemblrest/__init__.py index df54802..1b385d3 100644 --- a/pyensemblrest/__init__.py +++ b/pyensemblrest/__init__.py @@ -4,21 +4,30 @@ __copyright__ = "Copyright (C) 2013-2024, Steve Moss" __credits__ = ["Steve Moss"] __license__ = "GNU GPLv3" -__version__: str = importlib.metadata.version("pyensemblrest") +try: + __version__: str = importlib.metadata.version("pyensemblrest") +except importlib.metadata.PackageNotFoundError: + __version__ = "0.0.0" __maintainer__ = "Steve Moss" __email__ = "gawbul@gmail.com" __status__ = "beta" -__all__ = [ - "EnsemblRest", - "EnsemblRestError", - "EnsemblRestRateLimitError", - "EnsemblRestServiceUnavailable", -] - from .ensemblrest import EnsemblRest from .exceptions import ( + EnsemblRestBadRequestError, EnsemblRestError, + EnsemblRestNotFoundError, EnsemblRestRateLimitError, EnsemblRestServiceUnavailable, + EnsemblRestTimeoutError, ) + +__all__ = [ + "EnsemblRest", + "EnsemblRestBadRequestError", + "EnsemblRestError", + "EnsemblRestNotFoundError", + "EnsemblRestRateLimitError", + "EnsemblRestServiceUnavailable", + "EnsemblRestTimeoutError", +] diff --git a/pyensemblrest/ensemblrest.py b/pyensemblrest/ensemblrest.py index b73d166..aca15c8 100644 --- a/pyensemblrest/ensemblrest.py +++ b/pyensemblrest/ensemblrest.py @@ -1,14 +1,15 @@ +import collections import json import logging import re import time +import urllib.parse from typing import Any import requests from requests import Response from requests.structures import CaseInsensitiveDict -# import ensemblrest modules from .ensembl_config import ( ensembl_api_table, ensembl_content_type, @@ -19,200 +20,186 @@ ensembl_user_agent, ) from .exceptions import ( + EnsemblRestBadRequestError, EnsemblRestError, + EnsemblRestNotFoundError, EnsemblRestRateLimitError, EnsemblRestServiceUnavailable, + EnsemblRestTimeoutError, ) -# Logger instance logger = logging.getLogger(__name__) +PARAM_REGEX = re.compile(r"\{\{(?P[a-zA-Z0-9_]+)\}\}") + + +class FakeResponse: + """Mock Response object for timeout/error simulations.""" -# FakeResponse object -class FakeResponse(object): def __init__( self, headers: CaseInsensitiveDict[str] | dict[str, Any], status_code: int, text: str, - ): + ) -> None: self.headers = headers self.status_code = status_code self.text: str = text -# EnsEMBL REST API object -class EnsemblRest(object): - # class initialisation function +class EnsemblRest: + """ + EnsEMBL REST API Client. + + Provides a dynamic Pythonic interface to all Ensembl REST endpoints. + Handles rate limiting (15 req/s), automatic retries on transient errors, + and multiple response formats. + """ + def __init__( - self, api_table: dict[str, Any] = ensembl_api_table, **kwargs: dict[str, Any] + self, api_table: dict[str, Any] = ensembl_api_table, **kwargs: Any ) -> None: - # read args variable into object as session_args - self.session_args: dict[str, Any] = kwargs or {} + self.api_table = api_table + self.session_args: dict[str, Any] = kwargs.copy() - # In order to rate limit the requests, like https://github.com/Ensembl/ensembl-rest/wiki/Example-Python-Client - self.reqs_per_sec: int = 15 + # Rate limiting configuration (15 requests/second sliding window) + self.reqs_per_sec: int = int(self.session_args.pop("reqs_per_sec", 15)) + self.wall_time: float = float(self.session_args.pop("wall_time", 1.0)) + self._request_timestamps: collections.deque[float] = collections.deque() self.req_count: int = 0 - self.last_req: float = 0 - self.wall_time: int = 1 + self.last_req: float = 0.0 - # get rate limit parameters, if provided + # Rate limit metadata from response headers self.rate_reset: int | None = None self.rate_limit: int | None = None self.rate_remaining: int | None = None self.rate_period: int | None = None self.retry_after: float | None = None - # to record the last parameters used (in order to redo the query with an ensembl known error) + # Tracking state for retrying requests self.last_url: str = "" self.last_headers: CaseInsensitiveDict[str] | dict[str, Any] = {} self.last_params: dict[str, Any] = {} - self.last_data: dict[Any, Any] = {} + self.last_data: Any = {} self.last_method: str = "" self.last_attempt: int = 0 self.last_response: Response | FakeResponse = Response() - # the maximum number of attempts - self.max_attempts: int = 5 + # Request tuning + self.max_attempts: int = int(self.session_args.pop("max_attempts", 5)) + self.timeout: int | float = self.session_args.pop("timeout", 60) - # setting a timeout - self.timeout: int = 60 + # Base URL & Proxies + self.base_url: str = self.session_args.pop("base_url", ensembl_default_url) + proxies: dict[str, str] = self.session_args.pop("proxies", {}) - # set default values if those values are not provided - self.__set_default() - - # setup requests session + # Setup requests session self.session = requests.Session() + setattr(self.session, "base_url", self.base_url) + self.session.proxies.update(proxies) - # update headers - self.__update_headers() - - # add class methods relying api_table - self.__add_methods(api_table) - - def __set_default(self) -> None: - """Set default values""" - - # initialise default values - default_base_url = ensembl_default_url - default_headers = ensembl_header - default_content_type = ensembl_content_type - default_proxies: dict[str, str] = {} - - if "base_url" not in self.session_args: - self.session_args["base_url"] = default_base_url + # Update headers + self._setup_headers() - if "headers" not in self.session_args: - self.session_args["headers"] = default_headers + # Register dynamic API methods + self.__add_methods(self.api_table) - if "User-Agent" not in self.session_args["headers"]: - self.session_args["headers"].update(default_headers) - - if "Content-Type" not in self.session_args["headers"]: - self.session_args["headers"]["Content-Type"] = default_content_type - - if "proxies" not in self.session_args: - self.session_args["proxies"] = default_proxies - - def __update_headers(self) -> None: - """Update headers""" - - # update requests client with arguments - client_args_copy = self.session_args.copy() - for key, val in client_args_copy.items(): - if key in ("base_url", "proxies"): - setattr(self.session, key, val) - self.session_args.pop(key) - - # update headers as already exist within client - self.session.headers.update(self.session_args.pop("headers")) + def _setup_headers(self) -> None: + """Initialize session headers with default User-Agent and Content-Type.""" + headers: dict[str, str] = self.session_args.pop("headers", {}) + merged_headers = ensembl_header.copy() + merged_headers["Content-Type"] = ensembl_content_type + if headers: + merged_headers.update(headers) + self.session.headers.update(merged_headers) def __add_methods(self, api_table: dict[str, Any]) -> None: - """Add methods to class object""" - - # iterate over api_table keys and add key to class namespace - for fun_name in api_table.keys(): - # setattr(self, key, self.register_api_func(key)) - # Not as a class attribute, but a class method - self.__dict__[fun_name] = self.register_api_func(fun_name, api_table) - - # Set __doc__ for generic class method + """Add dynamic API endpoint methods to instance dictionary.""" + for fun_name in api_table: + func = self.register_api_func(fun_name, api_table) if "doc" in api_table[fun_name]: - self.__dict__[fun_name].__doc__ = api_table[fun_name]["doc"] - - # add function name to the class methods - self.__dict__[fun_name].__name__ = fun_name + func.__doc__ = api_table[fun_name]["doc"] + func.__name__ = fun_name + self.__dict__[fun_name] = func + + def __dir__(self) -> list[str]: + """Expose dynamic API methods for dir() and autocompletion tools.""" + return sorted(set(super().__dir__()) | set(self.api_table.keys())) + + def __getattr__(self, name: str) -> Any: + """Fallback to resolve API methods if not already bound.""" + if name in self.api_table: + func = self.register_api_func(name, self.api_table) + if "doc" in self.api_table[name]: + func.__doc__ = self.api_table[name]["doc"] + func.__name__ = name + self.__dict__[name] = func + return func + raise AttributeError( + f"'{self.__class__.__name__}' object has no attribute '{name}'" + ) - # dynamic api registration function def register_api_func(self, api_call: str, api_table: dict[str, Any]) -> Any: + """Create a callable lambda for an API endpoint.""" return lambda **kwargs: self.call_api_func(api_call, api_table, **kwargs) @staticmethod - def __check_params(func: Any, kwargs: Any) -> list[Any]: - """Check for mandatory parameters""" - - # Verify required variables and raise an Exception if needed - mandatory_params = re.findall(r"\{\{(?P[a-zA-Z1-9_]+)\}\}", func["url"]) - + def __check_params(func: dict[str, Any], kwargs: dict[str, Any]) -> list[str]: + """Check that all mandatory template parameters are provided.""" + mandatory_params = PARAM_REGEX.findall(func["url"]) for param in mandatory_params: if param not in kwargs: logger.critical( - "'%s' param not specified. Mandatory params are %s" - % (param, mandatory_params) + f"'{param}' param not specified. Mandatory params are {mandatory_params}" ) - raise Exception("mandatory param '%s' not specified" % param) - else: - logger.debug("Mandatory param %s found" % param) - + raise ValueError(f"mandatory param '{param}' not specified") + logger.debug(f"Mandatory param {param} found") return mandatory_params - # dynamic api call function + def _resolve_url(self, template: str, kwargs: dict[str, Any]) -> str: + """Interpolate path parameters into endpoint URL with proper quoting.""" + + def _replace(match: re.Match[str]) -> str: + key = match.group("m") + val = kwargs.get(key) + return urllib.parse.quote(str(val), safe=":") + + resolved_path = PARAM_REGEX.sub(_replace, template) + base = self.base_url.rstrip("/") + path = resolved_path.lstrip("/") + return f"{base}/{path}" + def call_api_func( - self, api_call: str, api_table: dict[str, Any], **kwargs: dict[str, Any] + self, api_call: str, api_table: dict[str, Any], **kwargs: Any ) -> Any: - # build url from api_table kwargs + """Execute dynamic API call, handling parameter resolution, HTTP method, and parsing.""" func = api_table[api_call] + call_kwargs = kwargs.copy() - # check mandatory params - mandatory_params = self.__check_params(func, kwargs) - - # resolving urls - url = re.sub( - r"\{\{(?P[a-zA-Z1-9_]+)\}\}", - lambda m: "%s" % kwargs.get(m.group(1)), - self.session.base_url + func["url"], # type: ignore[attr-defined] - ) + # Check and validate mandatory params + mandatory_params = self.__check_params(func, call_kwargs) - # debug - logger.debug("Resolved url: '%s'" % url) + # Build resolved URL + url = self._resolve_url(func["url"], call_kwargs) + logger.debug(f"Resolved url: '{url}'") - # Now I have to remove mandatory params from kwargs + # Remove mandatory path parameters from query/body kwargs for param in mandatory_params: - del kwargs[param] - - # Initialize with the ensembl default content type - content_type: str | dict[str, Any] = ensembl_content_type - - # Override content type if it is defined by function - if "content_type" in func: - content_type = func["content_type"] + call_kwargs.pop(param, None) - # Ovveride content type if it is provied when calling function - if "content_type" in kwargs: - content_type = kwargs["content_type"] - del kwargs["content_type"] + # Determine content type + content_type: str = func.get("content_type", ensembl_content_type) + if "content_type" in call_kwargs: + content_type = call_kwargs.pop("content_type") - # check the request type (GET or POST?) + # Handle GET or POST if func["method"] == "GET": logger.debug( - "Submitting a GET request: url = '%s', headers = %s, params = %s" - % (url, {"Content-Type": content_type}, kwargs) + f"Submitting a GET request: url = '{url}', headers = {{'Content-Type': '{content_type}'}}, params = {call_kwargs}" ) - - # record this request self.last_url = url self.last_headers = {"Content-Type": content_type} - self.last_params = kwargs + self.last_params = call_kwargs self.last_data = {} self.last_method = "GET" self.last_attempt = 0 @@ -220,24 +207,17 @@ def call_api_func( resp = self.__get_response() elif func["method"] == "POST": - # in a POST request, separate post parameters from other parameters - data = {} - - # pass key=value in POST data from kwargs - for key in func["post_parameters"]: - if key in kwargs: - data[key] = kwargs[key] - del kwargs[key] + data: dict[str, Any] = {} + for key in func.get("post_parameters", []): + if key in call_kwargs: + data[key] = call_kwargs.pop(key) logger.debug( - "Submitting a POST request: url = '%s', headers = %s, params = %s, data = %s" - % (url, {"Content-Type": content_type}, kwargs, data) + f"Submitting a POST request: url = '{url}', headers = {{'Content-Type': '{content_type}'}}, params = {call_kwargs}, data = {data}" ) - - # record this request self.last_url = url self.last_headers = {"Content-Type": content_type} - self.last_params = kwargs + self.last_params = call_kwargs self.last_data = data self.last_method = "POST" self.last_attempt = 0 @@ -245,41 +225,46 @@ def call_api_func( resp = self.__get_response() else: - raise NotImplementedError( - "Method '%s' not yet implemented" % (func["method"]) - ) + raise NotImplementedError(f"Method '{func['method']}' not yet implemented") - # call response and return content return self.parseResponse(resp, content_type) - # A function to get reponse from ensembl REST api - def __get_response(self) -> Response | FakeResponse: - """Call session get and post method. Return response""" - - # updating last_req time - self.last_req = time.time() - - # Increment the request counter to rate limit requests - self.req_count += 1 - - # Evaluating the numer of request in a second (according to EnsEMBL rest specification) - if self.req_count >= self.reqs_per_sec: - delta = time.time() - self.last_req - - # sleep upto wall_time - if delta < self.wall_time: - to_sleep = self.wall_time - delta - logger.debug("waiting %s" % to_sleep) + def _wait_for_rate_limit(self) -> None: + """Enforce rate limiting using a sliding window deque.""" + now = time.time() + # Purge timestamps outside the sliding window + while ( + self._request_timestamps + and (now - self._request_timestamps[0]) >= self.wall_time + ): + self._request_timestamps.popleft() + + # If window limit is reached, sleep until the oldest request expires + if len(self._request_timestamps) >= self.reqs_per_sec: + oldest = self._request_timestamps[0] + to_sleep = self.wall_time - (now - oldest) + if to_sleep > 0: + logger.debug( + f"Rate limit reached ({self.reqs_per_sec} req/s). Waiting {to_sleep:.4f}s" + ) time.sleep(to_sleep) + now = time.time() + while ( + self._request_timestamps + and (now - self._request_timestamps[0]) >= self.wall_time + ): + self._request_timestamps.popleft() - self.req_count = 0 + self._request_timestamps.append(now) + self.last_req = now + self.req_count = len(self._request_timestamps) - # my response - resp: Response | FakeResponse = Response() + def __get_response(self) -> Response | FakeResponse: + """Perform HTTP request with rate limiting and network exception handling.""" + self._wait_for_rate_limit() - # deal with exceptions + resp: Response | FakeResponse try: - # another request using the correct method if self.last_method == "GET": resp = self.session.get( self.last_url, @@ -288,7 +273,6 @@ def __get_response(self) -> Response | FakeResponse: timeout=self.timeout, ) elif self.last_method == "POST": - # post parameters are load as POST data, other parameters are url parameters as GET requests resp = self.session.post( self.last_url, headers=self.last_headers, @@ -296,40 +280,36 @@ def __get_response(self) -> Response | FakeResponse: params=self.last_params, timeout=self.timeout, ) - # other methods are verifiedby others functions + else: + raise NotImplementedError( + f"Method '{self.last_method}' not yet implemented" + ) except requests.ConnectionError as e: - raise EnsemblRestServiceUnavailable(e) + raise EnsemblRestServiceUnavailable(e) from e except requests.Timeout as e: - logger.error("%s request timeout: %s" % (self.last_method, e)) - - # create a fake response in order to redo the query + logger.error(f"{self.last_method} request timeout: {e}") resp = FakeResponse( - headers=self.last_response.headers, - status_code=400, + headers=getattr(self.last_response, "headers", {}), + status_code=408, text=json.dumps( - {"message": repr(e), "error": "%s timeout" % ensembl_user_agent} + {"message": repr(e), "error": f"{ensembl_user_agent} timeout"} ), ) - # return response return resp - # A function to deal with a generic response def parseResponse( self, resp: Response | FakeResponse, content_type: str | dict[str, Any] = "application/json", ) -> Any: - """Deal with a generic REST response""" - - logger.debug("Got %s" % resp.text) - - # Record response for debug intent + """Process API response, extract rate limits, handle retries, and parse content.""" + logger.debug(f"Got {resp.text}") self.last_response = resp - # Initialize some values. Check if I'm rate limited + # Extract rate limit headers ( self.rate_reset, self.rate_limit, @@ -338,51 +318,64 @@ def parseResponse( self.rate_period, ) = self.__get_rate_limit(resp.headers) - # parse status code + # Check for errors and retries if self.__check_retry(resp): return self.__retry_request() - # Handle content in different way relying on content-type + # Parse response content if content_type == "application/json": - content = json.loads(resp.text) + try: + content = json.loads(resp.text) + except (ValueError, json.JSONDecodeError): + content = resp.text else: - # Default content = resp.text return content def __check_retry(self, resp: Response | FakeResponse) -> bool: - """Parse status code and print warnings. Return True if a retry is needed""" - - # default status code - message = ensembl_http_status_codes[resp.status_code][1] + """Evaluate status codes for retrying or raising appropriate exceptions.""" + status_info = ensembl_http_status_codes.get( + resp.status_code, ("Error", "Unknown HTTP error") + ) + message = status_info[1] - # parse status codes if resp.status_code > 304: - ExceptionType = EnsemblRestError + exception_type = EnsemblRestError - # Try to derive a more useful message than ensembl default message - if resp.status_code == 400: + # Parse JSON error payload if available + try: json_message = json.loads(resp.text) - if "error" in json_message: + if isinstance(json_message, dict) and "error" in json_message: message = json_message["error"] + except (ValueError, json.JSONDecodeError): + pass - # TODO: deal with special cases errors + if resp.status_code == 400: if message in ensembl_known_errors: - # call a function that will re-execute the REST request and then call again parseResponse - # if everithing is ok, a processed content is returned - logger.warning("EnsEMBL REST Service returned: %s" % message) + logger.warning(f"EnsEMBL REST Service returned: {message}") + return True + exception_type = EnsemblRestBadRequestError + + elif resp.status_code == 404: + exception_type = EnsemblRestNotFoundError - # return true if retry needed + elif resp.status_code == 408: + if message in ensembl_known_errors or "timeout" in message: return True + exception_type = EnsemblRestTimeoutError + + elif resp.status_code == 429: + exception_type = EnsemblRestRateLimitError + elif resp.status_code == 500: - # Retrying when we get a 500 error. - # Due to Ensembl's condition on randomly returning 500s on valid requests. + # Retry transient Ensembl 500 errors return True - elif resp.status_code == 429: - ExceptionType = EnsemblRestRateLimitError - raise ExceptionType( + elif resp.status_code == 503: + exception_type = EnsemblRestServiceUnavailable + + raise exception_type( message, error_code=resp.status_code, rate_reset=self.rate_reset, @@ -391,122 +384,122 @@ def __check_retry(self, resp: Response | FakeResponse) -> bool: retry_after=self.retry_after, ) - # return a flag if status is ok return False @staticmethod def __get_rate_limit( headers: CaseInsensitiveDict[str] | dict[str, Any], ) -> tuple[int | None, int | None, int | None, float | None, int | None]: - """Read rate limited attributes""" + """Parse rate limit headers safely.""" + retry_after: float | None = None + rate_reset: int | None = None + rate_limit: int | None = None + rate_remaining: int | None = None + rate_period: int | None = None - # initialize some values - retry_after = None - rate_reset = None - rate_limit = None - rate_remaining = None - rate_period = None + if not headers: + return rate_reset, rate_limit, rate_remaining, retry_after, rate_period - # for semplicity - keys = [key.lower() for key in headers.keys()] + lower_headers = {k.lower(): v for k, v in headers.items()} - if "X-RateLimit-Reset".lower() in keys: - rate_reset = int(headers["X-RateLimit-Reset"]) - logger.debug("X-RateLimit-Reset: %s" % rate_reset) + if "x-ratelimit-reset" in lower_headers: + try: + rate_reset = int(lower_headers["x-ratelimit-reset"]) + logger.debug(f"X-RateLimit-Reset: {rate_reset}") + except (ValueError, TypeError): + pass - if "X-RateLimit-Period".lower() in keys: - rate_period = int(headers["X-RateLimit-Period"]) - logger.debug("X-RateLimit-Period: %s" % rate_period) + if "x-ratelimit-period" in lower_headers: + try: + rate_period = int(lower_headers["x-ratelimit-period"]) + logger.debug(f"X-RateLimit-Period: {rate_period}") + except (ValueError, TypeError): + pass - if "X-RateLimit-Limit".lower() in keys: - rate_limit = int(headers["X-RateLimit-Limit"]) - logger.debug("X-RateLimit-Limit: %s" % rate_limit) + if "x-ratelimit-limit" in lower_headers: + try: + rate_limit = int(lower_headers["x-ratelimit-limit"]) + logger.debug(f"X-RateLimit-Limit: {rate_limit}") + except (ValueError, TypeError): + pass - if "X-RateLimit-Remaining".lower() in keys: - rate_remaining = int(headers["X-RateLimit-Remaining"]) - logger.debug("X-RateLimit-Remaining: %s" % rate_remaining) + if "x-ratelimit-remaining" in lower_headers: + try: + rate_remaining = int(lower_headers["x-ratelimit-remaining"]) + logger.debug(f"X-RateLimit-Remaining: {rate_remaining}") + except (ValueError, TypeError): + pass - if "Retry-After".lower() in keys: - retry_after = float(headers["Retry-After"]) - logger.debug("Retry-After: %s" % retry_after) + if "retry-after" in lower_headers: + try: + retry_after = float(lower_headers["retry-after"]) + logger.debug(f"Retry-After: {retry_after}") + except (ValueError, TypeError): + pass return rate_reset, rate_limit, rate_remaining, retry_after, rate_period def __retry_request(self) -> Any: - """Retry last request in case of failure""" - - # update last attempt + """Retry the last request using exponential backoff.""" self.last_attempt += 1 - # a max of three attempts if self.last_attempt > self.max_attempts: - # default status code - message = ensembl_http_status_codes[self.last_response.status_code][1] + status_code = getattr(self.last_response, "status_code", 500) + status_info = ensembl_http_status_codes.get( + status_code, ("Error", "Unknown HTTP error") + ) + message = status_info[1] - # parse error if possible try: json_message = json.loads(self.last_response.text) - if "error" in json_message: + if isinstance(json_message, dict) and "error" in json_message: message = json_message["error"] - except ValueError: - # In this case we didn't even get a JSON back. + except (ValueError, json.JSONDecodeError, AttributeError): message = "Server returned invalid JSON." raise EnsemblRestError( - "Max number of retries attempts reached. Last message was: %s" - % message, - error_code=self.last_response.status_code, + f"Max number of retries attempts reached. Last message was: {message}", + error_code=status_code, rate_reset=self.rate_reset, rate_limit=self.rate_limit, rate_remaining=self.rate_remaining, retry_after=self.retry_after, ) - # sleep a while. Increment on each attempt to_sleep = (self.wall_time + 1) * self.last_attempt - - logger.debug("Sleeping %s" % to_sleep) + logger.debug( + f"Sleeping {to_sleep}s before retry {self.last_attempt}/{self.max_attempts}" + ) time.sleep(to_sleep) - # another request using the correct method if self.last_method == "GET": - # debug logger.debug( - "Retring last GET request (%s/%s): url = '%s', headers = %s, params = %s" - % ( - self.last_attempt, - self.max_attempts, - self.last_url, - self.last_headers, - self.last_params, - ) + f"Retrying last GET request ({self.last_attempt}/{self.max_attempts}): url = '{self.last_url}'" ) - resp = self.__get_response() - elif self.last_method == "POST": - # debug logger.debug( - "Retring last POST request (%s/%s): url = '%s', headers = %s, params = %s, data = %s" - % ( - self.last_attempt, - self.max_attempts, - self.last_url, - self.last_headers, - self.last_params, - self.last_data, - ) + f"Retrying last POST request ({self.last_attempt}/{self.max_attempts}): url = '{self.last_url}'" ) - resp = self.__get_response() else: raise NotImplementedError( - "Method '%s' not yet implemented" % (self.last_method) + f"Method '{self.last_method}' not yet implemented" ) - # call response and return content - return self.parseResponse(resp, self.last_headers["Content-Type"]) + content_type = self.last_headers.get("Content-Type", ensembl_content_type) + return self.parseResponse(resp, content_type) + + def close(self) -> None: + """Close the underlying HTTP session connection pools.""" + self.session.close() + + def __enter__(self) -> "EnsemblRest": + return self + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + self.close() def get_user_agent(self) -> str: - """Return the pyEnsemblRest user agent""" + """Return the pyEnsemblRest user agent string.""" return ensembl_user_agent diff --git a/pyensemblrest/ensemblrest.pyi b/pyensemblrest/ensemblrest.pyi new file mode 100644 index 0000000..06ba8a8 --- /dev/null +++ b/pyensemblrest/ensemblrest.pyi @@ -0,0 +1,604 @@ +import collections +from typing import Any + +import requests +from requests import Response +from requests.structures import CaseInsensitiveDict + +class FakeResponse: + headers: CaseInsensitiveDict[str] | dict[str, Any] + status_code: int + text: str + def __init__( + self, + headers: CaseInsensitiveDict[str] | dict[str, Any], + status_code: int, + text: str, + ) -> None: ... + +class EnsemblRest: + api_table: dict[str, Any] + session_args: dict[str, Any] + reqs_per_sec: int + wall_time: float + _request_timestamps: collections.deque[float] + req_count: int + last_req: float + rate_reset: int | None + rate_limit: int | None + rate_remaining: int | None + rate_period: int | None + retry_after: float | None + last_url: str + last_headers: CaseInsensitiveDict[str] | dict[str, Any] + last_params: dict[str, Any] + last_data: Any + last_method: str + last_attempt: int + last_response: Response | FakeResponse + max_attempts: int + timeout: int | float + base_url: str + session: requests.Session + + def __init__(self, api_table: dict[str, Any] = ..., **kwargs: Any) -> None: ... + def register_api_func(self, api_call: str, api_table: dict[str, Any]) -> Any: ... + def call_api_func( + self, api_call: str, api_table: dict[str, Any], **kwargs: Any + ) -> Any: ... + def parseResponse( + self, resp: Response | FakeResponse, content_type: str | dict[str, Any] = ... + ) -> Any: ... + def close(self) -> None: ... + def __enter__(self) -> EnsemblRest: ... + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: ... + def get_user_agent(self) -> str: ... + def getArchiveById(self, id: Any, **kwargs: Any) -> Any: + """Uses the given identifier to return its latest version""" + ... + + def getArchiveByMultipleIds(self, id: Any = ..., **kwargs: Any) -> Any: + """Retrieve the latest version for a set of identifiers""" + ... + + def getCafeGeneTreeById(self, id: Any, **kwargs: Any) -> Any: + """Retrieves a cafe tree of the gene tree using the gene tree stable identifier""" + ... + + def getCafeGeneTreeMemberBySymbol( + self, species: Any, symbol: Any, **kwargs: Any + ) -> Any: + """Retrieves the cafe tree of the gene tree that contains the gene identified by a symbol""" + ... + + def getCafeGeneTreeMemberById(self, species: Any, id: Any, **kwargs: Any) -> Any: + """Retrieves the cafe tree of the gene tree that contains the gene / transcript / translation stable identifier in the given species""" + ... + + def getGeneTreeById(self, id: Any, **kwargs: Any) -> Any: + """Retrieves a gene tree for a gene tree stable identifier""" + ... + + def getGeneTreeMemberBySymbol( + self, species: Any, symbol: Any, **kwargs: Any + ) -> Any: + """Retrieves the gene tree that contains the gene identified by a symbol""" + ... + + def getGeneTreeMemberById(self, species: Any, id: Any, **kwargs: Any) -> Any: + """Retrieves the gene tree that contains the gene / transcript / translation stable identifier in the given species""" + ... + + def getAlignmentByRegion(self, species: Any, region: Any, **kwargs: Any) -> Any: + """Retrieves genomic alignments as separate blocks based on a region and species""" + ... + + def getHomologyById(self, species: Any, id: Any, **kwargs: Any) -> Any: + """Retrieves homology information (orthologs) by species and Ensembl gene id""" + ... + + def getHomologyBySymbol(self, species: Any, symbol: Any, **kwargs: Any) -> Any: + """Retrieves homology information (orthologs) by symbol""" + ... + + def getXrefsBySymbol(self, species: Any, symbol: Any, **kwargs: Any) -> Any: + """Looks up an external symbol and returns all Ensembl objects linked to it. This can be a display name for a gene/transcript/translation, a synonym or an externally linked reference. If a gene's transcript is linked to the supplied symbol the service will return both gene and transcript (it supports transient links).""" + ... + + def getXrefsById(self, id: Any, **kwargs: Any) -> Any: + """Perform lookups of Ensembl Identifiers and retrieve their external references in other databases""" + ... + + def getXrefsByName(self, species: Any, name: Any, **kwargs: Any) -> Any: + """Performs a lookup based upon the primary accession or display label of an external reference and returning the information we hold about the entry""" + ... + + def getInfoAnalysis(self, species: Any, **kwargs: Any) -> Any: + """List the names of analyses involved in generating Ensembl data.""" + ... + + def getInfoAssembly(self, species: Any, **kwargs: Any) -> Any: + """List the currently available assemblies for a species, along with toplevel sequences, chromosomes and cytogenetic bands.""" + ... + + def getInfoAssemblyRegion( + self, species: Any, region_name: Any, **kwargs: Any + ) -> Any: + """Returns information about the specified toplevel sequence region for the given species.""" + ... + + def getInfoBiotypes(self, species: Any, **kwargs: Any) -> Any: + """List the functional classifications of gene models that Ensembl associates with a particular species. Useful for restricting the type of genes/transcripts retrieved by other endpoints.""" + ... + + def getInfoBiotypesByGroup( + self, group: Any, object_type: Any, **kwargs: Any + ) -> Any: + """Without argument the list of available biotype groups is returned. With :group argument provided, list the properties of biotypes within that group. Object type (gene or transcript) can be provided for filtering.""" + ... + + def getInfoBiotypesByName(self, name: Any, object_type: Any, **kwargs: Any) -> Any: + """List the properties of biotypes with a given name. Object type (gene or transcript) can be provided for filtering.""" + ... + + def getInfoComparaMethods(self, **kwargs: Any) -> Any: + """List all compara analyses available (an analysis defines the type of comparative data).""" + ... + + def getInfoComparaSpeciesSets(self, methods: Any, **kwargs: Any) -> Any: + """List all collections of species analysed with the specified compara method.""" + ... + + def getInfoComparas(self, **kwargs: Any) -> Any: + """Lists all available comparative genomics databases and their data release. DEPRECATED: use info/genomes/division instead.""" + ... + + def getInfoData(self, **kwargs: Any) -> Any: + """Shows the data releases available on this REST server. May return more than one release (unfrequent non-standard Ensembl configuration).""" + ... + + def getInfoEgVersion(self, **kwargs: Any) -> Any: + """Returns the Ensembl Genomes version of the databases backing this service""" + ... + + def getInfoExternalDbs(self, species: Any, **kwargs: Any) -> Any: + """Lists all available external sources for a species.""" + ... + + def getInfoDivisions(self, **kwargs: Any) -> Any: + """Get list of all Ensembl divisions for which information is available""" + ... + + def getInfoGenomesByName(self, name: Any, **kwargs: Any) -> Any: + """Find information about a given genome""" + ... + + def getInfoGenomesByAccession(self, accession: Any, **kwargs: Any) -> Any: + """Find information about genomes containing a specified INSDC accession""" + ... + + def getInfoGenomesByAssembly(self, assembly_id: Any, **kwargs: Any) -> Any: + """Find information about a genome with a specified assembly""" + ... + + def getInfoGenomesByDivision(self, division: Any, **kwargs: Any) -> Any: + """Find information about all genomes in a given division. May be large for Ensembl Bacteria.""" + ... + + def getInfoGenomesByTaxonomy(self, taxon_name: Any, **kwargs: Any) -> Any: + """Find information about all genomes beneath a given node of the taxonomy""" + ... + + def getInfoPing(self, **kwargs: Any) -> Any: + """Checks if the service is alive.""" + ... + + def getInfoRest(self, **kwargs: Any) -> Any: + """Shows the current version of the Ensembl REST API.""" + ... + + def getInfoSoftware(self, **kwargs: Any) -> Any: + """Shows the current version of the Ensembl API used by the REST server.""" + ... + + def getInfoSpecies(self, **kwargs: Any) -> Any: + """Lists all available species, their aliases, available adaptor groups and data release.""" + ... + + def getInfoVariationBySpecies(self, species: Any, **kwargs: Any) -> Any: + """List the variation sources used in Ensembl for a species.""" + ... + + def getInfoVariationConsequenceTypes(self, **kwargs: Any) -> Any: + """Lists all variant consequence types.""" + ... + + def getInfoVariationPopulationIndividuals( + self, species: Any, population_name: Any, **kwargs: Any + ) -> Any: + """List all individuals for a population from a species""" + ... + + def getInfoVariationPopulations(self, species: Any, **kwargs: Any) -> Any: + """List all populations for a species""" + ... + + def getLdId( + self, species: Any, id: Any, population_name: Any, **kwargs: Any + ) -> Any: + """Computes and returns LD values between the given variant and all other variants in a window centered around the given variant. The window size is set to 500 kb.""" + ... + + def getLdPairwise(self, species: Any, id1: Any, id2: Any, **kwargs: Any) -> Any: + """Computes and returns LD values between the given variants.""" + ... + + def getLdRegion( + self, species: Any, region: Any, population_name: Any, **kwargs: Any + ) -> Any: + """Computes and returns LD values between all pairs of variants in the defined region.""" + ... + + def getLookupById(self, id: Any, **kwargs: Any) -> Any: + """Find the species and database for a single identifier e.g. gene, transcript, protein""" + ... + + def getLookupByMultipleIds(self, ids: Any = ..., **kwargs: Any) -> Any: + """Find the species and database for several identifiers. IDs that are not found are returned with no data.""" + ... + + def getLookupBySymbol(self, species: Any, symbol: Any, **kwargs: Any) -> Any: + """Find the species and database for a symbol in a linked external database""" + ... + + def getLookupByMultipleSymbols( + self, species: Any, symbols: Any = ..., **kwargs: Any + ) -> Any: + """Find the species and database for a set of symbols in a linked external database. Unknown symbols are omitted from the response.""" + ... + + def getMapCdnaToRegion(self, id: Any, region: Any, **kwargs: Any) -> Any: + """Convert from cDNA coordinates to genomic coordinates. Output reflects forward orientation coordinates as returned from the Ensembl API.""" + ... + + def getMapCdsToRegion(self, id: Any, region: Any, **kwargs: Any) -> Any: + """Convert from CDS coordinates to genomic coordinates. Output reflects forward orientation coordinates as returned from the Ensembl API.""" + ... + + def getMapAssemblyOneToTwo( + self, species: Any, asm_one: Any, region: Any, asm_two: Any, **kwargs: Any + ) -> Any: + """Convert the co-ordinates of one assembly to another""" + ... + + def getMapTranslationToRegion(self, id: Any, region: Any, **kwargs: Any) -> Any: + """Convert from protein (translation) coordinates to genomic coordinates. Output reflects forward orientation coordinates as returned from the Ensembl API.""" + ... + + def getAncestorsById(self, id: Any, **kwargs: Any) -> Any: + """Reconstruct the entire ancestry of a term from is_a and part_of relationships""" + ... + + def getAncestorsChartById(self, id: Any, **kwargs: Any) -> Any: + """Reconstruct the entire ancestry of a term from is_a and part_of relationships.""" + ... + + def getDescendantsById(self, id: Any, **kwargs: Any) -> Any: + """Find all the terms descended from a given term. By default searches are conducted within the namespace of the given identifier""" + ... + + def getOntologyById(self, id: Any, **kwargs: Any) -> Any: + """Search for an ontological term by its namespaced identifier""" + ... + + def getOntologyByName(self, name: Any, **kwargs: Any) -> Any: + """Search for a list of ontological terms by their name""" + ... + + def getTaxonomyClassificationById(self, id: Any, **kwargs: Any) -> Any: + """Return the taxonomic classification of a taxon node""" + ... + + def getTaxonomyById(self, id: Any, **kwargs: Any) -> Any: + """Search for a taxonomic term by its identifier or name""" + ... + + def getTaxonomyByName(self, name: Any, **kwargs: Any) -> Any: + """Search for a taxonomic id by a non-scientific name""" + ... + + def getOverlapById(self, id: Any, **kwargs: Any) -> Any: + """Retrieves features (e.g. genes, transcripts, variants and more) that overlap a region defined by the given identifier.""" + ... + + def getOverlapByRegion(self, species: Any, region: Any, **kwargs: Any) -> Any: + """Retrieves features (e.g. genes, transcripts, variants and more) that overlap a given region.""" + ... + + def getOverlapByTranslation(self, id: Any, **kwargs: Any) -> Any: + """Retrieve features related to a specific Translation as described by its stable ID (e.g. domains, variants).""" + ... + + def getPhenotypeByAccession( + self, species: Any, accession: Any, **kwargs: Any + ) -> Any: + """Return phenotype annotations for genomic features given a phenotype ontology accession""" + ... + + def getPhenotypeByGene(self, species: Any, gene: Any, **kwargs: Any) -> Any: + """Return phenotype annotations for a given gene.""" + ... + + def getPhenotypeByRegion(self, species: Any, region: Any, **kwargs: Any) -> Any: + """Return phenotype annotations that overlap a given genomic region.""" + ... + + def getPhenotypeByTerm(self, species: Any, term: Any, **kwargs: Any) -> Any: + """Return phenotype annotations for genomic features given a phenotype ontology term""" + ... + + def getRegulationBindingMatrix( + self, species: Any, binding_matrix: Any, **kwargs: Any + ) -> Any: + """Return the specified binding matrix""" + ... + + def getSequenceById(self, id: Any, **kwargs: Any) -> Any: + """Request multiple types of sequence by stable identifier. Supports feature masking and expand options.""" + ... + + def getSequenceByMultipleIds(self, ids: Any = ..., **kwargs: Any) -> Any: + """Request multiple types of sequence by a stable identifier list.""" + ... + + def getSequenceByRegion(self, species: Any, region: Any, **kwargs: Any) -> Any: + """Returns the genomic sequence of the specified region of the given species. Supports feature masking and expand options.""" + ... + + def getSequenceByMultipleRegions( + self, species: Any, regions: Any = ..., **kwargs: Any + ) -> Any: + """Request multiple types of sequence by a list of regions.""" + ... + + def getTranscriptHaplotypes(self, species: Any, id: Any, **kwargs: Any) -> Any: + """Computes observed transcript haplotype sequences based on phased genotype data""" + ... + + def getVariantConsequencesByHGVSNotation( + self, species: Any, hgvs_notation: Any, **kwargs: Any + ) -> Any: + """Fetch variant consequences based on a HGVS notation""" + ... + + def getVariantConsequencesByMultipleHGVSNotations( + self, species: Any, hgvs_notations: Any = ..., **kwargs: Any + ) -> Any: + """Fetch variant consequences for multiple HGVS notations""" + ... + + def getVariantConsequencesById(self, species: Any, id: Any, **kwargs: Any) -> Any: + """Fetch variant consequences based on a variant identifier""" + ... + + def getVariantConsequencesByMultipleIds( + self, species: Any, ids: Any = ..., **kwargs: Any + ) -> Any: + """Fetch variant consequences for multiple ids""" + ... + + def getVariantConsequencesByRegion( + self, species: Any, region: Any, allele: Any, **kwargs: Any + ) -> Any: + """Fetch variant consequences""" + ... + + def getVariantConsequencesByMultipleRegions( + self, species: Any, variants: Any = ..., **kwargs: Any + ) -> Any: + """Fetch variant consequences for multiple regions""" + ... + + def getVariationRecoderById(self, species: Any, id: Any, **kwargs: Any) -> Any: + """Translate a variant identifier, HGVS notation or genomic SPDI notation to all possible variant IDs, HGVS and genomic SPDI""" + ... + + def getVariationRecoderByMultipleIds( + self, species: Any, ids: Any = ..., **kwargs: Any + ) -> Any: + """Translate a list of variant identifiers, HGVS notations or genomic SPDI notations to all possible variant IDs, HGVS and genomic SPDI""" + ... + + def getVariationById(self, species: Any, id: Any, **kwargs: Any) -> Any: + """Uses a variant identifier (e.g. rsID) to return the variation features including optional genotype, phenotype and population data""" + ... + + def getVariationByPMCID(self, species: Any, pmcid: Any, **kwargs: Any) -> Any: + """Uses a variant identifier (e.g. rsID) to return the variation features including optional genotype, phenotype and population data""" + ... + + def getVariationByPMID(self, species: Any, pmid: Any, **kwargs: Any) -> Any: + """Uses a variant identifier (e.g. rsID) to return the variation features including optional genotype, phenotype and population data""" + ... + + def getVariationByMultipleIds( + self, species: Any, ids: Any = ..., **kwargs: Any + ) -> Any: + """Uses a list of variant identifiers (e.g. rsID) to return the variation features including optional genotype, phenotype and population data""" + ... + + def getGA4GHBeacon(self, **kwargs: Any) -> Any: + """Return Beacon information""" + ... + + def getGA4GHBeaconQuery( + self, + alternateBases: Any, + assemblyId: Any, + referenceBases: Any, + referenceName: Any, + start: Any, + **kwargs: Any, + ) -> Any: + """Return the Beacon response for allele information""" + ... + + def postGA4GHBeaconQuery( + self, + alternateBases: Any = ..., + assemblyId: Any = ..., + end: Any = ..., + referenceBases: Any = ..., + referenceName: Any = ..., + start: Any = ..., + variantType: Any = ..., + **kwargs: Any, + ) -> Any: + """Return the Beacon response for allele information""" + ... + + def getGA4GHFeaturesById(self, id: Any, **kwargs: Any) -> Any: + """Return the GA4GH record for a specific sequence feature given its identifier""" + ... + + def searchGA4GHFeatures( + self, + end: Any = ..., + referenceName: Any = ..., + start: Any = ..., + featureSetId: Any = ..., + parentId: Any = ..., + **kwargs: Any, + ) -> Any: + """Return a list of sequence annotation features in GA4GH format""" + ... + + def searchGA4GHCallset( + self, + variantSetId: Any = ..., + name: Any = ..., + pageToken: Any = ..., + pageSize: Any = ..., + **kwargs: Any, + ) -> Any: + """Return a list of sets of genotype calls for specific samples in GA4GH format""" + ... + + def getGA4GHCallsetById(self, id: Any, **kwargs: Any) -> Any: + """Return the GA4GH record for a specific CallSet given its identifier""" + ... + + def searchGA4GHDatasets( + self, pageToken: Any = ..., pageSize: Any = ..., **kwargs: Any + ) -> Any: + """Return a list of datasets in GA4GH format""" + ... + + def getGA4GHDatasetsById(self, id: Any, **kwargs: Any) -> Any: + """Return the GA4GH record for a specific dataset given its identifier""" + ... + + def searchGA4GHFeaturesets( + self, + datasetId: Any = ..., + pageToken: Any = ..., + pageSize: Any = ..., + **kwargs: Any, + ) -> Any: + """Return a list of feature sets in GA4GH format""" + ... + + def getGA4GHFeaturesetsById(self, id: Any, **kwargs: Any) -> Any: + """Return the GA4GH record for a specific featureSet given its identifier""" + ... + + def getGA4GHVariantsById(self, id: Any, **kwargs: Any) -> Any: + """Return the GA4GH record for a specific variant given its identifier.""" + ... + + def searchGA4GHVariantAnnotations( + self, + variantAnnotationSetId: Any = ..., + effects: Any = ..., + end: Any = ..., + pageSize: Any = ..., + pageToken: Any = ..., + referenceId: Any = ..., + referenceName: Any = ..., + start: Any = ..., + **kwargs: Any, + ) -> Any: + """Return variant annotation information in GA4GH format for a region on a reference sequence""" + ... + + def searchGA4GHVariants( + self, + variantSetId: Any = ..., + callSetIds: Any = ..., + referenceName: Any = ..., + start: Any = ..., + end: Any = ..., + pageToken: Any = ..., + pageSize: Any = ..., + **kwargs: Any, + ) -> Any: + """Return variant call information in GA4GH format for a region on a reference sequence""" + ... + + def searchGA4GHVariantsets( + self, + datasetId: Any = ..., + pageToken: Any = ..., + pageSize: Any = ..., + **kwargs: Any, + ) -> Any: + """Return a list of variant sets in GA4GH format""" + ... + + def getGA4GHVariantsetsById(self, id: Any, **kwargs: Any) -> Any: + """Return the GA4GH record for a specific VariantSet given its identifier""" + ... + + def searchGA4GHReferences( + self, + referenceSetId: Any = ..., + md5checksum: Any = ..., + accession: Any = ..., + pageToken: Any = ..., + pageSize: Any = ..., + **kwargs: Any, + ) -> Any: + """Return a list of reference sequences in GA4GH format""" + ... + + def getGA4GHReferencesById(self, id: Any, **kwargs: Any) -> Any: + """Return data for a specific reference in GA4GH format by id""" + ... + + def searchGA4GHReferencesets( + self, + accession: Any = ..., + pageToken: Any = ..., + pageSize: Any = ..., + **kwargs: Any, + ) -> Any: + """Return a list of reference sets in GA4GH format""" + ... + + def getGA4GHReferencesetsById(self, id: Any, **kwargs: Any) -> Any: + """Return data for a specific reference set in GA4GH format""" + ... + + def searchGA4GHVariantAnnotationsets( + self, + variantSetId: Any = ..., + pageToken: Any = ..., + pageSize: Any = ..., + **kwargs: Any, + ) -> Any: + """Return a list of annotation sets in GA4GH format""" + ... + + def getGA4GHVariantAnnotationsetsById(self, id: Any, **kwargs: Any) -> Any: + """Return meta data for a specific annotation set in GA4GH format""" + ... diff --git a/pyensemblrest/exceptions.py b/pyensemblrest/exceptions.py index 0bbc8da..75fcda7 100644 --- a/pyensemblrest/exceptions.py +++ b/pyensemblrest/exceptions.py @@ -8,7 +8,7 @@ class EnsemblRestError(Exception): """ Generic error class, catch-all for most EnsemblRest issues. - Special cases are handled by EnsemblRestRateLimitError and EnsemblRestServiceUnavailable. + Special cases are handled by subclasses like EnsemblRestRateLimitError and EnsemblRestServiceUnavailable. """ def __init__( @@ -18,18 +18,19 @@ def __init__( rate_reset: int | None = None, rate_limit: int | None = None, rate_remaining: int | None = None, - retry_after: float | None = None, + retry_after: float | int | None = None, ) -> None: self.error_code = error_code + self.rate_reset = rate_reset + self.rate_limit = rate_limit + self.rate_remaining = rate_remaining + self.retry_after = float(retry_after) if retry_after is not None else None if error_code is not None and error_code in ensembl_http_status_codes: - msg = "EnsEMBL REST API returned a %s (%s): %s" % ( - error_code, - ensembl_http_status_codes[error_code][0], - msg, - ) + status_name = ensembl_http_status_codes[error_code][0] + msg = f"EnsEMBL REST API returned a {error_code} ({status_name}): {msg}" - super(EnsemblRestError, self).__init__(msg) + super().__init__(msg) @property def msg(self) -> Any: @@ -49,17 +50,48 @@ def __init__( rate_reset: int | None = None, rate_limit: int | None = None, rate_remaining: int | None = None, - retry_after: float | None = None, + retry_after: float | int | None = None, ) -> None: - if isinstance(retry_after, float): - msg = "%s (Rate limit hit: Retry after %d seconds)" % (msg, retry_after) + if isinstance(retry_after, (int, float)): + msg = f"{msg} (Rate limit hit: Retry after {int(retry_after)} seconds)" - EnsemblRestError.__init__(self, msg, error_code=error_code) + super().__init__( + msg, + error_code=error_code, + rate_reset=rate_reset, + rate_limit=rate_limit, + rate_remaining=rate_remaining, + retry_after=retry_after, + ) class EnsemblRestServiceUnavailable(EnsemblRestError): """ - Raised when the service is down. + Raised when the service is down or unreachable. + """ + + pass + + +class EnsemblRestTimeoutError(EnsemblRestError): + """ + Raised when a request times out after all retry attempts. + """ + + pass + + +class EnsemblRestNotFoundError(EnsemblRestError): + """ + Raised when a resource or URL is not found (HTTP 404). + """ + + pass + + +class EnsemblRestBadRequestError(EnsemblRestError): + """ + Raised when a bad request is submitted (HTTP 400). """ pass diff --git a/pyensemblrest/py.typed b/pyensemblrest/py.typed new file mode 100644 index 0000000..1242d43 --- /dev/null +++ b/pyensemblrest/py.typed @@ -0,0 +1 @@ +# Marker file for PEP 561. diff --git a/pyproject.toml b/pyproject.toml index ed25eb0..31c7264 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,8 +45,9 @@ style = "semver" [tool.poetry.dependencies] python = ">=3.10,<3.15" -coveralls = "^4.0.2" requests = "^2.32.5" + +[tool.poetry.group.dev.dependencies] mypy = "^1.18.0" ruff = "^0.14.0" pre-commit = "^4.4.0" @@ -56,13 +57,15 @@ types-requests = "^2.32.4.20250913" pytest = "^9.0.0" pytest-cov = "^7.0.0" pytest-rerunfailures = "^16.1" +responses = "^0.25.0" +coveralls = "^4.0.2" [tool.pytest.ini_options] markers = [ "live: tests that run live in CI (deselect with '-m \"not live\"')" ] # Automatically retry flaky live tests to handle API inconsistencies -addopts = "--reruns 2 --reruns-delay 2 --only-rerun AssertionError --only-rerun ConnectionError" +addopts = "--reruns 5 --reruns-delay 5" [tool.mypy] strict = true diff --git a/tests/test_ensemblrest.py b/tests/test_ensemblrest.py index e528b4b..7740e82 100644 --- a/tests/test_ensemblrest.py +++ b/tests/test_ensemblrest.py @@ -39,18 +39,25 @@ WAIT = 1 # Sometimes curl fails -MAX_RETRIES = 5 +MAX_RETRIES = 3 # curl timeouts -TIMEOUT = 60 +TIMEOUT = 90 +# The comparative genomics endpoints (/genetree/, /cafe/genetree/, /homology/) +# and GA4GH feature endpoints (/ga4gh/features/) can be extremely slow to resolve. +# The symbol variants have been measured between 98s and 176s on cold cache, while +# /ga4gh/features/search has been measured at ~270s on cold cache. Give these specific +# tests a 600s budget so they do not time out prematurely. +SLOW_ENDPOINT_TIMEOUT = 600 -def launch(cmd: str) -> str: + +def launch(cmd: str, timeout: int = TIMEOUT) -> str: """Calling a cmd with subprocess""" - # setting curl timeouts - pattern = re.compile("curl") - repl = "curl --connect-timeout %s --max-time %s" % (TIMEOUT, TIMEOUT * 2) + # setting curl timeouts and silent flag + pattern = re.compile(r"^curl\b") + repl = f"curl -s -S --connect-timeout {TIMEOUT} --max-time {timeout}" # Setting curl options cmd = re.sub(pattern, repl, cmd) @@ -66,13 +73,10 @@ def launch(cmd: str) -> str: if len(stderr) > 0: logger.debug(stderr) - # debug - # logger.debug("Got: %s" % (stdout)) - return stdout -def jsonFromCurl(curl_cmd: str) -> dict[Any, Any] | None: +def jsonFromCurl(curl_cmd: str, timeout: int = TIMEOUT) -> dict[Any, Any] | None: """Parsing a JSON curl result""" data = None @@ -83,7 +87,7 @@ def jsonFromCurl(curl_cmd: str) -> dict[Any, Any] | None: retry += 1 # execute the curl cmd - result = launch(curl_cmd) + result = launch(curl_cmd, timeout=timeout) # load it as a dictionary try: @@ -91,7 +95,7 @@ def jsonFromCurl(curl_cmd: str) -> dict[Any, Any] | None: except ValueError as e: logger.warning("Curl command failed: %s" % e) - time.sleep(WAIT * 10) + time.sleep(WAIT * 2) # next request continue @@ -99,7 +103,7 @@ def jsonFromCurl(curl_cmd: str) -> dict[Any, Any] | None: if isinstance(data, dict): if "error" in data: logger.warning("Curl command failed: %s" % (data["error"])) - time.sleep(WAIT * 10) + time.sleep(WAIT * 2) # next request continue @@ -273,12 +277,45 @@ def normalize_single(item: dict[str, Any]) -> dict[str, Any]: return normalize_single(data) +def versionedStableId(stable_id: str) -> str: + """ + Resolve an Ensembl stable ID to its currently versioned form. + + Some endpoints (e.g. /ga4gh/features/{id}) only accept an exact versioned + identifier and reject stale versions with a 400. Since versions are bumped + between Ensembl releases, look the current one up instead of hardcoding it. + + Args: + stable_id: An unversioned Ensembl stable ID (e.g. "ENST00000408937") + + Returns: + The stable ID with its current version appended (e.g. "ENST00000408937.8") + + Raises: + RuntimeError: If the current version could not be resolved + """ + + curl_cmd = ( + f"""curl 'https://rest.ensembl.org/lookup/id/{stable_id}' """ + """-H 'Content-type:application/json'""" + ) + + data = jsonFromCurl(curl_cmd) + + if not isinstance(data, dict) or "version" not in data: + raise RuntimeError( + "Could not resolve the current version for %s: %s" % (stable_id, data) + ) + + return "%s.%s" % (stable_id, data["version"]) + + class EnsemblRest(unittest.TestCase): """A class to test EnsemblRest methods""" def setUp(self) -> None: """Create a EnsemblRest object""" - self.EnsEMBL = pyensemblrest.EnsemblRest() + self.EnsEMBL = pyensemblrest.EnsemblRest(timeout=TIMEOUT, max_attempts=3) def tearDown(self) -> None: """Sleep a while before doing next request""" @@ -365,12 +402,12 @@ def test_wait4request(self) -> None: """Simulating max request per second""" self.EnsEMBL.getArchiveById(id="ENSG00000157764") - self.EnsEMBL.req_count = 15 - self.EnsEMBL.last_req += 2 + # Simulate window expiration + time.sleep(1.1) self.EnsEMBL.getArchiveById(id="ENSG00000157764") - # check request count has reset to zero - self.assertEqual(self.EnsEMBL.req_count, 0) + # check request count reflects active requests in current window + self.assertEqual(self.EnsEMBL.req_count, 1) @pytest.mark.live def test_methodNotImplemented(self) -> None: @@ -556,13 +593,16 @@ class EnsemblRestComparative(EnsemblRest): def test_getCafeGeneTreeById(self) -> None: """Test genetree by id GET method""" + # this endpoint is slow and highly variable, see SLOW_ENDPOINT_TIMEOUT + self.EnsEMBL.timeout = SLOW_ENDPOINT_TIMEOUT + curl_cmd = ( """curl 'https://rest.ensembl.org/cafe/genetree/id/ENSGT00390000003602?' """ """-H 'Content-type:application/json'""" ) # execute the curl cmd an get data as a dictionary - reference = jsonFromCurl(curl_cmd) + reference = jsonFromCurl(curl_cmd, timeout=SLOW_ENDPOINT_TIMEOUT) # execute EnsemblRest function. Dealing with application/json is simpler, # since text/x-phyloxml+xml may change elements order @@ -577,21 +617,22 @@ def test_getCafeGeneTreeById(self) -> None: def test_getCafeGeneTreeMemberBySymbol(self) -> None: """Test genetree by symbol GET method""" + # this endpoint is slow and highly variable, see SLOW_ENDPOINT_TIMEOUT + self.EnsEMBL.timeout = SLOW_ENDPOINT_TIMEOUT + curl_cmd = ( """curl 'https://rest.ensembl.org/cafe/genetree/member/symbol/homo_sapiens/""" - """BRCA2?prune_species=cow;prune_taxon=9526' -H 'Content-type:application/json'""" + """BRCA2?' -H 'Content-type:application/json'""" ) # execute the curl cmd an get data as a dictionary - reference = jsonFromCurl(curl_cmd) + reference = jsonFromCurl(curl_cmd, timeout=SLOW_ENDPOINT_TIMEOUT) # execute EnsemblRest function. Dealing with application/json is simpler, # since text/x-phyloxml+xml may change elements order test = self.EnsEMBL.getCafeGeneTreeMemberBySymbol( - species="human", + species="homo_sapiens", symbol="BRCA2", - prune_species="cow", - prune_taxon=9526, content_type="application/json", ) @@ -602,13 +643,16 @@ def test_getCafeGeneTreeMemberBySymbol(self) -> None: def test_getCafeGeneTreeMemberById(self) -> None: """Test genetree by member id GET method""" + # this endpoint is slow and highly variable, see SLOW_ENDPOINT_TIMEOUT + self.EnsEMBL.timeout = SLOW_ENDPOINT_TIMEOUT + curl_cmd = ( """curl 'https://rest.ensembl.org/cafe/genetree/member/id/""" """homo_sapiens/ENSG00000157764?' -H 'Content-type:application/json'""" ) # execute the curl cmd an get data as a dictionary - reference = jsonFromCurl(curl_cmd) + reference = jsonFromCurl(curl_cmd, timeout=SLOW_ENDPOINT_TIMEOUT) # Execute EnsemblRest function. Dealing with application/json is simpler, # since text/x-phyloxml+xml may change elements order @@ -651,13 +695,16 @@ def test_getGeneTreeById(self) -> None: def test_getGeneTreeMemberBySymbol(self) -> None: """Test genetree by symbol GET method""" + # resolving by symbol is slow, see SLOW_ENDPOINT_TIMEOUT + self.EnsEMBL.timeout = SLOW_ENDPOINT_TIMEOUT + curl_cmd = ( """curl 'https://rest.ensembl.org/genetree/member/symbol/homo_sapiens/""" """BRCA2?prune_species=cow;prune_taxon=9526' -H 'Content-type:application/json'""" ) # execute the curl cmd an get data as a dictionary - reference = jsonFromCurl(curl_cmd) + reference = jsonFromCurl(curl_cmd, timeout=SLOW_ENDPOINT_TIMEOUT) # execute EnsemblRest function. Dealing with application/json is simpler, # since text/x-phyloxml+xml may change elements order @@ -749,10 +796,13 @@ def test_getHomologyById(self) -> None: def test_getHomologyBySymbol(self) -> None: """test get homology by symbol""" + # resolving by symbol is slow, see SLOW_ENDPOINT_TIMEOUT + self.EnsEMBL.timeout = SLOW_ENDPOINT_TIMEOUT + curl_cmd = """curl 'https://rest.ensembl.org/homology/symbol/human/BRCA2?' -H 'Content-type:application/json'""" # execute the curl cmd an get data as a dictionary - reference = jsonFromCurl(curl_cmd) + reference = jsonFromCurl(curl_cmd, timeout=SLOW_ENDPOINT_TIMEOUT) # execute EnsemblRest function. Dealing with application/json is simpler, # since text/x-phyloxml+xml may change elements order @@ -2385,13 +2435,22 @@ def test_postGA4GHBeaconQuery(self) -> None: def test_getGA4GHFeatures(self) -> None: """Testing get GA4GH features GET method""" - curl_cmd = """curl 'https://rest.ensembl.org/ga4gh/features/ENST00000408937.7?' -H 'Content-type:application/json' """ + # this endpoint is slow and highly variable, see SLOW_ENDPOINT_TIMEOUT + self.EnsEMBL.timeout = SLOW_ENDPOINT_TIMEOUT + + # this endpoint needs an exact version, which changes between releases + feature_id = versionedStableId("ENST00000408937") + + curl_cmd = ( + f"""curl 'https://rest.ensembl.org/ga4gh/features/{feature_id}?' """ + """-H 'Content-type:application/json' """ + ) # execute the curl cmd an get data as a dictionary - reference = jsonFromCurl(curl_cmd) + reference = jsonFromCurl(curl_cmd, timeout=SLOW_ENDPOINT_TIMEOUT) # execute EnsemblRest function - test = self.EnsEMBL.getGA4GHFeaturesById(id="ENST00000408937.7") + test = self.EnsEMBL.getGA4GHFeaturesById(id=feature_id) # testing values self.assertTrue(compareNested(reference, test)) @@ -2400,6 +2459,9 @@ def test_getGA4GHFeatures(self) -> None: def test_searchGA4GHFeatures(self) -> None: """Testing GA4GH features search POST method""" + # this endpoint is slow and highly variable, see SLOW_ENDPOINT_TIMEOUT + self.EnsEMBL.timeout = SLOW_ENDPOINT_TIMEOUT + curl_cmd = ( """curl 'https://rest.ensembl.org/ga4gh/features/search' -H 'Content-type:application/json' """ """-H 'Accept:application/json' -X POST -d '{ "start":39657458, "end": 39753127, """ @@ -2407,7 +2469,7 @@ def test_searchGA4GHFeatures(self) -> None: ) # execute the curl cmd an get data as a dictionary - reference = jsonFromCurl(curl_cmd) + reference = jsonFromCurl(curl_cmd, timeout=SLOW_ENDPOINT_TIMEOUT) # execute EnsemblRest function test = self.EnsEMBL.searchGA4GHFeatures( diff --git a/tests/test_unit.py b/tests/test_unit.py new file mode 100644 index 0000000..3c8fa6b --- /dev/null +++ b/tests/test_unit.py @@ -0,0 +1,428 @@ +import json +import time +import unittest +from typing import Any +from unittest.mock import patch + +import requests +import responses + +from pyensemblrest import ( + EnsemblRest, + EnsemblRestBadRequestError, + EnsemblRestError, + EnsemblRestNotFoundError, + EnsemblRestRateLimitError, + EnsemblRestServiceUnavailable, +) +from pyensemblrest.ensembl_config import ( + ensembl_api_table, + ensembl_default_url, + ensembl_user_agent, +) + + +class TestEnsemblRestUnit(unittest.TestCase): + """Offline unit tests for EnsemblRest client with 100% mocked network calls.""" + + def setUp(self) -> None: + self.ens = EnsemblRest() + + def tearDown(self) -> None: + self.ens.close() + + # 1. Initialization and Session Management + def test_init_defaults(self) -> None: + """Test client initialization with default configuration.""" + client = EnsemblRest() + self.assertEqual(client.base_url, ensembl_default_url) + self.assertEqual(client.timeout, 60) + self.assertEqual(client.max_attempts, 5) + self.assertEqual(client.reqs_per_sec, 15) + self.assertEqual(client.wall_time, 1.0) + self.assertEqual(client.session.headers.get("User-Agent"), ensembl_user_agent) + self.assertEqual(client.session.headers.get("Content-Type"), "application/json") + self.assertEqual(client.get_user_agent(), ensembl_user_agent) + client.close() + + def test_init_custom_args(self) -> None: + """Test client initialization with custom parameters.""" + custom_headers = {"User-Agent": "CustomApp/1.0", "X-Custom": "Test"} + client = EnsemblRest( + base_url="https://custom.rest.org/", + timeout=30, + max_attempts=3, + reqs_per_sec=10, + wall_time=2.0, + headers=custom_headers, + ) + self.assertEqual(client.base_url, "https://custom.rest.org/") + self.assertEqual(client.timeout, 30) + self.assertEqual(client.max_attempts, 3) + self.assertEqual(client.reqs_per_sec, 10) + self.assertEqual(client.wall_time, 2.0) + self.assertEqual(client.session.headers.get("User-Agent"), "CustomApp/1.0") + self.assertEqual(client.session.headers.get("X-Custom"), "Test") + client.close() + + def test_context_manager(self) -> None: + """Test context manager __enter__ and __exit__ behavior.""" + with EnsemblRest() as client: + self.assertIsInstance(client, EnsemblRest) + self.assertIsNotNone(client.session) + + def test_dynamic_methods_registration(self) -> None: + """Test that all 106 API methods are registered and listed in dir().""" + dir_methods = dir(self.ens) + for method_name in ensembl_api_table: + self.assertTrue( + hasattr(self.ens, method_name), f"Missing method: {method_name}" + ) + self.assertIn(method_name, dir_methods) + method = getattr(self.ens, method_name) + self.assertTrue(callable(method)) + self.assertEqual(method.__name__, method_name) + + def test_getattr_fallback(self) -> None: + """Test __getattr__ fallback for non-existent attributes.""" + with self.assertRaises(AttributeError): + _ = self.ens.non_existent_method_xyz + + # 2. Parameter Validation and URL Resolution + def test_missing_mandatory_parameter(self) -> None: + """Test that missing mandatory URL parameters raise ValueError.""" + with self.assertRaisesRegex(Exception, "mandatory param 'id' not specified"): + self.ens.getArchiveById() + + def test_mandatory_parameter_with_digit_zero(self) -> None: + """Test regex handles parameters containing digit 0.""" + custom_table = { + "testMethod": { + "url": "/test/{{id0}}/{{param_0}}", + "method": "GET", + "content_type": "application/json", + } + } + client = EnsemblRest(api_table=custom_table) + with self.assertRaisesRegex(Exception, "mandatory param 'id0' not specified"): + client.testMethod(param_0="val0") + client.close() + + def test_url_path_quoting(self) -> None: + """Test URL path interpolation properly quotes characters while preserving colons.""" + resolved = self.ens._resolve_url( + "/sequence/region/{{species}}/{{region}}", + {"species": "homo sapiens", "region": "X:1000..2000:1"}, + ) + self.assertEqual( + resolved, + f"{ensembl_default_url}/sequence/region/homo%20sapiens/X:1000..2000:1", + ) + + def test_base_url_slash_handling(self) -> None: + """Test that trailing slashes on base_url do not produce double slashes.""" + client = EnsemblRest(base_url="https://rest.ensembl.org/") + resolved = client._resolve_url("/archive/id/{{id}}", {"id": "ENSG001"}) + self.assertEqual(resolved, "https://rest.ensembl.org/archive/id/ENSG001") + client.close() + + # 3. HTTP Methods, Payload and Query Parameters + @responses.activate + def test_get_request_json(self) -> None: + """Test successful GET request parsing JSON response.""" + url = f"{ensembl_default_url}/archive/id/ENSG00000157764" + responses.add( + responses.GET, + url, + json={"id": "ENSG00000157764", "latest": "ENSG00000157764.14"}, + status=200, + content_type="application/json", + ) + + res = self.ens.getArchiveById(id="ENSG00000157764") + self.assertEqual(res["id"], "ENSG00000157764") + self.assertEqual(res["latest"], "ENSG00000157764.14") + + @responses.activate + def test_get_request_plain_text(self) -> None: + """Test GET request with non-JSON content type returns raw string.""" + url = f"{ensembl_default_url}/sequence/id/ENSG00000157764" + responses.add( + responses.GET, + url, + body=">ENSG00000157764\nATGCATGC", + status=200, + content_type="text/x-fasta", + ) + + res = self.ens.getSequenceById( + id="ENSG00000157764", content_type="text/x-fasta" + ) + self.assertEqual(res, ">ENSG00000157764\nATGCATGC") + + @responses.activate + def test_post_request_with_body_and_params(self) -> None: + """Test POST request separates post_parameters into JSON body and other kwargs into query params.""" + url = f"{ensembl_default_url}/lookup/id" + responses.add( + responses.POST, + url, + json={ + "ENSG00000157764": {"id": "ENSG00000157764"}, + "ENSG00000248378": {"id": "ENSG00000248378"}, + }, + status=200, + content_type="application/json", + ) + + res = self.ens.getLookupByMultipleIds( + ids=["ENSG00000157764", "ENSG00000248378"], expand=1 + ) + self.assertIn("ENSG00000157764", res) + + # Inspect request + self.assertEqual(len(responses.calls), 1) + req = responses.calls[0].request + self.assertEqual( + json.loads(req.body), {"ids": ["ENSG00000157764", "ENSG00000248378"]} + ) + self.assertIn("expand=1", req.url) + + # 4. Rate Limiting Logic + def test_sliding_window_rate_limiter(self) -> None: + """Test that rate limiter records request timestamps and purges expired ones.""" + client = EnsemblRest(reqs_per_sec=5, wall_time=0.1) + + # Simulate 5 fast calls + for _ in range(5): + client._wait_for_rate_limit() + + self.assertEqual(len(client._request_timestamps), 5) + self.assertEqual(client.req_count, 5) + + # Wait for window to expire + time.sleep(0.12) + client._wait_for_rate_limit() + # Old timestamps purged, now should only have 1 active timestamp + self.assertEqual(len(client._request_timestamps), 1) + client.close() + + # 5. Header Parsing and Rate Limit Metadata + @responses.activate + def test_rate_limit_headers_extraction(self) -> None: + """Test extraction of X-RateLimit-* and Retry-After headers.""" + url = f"{ensembl_default_url}/info/ping" + headers = { + "X-RateLimit-Reset": "45", + "X-RateLimit-Period": "3600", + "X-RateLimit-Limit": "55000", + "X-RateLimit-Remaining": "54990", + "Retry-After": "12.5", + } + responses.add( + responses.GET, + url, + json={"ping": 1}, + status=200, + headers=headers, + ) + + res = self.ens.getInfoPing() + self.assertEqual(res, {"ping": 1}) + self.assertEqual(self.ens.rate_reset, 45) + self.assertEqual(self.ens.rate_period, 3600) + self.assertEqual(self.ens.rate_limit, 55000) + self.assertEqual(self.ens.rate_remaining, 54990) + self.assertEqual(self.ens.retry_after, 12.5) + + # 6. Error Handling and Exceptions + @responses.activate + def test_bad_request_json_error(self) -> None: + """Test HTTP 400 with JSON error message raises EnsemblRestBadRequestError.""" + url = f"{ensembl_default_url}/archive/id/INVALID_ID" + responses.add( + responses.GET, + url, + json={"error": "ID 'INVALID_ID' not found"}, + status=400, + ) + + with self.assertRaises(EnsemblRestBadRequestError) as ctx: + self.ens.getArchiveById(id="INVALID_ID") + + self.assertIn("400 (Bad Request)", str(ctx.exception)) + self.assertIn("ID 'INVALID_ID' not found", str(ctx.exception)) + self.assertEqual(ctx.exception.error_code, 400) + self.assertEqual(ctx.exception.msg, ctx.exception.args[0]) + + @responses.activate + def test_bad_request_html_error_safe_json(self) -> None: + """Test HTTP 400 with HTML/plain-text error safely raises without JSONDecodeError.""" + url = f"{ensembl_default_url}/archive/id/INVALID_ID" + responses.add( + responses.GET, + url, + body="Bad Request", + status=400, + ) + + with self.assertRaises(EnsemblRestBadRequestError) as ctx: + self.ens.getArchiveById(id="INVALID_ID") + + self.assertIn("400 (Bad Request)", str(ctx.exception)) + self.assertEqual(ctx.exception.error_code, 400) + + @responses.activate + def test_not_found_error(self) -> None: + """Test HTTP 404 raises EnsemblRestNotFoundError.""" + url = f"{ensembl_default_url}/archive/id/NOT_FOUND" + responses.add( + responses.GET, + url, + status=404, + ) + + with self.assertRaises(EnsemblRestNotFoundError) as ctx: + self.ens.getArchiveById(id="NOT_FOUND") + + self.assertIn("404 (Not Found)", str(ctx.exception)) + self.assertEqual(ctx.exception.error_code, 404) + + @responses.activate + def test_rate_limit_error_int_and_float_retry_after(self) -> None: + """Test HTTP 429 raises EnsemblRestRateLimitError with formatted retry seconds.""" + url = f"{ensembl_default_url}/info/ping" + responses.add( + responses.GET, + url, + status=429, + headers={"Retry-After": "40"}, + ) + + with self.assertRaises(EnsemblRestRateLimitError) as ctx: + self.ens.getInfoPing() + + self.assertIn("429 (Too Many Requests)", str(ctx.exception)) + self.assertIn("Retry after 40 seconds", str(ctx.exception)) + self.assertEqual(ctx.exception.retry_after, 40.0) + + @responses.activate + def test_service_unavailable_error(self) -> None: + """Test HTTP 503 raises EnsemblRestServiceUnavailable.""" + url = f"{ensembl_default_url}/info/ping" + responses.add( + responses.GET, + url, + status=503, + ) + + with self.assertRaises(EnsemblRestServiceUnavailable) as ctx: + self.ens.getInfoPing() + + self.assertIn("503 (Service Unavailable)", str(ctx.exception)) + self.assertEqual(ctx.exception.error_code, 503) + + @responses.activate + def test_connection_error_raises_service_unavailable(self) -> None: + """Test network connection error raises EnsemblRestServiceUnavailable.""" + url = f"{ensembl_default_url}/info/ping" + responses.add_callback( + responses.GET, + url, + callback=lambda req: (_ for _ in ()).throw( + requests.ConnectionError("Failed to connect") + ), + ) + + with self.assertRaises(EnsemblRestServiceUnavailable): + self.ens.getInfoPing() + + @responses.activate + @patch("time.sleep", return_value=None) + def test_retry_on_500_success(self, mock_sleep: Any) -> None: + """Test automatic retry on transient HTTP 500 error succeeding on 2nd attempt.""" + url = f"{ensembl_default_url}/archive/id/ENSG00000157764" + responses.add(responses.GET, url, status=500) + responses.add( + responses.GET, + url, + json={"id": "ENSG00000157764", "latest": "ENSG00000157764.14"}, + status=200, + ) + + res = self.ens.getArchiveById(id="ENSG00000157764") + self.assertEqual(res["id"], "ENSG00000157764") + self.assertEqual(len(responses.calls), 2) + mock_sleep.assert_called_once() + + @responses.activate + @patch("time.sleep", return_value=None) + def test_retry_max_attempts_exceeded(self, mock_sleep: Any) -> None: + """Test persistent HTTP 500 error raises EnsemblRestError after max retries.""" + self.ens.max_attempts = 2 + url = f"{ensembl_default_url}/archive/id/ENSG00000157764" + for _ in range(3): + responses.add(responses.GET, url, status=500) + + with self.assertRaisesRegex( + EnsemblRestError, "Max number of retries attempts reached" + ): + self.ens.getArchiveById(id="ENSG00000157764") + + self.assertEqual(len(responses.calls), 3) + + @responses.activate + @patch("time.sleep", return_value=None) + def test_known_error_retry(self, mock_sleep: Any) -> None: + """Test automatic retry on Ensembl known error strings in HTTP 400.""" + url = f"{ensembl_default_url}/archive/id/ENSG00000157764" + responses.add( + responses.GET, + url, + json={"error": "something bad has happened"}, + status=400, + ) + responses.add( + responses.GET, + url, + json={"id": "ENSG00000157764", "latest": "ENSG00000157764.14"}, + status=200, + ) + + res = self.ens.getArchiveById(id="ENSG00000157764") + self.assertEqual(res["id"], "ENSG00000157764") + self.assertEqual(len(responses.calls), 2) + + def test_fake_response_object(self) -> None: + """Test FakeResponse object creation and properties.""" + from pyensemblrest.ensemblrest import FakeResponse + + fake = FakeResponse(headers={"X-Test": "1"}, status_code=400, text="error") + self.assertEqual(fake.status_code, 400) + self.assertEqual(fake.text, "error") + self.assertEqual(fake.headers["X-Test"], "1") + + def test_getattr_dynamic_resolution(self) -> None: + """Test resolving an API method dynamically through __getattr__.""" + client = EnsemblRest() + # Remove getArchiveById from __dict__ to force __getattr__ resolution + client.__dict__.pop("getArchiveById", None) + method = client.getArchiveById + self.assertTrue(callable(method)) + self.assertEqual(method.__name__, "getArchiveById") + client.close() + + @patch("requests.Session.get", side_effect=requests.Timeout("Connection timed out")) + @patch("time.sleep", return_value=None) + def test_timeout_retry_and_error(self, mock_sleep: Any, mock_get: Any) -> None: + """Test that requests.Timeout is handled, retried, and raises EnsemblRestError.""" + client = EnsemblRest(max_attempts=2) + with self.assertRaisesRegex( + EnsemblRestError, "Max number of retries attempts reached.*timeout" + ): + client.getArchiveById(id="ENSG00000157764") + client.close() + + +if __name__ == "__main__": + unittest.main()