From 80741eb0a2ec7f0d6d28bf5cc69a5e33d3f735c0 Mon Sep 17 00:00:00 2001 From: Steve Moss Date: Sat, 15 Aug 2026 14:36:46 +0100 Subject: [PATCH 01/14] refactor: apply codebase review fixes, add type stubs, and offline unit tests --- AGENTS.md | 131 ++++++++ CLAUDE.md | 130 +------- poetry.lock | 92 ++++-- pyensemblrest/__init__.py | 25 +- pyensemblrest/ensemblrest.py | 539 +++++++++++++++--------------- pyensemblrest/ensemblrest.pyi | 604 ++++++++++++++++++++++++++++++++++ pyensemblrest/exceptions.py | 58 +++- pyensemblrest/py.typed | 1 + pyproject.toml | 5 +- tests/test_unit.py | 428 ++++++++++++++++++++++++ 10 files changed, 1553 insertions(+), 460 deletions(-) create mode 100644 AGENTS.md create mode 100644 pyensemblrest/ensemblrest.pyi create mode 100644 pyensemblrest/py.typed create mode 100644 tests/test_unit.py diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..34a4d95 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,131 @@ +# AGENTS.md + +This file provides guidance for AI agents when working with code in this repository. + +## Project Overview + +pyEnsemblRest is a Python client library for the Ensembl REST API. It provides a dynamic, Pythonic interface to all Ensembl REST endpoints without requiring explicit method definitions. The library handles rate limiting (15 requests/second), retry logic, and various response formats. + +## Development Commands + +### Setup +```bash +# Install dependencies +make install +# Or with Poetry directly +poetry install --sync +``` + +### Testing +```bash +# Run unit tests (excludes live API tests) +make unit-test +# Or +poetry run pytest -v -m "not live" + +# Run all tests with coverage (used in CI) +make ci-test +# Or +poetry run pytest -v --cov=pyensemblrest --cov-report lcov:./tests/lcov.info tests/ + +# Upload coverage to Coveralls +make coverage +``` + +### Code Quality +```bash +# Type checking +make type-check +# Or +poetry run mypy . --no-incremental + +# Linting +make lint +# Or +poetry run ruff check . --fix + +# Formatting +make format +# Or +poetry run ruff format . + +# Run all pre-commit hooks (except tests) +SKIP=unit-test poetry run pre-commit run --all-files +``` + +### Dependency Management +```bash +# Update lock file +make freeze +# Or +poetry lock +``` + +## Architecture + +### Dynamic Method Registration + +The library uses a dynamic method registration system rather than hard-coded methods. All REST API endpoints are defined in `pyensemblrest/ensembl_config.py` in the `ensembl_api_table` dictionary. Each entry specifies: +- `url`: URL template with `{{parameter}}` placeholders +- `method`: HTTP method (GET or POST) +- `content_type`: Default content type +- `post_parameters`: List of parameters passed in POST body (for POST methods) +- `doc`: Documentation string + +When `EnsemblRest` is instantiated, `__add_methods()` dynamically adds methods to the instance based on `ensembl_api_table`. This allows the library to support all Ensembl REST endpoints without explicit method definitions. + +### Request Flow + +1. User calls a dynamic method (e.g., `ensRest.getSequenceById(id='ENSG00000157764')`) +2. `call_api_func()` validates mandatory parameters from URL template +3. URL is constructed by replacing `{{parameter}}` placeholders +4. `__get_response()` handles rate limiting and executes the request +5. `parseResponse()` processes the response and handles retries if needed + +### Rate Limiting and Retry Logic + +The library implements automatic rate limiting (15 requests/second) in `__get_response()`: +- Tracks request count and timing +- Sleeps if rate limit would be exceeded +- Reads rate limit headers from Ensembl API responses + +Retry logic in `__retry_request()` handles: +- Ensembl known errors (defined in `ensembl_known_errors`) +- HTTP 500 errors (Ensembl sometimes returns 500 on valid requests) +- Timeouts +- Up to `max_attempts` (default 5) with exponential backoff + +### Exception Hierarchy + +- `EnsemblRestError`: Base exception for all REST API errors +- `EnsemblRestRateLimitError`: Raised on HTTP 429 (rate limit hit) +- `EnsemblRestServiceUnavailable`: Raised on connection errors + +### Key Files + +- `pyensemblrest/ensemblrest.py`: Main `EnsemblRest` class with dynamic method registration +- `pyensemblrest/ensembl_config.py`: API endpoint definitions, HTTP status codes, configuration +- `pyensemblrest/exceptions.py`: Custom exception classes +- `pyensemblrest/__init__.py`: Package exports and version metadata + +## Testing Notes + +The test suite uses pytest markers: +- `@pytest.mark.live`: Tests that call the live Ensembl REST API +- By default, `make unit-test` excludes live tests with `-m "not live"` +- CI runs all tests including live API calls + +## Version Management + +This project uses `poetry-dynamic-versioning` to automatically set the version from git tags. The version in `pyproject.toml` is set to `0.0.0` as a placeholder and is replaced at build time with the git tag version. + +## Type Checking + +mypy is configured with strict mode in `pyproject.toml`. The `tests/` directory and `examples.py` are excluded from type checking. + +## Code Style + +The project uses Ruff for both linting and formatting with: +- Line length: 88 characters +- Selected rules: E4, E7, E9, F, I (import sorting) +- Configured in `[tool.ruff]` section of `pyproject.toml` diff --git a/CLAUDE.md b/CLAUDE.md index aecbc58..83f00ce 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,131 +1,3 @@ # CLAUDE.md -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## Project Overview - -pyEnsemblRest is a Python client library for the Ensembl REST API. It provides a dynamic, Pythonic interface to all Ensembl REST endpoints without requiring explicit method definitions. The library handles rate limiting (15 requests/second), retry logic, and various response formats. - -## Development Commands - -### Setup -```bash -# Install dependencies -make install -# Or with Poetry directly -poetry install --sync -``` - -### Testing -```bash -# Run unit tests (excludes live API tests) -make unit-test -# Or -poetry run pytest -v -m "not live" - -# Run all tests with coverage (used in CI) -make ci-test -# Or -poetry run pytest -v --cov=pyensemblrest --cov-report lcov:./tests/lcov.info tests/ - -# Upload coverage to Coveralls -make coverage -``` - -### Code Quality -```bash -# Type checking -make type-check -# Or -poetry run mypy . --no-incremental - -# Linting -make lint -# Or -poetry run ruff check . --fix - -# Formatting -make format -# Or -poetry run ruff format . - -# Run all pre-commit hooks (except tests) -SKIP=unit-test poetry run pre-commit run --all-files -``` - -### Dependency Management -```bash -# Update lock file -make freeze -# Or -poetry lock -``` - -## Architecture - -### Dynamic Method Registration - -The library uses a dynamic method registration system rather than hard-coded methods. All REST API endpoints are defined in `pyensemblrest/ensembl_config.py` in the `ensembl_api_table` dictionary. Each entry specifies: -- `url`: URL template with `{{parameter}}` placeholders -- `method`: HTTP method (GET or POST) -- `content_type`: Default content type -- `post_parameters`: List of parameters passed in POST body (for POST methods) -- `doc`: Documentation string - -When `EnsemblRest` is instantiated, `__add_methods()` dynamically adds methods to the instance based on `ensembl_api_table`. This allows the library to support all Ensembl REST endpoints without explicit method definitions. - -### Request Flow - -1. User calls a dynamic method (e.g., `ensRest.getSequenceById(id='ENSG00000157764')`) -2. `call_api_func()` validates mandatory parameters from URL template -3. URL is constructed by replacing `{{parameter}}` placeholders -4. `__get_response()` handles rate limiting and executes the request -5. `parseResponse()` processes the response and handles retries if needed - -### Rate Limiting and Retry Logic - -The library implements automatic rate limiting (15 requests/second) in `__get_response()`: -- Tracks request count and timing -- Sleeps if rate limit would be exceeded -- Reads rate limit headers from Ensembl API responses - -Retry logic in `__retry_request()` handles: -- Ensembl known errors (defined in `ensembl_known_errors`) -- HTTP 500 errors (Ensembl sometimes returns 500 on valid requests) -- Timeouts -- Up to `max_attempts` (default 5) with exponential backoff - -### Exception Hierarchy - -- `EnsemblRestError`: Base exception for all REST API errors -- `EnsemblRestRateLimitError`: Raised on HTTP 429 (rate limit hit) -- `EnsemblRestServiceUnavailable`: Raised on connection errors - -### Key Files - -- `pyensemblrest/ensemblrest.py`: Main `EnsemblRest` class with dynamic method registration -- `pyensemblrest/ensembl_config.py`: API endpoint definitions, HTTP status codes, configuration -- `pyensemblrest/exceptions.py`: Custom exception classes -- `pyensemblrest/__init__.py`: Package exports and version metadata - -## Testing Notes - -The test suite uses pytest markers: -- `@pytest.mark.live`: Tests that call the live Ensembl REST API -- By default, `make unit-test` excludes live tests with `-m "not live"` -- CI runs all tests including live API calls - -## Version Management - -This project uses `poetry-dynamic-versioning` to automatically set the version from git tags. The version in `pyproject.toml` is set to `0.0.0` as a placeholder and is replaced at build time with the git tag version. - -## Type Checking - -mypy is configured with strict mode in `pyproject.toml`. The `tests/` directory and `examples.py` are excluded from type checking. - -## Code Style - -The project uses Ruff for both linting and formatting with: -- Line length: 88 characters -- Selected rules: E4, E7, E9, F, I (import sorting) -- Configured in `[tool.ruff]` section of `pyproject.toml` +See [AGENTS.md](AGENTS.md) for development guidelines, commands, and repository architecture. diff --git a/poetry.lock b/poetry.lock index 949bbad..9eea363 100644 --- a/poetry.lock +++ b/poetry.lock @@ -6,7 +6,7 @@ version = "0.0.4" description = "Document parameters, class attributes, return types, and variables inline, with Annotated." optional = false python-versions = ">=3.8" -groups = ["main"] +groups = ["test"] files = [ {file = "annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320"}, {file = "annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4"}, @@ -18,7 +18,7 @@ version = "2026.5.20" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" -groups = ["main"] +groups = ["main", "test"] files = [ {file = "certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897"}, {file = "certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d"}, @@ -30,7 +30,7 @@ version = "3.5.0" description = "Validate configuration and produce human readable error messages." optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["dev"] files = [ {file = "cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0"}, {file = "cfgv-3.5.0.tar.gz", hash = "sha256:d5b1034354820651caa73ede66a6294d6e95c1b00acc5e9b098e917404669132"}, @@ -42,7 +42,7 @@ version = "3.4.7" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7" -groups = ["main"] +groups = ["main", "test"] files = [ {file = "charset_normalizer-3.4.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d"}, {file = "charset_normalizer-3.4.7-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e17b8d5d6a8c47c85e68ca8379def1303fd360c3e22093a807cd34a71cd082b8"}, @@ -181,12 +181,12 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -groups = ["main", "test"] +groups = ["test"] +markers = "sys_platform == \"win32\" or platform_system == \"Windows\"" files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -markers = {main = "platform_system == \"Windows\"", test = "sys_platform == \"win32\""} [[package]] name = "coverage" @@ -194,7 +194,7 @@ version = "7.14.1" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.10" -groups = ["main", "test"] +groups = ["test"] files = [ {file = "coverage-7.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3e3680291c4a1d0dadfa84a2c459576a4af5133abb617905714339a0c73138cf"}, {file = "coverage-7.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a5274669f37f2343635a347b91a60777621341ab3378e9c6ac9335eee704bddf"}, @@ -316,7 +316,7 @@ version = "4.1.0" description = "Show coverage stats online via coveralls.io" optional = false python-versions = "<4.0,>=3.10" -groups = ["main"] +groups = ["test"] files = [ {file = "coveralls-4.1.0-py3-none-any.whl", hash = "sha256:bfacfda2d443c24fc90d67035027cec15015fff2dbd036427e8bf8f4953dda2e"}, {file = "coveralls-4.1.0.tar.gz", hash = "sha256:dab364025ba80cbb95ce56c6fc62cd9172d7fd637060ea235dde99d9b46a4494"}, @@ -336,7 +336,7 @@ version = "0.4.0" description = "Distribution utilities" optional = false python-versions = "*" -groups = ["main"] +groups = ["dev"] files = [ {file = "distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16"}, {file = "distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d"}, @@ -367,7 +367,7 @@ version = "3.29.0" description = "A platform independent file lock." optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["dev"] files = [ {file = "filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258"}, {file = "filelock-3.29.0.tar.gz", hash = "sha256:69974355e960702e789734cb4871f884ea6fe50bd8404051a3530bc07809cf90"}, @@ -379,7 +379,7 @@ version = "2.6.19" description = "File identification library for Python" optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["dev"] files = [ {file = "identify-2.6.19-py2.py3-none-any.whl", hash = "sha256:20e6a87f786f768c092a721ad107fc9df0eb89347be9396cadf3f4abbd1fb78a"}, {file = "identify-2.6.19.tar.gz", hash = "sha256:6be5020c38fcb07da56c53733538a3081ea5aa70d36a156f83044bfbf9173842"}, @@ -394,7 +394,7 @@ version = "3.17" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.9" -groups = ["main"] +groups = ["main", "test"] files = [ {file = "idna-3.17-py3-none-any.whl", hash = "sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c"}, {file = "idna-3.17.tar.gz", hash = "sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f"}, @@ -421,7 +421,7 @@ version = "0.11.0" description = "Mypyc runtime library" optional = false python-versions = ">=3.9" -groups = ["main"] +groups = ["dev"] markers = "platform_python_implementation != \"PyPy\"" files = [ {file = "librt-0.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6e94ebfcfa2d5e9926d6c3b9aa4617ffc42a845b4321fb84021b872358c82a0f"}, @@ -522,7 +522,7 @@ version = "4.2.0" description = "Python port of markdown-it. Markdown parsing, done right!" optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["test"] files = [ {file = "markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a"}, {file = "markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49"}, @@ -546,7 +546,7 @@ version = "0.1.2" description = "Markdown URL utilities" optional = false python-versions = ">=3.7" -groups = ["main"] +groups = ["test"] files = [ {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, @@ -558,7 +558,7 @@ version = "1.20.2" description = "Optional static typing for Python" optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["dev"] files = [ {file = "mypy-1.20.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cf5a4db6dca263010e2c7bff081c89383c72d187ba2cf4c44759aac970e2f0c4"}, {file = "mypy-1.20.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7b0e817b518bff7facd7f85ea05b643ad8bdcce684cf29784987b0a7c8e1f997"}, @@ -627,7 +627,7 @@ version = "1.1.0" description = "Type system extensions for programs checked with the mypy type checker." optional = false python-versions = ">=3.8" -groups = ["main"] +groups = ["dev"] files = [ {file = "mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505"}, {file = "mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558"}, @@ -639,7 +639,7 @@ version = "1.10.0" description = "Node.js virtual environment builder" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -groups = ["main"] +groups = ["dev"] files = [ {file = "nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827"}, {file = "nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb"}, @@ -663,7 +663,7 @@ version = "1.1.1" description = "Utility library for gitignore style pattern matching of file paths." optional = false python-versions = ">=3.9" -groups = ["main"] +groups = ["dev"] files = [ {file = "pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189"}, {file = "pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a"}, @@ -680,7 +680,7 @@ version = "4.10.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["dev"] files = [ {file = "platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a"}, {file = "platformdirs-4.10.0.tar.gz", hash = "sha256:31e761a6a0ca04faf7353ea759bdba55652be214725111e5aac52dfa29d4bef7"}, @@ -708,7 +708,7 @@ version = "4.6.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["dev"] files = [ {file = "pre_commit-4.6.0-py2.py3-none-any.whl", hash = "sha256:e2cf246f7299edcabcf15f9b0571fdce06058527f0a06535068a86d38089f29b"}, {file = "pre_commit-4.6.0.tar.gz", hash = "sha256:718d2208cef53fdc38206e40524a6d4d9576d103eb16f0fec11c875e7716e9d9"}, @@ -727,7 +727,7 @@ version = "2.20.0" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.9" -groups = ["main", "test"] +groups = ["test"] files = [ {file = "pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176"}, {file = "pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f"}, @@ -802,7 +802,7 @@ version = "1.4.0" description = "Python interpreter discovery" optional = false python-versions = ">=3.8" -groups = ["main"] +groups = ["dev"] files = [ {file = "python_discovery-1.4.0-py3-none-any.whl", hash = "sha256:26ed78d703e234879a66244c7d4114563fb13ec5cd30a2d1357e5fb4850782da"}, {file = "python_discovery-1.4.0.tar.gz", hash = "sha256:eb8bc7daad3c226c147e45bb4e970a1feb1bf4048ee178e6db59e197b8010ce3"}, @@ -822,7 +822,7 @@ version = "6.0.3" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" -groups = ["main"] +groups = ["dev", "test"] files = [ {file = "PyYAML-6.0.3-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c2514fceb77bc5e7a2f7adfaa1feb2fb311607c9cb518dbc378688ec73d8292f"}, {file = "PyYAML-6.0.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c57bb8c96f6d1808c030b1687b9b5fb476abaa47f0db9c0101f5e9f394e97f4"}, @@ -905,7 +905,7 @@ version = "2.34.2" description = "Python HTTP for Humans." optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["main", "test"] files = [ {file = "requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0"}, {file = "requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed"}, @@ -921,13 +921,33 @@ urllib3 = ">=1.26,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<8)"] +[[package]] +name = "responses" +version = "0.25.8" +description = "A utility library for mocking out the `requests` Python library." +optional = false +python-versions = ">=3.8" +groups = ["test"] +files = [ + {file = "responses-0.25.8-py3-none-any.whl", hash = "sha256:0c710af92def29c8352ceadff0c3fe340ace27cf5af1bbe46fb71275bcd2831c"}, + {file = "responses-0.25.8.tar.gz", hash = "sha256:9374d047a575c8f781b94454db5cab590b6029505f488d12899ddb10a4af1cf4"}, +] + +[package.dependencies] +pyyaml = "*" +requests = ">=2.30.0,<3.0" +urllib3 = ">=1.25.10,<3.0" + +[package.extras] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli ; python_version < \"3.11\"", "tomli-w", "types-PyYAML", "types-requests"] + [[package]] name = "rich" version = "15.0.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false python-versions = ">=3.9.0" -groups = ["main"] +groups = ["test"] files = [ {file = "rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb"}, {file = "rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36"}, @@ -946,7 +966,7 @@ version = "0.14.14" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" -groups = ["main"] +groups = ["dev"] files = [ {file = "ruff-0.14.14-py3-none-linux_armv6l.whl", hash = "sha256:7cfe36b56e8489dee8fbc777c61959f60ec0f1f11817e8f2415f429552846aed"}, {file = "ruff-0.14.14-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6006a0082336e7920b9573ef8a7f52eec837add1265cc74e04ea8a4368cd704c"}, @@ -975,7 +995,7 @@ version = "1.5.4" description = "Tool to Detect Surrounding Shell" optional = false python-versions = ">=3.7" -groups = ["main"] +groups = ["test"] files = [ {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, @@ -987,8 +1007,7 @@ version = "2.4.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" -groups = ["main", "test"] -markers = "python_full_version <= \"3.11.0a6\"" +groups = ["dev", "test"] files = [ {file = "tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30"}, {file = "tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a"}, @@ -1038,6 +1057,7 @@ files = [ {file = "tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe"}, {file = "tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f"}, ] +markers = {dev = "python_version == \"3.10\"", test = "python_full_version <= \"3.11.0a6\""} [[package]] name = "typer" @@ -1045,7 +1065,7 @@ version = "0.26.4" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["test"] files = [ {file = "typer-0.26.4-py3-none-any.whl", hash = "sha256:11bfd7b43557137e373c2b10f6967a555f9678a61ed72c808968b011d95534d6"}, {file = "typer-0.26.4.tar.gz", hash = "sha256:25b128964de66c5ea36d5ac82adc579e5e113509b17469edf9f5a4a1864ff2a9"}, @@ -1063,7 +1083,7 @@ version = "2.33.0.20260518" description = "Typing stubs for requests" optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["dev"] files = [ {file = "types_requests-2.33.0.20260518-py3-none-any.whl", hash = "sha256:626d697d1adaaff76e2044dc8c5c051d8f21abc157bdfe204a75558076fe0bf0"}, {file = "types_requests-2.33.0.20260518.tar.gz", hash = "sha256:df7bd3bfe0ca8402dfb841e7d9be714bb5578203283d66d7dc4ef69343449a5e"}, @@ -1078,7 +1098,7 @@ version = "4.15.0" description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" -groups = ["main", "test"] +groups = ["dev", "test"] files = [ {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, @@ -1091,7 +1111,7 @@ version = "2.7.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["main", "dev", "test"] files = [ {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, @@ -1109,7 +1129,7 @@ version = "21.4.1" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.8" -groups = ["main"] +groups = ["dev"] files = [ {file = "virtualenv-21.4.1-py3-none-any.whl", hash = "sha256:caf4ff72d1b4039057f41d8e8466e859513d67c0400d9c6b62c02c9d1ebc3e12"}, {file = "virtualenv-21.4.1.tar.gz", hash = "sha256:2ca543c713b72840ceffd94e9bdedfbd09a661defa1f7f69e5429ad4059442e2"}, @@ -1125,4 +1145,4 @@ typing-extensions = {version = ">=4.13.2", markers = "python_version < \"3.11\"" [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.15" -content-hash = "abc2c75fa8bba868f4691183b8edff7b25194b5242c26db8ace65f3108496e52" +content-hash = "80b4ad5798a64c316704bad0f15e0d588c82939391036e1d31844e4ae08fdf59" diff --git a/pyensemblrest/__init__.py b/pyensemblrest/__init__.py index df54802..1b385d3 100644 --- a/pyensemblrest/__init__.py +++ b/pyensemblrest/__init__.py @@ -4,21 +4,30 @@ __copyright__ = "Copyright (C) 2013-2024, Steve Moss" __credits__ = ["Steve Moss"] __license__ = "GNU GPLv3" -__version__: str = importlib.metadata.version("pyensemblrest") +try: + __version__: str = importlib.metadata.version("pyensemblrest") +except importlib.metadata.PackageNotFoundError: + __version__ = "0.0.0" __maintainer__ = "Steve Moss" __email__ = "gawbul@gmail.com" __status__ = "beta" -__all__ = [ - "EnsemblRest", - "EnsemblRestError", - "EnsemblRestRateLimitError", - "EnsemblRestServiceUnavailable", -] - from .ensemblrest import EnsemblRest from .exceptions import ( + EnsemblRestBadRequestError, EnsemblRestError, + EnsemblRestNotFoundError, EnsemblRestRateLimitError, EnsemblRestServiceUnavailable, + EnsemblRestTimeoutError, ) + +__all__ = [ + "EnsemblRest", + "EnsemblRestBadRequestError", + "EnsemblRestError", + "EnsemblRestNotFoundError", + "EnsemblRestRateLimitError", + "EnsemblRestServiceUnavailable", + "EnsemblRestTimeoutError", +] diff --git a/pyensemblrest/ensemblrest.py b/pyensemblrest/ensemblrest.py index b73d166..aca15c8 100644 --- a/pyensemblrest/ensemblrest.py +++ b/pyensemblrest/ensemblrest.py @@ -1,14 +1,15 @@ +import collections import json import logging import re import time +import urllib.parse from typing import Any import requests from requests import Response from requests.structures import CaseInsensitiveDict -# import ensemblrest modules from .ensembl_config import ( ensembl_api_table, ensembl_content_type, @@ -19,200 +20,186 @@ ensembl_user_agent, ) from .exceptions import ( + EnsemblRestBadRequestError, EnsemblRestError, + EnsemblRestNotFoundError, EnsemblRestRateLimitError, EnsemblRestServiceUnavailable, + EnsemblRestTimeoutError, ) -# Logger instance logger = logging.getLogger(__name__) +PARAM_REGEX = re.compile(r"\{\{(?P[a-zA-Z0-9_]+)\}\}") + + +class FakeResponse: + """Mock Response object for timeout/error simulations.""" -# FakeResponse object -class FakeResponse(object): def __init__( self, headers: CaseInsensitiveDict[str] | dict[str, Any], status_code: int, text: str, - ): + ) -> None: self.headers = headers self.status_code = status_code self.text: str = text -# EnsEMBL REST API object -class EnsemblRest(object): - # class initialisation function +class EnsemblRest: + """ + EnsEMBL REST API Client. + + Provides a dynamic Pythonic interface to all Ensembl REST endpoints. + Handles rate limiting (15 req/s), automatic retries on transient errors, + and multiple response formats. + """ + def __init__( - self, api_table: dict[str, Any] = ensembl_api_table, **kwargs: dict[str, Any] + self, api_table: dict[str, Any] = ensembl_api_table, **kwargs: Any ) -> None: - # read args variable into object as session_args - self.session_args: dict[str, Any] = kwargs or {} + self.api_table = api_table + self.session_args: dict[str, Any] = kwargs.copy() - # In order to rate limit the requests, like https://github.com/Ensembl/ensembl-rest/wiki/Example-Python-Client - self.reqs_per_sec: int = 15 + # Rate limiting configuration (15 requests/second sliding window) + self.reqs_per_sec: int = int(self.session_args.pop("reqs_per_sec", 15)) + self.wall_time: float = float(self.session_args.pop("wall_time", 1.0)) + self._request_timestamps: collections.deque[float] = collections.deque() self.req_count: int = 0 - self.last_req: float = 0 - self.wall_time: int = 1 + self.last_req: float = 0.0 - # get rate limit parameters, if provided + # Rate limit metadata from response headers self.rate_reset: int | None = None self.rate_limit: int | None = None self.rate_remaining: int | None = None self.rate_period: int | None = None self.retry_after: float | None = None - # to record the last parameters used (in order to redo the query with an ensembl known error) + # Tracking state for retrying requests self.last_url: str = "" self.last_headers: CaseInsensitiveDict[str] | dict[str, Any] = {} self.last_params: dict[str, Any] = {} - self.last_data: dict[Any, Any] = {} + self.last_data: Any = {} self.last_method: str = "" self.last_attempt: int = 0 self.last_response: Response | FakeResponse = Response() - # the maximum number of attempts - self.max_attempts: int = 5 + # Request tuning + self.max_attempts: int = int(self.session_args.pop("max_attempts", 5)) + self.timeout: int | float = self.session_args.pop("timeout", 60) - # setting a timeout - self.timeout: int = 60 + # Base URL & Proxies + self.base_url: str = self.session_args.pop("base_url", ensembl_default_url) + proxies: dict[str, str] = self.session_args.pop("proxies", {}) - # set default values if those values are not provided - self.__set_default() - - # setup requests session + # Setup requests session self.session = requests.Session() + setattr(self.session, "base_url", self.base_url) + self.session.proxies.update(proxies) - # update headers - self.__update_headers() - - # add class methods relying api_table - self.__add_methods(api_table) - - def __set_default(self) -> None: - """Set default values""" - - # initialise default values - default_base_url = ensembl_default_url - default_headers = ensembl_header - default_content_type = ensembl_content_type - default_proxies: dict[str, str] = {} - - if "base_url" not in self.session_args: - self.session_args["base_url"] = default_base_url + # Update headers + self._setup_headers() - if "headers" not in self.session_args: - self.session_args["headers"] = default_headers + # Register dynamic API methods + self.__add_methods(self.api_table) - if "User-Agent" not in self.session_args["headers"]: - self.session_args["headers"].update(default_headers) - - if "Content-Type" not in self.session_args["headers"]: - self.session_args["headers"]["Content-Type"] = default_content_type - - if "proxies" not in self.session_args: - self.session_args["proxies"] = default_proxies - - def __update_headers(self) -> None: - """Update headers""" - - # update requests client with arguments - client_args_copy = self.session_args.copy() - for key, val in client_args_copy.items(): - if key in ("base_url", "proxies"): - setattr(self.session, key, val) - self.session_args.pop(key) - - # update headers as already exist within client - self.session.headers.update(self.session_args.pop("headers")) + def _setup_headers(self) -> None: + """Initialize session headers with default User-Agent and Content-Type.""" + headers: dict[str, str] = self.session_args.pop("headers", {}) + merged_headers = ensembl_header.copy() + merged_headers["Content-Type"] = ensembl_content_type + if headers: + merged_headers.update(headers) + self.session.headers.update(merged_headers) def __add_methods(self, api_table: dict[str, Any]) -> None: - """Add methods to class object""" - - # iterate over api_table keys and add key to class namespace - for fun_name in api_table.keys(): - # setattr(self, key, self.register_api_func(key)) - # Not as a class attribute, but a class method - self.__dict__[fun_name] = self.register_api_func(fun_name, api_table) - - # Set __doc__ for generic class method + """Add dynamic API endpoint methods to instance dictionary.""" + for fun_name in api_table: + func = self.register_api_func(fun_name, api_table) if "doc" in api_table[fun_name]: - self.__dict__[fun_name].__doc__ = api_table[fun_name]["doc"] - - # add function name to the class methods - self.__dict__[fun_name].__name__ = fun_name + func.__doc__ = api_table[fun_name]["doc"] + func.__name__ = fun_name + self.__dict__[fun_name] = func + + def __dir__(self) -> list[str]: + """Expose dynamic API methods for dir() and autocompletion tools.""" + return sorted(set(super().__dir__()) | set(self.api_table.keys())) + + def __getattr__(self, name: str) -> Any: + """Fallback to resolve API methods if not already bound.""" + if name in self.api_table: + func = self.register_api_func(name, self.api_table) + if "doc" in self.api_table[name]: + func.__doc__ = self.api_table[name]["doc"] + func.__name__ = name + self.__dict__[name] = func + return func + raise AttributeError( + f"'{self.__class__.__name__}' object has no attribute '{name}'" + ) - # dynamic api registration function def register_api_func(self, api_call: str, api_table: dict[str, Any]) -> Any: + """Create a callable lambda for an API endpoint.""" return lambda **kwargs: self.call_api_func(api_call, api_table, **kwargs) @staticmethod - def __check_params(func: Any, kwargs: Any) -> list[Any]: - """Check for mandatory parameters""" - - # Verify required variables and raise an Exception if needed - mandatory_params = re.findall(r"\{\{(?P[a-zA-Z1-9_]+)\}\}", func["url"]) - + def __check_params(func: dict[str, Any], kwargs: dict[str, Any]) -> list[str]: + """Check that all mandatory template parameters are provided.""" + mandatory_params = PARAM_REGEX.findall(func["url"]) for param in mandatory_params: if param not in kwargs: logger.critical( - "'%s' param not specified. Mandatory params are %s" - % (param, mandatory_params) + f"'{param}' param not specified. Mandatory params are {mandatory_params}" ) - raise Exception("mandatory param '%s' not specified" % param) - else: - logger.debug("Mandatory param %s found" % param) - + raise ValueError(f"mandatory param '{param}' not specified") + logger.debug(f"Mandatory param {param} found") return mandatory_params - # dynamic api call function + def _resolve_url(self, template: str, kwargs: dict[str, Any]) -> str: + """Interpolate path parameters into endpoint URL with proper quoting.""" + + def _replace(match: re.Match[str]) -> str: + key = match.group("m") + val = kwargs.get(key) + return urllib.parse.quote(str(val), safe=":") + + resolved_path = PARAM_REGEX.sub(_replace, template) + base = self.base_url.rstrip("/") + path = resolved_path.lstrip("/") + return f"{base}/{path}" + def call_api_func( - self, api_call: str, api_table: dict[str, Any], **kwargs: dict[str, Any] + self, api_call: str, api_table: dict[str, Any], **kwargs: Any ) -> Any: - # build url from api_table kwargs + """Execute dynamic API call, handling parameter resolution, HTTP method, and parsing.""" func = api_table[api_call] + call_kwargs = kwargs.copy() - # check mandatory params - mandatory_params = self.__check_params(func, kwargs) - - # resolving urls - url = re.sub( - r"\{\{(?P[a-zA-Z1-9_]+)\}\}", - lambda m: "%s" % kwargs.get(m.group(1)), - self.session.base_url + func["url"], # type: ignore[attr-defined] - ) + # Check and validate mandatory params + mandatory_params = self.__check_params(func, call_kwargs) - # debug - logger.debug("Resolved url: '%s'" % url) + # Build resolved URL + url = self._resolve_url(func["url"], call_kwargs) + logger.debug(f"Resolved url: '{url}'") - # Now I have to remove mandatory params from kwargs + # Remove mandatory path parameters from query/body kwargs for param in mandatory_params: - del kwargs[param] - - # Initialize with the ensembl default content type - content_type: str | dict[str, Any] = ensembl_content_type - - # Override content type if it is defined by function - if "content_type" in func: - content_type = func["content_type"] + call_kwargs.pop(param, None) - # Ovveride content type if it is provied when calling function - if "content_type" in kwargs: - content_type = kwargs["content_type"] - del kwargs["content_type"] + # Determine content type + content_type: str = func.get("content_type", ensembl_content_type) + if "content_type" in call_kwargs: + content_type = call_kwargs.pop("content_type") - # check the request type (GET or POST?) + # Handle GET or POST if func["method"] == "GET": logger.debug( - "Submitting a GET request: url = '%s', headers = %s, params = %s" - % (url, {"Content-Type": content_type}, kwargs) + f"Submitting a GET request: url = '{url}', headers = {{'Content-Type': '{content_type}'}}, params = {call_kwargs}" ) - - # record this request self.last_url = url self.last_headers = {"Content-Type": content_type} - self.last_params = kwargs + self.last_params = call_kwargs self.last_data = {} self.last_method = "GET" self.last_attempt = 0 @@ -220,24 +207,17 @@ def call_api_func( resp = self.__get_response() elif func["method"] == "POST": - # in a POST request, separate post parameters from other parameters - data = {} - - # pass key=value in POST data from kwargs - for key in func["post_parameters"]: - if key in kwargs: - data[key] = kwargs[key] - del kwargs[key] + data: dict[str, Any] = {} + for key in func.get("post_parameters", []): + if key in call_kwargs: + data[key] = call_kwargs.pop(key) logger.debug( - "Submitting a POST request: url = '%s', headers = %s, params = %s, data = %s" - % (url, {"Content-Type": content_type}, kwargs, data) + f"Submitting a POST request: url = '{url}', headers = {{'Content-Type': '{content_type}'}}, params = {call_kwargs}, data = {data}" ) - - # record this request self.last_url = url self.last_headers = {"Content-Type": content_type} - self.last_params = kwargs + self.last_params = call_kwargs self.last_data = data self.last_method = "POST" self.last_attempt = 0 @@ -245,41 +225,46 @@ def call_api_func( resp = self.__get_response() else: - raise NotImplementedError( - "Method '%s' not yet implemented" % (func["method"]) - ) + raise NotImplementedError(f"Method '{func['method']}' not yet implemented") - # call response and return content return self.parseResponse(resp, content_type) - # A function to get reponse from ensembl REST api - def __get_response(self) -> Response | FakeResponse: - """Call session get and post method. Return response""" - - # updating last_req time - self.last_req = time.time() - - # Increment the request counter to rate limit requests - self.req_count += 1 - - # Evaluating the numer of request in a second (according to EnsEMBL rest specification) - if self.req_count >= self.reqs_per_sec: - delta = time.time() - self.last_req - - # sleep upto wall_time - if delta < self.wall_time: - to_sleep = self.wall_time - delta - logger.debug("waiting %s" % to_sleep) + def _wait_for_rate_limit(self) -> None: + """Enforce rate limiting using a sliding window deque.""" + now = time.time() + # Purge timestamps outside the sliding window + while ( + self._request_timestamps + and (now - self._request_timestamps[0]) >= self.wall_time + ): + self._request_timestamps.popleft() + + # If window limit is reached, sleep until the oldest request expires + if len(self._request_timestamps) >= self.reqs_per_sec: + oldest = self._request_timestamps[0] + to_sleep = self.wall_time - (now - oldest) + if to_sleep > 0: + logger.debug( + f"Rate limit reached ({self.reqs_per_sec} req/s). Waiting {to_sleep:.4f}s" + ) time.sleep(to_sleep) + now = time.time() + while ( + self._request_timestamps + and (now - self._request_timestamps[0]) >= self.wall_time + ): + self._request_timestamps.popleft() - self.req_count = 0 + self._request_timestamps.append(now) + self.last_req = now + self.req_count = len(self._request_timestamps) - # my response - resp: Response | FakeResponse = Response() + def __get_response(self) -> Response | FakeResponse: + """Perform HTTP request with rate limiting and network exception handling.""" + self._wait_for_rate_limit() - # deal with exceptions + resp: Response | FakeResponse try: - # another request using the correct method if self.last_method == "GET": resp = self.session.get( self.last_url, @@ -288,7 +273,6 @@ def __get_response(self) -> Response | FakeResponse: timeout=self.timeout, ) elif self.last_method == "POST": - # post parameters are load as POST data, other parameters are url parameters as GET requests resp = self.session.post( self.last_url, headers=self.last_headers, @@ -296,40 +280,36 @@ def __get_response(self) -> Response | FakeResponse: params=self.last_params, timeout=self.timeout, ) - # other methods are verifiedby others functions + else: + raise NotImplementedError( + f"Method '{self.last_method}' not yet implemented" + ) except requests.ConnectionError as e: - raise EnsemblRestServiceUnavailable(e) + raise EnsemblRestServiceUnavailable(e) from e except requests.Timeout as e: - logger.error("%s request timeout: %s" % (self.last_method, e)) - - # create a fake response in order to redo the query + logger.error(f"{self.last_method} request timeout: {e}") resp = FakeResponse( - headers=self.last_response.headers, - status_code=400, + headers=getattr(self.last_response, "headers", {}), + status_code=408, text=json.dumps( - {"message": repr(e), "error": "%s timeout" % ensembl_user_agent} + {"message": repr(e), "error": f"{ensembl_user_agent} timeout"} ), ) - # return response return resp - # A function to deal with a generic response def parseResponse( self, resp: Response | FakeResponse, content_type: str | dict[str, Any] = "application/json", ) -> Any: - """Deal with a generic REST response""" - - logger.debug("Got %s" % resp.text) - - # Record response for debug intent + """Process API response, extract rate limits, handle retries, and parse content.""" + logger.debug(f"Got {resp.text}") self.last_response = resp - # Initialize some values. Check if I'm rate limited + # Extract rate limit headers ( self.rate_reset, self.rate_limit, @@ -338,51 +318,64 @@ def parseResponse( self.rate_period, ) = self.__get_rate_limit(resp.headers) - # parse status code + # Check for errors and retries if self.__check_retry(resp): return self.__retry_request() - # Handle content in different way relying on content-type + # Parse response content if content_type == "application/json": - content = json.loads(resp.text) + try: + content = json.loads(resp.text) + except (ValueError, json.JSONDecodeError): + content = resp.text else: - # Default content = resp.text return content def __check_retry(self, resp: Response | FakeResponse) -> bool: - """Parse status code and print warnings. Return True if a retry is needed""" - - # default status code - message = ensembl_http_status_codes[resp.status_code][1] + """Evaluate status codes for retrying or raising appropriate exceptions.""" + status_info = ensembl_http_status_codes.get( + resp.status_code, ("Error", "Unknown HTTP error") + ) + message = status_info[1] - # parse status codes if resp.status_code > 304: - ExceptionType = EnsemblRestError + exception_type = EnsemblRestError - # Try to derive a more useful message than ensembl default message - if resp.status_code == 400: + # Parse JSON error payload if available + try: json_message = json.loads(resp.text) - if "error" in json_message: + if isinstance(json_message, dict) and "error" in json_message: message = json_message["error"] + except (ValueError, json.JSONDecodeError): + pass - # TODO: deal with special cases errors + if resp.status_code == 400: if message in ensembl_known_errors: - # call a function that will re-execute the REST request and then call again parseResponse - # if everithing is ok, a processed content is returned - logger.warning("EnsEMBL REST Service returned: %s" % message) + logger.warning(f"EnsEMBL REST Service returned: {message}") + return True + exception_type = EnsemblRestBadRequestError + + elif resp.status_code == 404: + exception_type = EnsemblRestNotFoundError - # return true if retry needed + elif resp.status_code == 408: + if message in ensembl_known_errors or "timeout" in message: return True + exception_type = EnsemblRestTimeoutError + + elif resp.status_code == 429: + exception_type = EnsemblRestRateLimitError + elif resp.status_code == 500: - # Retrying when we get a 500 error. - # Due to Ensembl's condition on randomly returning 500s on valid requests. + # Retry transient Ensembl 500 errors return True - elif resp.status_code == 429: - ExceptionType = EnsemblRestRateLimitError - raise ExceptionType( + elif resp.status_code == 503: + exception_type = EnsemblRestServiceUnavailable + + raise exception_type( message, error_code=resp.status_code, rate_reset=self.rate_reset, @@ -391,122 +384,122 @@ def __check_retry(self, resp: Response | FakeResponse) -> bool: retry_after=self.retry_after, ) - # return a flag if status is ok return False @staticmethod def __get_rate_limit( headers: CaseInsensitiveDict[str] | dict[str, Any], ) -> tuple[int | None, int | None, int | None, float | None, int | None]: - """Read rate limited attributes""" + """Parse rate limit headers safely.""" + retry_after: float | None = None + rate_reset: int | None = None + rate_limit: int | None = None + rate_remaining: int | None = None + rate_period: int | None = None - # initialize some values - retry_after = None - rate_reset = None - rate_limit = None - rate_remaining = None - rate_period = None + if not headers: + return rate_reset, rate_limit, rate_remaining, retry_after, rate_period - # for semplicity - keys = [key.lower() for key in headers.keys()] + lower_headers = {k.lower(): v for k, v in headers.items()} - if "X-RateLimit-Reset".lower() in keys: - rate_reset = int(headers["X-RateLimit-Reset"]) - logger.debug("X-RateLimit-Reset: %s" % rate_reset) + if "x-ratelimit-reset" in lower_headers: + try: + rate_reset = int(lower_headers["x-ratelimit-reset"]) + logger.debug(f"X-RateLimit-Reset: {rate_reset}") + except (ValueError, TypeError): + pass - if "X-RateLimit-Period".lower() in keys: - rate_period = int(headers["X-RateLimit-Period"]) - logger.debug("X-RateLimit-Period: %s" % rate_period) + if "x-ratelimit-period" in lower_headers: + try: + rate_period = int(lower_headers["x-ratelimit-period"]) + logger.debug(f"X-RateLimit-Period: {rate_period}") + except (ValueError, TypeError): + pass - if "X-RateLimit-Limit".lower() in keys: - rate_limit = int(headers["X-RateLimit-Limit"]) - logger.debug("X-RateLimit-Limit: %s" % rate_limit) + if "x-ratelimit-limit" in lower_headers: + try: + rate_limit = int(lower_headers["x-ratelimit-limit"]) + logger.debug(f"X-RateLimit-Limit: {rate_limit}") + except (ValueError, TypeError): + pass - if "X-RateLimit-Remaining".lower() in keys: - rate_remaining = int(headers["X-RateLimit-Remaining"]) - logger.debug("X-RateLimit-Remaining: %s" % rate_remaining) + if "x-ratelimit-remaining" in lower_headers: + try: + rate_remaining = int(lower_headers["x-ratelimit-remaining"]) + logger.debug(f"X-RateLimit-Remaining: {rate_remaining}") + except (ValueError, TypeError): + pass - if "Retry-After".lower() in keys: - retry_after = float(headers["Retry-After"]) - logger.debug("Retry-After: %s" % retry_after) + if "retry-after" in lower_headers: + try: + retry_after = float(lower_headers["retry-after"]) + logger.debug(f"Retry-After: {retry_after}") + except (ValueError, TypeError): + pass return rate_reset, rate_limit, rate_remaining, retry_after, rate_period def __retry_request(self) -> Any: - """Retry last request in case of failure""" - - # update last attempt + """Retry the last request using exponential backoff.""" self.last_attempt += 1 - # a max of three attempts if self.last_attempt > self.max_attempts: - # default status code - message = ensembl_http_status_codes[self.last_response.status_code][1] + status_code = getattr(self.last_response, "status_code", 500) + status_info = ensembl_http_status_codes.get( + status_code, ("Error", "Unknown HTTP error") + ) + message = status_info[1] - # parse error if possible try: json_message = json.loads(self.last_response.text) - if "error" in json_message: + if isinstance(json_message, dict) and "error" in json_message: message = json_message["error"] - except ValueError: - # In this case we didn't even get a JSON back. + except (ValueError, json.JSONDecodeError, AttributeError): message = "Server returned invalid JSON." raise EnsemblRestError( - "Max number of retries attempts reached. Last message was: %s" - % message, - error_code=self.last_response.status_code, + f"Max number of retries attempts reached. Last message was: {message}", + error_code=status_code, rate_reset=self.rate_reset, rate_limit=self.rate_limit, rate_remaining=self.rate_remaining, retry_after=self.retry_after, ) - # sleep a while. Increment on each attempt to_sleep = (self.wall_time + 1) * self.last_attempt - - logger.debug("Sleeping %s" % to_sleep) + logger.debug( + f"Sleeping {to_sleep}s before retry {self.last_attempt}/{self.max_attempts}" + ) time.sleep(to_sleep) - # another request using the correct method if self.last_method == "GET": - # debug logger.debug( - "Retring last GET request (%s/%s): url = '%s', headers = %s, params = %s" - % ( - self.last_attempt, - self.max_attempts, - self.last_url, - self.last_headers, - self.last_params, - ) + f"Retrying last GET request ({self.last_attempt}/{self.max_attempts}): url = '{self.last_url}'" ) - resp = self.__get_response() - elif self.last_method == "POST": - # debug logger.debug( - "Retring last POST request (%s/%s): url = '%s', headers = %s, params = %s, data = %s" - % ( - self.last_attempt, - self.max_attempts, - self.last_url, - self.last_headers, - self.last_params, - self.last_data, - ) + f"Retrying last POST request ({self.last_attempt}/{self.max_attempts}): url = '{self.last_url}'" ) - resp = self.__get_response() else: raise NotImplementedError( - "Method '%s' not yet implemented" % (self.last_method) + f"Method '{self.last_method}' not yet implemented" ) - # call response and return content - return self.parseResponse(resp, self.last_headers["Content-Type"]) + content_type = self.last_headers.get("Content-Type", ensembl_content_type) + return self.parseResponse(resp, content_type) + + def close(self) -> None: + """Close the underlying HTTP session connection pools.""" + self.session.close() + + def __enter__(self) -> "EnsemblRest": + return self + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + self.close() def get_user_agent(self) -> str: - """Return the pyEnsemblRest user agent""" + """Return the pyEnsemblRest user agent string.""" return ensembl_user_agent diff --git a/pyensemblrest/ensemblrest.pyi b/pyensemblrest/ensemblrest.pyi new file mode 100644 index 0000000..06ba8a8 --- /dev/null +++ b/pyensemblrest/ensemblrest.pyi @@ -0,0 +1,604 @@ +import collections +from typing import Any + +import requests +from requests import Response +from requests.structures import CaseInsensitiveDict + +class FakeResponse: + headers: CaseInsensitiveDict[str] | dict[str, Any] + status_code: int + text: str + def __init__( + self, + headers: CaseInsensitiveDict[str] | dict[str, Any], + status_code: int, + text: str, + ) -> None: ... + +class EnsemblRest: + api_table: dict[str, Any] + session_args: dict[str, Any] + reqs_per_sec: int + wall_time: float + _request_timestamps: collections.deque[float] + req_count: int + last_req: float + rate_reset: int | None + rate_limit: int | None + rate_remaining: int | None + rate_period: int | None + retry_after: float | None + last_url: str + last_headers: CaseInsensitiveDict[str] | dict[str, Any] + last_params: dict[str, Any] + last_data: Any + last_method: str + last_attempt: int + last_response: Response | FakeResponse + max_attempts: int + timeout: int | float + base_url: str + session: requests.Session + + def __init__(self, api_table: dict[str, Any] = ..., **kwargs: Any) -> None: ... + def register_api_func(self, api_call: str, api_table: dict[str, Any]) -> Any: ... + def call_api_func( + self, api_call: str, api_table: dict[str, Any], **kwargs: Any + ) -> Any: ... + def parseResponse( + self, resp: Response | FakeResponse, content_type: str | dict[str, Any] = ... + ) -> Any: ... + def close(self) -> None: ... + def __enter__(self) -> EnsemblRest: ... + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: ... + def get_user_agent(self) -> str: ... + def getArchiveById(self, id: Any, **kwargs: Any) -> Any: + """Uses the given identifier to return its latest version""" + ... + + def getArchiveByMultipleIds(self, id: Any = ..., **kwargs: Any) -> Any: + """Retrieve the latest version for a set of identifiers""" + ... + + def getCafeGeneTreeById(self, id: Any, **kwargs: Any) -> Any: + """Retrieves a cafe tree of the gene tree using the gene tree stable identifier""" + ... + + def getCafeGeneTreeMemberBySymbol( + self, species: Any, symbol: Any, **kwargs: Any + ) -> Any: + """Retrieves the cafe tree of the gene tree that contains the gene identified by a symbol""" + ... + + def getCafeGeneTreeMemberById(self, species: Any, id: Any, **kwargs: Any) -> Any: + """Retrieves the cafe tree of the gene tree that contains the gene / transcript / translation stable identifier in the given species""" + ... + + def getGeneTreeById(self, id: Any, **kwargs: Any) -> Any: + """Retrieves a gene tree for a gene tree stable identifier""" + ... + + def getGeneTreeMemberBySymbol( + self, species: Any, symbol: Any, **kwargs: Any + ) -> Any: + """Retrieves the gene tree that contains the gene identified by a symbol""" + ... + + def getGeneTreeMemberById(self, species: Any, id: Any, **kwargs: Any) -> Any: + """Retrieves the gene tree that contains the gene / transcript / translation stable identifier in the given species""" + ... + + def getAlignmentByRegion(self, species: Any, region: Any, **kwargs: Any) -> Any: + """Retrieves genomic alignments as separate blocks based on a region and species""" + ... + + def getHomologyById(self, species: Any, id: Any, **kwargs: Any) -> Any: + """Retrieves homology information (orthologs) by species and Ensembl gene id""" + ... + + def getHomologyBySymbol(self, species: Any, symbol: Any, **kwargs: Any) -> Any: + """Retrieves homology information (orthologs) by symbol""" + ... + + def getXrefsBySymbol(self, species: Any, symbol: Any, **kwargs: Any) -> Any: + """Looks up an external symbol and returns all Ensembl objects linked to it. This can be a display name for a gene/transcript/translation, a synonym or an externally linked reference. If a gene's transcript is linked to the supplied symbol the service will return both gene and transcript (it supports transient links).""" + ... + + def getXrefsById(self, id: Any, **kwargs: Any) -> Any: + """Perform lookups of Ensembl Identifiers and retrieve their external references in other databases""" + ... + + def getXrefsByName(self, species: Any, name: Any, **kwargs: Any) -> Any: + """Performs a lookup based upon the primary accession or display label of an external reference and returning the information we hold about the entry""" + ... + + def getInfoAnalysis(self, species: Any, **kwargs: Any) -> Any: + """List the names of analyses involved in generating Ensembl data.""" + ... + + def getInfoAssembly(self, species: Any, **kwargs: Any) -> Any: + """List the currently available assemblies for a species, along with toplevel sequences, chromosomes and cytogenetic bands.""" + ... + + def getInfoAssemblyRegion( + self, species: Any, region_name: Any, **kwargs: Any + ) -> Any: + """Returns information about the specified toplevel sequence region for the given species.""" + ... + + def getInfoBiotypes(self, species: Any, **kwargs: Any) -> Any: + """List the functional classifications of gene models that Ensembl associates with a particular species. Useful for restricting the type of genes/transcripts retrieved by other endpoints.""" + ... + + def getInfoBiotypesByGroup( + self, group: Any, object_type: Any, **kwargs: Any + ) -> Any: + """Without argument the list of available biotype groups is returned. With :group argument provided, list the properties of biotypes within that group. Object type (gene or transcript) can be provided for filtering.""" + ... + + def getInfoBiotypesByName(self, name: Any, object_type: Any, **kwargs: Any) -> Any: + """List the properties of biotypes with a given name. Object type (gene or transcript) can be provided for filtering.""" + ... + + def getInfoComparaMethods(self, **kwargs: Any) -> Any: + """List all compara analyses available (an analysis defines the type of comparative data).""" + ... + + def getInfoComparaSpeciesSets(self, methods: Any, **kwargs: Any) -> Any: + """List all collections of species analysed with the specified compara method.""" + ... + + def getInfoComparas(self, **kwargs: Any) -> Any: + """Lists all available comparative genomics databases and their data release. DEPRECATED: use info/genomes/division instead.""" + ... + + def getInfoData(self, **kwargs: Any) -> Any: + """Shows the data releases available on this REST server. May return more than one release (unfrequent non-standard Ensembl configuration).""" + ... + + def getInfoEgVersion(self, **kwargs: Any) -> Any: + """Returns the Ensembl Genomes version of the databases backing this service""" + ... + + def getInfoExternalDbs(self, species: Any, **kwargs: Any) -> Any: + """Lists all available external sources for a species.""" + ... + + def getInfoDivisions(self, **kwargs: Any) -> Any: + """Get list of all Ensembl divisions for which information is available""" + ... + + def getInfoGenomesByName(self, name: Any, **kwargs: Any) -> Any: + """Find information about a given genome""" + ... + + def getInfoGenomesByAccession(self, accession: Any, **kwargs: Any) -> Any: + """Find information about genomes containing a specified INSDC accession""" + ... + + def getInfoGenomesByAssembly(self, assembly_id: Any, **kwargs: Any) -> Any: + """Find information about a genome with a specified assembly""" + ... + + def getInfoGenomesByDivision(self, division: Any, **kwargs: Any) -> Any: + """Find information about all genomes in a given division. May be large for Ensembl Bacteria.""" + ... + + def getInfoGenomesByTaxonomy(self, taxon_name: Any, **kwargs: Any) -> Any: + """Find information about all genomes beneath a given node of the taxonomy""" + ... + + def getInfoPing(self, **kwargs: Any) -> Any: + """Checks if the service is alive.""" + ... + + def getInfoRest(self, **kwargs: Any) -> Any: + """Shows the current version of the Ensembl REST API.""" + ... + + def getInfoSoftware(self, **kwargs: Any) -> Any: + """Shows the current version of the Ensembl API used by the REST server.""" + ... + + def getInfoSpecies(self, **kwargs: Any) -> Any: + """Lists all available species, their aliases, available adaptor groups and data release.""" + ... + + def getInfoVariationBySpecies(self, species: Any, **kwargs: Any) -> Any: + """List the variation sources used in Ensembl for a species.""" + ... + + def getInfoVariationConsequenceTypes(self, **kwargs: Any) -> Any: + """Lists all variant consequence types.""" + ... + + def getInfoVariationPopulationIndividuals( + self, species: Any, population_name: Any, **kwargs: Any + ) -> Any: + """List all individuals for a population from a species""" + ... + + def getInfoVariationPopulations(self, species: Any, **kwargs: Any) -> Any: + """List all populations for a species""" + ... + + def getLdId( + self, species: Any, id: Any, population_name: Any, **kwargs: Any + ) -> Any: + """Computes and returns LD values between the given variant and all other variants in a window centered around the given variant. The window size is set to 500 kb.""" + ... + + def getLdPairwise(self, species: Any, id1: Any, id2: Any, **kwargs: Any) -> Any: + """Computes and returns LD values between the given variants.""" + ... + + def getLdRegion( + self, species: Any, region: Any, population_name: Any, **kwargs: Any + ) -> Any: + """Computes and returns LD values between all pairs of variants in the defined region.""" + ... + + def getLookupById(self, id: Any, **kwargs: Any) -> Any: + """Find the species and database for a single identifier e.g. gene, transcript, protein""" + ... + + def getLookupByMultipleIds(self, ids: Any = ..., **kwargs: Any) -> Any: + """Find the species and database for several identifiers. IDs that are not found are returned with no data.""" + ... + + def getLookupBySymbol(self, species: Any, symbol: Any, **kwargs: Any) -> Any: + """Find the species and database for a symbol in a linked external database""" + ... + + def getLookupByMultipleSymbols( + self, species: Any, symbols: Any = ..., **kwargs: Any + ) -> Any: + """Find the species and database for a set of symbols in a linked external database. Unknown symbols are omitted from the response.""" + ... + + def getMapCdnaToRegion(self, id: Any, region: Any, **kwargs: Any) -> Any: + """Convert from cDNA coordinates to genomic coordinates. Output reflects forward orientation coordinates as returned from the Ensembl API.""" + ... + + def getMapCdsToRegion(self, id: Any, region: Any, **kwargs: Any) -> Any: + """Convert from CDS coordinates to genomic coordinates. Output reflects forward orientation coordinates as returned from the Ensembl API.""" + ... + + def getMapAssemblyOneToTwo( + self, species: Any, asm_one: Any, region: Any, asm_two: Any, **kwargs: Any + ) -> Any: + """Convert the co-ordinates of one assembly to another""" + ... + + def getMapTranslationToRegion(self, id: Any, region: Any, **kwargs: Any) -> Any: + """Convert from protein (translation) coordinates to genomic coordinates. Output reflects forward orientation coordinates as returned from the Ensembl API.""" + ... + + def getAncestorsById(self, id: Any, **kwargs: Any) -> Any: + """Reconstruct the entire ancestry of a term from is_a and part_of relationships""" + ... + + def getAncestorsChartById(self, id: Any, **kwargs: Any) -> Any: + """Reconstruct the entire ancestry of a term from is_a and part_of relationships.""" + ... + + def getDescendantsById(self, id: Any, **kwargs: Any) -> Any: + """Find all the terms descended from a given term. By default searches are conducted within the namespace of the given identifier""" + ... + + def getOntologyById(self, id: Any, **kwargs: Any) -> Any: + """Search for an ontological term by its namespaced identifier""" + ... + + def getOntologyByName(self, name: Any, **kwargs: Any) -> Any: + """Search for a list of ontological terms by their name""" + ... + + def getTaxonomyClassificationById(self, id: Any, **kwargs: Any) -> Any: + """Return the taxonomic classification of a taxon node""" + ... + + def getTaxonomyById(self, id: Any, **kwargs: Any) -> Any: + """Search for a taxonomic term by its identifier or name""" + ... + + def getTaxonomyByName(self, name: Any, **kwargs: Any) -> Any: + """Search for a taxonomic id by a non-scientific name""" + ... + + def getOverlapById(self, id: Any, **kwargs: Any) -> Any: + """Retrieves features (e.g. genes, transcripts, variants and more) that overlap a region defined by the given identifier.""" + ... + + def getOverlapByRegion(self, species: Any, region: Any, **kwargs: Any) -> Any: + """Retrieves features (e.g. genes, transcripts, variants and more) that overlap a given region.""" + ... + + def getOverlapByTranslation(self, id: Any, **kwargs: Any) -> Any: + """Retrieve features related to a specific Translation as described by its stable ID (e.g. domains, variants).""" + ... + + def getPhenotypeByAccession( + self, species: Any, accession: Any, **kwargs: Any + ) -> Any: + """Return phenotype annotations for genomic features given a phenotype ontology accession""" + ... + + def getPhenotypeByGene(self, species: Any, gene: Any, **kwargs: Any) -> Any: + """Return phenotype annotations for a given gene.""" + ... + + def getPhenotypeByRegion(self, species: Any, region: Any, **kwargs: Any) -> Any: + """Return phenotype annotations that overlap a given genomic region.""" + ... + + def getPhenotypeByTerm(self, species: Any, term: Any, **kwargs: Any) -> Any: + """Return phenotype annotations for genomic features given a phenotype ontology term""" + ... + + def getRegulationBindingMatrix( + self, species: Any, binding_matrix: Any, **kwargs: Any + ) -> Any: + """Return the specified binding matrix""" + ... + + def getSequenceById(self, id: Any, **kwargs: Any) -> Any: + """Request multiple types of sequence by stable identifier. Supports feature masking and expand options.""" + ... + + def getSequenceByMultipleIds(self, ids: Any = ..., **kwargs: Any) -> Any: + """Request multiple types of sequence by a stable identifier list.""" + ... + + def getSequenceByRegion(self, species: Any, region: Any, **kwargs: Any) -> Any: + """Returns the genomic sequence of the specified region of the given species. Supports feature masking and expand options.""" + ... + + def getSequenceByMultipleRegions( + self, species: Any, regions: Any = ..., **kwargs: Any + ) -> Any: + """Request multiple types of sequence by a list of regions.""" + ... + + def getTranscriptHaplotypes(self, species: Any, id: Any, **kwargs: Any) -> Any: + """Computes observed transcript haplotype sequences based on phased genotype data""" + ... + + def getVariantConsequencesByHGVSNotation( + self, species: Any, hgvs_notation: Any, **kwargs: Any + ) -> Any: + """Fetch variant consequences based on a HGVS notation""" + ... + + def getVariantConsequencesByMultipleHGVSNotations( + self, species: Any, hgvs_notations: Any = ..., **kwargs: Any + ) -> Any: + """Fetch variant consequences for multiple HGVS notations""" + ... + + def getVariantConsequencesById(self, species: Any, id: Any, **kwargs: Any) -> Any: + """Fetch variant consequences based on a variant identifier""" + ... + + def getVariantConsequencesByMultipleIds( + self, species: Any, ids: Any = ..., **kwargs: Any + ) -> Any: + """Fetch variant consequences for multiple ids""" + ... + + def getVariantConsequencesByRegion( + self, species: Any, region: Any, allele: Any, **kwargs: Any + ) -> Any: + """Fetch variant consequences""" + ... + + def getVariantConsequencesByMultipleRegions( + self, species: Any, variants: Any = ..., **kwargs: Any + ) -> Any: + """Fetch variant consequences for multiple regions""" + ... + + def getVariationRecoderById(self, species: Any, id: Any, **kwargs: Any) -> Any: + """Translate a variant identifier, HGVS notation or genomic SPDI notation to all possible variant IDs, HGVS and genomic SPDI""" + ... + + def getVariationRecoderByMultipleIds( + self, species: Any, ids: Any = ..., **kwargs: Any + ) -> Any: + """Translate a list of variant identifiers, HGVS notations or genomic SPDI notations to all possible variant IDs, HGVS and genomic SPDI""" + ... + + def getVariationById(self, species: Any, id: Any, **kwargs: Any) -> Any: + """Uses a variant identifier (e.g. rsID) to return the variation features including optional genotype, phenotype and population data""" + ... + + def getVariationByPMCID(self, species: Any, pmcid: Any, **kwargs: Any) -> Any: + """Uses a variant identifier (e.g. rsID) to return the variation features including optional genotype, phenotype and population data""" + ... + + def getVariationByPMID(self, species: Any, pmid: Any, **kwargs: Any) -> Any: + """Uses a variant identifier (e.g. rsID) to return the variation features including optional genotype, phenotype and population data""" + ... + + def getVariationByMultipleIds( + self, species: Any, ids: Any = ..., **kwargs: Any + ) -> Any: + """Uses a list of variant identifiers (e.g. rsID) to return the variation features including optional genotype, phenotype and population data""" + ... + + def getGA4GHBeacon(self, **kwargs: Any) -> Any: + """Return Beacon information""" + ... + + def getGA4GHBeaconQuery( + self, + alternateBases: Any, + assemblyId: Any, + referenceBases: Any, + referenceName: Any, + start: Any, + **kwargs: Any, + ) -> Any: + """Return the Beacon response for allele information""" + ... + + def postGA4GHBeaconQuery( + self, + alternateBases: Any = ..., + assemblyId: Any = ..., + end: Any = ..., + referenceBases: Any = ..., + referenceName: Any = ..., + start: Any = ..., + variantType: Any = ..., + **kwargs: Any, + ) -> Any: + """Return the Beacon response for allele information""" + ... + + def getGA4GHFeaturesById(self, id: Any, **kwargs: Any) -> Any: + """Return the GA4GH record for a specific sequence feature given its identifier""" + ... + + def searchGA4GHFeatures( + self, + end: Any = ..., + referenceName: Any = ..., + start: Any = ..., + featureSetId: Any = ..., + parentId: Any = ..., + **kwargs: Any, + ) -> Any: + """Return a list of sequence annotation features in GA4GH format""" + ... + + def searchGA4GHCallset( + self, + variantSetId: Any = ..., + name: Any = ..., + pageToken: Any = ..., + pageSize: Any = ..., + **kwargs: Any, + ) -> Any: + """Return a list of sets of genotype calls for specific samples in GA4GH format""" + ... + + def getGA4GHCallsetById(self, id: Any, **kwargs: Any) -> Any: + """Return the GA4GH record for a specific CallSet given its identifier""" + ... + + def searchGA4GHDatasets( + self, pageToken: Any = ..., pageSize: Any = ..., **kwargs: Any + ) -> Any: + """Return a list of datasets in GA4GH format""" + ... + + def getGA4GHDatasetsById(self, id: Any, **kwargs: Any) -> Any: + """Return the GA4GH record for a specific dataset given its identifier""" + ... + + def searchGA4GHFeaturesets( + self, + datasetId: Any = ..., + pageToken: Any = ..., + pageSize: Any = ..., + **kwargs: Any, + ) -> Any: + """Return a list of feature sets in GA4GH format""" + ... + + def getGA4GHFeaturesetsById(self, id: Any, **kwargs: Any) -> Any: + """Return the GA4GH record for a specific featureSet given its identifier""" + ... + + def getGA4GHVariantsById(self, id: Any, **kwargs: Any) -> Any: + """Return the GA4GH record for a specific variant given its identifier.""" + ... + + def searchGA4GHVariantAnnotations( + self, + variantAnnotationSetId: Any = ..., + effects: Any = ..., + end: Any = ..., + pageSize: Any = ..., + pageToken: Any = ..., + referenceId: Any = ..., + referenceName: Any = ..., + start: Any = ..., + **kwargs: Any, + ) -> Any: + """Return variant annotation information in GA4GH format for a region on a reference sequence""" + ... + + def searchGA4GHVariants( + self, + variantSetId: Any = ..., + callSetIds: Any = ..., + referenceName: Any = ..., + start: Any = ..., + end: Any = ..., + pageToken: Any = ..., + pageSize: Any = ..., + **kwargs: Any, + ) -> Any: + """Return variant call information in GA4GH format for a region on a reference sequence""" + ... + + def searchGA4GHVariantsets( + self, + datasetId: Any = ..., + pageToken: Any = ..., + pageSize: Any = ..., + **kwargs: Any, + ) -> Any: + """Return a list of variant sets in GA4GH format""" + ... + + def getGA4GHVariantsetsById(self, id: Any, **kwargs: Any) -> Any: + """Return the GA4GH record for a specific VariantSet given its identifier""" + ... + + def searchGA4GHReferences( + self, + referenceSetId: Any = ..., + md5checksum: Any = ..., + accession: Any = ..., + pageToken: Any = ..., + pageSize: Any = ..., + **kwargs: Any, + ) -> Any: + """Return a list of reference sequences in GA4GH format""" + ... + + def getGA4GHReferencesById(self, id: Any, **kwargs: Any) -> Any: + """Return data for a specific reference in GA4GH format by id""" + ... + + def searchGA4GHReferencesets( + self, + accession: Any = ..., + pageToken: Any = ..., + pageSize: Any = ..., + **kwargs: Any, + ) -> Any: + """Return a list of reference sets in GA4GH format""" + ... + + def getGA4GHReferencesetsById(self, id: Any, **kwargs: Any) -> Any: + """Return data for a specific reference set in GA4GH format""" + ... + + def searchGA4GHVariantAnnotationsets( + self, + variantSetId: Any = ..., + pageToken: Any = ..., + pageSize: Any = ..., + **kwargs: Any, + ) -> Any: + """Return a list of annotation sets in GA4GH format""" + ... + + def getGA4GHVariantAnnotationsetsById(self, id: Any, **kwargs: Any) -> Any: + """Return meta data for a specific annotation set in GA4GH format""" + ... diff --git a/pyensemblrest/exceptions.py b/pyensemblrest/exceptions.py index 0bbc8da..75fcda7 100644 --- a/pyensemblrest/exceptions.py +++ b/pyensemblrest/exceptions.py @@ -8,7 +8,7 @@ class EnsemblRestError(Exception): """ Generic error class, catch-all for most EnsemblRest issues. - Special cases are handled by EnsemblRestRateLimitError and EnsemblRestServiceUnavailable. + Special cases are handled by subclasses like EnsemblRestRateLimitError and EnsemblRestServiceUnavailable. """ def __init__( @@ -18,18 +18,19 @@ def __init__( rate_reset: int | None = None, rate_limit: int | None = None, rate_remaining: int | None = None, - retry_after: float | None = None, + retry_after: float | int | None = None, ) -> None: self.error_code = error_code + self.rate_reset = rate_reset + self.rate_limit = rate_limit + self.rate_remaining = rate_remaining + self.retry_after = float(retry_after) if retry_after is not None else None if error_code is not None and error_code in ensembl_http_status_codes: - msg = "EnsEMBL REST API returned a %s (%s): %s" % ( - error_code, - ensembl_http_status_codes[error_code][0], - msg, - ) + status_name = ensembl_http_status_codes[error_code][0] + msg = f"EnsEMBL REST API returned a {error_code} ({status_name}): {msg}" - super(EnsemblRestError, self).__init__(msg) + super().__init__(msg) @property def msg(self) -> Any: @@ -49,17 +50,48 @@ def __init__( rate_reset: int | None = None, rate_limit: int | None = None, rate_remaining: int | None = None, - retry_after: float | None = None, + retry_after: float | int | None = None, ) -> None: - if isinstance(retry_after, float): - msg = "%s (Rate limit hit: Retry after %d seconds)" % (msg, retry_after) + if isinstance(retry_after, (int, float)): + msg = f"{msg} (Rate limit hit: Retry after {int(retry_after)} seconds)" - EnsemblRestError.__init__(self, msg, error_code=error_code) + super().__init__( + msg, + error_code=error_code, + rate_reset=rate_reset, + rate_limit=rate_limit, + rate_remaining=rate_remaining, + retry_after=retry_after, + ) class EnsemblRestServiceUnavailable(EnsemblRestError): """ - Raised when the service is down. + Raised when the service is down or unreachable. + """ + + pass + + +class EnsemblRestTimeoutError(EnsemblRestError): + """ + Raised when a request times out after all retry attempts. + """ + + pass + + +class EnsemblRestNotFoundError(EnsemblRestError): + """ + Raised when a resource or URL is not found (HTTP 404). + """ + + pass + + +class EnsemblRestBadRequestError(EnsemblRestError): + """ + Raised when a bad request is submitted (HTTP 400). """ pass diff --git a/pyensemblrest/py.typed b/pyensemblrest/py.typed new file mode 100644 index 0000000..1242d43 --- /dev/null +++ b/pyensemblrest/py.typed @@ -0,0 +1 @@ +# Marker file for PEP 561. diff --git a/pyproject.toml b/pyproject.toml index ed25eb0..bc1769e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,8 +45,9 @@ style = "semver" [tool.poetry.dependencies] python = ">=3.10,<3.15" -coveralls = "^4.0.2" requests = "^2.32.5" + +[tool.poetry.group.dev.dependencies] mypy = "^1.18.0" ruff = "^0.14.0" pre-commit = "^4.4.0" @@ -56,6 +57,8 @@ types-requests = "^2.32.4.20250913" pytest = "^9.0.0" pytest-cov = "^7.0.0" pytest-rerunfailures = "^16.1" +responses = "^0.25.0" +coveralls = "^4.0.2" [tool.pytest.ini_options] markers = [ diff --git a/tests/test_unit.py b/tests/test_unit.py new file mode 100644 index 0000000..3c8fa6b --- /dev/null +++ b/tests/test_unit.py @@ -0,0 +1,428 @@ +import json +import time +import unittest +from typing import Any +from unittest.mock import patch + +import requests +import responses + +from pyensemblrest import ( + EnsemblRest, + EnsemblRestBadRequestError, + EnsemblRestError, + EnsemblRestNotFoundError, + EnsemblRestRateLimitError, + EnsemblRestServiceUnavailable, +) +from pyensemblrest.ensembl_config import ( + ensembl_api_table, + ensembl_default_url, + ensembl_user_agent, +) + + +class TestEnsemblRestUnit(unittest.TestCase): + """Offline unit tests for EnsemblRest client with 100% mocked network calls.""" + + def setUp(self) -> None: + self.ens = EnsemblRest() + + def tearDown(self) -> None: + self.ens.close() + + # 1. Initialization and Session Management + def test_init_defaults(self) -> None: + """Test client initialization with default configuration.""" + client = EnsemblRest() + self.assertEqual(client.base_url, ensembl_default_url) + self.assertEqual(client.timeout, 60) + self.assertEqual(client.max_attempts, 5) + self.assertEqual(client.reqs_per_sec, 15) + self.assertEqual(client.wall_time, 1.0) + self.assertEqual(client.session.headers.get("User-Agent"), ensembl_user_agent) + self.assertEqual(client.session.headers.get("Content-Type"), "application/json") + self.assertEqual(client.get_user_agent(), ensembl_user_agent) + client.close() + + def test_init_custom_args(self) -> None: + """Test client initialization with custom parameters.""" + custom_headers = {"User-Agent": "CustomApp/1.0", "X-Custom": "Test"} + client = EnsemblRest( + base_url="https://custom.rest.org/", + timeout=30, + max_attempts=3, + reqs_per_sec=10, + wall_time=2.0, + headers=custom_headers, + ) + self.assertEqual(client.base_url, "https://custom.rest.org/") + self.assertEqual(client.timeout, 30) + self.assertEqual(client.max_attempts, 3) + self.assertEqual(client.reqs_per_sec, 10) + self.assertEqual(client.wall_time, 2.0) + self.assertEqual(client.session.headers.get("User-Agent"), "CustomApp/1.0") + self.assertEqual(client.session.headers.get("X-Custom"), "Test") + client.close() + + def test_context_manager(self) -> None: + """Test context manager __enter__ and __exit__ behavior.""" + with EnsemblRest() as client: + self.assertIsInstance(client, EnsemblRest) + self.assertIsNotNone(client.session) + + def test_dynamic_methods_registration(self) -> None: + """Test that all 106 API methods are registered and listed in dir().""" + dir_methods = dir(self.ens) + for method_name in ensembl_api_table: + self.assertTrue( + hasattr(self.ens, method_name), f"Missing method: {method_name}" + ) + self.assertIn(method_name, dir_methods) + method = getattr(self.ens, method_name) + self.assertTrue(callable(method)) + self.assertEqual(method.__name__, method_name) + + def test_getattr_fallback(self) -> None: + """Test __getattr__ fallback for non-existent attributes.""" + with self.assertRaises(AttributeError): + _ = self.ens.non_existent_method_xyz + + # 2. Parameter Validation and URL Resolution + def test_missing_mandatory_parameter(self) -> None: + """Test that missing mandatory URL parameters raise ValueError.""" + with self.assertRaisesRegex(Exception, "mandatory param 'id' not specified"): + self.ens.getArchiveById() + + def test_mandatory_parameter_with_digit_zero(self) -> None: + """Test regex handles parameters containing digit 0.""" + custom_table = { + "testMethod": { + "url": "/test/{{id0}}/{{param_0}}", + "method": "GET", + "content_type": "application/json", + } + } + client = EnsemblRest(api_table=custom_table) + with self.assertRaisesRegex(Exception, "mandatory param 'id0' not specified"): + client.testMethod(param_0="val0") + client.close() + + def test_url_path_quoting(self) -> None: + """Test URL path interpolation properly quotes characters while preserving colons.""" + resolved = self.ens._resolve_url( + "/sequence/region/{{species}}/{{region}}", + {"species": "homo sapiens", "region": "X:1000..2000:1"}, + ) + self.assertEqual( + resolved, + f"{ensembl_default_url}/sequence/region/homo%20sapiens/X:1000..2000:1", + ) + + def test_base_url_slash_handling(self) -> None: + """Test that trailing slashes on base_url do not produce double slashes.""" + client = EnsemblRest(base_url="https://rest.ensembl.org/") + resolved = client._resolve_url("/archive/id/{{id}}", {"id": "ENSG001"}) + self.assertEqual(resolved, "https://rest.ensembl.org/archive/id/ENSG001") + client.close() + + # 3. HTTP Methods, Payload and Query Parameters + @responses.activate + def test_get_request_json(self) -> None: + """Test successful GET request parsing JSON response.""" + url = f"{ensembl_default_url}/archive/id/ENSG00000157764" + responses.add( + responses.GET, + url, + json={"id": "ENSG00000157764", "latest": "ENSG00000157764.14"}, + status=200, + content_type="application/json", + ) + + res = self.ens.getArchiveById(id="ENSG00000157764") + self.assertEqual(res["id"], "ENSG00000157764") + self.assertEqual(res["latest"], "ENSG00000157764.14") + + @responses.activate + def test_get_request_plain_text(self) -> None: + """Test GET request with non-JSON content type returns raw string.""" + url = f"{ensembl_default_url}/sequence/id/ENSG00000157764" + responses.add( + responses.GET, + url, + body=">ENSG00000157764\nATGCATGC", + status=200, + content_type="text/x-fasta", + ) + + res = self.ens.getSequenceById( + id="ENSG00000157764", content_type="text/x-fasta" + ) + self.assertEqual(res, ">ENSG00000157764\nATGCATGC") + + @responses.activate + def test_post_request_with_body_and_params(self) -> None: + """Test POST request separates post_parameters into JSON body and other kwargs into query params.""" + url = f"{ensembl_default_url}/lookup/id" + responses.add( + responses.POST, + url, + json={ + "ENSG00000157764": {"id": "ENSG00000157764"}, + "ENSG00000248378": {"id": "ENSG00000248378"}, + }, + status=200, + content_type="application/json", + ) + + res = self.ens.getLookupByMultipleIds( + ids=["ENSG00000157764", "ENSG00000248378"], expand=1 + ) + self.assertIn("ENSG00000157764", res) + + # Inspect request + self.assertEqual(len(responses.calls), 1) + req = responses.calls[0].request + self.assertEqual( + json.loads(req.body), {"ids": ["ENSG00000157764", "ENSG00000248378"]} + ) + self.assertIn("expand=1", req.url) + + # 4. Rate Limiting Logic + def test_sliding_window_rate_limiter(self) -> None: + """Test that rate limiter records request timestamps and purges expired ones.""" + client = EnsemblRest(reqs_per_sec=5, wall_time=0.1) + + # Simulate 5 fast calls + for _ in range(5): + client._wait_for_rate_limit() + + self.assertEqual(len(client._request_timestamps), 5) + self.assertEqual(client.req_count, 5) + + # Wait for window to expire + time.sleep(0.12) + client._wait_for_rate_limit() + # Old timestamps purged, now should only have 1 active timestamp + self.assertEqual(len(client._request_timestamps), 1) + client.close() + + # 5. Header Parsing and Rate Limit Metadata + @responses.activate + def test_rate_limit_headers_extraction(self) -> None: + """Test extraction of X-RateLimit-* and Retry-After headers.""" + url = f"{ensembl_default_url}/info/ping" + headers = { + "X-RateLimit-Reset": "45", + "X-RateLimit-Period": "3600", + "X-RateLimit-Limit": "55000", + "X-RateLimit-Remaining": "54990", + "Retry-After": "12.5", + } + responses.add( + responses.GET, + url, + json={"ping": 1}, + status=200, + headers=headers, + ) + + res = self.ens.getInfoPing() + self.assertEqual(res, {"ping": 1}) + self.assertEqual(self.ens.rate_reset, 45) + self.assertEqual(self.ens.rate_period, 3600) + self.assertEqual(self.ens.rate_limit, 55000) + self.assertEqual(self.ens.rate_remaining, 54990) + self.assertEqual(self.ens.retry_after, 12.5) + + # 6. Error Handling and Exceptions + @responses.activate + def test_bad_request_json_error(self) -> None: + """Test HTTP 400 with JSON error message raises EnsemblRestBadRequestError.""" + url = f"{ensembl_default_url}/archive/id/INVALID_ID" + responses.add( + responses.GET, + url, + json={"error": "ID 'INVALID_ID' not found"}, + status=400, + ) + + with self.assertRaises(EnsemblRestBadRequestError) as ctx: + self.ens.getArchiveById(id="INVALID_ID") + + self.assertIn("400 (Bad Request)", str(ctx.exception)) + self.assertIn("ID 'INVALID_ID' not found", str(ctx.exception)) + self.assertEqual(ctx.exception.error_code, 400) + self.assertEqual(ctx.exception.msg, ctx.exception.args[0]) + + @responses.activate + def test_bad_request_html_error_safe_json(self) -> None: + """Test HTTP 400 with HTML/plain-text error safely raises without JSONDecodeError.""" + url = f"{ensembl_default_url}/archive/id/INVALID_ID" + responses.add( + responses.GET, + url, + body="Bad Request", + status=400, + ) + + with self.assertRaises(EnsemblRestBadRequestError) as ctx: + self.ens.getArchiveById(id="INVALID_ID") + + self.assertIn("400 (Bad Request)", str(ctx.exception)) + self.assertEqual(ctx.exception.error_code, 400) + + @responses.activate + def test_not_found_error(self) -> None: + """Test HTTP 404 raises EnsemblRestNotFoundError.""" + url = f"{ensembl_default_url}/archive/id/NOT_FOUND" + responses.add( + responses.GET, + url, + status=404, + ) + + with self.assertRaises(EnsemblRestNotFoundError) as ctx: + self.ens.getArchiveById(id="NOT_FOUND") + + self.assertIn("404 (Not Found)", str(ctx.exception)) + self.assertEqual(ctx.exception.error_code, 404) + + @responses.activate + def test_rate_limit_error_int_and_float_retry_after(self) -> None: + """Test HTTP 429 raises EnsemblRestRateLimitError with formatted retry seconds.""" + url = f"{ensembl_default_url}/info/ping" + responses.add( + responses.GET, + url, + status=429, + headers={"Retry-After": "40"}, + ) + + with self.assertRaises(EnsemblRestRateLimitError) as ctx: + self.ens.getInfoPing() + + self.assertIn("429 (Too Many Requests)", str(ctx.exception)) + self.assertIn("Retry after 40 seconds", str(ctx.exception)) + self.assertEqual(ctx.exception.retry_after, 40.0) + + @responses.activate + def test_service_unavailable_error(self) -> None: + """Test HTTP 503 raises EnsemblRestServiceUnavailable.""" + url = f"{ensembl_default_url}/info/ping" + responses.add( + responses.GET, + url, + status=503, + ) + + with self.assertRaises(EnsemblRestServiceUnavailable) as ctx: + self.ens.getInfoPing() + + self.assertIn("503 (Service Unavailable)", str(ctx.exception)) + self.assertEqual(ctx.exception.error_code, 503) + + @responses.activate + def test_connection_error_raises_service_unavailable(self) -> None: + """Test network connection error raises EnsemblRestServiceUnavailable.""" + url = f"{ensembl_default_url}/info/ping" + responses.add_callback( + responses.GET, + url, + callback=lambda req: (_ for _ in ()).throw( + requests.ConnectionError("Failed to connect") + ), + ) + + with self.assertRaises(EnsemblRestServiceUnavailable): + self.ens.getInfoPing() + + @responses.activate + @patch("time.sleep", return_value=None) + def test_retry_on_500_success(self, mock_sleep: Any) -> None: + """Test automatic retry on transient HTTP 500 error succeeding on 2nd attempt.""" + url = f"{ensembl_default_url}/archive/id/ENSG00000157764" + responses.add(responses.GET, url, status=500) + responses.add( + responses.GET, + url, + json={"id": "ENSG00000157764", "latest": "ENSG00000157764.14"}, + status=200, + ) + + res = self.ens.getArchiveById(id="ENSG00000157764") + self.assertEqual(res["id"], "ENSG00000157764") + self.assertEqual(len(responses.calls), 2) + mock_sleep.assert_called_once() + + @responses.activate + @patch("time.sleep", return_value=None) + def test_retry_max_attempts_exceeded(self, mock_sleep: Any) -> None: + """Test persistent HTTP 500 error raises EnsemblRestError after max retries.""" + self.ens.max_attempts = 2 + url = f"{ensembl_default_url}/archive/id/ENSG00000157764" + for _ in range(3): + responses.add(responses.GET, url, status=500) + + with self.assertRaisesRegex( + EnsemblRestError, "Max number of retries attempts reached" + ): + self.ens.getArchiveById(id="ENSG00000157764") + + self.assertEqual(len(responses.calls), 3) + + @responses.activate + @patch("time.sleep", return_value=None) + def test_known_error_retry(self, mock_sleep: Any) -> None: + """Test automatic retry on Ensembl known error strings in HTTP 400.""" + url = f"{ensembl_default_url}/archive/id/ENSG00000157764" + responses.add( + responses.GET, + url, + json={"error": "something bad has happened"}, + status=400, + ) + responses.add( + responses.GET, + url, + json={"id": "ENSG00000157764", "latest": "ENSG00000157764.14"}, + status=200, + ) + + res = self.ens.getArchiveById(id="ENSG00000157764") + self.assertEqual(res["id"], "ENSG00000157764") + self.assertEqual(len(responses.calls), 2) + + def test_fake_response_object(self) -> None: + """Test FakeResponse object creation and properties.""" + from pyensemblrest.ensemblrest import FakeResponse + + fake = FakeResponse(headers={"X-Test": "1"}, status_code=400, text="error") + self.assertEqual(fake.status_code, 400) + self.assertEqual(fake.text, "error") + self.assertEqual(fake.headers["X-Test"], "1") + + def test_getattr_dynamic_resolution(self) -> None: + """Test resolving an API method dynamically through __getattr__.""" + client = EnsemblRest() + # Remove getArchiveById from __dict__ to force __getattr__ resolution + client.__dict__.pop("getArchiveById", None) + method = client.getArchiveById + self.assertTrue(callable(method)) + self.assertEqual(method.__name__, "getArchiveById") + client.close() + + @patch("requests.Session.get", side_effect=requests.Timeout("Connection timed out")) + @patch("time.sleep", return_value=None) + def test_timeout_retry_and_error(self, mock_sleep: Any, mock_get: Any) -> None: + """Test that requests.Timeout is handled, retried, and raises EnsemblRestError.""" + client = EnsemblRest(max_attempts=2) + with self.assertRaisesRegex( + EnsemblRestError, "Max number of retries attempts reached.*timeout" + ): + client.getArchiveById(id="ENSG00000157764") + client.close() + + +if __name__ == "__main__": + unittest.main() From 5e07ea69d32af01ef1b9d213f94b36d5aa49b472 Mon Sep 17 00:00:00 2001 From: Steve Moss Date: Sat, 15 Aug 2026 14:45:44 +0100 Subject: [PATCH 02/14] ci: decouple PR workflows, pin GitHub Actions to latest SHAs, and add nightly drift check --- .github/workflows/nightly.yaml | 50 +++++++++++ .github/workflows/pull_request.yaml | 124 ++++++++++++++++++++++++---- .github/workflows/push_tag.yaml | 21 ++--- 3 files changed, 169 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/nightly.yaml diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml new file mode 100644 index 0000000..5adcafa --- /dev/null +++ b/.github/workflows/nightly.yaml @@ -0,0 +1,50 @@ +name: nightly + +on: + schedule: + - cron: '0 3 * * *' # Every night at 03:00 UTC + workflow_dispatch: + +jobs: + api-drift-check: + name: Ensembl Live API Drift Check + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python 3.14 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.14' + + - name: Set Poetry Version + id: set-poetry-version + run: echo "POETRY_VERSION=$(cat .poetry-version)" >> $GITHUB_OUTPUT + + - name: Install Poetry + id: install-poetry-version + uses: snok/install-poetry@a783c322200f0519c7926aa6faa857c4e23e9263 # v1.4.2 + with: + version: ${{ steps.set-poetry-version.outputs.POETRY_VERSION }} + virtualenvs-create: true + virtualenvs-in-project: true + virtualenvs-path: .venv + installer-parallel: true + + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .venv + key: venv-${{ runner.os }}-3.14-nightly-${{ hashFiles('**/poetry.lock') }} + + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: poetry install --no-interaction --no-root + + - name: Install project + run: poetry install --no-interaction + + - name: Run all tests against live Ensembl API + run: poetry run pytest -v tests/ diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index d046c82..4284c03 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -3,27 +3,77 @@ name: pull_request on: pull_request: paths: - - 'pyensemblrest/**.py' - - 'tests/**.py' + - 'pyensemblrest/**' + - 'tests/**' - 'pyproject.toml' - 'poetry.lock' - '.github/workflows/pull_request.yaml' jobs: - test: - name: Run Tests + lint: + name: Code Quality & Type Check + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python 3.14 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.14' + + - name: Set Poetry Version + id: set-poetry-version + run: echo "POETRY_VERSION=$(cat .poetry-version)" >> $GITHUB_OUTPUT + + - name: Install Poetry + id: install-poetry-version + uses: snok/install-poetry@a783c322200f0519c7926aa6faa857c4e23e9263 # v1.4.2 + with: + version: ${{ steps.set-poetry-version.outputs.POETRY_VERSION }} + virtualenvs-create: true + virtualenvs-in-project: true + virtualenvs-path: .venv + installer-parallel: true + + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .venv + key: venv-${{ runner.os }}-3.14-lint-${{ hashFiles('**/poetry.lock') }} + + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: poetry install --no-interaction --no-root + + - name: Install project + run: poetry install --no-interaction + + - name: Run Ruff Linter and Formatter Check + run: | + poetry run ruff check . + poetry run ruff format --check . + + - name: Run Mypy Type Check + run: poetry run mypy . --no-incremental + + - name: Run Poetry Config Check + run: poetry check + + unit-test: + name: Unit Tests (Python ${{ matrix.python-version }}) runs-on: ubuntu-latest - environment: dev strategy: + fail-fast: false matrix: python-version: ['3.10', '3.11', '3.12', '3.13', '3.14'] - timeout-minutes: 60 steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} @@ -33,7 +83,7 @@ jobs: - name: Install Poetry id: install-poetry-version - uses: snok/install-poetry@v1 + uses: snok/install-poetry@a783c322200f0519c7926aa6faa857c4e23e9263 # v1.4.2 with: version: ${{ steps.set-poetry-version.outputs.POETRY_VERSION }} virtualenvs-create: true @@ -43,7 +93,7 @@ jobs: - name: Load cached venv id: cached-poetry-dependencies - uses: actions/cache@v5 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} @@ -55,14 +105,56 @@ jobs: - name: Install project run: poetry install --no-interaction - - name: Run pre-commit hooks, excluding unit tests - run: SKIP=unit-test poetry run pre-commit run --all-files - - - name: Run unit tests with pytest and code coverage with pytest-cov - run: poetry run pytest -v --cov=pyensemblrest --cov-report lcov:./tests/lcov.info + - name: Run offline unit tests with coverage + run: poetry run pytest -v -m "not live" --cov=pyensemblrest --cov-report lcov:./tests/lcov.info tests/ - name: Upload Coverage Results - uses: coverallsapp/github-action@master + if: matrix.python-version == '3.14' + uses: coverallsapp/github-action@8d6379e14d29928660c4ba802d8e85393440b329 # v2.3.8 with: github-token: ${{ secrets.GITHUB_TOKEN }} path-to-lcov: ./tests/lcov.info + + live-test: + name: Live Ensembl API Smoke Test + needs: [unit-test] + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python 3.14 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.14' + + - name: Set Poetry Version + id: set-poetry-version + run: echo "POETRY_VERSION=$(cat .poetry-version)" >> $GITHUB_OUTPUT + + - name: Install Poetry + id: install-poetry-version + uses: snok/install-poetry@a783c322200f0519c7926aa6faa857c4e23e9263 # v1.4.2 + with: + version: ${{ steps.set-poetry-version.outputs.POETRY_VERSION }} + virtualenvs-create: true + virtualenvs-in-project: true + virtualenvs-path: .venv + installer-parallel: true + + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .venv + key: venv-${{ runner.os }}-3.14-live-${{ hashFiles('**/poetry.lock') }} + + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + run: poetry install --no-interaction --no-root + + - name: Install project + run: poetry install --no-interaction + + - name: Run Live Ensembl REST API tests (single runner to respect rate limits) + run: poetry run pytest -v -m "live" tests/ diff --git a/.github/workflows/push_tag.yaml b/.github/workflows/push_tag.yaml index 695e0f6..3645d5b 100644 --- a/.github/workflows/push_tag.yaml +++ b/.github/workflows/push_tag.yaml @@ -18,14 +18,14 @@ jobs: discussions: write steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-tags: true - ref: ${{ github.ref_name}} + ref: ${{ github.ref_name }} - name: Set up Python id: setup-python - uses: actions/setup-python@v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ">=3.10 <3.15" @@ -35,7 +35,7 @@ jobs: - name: Install Poetry id: install-poetry-version - uses: snok/install-poetry@v1 + uses: snok/install-poetry@a783c322200f0519c7926aa6faa857c4e23e9263 # v1.4.2 with: version: ${{ steps.set-poetry-version.outputs.POETRY_VERSION }} virtualenvs-create: true @@ -49,7 +49,7 @@ jobs: - name: Load cached venv id: cached-poetry-dependencies - uses: actions/cache@v5 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} @@ -65,19 +65,20 @@ jobs: run: poetry build - name: Upload artifacts - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: release-dists path: dist/ - name: Create release - uses: softprops/action-gh-release@v3 + uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 if: startsWith(github.ref, 'refs/tags/') with: files: dist/* generate_release_notes: true draft: false prerelease: false + publish: needs: - release @@ -89,14 +90,14 @@ jobs: name: dev steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set Poetry Version id: set-poetry-version run: echo "POETRY_VERSION=$(cat .poetry-version)" >> $GITHUB_OUTPUT - name: Install Poetry - uses: snok/install-poetry@v1 + uses: snok/install-poetry@a783c322200f0519c7926aa6faa857c4e23e9263 # v1.4.2 with: version: ${{ steps.set-poetry-version.outputs.POETRY_VERSION }} @@ -104,7 +105,7 @@ jobs: run: poetry self add poetry-dynamic-versioning[plugin] - name: Retrieve release distributions - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: release-dists path: dist From a0ef2ba3cf44ab5077def65561362b298f79e235 Mon Sep 17 00:00:00 2001 From: Steve Moss Date: Sat, 15 Aug 2026 14:53:47 +0100 Subject: [PATCH 03/14] test: update test_wait4request to verify sliding window rate limiter --- tests/test_ensemblrest.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_ensemblrest.py b/tests/test_ensemblrest.py index e528b4b..075f256 100644 --- a/tests/test_ensemblrest.py +++ b/tests/test_ensemblrest.py @@ -365,12 +365,12 @@ def test_wait4request(self) -> None: """Simulating max request per second""" self.EnsEMBL.getArchiveById(id="ENSG00000157764") - self.EnsEMBL.req_count = 15 - self.EnsEMBL.last_req += 2 + # Simulate window expiration + time.sleep(1.1) self.EnsEMBL.getArchiveById(id="ENSG00000157764") - # check request count has reset to zero - self.assertEqual(self.EnsEMBL.req_count, 0) + # check request count reflects active requests in current window + self.assertEqual(self.EnsEMBL.req_count, 1) @pytest.mark.live def test_methodNotImplemented(self) -> None: From 144aa4ee57bf8d32a0eb2acd50999ca5c0898dfb Mon Sep 17 00:00:00 2001 From: Steve Moss Date: Sat, 15 Aug 2026 14:55:24 +0100 Subject: [PATCH 04/14] ci: add workflow concurrency controls, explicit timeouts, and least privilege permissions --- .github/workflows/nightly.yaml | 8 ++++++++ .github/workflows/pull_request.yaml | 11 +++++++++++ .github/workflows/push_tag.yaml | 12 +++++++++--- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index 5adcafa..82e4090 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -5,10 +5,18 @@ on: - cron: '0 3 * * *' # Every night at 03:00 UTC workflow_dispatch: +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + jobs: api-drift-check: name: Ensembl Live API Drift Check runs-on: ubuntu-latest + timeout-minutes: 30 steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 4284c03..828d01c 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -9,10 +9,19 @@ on: - 'poetry.lock' - '.github/workflows/pull_request.yaml' +# Automatically cancel redundant in-progress workflow runs for new commits on the same PR branch +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + jobs: lint: name: Code Quality & Type Check runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -64,6 +73,7 @@ jobs: unit-test: name: Unit Tests (Python ${{ matrix.python-version }}) runs-on: ubuntu-latest + timeout-minutes: 10 strategy: fail-fast: false matrix: @@ -119,6 +129,7 @@ jobs: name: Live Ensembl API Smoke Test needs: [unit-test] runs-on: ubuntu-latest + timeout-minutes: 20 steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/push_tag.yaml b/.github/workflows/push_tag.yaml index 3645d5b..cb45d32 100644 --- a/.github/workflows/push_tag.yaml +++ b/.github/workflows/push_tag.yaml @@ -5,6 +5,10 @@ on: tags: - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + permissions: contents: read @@ -12,6 +16,7 @@ jobs: release: name: Release to GitHub runs-on: ubuntu-latest + timeout-minutes: 15 environment: dev permissions: contents: write @@ -82,12 +87,13 @@ jobs: publish: needs: - release - permissions: - id-token: write - name: Publish to PyPI runs-on: ubuntu-latest + timeout-minutes: 15 environment: name: dev + permissions: + contents: read + id-token: write steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 From c3c35beba44ac7e0cf8c91d6fef268845df658c7 Mon Sep 17 00:00:00 2001 From: Steve Moss Date: Sat, 15 Aug 2026 15:12:57 +0100 Subject: [PATCH 05/14] ci: allow pytest-rerunfailures to retry any transient live test failure --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bc1769e..767391b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ markers = [ "live: tests that run live in CI (deselect with '-m \"not live\"')" ] # Automatically retry flaky live tests to handle API inconsistencies -addopts = "--reruns 2 --reruns-delay 2 --only-rerun AssertionError --only-rerun ConnectionError" +addopts = "--reruns 3 --reruns-delay 3" [tool.mypy] strict = true From 021d3505fcba91e01fa04c6782d7c132aeb5f47e Mon Sep 17 00:00:00 2001 From: Steve Moss Date: Sat, 15 Aug 2026 15:15:41 +0100 Subject: [PATCH 06/14] test: optimize curl options and retry delays in live test helpers --- tests/test_ensemblrest.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/test_ensemblrest.py b/tests/test_ensemblrest.py index 075f256..2c9afc9 100644 --- a/tests/test_ensemblrest.py +++ b/tests/test_ensemblrest.py @@ -48,9 +48,9 @@ def launch(cmd: str) -> str: """Calling a cmd with subprocess""" - # setting curl timeouts - pattern = re.compile("curl") - repl = "curl --connect-timeout %s --max-time %s" % (TIMEOUT, TIMEOUT * 2) + # setting curl timeouts and silent flag + pattern = re.compile(r"^curl\b") + repl = f"curl -s -S --connect-timeout {TIMEOUT} --max-time {TIMEOUT}" # Setting curl options cmd = re.sub(pattern, repl, cmd) @@ -66,9 +66,6 @@ def launch(cmd: str) -> str: if len(stderr) > 0: logger.debug(stderr) - # debug - # logger.debug("Got: %s" % (stdout)) - return stdout @@ -91,7 +88,7 @@ def jsonFromCurl(curl_cmd: str) -> dict[Any, Any] | None: except ValueError as e: logger.warning("Curl command failed: %s" % e) - time.sleep(WAIT * 10) + time.sleep(WAIT * 2) # next request continue @@ -99,7 +96,7 @@ def jsonFromCurl(curl_cmd: str) -> dict[Any, Any] | None: if isinstance(data, dict): if "error" in data: logger.warning("Curl command failed: %s" % (data["error"])) - time.sleep(WAIT * 10) + time.sleep(WAIT * 2) # next request continue From 190ddab54b208b61dc043f3b8289d8433dff70b7 Mon Sep 17 00:00:00 2001 From: Steve Moss Date: Sat, 15 Aug 2026 15:41:01 +0100 Subject: [PATCH 07/14] ci: increase live test job timeout to 90 minutes --- .github/workflows/nightly.yaml | 2 +- .github/workflows/pull_request.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index 82e4090..8d8b324 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -16,7 +16,7 @@ jobs: api-drift-check: name: Ensembl Live API Drift Check runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 90 steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 828d01c..613f524 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -129,7 +129,7 @@ jobs: name: Live Ensembl API Smoke Test needs: [unit-test] runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 90 steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 From ec905ceae08af4f569e3962c58f45a1f6f2f0e3a Mon Sep 17 00:00:00 2001 From: Steve Moss Date: Sat, 15 Aug 2026 16:07:13 +0100 Subject: [PATCH 08/14] test: optimize live test timeouts and ensure explicit failures on error --- tests/test_ensemblrest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_ensemblrest.py b/tests/test_ensemblrest.py index 2c9afc9..9aa4b30 100644 --- a/tests/test_ensemblrest.py +++ b/tests/test_ensemblrest.py @@ -39,10 +39,10 @@ WAIT = 1 # Sometimes curl fails -MAX_RETRIES = 5 +MAX_RETRIES = 2 # curl timeouts -TIMEOUT = 60 +TIMEOUT = 20 def launch(cmd: str) -> str: @@ -275,7 +275,7 @@ class EnsemblRest(unittest.TestCase): def setUp(self) -> None: """Create a EnsemblRest object""" - self.EnsEMBL = pyensemblrest.EnsemblRest() + self.EnsEMBL = pyensemblrest.EnsemblRest(timeout=20, max_attempts=2) def tearDown(self) -> None: """Sleep a while before doing next request""" From d38fe51cf6d012ff668cd29f223ac68d4d380a8c Mon Sep 17 00:00:00 2001 From: Steve Moss Date: Sat, 15 Aug 2026 16:40:24 +0100 Subject: [PATCH 09/14] ci: configure 5 reruns with 5s delay, 60s live timeout, and standard CAFE test query --- pyproject.toml | 2 +- tests/test_ensemblrest.py | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 767391b..31c7264 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ markers = [ "live: tests that run live in CI (deselect with '-m \"not live\"')" ] # Automatically retry flaky live tests to handle API inconsistencies -addopts = "--reruns 3 --reruns-delay 3" +addopts = "--reruns 5 --reruns-delay 5" [tool.mypy] strict = true diff --git a/tests/test_ensemblrest.py b/tests/test_ensemblrest.py index 9aa4b30..5c8cc93 100644 --- a/tests/test_ensemblrest.py +++ b/tests/test_ensemblrest.py @@ -39,10 +39,10 @@ WAIT = 1 # Sometimes curl fails -MAX_RETRIES = 2 +MAX_RETRIES = 3 # curl timeouts -TIMEOUT = 20 +TIMEOUT = 60 def launch(cmd: str) -> str: @@ -275,7 +275,7 @@ class EnsemblRest(unittest.TestCase): def setUp(self) -> None: """Create a EnsemblRest object""" - self.EnsEMBL = pyensemblrest.EnsemblRest(timeout=20, max_attempts=2) + self.EnsEMBL = pyensemblrest.EnsemblRest(timeout=60, max_attempts=3) def tearDown(self) -> None: """Sleep a while before doing next request""" @@ -576,7 +576,7 @@ def test_getCafeGeneTreeMemberBySymbol(self) -> None: curl_cmd = ( """curl 'https://rest.ensembl.org/cafe/genetree/member/symbol/homo_sapiens/""" - """BRCA2?prune_species=cow;prune_taxon=9526' -H 'Content-type:application/json'""" + """BRCA2?' -H 'Content-type:application/json'""" ) # execute the curl cmd an get data as a dictionary @@ -585,10 +585,8 @@ def test_getCafeGeneTreeMemberBySymbol(self) -> None: # execute EnsemblRest function. Dealing with application/json is simpler, # since text/x-phyloxml+xml may change elements order test = self.EnsEMBL.getCafeGeneTreeMemberBySymbol( - species="human", + species="homo_sapiens", symbol="BRCA2", - prune_species="cow", - prune_taxon=9526, content_type="application/json", ) From 81774daf0b153b087d00c3fdb8ee007b95fa7f5c Mon Sep 17 00:00:00 2001 From: Steve Moss Date: Sat, 22 Aug 2026 11:43:22 +0100 Subject: [PATCH 10/14] test: resolve GA4GH feature transcript version dynamically Ensembl bumped ENST00000408937 (FOXP2-210) from version 7 to 8, and /ga4gh/features/{id} rejects stale versions with a 400. This broke test_getGA4GHFeatures deterministically, so the live-test reruns could not recover it. Add a versionedStableId() helper that looks up the current version via /lookup/id/ and have test_getGA4GHFeatures build its ID at runtime, so both the reference curl and the library call stay in sync across future releases. The helper raises rather than returning an unusable value, to avoid degrading into a vacuous comparison when the lookup fails. Also bump the hardcoded .7 to .8 in the examples.py and README.md snippets so they no longer error when copy-pasted. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 4 ++-- examples.py | 4 ++-- tests/test_ensemblrest.py | 43 +++++++++++++++++++++++++++++++++++++-- 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0389356..a658d20 100644 --- a/README.md +++ b/README.md @@ -527,11 +527,11 @@ print( variantType="DUP", ) ) -print(ensRest.getGA4GHFeaturesById(id="ENST00000408937.7")) +print(ensRest.getGA4GHFeaturesById(id="ENST00000408937.8")) ensRest.timeout = 180 print( ensRest.searchGA4GHFeatures( - parentId="ENST00000408937.7", + parentId="ENST00000408937.8", featureSetId="", featureTypes=["cds"], end=220023, diff --git a/examples.py b/examples.py index aba8ae8..5c514d0 100644 --- a/examples.py +++ b/examples.py @@ -250,11 +250,11 @@ variantType="DUP", ) ) -print(ensRest.getGA4GHFeaturesById(id="ENST00000408937.7")) +print(ensRest.getGA4GHFeaturesById(id="ENST00000408937.8")) ensRest.timeout = 180 print( ensRest.searchGA4GHFeatures( - parentId="ENST00000408937.7", + parentId="ENST00000408937.8", featureSetId="", featureTypes=["cds"], end=220023, diff --git a/tests/test_ensemblrest.py b/tests/test_ensemblrest.py index 5c8cc93..a9732ef 100644 --- a/tests/test_ensemblrest.py +++ b/tests/test_ensemblrest.py @@ -270,6 +270,39 @@ def normalize_single(item: dict[str, Any]) -> dict[str, Any]: return normalize_single(data) +def versionedStableId(stable_id: str) -> str: + """ + Resolve an Ensembl stable ID to its currently versioned form. + + Some endpoints (e.g. /ga4gh/features/{id}) only accept an exact versioned + identifier and reject stale versions with a 400. Since versions are bumped + between Ensembl releases, look the current one up instead of hardcoding it. + + Args: + stable_id: An unversioned Ensembl stable ID (e.g. "ENST00000408937") + + Returns: + The stable ID with its current version appended (e.g. "ENST00000408937.8") + + Raises: + RuntimeError: If the current version could not be resolved + """ + + curl_cmd = ( + f"""curl 'https://rest.ensembl.org/lookup/id/{stable_id}' """ + """-H 'Content-type:application/json'""" + ) + + data = jsonFromCurl(curl_cmd) + + if not isinstance(data, dict) or "version" not in data: + raise RuntimeError( + "Could not resolve the current version for %s: %s" % (stable_id, data) + ) + + return "%s.%s" % (stable_id, data["version"]) + + class EnsemblRest(unittest.TestCase): """A class to test EnsemblRest methods""" @@ -2380,13 +2413,19 @@ def test_postGA4GHBeaconQuery(self) -> None: def test_getGA4GHFeatures(self) -> None: """Testing get GA4GH features GET method""" - curl_cmd = """curl 'https://rest.ensembl.org/ga4gh/features/ENST00000408937.7?' -H 'Content-type:application/json' """ + # this endpoint needs an exact version, which changes between releases + feature_id = versionedStableId("ENST00000408937") + + curl_cmd = ( + f"""curl 'https://rest.ensembl.org/ga4gh/features/{feature_id}?' """ + """-H 'Content-type:application/json' """ + ) # execute the curl cmd an get data as a dictionary reference = jsonFromCurl(curl_cmd) # execute EnsemblRest function - test = self.EnsEMBL.getGA4GHFeaturesById(id="ENST00000408937.7") + test = self.EnsEMBL.getGA4GHFeaturesById(id=feature_id) # testing values self.assertTrue(compareNested(reference, test)) From e01f7bcac7cd71b50720d462a5be091d3e836493 Mon Sep 17 00:00:00 2001 From: Steve Moss Date: Sat, 22 Aug 2026 13:17:04 +0100 Subject: [PATCH 11/14] test: give the slow CAFE endpoints a longer timeout The /cafe/ endpoints return the whole species tree (~400 taxa, ~100KB) and their latency is driven by server load, not payload size: the same request measured between 9s and 122s across samples, with the smallest response being the slowest. The 60s budget lost that race often enough to make the CAFE tests flaky, and the configured reruns could not help because every attempt hit the same wall. Raise the global TIMEOUT to 90s and add a CAFE_TIMEOUT of 180s, applied to all three CAFE tests on both sides of the comparison via a new optional timeout parameter on launch()/jsonFromCurl(). Only --max-time is extended; --connect-timeout stays on the global value since the handshake is not what is slow here. Also derive the library timeout in setUp() from TIMEOUT rather than repeating the literal, so the two cannot drift apart. Note this does not address the intermittent 503s from the same endpoints, which the existing reruns already handle. Co-Authored-By: Claude Opus 5 (1M context) --- tests/test_ensemblrest.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/tests/test_ensemblrest.py b/tests/test_ensemblrest.py index a9732ef..4c69ba6 100644 --- a/tests/test_ensemblrest.py +++ b/tests/test_ensemblrest.py @@ -42,15 +42,20 @@ MAX_RETRIES = 3 # curl timeouts -TIMEOUT = 60 +TIMEOUT = 90 +# The /cafe/ endpoints return the whole species tree (~400 taxa, ~100KB) and +# their latency is driven by server load rather than payload size: the same +# request has been measured anywhere between 9s and 122s. Give them more room. +CAFE_TIMEOUT = 180 -def launch(cmd: str) -> str: + +def launch(cmd: str, timeout: int = TIMEOUT) -> str: """Calling a cmd with subprocess""" # setting curl timeouts and silent flag pattern = re.compile(r"^curl\b") - repl = f"curl -s -S --connect-timeout {TIMEOUT} --max-time {TIMEOUT}" + repl = f"curl -s -S --connect-timeout {TIMEOUT} --max-time {timeout}" # Setting curl options cmd = re.sub(pattern, repl, cmd) @@ -69,7 +74,7 @@ def launch(cmd: str) -> str: return stdout -def jsonFromCurl(curl_cmd: str) -> dict[Any, Any] | None: +def jsonFromCurl(curl_cmd: str, timeout: int = TIMEOUT) -> dict[Any, Any] | None: """Parsing a JSON curl result""" data = None @@ -80,7 +85,7 @@ def jsonFromCurl(curl_cmd: str) -> dict[Any, Any] | None: retry += 1 # execute the curl cmd - result = launch(curl_cmd) + result = launch(curl_cmd, timeout=timeout) # load it as a dictionary try: @@ -308,7 +313,7 @@ class EnsemblRest(unittest.TestCase): def setUp(self) -> None: """Create a EnsemblRest object""" - self.EnsEMBL = pyensemblrest.EnsemblRest(timeout=60, max_attempts=3) + self.EnsEMBL = pyensemblrest.EnsemblRest(timeout=TIMEOUT, max_attempts=3) def tearDown(self) -> None: """Sleep a while before doing next request""" @@ -586,13 +591,16 @@ class EnsemblRestComparative(EnsemblRest): def test_getCafeGeneTreeById(self) -> None: """Test genetree by id GET method""" + # this endpoint is slow and highly variable, see CAFE_TIMEOUT + self.EnsEMBL.timeout = CAFE_TIMEOUT + curl_cmd = ( """curl 'https://rest.ensembl.org/cafe/genetree/id/ENSGT00390000003602?' """ """-H 'Content-type:application/json'""" ) # execute the curl cmd an get data as a dictionary - reference = jsonFromCurl(curl_cmd) + reference = jsonFromCurl(curl_cmd, timeout=CAFE_TIMEOUT) # execute EnsemblRest function. Dealing with application/json is simpler, # since text/x-phyloxml+xml may change elements order @@ -607,13 +615,16 @@ def test_getCafeGeneTreeById(self) -> None: def test_getCafeGeneTreeMemberBySymbol(self) -> None: """Test genetree by symbol GET method""" + # this endpoint is slow and highly variable, see CAFE_TIMEOUT + self.EnsEMBL.timeout = CAFE_TIMEOUT + curl_cmd = ( """curl 'https://rest.ensembl.org/cafe/genetree/member/symbol/homo_sapiens/""" """BRCA2?' -H 'Content-type:application/json'""" ) # execute the curl cmd an get data as a dictionary - reference = jsonFromCurl(curl_cmd) + reference = jsonFromCurl(curl_cmd, timeout=CAFE_TIMEOUT) # execute EnsemblRest function. Dealing with application/json is simpler, # since text/x-phyloxml+xml may change elements order @@ -630,13 +641,16 @@ def test_getCafeGeneTreeMemberBySymbol(self) -> None: def test_getCafeGeneTreeMemberById(self) -> None: """Test genetree by member id GET method""" + # this endpoint is slow and highly variable, see CAFE_TIMEOUT + self.EnsEMBL.timeout = CAFE_TIMEOUT + curl_cmd = ( """curl 'https://rest.ensembl.org/cafe/genetree/member/id/""" """homo_sapiens/ENSG00000157764?' -H 'Content-type:application/json'""" ) # execute the curl cmd an get data as a dictionary - reference = jsonFromCurl(curl_cmd) + reference = jsonFromCurl(curl_cmd, timeout=CAFE_TIMEOUT) # Execute EnsemblRest function. Dealing with application/json is simpler, # since text/x-phyloxml+xml may change elements order From 6e0c34a4c48b15bca5b4dd7b9c59c0cb6887591a Mon Sep 17 00:00:00 2001 From: Steve Moss Date: Sat, 22 Aug 2026 14:01:31 +0100 Subject: [PATCH 12/14] test: extend the slow-endpoint timeout to the genetree symbol lookup test_getGeneTreeMemberBySymbol hits the same wall the CAFE tests did, measuring 118s and 140s against the 90s global budget. Measuring the neighbouring queries shows the previous commit blamed the wrong thing: the cost is symbol resolution, not payload size or pruning. The same BRCA2 tree returns byte-identical content in 5.6s via /genetree/id but takes 176s via /genetree/member/symbol, and a pruned /genetree/member/id query was the fastest of all at 2.3s. Rename CAFE_TIMEOUT to SLOW_ENDPOINT_TIMEOUT since it is not specific to CAFE, raise it to 300s to clear the 176s worst case with margin, and apply it to test_getGeneTreeMemberBySymbol as well. Correct the comment to record what was actually measured. test_getGeneTreeById and test_getGeneTreeMemberById stay on the global timeout, having measured 5.6s and 13.6s. Co-Authored-By: Claude Opus 5 (1M context) --- tests/test_ensemblrest.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/tests/test_ensemblrest.py b/tests/test_ensemblrest.py index 4c69ba6..cc5c985 100644 --- a/tests/test_ensemblrest.py +++ b/tests/test_ensemblrest.py @@ -44,10 +44,13 @@ # curl timeouts TIMEOUT = 90 -# The /cafe/ endpoints return the whole species tree (~400 taxa, ~100KB) and -# their latency is driven by server load rather than payload size: the same -# request has been measured anywhere between 9s and 122s. Give them more room. -CAFE_TIMEOUT = 180 +# The genetree endpoints resolve much more slowly by symbol than by id, and the +# cost is neither the payload nor the pruning: the same BRCA2 tree came back in +# 5.6s via /genetree/id but took 176s via /genetree/member/symbol, and a pruned +# /genetree/member/id query was the fastest of all at 2.3s. The symbol variants +# of /genetree/ and /cafe/genetree/ have been measured between 92s and 176s, so +# give those specific tests a much longer budget than the global one. +SLOW_ENDPOINT_TIMEOUT = 300 def launch(cmd: str, timeout: int = TIMEOUT) -> str: @@ -591,8 +594,8 @@ class EnsemblRestComparative(EnsemblRest): def test_getCafeGeneTreeById(self) -> None: """Test genetree by id GET method""" - # this endpoint is slow and highly variable, see CAFE_TIMEOUT - self.EnsEMBL.timeout = CAFE_TIMEOUT + # this endpoint is slow and highly variable, see SLOW_ENDPOINT_TIMEOUT + self.EnsEMBL.timeout = SLOW_ENDPOINT_TIMEOUT curl_cmd = ( """curl 'https://rest.ensembl.org/cafe/genetree/id/ENSGT00390000003602?' """ @@ -600,7 +603,7 @@ def test_getCafeGeneTreeById(self) -> None: ) # execute the curl cmd an get data as a dictionary - reference = jsonFromCurl(curl_cmd, timeout=CAFE_TIMEOUT) + reference = jsonFromCurl(curl_cmd, timeout=SLOW_ENDPOINT_TIMEOUT) # execute EnsemblRest function. Dealing with application/json is simpler, # since text/x-phyloxml+xml may change elements order @@ -615,8 +618,8 @@ def test_getCafeGeneTreeById(self) -> None: def test_getCafeGeneTreeMemberBySymbol(self) -> None: """Test genetree by symbol GET method""" - # this endpoint is slow and highly variable, see CAFE_TIMEOUT - self.EnsEMBL.timeout = CAFE_TIMEOUT + # this endpoint is slow and highly variable, see SLOW_ENDPOINT_TIMEOUT + self.EnsEMBL.timeout = SLOW_ENDPOINT_TIMEOUT curl_cmd = ( """curl 'https://rest.ensembl.org/cafe/genetree/member/symbol/homo_sapiens/""" @@ -624,7 +627,7 @@ def test_getCafeGeneTreeMemberBySymbol(self) -> None: ) # execute the curl cmd an get data as a dictionary - reference = jsonFromCurl(curl_cmd, timeout=CAFE_TIMEOUT) + reference = jsonFromCurl(curl_cmd, timeout=SLOW_ENDPOINT_TIMEOUT) # execute EnsemblRest function. Dealing with application/json is simpler, # since text/x-phyloxml+xml may change elements order @@ -641,8 +644,8 @@ def test_getCafeGeneTreeMemberBySymbol(self) -> None: def test_getCafeGeneTreeMemberById(self) -> None: """Test genetree by member id GET method""" - # this endpoint is slow and highly variable, see CAFE_TIMEOUT - self.EnsEMBL.timeout = CAFE_TIMEOUT + # this endpoint is slow and highly variable, see SLOW_ENDPOINT_TIMEOUT + self.EnsEMBL.timeout = SLOW_ENDPOINT_TIMEOUT curl_cmd = ( """curl 'https://rest.ensembl.org/cafe/genetree/member/id/""" @@ -650,7 +653,7 @@ def test_getCafeGeneTreeMemberById(self) -> None: ) # execute the curl cmd an get data as a dictionary - reference = jsonFromCurl(curl_cmd, timeout=CAFE_TIMEOUT) + reference = jsonFromCurl(curl_cmd, timeout=SLOW_ENDPOINT_TIMEOUT) # Execute EnsemblRest function. Dealing with application/json is simpler, # since text/x-phyloxml+xml may change elements order @@ -693,13 +696,16 @@ def test_getGeneTreeById(self) -> None: def test_getGeneTreeMemberBySymbol(self) -> None: """Test genetree by symbol GET method""" + # resolving by symbol is slow, see SLOW_ENDPOINT_TIMEOUT + self.EnsEMBL.timeout = SLOW_ENDPOINT_TIMEOUT + curl_cmd = ( """curl 'https://rest.ensembl.org/genetree/member/symbol/homo_sapiens/""" """BRCA2?prune_species=cow;prune_taxon=9526' -H 'Content-type:application/json'""" ) # execute the curl cmd an get data as a dictionary - reference = jsonFromCurl(curl_cmd) + reference = jsonFromCurl(curl_cmd, timeout=SLOW_ENDPOINT_TIMEOUT) # execute EnsemblRest function. Dealing with application/json is simpler, # since text/x-phyloxml+xml may change elements order From 0f0333f29e8d2805705154806ff6ef58ab9dd577 Mon Sep 17 00:00:00 2001 From: Steve Moss Date: Sat, 22 Aug 2026 15:43:24 +0100 Subject: [PATCH 13/14] ci: raise live job timeout to 120m and extend the homology symbol test The live job timed out at 90 minutes having reached 80%, so raise both live jobs to 120 minutes. The nightly drift check runs the same suite against the same API and would have hit the identical wall. test_getHomologyBySymbol is the last test over the global budget, measuring 104s and 112s on a cold cache, so give it the extended timeout as well. Correct the SLOW_ENDPOINT_TIMEOUT comment. Re-measuring showed these responses are cached by Ensembl, which made some earlier readings look fast on a second sample; with untouched genes as controls the symbol cost holds up, a 929KB homology query by id taking 10.5s against 104s for a smaller 372KB one by symbol. The rest of the suite's symbol endpoints are cheap and stay on the global timeout: /xrefs/symbol and /lookup/symbol both measured around 0.1s. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/nightly.yaml | 2 +- .github/workflows/pull_request.yaml | 2 +- tests/test_ensemblrest.py | 19 ++++++++++++------- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index 8d8b324..b5e8fc6 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -16,7 +16,7 @@ jobs: api-drift-check: name: Ensembl Live API Drift Check runs-on: ubuntu-latest - timeout-minutes: 90 + timeout-minutes: 120 steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 613f524..fafe6ea 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -129,7 +129,7 @@ jobs: name: Live Ensembl API Smoke Test needs: [unit-test] runs-on: ubuntu-latest - timeout-minutes: 90 + timeout-minutes: 120 steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/tests/test_ensemblrest.py b/tests/test_ensemblrest.py index cc5c985..5caa738 100644 --- a/tests/test_ensemblrest.py +++ b/tests/test_ensemblrest.py @@ -44,12 +44,14 @@ # curl timeouts TIMEOUT = 90 -# The genetree endpoints resolve much more slowly by symbol than by id, and the -# cost is neither the payload nor the pruning: the same BRCA2 tree came back in -# 5.6s via /genetree/id but took 176s via /genetree/member/symbol, and a pruned -# /genetree/member/id query was the fastest of all at 2.3s. The symbol variants -# of /genetree/ and /cafe/genetree/ have been measured between 92s and 176s, so -# give those specific tests a much longer budget than the global one. +# The comparative genomics endpoints (/genetree/, /cafe/genetree/, /homology/) +# are far more expensive to resolve by symbol than by id, and the cost is the +# symbol resolution rather than the payload: an untouched BRCA1 homology query +# returned 929KB by id in 10.5s, while an untouched TP53 one returned 372KB by +# symbol in 104s. The symbol variants have been measured between 98s and 176s +# on a cold cache; a repeat of the same URL is served from Ensembl's cache in +# ~0.15s, so each test pays this once. Their id-based counterparts stay on the +# global timeout, having measured between 2s and 14s cold. SLOW_ENDPOINT_TIMEOUT = 300 @@ -797,10 +799,13 @@ def test_getHomologyById(self) -> None: def test_getHomologyBySymbol(self) -> None: """test get homology by symbol""" + # resolving by symbol is slow, see SLOW_ENDPOINT_TIMEOUT + self.EnsEMBL.timeout = SLOW_ENDPOINT_TIMEOUT + curl_cmd = """curl 'https://rest.ensembl.org/homology/symbol/human/BRCA2?' -H 'Content-type:application/json'""" # execute the curl cmd an get data as a dictionary - reference = jsonFromCurl(curl_cmd) + reference = jsonFromCurl(curl_cmd, timeout=SLOW_ENDPOINT_TIMEOUT) # execute EnsemblRest function. Dealing with application/json is simpler, # since text/x-phyloxml+xml may change elements order From b7f6e47e7ff7a8e0bfc7b93676e368f607c19761 Mon Sep 17 00:00:00 2001 From: Steve Moss Date: Sat, 22 Aug 2026 21:01:53 +0100 Subject: [PATCH 14/14] test: extend slow-endpoint timeout to GA4GH feature endpoints and increase to 600s --- tests/test_ensemblrest.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/test_ensemblrest.py b/tests/test_ensemblrest.py index 5caa738..7740e82 100644 --- a/tests/test_ensemblrest.py +++ b/tests/test_ensemblrest.py @@ -45,14 +45,11 @@ TIMEOUT = 90 # The comparative genomics endpoints (/genetree/, /cafe/genetree/, /homology/) -# are far more expensive to resolve by symbol than by id, and the cost is the -# symbol resolution rather than the payload: an untouched BRCA1 homology query -# returned 929KB by id in 10.5s, while an untouched TP53 one returned 372KB by -# symbol in 104s. The symbol variants have been measured between 98s and 176s -# on a cold cache; a repeat of the same URL is served from Ensembl's cache in -# ~0.15s, so each test pays this once. Their id-based counterparts stay on the -# global timeout, having measured between 2s and 14s cold. -SLOW_ENDPOINT_TIMEOUT = 300 +# and GA4GH feature endpoints (/ga4gh/features/) can be extremely slow to resolve. +# The symbol variants have been measured between 98s and 176s on cold cache, while +# /ga4gh/features/search has been measured at ~270s on cold cache. Give these specific +# tests a 600s budget so they do not time out prematurely. +SLOW_ENDPOINT_TIMEOUT = 600 def launch(cmd: str, timeout: int = TIMEOUT) -> str: @@ -2438,6 +2435,9 @@ def test_postGA4GHBeaconQuery(self) -> None: def test_getGA4GHFeatures(self) -> None: """Testing get GA4GH features GET method""" + # this endpoint is slow and highly variable, see SLOW_ENDPOINT_TIMEOUT + self.EnsEMBL.timeout = SLOW_ENDPOINT_TIMEOUT + # this endpoint needs an exact version, which changes between releases feature_id = versionedStableId("ENST00000408937") @@ -2447,7 +2447,7 @@ def test_getGA4GHFeatures(self) -> None: ) # execute the curl cmd an get data as a dictionary - reference = jsonFromCurl(curl_cmd) + reference = jsonFromCurl(curl_cmd, timeout=SLOW_ENDPOINT_TIMEOUT) # execute EnsemblRest function test = self.EnsEMBL.getGA4GHFeaturesById(id=feature_id) @@ -2459,6 +2459,9 @@ def test_getGA4GHFeatures(self) -> None: def test_searchGA4GHFeatures(self) -> None: """Testing GA4GH features search POST method""" + # this endpoint is slow and highly variable, see SLOW_ENDPOINT_TIMEOUT + self.EnsEMBL.timeout = SLOW_ENDPOINT_TIMEOUT + curl_cmd = ( """curl 'https://rest.ensembl.org/ga4gh/features/search' -H 'Content-type:application/json' """ """-H 'Accept:application/json' -X POST -d '{ "start":39657458, "end": 39753127, """ @@ -2466,7 +2469,7 @@ def test_searchGA4GHFeatures(self) -> None: ) # execute the curl cmd an get data as a dictionary - reference = jsonFromCurl(curl_cmd) + reference = jsonFromCurl(curl_cmd, timeout=SLOW_ENDPOINT_TIMEOUT) # execute EnsemblRest function test = self.EnsEMBL.searchGA4GHFeatures(