From f761b38c8871559dcf02b1d11c672cb51d31a7c8 Mon Sep 17 00:00:00 2001 From: Elarwei Date: Wed, 24 Jun 2026 22:48:31 +0800 Subject: [PATCH 1/6] feat(ref): support fetching GENCODE GTFs and FASTAs (#73) Add a `source` argument to `gget ref` (Python API and CLI) to select the reference database: "ensembl" (default, unchanged) or "gencode" (human and mouse only). - With source="gencode", `release` is the GENCODE release number (mouse "M" prefix added automatically) and defaults to the latest release. - `which` supports gtf, dna (primary assembly genome), cdna (transcripts), ncrna (lncRNA transcripts) and pep (translations); cds is not provided by GENCODE and raises a clear error. - Returns the same dict/list structure as the Ensembl path (with a `gencode_release` field). Tests + fixtures (live GENCODE FTP) and docs. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/src/en/ref.md | 23 +++++- docs/src/en/updates.md | 3 + gget/constants.py | 3 + gget/gget_ref.py | 156 ++++++++++++++++++++++++++++++++++- gget/main.py | 13 +++ tests/fixtures/test_ref.json | 59 +++++++++++++ 6 files changed, 251 insertions(+), 6 deletions(-) diff --git a/docs/src/en/ref.md b/docs/src/en/ref.md index 0d799ab64..f4d1169df 100644 --- a/docs/src/en/ref.md +++ b/docs/src/en/ref.md @@ -2,13 +2,14 @@ > Python arguments are equivalent to long-option arguments (`--arg`), unless otherwise specified. Flags are True/False arguments in Python. The manual for any gget tool can be called from the command-line using the `-h` `--help` flag. # gget ref 📖 -Fetch download links and metadata for [Ensembl](https://www.ensembl.org/) reference genomes. +Fetch download links and metadata for [Ensembl](https://www.ensembl.org/) and [GENCODE](https://www.gencodegenes.org/) reference genomes. Return format: dictionary/JSON. **Positional argument** `species` Species for which the FTPs will be fetched in the format genus_species, e.g. homo_sapiens. Supports all available vertebrate and invertebrate (plants, fungi, protists, and invertebrate metazoa) genomes from Ensembl, except bacteria. +For `source=gencode`, only human ('human'/'homo_sapiens') and mouse ('mouse'/'mus_musculus') are supported. Note: Not required when using flags `--list_species` or `--list_iv_species`. Supported shortcuts: 'human', 'mouse', 'human_grch37' (accesses the GRCh37 genome assembly) @@ -22,9 +23,14 @@ Possible entries are one or a combination (as comma-separated list) of the follo 'cds' - Returns the coding sequences corresponding to Ensembl genes. (Does not contain UTR or intronic sequence.) 'cdrna' - Returns transcript sequences corresponding to non-coding RNA genes (ncRNA). 'pep' - Returns the protein translations of Ensembl genes. +Note: `source=gencode` does not provide 'cds'; with GENCODE, 'cdna' returns transcript sequences and 'ncrna' returns long non-coding RNA transcript sequences. `-r` `--release` -Defines the Ensembl release number from which the files are fetched, e.g. 104. Default: latest Ensembl release. +Defines the release number from which the files are fetched, e.g. 104. Default: latest release. +For `source=gencode`, this is the GENCODE release number (e.g. 46); the mouse 'M' prefix is added automatically. + +`-src` `--source` +Reference database to fetch from: 'ensembl' (default) or 'gencode'. GENCODE provides human and mouse references only. `-od` `--out_dir` Path to the directory where the FTPs will be saved, e.g. path/to/directory/. Default: Current working directory. @@ -98,6 +104,19 @@ gget.ref(species=None, list_species=True, release=103)

+**Fetch human reference GTF and genome FASTA from GENCODE (release 46):** +```bash +gget ref -w gtf,dna -r 46 --source gencode human +``` +```python +# Python +gget.ref("human", which=["gtf", "dna"], release=46, source="gencode") +``` +→ Returns a JSON with the GENCODE v46 human GTF and genome FASTA FTPs and their metadata. +(GENCODE is available for human and mouse only. Omit `--release` to use the latest GENCODE release.) + +

+ **Use `gget ref` in combination with [kallisto | bustools](https://www.kallistobus.tools/kb_usage/kb_ref/) to build a reference index:** ```bash kb ref \ diff --git a/docs/src/en/updates.md b/docs/src/en/updates.md index 89be22281..a76f615d2 100644 --- a/docs/src/en/updates.md +++ b/docs/src/en/updates.md @@ -5,6 +5,9 @@ #### *gget* officially became part of [*scverse*](https://scverse.org/) on June 9, 2026. 🥳🥳🥳 **Version ≥ 0.30.9** (XXX XX, 2026): +- [`gget ref`](ref.md): Added support for fetching reference GTFs and FASTAs from [GENCODE](https://www.gencodegenes.org/) (fixes [issue 73](https://github.com/scverse/gget/issues/73)). + - New `source` argument (command line: `--source`/`-src`) selects the reference database: `"ensembl"` (default, unchanged behavior) or `"gencode"` (human and mouse only). + - With `source="gencode"`, the `release` argument is the GENCODE release number (e.g. `46`; the mouse `M` prefix is added automatically) and defaults to the latest release. `which` supports `gtf`, `dna` (primary assembly genome), `cdna` (transcript sequences), `ncrna` (long non-coding RNA transcripts), and `pep` (protein translations); GENCODE does not provide `cds`. **Version ≥ 0.30.8** (Jun 28, 2026): - [`gget g2p`](g2p.md): Either `gene` or `--uniprot_id` is now sufficient — whichever is missing is resolved via UniProt and cached. Gene→UniProt picks the canonical reviewed human Swiss-Prot entry; the resolution and its limitations are logged. The canonical pair is **always** prepended to the result as `gene_name` / `uniprot_id` columns (and stored on `df.attrs`), so the output schema is invariant regardless of input mode. Existing call sites continue to work. diff --git a/gget/constants.py b/gget/constants.py index 38f90119f..fe10a563f 100644 --- a/gget/constants.py +++ b/gget/constants.py @@ -14,6 +14,9 @@ # Non-vertebrate server ENSEMBL_FTP_URL_NV = "http://ftp.ensemblgenomes.org/pub/" +# GENCODE FTP root for gget ref (human and mouse reference GTFs/FASTAs) +GENCODE_FTP_URL = "https://ftp.ebi.ac.uk/pub/databases/gencode/" + # NCBI URL for gget info NCBI_URL = "https://www.ncbi.nlm.nih.gov" diff --git a/gget/gget_ref.py b/gget/gget_ref.py index b16391c41..13f277257 100644 --- a/gget/gget_ref.py +++ b/gget/gget_ref.py @@ -21,8 +21,21 @@ ENSEMBL_FTP_URL, ENSEMBL_FTP_URL_GRCH37, ENSEMBL_FTP_URL_NV, + GENCODE_FTP_URL, ) +# Mapping of `which` keys to (GENCODE file-name substring template, output dict key) for gget ref. +# {ver} is filled with the GENCODE version string (e.g. "v46" for human, "vM35" for mouse). +_GENCODE_FILES: dict[str, tuple[str, str]] = { + "gtf": ("gencode.{ver}.annotation.gtf.gz", "annotation_gtf"), + "dna": ("primary_assembly.genome.fa.gz", "genome_dna"), + "cdna": ("gencode.{ver}.transcripts.fa.gz", "transcriptome_cdna"), + "ncrna": ("gencode.{ver}.lncRNA_transcripts.fa.gz", "non-coding_seq_ncRNA"), + "pep": ("gencode.{ver}.pc_translations.fa.gz", "protein_translation_pep"), +} +# Order used when `which="all"` is requested for GENCODE (mirrors the Ensembl ordering, minus 'cds'). +_GENCODE_ALL_ORDER = ["cdna", "dna", "gtf", "ncrna", "pep"] + def find_FTP_link(url: str, link_substring: str) -> tuple[str | None, str | None, str | None]: """Helper function for gget ref to find an FTP link, its release date and size. @@ -58,6 +71,127 @@ def find_FTP_link(url: str, link_substring: str) -> tuple[str | None, str | None return link_str, date_str, size_str +def _find_latest_gencode_release(organism: str) -> str: + """Return the latest GENCODE release identifier for 'human' (e.g. '46') or 'mouse' (e.g. 'M35').""" + base_url = f"{GENCODE_FTP_URL}Gencode_{organism}/" + html = requests.get(base_url, timeout=DEFAULT_REQUESTS_TIMEOUT) + if html.status_code != 200: + raise RuntimeError( + f"GENCODE FTP returned status code {html.status_code} for {base_url}. Please try again.\n" + ) + + soup = BeautifulSoup(html.text, "html.parser") + releases = [ + href.strip("/").replace("release_", "") + for href in (a.get("href", "") for a in soup.find_all("a")) + if href.startswith("release_") + ] + + if organism == "mouse": + # Mouse releases are prefixed with "M" (e.g. "M35") + nums = [int(r[1:]) for r in releases if r.startswith("M") and r[1:].isdigit()] + if not nums: + raise RuntimeError(f"Could not determine the latest GENCODE mouse release from {base_url}.\n") + return f"M{max(nums)}" + + nums = [int(r) for r in releases if r.isdigit()] + if not nums: + raise RuntimeError(f"Could not determine the latest GENCODE human release from {base_url}.\n") + return str(max(nums)) + + +def _gencode_ref( + species: str, + which: str | list[str], + release: int | None, + ftp: bool, + save: bool, + verbose: bool, +) -> Any: + """Fetch reference GTF/FASTA FTP links from GENCODE (human and mouse only). See `ref` for details.""" + # Resolve organism (GENCODE only provides human and mouse references) + species_lower = species.lower() + if species_lower in ("human", "homo_sapiens"): + organism = "human" + elif species_lower in ("mouse", "mus_musculus"): + organism = "mouse" + else: + raise ValueError( + f"GENCODE only provides reference files for human and mouse, but species '{species}' was passed.\n" + "Use species 'human'/'homo_sapiens' or 'mouse'/'mus_musculus' with source='gencode', " + "or use source='ensembl' (default) for other species.\n" + ) + + # Resolve the GENCODE release identifier ("46" for human, "M35" for mouse) + if release is None: + rel = _find_latest_gencode_release(organism) + else: + rel = f"M{release}" if organism == "mouse" else str(release) + ver = f"v{rel}" + base_url = f"{GENCODE_FTP_URL}Gencode_{organism}/release_{rel}/" + + # Normalize and validate the 'which' parameter + if isinstance(which, str): + which = [which] + if len(which) > 1 and "all" in which: + raise ValueError( + "Parameter 'which' must be 'all', or any one or a combination of the following: " + "'gtf', 'cdna', 'dna', 'ncrna', 'pep'.\n" + ) + which_allowed = ["all", *_GENCODE_FILES.keys()] + bad = [x for x in which if x not in which_allowed] + if bad: + extra = "" + if "cds" in bad: + extra = " (GENCODE does not provide a CDS file; use 'cdna' for transcripts or 'pep' for translations)" + raise ValueError( + "For source='gencode', parameter 'which' must be 'all', or any one or a combination of the " + f"following: 'gtf', 'cdna', 'dna', 'ncrna', 'pep'.{extra}\n" + ) + + keys = _GENCODE_ALL_ORDER if "all" in which else which + + if verbose: + logger.info(f"Fetching GENCODE reference information for {organism} from release {rel}.") + + ref_dict: dict[str, dict[str, Any]] = {species_lower: {}} + urls: list[str] = [] + for key in keys: + substring_template, out_key = _GENCODE_FILES[key] + link_substring = substring_template.format(ver=ver) + link_str, date_str, size_str = find_FTP_link(url=base_url, link_substring=link_substring) + if link_str is not None: + file_url = base_url + link_str + date_part = (date_str or " ").split(" ")[0] + time_part = (date_str or " ").split(" ")[1] if date_str and " " in date_str else "" + size_part = size_str or "" + else: + file_url = "" + date_part = "" + time_part = "" + size_part = "" + + urls.append(file_url) + ref_dict[species_lower][out_key] = { + "ftp": file_url, + "gencode_release": rel, + "release_date": date_part, + "release_time": time_part, + "bytes": size_part, + } + + if ftp: + if save: + with open("gget_ref_results.txt", "w") as tfile: + tfile.write("\n".join(urls)) + return urls + + if save: + with open("gget_ref_results.json", "w", encoding="utf-8") as file: + json.dump(ref_dict, file, ensure_ascii=False, indent=4) + return ref_dict + + def ref( species: str | None, which: str | list[str] = "all", @@ -66,14 +200,16 @@ def ref( save: bool = False, list_species: bool = False, list_iv_species: bool = False, + source: str = "ensembl", verbose: bool = True, ) -> Any: - """Fetch FTPs for reference genomes and annotations by species from Ensembl. + """Fetch FTPs for reference genomes and annotations by species from Ensembl or GENCODE. Args: - species Defines the species for which the reference should be fetched in the format "_", e.g. species = "homo_sapiens". Supported shortcuts: "human", "mouse", "human_grch37" (accesses the GRCh37 genome assembly) + For source='gencode', only human ('human'/'homo_sapiens') and mouse ('mouse'/'mus_musculus') are supported. - which Defines which results to return. Default: 'all' -> Returns all available results. Possible entries are one or a combination (as a list of strings) of the following: @@ -83,19 +219,31 @@ def ref( 'cds - Returns the coding sequences corresponding to Ensembl genes. (Does not contain UTR or intronic sequence.) 'cdrna' - Returns transcript sequences corresponding to non-coding RNA genes (ncRNA). 'pep' - Returns the protein translations of Ensembl genes. - - release Defines the Ensembl release number from which the files are fetched, e.g. release = 104. - Default: None -> latest Ensembl release is used + Note: source='gencode' does not provide 'cds'; 'cdna' returns GENCODE transcript sequences and + 'ncrna' returns GENCODE long non-coding RNA transcript sequences. + - release Defines the release number from which the files are fetched, e.g. release = 104. + Default: None -> latest release is used + For source='gencode', this is the GENCODE release number (e.g. 46); the mouse 'M' prefix is added automatically. - ftp Return only the requested FTP links in a list (default: False). - save Save the results in the local directory (default: False). - list_species If True and `species=None`, returns a list of all available VERTEBRATE species from the Ensembl database (default: False). (Can be combined with the `release` argument to get the available species from a specific Ensembl release.) - list_iv_species If True and `species=None`, returns a list of all available INVERTEBRATE species from the Ensembl database (default: False). (Can be combined with the `release` argument to get the available species from a specific Ensembl release.) + - source Reference database to fetch from: 'ensembl' (default) or 'gencode' (human and mouse only). - verbose True/False whether to print progress information (default: True). - Returns a dictionary containing the requested URLs with their respective Ensembl version and release date and time. + Returns a dictionary containing the requested URLs with their respective Ensembl/GENCODE version and release date and time. (If FTP=True, returns a list containing only the URLs.) """ + # Fetch from GENCODE instead of Ensembl + if source != "ensembl": + if source != "gencode": + raise ValueError(f"Parameter 'source' must be 'ensembl' or 'gencode', but '{source}' was passed.\n") + if species is None: + raise ValueError("A species ('human'/'homo_sapiens' or 'mouse'/'mus_musculus') must be provided for source='gencode'.\n") + return _gencode_ref(species, which, release, ftp, save, verbose) + # Return list of all available species if list_species: if release is None: diff --git a/gget/main.py b/gget/main.py index 7a2944b09..9f0f2ec75 100644 --- a/gget/main.py +++ b/gget/main.py @@ -199,6 +199,18 @@ def main() -> None: required=False, help="Return only the FTP link(s).", ) + parser_ref.add_argument( + "-src", + "--source", + default="ensembl", + type=str, + choices=["ensembl", "gencode"], + required=False, + help=( + "Reference database to fetch from: 'ensembl' (default) or 'gencode'.\n" + "GENCODE provides human and mouse references only." + ), + ) parser_ref.add_argument( "-d", "--download", @@ -3380,6 +3392,7 @@ def main() -> None: which=which_clean, release=args.release, ftp=args.ftp, + source=args.source, verbose=args.quiet, ) diff --git a/tests/fixtures/test_ref.json b/tests/fixtures/test_ref.json index c88a364db..df2d44ff2 100644 --- a/tests/fixtures/test_ref.json +++ b/tests/fixtures/test_ref.json @@ -648,5 +648,64 @@ "ftp": false }, "expected_result": "RuntimeError" + }, + "test_ref_gencode": { + "type": "assert_equal", + "args": { + "species": "human", + "which": "gtf", + "release": 46, + "source": "gencode", + "verbose": false + }, + "expected_result": { + "human": { + "annotation_gtf": { + "ftp": "https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_46/gencode.v46.annotation.gtf.gz", + "gencode_release": "46", + "release_date": "2024-05-13", + "release_time": "16:01", + "bytes": "49M" + } + } + } + }, + "test_ref_gencode_ftp": { + "type": "assert_equal", + "args": { + "species": "mouse", + "which": [ + "gtf", + "dna" + ], + "release": 35, + "source": "gencode", + "ftp": true, + "verbose": false + }, + "expected_result": [ + "https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_mouse/release_M35/gencode.vM35.annotation.gtf.gz", + "https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_mouse/release_M35/GRCm39.primary_assembly.genome.fa.gz" + ] + }, + "test_ref_gencode_bad_species": { + "type": "error", + "args": { + "species": "zebrafish", + "which": "gtf", + "release": 46, + "source": "gencode" + }, + "expected_result": "ValueError" + }, + "test_ref_gencode_no_cds": { + "type": "error", + "args": { + "species": "human", + "which": "cds", + "release": 46, + "source": "gencode" + }, + "expected_result": "ValueError" } } \ No newline at end of file From 4f78b0c609367bd0370b99e323804bdd92ee4f71 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 24 Jun 2026 15:42:03 +0000 Subject: [PATCH 2/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- gget/gget_ref.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gget/gget_ref.py b/gget/gget_ref.py index 13f277257..fba2c426c 100644 --- a/gget/gget_ref.py +++ b/gget/gget_ref.py @@ -76,9 +76,7 @@ def _find_latest_gencode_release(organism: str) -> str: base_url = f"{GENCODE_FTP_URL}Gencode_{organism}/" html = requests.get(base_url, timeout=DEFAULT_REQUESTS_TIMEOUT) if html.status_code != 200: - raise RuntimeError( - f"GENCODE FTP returned status code {html.status_code} for {base_url}. Please try again.\n" - ) + raise RuntimeError(f"GENCODE FTP returned status code {html.status_code} for {base_url}. Please try again.\n") soup = BeautifulSoup(html.text, "html.parser") releases = [ @@ -241,7 +239,9 @@ def ref( if source != "gencode": raise ValueError(f"Parameter 'source' must be 'ensembl' or 'gencode', but '{source}' was passed.\n") if species is None: - raise ValueError("A species ('human'/'homo_sapiens' or 'mouse'/'mus_musculus') must be provided for source='gencode'.\n") + raise ValueError( + "A species ('human'/'homo_sapiens' or 'mouse'/'mus_musculus') must be provided for source='gencode'.\n" + ) return _gencode_ref(species, which, release, ftp, save, verbose) # Return list of all available species From 7b932f0d0737248cfe1e203d111cb8b9ea1a6246 Mon Sep 17 00:00:00 2001 From: Elarwei Date: Thu, 25 Jun 2026 00:59:48 +0800 Subject: [PATCH 3/6] test(ref): cover remaining lines for codecov (#73) Add a network-free TestGencodeRefOffline class that mocks requests and find_FTP_link to cover _find_latest_gencode_release (human/mouse release resolution and error paths) and _gencode_ref (species and 'which' validation, link found/not-found, ftp/json save, release resolution), plus the ref() source validation and GENCODE delegation. All PR-added lines are now covered. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/test_ref.py | 110 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/tests/test_ref.py b/tests/test_ref.py index c6ab3e0e8..d6247d5ba 100644 --- a/tests/test_ref.py +++ b/tests/test_ref.py @@ -1,6 +1,10 @@ import json +import os +import tempfile import unittest +from unittest.mock import patch +import gget.gget_ref as gget_ref from gget.gget_ref import ref from .from_json import from_json @@ -12,3 +16,109 @@ class TestRef(unittest.TestCase, metaclass=from_json(ref_dict, ref)): pass # all tests are loaded from json + + +class _FakeResp: + """Minimal stand-in for a requests.Response used to test GENCODE helpers offline.""" + + def __init__(self, text="", status_code=200): + self.text = text + self.status_code = status_code + + +class TestGencodeRefOffline(unittest.TestCase): + """Network-free tests of the GENCODE reference helpers (issue #73).""" + + @patch.object(gget_ref.requests, "get") + def test_find_latest_human(self, mock_get): + mock_get.return_value = _FakeResp('ab') + self.assertEqual(gget_ref._find_latest_gencode_release("human"), "46") + + @patch.object(gget_ref.requests, "get") + def test_find_latest_mouse(self, mock_get): + mock_get.return_value = _FakeResp('ab') + self.assertEqual(gget_ref._find_latest_gencode_release("mouse"), "M35") + + @patch.object(gget_ref.requests, "get") + def test_find_latest_bad_status(self, mock_get): + mock_get.return_value = _FakeResp("", status_code=500) + with self.assertRaises(RuntimeError): + gget_ref._find_latest_gencode_release("human") + + @patch.object(gget_ref.requests, "get") + def test_find_latest_human_no_releases(self, mock_get): + mock_get.return_value = _FakeResp('a') + with self.assertRaises(RuntimeError): + gget_ref._find_latest_gencode_release("human") + + @patch.object(gget_ref.requests, "get") + def test_find_latest_mouse_no_releases(self, mock_get): + # A non-"M" release listing yields no mouse release numbers. + mock_get.return_value = _FakeResp('a') + with self.assertRaises(RuntimeError): + gget_ref._find_latest_gencode_release("mouse") + + def test_gencode_bad_species(self): + with self.assertRaises(ValueError): + gget_ref._gencode_ref("zebrafish", "gtf", 46, False, False, False) + + def test_gencode_which_all_conflict(self): + with self.assertRaises(ValueError): + gget_ref._gencode_ref("human", ["gtf", "all"], 46, False, False, False) + + def test_gencode_bad_which_cds(self): + # 'cds' is unsupported by GENCODE and triggers the extra-hint ValueError. + with self.assertRaises(ValueError): + gget_ref._gencode_ref("human", ["cds"], 46, False, False, False) + + @patch.object(gget_ref, "find_FTP_link") + def test_gencode_all_verbose_link_found(self, mock_ftp): + mock_ftp.return_value = ("gencode.v46.annotation.gtf.gz", "01-Jan-2024 12:00", "1.2G") + result = gget_ref._gencode_ref("human", "all", 46, False, False, True) + self.assertIn("human", result) + self.assertTrue(any(v["ftp"] for v in result["human"].values())) + + @patch.object(gget_ref, "find_FTP_link") + def test_gencode_link_not_found(self, mock_ftp): + mock_ftp.return_value = (None, None, None) + result = gget_ref._gencode_ref("mouse", "gtf", 35, False, False, False) + self.assertIn("mouse", result) + self.assertEqual(next(iter(result["mouse"].values()))["ftp"], "") + + @patch.object(gget_ref, "find_FTP_link") + def test_gencode_ftp_and_save(self, mock_ftp): + mock_ftp.return_value = ("f.gtf.gz", "01-Jan-2024 12:00", "1G") + with tempfile.TemporaryDirectory() as tmp: + cwd = os.getcwd() + os.chdir(tmp) + try: + urls = gget_ref._gencode_ref("human", "gtf", 46, True, True, False) # ftp + save + self.assertIsInstance(urls, list) + self.assertTrue(os.path.exists("gget_ref_results.txt")) + gget_ref._gencode_ref("human", "gtf", 46, False, True, False) # json save + self.assertTrue(os.path.exists("gget_ref_results.json")) + finally: + os.chdir(cwd) + + @patch.object(gget_ref, "_find_latest_gencode_release") + @patch.object(gget_ref, "find_FTP_link") + def test_gencode_release_none_resolves_latest(self, mock_ftp, mock_rel): + mock_rel.return_value = "46" + mock_ftp.return_value = ("f.gtf.gz", "01-Jan-2024 12:00", "1G") + gget_ref._gencode_ref("human", "gtf", None, False, False, False) + mock_rel.assert_called_once() + + def test_ref_bad_source(self): + with self.assertRaises(ValueError): + ref("human", source="banana", verbose=False) + + def test_ref_gencode_requires_species(self): + with self.assertRaises(ValueError): + ref(None, source="gencode", verbose=False) + + @patch.object(gget_ref, "_gencode_ref") + def test_ref_delegates_to_gencode(self, mock_gencode): + mock_gencode.return_value = {"human": {}} + result = ref("human", source="gencode", verbose=False) + self.assertEqual(result, {"human": {}}) + mock_gencode.assert_called_once() From 1115c7f74d2b65554759ec7972b46e925a4a93ac Mon Sep 17 00:00:00 2001 From: Elarwei Date: Fri, 3 Jul 2026 22:14:07 +0800 Subject: [PATCH 4/6] test(ref): harden GENCODE live tests + warn on missing file (#73) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address live-data-test robustness for the GENCODE path: 1. Move the two network-touching fixtures (test_ref_gencode / test_ref_gencode_ftp) out of test_ref.json into an explicit TestGencodeRefLive class. They still assert exact values (anchored to the frozen releases human v46 / mouse M35, whose files never change), but now skip on a network error or a transient non-200 (rate-limit / HTML page) instead of reddening CI. The two validation-only fixtures (bad_species / no_cds) stay in test_ref.json — they raise before any request. 2. Add live tests for the HTML-scraping "latest release" path (_find_latest_gencode_release) with loose assertions (human -> integer string >= 46; mouse -> 'M'+integer >= 35), skipping on transient errors. This is the only regression net for upstream listing changes. 3. Warn (logger.warning) when a GENCODE file name no longer matches, instead of silently returning an empty entry. Co-Authored-By: Claude Opus 4.8 (1M context) --- gget/gget_ref.py | 4 +++ tests/fixtures/test_ref.json | 39 ---------------------- tests/test_ref.py | 64 ++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 39 deletions(-) diff --git a/gget/gget_ref.py b/gget/gget_ref.py index fba2c426c..61a8663d9 100644 --- a/gget/gget_ref.py +++ b/gget/gget_ref.py @@ -164,6 +164,10 @@ def _gencode_ref( time_part = (date_str or " ").split(" ")[1] if date_str and " " in date_str else "" size_part = size_str or "" else: + logger.warning( + f"No GENCODE file matching '{link_substring}' was found in {base_url} " + f"(the '{key}' entry will be empty). The upstream file naming may have changed." + ) file_url = "" date_part = "" time_part = "" diff --git a/tests/fixtures/test_ref.json b/tests/fixtures/test_ref.json index df2d44ff2..7fdf23df5 100644 --- a/tests/fixtures/test_ref.json +++ b/tests/fixtures/test_ref.json @@ -649,45 +649,6 @@ }, "expected_result": "RuntimeError" }, - "test_ref_gencode": { - "type": "assert_equal", - "args": { - "species": "human", - "which": "gtf", - "release": 46, - "source": "gencode", - "verbose": false - }, - "expected_result": { - "human": { - "annotation_gtf": { - "ftp": "https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_46/gencode.v46.annotation.gtf.gz", - "gencode_release": "46", - "release_date": "2024-05-13", - "release_time": "16:01", - "bytes": "49M" - } - } - } - }, - "test_ref_gencode_ftp": { - "type": "assert_equal", - "args": { - "species": "mouse", - "which": [ - "gtf", - "dna" - ], - "release": 35, - "source": "gencode", - "ftp": true, - "verbose": false - }, - "expected_result": [ - "https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_mouse/release_M35/gencode.vM35.annotation.gtf.gz", - "https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_mouse/release_M35/GRCm39.primary_assembly.genome.fa.gz" - ] - }, "test_ref_gencode_bad_species": { "type": "error", "args": { diff --git a/tests/test_ref.py b/tests/test_ref.py index d6247d5ba..230b3c74d 100644 --- a/tests/test_ref.py +++ b/tests/test_ref.py @@ -5,6 +5,7 @@ from unittest.mock import patch import gget.gget_ref as gget_ref +import requests from gget.gget_ref import ref from .from_json import from_json @@ -122,3 +123,66 @@ def test_ref_delegates_to_gencode(self, mock_gencode): result = ref("human", source="gencode", verbose=False) self.assertEqual(result, {"human": {}}) mock_gencode.assert_called_once() + + +class TestGencodeRefLive(unittest.TestCase): + """Live tests against the real GENCODE FTP (ftp.ebi.ac.uk) for issue #73. + + Unlike the offline tests above, these actually reach the network. They anchor + a frozen GENCODE release (human v46 / mouse M35), whose files never change, so + the exact assertions are stable over time. If the source is temporarily + unreachable or rate-limits the request, the resource is skipped rather than + failing the build. + """ + + def _skip_if_transient(self, fn, *args, **kwargs): + """Run fn, converting a network error or a transient HTTP error into a skip.""" + try: + return fn(*args, **kwargs) + except requests.RequestException as e: + self.skipTest(f"Network error reaching GENCODE FTP: {e}") + except RuntimeError as e: + # find_FTP_link / _find_latest raise RuntimeError on a non-200 (e.g. 5xx, rate-limit). + self.skipTest(f"GENCODE FTP transient error: {e}") + + def test_human_v46_gtf(self): + result = self._skip_if_transient(ref, "human", which="gtf", release=46, source="gencode", verbose=False) + entry = result["human"]["annotation_gtf"] + if not entry["ftp"]: + self.skipTest("GENCODE returned no matching link (likely a transient/rate-limit HTML page).") + self.assertEqual( + entry["ftp"], + "https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_46/gencode.v46.annotation.gtf.gz", + ) + # Frozen release -> these metadata values do not change. + self.assertEqual(entry["gencode_release"], "46") + self.assertEqual(entry["release_date"], "2024-05-13") + self.assertEqual(entry["release_time"], "16:01") + self.assertEqual(entry["bytes"], "49M") + + def test_mouse_M35_ftp(self): + result = self._skip_if_transient( + ref, "mouse", which=["gtf", "dna"], release=35, source="gencode", ftp=True, verbose=False + ) + if not all(result): + self.skipTest("GENCODE returned an empty link (likely a transient/rate-limit HTML page).") + self.assertEqual( + result, + [ + "https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_mouse/release_M35/gencode.vM35.annotation.gtf.gz", + "https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_mouse/release_M35/GRCm39.primary_assembly.genome.fa.gz", + ], + ) + + def test_latest_human_release_is_plausible(self): + # Guards the HTML-scraping path against upstream listing changes. Loose on purpose: + # a bare integer string, >= a release we know exists (releases only grow). + rel = self._skip_if_transient(gget_ref._find_latest_gencode_release, "human") + self.assertTrue(rel.isdigit(), f"expected an integer human release, got {rel!r}") + self.assertGreaterEqual(int(rel), 46) + + def test_latest_mouse_release_is_plausible(self): + # Mouse releases look like 'M35', 'M39', ... (M + integer). + rel = self._skip_if_transient(gget_ref._find_latest_gencode_release, "mouse") + self.assertTrue(rel.startswith("M") and rel[1:].isdigit(), f"expected 'M', got {rel!r}") + self.assertGreaterEqual(int(rel[1:]), 35) From 31967a48c33edd5c10505ff1569831c0751c0750 Mon Sep 17 00:00:00 2001 From: Elarwei Date: Fri, 3 Jul 2026 22:39:20 +0800 Subject: [PATCH 5/6] docs(ref): sync Spanish GENCODE docs; drop issue link from changelog - es/ref.md: mirror the GENCODE additions from en/ref.md (source arg, species restriction, which/cds note, release note, and the GENCODE fetch example). - es/updates.md: add the Spanish 0.30.9 GENCODE changelog entry. - en/updates.md: drop the "(fixes issue 73)" parenthetical from the GENCODE entry (readers of the changelog don't need the issue link). Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/src/en/updates.md | 2 +- docs/src/es/ref.md | 23 +++++++++++++++++++++-- docs/src/es/updates.md | 5 +++++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/docs/src/en/updates.md b/docs/src/en/updates.md index a76f615d2..65a04756e 100644 --- a/docs/src/en/updates.md +++ b/docs/src/en/updates.md @@ -5,7 +5,7 @@ #### *gget* officially became part of [*scverse*](https://scverse.org/) on June 9, 2026. 🥳🥳🥳 **Version ≥ 0.30.9** (XXX XX, 2026): -- [`gget ref`](ref.md): Added support for fetching reference GTFs and FASTAs from [GENCODE](https://www.gencodegenes.org/) (fixes [issue 73](https://github.com/scverse/gget/issues/73)). +- [`gget ref`](ref.md): Added support for fetching reference GTFs and FASTAs from [GENCODE](https://www.gencodegenes.org/). - New `source` argument (command line: `--source`/`-src`) selects the reference database: `"ensembl"` (default, unchanged behavior) or `"gencode"` (human and mouse only). - With `source="gencode"`, the `release` argument is the GENCODE release number (e.g. `46`; the mouse `M` prefix is added automatically) and defaults to the latest release. `which` supports `gtf`, `dna` (primary assembly genome), `cdna` (transcript sequences), `ncrna` (long non-coding RNA transcripts), and `pep` (protein translations); GENCODE does not provide `cds`. diff --git a/docs/src/es/ref.md b/docs/src/es/ref.md index effd646f5..3bfb3f329 100644 --- a/docs/src/es/ref.md +++ b/docs/src/es/ref.md @@ -2,7 +2,7 @@ > Parámetros de Python són iguales a los parámetros largos (`--parámetro`) de Terminal, si no especificado de otra manera. Banderas son parámetros de verdadero o falso (True/False) en Python. El manuál para cualquier modulo de gget se puede llamar desde la Terminal con la bandera `-h` `--help`. # gget ref 📖 -Obtenga enlaces de descarga y metadatos para los genomas de referencia de [Ensembl](https://www.ensembl.org/). +Obtenga enlaces de descarga y metadatos para los genomas de referencia de [Ensembl](https://www.ensembl.org/) y [GENCODE](https://www.gencodegenes.org/). Regresa: Resultados en formato JSON. **Parámetro posicional** @@ -10,6 +10,7 @@ Regresa: Resultados en formato JSON. La especie por la cual que se buscará los FTP en el formato género_especies, p. ej. homo_sapiens. Nota: No se requiere cuando se llama a la bandera `--list_species`. Accesos directos: 'human', 'mouse', 'human_grch37' (accede al ensamblaje del genoma GRCh37) +Para `source=gencode`, solo se admiten humano ('human'/'homo_sapiens') y ratón ('mouse'/'mus_musculus'). **Parámetros optionales** `-w` `--which` @@ -21,9 +22,14 @@ Las entradas posibles son uno solo o una combinación de las siguientes (como li 'cds' - Regresa las secuencias codificantes correspondientes a los genes Ensembl. (No contiene UTR ni secuencia intrónica). 'cdrna' - Regresa secuencias de transcripción correspondientes a genes de ARN no codificantes (ncRNA). 'pep' - Regresa las traducciones de proteínas de los genes Ensembl. +Nota: `source=gencode` no proporciona 'cds'; con GENCODE, 'cdna' regresa secuencias de transcripción y 'ncrna' regresa secuencias de transcripción de ARN largo no codificante. `-r` `--release` -Define el número de versión de Ensembl desde el que se obtienen los archivos, p. ej. 104. Default: latest Ensembl release. +Define el número de versión desde el que se obtienen los archivos, p. ej. 104. Por defecto: la última versión. +Para `source=gencode`, este es el número de versión de GENCODE (p. ej. 46); el prefijo 'M' para ratón se añade automáticamente. + +`-src` `--source` +Base de datos de referencia de la que obtener los archivos: 'ensembl' (por defecto) o 'gencode'. GENCODE proporciona referencias solo para humano y ratón. `-od` `--out_dir` Ruta al directorio donde se guardarán los archivos FTP, p. ruta/al/directorio/. Por defecto: directorio de trabajo actual. @@ -68,6 +74,19 @@ gget.ref(species=None, list_species=True, release=103)

+**Obtenga el GTF de referencia humano y el FASTA del genoma desde GENCODE (versión 46):** +```bash +gget ref -w gtf,dna -r 46 --source gencode human +``` +```python +# Python +gget.ref("human", which=["gtf", "dna"], release=46, source="gencode") +``` +→ Regresa un JSON con los FTP del GTF humano y el FASTA del genoma de GENCODE v46 y sus metadatos. +(GENCODE está disponible solo para humano y ratón. Omita `--release` para usar la última versión de GENCODE). + +

+ **Obtenga la referencia del genoma para una especie específica:** ```bash gget ref -w gtf,dna homo_sapiens diff --git a/docs/src/es/updates.md b/docs/src/es/updates.md index 0aa103023..af4d004fc 100644 --- a/docs/src/es/updates.md +++ b/docs/src/es/updates.md @@ -4,6 +4,11 @@ #### *gget* se unió oficialmente a *scverse* el 9 de junio de 2026. 🥳🥳🥳 +**Versión ≥ 0.30.9** (XXX XX, 2026): +- [`gget ref`](ref.md): Se añadió soporte para obtener GTFs y FASTAs de referencia desde [GENCODE](https://www.gencodegenes.org/). + - Nuevo argumento `source` (línea de comandos: `--source`/`-src`) que selecciona la base de datos de referencia: `"ensembl"` (por defecto, comportamiento sin cambios) o `"gencode"` (solo humano y ratón). + - Con `source="gencode"`, el argumento `release` es el número de versión de GENCODE (p. ej. `46`; el prefijo `M` para ratón se añade automáticamente) y por defecto usa la última versión. `which` admite `gtf`, `dna` (genoma del ensamblaje primario), `cdna` (secuencias de transcripción), `ncrna` (transcripciones de ARN largo no codificante) y `pep` (traducciones de proteínas); GENCODE no proporciona `cds`. + **Versión ≥ 0.30.8** (28 de junio de 2026): - [`gget g2p`](g2p.md): Ahora es suficiente con `gene` o `--uniprot_id` — el que falte se resuelve a través de UniProt y se almacena en caché. Gene→UniProt selecciona la entrada canónica revisada humana de Swiss-Prot; la resolución y sus limitaciones se registran. El par canónico **siempre** se añade al inicio del resultado como columnas `gene_name` / `uniprot_id` (y se almacena en `df.attrs`), por lo que el esquema de salida es invariante independientemente del modo de entrada. Los sitios de llamada existentes siguen funcionando. - Nuevo filtro `residues=` (Python: int / list / range / set; CLI `--residues 100,200,300` o `100-200`) restringe `features` / `alignment` a posiciones específicas del lado del cliente. From 5d5aebdb9449c10dcdb4fdbf44b2a8da7b39818a Mon Sep 17 00:00:00 2001 From: Elarwei Date: Fri, 3 Jul 2026 22:53:44 +0800 Subject: [PATCH 6/6] feat(ref): add list_files escape hatch for GENCODE full directory (#73) The curated `which` set (gtf/dna/cdna/ncrna/pep) mirrors Ensembl and covers the common needs, but GENCODE ships many more files (GFF3, basic/comprehensive annotation, pc_transcripts, metadata, ...). Rather than growing `which` into a source-specific, non-portable vocabulary, add a `list_files` escape hatch. - `ref(..., source="gencode", list_files=True)` returns every file in the GENCODE release directory keyed by filename, each with its FTP link, release date/time, and size. Honors `ftp=True` (URL list) and `save=True`. - CLI: `--list_files`/`-lf`. Requires `source="gencode"`; using it with Ensembl raises a clear ValueError (Ensembl splits files across per-type subdirectories, so there is no single directory to list). - New `_list_gencode_files` parses the release directory HTML, skipping the parent-directory link and subdirectories. - Tests: offline (parse/filter, bad status, dict/ftp shapes, ensembl-guard) and a live test asserting the listing includes pc_transcripts (a file with no `which` key), skipping on transient errors. - Docs (en/es ref.md + updates.md) document the flag and add an example. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/src/en/ref.md | 15 +++++++++ docs/src/en/updates.md | 1 + docs/src/es/ref.md | 15 +++++++++ docs/src/es/updates.md | 1 + gget/gget_ref.py | 70 +++++++++++++++++++++++++++++++++++++++++- gget/main.py | 13 ++++++++ tests/test_ref.py | 61 ++++++++++++++++++++++++++++++++++++ 7 files changed, 175 insertions(+), 1 deletion(-) diff --git a/docs/src/en/ref.md b/docs/src/en/ref.md index f4d1169df..ef5e28f2e 100644 --- a/docs/src/en/ref.md +++ b/docs/src/en/ref.md @@ -46,6 +46,9 @@ Lists all available vertebrate species. (Python: combine with `species=None`.) `-liv` `--list_iv_species` Lists all available invertebrate species. (Python: combine with `species=None`.) +`-lf` `--list_files` +Only for `--source gencode`. Lists every file in the GENCODE release directory for the given species (beyond the curated `-w`/`--which` set), e.g. GFF3, basic/comprehensive annotation, `pc_transcripts`, and metadata files. Combine with `--release` to target a specific GENCODE release. + `-ftp` `--ftp` Returns only the requested FTP links. @@ -117,6 +120,18 @@ gget.ref("human", which=["gtf", "dna"], release=46, source="gencode")

+**List every file in a GENCODE release directory (to fetch files beyond the `which` set):** +```bash +gget ref --list_files -r 46 --source gencode human +``` +```python +# Python +gget.ref("human", list_files=True, release=46, source="gencode") +``` +→ Returns a JSON keyed by filename with every file in the GENCODE v46 human release directory (GFF3, basic/comprehensive annotation, `pc_transcripts`, metadata, etc.) and its FTP link, date, and size. Use this to reach GENCODE-specific files that `which` does not expose. + +

+ **Use `gget ref` in combination with [kallisto | bustools](https://www.kallistobus.tools/kb_usage/kb_ref/) to build a reference index:** ```bash kb ref \ diff --git a/docs/src/en/updates.md b/docs/src/en/updates.md index 65a04756e..0f5b85d28 100644 --- a/docs/src/en/updates.md +++ b/docs/src/en/updates.md @@ -8,6 +8,7 @@ - [`gget ref`](ref.md): Added support for fetching reference GTFs and FASTAs from [GENCODE](https://www.gencodegenes.org/). - New `source` argument (command line: `--source`/`-src`) selects the reference database: `"ensembl"` (default, unchanged behavior) or `"gencode"` (human and mouse only). - With `source="gencode"`, the `release` argument is the GENCODE release number (e.g. `46`; the mouse `M` prefix is added automatically) and defaults to the latest release. `which` supports `gtf`, `dna` (primary assembly genome), `cdna` (transcript sequences), `ncrna` (long non-coding RNA transcripts), and `pep` (protein translations); GENCODE does not provide `cds`. + - New `list_files` flag (command line: `--list_files`/`-lf`), only for `source="gencode"`, lists every file in a GENCODE release directory (keyed by filename) so GENCODE-specific files beyond the `which` set — GFF3, basic/comprehensive annotation, `pc_transcripts`, metadata, etc. — can be fetched. **Version ≥ 0.30.8** (Jun 28, 2026): - [`gget g2p`](g2p.md): Either `gene` or `--uniprot_id` is now sufficient — whichever is missing is resolved via UniProt and cached. Gene→UniProt picks the canonical reviewed human Swiss-Prot entry; the resolution and its limitations are logged. The canonical pair is **always** prepended to the result as `gene_name` / `uniprot_id` columns (and stored on `df.attrs`), so the output schema is invariant regardless of input mode. Existing call sites continue to work. diff --git a/docs/src/es/ref.md b/docs/src/es/ref.md index 3bfb3f329..65c8691ca 100644 --- a/docs/src/es/ref.md +++ b/docs/src/es/ref.md @@ -42,6 +42,9 @@ Para Python, usa `save=True` para guardar los resultados en el directorio de tra `-l` `--list_species` Enumera todas las especies disponibles. (Para Python: combina con `species=None`.) +`-lf` `--list_files` +Solo para `--source gencode`. Enumera todos los archivos del directorio de la versión de GENCODE para la especie indicada (más allá del conjunto seleccionado por `-w`/`--which`), p. ej. GFF3, anotación básica/completa, `pc_transcripts` y archivos de metadatos. Combínalo con `--release` para apuntar a una versión específica de GENCODE. + `-ftp` `--ftp` Regresa solo los enlaces FTP solicitados. @@ -87,6 +90,18 @@ gget.ref("human", which=["gtf", "dna"], release=46, source="gencode")

+**Enumere todos los archivos de un directorio de versión de GENCODE (para obtener archivos más allá del conjunto `which`):** +```bash +gget ref --list_files -r 46 --source gencode human +``` +```python +# Python +gget.ref("human", list_files=True, release=46, source="gencode") +``` +→ Regresa un JSON indexado por nombre de archivo con todos los archivos del directorio de la versión 46 humana de GENCODE (GFF3, anotación básica/completa, `pc_transcripts`, metadatos, etc.) junto con su enlace FTP, fecha y tamaño. Úselo para alcanzar archivos específicos de GENCODE que `which` no expone. + +

+ **Obtenga la referencia del genoma para una especie específica:** ```bash gget ref -w gtf,dna homo_sapiens diff --git a/docs/src/es/updates.md b/docs/src/es/updates.md index af4d004fc..8471611bb 100644 --- a/docs/src/es/updates.md +++ b/docs/src/es/updates.md @@ -8,6 +8,7 @@ - [`gget ref`](ref.md): Se añadió soporte para obtener GTFs y FASTAs de referencia desde [GENCODE](https://www.gencodegenes.org/). - Nuevo argumento `source` (línea de comandos: `--source`/`-src`) que selecciona la base de datos de referencia: `"ensembl"` (por defecto, comportamiento sin cambios) o `"gencode"` (solo humano y ratón). - Con `source="gencode"`, el argumento `release` es el número de versión de GENCODE (p. ej. `46`; el prefijo `M` para ratón se añade automáticamente) y por defecto usa la última versión. `which` admite `gtf`, `dna` (genoma del ensamblaje primario), `cdna` (secuencias de transcripción), `ncrna` (transcripciones de ARN largo no codificante) y `pep` (traducciones de proteínas); GENCODE no proporciona `cds`. + - Nueva bandera `list_files` (línea de comandos: `--list_files`/`-lf`), solo para `source="gencode"`, enumera todos los archivos de un directorio de versión de GENCODE (indexados por nombre de archivo) para poder obtener archivos específicos de GENCODE más allá del conjunto `which` — GFF3, anotación básica/completa, `pc_transcripts`, metadatos, etc. **Versión ≥ 0.30.8** (28 de junio de 2026): - [`gget g2p`](g2p.md): Ahora es suficiente con `gene` o `--uniprot_id` — el que falte se resuelve a través de UniProt y se almacena en caché. Gene→UniProt selecciona la entrada canónica revisada humana de Swiss-Prot; la resolución y sus limitaciones se registran. El par canónico **siempre** se añade al inicio del resultado como columnas `gene_name` / `uniprot_id` (y se almacena en `df.attrs`), por lo que el esquema de salida es invariante independientemente del modo de entrada. Los sitios de llamada existentes siguen funcionando. diff --git a/gget/gget_ref.py b/gget/gget_ref.py index 61a8663d9..f484e328f 100644 --- a/gget/gget_ref.py +++ b/gget/gget_ref.py @@ -98,6 +98,37 @@ def _find_latest_gencode_release(organism: str) -> str: return str(max(nums)) +def _list_gencode_files(base_url: str) -> list[tuple[str, str, str]]: + """List every file in a GENCODE release directory as (filename, date, size) tuples. + + Powers the `list_files` escape hatch of gget ref, exposing the full GENCODE release + directory (beyond the curated `which` set). The parent-directory link and any + subdirectories are skipped; only files are returned. + """ + html = requests.get(base_url, timeout=DEFAULT_REQUESTS_TIMEOUT) + if html.status_code != 200: + raise RuntimeError(f"GENCODE FTP returned status code {html.status_code} for {base_url}. Please try again.\n") + + soup = BeautifulSoup(html.text, "html.parser") + files: list[tuple[str, str, str]] = [] + for row in soup.find_all("tr"): + cells = row.find_all("td") + if len(cells) < 4: + continue + link = cells[1].find("a") + if link is None: + continue + name = link.get("href", "") + # Skip the parent-directory link ("../"), subdirectories (end in "/"), + # absolute-path links, and column-sort links ("?C=..."). + if not name or name.endswith("/") or name.startswith(("/", "?")): + continue + date_str = cells[2].get_text().strip() + size_str = cells[3].get_text().strip() + files.append((name, date_str, size_str)) + return files + + def _gencode_ref( species: str, which: str | list[str], @@ -105,6 +136,7 @@ def _gencode_ref( ftp: bool, save: bool, verbose: bool, + list_files: bool = False, ) -> Any: """Fetch reference GTF/FASTA FTP links from GENCODE (human and mouse only). See `ref` for details.""" # Resolve organism (GENCODE only provides human and mouse references) @@ -128,6 +160,31 @@ def _gencode_ref( ver = f"v{rel}" base_url = f"{GENCODE_FTP_URL}Gencode_{organism}/release_{rel}/" + # Escape hatch: list every file in the release directory (beyond the curated `which` set) + if list_files: + if verbose: + logger.info(f"Listing all files in the GENCODE {organism} release {rel} directory.") + all_files = _list_gencode_files(base_url) + file_urls = [base_url + fname for fname, _, _ in all_files] + listing: dict[str, dict[str, Any]] = {species_lower: {}} + for fname, f_date, f_size in all_files: + listing[species_lower][fname] = { + "ftp": base_url + fname, + "gencode_release": rel, + "release_date": f_date.split(" ")[0] if f_date else "", + "release_time": f_date.split(" ")[1] if f_date and " " in f_date else "", + "bytes": f_size or "", + } + if ftp: + if save: + with open("gget_ref_results.txt", "w") as tfile: + tfile.write("\n".join(file_urls)) + return file_urls + if save: + with open("gget_ref_results.json", "w", encoding="utf-8") as file: + json.dump(listing, file, ensure_ascii=False, indent=4) + return listing + # Normalize and validate the 'which' parameter if isinstance(which, str): which = [which] @@ -203,6 +260,7 @@ def ref( list_species: bool = False, list_iv_species: bool = False, source: str = "ensembl", + list_files: bool = False, verbose: bool = True, ) -> Any: """Fetch FTPs for reference genomes and annotations by species from Ensembl or GENCODE. @@ -233,6 +291,10 @@ def ref( - list_iv_species If True and `species=None`, returns a list of all available INVERTEBRATE species from the Ensembl database (default: False). (Can be combined with the `release` argument to get the available species from a specific Ensembl release.) - source Reference database to fetch from: 'ensembl' (default) or 'gencode' (human and mouse only). + - list_files Only for source='gencode'. If True, returns every file in the GENCODE release directory + for the given species (not just the curated `which` set), keyed by filename. Handy for + fetching GENCODE-specific files (GFF3, basic/comprehensive annotation, pc_transcripts, + metadata, ...) that `which` does not expose (default: False). - verbose True/False whether to print progress information (default: True). Returns a dictionary containing the requested URLs with their respective Ensembl/GENCODE version and release date and time. @@ -246,7 +308,13 @@ def ref( raise ValueError( "A species ('human'/'homo_sapiens' or 'mouse'/'mus_musculus') must be provided for source='gencode'.\n" ) - return _gencode_ref(species, which, release, ftp, save, verbose) + return _gencode_ref(species, which, release, ftp, save, verbose, list_files=list_files) + + if list_files: + raise ValueError( + "list_files=True is only supported with source='gencode' (it lists the full GENCODE release " + "directory). Ensembl organizes files in per-type subdirectories; use the 'which' argument instead.\n" + ) # Return list of all available species if list_species: diff --git a/gget/main.py b/gget/main.py index 9f0f2ec75..ef1f3263f 100644 --- a/gget/main.py +++ b/gget/main.py @@ -211,6 +211,18 @@ def main() -> None: "GENCODE provides human and mouse references only." ), ) + parser_ref.add_argument( + "-lf", + "--list_files", + default=False, + action="store_true", + required=False, + help=( + "Only for --source gencode. List every file in the GENCODE release directory for the\n" + "given species (beyond the curated -w/--which set), e.g. GFF3, basic/comprehensive\n" + "annotation, pc_transcripts, and metadata files." + ), + ) parser_ref.add_argument( "-d", "--download", @@ -3393,6 +3405,7 @@ def main() -> None: release=args.release, ftp=args.ftp, source=args.source, + list_files=args.list_files, verbose=args.quiet, ) diff --git a/tests/test_ref.py b/tests/test_ref.py index 230b3c74d..030ae2b3a 100644 --- a/tests/test_ref.py +++ b/tests/test_ref.py @@ -124,6 +124,57 @@ def test_ref_delegates_to_gencode(self, mock_gencode): self.assertEqual(result, {"human": {}}) mock_gencode.assert_called_once() + @patch.object(gget_ref.requests, "get") + def test_list_gencode_files_parses_and_filters(self, mock_get): + html = ( + "" + '' + '' + '' + '' + "
Parent Directory -
GRCh37_mapping/2024-05-13 16:01 -
gencode.v46.annotation.gtf.gz2024-05-13 16:01 49M
README.TXT2024-05-13 15:55 67K
" + ) + mock_get.return_value = _FakeResp(html) + files = gget_ref._list_gencode_files("https://example/release_46/") + names = [f[0] for f in files] + self.assertIn("gencode.v46.annotation.gtf.gz", names) + self.assertIn("README.TXT", names) # non-.gz files are listed too + # The parent-directory link and subdirectories are filtered out. + self.assertNotIn("../", names) + self.assertNotIn("GRCh37_mapping/", names) + + @patch.object(gget_ref.requests, "get") + def test_list_gencode_files_bad_status_raises(self, mock_get): + mock_get.return_value = _FakeResp("", status_code=503) + with self.assertRaises(RuntimeError): + gget_ref._list_gencode_files("https://example/release_46/") + + @patch.object(gget_ref, "_list_gencode_files") + def test_gencode_list_files_returns_dict(self, mock_list): + mock_list.return_value = [ + ("gencode.v46.annotation.gtf.gz", "2024-05-13 16:01", "49M"), + ("README.TXT", "2024-05-13 15:55", "67K"), + ] + result = gget_ref._gencode_ref("human", "all", 46, False, False, False, list_files=True) + self.assertIn("gencode.v46.annotation.gtf.gz", result["human"]) + entry = result["human"]["gencode.v46.annotation.gtf.gz"] + self.assertTrue(entry["ftp"].endswith("release_46/gencode.v46.annotation.gtf.gz")) + self.assertEqual(entry["release_date"], "2024-05-13") + self.assertEqual(entry["release_time"], "16:01") + self.assertEqual(entry["bytes"], "49M") + + @patch.object(gget_ref, "_list_gencode_files") + def test_gencode_list_files_ftp_returns_urls(self, mock_list): + mock_list.return_value = [("gencode.v46.annotation.gtf.gz", "2024-05-13 16:01", "49M")] + urls = gget_ref._gencode_ref("human", "all", 46, True, False, False, list_files=True) + self.assertIsInstance(urls, list) + self.assertTrue(urls[0].endswith("gencode.v46.annotation.gtf.gz")) + + def test_ref_list_files_requires_gencode(self): + # list_files is only valid with source='gencode'; the default source='ensembl' must reject it. + with self.assertRaises(ValueError): + ref("human", list_files=True, verbose=False) + class TestGencodeRefLive(unittest.TestCase): """Live tests against the real GENCODE FTP (ftp.ebi.ac.uk) for issue #73. @@ -186,3 +237,13 @@ def test_latest_mouse_release_is_plausible(self): rel = self._skip_if_transient(gget_ref._find_latest_gencode_release, "mouse") self.assertTrue(rel.startswith("M") and rel[1:].isdigit(), f"expected 'M', got {rel!r}") self.assertGreaterEqual(int(rel[1:]), 35) + + def test_list_files_exposes_more_than_which(self): + # The escape hatch must surface files the curated `which` set does not expose. + result = self._skip_if_transient(ref, "human", release=46, source="gencode", list_files=True, verbose=False) + files = result["human"] + if not files: + self.skipTest("GENCODE returned an empty listing (likely a transient/rate-limit HTML page).") + self.assertIn("gencode.v46.annotation.gtf.gz", files) + # pc_transcripts is a real GENCODE file with no `which` key -> proves list_files goes beyond it. + self.assertIn("gencode.v46.pc_transcripts.fa.gz", files)