From 9e28f5b870b22cf32127cd8537ce51596cc0d6f1 Mon Sep 17 00:00:00 2001 From: Eric Hackathorn Date: Thu, 18 Jun 2026 05:53:26 +0000 Subject: [PATCH] Add THREDDS catalog connector to the acquire stage Implements `zyra acquire thredds` to enumerate datasets from a THREDDS catalog.xml, map each dataset's urlPath to its HTTPServer (fileServer) download URL, and list, sync, or batch-fetch matches. Supports opt-in recursion into nested catalogRef entries (with a depth cap and cycle guard), plus pattern/date filtering and sync replacement options that mirror the existing FTP connector vocabulary. - New backend: connectors/backends/thredds.py (hermetic; supports a fetcher/catalog_xml injection so tests need no network) - CLI: acquire/import `thredds` subcommand - API: AcquireThreddsArgs/AcquireThreddsRun wired into POST /acquire - Tests: backend unit tests + CLI tests using offline catalog fixtures - Docs: ingest README THREDDS section; sample pipeline thredds_to_local.yaml - Regenerated capabilities manifest and OpenAPI sha256 snapshot Addresses NOAA-GSL/zyra#283 Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Vfz8KKCKV7AGZ9RSPCiRoH Signed-off-by: Eric Hackathorn --- samples/pipelines/thredds_to_local.yaml | 41 ++ src/zyra/api/models/domain_api.py | 5 + src/zyra/api/routers/domain_acquire.py | 9 +- src/zyra/api/schemas/domain_args.py | 30 ++ src/zyra/connectors/backends/thredds.py | 371 ++++++++++++++++++ src/zyra/connectors/ingest/README.md | 11 + src/zyra/connectors/ingest/__init__.py | 167 ++++++++ src/zyra/wizard/zyra_capabilities.json | 288 ++++++++++++++ .../wizard/zyra_capabilities/acquire.json | 288 ++++++++++++++ tests/cli/test_acquire_thredds.py | 56 +++ tests/connectors/test_thredds_backend.py | 178 +++++++++ tests/snapshots/openapi_sha256.txt | 2 +- 12 files changed, 1444 insertions(+), 2 deletions(-) create mode 100644 samples/pipelines/thredds_to_local.yaml create mode 100644 src/zyra/connectors/backends/thredds.py create mode 100644 tests/cli/test_acquire_thredds.py create mode 100644 tests/connectors/test_thredds_backend.py diff --git a/samples/pipelines/thredds_to_local.yaml b/samples/pipelines/thredds_to_local.yaml new file mode 100644 index 00000000..7765b39b --- /dev/null +++ b/samples/pipelines/thredds_to_local.yaml @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: Apache-2.0 +name: THREDDS → Transform → Video → Local +stages: + # 1) Acquire frames by syncing a THREDDS catalog to local. + # Datasets are enumerated from catalog.xml and downloaded via the + # HTTPServer (fileServer) service. Self-updating: existing non-empty + # files are skipped on re-runs. + - stage: acquire + command: thredds + args: + catalog_url: https://gsl.noaa.gov/thredds/catalog/fv3-chem-0p25deg-grib2/catalog.xml + sync_dir: ./frames + pattern: "\\.grib2$" + since_period: "P1D" + # Follow nested catalogRef entries (optional) + recursive: false + max_depth: 3 + + # 2) Compute frames metadata (JSON) + - stage: transform + command: metadata + args: + frames_dir: ./frames + datetime_format: "%Y%m%d" + period_seconds: 3600 + output: frames_meta.json + + # 3) Compose frames into a video + - stage: visualize + command: compose-video + args: + frames: ./frames + output: video.mp4 + fps: 24 + + # 4) Copy outputs to local filesystem (no network required) + - stage: export + command: local + args: + input: video.mp4 + path: /tmp/video.mp4 diff --git a/src/zyra/api/models/domain_api.py b/src/zyra/api/models/domain_api.py index 92899aea..e67ca69c 100644 --- a/src/zyra/api/models/domain_api.py +++ b/src/zyra/api/models/domain_api.py @@ -180,3 +180,8 @@ class AcquireFtpRun(DomainRunRequest): class AcquireApiRun(DomainRunRequest): tool: Literal["api"] args: da.AcquireApiArgs # type: ignore[assignment] + + +class AcquireThreddsRun(DomainRunRequest): + tool: Literal["thredds"] + args: da.AcquireThreddsArgs # type: ignore[assignment] diff --git a/src/zyra/api/routers/domain_acquire.py b/src/zyra/api/routers/domain_acquire.py index 9188cea6..9ff9d89c 100644 --- a/src/zyra/api/routers/domain_acquire.py +++ b/src/zyra/api/routers/domain_acquire.py @@ -25,6 +25,7 @@ AcquireFtpRun, AcquireHttpRun, AcquireS3Run, + AcquireThreddsRun, DomainRunResponse, ) from zyra.api.routers.cli import get_cli_matrix, run_cli_endpoint @@ -37,7 +38,13 @@ AcquireRequest = Annotated[ - Union[AcquireHttpRun, AcquireS3Run, AcquireFtpRun, AcquireApiRun], + Union[ + AcquireHttpRun, + AcquireS3Run, + AcquireFtpRun, + AcquireApiRun, + AcquireThreddsRun, + ], Body(discriminator="tool"), ] diff --git a/src/zyra/api/schemas/domain_args.py b/src/zyra/api/schemas/domain_args.py index 9c2db49e..fdcf47ca 100644 --- a/src/zyra/api/schemas/domain_args.py +++ b/src/zyra/api/schemas/domain_args.py @@ -186,6 +186,30 @@ def _require_path_or_listing(self): # type: ignore[override] return self +class AcquireThreddsArgs(BaseModel): + catalog_url: str + output: str | None = None + # Listing/sync/batch + list_mode: bool | None = Field(default=None, alias="list") + sync_dir: str | None = None + output_dir: str | None = None + # Enumeration + recursive: bool | None = None + max_depth: int | None = None + pattern: str | None = None + since: str | None = None + since_period: str | None = None + until: str | None = None + date_format: str | None = None + header: list[str] | None = None + # Sync mode options (subset meaningful over HTTP) + overwrite_existing: bool | None = None + recheck_existing: bool | None = None + min_remote_size: str | int | None = None + prefer_remote: bool | None = None + skip_if_local_done: bool | None = None + + # New: acquire api (generic REST) class AcquireApiArgs(BaseModel): url: str | None = None @@ -303,6 +327,8 @@ def normalize_and_validate(stage: str, tool: str, args: dict) -> dict: out = _normalize_credentials(out) elif stage == "acquire" and tool == "ftp": out = _normalize_credentials(out) + elif stage == "acquire" and tool == "thredds": + out = _normalize_headers(out) elif stage == "decimate" and tool == "post": out = _normalize_headers(out) out = _normalize_credentials(out) @@ -651,11 +677,15 @@ def resolve_model(stage: str, tool: str) -> type[BaseModel] | None: return AcquireHttpArgs if key == ("acquire", "api"): return AcquireApiArgs + if key == ("acquire", "thredds"): + return AcquireThreddsArgs # Aliases for acquire if key == ("import", "http"): return AcquireHttpArgs if key == ("import", "api"): return AcquireApiArgs + if key == ("import", "thredds"): + return AcquireThreddsArgs if key == ("process", "convert-format"): return ProcessConvertFormatArgs if key == ("process", "decode-grib2"): diff --git a/src/zyra/connectors/backends/thredds.py b/src/zyra/connectors/backends/thredds.py new file mode 100644 index 00000000..7fdef95e --- /dev/null +++ b/src/zyra/connectors/backends/thredds.py @@ -0,0 +1,371 @@ +# SPDX-License-Identifier: Apache-2.0 +"""THREDDS Data Server (TDS) connector backend. + +Functional helpers to enumerate datasets from THREDDS ``catalog.xml`` documents +and map them to ``fileServer`` download URLs, with optional recursion into +nested ``catalogRef`` entries. + +Design goals mirror the other connector backends (``http``/``ftp``/``s3``): + +- Network access is optional so tests stay hermetic. Callers may inject a + ``fetcher`` callable (or a single ``catalog_xml`` string) instead of hitting + the network; otherwise the HTTP backend is used. +- Functions are small and dependency-light. Parsing relies on the stdlib + ``xml.etree.ElementTree`` and is namespace-agnostic (THREDDS InvCatalog and + xlink namespaces are matched by local tag name). + +A THREDDS catalog declares one or more ```` entries (we use the +``HTTPServer`` service to build download URLs), a tree of ```` +elements (downloadable ones carry a ``urlPath`` attribute), and +```` links to nested catalogs. +""" + +from __future__ import annotations + +import logging +import re +import xml.etree.ElementTree as ET +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from urllib.parse import urljoin, urlparse + +from zyra.connectors.backends import ftp as ftp_backend +from zyra.connectors.backends import http as http_backend +from zyra.utils.date_manager import DateManager + +# Re-export the shared sync option surface so callers can configure THREDDS +# sync behavior with the same vocabulary used by the FTP backend. +SyncOptions = ftp_backend.SyncOptions + +logger = logging.getLogger(__name__) + +#: Maximum recursion depth applied when ``recursive=True`` and the caller does +#: not specify ``max_depth``. +DEFAULT_MAX_DEPTH = 3 + + +@dataclass(frozen=True) +class ThreddsDataset: + """A single downloadable THREDDS dataset. + + Attributes + - name: Human-readable dataset name (``name`` attribute). + - url_path: The service-relative ``urlPath`` used to build the download URL. + - download_url: Absolute HTTP(S) URL served by the ``fileServer`` service. + - dataset_id: Optional THREDDS dataset ``ID`` attribute. + - date: Optional ISO date string parsed from a ```` element, if any. + """ + + name: str + url_path: str + download_url: str + dataset_id: str | None = None + date: str | None = None + + +def _local(tag: str) -> str: + """Return the local name of a possibly namespaced XML tag.""" + return tag.rsplit("}", 1)[-1] if "}" in tag else tag + + +def _attr(el: ET.Element, name: str) -> str | None: + """Return an attribute by local name, ignoring any XML namespace prefix.""" + val = el.get(name) + if val is not None: + return val + for key, value in el.attrib.items(): + if _local(key) == name: + return value + return None + + +def _iter_services(root: ET.Element) -> Iterable[ET.Element]: + """Yield all ```` elements, including those nested in compounds.""" + for el in root.iter(): + if _local(el.tag) == "service": + yield el + + +def _http_server_base(root: ET.Element) -> str: + """Return the ``HTTPServer`` service base path (e.g. ``/thredds/fileServer/``). + + Falls back to the conventional default when no HTTPServer service is found. + """ + for svc in _iter_services(root): + stype = (_attr(svc, "serviceType") or "").lower() + if stype == "httpserver": + base = _attr(svc, "base") or "" + if base: + return base + return "/thredds/fileServer/" + + +def _server_root(catalog_url: str) -> str: + """Return ``scheme://netloc`` for the given catalog URL.""" + pr = urlparse(catalog_url) + return f"{pr.scheme}://{pr.netloc}" + + +def _build_download_url(catalog_url: str, base: str, url_path: str) -> str: + """Build an absolute fileServer download URL from a dataset ``urlPath``.""" + root = _server_root(catalog_url) + base_path = base if base.startswith("/") else "/" + base + if not base_path.endswith("/"): + base_path += "/" + return f"{root}{base_path}{url_path.lstrip('/')}" + + +def _dataset_date(el: ET.Element) -> str | None: + """Return the text of the first ```` child element, if present.""" + for child in el: + if _local(child.tag) == "date" and (child.text or "").strip(): + return child.text.strip() + return None + + +def parse_catalog( + catalog_xml: str, catalog_url: str +) -> tuple[list[ThreddsDataset], list[str]]: + """Parse a THREDDS catalog document. + + Parameters + - catalog_xml: Raw catalog XML text. + - catalog_url: URL the catalog was fetched from (used to resolve relative + ``catalogRef`` hrefs and to build absolute download URLs). + + Returns a tuple ``(datasets, catalog_refs)`` where ``datasets`` are the + downloadable datasets (those carrying a ``urlPath``) and ``catalog_refs`` + are absolute URLs of nested catalogs referenced via ````. + """ + root = ET.fromstring(catalog_xml) + base = _http_server_base(root) + + datasets: list[ThreddsDataset] = [] + catalog_refs: list[str] = [] + for el in root.iter(): + local = _local(el.tag) + if local == "dataset": + url_path = _attr(el, "urlPath") + if not url_path: + continue + datasets.append( + ThreddsDataset( + name=_attr(el, "name") or Path(url_path).name, + url_path=url_path, + download_url=_build_download_url(catalog_url, base, url_path), + dataset_id=_attr(el, "ID"), + date=_dataset_date(el), + ) + ) + elif local == "catalogRef": + href = _attr(el, "href") + if href: + catalog_refs.append(urljoin(catalog_url, href)) + return datasets, catalog_refs + + +def _default_fetcher( + *, headers: dict[str, str] | None, timeout: int +) -> Callable[[str], str]: + def _fetch(url: str) -> str: + return http_backend.fetch_text(url, timeout=timeout, headers=headers) + + return _fetch + + +def enumerate_datasets( + catalog_url: str, + *, + catalog_xml: str | None = None, + recursive: bool = False, + max_depth: int = DEFAULT_MAX_DEPTH, + pattern: str | None = None, + since: str | None = None, + until: str | None = None, + date_format: str | None = None, + headers: dict[str, str] | None = None, + timeout: int = 60, + fetcher: Callable[[str], str] | None = None, +) -> list[ThreddsDataset]: + """Enumerate downloadable datasets from a THREDDS catalog. + + When ``recursive`` is True, nested ``catalogRef`` entries are followed up to + ``max_depth`` levels (the root catalog is depth 0). A visited set guards + against catalog reference cycles. + + Filtering mirrors the other connectors: ``pattern`` is a regex applied to + the dataset ``urlPath``; ``since``/``until`` are ISO dates applied to dates + parsed from the dataset basename via :class:`DateManager`. + """ + fetch = fetcher or _default_fetcher(headers=headers, timeout=timeout) + + results: list[ThreddsDataset] = [] + seen_urls: set[str] = set() + seen_paths: set[str] = set() + + def _walk(url: str, xml: str | None, depth: int) -> None: + if url in seen_urls: + return + seen_urls.add(url) + try: + text = xml if xml is not None else fetch(url) + except Exception as exc: # pragma: no cover - network/error path + logger.warning("Failed to fetch THREDDS catalog %s: %s", url, exc) + return + datasets, refs = parse_catalog(text, url) + for ds in datasets: + if ds.url_path not in seen_paths: + seen_paths.add(ds.url_path) + results.append(ds) + if recursive and depth < max_depth: + for ref in refs: + _walk(ref, None, depth + 1) + + _walk(catalog_url, catalog_xml, 0) + + if pattern: + rx = re.compile(pattern) + results = [ds for ds in results if rx.search(ds.url_path)] + + if since or until: + dm = DateManager([date_format] if date_format else None) + start = datetime.min if not since else datetime.fromisoformat(since) + end = datetime.max if not until else datetime.fromisoformat(until) + results = [ + ds + for ds in results + if dm.is_date_in_range(Path(ds.url_path).name, start, end) + ] + return results + + +def list_files( + catalog_url: str, + *, + catalog_xml: str | None = None, + recursive: bool = False, + max_depth: int = DEFAULT_MAX_DEPTH, + pattern: str | None = None, + since: str | None = None, + until: str | None = None, + date_format: str | None = None, + headers: dict[str, str] | None = None, + timeout: int = 60, + fetcher: Callable[[str], str] | None = None, +) -> list[str]: + """Return absolute fileServer download URLs for matching datasets.""" + datasets = enumerate_datasets( + catalog_url, + catalog_xml=catalog_xml, + recursive=recursive, + max_depth=max_depth, + pattern=pattern, + since=since, + until=until, + date_format=date_format, + headers=headers, + timeout=timeout, + fetcher=fetcher, + ) + return [ds.download_url for ds in datasets] + + +def fetch_bytes( + url: str, *, timeout: int = 60, headers: dict[str, str] | None = None +) -> bytes: + """Fetch a single dataset over HTTP(S) (delegates to the HTTP backend).""" + return http_backend.fetch_bytes(url, timeout=timeout, headers=headers) + + +def _should_download( + download_url: str, + local_path: Path, + options: SyncOptions, + *, + headers: dict[str, str] | None, + timeout: int, +) -> bool: + """Decide whether to (re)download a dataset to ``local_path``. + + Precedence mirrors the FTP backend for the options meaningful over HTTP: + ``skip_if_local_done`` -> missing/zero-byte -> ``overwrite_existing`` / + ``prefer_remote`` -> ``min_remote_size`` / ``recheck_existing`` (via the + HTTP ``Content-Length`` header) -> default skip when a non-empty local copy + already exists. + """ + if options.skip_if_local_done and ftp_backend._has_done_marker(local_path): + return False + if not local_path.exists(): + return True + local_size = local_path.stat().st_size + if local_size == 0: + return True + if options.overwrite_existing or options.prefer_remote: + return True + if options.min_remote_size is not None or options.recheck_existing: + remote_size = http_backend.get_size( + download_url, headers=headers, timeout=timeout + ) + if remote_size is None: + return options.recheck_existing + threshold = ftp_backend._parse_min_size(options.min_remote_size, local_size) + if threshold is not None: + return remote_size >= threshold + if options.recheck_existing: + return remote_size != local_size + return False + + +def sync_directory( + catalog_url: str, + local_dir: str, + *, + catalog_xml: str | None = None, + recursive: bool = False, + max_depth: int = DEFAULT_MAX_DEPTH, + pattern: str | None = None, + since: str | None = None, + until: str | None = None, + date_format: str | None = None, + headers: dict[str, str] | None = None, + timeout: int = 60, + sync_options: SyncOptions | None = None, + fetcher: Callable[[str], str] | None = None, +) -> list[str]: + """Download matching THREDDS datasets into ``local_dir``. + + Returns the list of local file paths that were downloaded (skipped files are + not included). Files are named by the dataset ``urlPath`` basename. + """ + options = sync_options or SyncOptions() + datasets = enumerate_datasets( + catalog_url, + catalog_xml=catalog_xml, + recursive=recursive, + max_depth=max_depth, + pattern=pattern, + since=since, + until=until, + date_format=date_format, + headers=headers, + timeout=timeout, + fetcher=fetcher, + ) + out_dir = Path(local_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + downloaded: list[str] = [] + for ds in datasets: + name = Path(ds.url_path).name or "download.bin" + local_path = out_dir / name + if not _should_download( + ds.download_url, local_path, options, headers=headers, timeout=timeout + ): + logger.debug("Skipping existing THREDDS dataset %s", name) + continue + data = fetch_bytes(ds.download_url, timeout=timeout, headers=headers) + local_path.write_bytes(data) + downloaded.append(str(local_path)) + return downloaded diff --git a/src/zyra/connectors/ingest/README.md b/src/zyra/connectors/ingest/README.md index de6ae254..db2a6ac8 100644 --- a/src/zyra/connectors/ingest/README.md +++ b/src/zyra/connectors/ingest/README.md @@ -4,6 +4,7 @@ Commands - `zyra acquire http` — Download via HTTP(S), list/filter directory pages, batch with `--inputs/--manifest`. - `zyra acquire s3` — Download from S3 by URL (`s3://bucket/key`) or bucket/key. - `zyra acquire ftp` — Fetch from FTP (single path or list/sync directories). +- `zyra acquire thredds` — Enumerate a THREDDS catalog and fetch datasets via the fileServer service. - `zyra acquire vimeo` — Placeholder for Vimeo fetch by id (not implemented). - `zyra acquire api` — Generic REST API fetch (headers/params/body, pagination, streaming). @@ -23,6 +24,16 @@ FTP - List/sync directory: `zyra acquire ftp ftp://host/path/ --list` or `--sync-dir local_dir` - Credentials: `--user demo --credential password=$FTP_PASS` (aliases for `--credential user=...` / `password=...`) apply to fetch, list, and sync operations without embedding secrets in the URL. +THREDDS +- Reads a THREDDS Data Server `catalog.xml`, maps each dataset's `urlPath` to its `HTTPServer` (fileServer) download URL, and lists, syncs, or batch-fetches matches. +- List URLs: `zyra acquire thredds https://host/thredds/catalog/foo/catalog.xml --list --pattern '\\.grib2$'` +- Sync to a frames dir (self-updating; skips existing non-empty files): `zyra acquire thredds https://host/thredds/catalog/foo/catalog.xml --sync-dir ./frames --pattern '\\.grib2$' --since-period P1D` +- Batch fetch: add `--output-dir downloads/` (without `--list`/`--sync-dir`) to download all matching datasets. +- Nested catalogs: `--recursive` follows `catalogRef` entries up to `--max-depth` (default 3). +- Filtering mirrors the other connectors: `--pattern` (regex on `urlPath`), `--since/--until/--since-period`, and `--date-format` (strftime tokens, e.g. `%Y%m%d`) matched against the dataset filename. +- Sync replacement options: `--overwrite-existing`, `--recheck-existing` (size compare via HTTP `Content-Length`), `--min-remote-size`, `--prefer-remote`, `--skip-if-local-done`. +- Example (GSL FV3-Chem GRIB2): `zyra acquire thredds https://gsl.noaa.gov/thredds/catalog/fv3-chem-0p25deg-grib2/catalog.xml --sync-dir ./frames --pattern '\\.grib2$' --since-period P1D` + Credential helper (all connectors opting in) - `--credential field=value` (repeatable) resolves secrets via literals, `$ENV`, or `@KEY` (using `CredentialManager` / dotenv). Common slots: `token`, `user`, `password`, and `header.`. - `--credential-file path/to/.env` points to a specific dotenv file when using `@KEY` lookups. diff --git a/src/zyra/connectors/ingest/__init__.py b/src/zyra/connectors/ingest/__init__.py index 0e93d7d9..56176f62 100644 --- a/src/zyra/connectors/ingest/__init__.py +++ b/src/zyra/connectors/ingest/__init__.py @@ -9,6 +9,7 @@ from zyra.connectors.backends import ftp as ftp_backend from zyra.connectors.backends import http as http_backend from zyra.connectors.backends import s3 as s3_backend +from zyra.connectors.backends import thredds as thredds_backend from zyra.connectors.credentials import ( CredentialResolutionError, apply_auth_header, @@ -312,6 +313,72 @@ def _cmd_ftp(ns: argparse.Namespace) -> int: return 0 +def _cmd_thredds(ns: argparse.Namespace) -> int: + """Acquire data from a THREDDS catalog (enumerate + fileServer fetch).""" + if getattr(ns, "verbose", False): + os.environ["ZYRA_VERBOSITY"] = "debug" + elif getattr(ns, "quiet", False): + os.environ["ZYRA_VERBOSITY"] = "quiet" + if getattr(ns, "trace", False): + os.environ["ZYRA_SHELL_TRACE"] = "1" + configure_logging_from_env() + headers = parse_header_strings(getattr(ns, "header", None)) + + def _since() -> str | None: + sp = getattr(ns, "since_period", None) + s = getattr(ns, "since", None) + if sp and not s: + return DateManager().get_date_range_iso(sp)[0].isoformat() + return s + + common = dict( + recursive=getattr(ns, "recursive", False), + max_depth=int(getattr(ns, "max_depth", thredds_backend.DEFAULT_MAX_DEPTH)), + pattern=getattr(ns, "pattern", None), + since=_since(), + until=getattr(ns, "until", None), + date_format=getattr(ns, "date_format", None), + headers=headers or None, + ) + + # Listing mode + if getattr(ns, "list", False): + for url in thredds_backend.list_files(ns.catalog_url, **common): + print(url) + return 0 + + # Sync mode + if getattr(ns, "sync_dir", None): + sync_opts = thredds_backend.SyncOptions( + overwrite_existing=getattr(ns, "overwrite_existing", False), + recheck_existing=getattr(ns, "recheck_existing", False), + min_remote_size=getattr(ns, "min_remote_size", None), + prefer_remote=getattr(ns, "prefer_remote", False), + skip_if_local_done=getattr(ns, "skip_if_local_done", False), + ) + thredds_backend.sync_directory( + ns.catalog_url, ns.sync_dir, sync_options=sync_opts, **common + ) + return 0 + + # Batch fetch of enumerated datasets + from pathlib import Path + + urls = thredds_backend.list_files(ns.catalog_url, **common) + if ns.output_dir is None: + raise SystemExit( + "--output-dir is required when fetching from a THREDDS catalog" + ) + outdir = Path(ns.output_dir) + outdir.mkdir(parents=True, exist_ok=True) + for u in urls: + data = thredds_backend.fetch_bytes(u, headers=headers or None) + name = Path(u).name or "download.bin" + with (outdir / name).open("wb") as f: + f.write(data) + return 0 + + def _cmd_vimeo(ns: argparse.Namespace) -> int: # pragma: no cover - placeholder """Placeholder for Vimeo acquisition; not implemented.""" if getattr(ns, "verbose", False): @@ -1110,6 +1177,106 @@ def register_cli(acq_subparsers: Any) -> None: ) p_ftp.set_defaults(func=_cmd_ftp) + # thredds (THREDDS Data Server catalogs) + p_thr = acq_subparsers.add_parser( + "thredds", + help="Enumerate and fetch from THREDDS catalogs", + description=( + "Read a THREDDS catalog.xml, map datasets to fileServer download URLs, " + "and list, sync, or fetch matching datasets. Optionally recurse into " + "nested catalogRef entries." + ), + ) + p_thr.add_argument("catalog_url", help="URL to a THREDDS catalog.xml document") + add_output_option(p_thr) + p_thr.add_argument( + "--list", + action="store_true", + help="List fileServer download URLs for matching datasets", + ) + p_thr.add_argument( + "--sync-dir", + dest="sync_dir", + help="Sync matching datasets to a local directory", + ) + p_thr.add_argument( + "--output-dir", + dest="output_dir", + help="Directory to write outputs when fetching enumerated datasets", + ) + p_thr.add_argument( + "--recursive", + action="store_true", + help="Follow nested catalogRef entries", + ) + p_thr.add_argument( + "--max-depth", + dest="max_depth", + type=int, + default=thredds_backend.DEFAULT_MAX_DEPTH, + help="Maximum recursion depth for --recursive (default: 3)", + ) + p_thr.add_argument("--pattern", help="Regex to filter dataset urlPath") + p_thr.add_argument("--since", help="ISO date filter (matched against dataset name)") + p_thr.add_argument( + "--since-period", + dest="since_period", + help="ISO-8601 duration for lookback (e.g., P1Y, P6M, P7D, PT24H)", + ) + p_thr.add_argument("--until", help="ISO date filter (matched against dataset name)") + p_thr.add_argument( + "--date-format", + dest="date_format", + help="Filename date format for filtering (e.g., YYYYMMDD)", + ) + p_thr.add_argument( + "--header", + action="append", + help="Add custom HTTP header 'Name: Value' (repeatable)", + ) + p_thr.add_argument( + "--verbose", action="store_true", help="Verbose logging for this command" + ) + p_thr.add_argument( + "--quiet", action="store_true", help="Quiet logging for this command" + ) + p_thr.add_argument( + "--trace", + action="store_true", + help="Shell-style trace of key steps and external commands", + ) + # Sync mode replacement options (subset meaningful over HTTP) + p_thr.add_argument( + "--overwrite-existing", + dest="overwrite_existing", + action="store_true", + help="Replace local files unconditionally", + ) + p_thr.add_argument( + "--recheck-existing", + dest="recheck_existing", + action="store_true", + help="Compare sizes via HTTP Content-Length when deciding to re-download", + ) + p_thr.add_argument( + "--min-remote-size", + dest="min_remote_size", + help="Threshold for replacement (absolute bytes or relative percentage)", + ) + p_thr.add_argument( + "--prefer-remote", + dest="prefer_remote", + action="store_true", + help="Always prioritize remote versions over local copies", + ) + p_thr.add_argument( + "--skip-if-local-done", + dest="skip_if_local_done", + action="store_true", + help="Skip files that have a .done marker file", + ) + p_thr.set_defaults(func=_cmd_thredds) + # vimeo (placeholder) p_vimeo = acq_subparsers.add_parser( "vimeo", diff --git a/src/zyra/wizard/zyra_capabilities.json b/src/zyra/wizard/zyra_capabilities.json index 4560da55..87c33f04 100644 --- a/src/zyra/wizard/zyra_capabilities.json +++ b/src/zyra/wizard/zyra_capabilities.json @@ -422,6 +422,150 @@ "--output-dir" ] }, + "acquire thredds": { + "description": "Read a THREDDS catalog.xml, map datasets to fileServer download URLs, and list, sync, or fetch matching datasets. Optionally recurse into nested catalogRef entries.", + "doc": "Read a THREDDS catalog.xml, map datasets to fileServer download URLs, and list, sync, or fetch matching datasets. Optionally recurse into nested catalogRef entries.", + "epilog": "", + "groups": [ + { + "title": "options", + "options": [ + "--help", + "--output", + "--list", + "--sync-dir", + "--output-dir", + "--recursive", + "--max-depth", + "--pattern", + "--since", + "--since-period", + "--until", + "--date-format", + "--header", + "--verbose", + "--quiet", + "--trace", + "--overwrite-existing", + "--recheck-existing", + "--min-remote-size", + "--prefer-remote", + "--skip-if-local-done" + ] + } + ], + "options": { + "--help": "show this help message and exit", + "--output": { + "help": "Output path or '-' for stdout", + "path_arg": true, + "type": "path", + "default": "-" + }, + "--list": { + "help": "List fileServer download URLs for matching datasets", + "type": "bool", + "default": false + }, + "--sync-dir": "Sync matching datasets to a local directory", + "--output-dir": { + "help": "Directory to write outputs when fetching enumerated datasets", + "path_arg": true, + "type": "path" + }, + "--recursive": { + "help": "Follow nested catalogRef entries", + "type": "bool", + "default": false + }, + "--max-depth": { + "help": "Maximum recursion depth for --recursive (default: 3)", + "type": "int", + "default": 3 + }, + "--pattern": "Regex to filter dataset urlPath", + "--since": "ISO date filter (matched against dataset name)", + "--since-period": "ISO-8601 duration for lookback (e.g., P1Y, P6M, P7D, PT24H)", + "--until": "ISO date filter (matched against dataset name)", + "--date-format": "Filename date format for filtering (e.g., YYYYMMDD)", + "--header": "Add custom HTTP header 'Name: Value' (repeatable)", + "--verbose": { + "help": "Verbose logging for this command", + "type": "bool", + "default": false + }, + "--quiet": { + "help": "Quiet logging for this command", + "type": "bool", + "default": false + }, + "--trace": { + "help": "Shell-style trace of key steps and external commands", + "type": "bool", + "default": false + }, + "--overwrite-existing": { + "help": "Replace local files unconditionally", + "type": "bool", + "default": false + }, + "--recheck-existing": { + "help": "Compare sizes via HTTP Content-Length when deciding to re-download", + "type": "bool", + "default": false + }, + "--min-remote-size": "Threshold for replacement (absolute bytes or relative percentage)", + "--prefer-remote": { + "help": "Always prioritize remote versions over local copies", + "type": "bool", + "default": false + }, + "--skip-if-local-done": { + "help": "Skip files that have a .done marker file", + "type": "bool", + "default": false + } + }, + "positionals": [ + { + "name": "catalog_url", + "help": "URL to a THREDDS catalog.xml document", + "type": "str", + "required": true + } + ], + "domain": "acquire", + "args_schema": { + "required": [ + "catalog_url" + ], + "optional": [ + "date_format", + "header", + "list_mode", + "max_depth", + "min_remote_size", + "output", + "output_dir", + "overwrite_existing", + "pattern", + "prefer_remote", + "recheck_existing", + "recursive", + "since", + "since_period", + "skip_if_local_done", + "sync_dir", + "until" + ] + }, + "example_args": null, + "impl": { + "module": "zyra.connectors.ingest", + "callable": "_cmd_thredds" + }, + "returns": "bytes" + }, "acquire vimeo": { "description": "Placeholder for fetching Vimeo videos by id. Not implemented yet.", "doc": "Placeholder for fetching Vimeo videos by id. Not implemented yet.", @@ -1130,6 +1274,150 @@ "--output-dir" ] }, + "import thredds": { + "description": "Read a THREDDS catalog.xml, map datasets to fileServer download URLs, and list, sync, or fetch matching datasets. Optionally recurse into nested catalogRef entries.", + "doc": "Read a THREDDS catalog.xml, map datasets to fileServer download URLs, and list, sync, or fetch matching datasets. Optionally recurse into nested catalogRef entries.", + "epilog": "", + "groups": [ + { + "title": "options", + "options": [ + "--help", + "--output", + "--list", + "--sync-dir", + "--output-dir", + "--recursive", + "--max-depth", + "--pattern", + "--since", + "--since-period", + "--until", + "--date-format", + "--header", + "--verbose", + "--quiet", + "--trace", + "--overwrite-existing", + "--recheck-existing", + "--min-remote-size", + "--prefer-remote", + "--skip-if-local-done" + ] + } + ], + "options": { + "--help": "show this help message and exit", + "--output": { + "help": "Output path or '-' for stdout", + "path_arg": true, + "type": "path", + "default": "-" + }, + "--list": { + "help": "List fileServer download URLs for matching datasets", + "type": "bool", + "default": false + }, + "--sync-dir": "Sync matching datasets to a local directory", + "--output-dir": { + "help": "Directory to write outputs when fetching enumerated datasets", + "path_arg": true, + "type": "path" + }, + "--recursive": { + "help": "Follow nested catalogRef entries", + "type": "bool", + "default": false + }, + "--max-depth": { + "help": "Maximum recursion depth for --recursive (default: 3)", + "type": "int", + "default": 3 + }, + "--pattern": "Regex to filter dataset urlPath", + "--since": "ISO date filter (matched against dataset name)", + "--since-period": "ISO-8601 duration for lookback (e.g., P1Y, P6M, P7D, PT24H)", + "--until": "ISO date filter (matched against dataset name)", + "--date-format": "Filename date format for filtering (e.g., YYYYMMDD)", + "--header": "Add custom HTTP header 'Name: Value' (repeatable)", + "--verbose": { + "help": "Verbose logging for this command", + "type": "bool", + "default": false + }, + "--quiet": { + "help": "Quiet logging for this command", + "type": "bool", + "default": false + }, + "--trace": { + "help": "Shell-style trace of key steps and external commands", + "type": "bool", + "default": false + }, + "--overwrite-existing": { + "help": "Replace local files unconditionally", + "type": "bool", + "default": false + }, + "--recheck-existing": { + "help": "Compare sizes via HTTP Content-Length when deciding to re-download", + "type": "bool", + "default": false + }, + "--min-remote-size": "Threshold for replacement (absolute bytes or relative percentage)", + "--prefer-remote": { + "help": "Always prioritize remote versions over local copies", + "type": "bool", + "default": false + }, + "--skip-if-local-done": { + "help": "Skip files that have a .done marker file", + "type": "bool", + "default": false + } + }, + "positionals": [ + { + "name": "catalog_url", + "help": "URL to a THREDDS catalog.xml document", + "type": "str", + "required": true + } + ], + "domain": "import", + "args_schema": { + "required": [ + "catalog_url" + ], + "optional": [ + "date_format", + "header", + "list_mode", + "max_depth", + "min_remote_size", + "output", + "output_dir", + "overwrite_existing", + "pattern", + "prefer_remote", + "recheck_existing", + "recursive", + "since", + "since_period", + "skip_if_local_done", + "sync_dir", + "until" + ] + }, + "example_args": null, + "impl": { + "module": "zyra.connectors.ingest", + "callable": "_cmd_thredds" + }, + "returns": "bytes" + }, "import vimeo": { "description": "Placeholder for fetching Vimeo videos by id. Not implemented yet.", "doc": "Placeholder for fetching Vimeo videos by id. Not implemented yet.", diff --git a/src/zyra/wizard/zyra_capabilities/acquire.json b/src/zyra/wizard/zyra_capabilities/acquire.json index a55a0c9b..263f445c 100644 --- a/src/zyra/wizard/zyra_capabilities/acquire.json +++ b/src/zyra/wizard/zyra_capabilities/acquire.json @@ -699,6 +699,150 @@ "--output-dir" ] }, + "acquire thredds": { + "description": "Read a THREDDS catalog.xml, map datasets to fileServer download URLs, and list, sync, or fetch matching datasets. Optionally recurse into nested catalogRef entries.", + "doc": "Read a THREDDS catalog.xml, map datasets to fileServer download URLs, and list, sync, or fetch matching datasets. Optionally recurse into nested catalogRef entries.", + "epilog": "", + "groups": [ + { + "title": "options", + "options": [ + "--help", + "--output", + "--list", + "--sync-dir", + "--output-dir", + "--recursive", + "--max-depth", + "--pattern", + "--since", + "--since-period", + "--until", + "--date-format", + "--header", + "--verbose", + "--quiet", + "--trace", + "--overwrite-existing", + "--recheck-existing", + "--min-remote-size", + "--prefer-remote", + "--skip-if-local-done" + ] + } + ], + "options": { + "--help": "show this help message and exit", + "--output": { + "help": "Output path or '-' for stdout", + "path_arg": true, + "type": "path", + "default": "-" + }, + "--list": { + "help": "List fileServer download URLs for matching datasets", + "type": "bool", + "default": false + }, + "--sync-dir": "Sync matching datasets to a local directory", + "--output-dir": { + "help": "Directory to write outputs when fetching enumerated datasets", + "path_arg": true, + "type": "path" + }, + "--recursive": { + "help": "Follow nested catalogRef entries", + "type": "bool", + "default": false + }, + "--max-depth": { + "help": "Maximum recursion depth for --recursive (default: 3)", + "type": "int", + "default": 3 + }, + "--pattern": "Regex to filter dataset urlPath", + "--since": "ISO date filter (matched against dataset name)", + "--since-period": "ISO-8601 duration for lookback (e.g., P1Y, P6M, P7D, PT24H)", + "--until": "ISO date filter (matched against dataset name)", + "--date-format": "Filename date format for filtering (e.g., YYYYMMDD)", + "--header": "Add custom HTTP header 'Name: Value' (repeatable)", + "--verbose": { + "help": "Verbose logging for this command", + "type": "bool", + "default": false + }, + "--quiet": { + "help": "Quiet logging for this command", + "type": "bool", + "default": false + }, + "--trace": { + "help": "Shell-style trace of key steps and external commands", + "type": "bool", + "default": false + }, + "--overwrite-existing": { + "help": "Replace local files unconditionally", + "type": "bool", + "default": false + }, + "--recheck-existing": { + "help": "Compare sizes via HTTP Content-Length when deciding to re-download", + "type": "bool", + "default": false + }, + "--min-remote-size": "Threshold for replacement (absolute bytes or relative percentage)", + "--prefer-remote": { + "help": "Always prioritize remote versions over local copies", + "type": "bool", + "default": false + }, + "--skip-if-local-done": { + "help": "Skip files that have a .done marker file", + "type": "bool", + "default": false + } + }, + "positionals": [ + { + "name": "catalog_url", + "help": "URL to a THREDDS catalog.xml document", + "type": "str", + "required": true + } + ], + "domain": "acquire", + "args_schema": { + "required": [ + "catalog_url" + ], + "optional": [ + "date_format", + "header", + "list_mode", + "max_depth", + "min_remote_size", + "output", + "output_dir", + "overwrite_existing", + "pattern", + "prefer_remote", + "recheck_existing", + "recursive", + "since", + "since_period", + "skip_if_local_done", + "sync_dir", + "until" + ] + }, + "example_args": null, + "impl": { + "module": "zyra.connectors.ingest", + "callable": "_cmd_thredds" + }, + "returns": "bytes" + }, "acquire vimeo": { "description": "Placeholder for fetching Vimeo videos by id. Not implemented yet.", "doc": "Placeholder for fetching Vimeo videos by id. Not implemented yet.", @@ -1407,6 +1551,150 @@ "--output-dir" ] }, + "import thredds": { + "description": "Read a THREDDS catalog.xml, map datasets to fileServer download URLs, and list, sync, or fetch matching datasets. Optionally recurse into nested catalogRef entries.", + "doc": "Read a THREDDS catalog.xml, map datasets to fileServer download URLs, and list, sync, or fetch matching datasets. Optionally recurse into nested catalogRef entries.", + "epilog": "", + "groups": [ + { + "title": "options", + "options": [ + "--help", + "--output", + "--list", + "--sync-dir", + "--output-dir", + "--recursive", + "--max-depth", + "--pattern", + "--since", + "--since-period", + "--until", + "--date-format", + "--header", + "--verbose", + "--quiet", + "--trace", + "--overwrite-existing", + "--recheck-existing", + "--min-remote-size", + "--prefer-remote", + "--skip-if-local-done" + ] + } + ], + "options": { + "--help": "show this help message and exit", + "--output": { + "help": "Output path or '-' for stdout", + "path_arg": true, + "type": "path", + "default": "-" + }, + "--list": { + "help": "List fileServer download URLs for matching datasets", + "type": "bool", + "default": false + }, + "--sync-dir": "Sync matching datasets to a local directory", + "--output-dir": { + "help": "Directory to write outputs when fetching enumerated datasets", + "path_arg": true, + "type": "path" + }, + "--recursive": { + "help": "Follow nested catalogRef entries", + "type": "bool", + "default": false + }, + "--max-depth": { + "help": "Maximum recursion depth for --recursive (default: 3)", + "type": "int", + "default": 3 + }, + "--pattern": "Regex to filter dataset urlPath", + "--since": "ISO date filter (matched against dataset name)", + "--since-period": "ISO-8601 duration for lookback (e.g., P1Y, P6M, P7D, PT24H)", + "--until": "ISO date filter (matched against dataset name)", + "--date-format": "Filename date format for filtering (e.g., YYYYMMDD)", + "--header": "Add custom HTTP header 'Name: Value' (repeatable)", + "--verbose": { + "help": "Verbose logging for this command", + "type": "bool", + "default": false + }, + "--quiet": { + "help": "Quiet logging for this command", + "type": "bool", + "default": false + }, + "--trace": { + "help": "Shell-style trace of key steps and external commands", + "type": "bool", + "default": false + }, + "--overwrite-existing": { + "help": "Replace local files unconditionally", + "type": "bool", + "default": false + }, + "--recheck-existing": { + "help": "Compare sizes via HTTP Content-Length when deciding to re-download", + "type": "bool", + "default": false + }, + "--min-remote-size": "Threshold for replacement (absolute bytes or relative percentage)", + "--prefer-remote": { + "help": "Always prioritize remote versions over local copies", + "type": "bool", + "default": false + }, + "--skip-if-local-done": { + "help": "Skip files that have a .done marker file", + "type": "bool", + "default": false + } + }, + "positionals": [ + { + "name": "catalog_url", + "help": "URL to a THREDDS catalog.xml document", + "type": "str", + "required": true + } + ], + "domain": "import", + "args_schema": { + "required": [ + "catalog_url" + ], + "optional": [ + "date_format", + "header", + "list_mode", + "max_depth", + "min_remote_size", + "output", + "output_dir", + "overwrite_existing", + "pattern", + "prefer_remote", + "recheck_existing", + "recursive", + "since", + "since_period", + "skip_if_local_done", + "sync_dir", + "until" + ] + }, + "example_args": null, + "impl": { + "module": "zyra.connectors.ingest", + "callable": "_cmd_thredds" + }, + "returns": "bytes" + }, "import vimeo": { "description": "Placeholder for fetching Vimeo videos by id. Not implemented yet.", "doc": "Placeholder for fetching Vimeo videos by id. Not implemented yet.", diff --git a/tests/cli/test_acquire_thredds.py b/tests/cli/test_acquire_thredds.py new file mode 100644 index 00000000..898eb2a7 --- /dev/null +++ b/tests/cli/test_acquire_thredds.py @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: Apache-2.0 +"""CLI-level tests for ``zyra acquire thredds`` (hermetic, no network).""" + +from __future__ import annotations + +from unittest.mock import patch + +from zyra.cli import main as cli_main + +CATALOG = """ + + + + + +""" + + +def test_acquire_thredds_list(capsys): + with patch("zyra.connectors.backends.http.fetch_text", return_value=CATALOG): + rc = cli_main( + [ + "acquire", + "thredds", + "https://thredds.example.com/thredds/catalog/foo/catalog.xml", + "--list", + "--pattern", + r"\.grib2$", + ] + ) + assert rc in (0, None) + out = capsys.readouterr().out + assert "https://thredds.example.com/thredds/fileServer/foo/a.grib2" in out + assert "b.txt" not in out + + +def test_acquire_thredds_sync(tmp_path): + with ( + patch("zyra.connectors.backends.http.fetch_text", return_value=CATALOG), + patch("zyra.connectors.backends.http.fetch_bytes", return_value=b"DATA"), + ): + rc = cli_main( + [ + "acquire", + "thredds", + "https://thredds.example.com/thredds/catalog/foo/catalog.xml", + "--sync-dir", + str(tmp_path), + "--pattern", + r"\.grib2$", + ] + ) + assert rc in (0, None) + assert (tmp_path / "a.grib2").read_bytes() == b"DATA" + assert not (tmp_path / "b.txt").exists() diff --git a/tests/connectors/test_thredds_backend.py b/tests/connectors/test_thredds_backend.py new file mode 100644 index 00000000..bbbe52cb --- /dev/null +++ b/tests/connectors/test_thredds_backend.py @@ -0,0 +1,178 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for the THREDDS connector backend (hermetic, no network).""" + +from __future__ import annotations + +import pytest + +from zyra.connectors.backends import thredds as thr + +CATALOG_URL = "https://thredds.example.com/thredds/catalog/foo/catalog.xml" + +FLAT_CATALOG = """ + + + + + + + + 2024-01-01T00:00:00Z + + + + + +""" + +ROOT_CATALOG = """ + + + + + +""" + +SUB_CATALOG = """ + + + + +""" + + +def test_parse_catalog_builds_fileserver_urls(): + datasets, refs = thr.parse_catalog(FLAT_CATALOG, CATALOG_URL) + assert refs == [] + urls = {d.url_path: d.download_url for d in datasets} + assert ( + urls["foo/foo_20240101.grib2"] + == "https://thredds.example.com/thredds/fileServer/foo/foo_20240101.grib2" + ) + # The collection wrapper (no urlPath) is excluded; only leaf datasets remain. + assert len(datasets) == 3 + # Date element is captured. + d0 = next(d for d in datasets if d.url_path == "foo/foo_20240101.grib2") + assert d0.date == "2024-01-01T00:00:00Z" + + +def test_pattern_filter(): + out = thr.list_files(CATALOG_URL, catalog_xml=FLAT_CATALOG, pattern=r"\.grib2$") + assert all(u.endswith(".grib2") for u in out) + assert len(out) == 2 + + +def test_date_filter(): + out = thr.enumerate_datasets( + CATALOG_URL, + catalog_xml=FLAT_CATALOG, + pattern=r"\.grib2$", + since="2024-01-02", + date_format="%Y%m%d", + ) + assert [d.url_path for d in out] == ["foo/foo_20240102.grib2"] + + +def test_no_recursion_by_default(): + out = thr.list_files(CATALOG_URL, catalog_xml=ROOT_CATALOG) + assert out == ["https://thredds.example.com/thredds/fileServer/root/top.grib2"] + + +def test_recursion_follows_catalog_ref(): + def fetcher(url: str) -> str: + if url.endswith("sub/catalog.xml"): + return SUB_CATALOG + raise AssertionError(f"unexpected fetch: {url}") + + out = thr.list_files( + ROOT_CATALOG_URL + := "https://thredds.example.com/thredds/catalog/root/catalog.xml", + catalog_xml=ROOT_CATALOG, + recursive=True, + fetcher=fetcher, + ) + assert out == [ + "https://thredds.example.com/thredds/fileServer/root/top.grib2", + "https://thredds.example.com/thredds/fileServer/root/sub/nested.grib2", + ] + + +def test_recursion_respects_max_depth(): + def fetcher(url: str) -> str: + return SUB_CATALOG + + out = thr.list_files( + "https://thredds.example.com/thredds/catalog/root/catalog.xml", + catalog_xml=ROOT_CATALOG, + recursive=True, + max_depth=0, + fetcher=fetcher, + ) + # Depth 0 means only the root catalog is read. + assert out == ["https://thredds.example.com/thredds/fileServer/root/top.grib2"] + + +def test_sync_directory_downloads_and_skips(tmp_path, monkeypatch): + calls: list[str] = [] + + def fake_fetch_bytes(url, *, timeout=60, headers=None): + calls.append(url) + return b"DATA" + + monkeypatch.setattr(thr.http_backend, "fetch_bytes", fake_fetch_bytes) + + written = thr.sync_directory( + CATALOG_URL, + str(tmp_path), + catalog_xml=FLAT_CATALOG, + pattern=r"\.grib2$", + ) + assert len(written) == 2 + assert (tmp_path / "foo_20240101.grib2").read_bytes() == b"DATA" + + # Second sync skips existing non-empty files (no new downloads). + calls.clear() + written2 = thr.sync_directory( + CATALOG_URL, + str(tmp_path), + catalog_xml=FLAT_CATALOG, + pattern=r"\.grib2$", + ) + assert written2 == [] + assert calls == [] + + +def test_sync_directory_overwrite(tmp_path, monkeypatch): + monkeypatch.setattr( + thr.http_backend, + "fetch_bytes", + lambda url, *, timeout=60, headers=None: b"NEW", + ) + (tmp_path / "foo_20240102.grib2").write_bytes(b"OLD") + written = thr.sync_directory( + CATALOG_URL, + str(tmp_path), + catalog_xml=FLAT_CATALOG, + pattern=r"foo_20240102", + sync_options=thr.SyncOptions(overwrite_existing=True), + ) + assert (tmp_path / "foo_20240102.grib2").read_bytes() == b"NEW" + assert len(written) == 1 + + +def test_missing_http_server_defaults_base(): + xml = ( + '' + '' + ) + datasets, _ = thr.parse_catalog(xml, CATALOG_URL) + assert datasets[0].download_url.endswith("/thredds/fileServer/a/x.grib2") + + +if __name__ == "__main__": # pragma: no cover + pytest.main([__file__, "-v"]) diff --git a/tests/snapshots/openapi_sha256.txt b/tests/snapshots/openapi_sha256.txt index 99e57075..6effd2f4 100644 --- a/tests/snapshots/openapi_sha256.txt +++ b/tests/snapshots/openapi_sha256.txt @@ -1 +1 @@ -afddec39bddbd2ec4340f6beddf77275ce324618d99b7ff3b9a765071983b569 +a82847af652ef97e8ff4c8835e8c898fc29c23f997eff6fa2984d409e833efaf