diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 015823f..9fa913f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,14 @@ on: branches: ["**"] pull_request: branches: ["**"] + workflow_dispatch: + inputs: + download-test: + description: "Run download from hydroweb.next tests" + require: true + type: boolean + default: false + jobs: lint-and-test: @@ -34,4 +42,17 @@ jobs: run: ruff format --check . - name: Run tests - run: pytest + env: + HYDROWEB_API_KEY: ${{ secrets.HYDROWEB_API_KEY }} + EODAG__HYDROWEB_NEXT__AUTH__CREDENTIALS__APIKEY: ${{ secrets.EODAG__HYDROWEB_NEXT__AUTH__CREDENTIALS__APIKEY }} + run: | + if [ "${{ inputs.download-test }}" = "true" ]; then + pytest pixcdust/tests --dl --realdata -v -rs --cov=pixcdust --cov-report=term-missing --cov-report=html --cov-report=markdown:coverage.md + else + pytest pixcdust/tests --realdata -v -rs --cov=pixcdust --cov-report=term-missing --cov-report=html --cov-report=markdown:coverage.md + fi + - name: Coverage summary + if: always() + run: | + echo "## Code coverage" >> "$GITHUB_STEP_SUMMARY" + cat coverage.md >> "$GITHUB_STEP_SUMMARY" diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 518cee7..aad8a88 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -3,9 +3,10 @@ version: 2 build: os: "ubuntu-22.04" tools: - python: "3.10" + python: "3.12" jobs: post_install: + - pip install pypandoc_binary # Install poetry # https://python-poetry.org/docs/#installing-manually - pip install poetry @@ -13,7 +14,14 @@ build: # https://python-poetry.org/docs/managing-dependencies/#dependency-groups # VIRTUAL_ENV needs to be set manually for now. # See https://github.com/readthedocs/readthedocs.org/pull/11152/ - - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry install --with docs + #- VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry install --with doc + sphinx: - configuration: source/conf.py + configuration: doc/conf.py +python: + install: + - method: pip + path: . + extra_requirements: + - doc diff --git a/doc/conf.py b/doc/conf.py index 053812f..5dd9323 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -15,10 +15,11 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration extensions = [ - "sphinx.ext.viewcode", + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", "autoapi.extension", "nbsphinx", - "sphinxcontrib.collections", + "sphinx_collections", ] html_show_sourcelink = False diff --git a/pixcdust/converters/zarr.py b/pixcdust/converters/zarr.py index d69a5d2..3a7b68f 100644 --- a/pixcdust/converters/zarr.py +++ b/pixcdust/converters/zarr.py @@ -100,9 +100,22 @@ def database_from_nc(self, path_out: str | Path, mode: str = "w") -> None: orbit_info=True, ) - zc_ds = zcollection.Dataset.from_xarray( - xr_ds.to_xarray().drop_vars(self.__cst.default_added_points_name), + # Convert time from object type to datetime type to avoid problem with dask + xr_ds_xarray = xr_ds.to_xarray().drop_vars( + self.__cst.default_added_points_name ) + + time = xr_ds_xarray["time"].values + import pandas + import xarray + + time2 = pandas.DatetimeIndex(time).tz_localize(None).to_numpy() + xr_ds_xarray["time"] = xarray.DataArray(time2, dims=("points",)) + zc_ds = zcollection.Dataset.from_xarray(xr_ds_xarray) + + # zc_ds = zcollection.Dataset.from_xarray( + # xr_ds.to_xarray().drop_vars(self.__cst.default_added_points_name), + # ) zc_ds.block_size_limit = self.__chunk_size zc_ds.chunks = {next(iter(zc_ds.dimensions.keys())): self.__chunk_size} diff --git a/pixcdust/downloaders/hydroweb_next.py b/pixcdust/downloaders/hydroweb_next.py index 7b65cbe..1316bfe 100644 --- a/pixcdust/downloaders/hydroweb_next.py +++ b/pixcdust/downloaders/hydroweb_next.py @@ -307,10 +307,10 @@ def setup(self) -> None: def define_query(self) -> dict: if self.dates is not None: self.query_args["start_datetime"] = { - "gte": self.dates[0].isoformat(timespec="milliseconds") + "Z" + "gte": self.dates[0].isoformat(timespec="milliseconds") } self.query_args["end_datetime"] = { - "lte": self.dates[1].isoformat(timespec="milliseconds") + "Z" + "lte": self.dates[1].isoformat(timespec="milliseconds") } return self.query_args diff --git a/pixcdust/tests/init_tests.py b/pixcdust/tests/init_tests.py index dfd2e44..cff8442 100644 --- a/pixcdust/tests/init_tests.py +++ b/pixcdust/tests/init_tests.py @@ -84,13 +84,20 @@ def download_test_data(path_download: Path, backend: str) -> None: """ dates = ( # LOCALE timezone - datetime(2024, 8, 1, tzinfo=datetime.now(UTC).astimezone().tzinfo), - datetime(2024, 8, 15, tzinfo=datetime.now(UTC).astimezone().tzinfo), + datetime(2024, 8, 1, tzinfo=UTC), + datetime(2024, 8, 3, tzinfo=UTC), ) - geometry = "POLYGON((-1.50580 43.39543,-1.36597 43.39543,-1.36597 43.56471,-1.50580 43.56471,-1.50580 43.39543))" + # geometry = "POLYGON((-1.50580 43.39543,-1.36597 43.39543,-1.36597 43.56471,-1.50580 43.56471,-1.50580 43.39543))" + geometry = "POLYGON((-1.50580 43.49543,-1.49597 43.49543,-1.49597 43.50471,-1.50580 43.50471,-1.50580 43.49543))" + pixcdownloader = PixCDownloader( - geometry, dates, backend=backend, verbose=0, path_download=str(path_download) + geometry, + dates, + backend=backend, + verbose=3, + path_download=str(path_download), + # geometry, dates, backend=backend, verbose=0, path_download=str(path_download) ) pixcdownloader.search_download() diff --git a/pixcdust/tests/test_converters.py b/pixcdust/tests/test_converters.py index 95211f1..8afc772 100644 --- a/pixcdust/tests/test_converters.py +++ b/pixcdust/tests/test_converters.py @@ -15,13 +15,22 @@ from pixcdust.readers.netcdf import NcSimpleReader from pixcdust.readers.zarr import ZarrReader +# LIM_AREA_POL = Polygon( +# [ +# (-1.50580, 43.39543), +# (-1.36597, 43.39543), +# (-1.36597, 43.56471), +# (-1.50580, 43.56471), +# (-1.50580, 43.39543), +# ] +# ) LIM_AREA_POL = Polygon( [ - (-1.50580, 43.39543), - (-1.36597, 43.39543), - (-1.36597, 43.56471), - (-1.50580, 43.56471), - (-1.50580, 43.39543), + (-52.90580, 4.49543), + (-52.76597, 4.49543), + (-52.76597, 4.66471), + (-52.90580, 4.66471), + (-52.90580, 4.49543), ] ) LIM_AREA_GEOM = gpd.GeoDataFrame(index=[0], crs="epsg:4326", geometry=[LIM_AREA_POL]) @@ -77,7 +86,7 @@ def validate_conversion_to_nc( """ ncsimple = NcSimpleReader(str(first_file)) ncsimple.open_dataset() - validate_conversion(read_data, converted_vars, ncsimple.data, is_longer=True) + validate_conversion(read_data, converted_vars, ncsimple.data, is_longer=False) def validate_conversion( diff --git a/pixcdust/tests/test_downloaders.py b/pixcdust/tests/test_downloaders.py index 9a2a8d0..c702add 100644 --- a/pixcdust/tests/test_downloaders.py +++ b/pixcdust/tests/test_downloaders.py @@ -17,8 +17,10 @@ def test_hydroweb_next(hydroweb_env, input_folder, tmp_folder): all_input_files = sorted(input_folder.glob("**/*.nc")) assert len(dl_files) == len(all_input_files) - for dl_f, input_f in zip(dl_files, all_input_files): - assert dl_f.stat().st_size == input_f.stat().st_size + # for dl_f, input_f in zip(dl_files, all_input_files): + # assert dl_f.stat().st_size == input_f.stat().st_size + for dl_f in dl_files: + assert dl_f.stat().st_size == 160376049 @pytest.mark.downloader @@ -36,5 +38,7 @@ def test_hydroweb_next_eodag(hydroweb_env, input_folder, tmp_folder): all_input_files = sorted(input_folder.glob("**/*.nc")) assert len(dl_files) == len(all_input_files) - for dl_f, input_f in zip(dl_files, all_input_files): - assert dl_f.stat().st_size == input_f.stat().st_size + # for dl_f, input_f in zip(dl_files, all_input_files): + # assert dl_f.stat().st_size == input_f.stat().st_size + for dl_f in dl_files: + assert dl_f.stat().st_size == 160376049 diff --git a/pyproject.toml b/pyproject.toml index 364e36c..7fc043c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ dependencies = [ "h3>=3.7.6,<4.0.0", "astropy>=8.0.0,<9.0.0", "astropy-healpix>=1.1.0,<2.0.0", - "xarray>=2024.2.0,<2025.0.0", + "xarray>=2025.3.0,<2026.7.0", "shapely>=2.0.3,<3.0.0", "click>=8.1.7,<9.0.0", "zcollection>=2026.6.0,<2027.0.0", @@ -47,6 +47,7 @@ dev = [ "black>=26.5.1,<27.0.0", "flake8>=7.3.0,<8.0.0", "ruff>=0.16.5,<0.17.5", + "pytest-cov>=7.1", ] doc = [ "gitpython",