diff --git a/README.md b/README.md index 706d880..6169f04 100644 --- a/README.md +++ b/README.md @@ -47,20 +47,7 @@ This library doesn't have knowledge of those datasets, it forms the basis for pr ## Limitations -There are some major limitations to this as a general purpose tool, these will hopefully be dealt with in time! I'm raising issues as I go - -**This is currently very heavy development functionality, but the following commands already work**: - -* preprocess_missing_spatial - poorly at present due to missing mask backref implementation -* preprocess_missing_time -* preprocess_regrid -* preprocess_rotate -* preprocess_dataset -* preprocess_loader_init -* preprocess_add_mask -* preprocess_add_channel - -Other stubs probably don't work, unless I forgot to update these docs! +**This is currently active development, but the commands work to most degrees.** ## Contributing diff --git a/docs/acknowledgements.md b/docs/acknowledgements.md new file mode 100644 index 0000000..68f2734 --- /dev/null +++ b/docs/acknowledgements.md @@ -0,0 +1,4 @@ +# Acknowledgements +These are automatically generated from the git repository. + +{{ git_site_authors }} \ No newline at end of file diff --git a/docs/change_log.md b/docs/change_log.md new file mode 100644 index 0000000..90cb31c --- /dev/null +++ b/docs/change_log.md @@ -0,0 +1 @@ +--8<-- "CHANGELOG.md" \ No newline at end of file diff --git a/docs/commands.md b/docs/commands.md new file mode 100644 index 0000000..d0580f5 --- /dev/null +++ b/docs/commands.md @@ -0,0 +1,19 @@ +# Commands + +## Datasets +### preprocess_missing_spatial +### preprocess_missing_time +### preprocess_regrid +### preprocess_reproject +### preprocess_rotate + +## Loaders +### preprocess_loader_init +### preprocess_loader_copy + +### preprocess_add_mask +### preprocess_add_channel + +## AI-ready datasets +### preprocess_dataset +### preprocess_add_processed diff --git a/docs/index.md b/docs/index.md index fc9d69a..80084f0 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,8 +4,4 @@ Check out the [usage](usage.md) section for further information, including how t Check out the [API](preprocess_toolbox/base.md) if you're keen to go deep. -!!! note - - This project is under active development. - {!README.md!} diff --git a/docs/license.md b/docs/license.md new file mode 100644 index 0000000..64ad9b4 --- /dev/null +++ b/docs/license.md @@ -0,0 +1 @@ +--8<-- "LICENSE" \ No newline at end of file diff --git a/docs/preprocess_toolbox/base.md b/docs/preprocess_toolbox/base.md deleted file mode 100644 index acacc64..0000000 --- a/docs/preprocess_toolbox/base.md +++ /dev/null @@ -1,4 +0,0 @@ -title: base ---- -::: preprocess_toolbox.base - diff --git a/docs/preprocess_toolbox/cli.md b/docs/preprocess_toolbox/cli.md deleted file mode 100644 index ca16186..0000000 --- a/docs/preprocess_toolbox/cli.md +++ /dev/null @@ -1,4 +0,0 @@ -title: cli ---- -::: preprocess_toolbox.cli - diff --git a/docs/preprocess_toolbox/dataset/cli.md b/docs/preprocess_toolbox/dataset/cli.md deleted file mode 100644 index 6c99446..0000000 --- a/docs/preprocess_toolbox/dataset/cli.md +++ /dev/null @@ -1,4 +0,0 @@ -title: preprocess_toolbox.dataset.cli ---- -::: preprocess_toolbox.dataset.cli - diff --git a/docs/preprocess_toolbox/dataset/process.md b/docs/preprocess_toolbox/dataset/process.md deleted file mode 100644 index bb2799a..0000000 --- a/docs/preprocess_toolbox/dataset/process.md +++ /dev/null @@ -1,4 +0,0 @@ -title: preprocess_toolbox.dataset.process ---- -::: preprocess_toolbox.dataset.process - diff --git a/docs/preprocess_toolbox/dataset/spatial.md b/docs/preprocess_toolbox/dataset/spatial.md deleted file mode 100644 index 0fe7871..0000000 --- a/docs/preprocess_toolbox/dataset/spatial.md +++ /dev/null @@ -1,4 +0,0 @@ -title: preprocess_toolbox.dataset.spatial ---- -::: preprocess_toolbox.dataset.spatial - diff --git a/docs/preprocess_toolbox/dataset/time.md b/docs/preprocess_toolbox/dataset/time.md deleted file mode 100644 index 93e072e..0000000 --- a/docs/preprocess_toolbox/dataset/time.md +++ /dev/null @@ -1,4 +0,0 @@ -title: preprocess_toolbox.dataset.time ---- -::: preprocess_toolbox.dataset.time - diff --git a/docs/preprocess_toolbox/interface.md b/docs/preprocess_toolbox/interface.md deleted file mode 100644 index 87f8dbc..0000000 --- a/docs/preprocess_toolbox/interface.md +++ /dev/null @@ -1,4 +0,0 @@ -title: interface ---- -::: preprocess_toolbox.interface - diff --git a/docs/preprocess_toolbox/loader/cli.md b/docs/preprocess_toolbox/loader/cli.md deleted file mode 100644 index a3ca342..0000000 --- a/docs/preprocess_toolbox/loader/cli.md +++ /dev/null @@ -1,4 +0,0 @@ -title: preprocess_toolbox.loader.cli ---- -::: preprocess_toolbox.loader.cli - diff --git a/docs/preprocess_toolbox/loader/utils.md b/docs/preprocess_toolbox/loader/utils.md deleted file mode 100644 index 897294b..0000000 --- a/docs/preprocess_toolbox/loader/utils.md +++ /dev/null @@ -1,4 +0,0 @@ -title: preprocess_toolbox.loader.utils ---- -::: preprocess_toolbox.loader.utils - diff --git a/docs/preprocess_toolbox/models.md b/docs/preprocess_toolbox/models.md deleted file mode 100644 index 610069a..0000000 --- a/docs/preprocess_toolbox/models.md +++ /dev/null @@ -1,4 +0,0 @@ -title: models ---- -::: preprocess_toolbox.models - diff --git a/docs/preprocess_toolbox/processor.md b/docs/preprocess_toolbox/processor.md deleted file mode 100644 index 1f5f9c6..0000000 --- a/docs/preprocess_toolbox/processor.md +++ /dev/null @@ -1,4 +0,0 @@ -title: processor ---- -::: preprocess_toolbox.processor - diff --git a/docs/preprocess_toolbox/utils.md b/docs/preprocess_toolbox/utils.md deleted file mode 100644 index 51e574c..0000000 --- a/docs/preprocess_toolbox/utils.md +++ /dev/null @@ -1,4 +0,0 @@ -title: utils ---- -::: preprocess_toolbox.utils - diff --git a/mkdocs.yml b/mkdocs.yml index fb7e6f8..65c27ca 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -4,35 +4,45 @@ theme: highlightjs: true nav: - Home: 'index.md' + - 'commands.md' - 'usage.md' - - API: - - preprocess.dataset: - - "preprocess_toolbox/dataset/cli.md" - - "preprocess_toolbox/dataset/process.md" - - "preprocess_toolbox/dataset/spatial.md" - - "preprocess_toolbox/dataset/time.md" - - preprocess.loader: - - "preprocess_toolbox/loader/cli.md" - - "preprocess_toolbox/loader/utils.md" - - "preprocess_toolbox/base.md" - - "preprocess_toolbox/interface.md" - - "preprocess_toolbox/models.md" - - "preprocess_toolbox/processor.md" - - "preprocess_toolbox/utils.md" - - "preprocess_toolbox/cli.md" - GitHub: 'https://github.com/environmental-forecasting/preprocess-toolbox' plugins: - search + # Enables automatically generating authors from git repo, see acknowledgements.md + - git-authors + # Automatically generates API section from docstrings + - mkdocs-autoapi: + autoapi_add_nav_entry: API + autoapi_ignore: + - setup.py + - autorefs: + resolve_closest: true - mkdocstrings: handlers: python: + paths: + - . options: - docstring_style: sphinx + show_submodules: true + docstring_style: google heading_level: 3 markdown_extensions: - markdown_include.include: base_path: . - admonition + # Enables code block highlighting + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.superfences + - pymdownx.blocks.caption + - pymdownx.tabbed: + alternate_style: true +copyright: Copyright © 2025 British Antarctic Survey / Alan Turing Institute # Include extra CSS to make some style adjustments extra_css: diff --git a/preprocess_toolbox/base.py b/preprocess_toolbox/base.py index 4c74da3..2f2417d 100644 --- a/preprocess_toolbox/base.py +++ b/preprocess_toolbox/base.py @@ -144,7 +144,7 @@ def get_dataset(self, for var_filepaths in self.processed_files[vn].values()] logging.info("Got {} filenames to open dataset with!".format(len(var_files))) - logging.debug(pformat(var_files)) + # logging.debug(pformat(var_files)) # TODO: where's my parallel mfdataset please!? with (dask.config.set(**{'array.slicing.split_large_chunks': True})): diff --git a/preprocess_toolbox/dataset/process.py b/preprocess_toolbox/dataset/process.py index 2cda6ad..9f52582 100644 --- a/preprocess_toolbox/dataset/process.py +++ b/preprocess_toolbox/dataset/process.py @@ -150,6 +150,9 @@ def rotate_dataset(ref_file: os.PathLike, wind_cubes[vars_to_rotate[1]], angles, ) + if len(wind_cubes_r[vars_to_rotate[0]].shape) == 2 and len(wind_cubes_r[vars_to_rotate[1]].shape) == 2: + wind_cubes_r[vars_to_rotate[0]] = iris.util.new_axis(wind_cubes_r[vars_to_rotate[0]], "time") + wind_cubes_r[vars_to_rotate[1]] = iris.util.new_axis(wind_cubes_r[vars_to_rotate[1]], "time") except iris.exceptions.CoordinateNotFoundError: logging.exception("Failure to rotate due to coordinate issues. " "moving onto next file") diff --git a/preprocess_toolbox/interface.py b/preprocess_toolbox/interface.py index cddc5e3..b2552b1 100644 --- a/preprocess_toolbox/interface.py +++ b/preprocess_toolbox/interface.py @@ -34,7 +34,6 @@ def get_processor_implementation(config: os.PathLike) -> object: create_kwargs = dict(**remaining) logging.info("Attempting to instantiate {} with loaded configuration".format(implementation)) - logging.debug("Converted kwargs from the retrieved configuration: {}".format(create_kwargs)) return implementation(**create_kwargs) @@ -57,7 +56,6 @@ def get_processor_from_source(identifier: str, source_cfg: dict) -> object: create_kwargs = {k: v for k, v in source_cfg.items() if k not in ["dataset_config", "implementation"]} logging.info("Attempting to instantiate {} with loaded configuration".format(source_cfg["implementation"])) - logging.debug("Converted kwargs from the retrieved configuration: {}".format(create_kwargs)) return get_implementation(source_cfg["implementation"])( get_dataset_config_implementation(source_cfg["dataset_config"]), diff --git a/preprocess_toolbox/processor.py b/preprocess_toolbox/processor.py index 4aa9778..8a483b9 100644 --- a/preprocess_toolbox/processor.py +++ b/preprocess_toolbox/processor.py @@ -95,8 +95,16 @@ def __init__(self, self._normalisation_splits = [] if normalisation_splits is None else normalisation_splits self._parallel = parallel_opens self._refdir = ref_procdir + + ## + # Split dates - + # + # TODO: splits -> { dates, sources }, but currently sources are separate... self._splits = splits + self._dropped_split_dates = {} + # TODO: add self._dropped_dates based on DATA + self._source_files = dict() if init_source: @@ -223,45 +231,49 @@ def _init_source_data(self, :return: """ - split_dates_required = dict() drop_dates = dict() + all_dates = dict() for split in self._splits.keys(): - dates = sorted(self._splits[split]) + all_dates[split] = sorted(self._splits[split]) drop_dates[split] = list() - if dates: + if all_dates[split]: logging.info("Processing {} dates for {} category: {} - {}". - format(len(dates), split, min(dates), max(dates))) + format(len(all_dates[split]), split, min(all_dates[split]), max(all_dates[split]))) else: logging.info("No {} dates for this processor".format(split)) continue # Calculating lead and lag dates that aren't already accounted for in splits - if self._lag_time > 0: + if self._lag_time >= 0: logging.info("Including lag of {} {}s".format(self._lag_time, ds_config.frequency.attribute)) - additional_lag_dates, dropped_lag_dates = get_extension_dates(ds_config, dates, self._lag_time, reverse=True) - dates += additional_lag_dates + additional_lag_dates, dropped_lag_dates = get_extension_dates( + ds_config, all_dates[split], + # We offset by two, because -1 is channel one, so we need to account for lag == 1 being -2 + self._lag_time + 2, + start_step=1, reverse=True) + all_dates[split] += additional_lag_dates drop_dates[split] += dropped_lag_dates logging.info("Lag added {} dates for {} category: {} - {}". - format(len(dates), split, min(dates), max(dates))) + format(len(all_dates[split]), split, min(all_dates[split]), max(all_dates[split]))) if self._lead_time > 0: logging.info("Including lead of {} {}s".format(self._lead_time, ds_config.frequency.attribute)) - additional_lead_dates, dropped_lead_dates = get_extension_dates(ds_config, dates, self._lead_time) - dates += additional_lead_dates + additional_lead_dates, dropped_lead_dates = get_extension_dates(ds_config, all_dates[split], self._lead_time) + all_dates[split] += additional_lead_dates drop_dates[split] += dropped_lead_dates logging.info("Lead added {} dates for {} category: {} - {}". - format(len(dates), split, min(dates), max(dates))) + format(len(all_dates[split]), split, min(all_dates[split]), max(all_dates[split]))) - split_dates_required[split] = sorted([_ for _ in dates if _ not in drop_dates[split]]) + self._dropped_split_dates[split] = sorted(drop_dates[split]) + all_dates[split] = sorted([_ for _ in all_dates[split] if _ not in drop_dates[split]]) for split in self._splits.keys(): - self._source_files[split] = {var_config.name: ds_config.var_filepaths(var_config, split_dates_required[split]) + self._source_files[split] = {var_config.name: ds_config.var_filepaths(var_config, all_dates[split]) for var_config in ds_config.variables} for var_name, var_files in self._source_files[split].items(): logging.info("Got {} files for {}:{}".format(len(var_files), split, var_name)) - logging.debug(pformat(self._source_files)) def _normalise_array_mean(self, var_name: str, da: object, denormalise: bool=False): """ @@ -341,7 +353,6 @@ def _normalise_array_scaling(self, var_name: str, da: object, denormalise: bool= elif self.norm_split_dates: logging.debug("Generating norm-scaling min-max from {} training " "dates".format(len(self.norm_split_dates))) - norm_samples = da.sel(time=self.norm_split_dates).data norm_samples = norm_samples.ravel() @@ -375,7 +386,8 @@ def _process_channel(self, for split, var_files in self.source_files.items() for vn, files in var_files.items() for file in files - if var_name == vn]))) + if var_name == vn + and os.path.exists(file)]))) if len(source_files) > 0: logging.info("Opening {} files for {}".format(len(source_files), var_name)) @@ -384,22 +396,15 @@ def _process_channel(self, # data so this was harder. Now we work with whatever we get from download-toolbox ds = xr.open_mfdataset( source_files, - # Solves issue with inheriting files without - # time dimension (only having coordinate) - combine="nested", - concat_dim="time", - coords="minimal", - compat="override", - # TODO: review this, but if lat-lon is in the file, it's signalling bigger issues - # drop_variables=("lat", "lon"), - parallel=self._parallel) + parallel=self._parallel, + lock=False) da = getattr(ds, var_name) da = da.astype(self.dtype) + logging.debug("Files to be opened: {}".format(da.dims)) # FIXME: we should ideally store train dates against the # normalisation and climatology, to ensure recalculation on # reprocess. All this need be is in the path, to be honest - if var_suffix == "anom": if len(self._anom_clim_splits) < 1 and self._refdir is None: raise ProcessingError("You must provide a list of splits via " @@ -507,7 +512,9 @@ def get_config(self, **kwargs): "path": self.path, "processed_files": self._processed_files, "source_files": self._source_files, - "splits": self.splits, + "splits": {split: [ + date for date in dates if date not in self._dropped_split_dates[split] + ] for split, dates in self._splits.items()}, } @staticmethod @@ -588,7 +595,6 @@ def lead_time(self) -> int: @property def norm_split_dates(self): - # TODO: functools.cached_property, though slightly odd behaviour re. write-ability return [date for clim_split in self._normalisation_splits for date in self._splits[clim_split]] diff --git a/preprocess_toolbox/utils.py b/preprocess_toolbox/utils.py index 5ecc4dc..7862dfb 100644 --- a/preprocess_toolbox/utils.py +++ b/preprocess_toolbox/utils.py @@ -6,8 +6,10 @@ from dateutil.relativedelta import relativedelta import orjson +import pandas as pd +import xarray as xr -from download_toolbox.interface import DatasetConfig +from download_toolbox.interface import DatasetConfig, Frequency def get_config(config_path: os.PathLike): @@ -35,25 +37,43 @@ def get_config_filename(args: argparse.Namespace, prefix: str = "loader"): def get_extension_dates(ds_config: DatasetConfig, dates: list, num_steps: int, - reverse=False): + start_step: int = 0, + reverse: bool = False): additional_dates, dropped_dates = [], [] for date in dates: - for time in range(num_steps): - attrs = {"{}s".format(ds_config.frequency.attribute): time + 1} + for time in range(start_step, num_steps): + attrs = {"{}s".format(ds_config.frequency.attribute): time} op = operator.sub if reverse else operator.add extended_date = op(date, relativedelta(**attrs)) - if extended_date not in dates: - if all([os.path.exists(ds_config.var_filepath(var_config, [extended_date])) - for var_config in ds_config.variables]): - # We only add these dates into the mix if all necessary files exist - additional_dates.append(extended_date) + if ds_config.frequency == Frequency.MONTH: + extended_date = pd.to_datetime(extended_date + pd.offsets.MonthEnd(0)).date() + + # Check we don't know we have data, and also ignore previous occurrences + if extended_date not in dates and extended_date not in additional_dates: + extended_date_var_files = [ds_config.var_filepath(var_config, [extended_date]) + for var_config in ds_config.variables] + if all([os.path.exists(df) for df in extended_date_var_files]): + # The above will catch those items that fall outside the file output boundary, but not missing + # dates within ALL files. This next clause is more expensive, but necessary to catch everything! + logging.debug("Files exist, double checking whether {} appears in data itself across {} files". + format(extended_date, len(extended_date_var_files))) + + # TODO: this won't catch partially available dates where not all files have the date, but some do + if pd.Timestamp(extended_date) in xr.open_mfdataset(extended_date_var_files).time.values: + # We only add these dates into the mix if all necessary files exist + additional_dates.append(extended_date) + else: + logging.warning("Nope, {} not in data itself so dropping {}".format(extended_date, date)) + dropped_dates.append(date) + break else: # Otherwise, warn that the lag data means this is being dropped logging.warning("{} will be dropped due to missing data {}". format(date, extended_date)) dropped_dates.append(date) + break return sorted(list(set(additional_dates))), sorted(list(set(dropped_dates))) diff --git a/pyproject.toml b/pyproject.toml index 68c0951..a3b3038 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,16 @@ docs = [ "mkdocs", "mkdocstrings[python]", "markdown-include", - "mkdocs-material" + "mkdocs-material", + "mkdocs-autoapi", + "mkdocs-git-authors-plugin", +] +lint = [ + "flake8" +] +tests = [ + "pytest", + "pytest-cov" ] [tool.setuptools.packages.find]