From 2f9379e441a35c403caf55795916f11602b4cdd2 Mon Sep 17 00:00:00 2001 From: James Byrne Date: Tue, 27 May 2025 21:18:03 +0100 Subject: [PATCH 01/15] Filtering non-existent files in configuration --- preprocess_toolbox/processor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/preprocess_toolbox/processor.py b/preprocess_toolbox/processor.py index 4aa9778..56a3e55 100644 --- a/preprocess_toolbox/processor.py +++ b/preprocess_toolbox/processor.py @@ -375,7 +375,8 @@ def _process_channel(self, for split, var_files in self.source_files.items() for vn, files in var_files.items() for file in files - if var_name == vn]))) + if var_name == vn + and os.path.exists(file)]))) if len(source_files) > 0: logging.info("Opening {} files for {}".format(len(source_files), var_name)) From 0189534500e3320cc3826f0b401f8212d6000d88 Mon Sep 17 00:00:00 2001 From: James Byrne Date: Wed, 28 May 2025 12:53:13 +0100 Subject: [PATCH 02/15] Removing further crazy debugging --- preprocess_toolbox/base.py | 2 +- preprocess_toolbox/interface.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/preprocess_toolbox/base.py b/preprocess_toolbox/base.py index 4c74da3..2f2417d 100644 --- a/preprocess_toolbox/base.py +++ b/preprocess_toolbox/base.py @@ -144,7 +144,7 @@ def get_dataset(self, for var_filepaths in self.processed_files[vn].values()] logging.info("Got {} filenames to open dataset with!".format(len(var_files))) - logging.debug(pformat(var_files)) + # logging.debug(pformat(var_files)) # TODO: where's my parallel mfdataset please!? with (dask.config.set(**{'array.slicing.split_large_chunks': True})): diff --git a/preprocess_toolbox/interface.py b/preprocess_toolbox/interface.py index cddc5e3..b2552b1 100644 --- a/preprocess_toolbox/interface.py +++ b/preprocess_toolbox/interface.py @@ -34,7 +34,6 @@ def get_processor_implementation(config: os.PathLike) -> object: create_kwargs = dict(**remaining) logging.info("Attempting to instantiate {} with loaded configuration".format(implementation)) - logging.debug("Converted kwargs from the retrieved configuration: {}".format(create_kwargs)) return implementation(**create_kwargs) @@ -57,7 +56,6 @@ def get_processor_from_source(identifier: str, source_cfg: dict) -> object: create_kwargs = {k: v for k, v in source_cfg.items() if k not in ["dataset_config", "implementation"]} logging.info("Attempting to instantiate {} with loaded configuration".format(source_cfg["implementation"])) - logging.debug("Converted kwargs from the retrieved configuration: {}".format(create_kwargs)) return get_implementation(source_cfg["implementation"])( get_dataset_config_implementation(source_cfg["dataset_config"]), From cec39cc8952128856131d30e54527505e84b1777 Mon Sep 17 00:00:00 2001 From: James Byrne Date: Thu, 29 May 2025 00:38:27 +0100 Subject: [PATCH 03/15] Updating to ensure dates are month ends when extending ranges --- preprocess_toolbox/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/preprocess_toolbox/utils.py b/preprocess_toolbox/utils.py index 5ecc4dc..5d06aca 100644 --- a/preprocess_toolbox/utils.py +++ b/preprocess_toolbox/utils.py @@ -6,8 +6,9 @@ from dateutil.relativedelta import relativedelta import orjson +import pandas as pd -from download_toolbox.interface import DatasetConfig +from download_toolbox.interface import DatasetConfig, Frequency def get_config(config_path: os.PathLike): @@ -44,6 +45,9 @@ def get_extension_dates(ds_config: DatasetConfig, op = operator.sub if reverse else operator.add extended_date = op(date, relativedelta(**attrs)) + if ds_config.frequency.attribute == Frequency.MONTH: + extended_date = extended_date + pd.offsets.MonthEnd(0) + if extended_date not in dates: if all([os.path.exists(ds_config.var_filepath(var_config, [extended_date])) for var_config in ds_config.variables]): From 143e8f8cd1c44a01568b481134afc7db6170159c Mon Sep 17 00:00:00 2001 From: James Byrne Date: Fri, 30 May 2025 10:22:10 +0100 Subject: [PATCH 04/15] Removing concatenation on time dimension as will produce duplicates and thus non-monotonic indexes in some situations --- preprocess_toolbox/processor.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/preprocess_toolbox/processor.py b/preprocess_toolbox/processor.py index 56a3e55..2a1d121 100644 --- a/preprocess_toolbox/processor.py +++ b/preprocess_toolbox/processor.py @@ -261,7 +261,6 @@ def _init_source_data(self, for var_name, var_files in self._source_files[split].items(): logging.info("Got {} files for {}:{}".format(len(var_files), split, var_name)) - logging.debug(pformat(self._source_files)) def _normalise_array_mean(self, var_name: str, da: object, denormalise: bool=False): """ @@ -341,7 +340,6 @@ def _normalise_array_scaling(self, var_name: str, da: object, denormalise: bool= elif self.norm_split_dates: logging.debug("Generating norm-scaling min-max from {} training " "dates".format(len(self.norm_split_dates))) - norm_samples = da.sel(time=self.norm_split_dates).data norm_samples = norm_samples.ravel() @@ -380,19 +378,15 @@ def _process_channel(self, if len(source_files) > 0: logging.info("Opening {} files for {}".format(len(source_files), var_name)) + logging.debug("Files to be opened:\n{}".format(pformat(source_files))) # In the old IceNet library there was dubiousness about the source of the # data so this was harder. Now we work with whatever we get from download-toolbox ds = xr.open_mfdataset( source_files, - # Solves issue with inheriting files without - # time dimension (only having coordinate) combine="nested", - concat_dim="time", coords="minimal", compat="override", - # TODO: review this, but if lat-lon is in the file, it's signalling bigger issues - # drop_variables=("lat", "lon"), parallel=self._parallel) da = getattr(ds, var_name) da = da.astype(self.dtype) @@ -400,7 +394,6 @@ def _process_channel(self, # FIXME: we should ideally store train dates against the # normalisation and climatology, to ensure recalculation on # reprocess. All this need be is in the path, to be honest - if var_suffix == "anom": if len(self._anom_clim_splits) < 1 and self._refdir is None: raise ProcessingError("You must provide a list of splits via " @@ -589,7 +582,6 @@ def lead_time(self) -> int: @property def norm_split_dates(self): - # TODO: functools.cached_property, though slightly odd behaviour re. write-ability return [date for clim_split in self._normalisation_splits for date in self._splits[clim_split]] From 3f1c25d6e6e8c798359e50b92cfb7693c2cd0787 Mon Sep 17 00:00:00 2001 From: James Byrne Date: Fri, 30 May 2025 11:54:14 +0100 Subject: [PATCH 05/15] Fixes #32: resolving the date checking in monthly handling --- preprocess_toolbox/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/preprocess_toolbox/utils.py b/preprocess_toolbox/utils.py index 5d06aca..fe59667 100644 --- a/preprocess_toolbox/utils.py +++ b/preprocess_toolbox/utils.py @@ -45,7 +45,7 @@ def get_extension_dates(ds_config: DatasetConfig, op = operator.sub if reverse else operator.add extended_date = op(date, relativedelta(**attrs)) - if ds_config.frequency.attribute == Frequency.MONTH: + if ds_config.frequency == Frequency.MONTH: extended_date = extended_date + pd.offsets.MonthEnd(0) if extended_date not in dates: @@ -58,6 +58,7 @@ def get_extension_dates(ds_config: DatasetConfig, logging.warning("{} will be dropped due to missing data {}". format(date, extended_date)) dropped_dates.append(date) + break return sorted(list(set(additional_dates))), sorted(list(set(dropped_dates))) From 6e9274c83ed7f42c6e45387cd180ba689b069205 Mon Sep 17 00:00:00 2001 From: James Byrne Date: Fri, 30 May 2025 12:08:16 +0100 Subject: [PATCH 06/15] Fixes #32: fixes resolutions monthly handling --- preprocess_toolbox/processor.py | 3 ++- preprocess_toolbox/utils.py | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/preprocess_toolbox/processor.py b/preprocess_toolbox/processor.py index 2a1d121..b95969d 100644 --- a/preprocess_toolbox/processor.py +++ b/preprocess_toolbox/processor.py @@ -240,7 +240,8 @@ def _init_source_data(self, # Calculating lead and lag dates that aren't already accounted for in splits if self._lag_time > 0: logging.info("Including lag of {} {}s".format(self._lag_time, ds_config.frequency.attribute)) - additional_lag_dates, dropped_lag_dates = get_extension_dates(ds_config, dates, self._lag_time, reverse=True) + additional_lag_dates, dropped_lag_dates = get_extension_dates(ds_config, dates, self._lag_time + 1, + start_step=1, reverse=True) dates += additional_lag_dates drop_dates[split] += dropped_lag_dates logging.info("Lag added {} dates for {} category: {} - {}". diff --git a/preprocess_toolbox/utils.py b/preprocess_toolbox/utils.py index fe59667..cadc610 100644 --- a/preprocess_toolbox/utils.py +++ b/preprocess_toolbox/utils.py @@ -36,17 +36,18 @@ def get_config_filename(args: argparse.Namespace, prefix: str = "loader"): def get_extension_dates(ds_config: DatasetConfig, dates: list, num_steps: int, - reverse=False): + start_step: int = 0, + reverse: bool = False): additional_dates, dropped_dates = [], [] for date in dates: - for time in range(num_steps): - attrs = {"{}s".format(ds_config.frequency.attribute): time + 1} + for time in range(start_step, num_steps): + attrs = {"{}s".format(ds_config.frequency.attribute): time} op = operator.sub if reverse else operator.add extended_date = op(date, relativedelta(**attrs)) if ds_config.frequency == Frequency.MONTH: - extended_date = extended_date + pd.offsets.MonthEnd(0) + extended_date = pd.to_datetime(extended_date + pd.offsets.MonthEnd(0)).date() if extended_date not in dates: if all([os.path.exists(ds_config.var_filepath(var_config, [extended_date])) From bcd70ff5daa4e74be287634d3ebd3fdca703b289 Mon Sep 17 00:00:00 2001 From: James Byrne Date: Fri, 30 May 2025 12:43:46 +0100 Subject: [PATCH 07/15] Adding lock=False to open_mfdataset as per xarray#3961 - we do not specify engine --- preprocess_toolbox/processor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/preprocess_toolbox/processor.py b/preprocess_toolbox/processor.py index b95969d..598dde9 100644 --- a/preprocess_toolbox/processor.py +++ b/preprocess_toolbox/processor.py @@ -388,7 +388,8 @@ def _process_channel(self, combine="nested", coords="minimal", compat="override", - parallel=self._parallel) + parallel=self._parallel, + lock=False) da = getattr(ds, var_name) da = da.astype(self.dtype) From 84732a06bf74524b9ee61dfde602298c99761ed1 Mon Sep 17 00:00:00 2001 From: James Byrne Date: Wed, 4 Jun 2025 13:17:36 +0100 Subject: [PATCH 08/15] Updating open and split analysis to better handle lag/lead --- preprocess_toolbox/processor.py | 38 +++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/preprocess_toolbox/processor.py b/preprocess_toolbox/processor.py index 598dde9..f97c051 100644 --- a/preprocess_toolbox/processor.py +++ b/preprocess_toolbox/processor.py @@ -95,8 +95,16 @@ def __init__(self, self._normalisation_splits = [] if normalisation_splits is None else normalisation_splits self._parallel = parallel_opens self._refdir = ref_procdir + + ## + # Split dates - + # + # TODO: splits -> { dates, sources }, but currently sources are separate... self._splits = splits + self._valid_split_dates = splits + # TODO: add self._dropped_dates based on DATA + self._source_files = dict() if init_source: @@ -223,16 +231,17 @@ def _init_source_data(self, :return: """ - split_dates_required = dict() + # TODO: distracted, but this needs to be better written drop_dates = dict() + all_dates = dict() for split in self._splits.keys(): - dates = sorted(self._splits[split]) + all_dates[split] = sorted(self._splits[split]) drop_dates[split] = list() - if dates: + if all_dates[split]: logging.info("Processing {} dates for {} category: {} - {}". - format(len(dates), split, min(dates), max(dates))) + format(len(all_dates[split]), split, min(all_dates[split]), max(all_dates[split]))) else: logging.info("No {} dates for this processor".format(split)) continue @@ -240,24 +249,24 @@ def _init_source_data(self, # Calculating lead and lag dates that aren't already accounted for in splits if self._lag_time > 0: logging.info("Including lag of {} {}s".format(self._lag_time, ds_config.frequency.attribute)) - additional_lag_dates, dropped_lag_dates = get_extension_dates(ds_config, dates, self._lag_time + 1, + additional_lag_dates, dropped_lag_dates = get_extension_dates(ds_config, all_dates[split], self._lag_time + 2, start_step=1, reverse=True) - dates += additional_lag_dates + all_dates[split] += additional_lag_dates drop_dates[split] += dropped_lag_dates logging.info("Lag added {} dates for {} category: {} - {}". - format(len(dates), split, min(dates), max(dates))) + format(len(all_dates[split]), split, min(all_dates[split]), max(all_dates[split]))) if self._lead_time > 0: logging.info("Including lead of {} {}s".format(self._lead_time, ds_config.frequency.attribute)) - additional_lead_dates, dropped_lead_dates = get_extension_dates(ds_config, dates, self._lead_time) - dates += additional_lead_dates + additional_lead_dates, dropped_lead_dates = get_extension_dates(ds_config, all_dates[split], self._lead_time) + all_dates[split] += additional_lead_dates drop_dates[split] += dropped_lead_dates logging.info("Lead added {} dates for {} category: {} - {}". - format(len(dates), split, min(dates), max(dates))) + format(len(all_dates[split]), split, min(all_dates[split]), max(all_dates[split]))) - split_dates_required[split] = sorted([_ for _ in dates if _ not in drop_dates[split]]) + self._valid_split_dates[split] = sorted([_ for _ in all_dates[split] if _ not in drop_dates[split]]) for split in self._splits.keys(): - self._source_files[split] = {var_config.name: ds_config.var_filepaths(var_config, split_dates_required[split]) + self._source_files[split] = {var_config.name: ds_config.var_filepaths(var_config, all_dates[split]) for var_config in ds_config.variables} for var_name, var_files in self._source_files[split].items(): @@ -385,9 +394,6 @@ def _process_channel(self, # data so this was harder. Now we work with whatever we get from download-toolbox ds = xr.open_mfdataset( source_files, - combine="nested", - coords="minimal", - compat="override", parallel=self._parallel, lock=False) da = getattr(ds, var_name) @@ -503,7 +509,7 @@ def get_config(self, **kwargs): "path": self.path, "processed_files": self._processed_files, "source_files": self._source_files, - "splits": self.splits, + "splits": self._valid_split_dates, } @staticmethod From 728bdd85c68c652676e722b11de0b24ced8b68bb Mon Sep 17 00:00:00 2001 From: James Byrne Date: Tue, 10 Jun 2025 00:16:25 +0100 Subject: [PATCH 09/15] Fixing single time entry for rotation --- preprocess_toolbox/dataset/process.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/preprocess_toolbox/dataset/process.py b/preprocess_toolbox/dataset/process.py index 2cda6ad..9f52582 100644 --- a/preprocess_toolbox/dataset/process.py +++ b/preprocess_toolbox/dataset/process.py @@ -150,6 +150,9 @@ def rotate_dataset(ref_file: os.PathLike, wind_cubes[vars_to_rotate[1]], angles, ) + if len(wind_cubes_r[vars_to_rotate[0]].shape) == 2 and len(wind_cubes_r[vars_to_rotate[1]].shape) == 2: + wind_cubes_r[vars_to_rotate[0]] = iris.util.new_axis(wind_cubes_r[vars_to_rotate[0]], "time") + wind_cubes_r[vars_to_rotate[1]] = iris.util.new_axis(wind_cubes_r[vars_to_rotate[1]], "time") except iris.exceptions.CoordinateNotFoundError: logging.exception("Failure to rotate due to coordinate issues. " "moving onto next file") From 7937d9819096d08da606dc7309c394743adc3595 Mon Sep 17 00:00:00 2001 From: James Byrne Date: Tue, 10 Jun 2025 00:16:54 +0100 Subject: [PATCH 10/15] Resolving some of the intricacies of lag dates with predictions --- preprocess_toolbox/processor.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/preprocess_toolbox/processor.py b/preprocess_toolbox/processor.py index f97c051..f420bf2 100644 --- a/preprocess_toolbox/processor.py +++ b/preprocess_toolbox/processor.py @@ -102,7 +102,7 @@ def __init__(self, # TODO: splits -> { dates, sources }, but currently sources are separate... self._splits = splits - self._valid_split_dates = splits + self._dropped_split_dates = {} # TODO: add self._dropped_dates based on DATA self._source_files = dict() @@ -231,7 +231,6 @@ def _init_source_data(self, :return: """ - # TODO: distracted, but this needs to be better written drop_dates = dict() all_dates = dict() @@ -247,9 +246,9 @@ def _init_source_data(self, continue # Calculating lead and lag dates that aren't already accounted for in splits - if self._lag_time > 0: + if self._lag_time >= 0: logging.info("Including lag of {} {}s".format(self._lag_time, ds_config.frequency.attribute)) - additional_lag_dates, dropped_lag_dates = get_extension_dates(ds_config, all_dates[split], self._lag_time + 2, + additional_lag_dates, dropped_lag_dates = get_extension_dates(ds_config, all_dates[split], self._lag_time + 1, start_step=1, reverse=True) all_dates[split] += additional_lag_dates drop_dates[split] += dropped_lag_dates @@ -263,7 +262,8 @@ def _init_source_data(self, logging.info("Lead added {} dates for {} category: {} - {}". format(len(all_dates[split]), split, min(all_dates[split]), max(all_dates[split]))) - self._valid_split_dates[split] = sorted([_ for _ in all_dates[split] if _ not in drop_dates[split]]) + self._dropped_split_dates[split] = sorted(drop_dates[split]) + all_dates[split] = sorted([_ for _ in all_dates[split] if _ not in drop_dates[split]]) for split in self._splits.keys(): self._source_files[split] = {var_config.name: ds_config.var_filepaths(var_config, all_dates[split]) @@ -388,7 +388,6 @@ def _process_channel(self, if len(source_files) > 0: logging.info("Opening {} files for {}".format(len(source_files), var_name)) - logging.debug("Files to be opened:\n{}".format(pformat(source_files))) # In the old IceNet library there was dubiousness about the source of the # data so this was harder. Now we work with whatever we get from download-toolbox @@ -398,6 +397,7 @@ def _process_channel(self, lock=False) da = getattr(ds, var_name) da = da.astype(self.dtype) + logging.debug("Files to be opened: {}".format(da.dims)) # FIXME: we should ideally store train dates against the # normalisation and climatology, to ensure recalculation on @@ -509,7 +509,9 @@ def get_config(self, **kwargs): "path": self.path, "processed_files": self._processed_files, "source_files": self._source_files, - "splits": self._valid_split_dates, + "splits": {split: [ + date for date in dates if date not in self._dropped_split_dates[split] + ] for split, dates in self._splits.items()}, } @staticmethod From 681e7aaf1ded2dc978ad70c1e6553cacb8b919bd Mon Sep 17 00:00:00 2001 From: James Byrne Date: Thu, 12 Jun 2025 11:19:14 +0100 Subject: [PATCH 11/15] Improving date awareness when processing datasets --- preprocess_toolbox/utils.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/preprocess_toolbox/utils.py b/preprocess_toolbox/utils.py index cadc610..7862dfb 100644 --- a/preprocess_toolbox/utils.py +++ b/preprocess_toolbox/utils.py @@ -7,6 +7,7 @@ import orjson import pandas as pd +import xarray as xr from download_toolbox.interface import DatasetConfig, Frequency @@ -49,11 +50,24 @@ def get_extension_dates(ds_config: DatasetConfig, if ds_config.frequency == Frequency.MONTH: extended_date = pd.to_datetime(extended_date + pd.offsets.MonthEnd(0)).date() - if extended_date not in dates: - if all([os.path.exists(ds_config.var_filepath(var_config, [extended_date])) - for var_config in ds_config.variables]): - # We only add these dates into the mix if all necessary files exist - additional_dates.append(extended_date) + # Check we don't know we have data, and also ignore previous occurrences + if extended_date not in dates and extended_date not in additional_dates: + extended_date_var_files = [ds_config.var_filepath(var_config, [extended_date]) + for var_config in ds_config.variables] + if all([os.path.exists(df) for df in extended_date_var_files]): + # The above will catch those items that fall outside the file output boundary, but not missing + # dates within ALL files. This next clause is more expensive, but necessary to catch everything! + logging.debug("Files exist, double checking whether {} appears in data itself across {} files". + format(extended_date, len(extended_date_var_files))) + + # TODO: this won't catch partially available dates where not all files have the date, but some do + if pd.Timestamp(extended_date) in xr.open_mfdataset(extended_date_var_files).time.values: + # We only add these dates into the mix if all necessary files exist + additional_dates.append(extended_date) + else: + logging.warning("Nope, {} not in data itself so dropping {}".format(extended_date, date)) + dropped_dates.append(date) + break else: # Otherwise, warn that the lag data means this is being dropped logging.warning("{} will be dropped due to missing data {}". From 35f57eecd8017fae0bf1c7a4a4ca80ca77e905d4 Mon Sep 17 00:00:00 2001 From: James Byrne Date: Thu, 12 Jun 2025 11:27:06 +0100 Subject: [PATCH 12/15] Correcting lag indexing --- preprocess_toolbox/processor.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/preprocess_toolbox/processor.py b/preprocess_toolbox/processor.py index f420bf2..8a483b9 100644 --- a/preprocess_toolbox/processor.py +++ b/preprocess_toolbox/processor.py @@ -248,8 +248,11 @@ def _init_source_data(self, # Calculating lead and lag dates that aren't already accounted for in splits if self._lag_time >= 0: logging.info("Including lag of {} {}s".format(self._lag_time, ds_config.frequency.attribute)) - additional_lag_dates, dropped_lag_dates = get_extension_dates(ds_config, all_dates[split], self._lag_time + 1, - start_step=1, reverse=True) + additional_lag_dates, dropped_lag_dates = get_extension_dates( + ds_config, all_dates[split], + # We offset by two, because -1 is channel one, so we need to account for lag == 1 being -2 + self._lag_time + 2, + start_step=1, reverse=True) all_dates[split] += additional_lag_dates drop_dates[split] += dropped_lag_dates logging.info("Lag added {} dates for {} category: {} - {}". From e72508f843e44d5bed03e69338644ca77be2cf22 Mon Sep 17 00:00:00 2001 From: James Byrne Date: Fri, 30 May 2025 23:32:16 +0100 Subject: [PATCH 13/15] Documentation stubs drafted --- README.md | 15 +-------------- docs/commands.md | 19 +++++++++++++++++++ docs/index.md | 4 ---- docs/preprocess_toolbox/base.md | 4 ---- docs/preprocess_toolbox/cli.md | 4 ---- docs/preprocess_toolbox/dataset/cli.md | 4 ---- docs/preprocess_toolbox/dataset/process.md | 4 ---- docs/preprocess_toolbox/dataset/spatial.md | 4 ---- docs/preprocess_toolbox/dataset/time.md | 4 ---- docs/preprocess_toolbox/interface.md | 4 ---- docs/preprocess_toolbox/loader/cli.md | 4 ---- docs/preprocess_toolbox/loader/utils.md | 4 ---- docs/preprocess_toolbox/models.md | 4 ---- docs/preprocess_toolbox/processor.md | 4 ---- docs/preprocess_toolbox/utils.md | 4 ---- 15 files changed, 20 insertions(+), 66 deletions(-) create mode 100644 docs/commands.md delete mode 100644 docs/preprocess_toolbox/base.md delete mode 100644 docs/preprocess_toolbox/cli.md delete mode 100644 docs/preprocess_toolbox/dataset/cli.md delete mode 100644 docs/preprocess_toolbox/dataset/process.md delete mode 100644 docs/preprocess_toolbox/dataset/spatial.md delete mode 100644 docs/preprocess_toolbox/dataset/time.md delete mode 100644 docs/preprocess_toolbox/interface.md delete mode 100644 docs/preprocess_toolbox/loader/cli.md delete mode 100644 docs/preprocess_toolbox/loader/utils.md delete mode 100644 docs/preprocess_toolbox/models.md delete mode 100644 docs/preprocess_toolbox/processor.md delete mode 100644 docs/preprocess_toolbox/utils.md diff --git a/README.md b/README.md index 706d880..6169f04 100644 --- a/README.md +++ b/README.md @@ -47,20 +47,7 @@ This library doesn't have knowledge of those datasets, it forms the basis for pr ## Limitations -There are some major limitations to this as a general purpose tool, these will hopefully be dealt with in time! I'm raising issues as I go - -**This is currently very heavy development functionality, but the following commands already work**: - -* preprocess_missing_spatial - poorly at present due to missing mask backref implementation -* preprocess_missing_time -* preprocess_regrid -* preprocess_rotate -* preprocess_dataset -* preprocess_loader_init -* preprocess_add_mask -* preprocess_add_channel - -Other stubs probably don't work, unless I forgot to update these docs! +**This is currently active development, but the commands work to most degrees.** ## Contributing diff --git a/docs/commands.md b/docs/commands.md new file mode 100644 index 0000000..d0580f5 --- /dev/null +++ b/docs/commands.md @@ -0,0 +1,19 @@ +# Commands + +## Datasets +### preprocess_missing_spatial +### preprocess_missing_time +### preprocess_regrid +### preprocess_reproject +### preprocess_rotate + +## Loaders +### preprocess_loader_init +### preprocess_loader_copy + +### preprocess_add_mask +### preprocess_add_channel + +## AI-ready datasets +### preprocess_dataset +### preprocess_add_processed diff --git a/docs/index.md b/docs/index.md index fc9d69a..80084f0 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,8 +4,4 @@ Check out the [usage](usage.md) section for further information, including how t Check out the [API](preprocess_toolbox/base.md) if you're keen to go deep. -!!! note - - This project is under active development. - {!README.md!} diff --git a/docs/preprocess_toolbox/base.md b/docs/preprocess_toolbox/base.md deleted file mode 100644 index acacc64..0000000 --- a/docs/preprocess_toolbox/base.md +++ /dev/null @@ -1,4 +0,0 @@ -title: base ---- -::: preprocess_toolbox.base - diff --git a/docs/preprocess_toolbox/cli.md b/docs/preprocess_toolbox/cli.md deleted file mode 100644 index ca16186..0000000 --- a/docs/preprocess_toolbox/cli.md +++ /dev/null @@ -1,4 +0,0 @@ -title: cli ---- -::: preprocess_toolbox.cli - diff --git a/docs/preprocess_toolbox/dataset/cli.md b/docs/preprocess_toolbox/dataset/cli.md deleted file mode 100644 index 6c99446..0000000 --- a/docs/preprocess_toolbox/dataset/cli.md +++ /dev/null @@ -1,4 +0,0 @@ -title: preprocess_toolbox.dataset.cli ---- -::: preprocess_toolbox.dataset.cli - diff --git a/docs/preprocess_toolbox/dataset/process.md b/docs/preprocess_toolbox/dataset/process.md deleted file mode 100644 index bb2799a..0000000 --- a/docs/preprocess_toolbox/dataset/process.md +++ /dev/null @@ -1,4 +0,0 @@ -title: preprocess_toolbox.dataset.process ---- -::: preprocess_toolbox.dataset.process - diff --git a/docs/preprocess_toolbox/dataset/spatial.md b/docs/preprocess_toolbox/dataset/spatial.md deleted file mode 100644 index 0fe7871..0000000 --- a/docs/preprocess_toolbox/dataset/spatial.md +++ /dev/null @@ -1,4 +0,0 @@ -title: preprocess_toolbox.dataset.spatial ---- -::: preprocess_toolbox.dataset.spatial - diff --git a/docs/preprocess_toolbox/dataset/time.md b/docs/preprocess_toolbox/dataset/time.md deleted file mode 100644 index 93e072e..0000000 --- a/docs/preprocess_toolbox/dataset/time.md +++ /dev/null @@ -1,4 +0,0 @@ -title: preprocess_toolbox.dataset.time ---- -::: preprocess_toolbox.dataset.time - diff --git a/docs/preprocess_toolbox/interface.md b/docs/preprocess_toolbox/interface.md deleted file mode 100644 index 87f8dbc..0000000 --- a/docs/preprocess_toolbox/interface.md +++ /dev/null @@ -1,4 +0,0 @@ -title: interface ---- -::: preprocess_toolbox.interface - diff --git a/docs/preprocess_toolbox/loader/cli.md b/docs/preprocess_toolbox/loader/cli.md deleted file mode 100644 index a3ca342..0000000 --- a/docs/preprocess_toolbox/loader/cli.md +++ /dev/null @@ -1,4 +0,0 @@ -title: preprocess_toolbox.loader.cli ---- -::: preprocess_toolbox.loader.cli - diff --git a/docs/preprocess_toolbox/loader/utils.md b/docs/preprocess_toolbox/loader/utils.md deleted file mode 100644 index 897294b..0000000 --- a/docs/preprocess_toolbox/loader/utils.md +++ /dev/null @@ -1,4 +0,0 @@ -title: preprocess_toolbox.loader.utils ---- -::: preprocess_toolbox.loader.utils - diff --git a/docs/preprocess_toolbox/models.md b/docs/preprocess_toolbox/models.md deleted file mode 100644 index 610069a..0000000 --- a/docs/preprocess_toolbox/models.md +++ /dev/null @@ -1,4 +0,0 @@ -title: models ---- -::: preprocess_toolbox.models - diff --git a/docs/preprocess_toolbox/processor.md b/docs/preprocess_toolbox/processor.md deleted file mode 100644 index 1f5f9c6..0000000 --- a/docs/preprocess_toolbox/processor.md +++ /dev/null @@ -1,4 +0,0 @@ -title: processor ---- -::: preprocess_toolbox.processor - diff --git a/docs/preprocess_toolbox/utils.md b/docs/preprocess_toolbox/utils.md deleted file mode 100644 index 51e574c..0000000 --- a/docs/preprocess_toolbox/utils.md +++ /dev/null @@ -1,4 +0,0 @@ -title: utils ---- -::: preprocess_toolbox.utils - From 97aad7dbea5a585ea005d73efbc79a221696ab25 Mon Sep 17 00:00:00 2001 From: James Byrne Date: Fri, 30 May 2025 23:33:46 +0100 Subject: [PATCH 14/15] updating menus --- mkdocs.yml | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index fb7e6f8..b8389a1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -4,22 +4,8 @@ theme: highlightjs: true nav: - Home: 'index.md' + - 'commands.md' - 'usage.md' - - API: - - preprocess.dataset: - - "preprocess_toolbox/dataset/cli.md" - - "preprocess_toolbox/dataset/process.md" - - "preprocess_toolbox/dataset/spatial.md" - - "preprocess_toolbox/dataset/time.md" - - preprocess.loader: - - "preprocess_toolbox/loader/cli.md" - - "preprocess_toolbox/loader/utils.md" - - "preprocess_toolbox/base.md" - - "preprocess_toolbox/interface.md" - - "preprocess_toolbox/models.md" - - "preprocess_toolbox/processor.md" - - "preprocess_toolbox/utils.md" - - "preprocess_toolbox/cli.md" - GitHub: 'https://github.com/environmental-forecasting/preprocess-toolbox' plugins: - search From 835e07204b84b8b16239bf5ec9f6504fabf408cd Mon Sep 17 00:00:00 2001 From: James Byrne Date: Thu, 29 May 2025 22:54:18 +0100 Subject: [PATCH 15/15] Documentation setup --- docs/acknowledgements.md | 4 ++++ docs/change_log.md | 1 + docs/license.md | 1 + mkdocs.yml | 26 +++++++++++++++++++++++++- pyproject.toml | 11 ++++++++++- 5 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 docs/acknowledgements.md create mode 100644 docs/change_log.md create mode 100644 docs/license.md diff --git a/docs/acknowledgements.md b/docs/acknowledgements.md new file mode 100644 index 0000000..68f2734 --- /dev/null +++ b/docs/acknowledgements.md @@ -0,0 +1,4 @@ +# Acknowledgements +These are automatically generated from the git repository. + +{{ git_site_authors }} \ No newline at end of file diff --git a/docs/change_log.md b/docs/change_log.md new file mode 100644 index 0000000..90cb31c --- /dev/null +++ b/docs/change_log.md @@ -0,0 +1 @@ +--8<-- "CHANGELOG.md" \ No newline at end of file diff --git a/docs/license.md b/docs/license.md new file mode 100644 index 0000000..64ad9b4 --- /dev/null +++ b/docs/license.md @@ -0,0 +1 @@ +--8<-- "LICENSE" \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index b8389a1..65c27ca 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -9,16 +9,40 @@ nav: - GitHub: 'https://github.com/environmental-forecasting/preprocess-toolbox' plugins: - search + # Enables automatically generating authors from git repo, see acknowledgements.md + - git-authors + # Automatically generates API section from docstrings + - mkdocs-autoapi: + autoapi_add_nav_entry: API + autoapi_ignore: + - setup.py + - autorefs: + resolve_closest: true - mkdocstrings: handlers: python: + paths: + - . options: - docstring_style: sphinx + show_submodules: true + docstring_style: google heading_level: 3 markdown_extensions: - markdown_include.include: base_path: . - admonition + # Enables code block highlighting + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.superfences + - pymdownx.blocks.caption + - pymdownx.tabbed: + alternate_style: true +copyright: Copyright © 2025 British Antarctic Survey / Alan Turing Institute # Include extra CSS to make some style adjustments extra_css: diff --git a/pyproject.toml b/pyproject.toml index 68c0951..a3b3038 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,16 @@ docs = [ "mkdocs", "mkdocstrings[python]", "markdown-include", - "mkdocs-material" + "mkdocs-material", + "mkdocs-autoapi", + "mkdocs-git-authors-plugin", +] +lint = [ + "flake8" +] +tests = [ + "pytest", + "pytest-cov" ] [tool.setuptools.packages.find]