From 21740d742eb01289aa55f11ee328ef1c0c24023c Mon Sep 17 00:00:00 2001 From: Tonio Fincke Date: Thu, 14 Nov 2019 17:06:32 +0100 Subject: [PATCH 1/3] use aux data providers --- .../soilmoisture_prior_creator.py | 46 ++++++++++--------- .../vegetation_prior_creator.py | 30 +++++++++--- 2 files changed, 48 insertions(+), 28 deletions(-) diff --git a/multiply_prior_engine/soilmoisture_prior_creator.py b/multiply_prior_engine/soilmoisture_prior_creator.py index 4d93c31..687d069 100644 --- a/multiply_prior_engine/soilmoisture_prior_creator.py +++ b/multiply_prior_engine/soilmoisture_prior_creator.py @@ -18,9 +18,11 @@ import numpy as np import shapely import shapely.wkt +from multiply_core.util import get_aux_data_provider from netCDF4 import Dataset from osgeo import gdal from scipy import spatial +from typing import List, Optional from .prior_creator import PriorCreator @@ -76,32 +78,28 @@ def compute_prior_file(self): f"({prior_file_type}). Please use multiple ptypes (user1, " f"user2, ..).") try: - data_dir = \ - self.config['Prior'][self.variable][self.ptype]['dir'] + data_dir = self.config['Prior'][self.variable][self.ptype]['dir'] self.data_dir = data_dir assert os.path.isdir(self.data_dir), \ - ('Directory does not exist or cannot be found: {}' - .format(self.data_dir)) + ('Directory does not exist or cannot be found: {}'.format(self.data_dir)) except KeyError as e: assert self.data_dir is not None, \ - ('Cannot find directory information for ' - '"{}" prior in config file!'.format(self.ptype)) + ('Cannot find directory information for "{}" prior in config file!'.format(self.ptype)) else: return self._provide_prior_file() elif 'file' in prior_file_type: assert len(prior_file_type) == 1, \ - (f"More than one user prior file type mentioned in config " - f"({prior_file_type}). Please use multiple ptypes (user1, " - f"user2, ..).") + (f"More than one user prior file type mentioned in config ({prior_file_type}). " + f"Please use multiple ptypes (user1, user2, ..).") try: - data_file = \ - (self.config['Prior'][self.variable][self.ptype]['file']) - self.data_file = data_file - except KeyError as e: + aux_data_provider = get_aux_data_provider() + self.data_file = (self.config['Prior'][self.variable][self.ptype]['file']) + if not aux_data_provider.assure_element_provided(self.data_file): + raise ValueError(f'Could not locate {self.data_file}.') + except KeyError: assert self.data_file is not None, \ - ('Cannot find file name for ' - '"{ptype}" prior in config file (under' - ' \'Prior/sm/{ptype}/file:\')!'.format(ptype=self.ptype)) + ('Cannot find file name for "{ptype}" prior in config file (under \'Prior/sm/{ptype}/file:\')!'. + format(ptype=self.ptype)) else: return self._provide_prior_file() elif 'mean' and 'unc' in prior_file_type: @@ -205,8 +203,7 @@ def _provide_prior_file(self): if self.data_dir is not None: self.data_file = self._get_prior_file_from_dir(self.data_dir) - else: - assert self.data_file is not None + assert self.data_file is not None ext = os.path.splitext(self.data_file)[-1].lower() if ext == 'vrt': @@ -222,6 +219,10 @@ def _provide_prior_file(self): self.data_file)) return self.data_file + def list_elements(self, base_folder: str, pattern: [Optional[str]] = '*') -> List[str]: + pattern_in_absolute_path = os.path.join(os.path.abspath(base_folder), pattern) + return glob.glob(f'{pattern_in_absolute_path}') + def _get_prior_file_from_dir(self, directory, return_vrt=True): """Get filename(s) of prior file(s) from directory. If multiple files are found self._merge_multiple_prior_files is called. @@ -256,9 +257,9 @@ def _get_prior_file_from_dir(self, directory, return_vrt=True): .format(self.date8)) else: pattern = (r"*") - - srch = os.path.join(os.path.abspath(directory), pattern) - fn_list = sorted(glob.glob(f'{srch}', recursive=True)) + aux_data_provider = get_aux_data_provider() + srch = aux_data_provider.list_elements(directory, pattern) + fn_list = sorted(srch) logging.info(f"Searching for files with expression: {srch}") # AssertionError is caught by the prior engine: @@ -271,7 +272,8 @@ def _get_prior_file_from_dir(self, directory, return_vrt=True): fn = self._merge_multiple_prior_files(fn_list) else: fn = fn_list[0] - + if not aux_data_provider.assure_element_provided(fn): + return self._check_gdal_compliance(fn) return '{}'.format(fn) diff --git a/multiply_prior_engine/vegetation_prior_creator.py b/multiply_prior_engine/vegetation_prior_creator.py index 8ad3a33..e5c546d 100644 --- a/multiply_prior_engine/vegetation_prior_creator.py +++ b/multiply_prior_engine/vegetation_prior_creator.py @@ -4,16 +4,16 @@ __email__ = "j.timmermans@cml.leidenuniv.nl" import glob +import logging import os -import subprocess import time -import datetime import multiprocessing import numpy as np import yaml from dateutil.parser import parse from matplotlib import pyplot as plt +from multiply_core.util import get_aux_data_provider from scipy import interpolate as RegularGridInterpolator from netCDF4 import Dataset from shapely.wkt import loads @@ -199,12 +199,19 @@ def __init__(self, **kwargs): self.lat_study = [-90, 90] # 1.2 Define paths + aux_data_provider = get_aux_data_provider() self.directory_data = self.config['Prior']['General']['directory_data'] self.path2LCC_file = (self.directory_data + 'LCC/' + 'ESACCI-LC-L4-LCCS-Map-300m-P1Y-2015-v2.0.7_updated.nc') self.path2Climate_file = (self.directory_data + 'Climate/' + 'sdat_10012_1_20171030_081458445.tif') self.path2Meteo_file = (self.directory_data + 'Meteorological/' + 'Meteo_.nc') self.path2Trait_file = (self.directory_data + 'Trait_Database/' + 'Traits.nc') self.path2Traitmap_file = self.directory_data + 'Priors/' + 'Priors.nc' + if not aux_data_provider.assure_element_provided(self.path2LCC_file): + logging.warning(f'{self.path2LCC_file} could not be found!') + if not aux_data_provider.assure_element_provided(self.path2Climate_file): + logging.warning(f'{self.path2Climate_file} could not be found!') + if not aux_data_provider.assure_element_provided(self.path2Trait_file): + logging.warning(f'{self.path2Trait_file} could not be found!') self.output_directory = self.config['Prior']['output_directory'] if not os.path.exists(self.output_directory): @@ -749,7 +756,11 @@ def ReadTryDatabase(self): import csv Rows = [] - with open(self.directory_data + 'Try_Database/' + filename, 'r') as csvfile: + aux_data_provider = get_aux_data_provider() + path_to_file = os.path.abspath(os.path.join(self.directory_data, 'Try_Database', filename)) + if not aux_data_provider.assure_element_provided(path_to_file): + logging.warning(f'Could not find {path_to_file}') + with open(path_to_file, 'r') as csvfile: spamreader = csv.reader(csvfile, delimiter=',') # spamreader = csv.reader(csvfile) # reader = csv.DictReader(csvfile) @@ -847,7 +858,11 @@ def ReadTryFile(self): import csv Rows = [] - with open(self.directory_data + 'Try_Database/PROSAILTraits.txt', 'r') as csvfile: + aux_data_provider = get_aux_data_provider() + path_to_file = os.path.abspath(os.path.join(self.directory_data, 'Try_Database', 'PROSAILTraits.txt')) + if not aux_data_provider.assure_element_provided(path_to_file): + logging.warning(f'Could not find {path_to_file}') + with open(path_to_file, 'r') as csvfile: spamreader = csv.reader(csvfile, delimiter=',') # spamreader = csv.reader(csvfile) # reader = csv.DictReader(csvfile) @@ -1527,16 +1542,19 @@ def CombineTiles2Virtualfile(self, variable, doystr, directory_data): :rtype: """ + aux_data_provider = get_aux_data_provider() dir = directory_data + 'Priors/' file_name = 'Priors_' + variable + '_' + doystr + '_global.vrt' # todo exchange 125 in upcoming versions with doy - list_of_files = glob.glob(dir + 'Priors*_' + variable + '_*125*.tiff') + # list_of_files = glob.glob(dir + 'Priors*_' + variable + '_*125*.tiff') + list_of_files = aux_data_provider.list_elements(dir, f'Priors*_{variable}_*125*.tiff') if len(list_of_files) == 0: raise UserWarning('No input files found for variable {}'.format(variable)) list_of_files_as_strings = [] for filename in list_of_files: - list_of_files_as_strings.append('"' + filename + '"') + if aux_data_provider.assure_element_provided(filename): + list_of_files_as_strings.append('"' + filename + '"') files = " ".join(list_of_files_as_strings) output_file_name = '{}{}'.format(self.output_directory, file_name) From 92c7c8bbd45ca25323e318bd14a3155e91159f9c Mon Sep 17 00:00:00 2001 From: Tonio Fincke Date: Thu, 14 Nov 2019 17:10:28 +0100 Subject: [PATCH 2/3] made test run --- multiply_prior_engine/soilmoisture_prior_creator.py | 2 +- test/prior_engine/test_prior.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/multiply_prior_engine/soilmoisture_prior_creator.py b/multiply_prior_engine/soilmoisture_prior_creator.py index 687d069..13efc94 100644 --- a/multiply_prior_engine/soilmoisture_prior_creator.py +++ b/multiply_prior_engine/soilmoisture_prior_creator.py @@ -95,7 +95,7 @@ def compute_prior_file(self): aux_data_provider = get_aux_data_provider() self.data_file = (self.config['Prior'][self.variable][self.ptype]['file']) if not aux_data_provider.assure_element_provided(self.data_file): - raise ValueError(f'Could not locate {self.data_file}.') + raise FileNotFoundError(f'Could not locate {self.data_file}.') except KeyError: assert self.data_file is not None, \ ('Cannot find file name for "{ptype}" prior in config file (under \'Prior/sm/{ptype}/file:\')!'. diff --git a/test/prior_engine/test_prior.py b/test/prior_engine/test_prior.py index 470bf59..24feb11 100644 --- a/test/prior_engine/test_prior.py +++ b/test/prior_engine/test_prior.py @@ -133,8 +133,8 @@ def test_user_prior_initialization(): datestr='2017-03-01', var="sm", ptype='user1') - with pytest.raises(AssertionError, - # Expecting AssertionError + with pytest.raises(FileNotFoundError, + # Expecting FileNotFoundError # --> wrong path in config file. match=r'.*path/to/file*'): S.compute_prior_file() From 798f1eb1bbaef5f5c14f62a38432682b5b7bc193 Mon Sep 17 00:00:00 2001 From: Tonio Fincke Date: Thu, 14 Nov 2019 17:11:14 +0100 Subject: [PATCH 3/3] added requirement --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index e87ea42..0f07692 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,4 @@ PyYAML==5.1 typing python_dateutil recommonmark +multiply_core.util