Toniof auxdata provision

multiply_prior_engine/soilmoisture_prior_creator.py

@@ -18,9 +18,11 @@
 import numpy as np
 import shapely
 import shapely.wkt
+from multiply_core.util import get_aux_data_provider
 from netCDF4 import Dataset
 from osgeo import gdal
 from scipy import spatial
+from typing import List, Optional
 
 from .prior_creator import PriorCreator
 
@@ -76,32 +78,28 @@ def compute_prior_file(self):
                  f"({prior_file_type}). Please use multiple ptypes (user1, "
                  f"user2, ..).")
             try:
-                data_dir = \
-                    self.config['Prior'][self.variable][self.ptype]['dir']
+                data_dir = self.config['Prior'][self.variable][self.ptype]['dir']
                 self.data_dir = data_dir
                 assert os.path.isdir(self.data_dir), \
-                    ('Directory does not exist or cannot be found: {}'
-                     .format(self.data_dir))
+                    ('Directory does not exist or cannot be found: {}'.format(self.data_dir))
             except KeyError as e:
                 assert self.data_dir is not None, \
-                  ('Cannot find directory information for '
-                   '"{}" prior in config file!'.format(self.ptype))
+                  ('Cannot find directory information for "{}" prior in config file!'.format(self.ptype))
             else:
                 return self._provide_prior_file()
         elif 'file' in prior_file_type:
             assert len(prior_file_type) == 1, \
-                (f"More than one user prior file type mentioned in config "
-                 f"({prior_file_type}). Please use multiple ptypes (user1, "
-                 f"user2, ..).")
+                (f"More than one user prior file type mentioned in config ({prior_file_type}). "
+                 f"Please use multiple ptypes (user1, user2, ..).")
             try:
-                data_file = \
-                    (self.config['Prior'][self.variable][self.ptype]['file'])
-                self.data_file = data_file
-            except KeyError as e:
+                aux_data_provider = get_aux_data_provider()
+                self.data_file = (self.config['Prior'][self.variable][self.ptype]['file'])
+                if not aux_data_provider.assure_element_provided(self.data_file):
+                    raise FileNotFoundError(f'Could not locate {self.data_file}.')
+            except KeyError:
                 assert self.data_file is not None, \
-                  ('Cannot find file name for '
-                   '"{ptype}" prior in config file (under'
-                   ' \'Prior/sm/{ptype}/file:\')!'.format(ptype=self.ptype))
+                  ('Cannot find file name for "{ptype}" prior in config file (under \'Prior/sm/{ptype}/file:\')!'.
+                   format(ptype=self.ptype))
             else:
                 return self._provide_prior_file()
         elif 'mean' and 'unc' in prior_file_type:
@@ -205,8 +203,7 @@ def _provide_prior_file(self):
 
         if self.data_dir is not None:
             self.data_file = self._get_prior_file_from_dir(self.data_dir)
-        else:
-            assert self.data_file is not None
+        assert self.data_file is not None
 
         ext = os.path.splitext(self.data_file)[-1].lower()
         if ext == 'vrt':
@@ -222,6 +219,10 @@ def _provide_prior_file(self):
                                               self.data_file))
         return self.data_file
 
+    def list_elements(self, base_folder: str, pattern: [Optional[str]] = '*') -> List[str]:
+        pattern_in_absolute_path = os.path.join(os.path.abspath(base_folder), pattern)
+        return glob.glob(f'{pattern_in_absolute_path}')
+
     def _get_prior_file_from_dir(self, directory, return_vrt=True):
         """Get filename(s) of prior file(s) from directory.
         If multiple files are found self._merge_multiple_prior_files is called.
@@ -256,9 +257,9 @@ def _get_prior_file_from_dir(self, directory, return_vrt=True):
                        .format(self.date8))
         else:
             pattern = (r"*")
-
-        srch = os.path.join(os.path.abspath(directory), pattern)
-        fn_list = sorted(glob.glob(f'{srch}', recursive=True))
+        aux_data_provider = get_aux_data_provider()
+        srch = aux_data_provider.list_elements(directory, pattern)
+        fn_list = sorted(srch)
         logging.info(f"Searching for files with expression: {srch}")
 
         # AssertionError is caught by the prior engine:
@@ -271,7 +272,8 @@ def _get_prior_file_from_dir(self, directory, return_vrt=True):
             fn = self._merge_multiple_prior_files(fn_list)
         else:
             fn = fn_list[0]
-
+        if not aux_data_provider.assure_element_provided(fn):
+            return
         self._check_gdal_compliance(fn)
         return '{}'.format(fn)
 

multiply_prior_engine/vegetation_prior_creator.py

@@ -4,16 +4,16 @@
 __email__ = "j.timmermans@cml.leidenuniv.nl"
 
 import glob
+import logging
 import os
-import subprocess
 import time
-import datetime
 
 import multiprocessing
 import numpy as np
 import yaml
 from dateutil.parser import parse
 from matplotlib import pyplot as plt
+from multiply_core.util import get_aux_data_provider
 from scipy import interpolate as RegularGridInterpolator
 from netCDF4 import Dataset
 from shapely.wkt import loads
@@ -199,12 +199,19 @@ def __init__(self, **kwargs):
             self.lat_study = [-90, 90]
 
         # 1.2 Define paths
+        aux_data_provider = get_aux_data_provider()
         self.directory_data = self.config['Prior']['General']['directory_data']
         self.path2LCC_file = (self.directory_data + 'LCC/' + 'ESACCI-LC-L4-LCCS-Map-300m-P1Y-2015-v2.0.7_updated.nc')
         self.path2Climate_file = (self.directory_data + 'Climate/' + 'sdat_10012_1_20171030_081458445.tif')
         self.path2Meteo_file = (self.directory_data + 'Meteorological/' + 'Meteo_.nc')
         self.path2Trait_file = (self.directory_data + 'Trait_Database/' + 'Traits.nc')
         self.path2Traitmap_file = self.directory_data + 'Priors/' + 'Priors.nc'
+        if not aux_data_provider.assure_element_provided(self.path2LCC_file):
+            logging.warning(f'{self.path2LCC_file} could not be found!')
+        if not aux_data_provider.assure_element_provided(self.path2Climate_file):
+            logging.warning(f'{self.path2Climate_file} could not be found!')
+        if not aux_data_provider.assure_element_provided(self.path2Trait_file):
+            logging.warning(f'{self.path2Trait_file} could not be found!')
 
         self.output_directory = self.config['Prior']['output_directory']
         if not os.path.exists(self.output_directory):
@@ -749,7 +756,11 @@ def ReadTryDatabase(self):
         import csv
 
         Rows = []
-        with open(self.directory_data + 'Try_Database/' + filename, 'r') as csvfile:
+        aux_data_provider = get_aux_data_provider()
+        path_to_file = os.path.abspath(os.path.join(self.directory_data, 'Try_Database', filename))
+        if not aux_data_provider.assure_element_provided(path_to_file):
+            logging.warning(f'Could not find {path_to_file}')
+        with open(path_to_file, 'r') as csvfile:
             spamreader = csv.reader(csvfile, delimiter=',')
             # spamreader = csv.reader(csvfile)
             # reader = csv.DictReader(csvfile)
@@ -847,7 +858,11 @@ def ReadTryFile(self):
         import csv
 
         Rows = []
-        with open(self.directory_data + 'Try_Database/PROSAILTraits.txt', 'r') as csvfile:
+        aux_data_provider = get_aux_data_provider()
+        path_to_file = os.path.abspath(os.path.join(self.directory_data, 'Try_Database', 'PROSAILTraits.txt'))
+        if not aux_data_provider.assure_element_provided(path_to_file):
+            logging.warning(f'Could not find {path_to_file}')
+        with open(path_to_file, 'r') as csvfile:
             spamreader = csv.reader(csvfile, delimiter=',')
             # spamreader = csv.reader(csvfile)
             # reader = csv.DictReader(csvfile)
@@ -1527,16 +1542,19 @@ def CombineTiles2Virtualfile(self, variable, doystr, directory_data):
         :rtype:
 
         """
+        aux_data_provider = get_aux_data_provider()
         dir = directory_data + 'Priors/'
         file_name = 'Priors_' + variable + '_' + doystr + '_global.vrt'
         # todo exchange 125 in upcoming versions with doy
-        list_of_files = glob.glob(dir + 'Priors*_' + variable + '_*125*.tiff')
+        # list_of_files = glob.glob(dir + 'Priors*_' + variable + '_*125*.tiff')
+        list_of_files = aux_data_provider.list_elements(dir, f'Priors*_{variable}_*125*.tiff')
         if len(list_of_files) == 0:
             raise UserWarning('No input files found for variable {}'.format(variable))
 
         list_of_files_as_strings = []
         for filename in list_of_files:
-            list_of_files_as_strings.append('"' + filename + '"')
+            if aux_data_provider.assure_element_provided(filename):
+                list_of_files_as_strings.append('"' + filename + '"')
 
         files = " ".join(list_of_files_as_strings)
         output_file_name = '{}{}'.format(self.output_directory, file_name)

requirements.txt

@@ -13,3 +13,4 @@ PyYAML==5.1
 typing
 python_dateutil
 recommonmark
+multiply_core.util

test/prior_engine/test_prior.py

@@ -133,8 +133,8 @@ def test_user_prior_initialization():
                                  datestr='2017-03-01',
                                  var="sm",
                                  ptype='user1')
-    with pytest.raises(AssertionError,
-                       # Expecting AssertionError
+    with pytest.raises(FileNotFoundError,
+                       # Expecting FileNotFoundError
                        # --> wrong path in config file.
                        match=r'.*path/to/file*'):
         S.compute_prior_file()