Added handling for stub JSON files

oceannavigator/dataset_config.py

@@ -32,12 +32,25 @@ def get_datasets() -> List[str]:
     @lru_cache()
     def _get_dataset_config() -> dict:
         settings = get_settings()
+
         with open(settings.dataset_config_file, "r") as f:
             config = json.load(f)
             log().debug(
                 f"Loaded dataset config file from: {settings.dataset_config_file}"
             )
-            return config
+
+        for name in config: 
+            stub_file = f"{settings.dataset_config_stub_path}{name}.json"
+
+            with open(stub_file, "r") as f: 
+                config[name] = json.load(f)[name]
+                log().debug(
+                    f"Loaded stub dataset config file from: {stub_file}"
+                )
+
+        return config
+
+
 
     def _get_attribute(self, key: str) -> Union[str, dict]:
         return self._config.get(key) if not None else ""
@@ -111,21 +124,6 @@ def bathymetry_file_url(self) -> str:
         """
         return self._get_attribute("bathymetry_file_url")
 
-    @property
-    def vector_arrow_stride(self) -> int:
-        """
-        Returns the stride used to slice the dataset for generating geojson
-        for the vector arrows (i.e. every n-th value).
-
-        Defaults to 4.
-        """
-
-        stride = self._get_attribute("vector_arrow_stride")
-        if stride:
-            return stride
-
-        return 4
-
     @property
     def model_class(self) -> str:
         return self._get_attribute("model_class")
@@ -178,17 +176,6 @@ def attribution(self) -> str:
         except KeyError:
             return ""
 
-    @property
-    def cache(self) -> int:
-        """
-        Returns the cache value for a dataset as defined in dataset config file
-        """
-        cache = self._get_attribute("cache")
-        if cache is not None and isinstance(cache, str):
-            cache = int(cache)
-
-        return cache
-
     @property
     def default_location(self) -> list:
         """

oceannavigator/settings.py

@@ -34,6 +34,7 @@ class Settings(BaseSettings):
     dask_scheduler: str = ""
     dataset_shape_file_dir: str = ""
     dataset_config_file: str = ""
+    dataset_config_stub_path: str = ""
     debug: bool = False
     drifter_agg_url: str = ""
     drifter_catalog_url: str = ""

tests/test_dataset_config.py

@@ -28,7 +28,6 @@ def test_datasetconfig_object(self, m):
                 "model_class": "my_model_class",
                 "time_dim_units": "my_time_units",
                 "attribution": "my_<b>attribution</b>",
-                "cache": "123",
                 "lat_var_key": "my_lat",
                 "lon_var_key": "my_lon",
                 "variables": {
@@ -58,7 +57,6 @@ def test_datasetconfig_object(self, m):
         assert result.lat_var_key == "my_lat"
         assert result.lon_var_key == "my_lon"
         assert result.attribution == "my_attribution"
-        assert result.cache == 123
 
         assert not result.variable[Mock(key="var")].is_hidden
 
@@ -135,17 +133,14 @@ def test_get_dataset_misc(self, m):
                 "url": "the_url",
                 "attribution": "My attribution <b>bold</b>",
                 "climatology": "climatology_url",
-                "cache": 5,
             }
         }
 
         assert DatasetConfig("dataset").url == "the_url"
         assert DatasetConfig("dataset").climatology == "climatology_url"
         assert DatasetConfig("dataset").attribution == "My attribution bold"
-        assert DatasetConfig("dataset").cache == 5
 
         m.return_value = {"dataset2": {}}
-        assert DatasetConfig("dataset2").cache is None
 
     @patch.object(DatasetConfig, "_get_dataset_config")
     def test_get_variables(self, m):
@@ -181,7 +176,6 @@ def test_get_variable_misc(self, m):
                 "url": "the_url",
                 "attribution": "My attribution <b>bold</b>",
                 "climatology": "climatology_url",
-                "cache": 5,
                 "variables": {
                     "var": {
                         "name": "the_name",

tests/test_netcdf_data.py

@@ -20,8 +20,15 @@
 
 
 class TestNetCDFData(unittest.TestCase):
-    with open("tests/testdata/datasetconfigpatch.json") as dataPatch:
-        patch_dataset_config_ret_val = json.load(dataPatch)
+
+    with open("tests/testdata/datasetconfigpatch.json", "r") as f:
+        patch_dataset_config_ret_val = json.load(f)
+
+    for name in patch_dataset_config_ret_val: 
+        stub_file = f"tests/testdata/datasetconfigpatch-stubs/{name}.json"
+
+        with open(stub_file, "r") as f: 
+            patch_dataset_config_ret_val[name] = json.load(f)[name]
 
     @pytest.fixture(scope="class", autouse=True)
     def setUp_teardown(self):

tests/testdata/datasetconfigpatch-stubs/giops.json

@@ -0,0 +1,17 @@
+{
+    "giops": {
+        "enabled": 1,
+        "url": "tests/testdata/nemo_test.nc",
+        "time_dim_units": "seconds since 1950-01-01 00:00:00",
+        "model_class": "Mercator",
+        "quantum": "day",
+        "name": "GIOPS",
+        "help": "help",
+        "attribution": "attrib",
+        "lat_var_key": "nav_lat",
+        "lon_var_key": "nav_lon",
+        "variables": {
+            "votemper": { "name": "Temperature", "units": "Celsius", "scale": [-5, 30] }
+        }
+    }
+}

tests/testdata/datasetconfigpatch-stubs/giops_icechunk.json

@@ -0,0 +1,20 @@
+{
+    "giops_icechunk": {
+        "group": "GIOPS Forecast",    
+        "url": "icechunk",
+        "name": "GIOPS 10 Day Daily Mean Forecast 3D - LatLon",
+        "quantum": "hour",
+        "type": "forecast",
+        "time_dim_units": "seconds since 1950-01-01 00:00:00",
+        "model_class": "Mercator",
+        "enabled": true,
+        "attribution": "The Canadian Centre for Meteorological and Environmental Prediction",
+        "lat_var_key": "latitude",
+        "lon_var_key": "longitude",
+        "help": "./data-help/giops-10day.html",
+        "vector_variables": ["magwatervel"],
+        "variables": {
+            "votemper": { "name": "Potential Temperature", "units": "Celsius", "scale": [-5, 30], "equation": "votemper - 273.15", "dims": ["time", "depth", "latitude", "longitude"], "valid_min": -100, "valid_max": 100 }
+        }
+    }
+}

tests/testdata/datasetconfigpatch-stubs/giops_no_url.json

@@ -0,0 +1,14 @@
+{
+    "giops_no_url": {
+        "enabled": 1,
+        "time_dim_units": "seconds since 1950-01-01 00:00:00",
+        "model_class": "Mercator",
+        "quantum": "day",
+        "name": "GIOPS",
+        "help": "help",
+        "attribution": "attrib",
+        "variables": {
+            "votemper": { "name": "Temperature", "units": "Celsius", "scale": [-5, 30] }
+        }
+    }
+}

tests/testdata/datasetconfigpatch-stubs/giops_real.json

@@ -0,0 +1,23 @@
+{
+    "giops_real": {
+        "enabled": 1,
+        "url": "tests/testdata/giops_test.nc",
+        "name": "GIOPS Forecast 3D - Polar Stereographic",
+        "model_class": "Mercator",
+        "quantum": "day",
+        "type": "forecast",
+        "lat_var_key": "latitude",
+        "lon_var_key": "longitude",
+        "time_dim_units": "seconds since 1950-01-01 00:00:00",
+        "attribution": "The Canadian Centre for Meteorological and Environmental Prediction",
+        "vector_variables": ["magwatervel"],
+        "variables": {
+            "votemper": { "name": "Temperature", "units": "Celsius", "scale": [-5, 30] },
+            "vosaline": { "name": "Salinity", "units": "PSU", "scale": [30, 40] },
+            "vozocrtx": { "name": "Water Y Velocity", "units": "m/s", "scale": [-3, 3], "zero_centered": "true" },
+            "vomecrty": { "name": "Water X Velocity", "units": "m/s", "scale": [-3, 3], "zero_centered": "true" },
+            "magwatervel": { "name": "Water Velocity", "units": "m/s", "scale": [0, 3], "equation": "magnitude(vozocrtx, vomecrty)",  "dims": ["time", "depth", "latitude", "longitude"], "east_vector_component": "vozocrtx", "north_vector_component": "vomecrty" },
+            "bearing": { "hide": "true", "name": "Water Velocity Bearing (deg clockwise +tive from N)", "units": "degrees", "scale": [0, 360], "equation": "bearing(vomecrty, vozocrtx)" ,"dims": ["time", "depth", "latitude", "longitude"] }
+        }
+    }
+}

tests/testdata/datasetconfigpatch-stubs/mercator_test.json

@@ -0,0 +1,16 @@
+{
+    "mercator_test": {
+        "enabled": 1,
+        "url": "tests/testdata/mercator_test.nc",
+        "name": "Mercator test",
+        "time_dim_units": "seconds since 1950-01-01 00:00:00",
+        "model_class": "Mercator",
+        "quantum": "day",
+        "lat_var_key": "latitude",
+        "lon_var_key": "longitude",
+        "attribution": "attrib",
+        "variables": {
+            "votemper": { "name": "Temperature", "units": "Celsius", "scale": [-5, 30] }
+        }
+    }
+}

tests/testdata/datasetconfigpatch-stubs/nemo_sqlite3.json

@@ -0,0 +1,14 @@
+{
+    "nemo_sqlite3": {
+        "enabled": 1,
+        "name": "Nemo SQLITE3",
+        "url": "tests/testdata/databases/test-nemo.sqlite3",
+        "quantum": "day",
+        "time_dim_units": "seconds since 1950-01-01 00:00:00",
+        "model_class": "Nemo",
+        "attribution": "attrib",
+        "variables": {
+            "votemper": { "name": "Temperature", "units": "Celsius", "scale": [-5, 30] }
+        }
+    }
+}

tests/testdata/datasetconfigpatch-stubs/salishseacast_currents.json

@@ -0,0 +1,25 @@
+{
+    "salishseacast_currents": {
+        "enabled": true,
+        "name": "SalishSeaCast 3D Currents",
+        "type": "historical",
+        "url": [
+            "https://salishsea.eos.ubc.ca/erddap/griddap/ubcSSg3DuGridFields1hV21-11",
+            "https://salishsea.eos.ubc.ca/erddap/griddap/ubcSSg3DvGridFields1hV21-11"
+        ],
+        "geo_ref": {
+            "url": "https://salishsea.eos.ubc.ca/erddap/griddap/ubcSSnBathymetryV21-08",
+            "drop_variables": ["bathymetry"]
+        },
+        "quantum": "hour",
+        "time_dim_units": "seconds since 1970-01-01 00:00:00",
+        "model_class": "Nemo",
+        "lat_var_key": "nav_lat",
+        "lon_var_key": "nav_lon",
+        "attribution": "UBC-MOAD",
+        "variables": {
+            "uVelocity": { "name":  "Eastward Current", "units":  "m/s", "scale":  [-8, 8], "zero_centered": "true" },
+            "vVelocity": { "name":  "Northward Current", "units":  "m/s", "scale":  [-8, 8], "zero_centered": "true" }
+        }
+    }
+}

tests/testdata/datasetconfigpatch-stubs/salishseacast_ssh.json

@@ -0,0 +1,21 @@
+{
+    "salishseacast_ssh": {
+        "enabled": true,
+        "url": "https://salishsea.eos.ubc.ca/erddap/griddap/ubcSSgSurfaceTracerFields1hV19-05",
+        "geo_ref": {
+            "url": "https://salishsea.eos.ubc.ca/erddap/griddap/ubcSSnBathymetryV21-08",
+            "drop_variables": ["bathymetry"]
+        },
+        "name": "SalishSeaCast Sea Surface Height",
+        "quantum": "hour",
+        "type": "historical",
+        "time_dim_units": "seconds since 1970-01-01 00:00:00",
+        "model_class": "Nemo",
+        "attribution": "UBC-MOAD",
+        "lat_var_key": "nav_lat",
+        "lon_var_key": "nav_lon",
+        "variables": {
+            "ssh": { "name":  "Sea Surface Height", "units":  "m", "scale":  [-4, 4], "zero_centered": "true" }
+        }
+    }
+}