diff --git a/oceannavigator/configs b/oceannavigator/configs
index 7246bd933..9733b8f05 160000
--- a/oceannavigator/configs
+++ b/oceannavigator/configs
@@ -1 +1 @@
-Subproject commit 7246bd933743bfbdd0e11cf3dcd3361bc38229a6
+Subproject commit 9733b8f0500b87655dc331dee0bf0f2cce398a8a
diff --git a/oceannavigator/dataset_config.py b/oceannavigator/dataset_config.py
index 7b4bd919d..c4979b4c6 100644
--- a/oceannavigator/dataset_config.py
+++ b/oceannavigator/dataset_config.py
@@ -32,12 +32,25 @@ def get_datasets() -> List[str]:
@lru_cache()
def _get_dataset_config() -> dict:
settings = get_settings()
+
with open(settings.dataset_config_file, "r") as f:
config = json.load(f)
log().debug(
f"Loaded dataset config file from: {settings.dataset_config_file}"
)
- return config
+
+ for name in config:
+ stub_file = f"{settings.dataset_config_stub_path}{name}.json"
+
+ with open(stub_file, "r") as f:
+ config[name] = json.load(f)[name]
+ log().debug(
+ f"Loaded stub dataset config file from: {stub_file}"
+ )
+
+ return config
+
+
def _get_attribute(self, key: str) -> Union[str, dict]:
return self._config.get(key) if not None else ""
@@ -111,21 +124,6 @@ def bathymetry_file_url(self) -> str:
"""
return self._get_attribute("bathymetry_file_url")
- @property
- def vector_arrow_stride(self) -> int:
- """
- Returns the stride used to slice the dataset for generating geojson
- for the vector arrows (i.e. every n-th value).
-
- Defaults to 4.
- """
-
- stride = self._get_attribute("vector_arrow_stride")
- if stride:
- return stride
-
- return 4
-
@property
def model_class(self) -> str:
return self._get_attribute("model_class")
@@ -178,17 +176,6 @@ def attribution(self) -> str:
except KeyError:
return ""
- @property
- def cache(self) -> int:
- """
- Returns the cache value for a dataset as defined in dataset config file
- """
- cache = self._get_attribute("cache")
- if cache is not None and isinstance(cache, str):
- cache = int(cache)
-
- return cache
-
@property
def default_location(self) -> list:
"""
diff --git a/oceannavigator/settings.py b/oceannavigator/settings.py
index 05c12be4b..bf9c425aa 100644
--- a/oceannavigator/settings.py
+++ b/oceannavigator/settings.py
@@ -34,6 +34,7 @@ class Settings(BaseSettings):
dask_scheduler: str = ""
dataset_shape_file_dir: str = ""
dataset_config_file: str = ""
+ dataset_config_stub_path: str = ""
debug: bool = False
drifter_agg_url: str = ""
drifter_catalog_url: str = ""
diff --git a/tests/test_dataset_config.py b/tests/test_dataset_config.py
index 707df6071..770d06520 100644
--- a/tests/test_dataset_config.py
+++ b/tests/test_dataset_config.py
@@ -28,7 +28,6 @@ def test_datasetconfig_object(self, m):
"model_class": "my_model_class",
"time_dim_units": "my_time_units",
"attribution": "my_attribution",
- "cache": "123",
"lat_var_key": "my_lat",
"lon_var_key": "my_lon",
"variables": {
@@ -58,7 +57,6 @@ def test_datasetconfig_object(self, m):
assert result.lat_var_key == "my_lat"
assert result.lon_var_key == "my_lon"
assert result.attribution == "my_attribution"
- assert result.cache == 123
assert not result.variable[Mock(key="var")].is_hidden
@@ -135,17 +133,14 @@ def test_get_dataset_misc(self, m):
"url": "the_url",
"attribution": "My attribution bold",
"climatology": "climatology_url",
- "cache": 5,
}
}
assert DatasetConfig("dataset").url == "the_url"
assert DatasetConfig("dataset").climatology == "climatology_url"
assert DatasetConfig("dataset").attribution == "My attribution bold"
- assert DatasetConfig("dataset").cache == 5
m.return_value = {"dataset2": {}}
- assert DatasetConfig("dataset2").cache is None
@patch.object(DatasetConfig, "_get_dataset_config")
def test_get_variables(self, m):
@@ -181,7 +176,6 @@ def test_get_variable_misc(self, m):
"url": "the_url",
"attribution": "My attribution bold",
"climatology": "climatology_url",
- "cache": 5,
"variables": {
"var": {
"name": "the_name",
diff --git a/tests/test_netcdf_data.py b/tests/test_netcdf_data.py
index 8117c73ce..0406b321f 100644
--- a/tests/test_netcdf_data.py
+++ b/tests/test_netcdf_data.py
@@ -20,8 +20,15 @@
class TestNetCDFData(unittest.TestCase):
- with open("tests/testdata/datasetconfigpatch.json") as dataPatch:
- patch_dataset_config_ret_val = json.load(dataPatch)
+
+ with open("tests/testdata/datasetconfigpatch.json", "r") as f:
+ patch_dataset_config_ret_val = json.load(f)
+
+ for name in patch_dataset_config_ret_val:
+ stub_file = f"tests/testdata/datasetconfigpatch-stubs/{name}.json"
+
+ with open(stub_file, "r") as f:
+ patch_dataset_config_ret_val[name] = json.load(f)[name]
@pytest.fixture(scope="class", autouse=True)
def setUp_teardown(self):
diff --git a/tests/testdata/datasetconfigpatch-stubs/giops.json b/tests/testdata/datasetconfigpatch-stubs/giops.json
new file mode 100644
index 000000000..9f94790b0
--- /dev/null
+++ b/tests/testdata/datasetconfigpatch-stubs/giops.json
@@ -0,0 +1,17 @@
+{
+ "giops": {
+ "enabled": 1,
+ "url": "tests/testdata/nemo_test.nc",
+ "time_dim_units": "seconds since 1950-01-01 00:00:00",
+ "model_class": "Mercator",
+ "quantum": "day",
+ "name": "GIOPS",
+ "help": "help",
+ "attribution": "attrib",
+ "lat_var_key": "nav_lat",
+ "lon_var_key": "nav_lon",
+ "variables": {
+ "votemper": { "name": "Temperature", "units": "Celsius", "scale": [-5, 30] }
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/testdata/datasetconfigpatch-stubs/giops_icechunk.json b/tests/testdata/datasetconfigpatch-stubs/giops_icechunk.json
new file mode 100644
index 000000000..54a6f034e
--- /dev/null
+++ b/tests/testdata/datasetconfigpatch-stubs/giops_icechunk.json
@@ -0,0 +1,20 @@
+{
+ "giops_icechunk": {
+ "group": "GIOPS Forecast",
+ "url": "icechunk",
+ "name": "GIOPS 10 Day Daily Mean Forecast 3D - LatLon",
+ "quantum": "hour",
+ "type": "forecast",
+ "time_dim_units": "seconds since 1950-01-01 00:00:00",
+ "model_class": "Mercator",
+ "enabled": true,
+ "attribution": "The Canadian Centre for Meteorological and Environmental Prediction",
+ "lat_var_key": "latitude",
+ "lon_var_key": "longitude",
+ "help": "./data-help/giops-10day.html",
+ "vector_variables": ["magwatervel"],
+ "variables": {
+ "votemper": { "name": "Potential Temperature", "units": "Celsius", "scale": [-5, 30], "equation": "votemper - 273.15", "dims": ["time", "depth", "latitude", "longitude"], "valid_min": -100, "valid_max": 100 }
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/testdata/datasetconfigpatch-stubs/giops_no_url.json b/tests/testdata/datasetconfigpatch-stubs/giops_no_url.json
new file mode 100644
index 000000000..e79324883
--- /dev/null
+++ b/tests/testdata/datasetconfigpatch-stubs/giops_no_url.json
@@ -0,0 +1,14 @@
+{
+ "giops_no_url": {
+ "enabled": 1,
+ "time_dim_units": "seconds since 1950-01-01 00:00:00",
+ "model_class": "Mercator",
+ "quantum": "day",
+ "name": "GIOPS",
+ "help": "help",
+ "attribution": "attrib",
+ "variables": {
+ "votemper": { "name": "Temperature", "units": "Celsius", "scale": [-5, 30] }
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/testdata/datasetconfigpatch-stubs/giops_real.json b/tests/testdata/datasetconfigpatch-stubs/giops_real.json
new file mode 100644
index 000000000..8cb54d39d
--- /dev/null
+++ b/tests/testdata/datasetconfigpatch-stubs/giops_real.json
@@ -0,0 +1,23 @@
+{
+ "giops_real": {
+ "enabled": 1,
+ "url": "tests/testdata/giops_test.nc",
+ "name": "GIOPS Forecast 3D - Polar Stereographic",
+ "model_class": "Mercator",
+ "quantum": "day",
+ "type": "forecast",
+ "lat_var_key": "latitude",
+ "lon_var_key": "longitude",
+ "time_dim_units": "seconds since 1950-01-01 00:00:00",
+ "attribution": "The Canadian Centre for Meteorological and Environmental Prediction",
+ "vector_variables": ["magwatervel"],
+ "variables": {
+ "votemper": { "name": "Temperature", "units": "Celsius", "scale": [-5, 30] },
+ "vosaline": { "name": "Salinity", "units": "PSU", "scale": [30, 40] },
+ "vozocrtx": { "name": "Water Y Velocity", "units": "m/s", "scale": [-3, 3], "zero_centered": "true" },
+ "vomecrty": { "name": "Water X Velocity", "units": "m/s", "scale": [-3, 3], "zero_centered": "true" },
+ "magwatervel": { "name": "Water Velocity", "units": "m/s", "scale": [0, 3], "equation": "magnitude(vozocrtx, vomecrty)", "dims": ["time", "depth", "latitude", "longitude"], "east_vector_component": "vozocrtx", "north_vector_component": "vomecrty" },
+ "bearing": { "hide": "true", "name": "Water Velocity Bearing (deg clockwise +tive from N)", "units": "degrees", "scale": [0, 360], "equation": "bearing(vomecrty, vozocrtx)" ,"dims": ["time", "depth", "latitude", "longitude"] }
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/testdata/datasetconfigpatch-stubs/mercator_test.json b/tests/testdata/datasetconfigpatch-stubs/mercator_test.json
new file mode 100644
index 000000000..10582d00e
--- /dev/null
+++ b/tests/testdata/datasetconfigpatch-stubs/mercator_test.json
@@ -0,0 +1,16 @@
+{
+ "mercator_test": {
+ "enabled": 1,
+ "url": "tests/testdata/mercator_test.nc",
+ "name": "Mercator test",
+ "time_dim_units": "seconds since 1950-01-01 00:00:00",
+ "model_class": "Mercator",
+ "quantum": "day",
+ "lat_var_key": "latitude",
+ "lon_var_key": "longitude",
+ "attribution": "attrib",
+ "variables": {
+ "votemper": { "name": "Temperature", "units": "Celsius", "scale": [-5, 30] }
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/testdata/datasetconfigpatch-stubs/nemo_sqlite3.json b/tests/testdata/datasetconfigpatch-stubs/nemo_sqlite3.json
new file mode 100644
index 000000000..b4b2811e8
--- /dev/null
+++ b/tests/testdata/datasetconfigpatch-stubs/nemo_sqlite3.json
@@ -0,0 +1,14 @@
+{
+ "nemo_sqlite3": {
+ "enabled": 1,
+ "name": "Nemo SQLITE3",
+ "url": "tests/testdata/databases/test-nemo.sqlite3",
+ "quantum": "day",
+ "time_dim_units": "seconds since 1950-01-01 00:00:00",
+ "model_class": "Nemo",
+ "attribution": "attrib",
+ "variables": {
+ "votemper": { "name": "Temperature", "units": "Celsius", "scale": [-5, 30] }
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/testdata/datasetconfigpatch-stubs/salishseacast_currents.json b/tests/testdata/datasetconfigpatch-stubs/salishseacast_currents.json
new file mode 100644
index 000000000..b5c063d3a
--- /dev/null
+++ b/tests/testdata/datasetconfigpatch-stubs/salishseacast_currents.json
@@ -0,0 +1,25 @@
+{
+ "salishseacast_currents": {
+ "enabled": true,
+ "name": "SalishSeaCast 3D Currents",
+ "type": "historical",
+ "url": [
+ "https://salishsea.eos.ubc.ca/erddap/griddap/ubcSSg3DuGridFields1hV21-11",
+ "https://salishsea.eos.ubc.ca/erddap/griddap/ubcSSg3DvGridFields1hV21-11"
+ ],
+ "geo_ref": {
+ "url": "https://salishsea.eos.ubc.ca/erddap/griddap/ubcSSnBathymetryV21-08",
+ "drop_variables": ["bathymetry"]
+ },
+ "quantum": "hour",
+ "time_dim_units": "seconds since 1970-01-01 00:00:00",
+ "model_class": "Nemo",
+ "lat_var_key": "nav_lat",
+ "lon_var_key": "nav_lon",
+ "attribution": "UBC-MOAD",
+ "variables": {
+ "uVelocity": { "name": "Eastward Current", "units": "m/s", "scale": [-8, 8], "zero_centered": "true" },
+ "vVelocity": { "name": "Northward Current", "units": "m/s", "scale": [-8, 8], "zero_centered": "true" }
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/testdata/datasetconfigpatch-stubs/salishseacast_ssh.json b/tests/testdata/datasetconfigpatch-stubs/salishseacast_ssh.json
new file mode 100644
index 000000000..7b26150e5
--- /dev/null
+++ b/tests/testdata/datasetconfigpatch-stubs/salishseacast_ssh.json
@@ -0,0 +1,21 @@
+{
+ "salishseacast_ssh": {
+ "enabled": true,
+ "url": "https://salishsea.eos.ubc.ca/erddap/griddap/ubcSSgSurfaceTracerFields1hV19-05",
+ "geo_ref": {
+ "url": "https://salishsea.eos.ubc.ca/erddap/griddap/ubcSSnBathymetryV21-08",
+ "drop_variables": ["bathymetry"]
+ },
+ "name": "SalishSeaCast Sea Surface Height",
+ "quantum": "hour",
+ "type": "historical",
+ "time_dim_units": "seconds since 1970-01-01 00:00:00",
+ "model_class": "Nemo",
+ "attribution": "UBC-MOAD",
+ "lat_var_key": "nav_lat",
+ "lon_var_key": "nav_lon",
+ "variables": {
+ "ssh": { "name": "Sea Surface Height", "units": "m", "scale": [-4, 4], "zero_centered": "true" }
+ }
+ }
+}
\ No newline at end of file
diff --git a/tests/testdata/datasetconfigpatch.json b/tests/testdata/datasetconfigpatch.json
index 71131bbee..a1e8015b5 100644
--- a/tests/testdata/datasetconfigpatch.json
+++ b/tests/testdata/datasetconfigpatch.json
@@ -1,143 +1,17 @@
{
- "giops": {
- "enabled": 1,
- "url": "tests/testdata/nemo_test.nc",
- "time_dim_units": "seconds since 1950-01-01 00:00:00",
- "model_class": "Mercator",
- "quantum": "day",
- "name": "GIOPS",
- "help": "help",
- "attribution": "attrib",
- "lat_var_key": "nav_lat",
- "lon_var_key": "nav_lon",
- "variables": {
- "votemper": { "name": "Temperature", "units": "Celsius", "scale": [-5, 30] }
- }
- },
+ "giops": {},
- "giops_no_url": {
- "enabled": 1,
- "time_dim_units": "seconds since 1950-01-01 00:00:00",
- "model_class": "Mercator",
- "quantum": "day",
- "name": "GIOPS",
- "help": "help",
- "attribution": "attrib",
- "variables": {
- "votemper": { "name": "Temperature", "units": "Celsius", "scale": [-5, 30] }
- }
- },
+ "giops_no_url": {},
- "giops_real": {
- "enabled": 1,
- "url": "tests/testdata/giops_test.nc",
- "name": "GIOPS Forecast 3D - Polar Stereographic",
- "model_class": "Mercator",
- "quantum": "day",
- "type": "forecast",
- "lat_var_key": "latitude",
- "lon_var_key": "longitude",
- "time_dim_units": "seconds since 1950-01-01 00:00:00",
- "attribution": "The Canadian Centre for Meteorological and Environmental Prediction",
- "vector_variables": ["magwatervel"],
- "variables": {
- "votemper": { "name": "Temperature", "units": "Celsius", "scale": [-5, 30] },
- "vosaline": { "name": "Salinity", "units": "PSU", "scale": [30, 40] },
- "vozocrtx": { "name": "Water Y Velocity", "units": "m/s", "scale": [-3, 3], "zero_centered": "true" },
- "vomecrty": { "name": "Water X Velocity", "units": "m/s", "scale": [-3, 3], "zero_centered": "true" },
- "magwatervel": { "name": "Water Velocity", "units": "m/s", "scale": [0, 3], "equation": "magnitude(vozocrtx, vomecrty)", "dims": ["time", "depth", "latitude", "longitude"], "east_vector_component": "vozocrtx", "north_vector_component": "vomecrty" },
- "bearing": { "hide": "true", "name": "Water Velocity Bearing (deg clockwise +tive from N)", "units": "degrees", "scale": [0, 360], "equation": "bearing(vomecrty, vozocrtx)" ,"dims": ["time", "depth", "latitude", "longitude"] }
- }
- },
+ "giops_real": {},
- "giops_icechunk": {
- "group": "GIOPS Forecast",
- "url": "icechunk",
- "name": "GIOPS 10 Day Daily Mean Forecast 3D - LatLon",
- "quantum": "hour",
- "type": "forecast",
- "time_dim_units": "seconds since 1950-01-01 00:00:00",
- "model_class": "Mercator",
- "enabled": true,
- "attribution": "The Canadian Centre for Meteorological and Environmental Prediction",
- "lat_var_key": "latitude",
- "lon_var_key": "longitude",
- "help": "./data-help/giops-10day.html",
- "vector_variables": ["magwatervel"],
- "variables": {
- "votemper": { "name": "Potential Temperature", "units": "Celsius", "scale": [-5, 30], "equation": "votemper - 273.15", "dims": ["time", "depth", "latitude", "longitude"], "valid_min": -100, "valid_max": 100 }
- }
- },
+ "giops_icechunk": {},
- "nemo_sqlite3": {
- "enabled": 1,
- "name": "Nemo SQLITE3",
- "url": "tests/testdata/databases/test-nemo.sqlite3",
- "quantum": "day",
- "time_dim_units": "seconds since 1950-01-01 00:00:00",
- "model_class": "Nemo",
- "attribution": "attrib",
- "variables": {
- "votemper": { "name": "Temperature", "units": "Celsius", "scale": [-5, 30] }
- }
- },
+ "nemo_sqlite3": {},
- "salishseacast_ssh": {
- "enabled": true,
- "url": "https://salishsea.eos.ubc.ca/erddap/griddap/ubcSSgSurfaceTracerFields1hV19-05",
- "geo_ref": {
- "url": "https://salishsea.eos.ubc.ca/erddap/griddap/ubcSSnBathymetryV21-08",
- "drop_variables": ["bathymetry"]
- },
- "name": "SalishSeaCast Sea Surface Height",
- "quantum": "hour",
- "type": "historical",
- "time_dim_units": "seconds since 1970-01-01 00:00:00",
- "model_class": "Nemo",
- "attribution": "UBC-MOAD",
- "lat_var_key": "nav_lat",
- "lon_var_key": "nav_lon",
- "variables": {
- "ssh": { "name": "Sea Surface Height", "units": "m", "scale": [-4, 4], "zero_centered": "true" }
- }
- },
+ "salishseacast_ssh": {},
- "salishseacast_currents": {
- "enabled": true,
- "name": "SalishSeaCast 3D Currents",
- "type": "historical",
- "url": [
- "https://salishsea.eos.ubc.ca/erddap/griddap/ubcSSg3DuGridFields1hV21-11",
- "https://salishsea.eos.ubc.ca/erddap/griddap/ubcSSg3DvGridFields1hV21-11"
- ],
- "geo_ref": {
- "url": "https://salishsea.eos.ubc.ca/erddap/griddap/ubcSSnBathymetryV21-08",
- "drop_variables": ["bathymetry"]
- },
- "quantum": "hour",
- "time_dim_units": "seconds since 1970-01-01 00:00:00",
- "model_class": "Nemo",
- "lat_var_key": "nav_lat",
- "lon_var_key": "nav_lon",
- "attribution": "UBC-MOAD",
- "variables": {
- "uVelocity": { "name": "Eastward Current", "units": "m/s", "scale": [-8, 8], "zero_centered": "true" },
- "vVelocity": { "name": "Northward Current", "units": "m/s", "scale": [-8, 8], "zero_centered": "true" }
- }
- },
+ "salishseacast_currents": {},
- "mercator_test": {
- "enabled": 1,
- "url": "tests/testdata/mercator_test.nc",
- "name": "Mercator test",
- "time_dim_units": "seconds since 1950-01-01 00:00:00",
- "model_class": "Mercator",
- "quantum": "day",
- "lat_var_key": "latitude",
- "lon_var_key": "longitude",
- "attribution": "attrib",
- "variables": {
- "votemper": { "name": "Temperature", "units": "Celsius", "scale": [-5, 30] }
- }
- }
+ "mercator_test": {}
}