From e5a22dca474572708960e6c95c74c06ca9d19199 Mon Sep 17 00:00:00 2001 From: Paul Slavin Date: Fri, 15 Nov 2024 17:03:14 +0000 Subject: [PATCH 01/28] ConfigKey types and lib prototypes --- hydra_base/db/model/hydraconfig/__init__.py | 4 ++ .../db/model/hydraconfig/config_key_types.py | 50 +++++++++++++++++ .../db/model/hydraconfig/hydraconfig.py | 26 +++++++++ hydra_base/lib/hydraconfig.py | 56 +++++++++++++++++++ 4 files changed, 136 insertions(+) create mode 100644 hydra_base/db/model/hydraconfig/__init__.py create mode 100644 hydra_base/db/model/hydraconfig/config_key_types.py create mode 100644 hydra_base/db/model/hydraconfig/hydraconfig.py create mode 100644 hydra_base/lib/hydraconfig.py diff --git a/hydra_base/db/model/hydraconfig/__init__.py b/hydra_base/db/model/hydraconfig/__init__.py new file mode 100644 index 00000000..4e18f054 --- /dev/null +++ b/hydra_base/db/model/hydraconfig/__init__.py @@ -0,0 +1,4 @@ +from config_key_types import ( + ConfigKey, + config_key_type_map +) diff --git a/hydra_base/db/model/hydraconfig/config_key_types.py b/hydra_base/db/model/hydraconfig/config_key_types.py new file mode 100644 index 00000000..18ccd228 --- /dev/null +++ b/hydra_base/db/model/hydraconfig/config_key_types.py @@ -0,0 +1,50 @@ +import logging +from abc import ABC, abstractmethod + + +log = logging.getLogger(__name__) +config_key_type_map = {} + + +class ConfigKey(ABC): + subclass_type_key = "config_key_type" + key_name_max_length = 200 + key_desc_max_length = 1000 + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + keyattr = __class__.subclass_type_key + configkey_type = getattr(cls, keyattr, None) + if not configkey_type or not isinstance(configkey_type, str): + raise NotImplementedError(f"ConfigKey subclass {cls.__name__} does not define a '{keyattr}' attribute") + config_key_type_map[configkey_type] = cls + log.debug(f"Registered ConfigKey type '{cls.__name__}' with key '{configkey_type}'") + + +class ConfigKey_Integer(ConfigKey): + config_key_type = "integer" + # min, max + + +class ConfigKey_String(ConfigKey): + config_key_type = "string" + # max_len + + +class ConfigKey_Boolean(ConfigKey): + config_key_type = "boolean" + # True, False only: not 'Y' 'N' 'X' etc + + +class ConfigKey_Uri(ConfigKey): + config_key_type = "uri" + # Includes paths with file:// scheme + + +class ConfigKey_Json(ConfigKey): + config_key_type = "json" + # Unvalidated json object + + +if __name__ == "__main__": + print(config_key_type_map) diff --git a/hydra_base/db/model/hydraconfig/hydraconfig.py b/hydra_base/db/model/hydraconfig/hydraconfig.py new file mode 100644 index 00000000..2224192e --- /dev/null +++ b/hydra_base/db/model/hydraconfig/hydraconfig.py @@ -0,0 +1,26 @@ +""" + Types and definitions for Hydra config values +""" + +from hydra_base.db.model.base import * +from hydra_base.db.model.hydraconfig import ( + ConfigKey, + config_key_type_map +) + +from sqlalchemy import Enum + +__all__ = ["ConfigKeyRecord",] + + +class ConfigKeyRecord(AuditMixin, Base, Inspect): + __tablename__ = "tConfigKey" + + id = Column(Integer(), primary_key=True, nullable=False) + name = Column(String(ConfigKey.key_name_max_length), nullable=False, unique=True) + description = Column(String(ConfigKey.key_desc_max_length)) + type = Column(Enum(*config_key_type_map.keys())) + + +if __name__ == "__main__": + print(config_key_type_map) diff --git a/hydra_base/lib/hydraconfig.py b/hydra_base/lib/hydraconfig.py new file mode 100644 index 00000000..8e82eb03 --- /dev/null +++ b/hydra_base/lib/hydraconfig.py @@ -0,0 +1,56 @@ +""" + Library functions for Hydra configuration +""" + +""" Config Keys: Key:Value pairs of config settings """ + +def register_config_key(key): + pass + +def unregister_config_key(key): + pass + +def list_config_keys(): + pass + +def set_config_value(key, value): + pass + +def get_config_value(key): + pass + + +""" Config Sets: Archived versions of complete configurations """ + +def create_configset(set_name): + pass + +def delete_configset(set_name): + pass + +def apply_configset(set_name): + pass + +def list_configsets(): + pass + +def list_configset_versions(set_name): + pass + + +""" Config Groups: A named collection of Config Keys """ + +def create_config_group(group_name): + pass + +def delete_config_group(group_name): + pass + +def list_config_groups(): + pass + +def add_config_key_to_group(key_name, group_name): + pass + +def remove_config_key_from_group(key_name, group_name): + pass From 343cae0b9a2d0347c51349e2ac4f4da033d91c4b Mon Sep 17 00:00:00 2001 From: Paul Slavin Date: Mon, 18 Nov 2024 17:08:36 +0000 Subject: [PATCH 02/28] Add tests; Before STI --- hydra_base/__init__.py | 1 + hydra_base/db/model/hydraconfig/__init__.py | 4 ++- .../db/model/hydraconfig/config_key_types.py | 11 ++++++ hydra_base/lib/hydraconfig.py | 36 ++++++++++++++++--- 4 files changed, 47 insertions(+), 5 deletions(-) diff --git a/hydra_base/__init__.py b/hydra_base/__init__.py index b8dff950..99fc214a 100644 --- a/hydra_base/__init__.py +++ b/hydra_base/__init__.py @@ -72,3 +72,4 @@ from .lib.units import * from .lib.users import * from .lib.service import * +from .lib.hydraconfig import * diff --git a/hydra_base/db/model/hydraconfig/__init__.py b/hydra_base/db/model/hydraconfig/__init__.py index 4e18f054..fc46e6a3 100644 --- a/hydra_base/db/model/hydraconfig/__init__.py +++ b/hydra_base/db/model/hydraconfig/__init__.py @@ -1,4 +1,6 @@ -from config_key_types import ( +from .config_key_types import ( ConfigKey, config_key_type_map ) + +from .hydraconfig import * diff --git a/hydra_base/db/model/hydraconfig/config_key_types.py b/hydra_base/db/model/hydraconfig/config_key_types.py index 18ccd228..7a56bd18 100644 --- a/hydra_base/db/model/hydraconfig/config_key_types.py +++ b/hydra_base/db/model/hydraconfig/config_key_types.py @@ -20,10 +20,21 @@ def __init_subclass__(cls, **kwargs): config_key_type_map[configkey_type] = cls log.debug(f"Registered ConfigKey type '{cls.__name__}' with key '{configkey_type}'") + def __init__(self, name, desc=None): + if not name: + raise ValueError(f"ConfigKey requires a valid name, not '{name}'") + + self.name = name + self.description = desc if desc else "" + + class ConfigKey_Integer(ConfigKey): config_key_type = "integer" # min, max + def __init__(self, name, desc=None): + super().__init__(name, desc) + class ConfigKey_String(ConfigKey): diff --git a/hydra_base/lib/hydraconfig.py b/hydra_base/lib/hydraconfig.py index 8e82eb03..d3ffb179 100644 --- a/hydra_base/lib/hydraconfig.py +++ b/hydra_base/lib/hydraconfig.py @@ -2,16 +2,44 @@ Library functions for Hydra configuration """ +from hydra_base import db +from hydra_base.exceptions import ( + HydraError, + ResourceNotFoundError, + PermissionError +) + +from hydra_base.db.model.hydraconfig import ( + ConfigKeyRecord, + config_key_type_map +) + + """ Config Keys: Key:Value pairs of config settings """ -def register_config_key(key): - pass +def register_config_key(key_name, key_type, **kwargs): + + if not (key_cls := config_key_type_map.get(key_type, None)): + raise HydraError(f"Invalid ConfigKey type '{key_type}'") + + key = key_cls(key_name) + key_record = ConfigKeyRecord(name=key.name, type=key.config_key_type, description=key.description) + db.DBSession.add(key_record) + db.DBSession.flush() + + return key_record + #return f"Register: {key} {kwargs}" def unregister_config_key(key): pass -def list_config_keys(): - pass +def list_config_keys(like=None, **kwargs): + query = db.DBSession.query(ConfigKeyRecord) + if like: + query = query.filter(ConfigKeyRecord.name.like(f"%{like}%")) + + keys = query.all() + return [key.name for key in keys] def set_config_value(key, value): pass From 17a503e73996a368b8eaa90317b185c3c4d489c9 Mon Sep 17 00:00:00 2001 From: Paul Slavin Date: Tue, 19 Nov 2024 10:59:21 +0000 Subject: [PATCH 03/28] STI with init_subclass --- hydra_base/db/model/hydraconfig/__init__.py | 5 - .../db/model/hydraconfig/config_key_types.py | 61 ------------ .../db/model/hydraconfig/hydraconfig.py | 95 ++++++++++++++++--- hydra_base/lib/hydraconfig.py | 15 ++- 4 files changed, 88 insertions(+), 88 deletions(-) delete mode 100644 hydra_base/db/model/hydraconfig/config_key_types.py diff --git a/hydra_base/db/model/hydraconfig/__init__.py b/hydra_base/db/model/hydraconfig/__init__.py index fc46e6a3..4cf6bea0 100644 --- a/hydra_base/db/model/hydraconfig/__init__.py +++ b/hydra_base/db/model/hydraconfig/__init__.py @@ -1,6 +1 @@ -from .config_key_types import ( - ConfigKey, - config_key_type_map -) - from .hydraconfig import * diff --git a/hydra_base/db/model/hydraconfig/config_key_types.py b/hydra_base/db/model/hydraconfig/config_key_types.py deleted file mode 100644 index 7a56bd18..00000000 --- a/hydra_base/db/model/hydraconfig/config_key_types.py +++ /dev/null @@ -1,61 +0,0 @@ -import logging -from abc import ABC, abstractmethod - - -log = logging.getLogger(__name__) -config_key_type_map = {} - - -class ConfigKey(ABC): - subclass_type_key = "config_key_type" - key_name_max_length = 200 - key_desc_max_length = 1000 - - def __init_subclass__(cls, **kwargs): - super().__init_subclass__(**kwargs) - keyattr = __class__.subclass_type_key - configkey_type = getattr(cls, keyattr, None) - if not configkey_type or not isinstance(configkey_type, str): - raise NotImplementedError(f"ConfigKey subclass {cls.__name__} does not define a '{keyattr}' attribute") - config_key_type_map[configkey_type] = cls - log.debug(f"Registered ConfigKey type '{cls.__name__}' with key '{configkey_type}'") - - def __init__(self, name, desc=None): - if not name: - raise ValueError(f"ConfigKey requires a valid name, not '{name}'") - - self.name = name - self.description = desc if desc else "" - - - -class ConfigKey_Integer(ConfigKey): - config_key_type = "integer" - # min, max - def __init__(self, name, desc=None): - super().__init__(name, desc) - - - -class ConfigKey_String(ConfigKey): - config_key_type = "string" - # max_len - - -class ConfigKey_Boolean(ConfigKey): - config_key_type = "boolean" - # True, False only: not 'Y' 'N' 'X' etc - - -class ConfigKey_Uri(ConfigKey): - config_key_type = "uri" - # Includes paths with file:// scheme - - -class ConfigKey_Json(ConfigKey): - config_key_type = "json" - # Unvalidated json object - - -if __name__ == "__main__": - print(config_key_type_map) diff --git a/hydra_base/db/model/hydraconfig/hydraconfig.py b/hydra_base/db/model/hydraconfig/hydraconfig.py index 2224192e..0e445d9a 100644 --- a/hydra_base/db/model/hydraconfig/hydraconfig.py +++ b/hydra_base/db/model/hydraconfig/hydraconfig.py @@ -1,26 +1,95 @@ """ Types and definitions for Hydra config values """ - from hydra_base.db.model.base import * -from hydra_base.db.model.hydraconfig import ( - ConfigKey, - config_key_type_map -) -from sqlalchemy import Enum -__all__ = ["ConfigKeyRecord",] +__all__ = ["ConfigKey", "config_key_type_map"] + +config_key_type_map = {} -class ConfigKeyRecord(AuditMixin, Base, Inspect): +class ConfigKey(Base): __tablename__ = "tConfigKey" + key_name_max_length = 200 + key_desc_max_length = 1000 + id = Column(Integer(), primary_key=True, nullable=False) - name = Column(String(ConfigKey.key_name_max_length), nullable=False, unique=True) - description = Column(String(ConfigKey.key_desc_max_length)) - type = Column(Enum(*config_key_type_map.keys())) + name = Column(String(key_name_max_length), nullable=False, unique=True) + description = Column(String(key_desc_max_length)) + type = Column(String(40)) + + + __mapper_args__ = { + "polymorphic_on": type, + "polymorphic_identity": "configkey" + } + + subclass_type_key = "config_key_type" + + @classmethod + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + keyattr = __class__.subclass_type_key + configkey_type = getattr(cls, keyattr, None) + if not configkey_type or not isinstance(configkey_type, str): + raise NotImplementedError(f"ConfigKey subclass {cls.__name__} does not define a '{keyattr}' attribute") + config_key_type_map[configkey_type] = cls + log.debug(f"Registered ConfigKey type '{cls.__name__}' with key '{configkey_type}'") + + def __init__(self, name, desc=None): + if not name: + raise ValueError(f"ConfigKey requires a valid name, not '{name}'") + + self.name = name + self.description = desc if desc else "" + + +class ConfigKey_Integer(ConfigKey): + config_key_type = "integer" + + __mapper_args__ = { + "polymorphic_identity": config_key_type + } + + # min, max + def __init__(self, name, desc=None): + super().__init__(name, desc) + + + +class ConfigKey_String(ConfigKey): + config_key_type = "string" + + __mapper_args__ = { + "polymorphic_identity": config_key_type + } + # max_len + + +class ConfigKey_Boolean(ConfigKey): + config_key_type = "boolean" + + __mapper_args__ = { + "polymorphic_identity": config_key_type + } + # True, False only: not 'Y' 'N' 'X' etc + + +class ConfigKey_Uri(ConfigKey): + config_key_type = "uri" + + __mapper_args__ = { + "polymorphic_identity": config_key_type + } + # Includes paths with file:// scheme + +class ConfigKey_Json(ConfigKey): + config_key_type = "json" -if __name__ == "__main__": - print(config_key_type_map) + __mapper_args__ = { + "polymorphic_identity": config_key_type + } + # Unvalidated json object diff --git a/hydra_base/lib/hydraconfig.py b/hydra_base/lib/hydraconfig.py index d3ffb179..c28ff389 100644 --- a/hydra_base/lib/hydraconfig.py +++ b/hydra_base/lib/hydraconfig.py @@ -10,7 +10,7 @@ ) from hydra_base.db.model.hydraconfig import ( - ConfigKeyRecord, + ConfigKey, config_key_type_map ) @@ -18,25 +18,22 @@ """ Config Keys: Key:Value pairs of config settings """ def register_config_key(key_name, key_type, **kwargs): - if not (key_cls := config_key_type_map.get(key_type, None)): raise HydraError(f"Invalid ConfigKey type '{key_type}'") - key = key_cls(key_name) - key_record = ConfigKeyRecord(name=key.name, type=key.config_key_type, description=key.description) - db.DBSession.add(key_record) + key = key_cls(name=key_name) + db.DBSession.add(key) db.DBSession.flush() - return key_record - #return f"Register: {key} {kwargs}" + return key def unregister_config_key(key): pass def list_config_keys(like=None, **kwargs): - query = db.DBSession.query(ConfigKeyRecord) + query = db.DBSession.query(ConfigKey) if like: - query = query.filter(ConfigKeyRecord.name.like(f"%{like}%")) + query = query.filter(ConfigKey.name.like(f"%{like}%")) keys = query.all() return [key.name for key in keys] From b4ff786931875179c5a65156ef21e2952b0dd95f Mon Sep 17 00:00:00 2001 From: Paul Slavin Date: Tue, 19 Nov 2024 16:53:05 +0000 Subject: [PATCH 04/28] Add tests; base64; value get/set --- .../db/model/hydraconfig/hydraconfig.py | 95 ++++++++++++++++-- hydra_base/lib/hydraconfig.py | 11 ++- tests/test_hydraconfig.py | 97 +++++++++++++++++++ 3 files changed, 191 insertions(+), 12 deletions(-) create mode 100644 tests/test_hydraconfig.py diff --git a/hydra_base/db/model/hydraconfig/hydraconfig.py b/hydra_base/db/model/hydraconfig/hydraconfig.py index 0e445d9a..fce6a09d 100644 --- a/hydra_base/db/model/hydraconfig/hydraconfig.py +++ b/hydra_base/db/model/hydraconfig/hydraconfig.py @@ -1,7 +1,16 @@ """ Types and definitions for Hydra config values """ +import base64 +import binascii + from hydra_base.db.model.base import * +from hydra_base.exceptions import HydraError + +from sqlalchemy.orm import ( + Mapped, + mapped_column +) __all__ = ["ConfigKey", "config_key_type_map"] @@ -46,38 +55,108 @@ def __init__(self, name, desc=None): self.description = desc if desc else "" -class ConfigKey_Integer(ConfigKey): +class HasValue: + _value: Mapped[str] = mapped_column(String(2000), nullable=True, use_existing_column=True) + + +class ConfigKey_Integer(ConfigKey, HasValue): config_key_type = "integer" __mapper_args__ = { "polymorphic_identity": config_key_type } - # min, max + def __init__(self, name, desc=None): super().__init__(name, desc) + @property + def value(self): + if self._value is None: + return None + + return int(self._value) + + @value.setter + def value(self, val): + if val is None: + return + + try: + _ = int(val) + except (TypeError, ValueError): + raise HydraError(f"Config Key {self.name} requires an integer value, not {val}") + self._value = val -class ConfigKey_String(ConfigKey): + +class ConfigKey_String(ConfigKey, HasValue): config_key_type = "string" __mapper_args__ = { "polymorphic_identity": config_key_type } - # max_len + def __init__(self, name, desc=None): + super().__init__(name, desc) + + @property + def value(self): + return str(self._value) -class ConfigKey_Boolean(ConfigKey): + @value.setter + def value(self, val): + self._value = str(val) + + +class ConfigKey_Boolean(ConfigKey, HasValue): config_key_type = "boolean" __mapper_args__ = { "polymorphic_identity": config_key_type } - # True, False only: not 'Y' 'N' 'X' etc + + def __init__(self, name, desc=None): + super().__init__(name, desc) + + @property + def value(self): + return self._value == "True" + + @value.setter + def value(self, val): + if val not in (True, False): + raise HydraError(f"Config Key {self.name} with Boolean type accepts only True or False value") + + self._value = "True" if val else "False" + + +class ConfigKey_Base64(ConfigKey, HasValue): + config_key_type = "base64" + + __mapper_args__ = { + "polymorphic_identity": config_key_type + } + + def __init__(self, name, desc=None): + super().__init__(name, desc) + + @property + def value(self): + return self._value + + @value.setter + def value(self, val): + try: + _ = base64.b64decode(val, validate=True) + except (AttributeError, TypeError, binascii.Error) as e: + raise HydraError(f"Config Key {self.name} with Base64 " + f"type accepts only valid Base64 strings") from e + + self._value = val -class ConfigKey_Uri(ConfigKey): +class ConfigKey_Uri(ConfigKey, HasValue): config_key_type = "uri" __mapper_args__ = { @@ -86,7 +165,7 @@ class ConfigKey_Uri(ConfigKey): # Includes paths with file:// scheme -class ConfigKey_Json(ConfigKey): +class ConfigKey_Json(ConfigKey, HasValue): config_key_type = "json" __mapper_args__ = { diff --git a/hydra_base/lib/hydraconfig.py b/hydra_base/lib/hydraconfig.py index c28ff389..9a117e74 100644 --- a/hydra_base/lib/hydraconfig.py +++ b/hydra_base/lib/hydraconfig.py @@ -38,11 +38,14 @@ def list_config_keys(like=None, **kwargs): keys = query.all() return [key.name for key in keys] -def set_config_value(key, value): - pass +def set_config_key_value(key_name, value, **kwargs): + key = db.DBSession.query(ConfigKey).filter(ConfigKey.name == key_name).one() + key.value = value + db.DBSession.flush() -def get_config_value(key): - pass +def get_config_key_value(key_name, **kwargs): + key = db.DBSession.query(ConfigKey).filter(ConfigKey.name == key_name).one() + return key.value """ Config Sets: Archived versions of complete configurations """ diff --git a/tests/test_hydraconfig.py b/tests/test_hydraconfig.py new file mode 100644 index 00000000..ee4cb763 --- /dev/null +++ b/tests/test_hydraconfig.py @@ -0,0 +1,97 @@ +import base64 +import math +import pytest + +from hydra_base.exceptions import HydraError +from hydra_base.lib.hydraconfig import ( + config_key_type_map, + ConfigKey +) + +class TestHydraConfig(): + def test_valid_config_key_type_map(self): + """ + Ensure that all basic config key subclasses are + registered and map to the correct type + """ + assert config_key_type_map is not None + + required_types = ("integer", "string", "boolean", "base64", "uri", "json") + for rtype in required_types: + assert rtype in config_key_type_map + assert issubclass(config_key_type_map[rtype], ConfigKey) + + def test_define_config_key(self, client): + """ + Can only keys with valid names and types be registered + and are these correctly retrieved? + """ + with pytest.raises(HydraError): + client.register_config_key("test_key_name", "invalid type") + + with pytest.raises(ValueError): + client.register_config_key("", "integer") + + itk = client.register_config_key("integer_test_key", "integer") + assert itk.name == "integer_test_key" + assert itk.type == "integer" + + all_keys = client.list_config_keys() + assert itk.name in all_keys + + def test_set_config_key_value(self, client): + """ + Can a key be assigned an appropriate value and + this value then retrieved as the correct type? + """ + # ConfigKey_Integer + key_name = "integer_value_test_key" + key_value = 46 + _ = client.register_config_key(key_name, "integer") + client.set_config_key_value(key_name, key_value) + ret_value = client.get_config_key_value(key_name) + assert isinstance(ret_value, int) + assert ret_value == key_value + + # Invalid int values should be rejected + with pytest.raises(HydraError): + client.set_config_key_value(key_name, math.nan) + + # ConfigKey_String + key_name = "string_value_test_key" + key_value = "A string value" + _ = client.register_config_key(key_name, "string") + client.set_config_key_value(key_name, key_value) + ret_value = client.get_config_key_value(key_name) + assert isinstance(ret_value, str) + assert ret_value == key_value + + + # ConfigKey_Boolean + key_name = "boolean_value_test_key" + key_value = True + _ = client.register_config_key(key_name, "boolean") + client.set_config_key_value(key_name, key_value) + ret_value = client.get_config_key_value(key_name) + assert isinstance(ret_value, bool) + assert ret_value == key_value + + # Invalid bool values should be rejected + with pytest.raises(HydraError): + client.set_config_key_value(key_name, 'Y') + + # ConfigKey_Base64 + key_name = "base64_value_test_key" + raw_bytes = bytes([73, 110, 112, 117, 116, 32, 118, 97, 108, 117, 101]) + b64_bytes = base64.b64encode(raw_bytes) + key_value = b64_bytes.decode("utf8") + _ = client.register_config_key(key_name, "base64") + client.set_config_key_value(key_name, key_value) + ret_value = client.get_config_key_value(key_name) + assert isinstance(ret_value, str) + ret_bytes = base64.b64decode(ret_value, validate=True) + assert ret_bytes == raw_bytes + + # Non-b64 strings should be rejected + with pytest.raises(HydraError): + client.set_config_key_value(key_name, "Not Base64") From 118b555efd4657d88bb1cf9712b59f641c171a04 Mon Sep 17 00:00:00 2001 From: Paul Slavin Date: Wed, 20 Nov 2024 17:22:15 +0000 Subject: [PATCH 05/28] Add validator classes --- .../db/model/hydraconfig/hydraconfig.py | 25 +++-- hydra_base/db/model/hydraconfig/validators.py | 102 ++++++++++++++++++ tests/test_hydraconfig.py | 74 +++++++++++++ 3 files changed, 195 insertions(+), 6 deletions(-) create mode 100644 hydra_base/db/model/hydraconfig/validators.py diff --git a/hydra_base/db/model/hydraconfig/hydraconfig.py b/hydra_base/db/model/hydraconfig/hydraconfig.py index fce6a09d..76434782 100644 --- a/hydra_base/db/model/hydraconfig/hydraconfig.py +++ b/hydra_base/db/model/hydraconfig/hydraconfig.py @@ -7,6 +7,11 @@ from hydra_base.db.model.base import * from hydra_base.exceptions import HydraError +from hydra_base.db.model.hydraconfig.validators import ( + ConfigKeyIntegerValidator, + ConfigKeyStringValidator +) + from sqlalchemy.orm import ( Mapped, mapped_column @@ -22,12 +27,13 @@ class ConfigKey(Base): __tablename__ = "tConfigKey" key_name_max_length = 200 + key_type_tag_max_length = 40 key_desc_max_length = 1000 id = Column(Integer(), primary_key=True, nullable=False) name = Column(String(key_name_max_length), nullable=False, unique=True) description = Column(String(key_desc_max_length)) - type = Column(String(40)) + type = Column(String(key_type_tag_max_length)) __mapper_args__ = { @@ -54,13 +60,19 @@ def __init__(self, name, desc=None): self.name = name self.description = desc if desc else "" + if vcls := getattr(self.__class__, "validator_type", None): + self.validator = vcls() + class HasValue: - _value: Mapped[str] = mapped_column(String(2000), nullable=True, use_existing_column=True) + value_max_length = 2000 + _value: Mapped[str] = mapped_column(String(value_max_length), nullable=True, use_existing_column=True) + class ConfigKey_Integer(ConfigKey, HasValue): config_key_type = "integer" + validator_type = ConfigKeyIntegerValidator __mapper_args__ = { "polymorphic_identity": config_key_type @@ -70,7 +82,7 @@ class ConfigKey_Integer(ConfigKey, HasValue): def __init__(self, name, desc=None): super().__init__(name, desc) - @property + @hybrid_property def value(self): if self._value is None: return None @@ -92,6 +104,7 @@ def value(self, val): class ConfigKey_String(ConfigKey, HasValue): config_key_type = "string" + validator_type = ConfigKeyStringValidator __mapper_args__ = { "polymorphic_identity": config_key_type @@ -100,7 +113,7 @@ class ConfigKey_String(ConfigKey, HasValue): def __init__(self, name, desc=None): super().__init__(name, desc) - @property + @hybrid_property def value(self): return str(self._value) @@ -119,7 +132,7 @@ class ConfigKey_Boolean(ConfigKey, HasValue): def __init__(self, name, desc=None): super().__init__(name, desc) - @property + @hybrid_property def value(self): return self._value == "True" @@ -141,7 +154,7 @@ class ConfigKey_Base64(ConfigKey, HasValue): def __init__(self, name, desc=None): super().__init__(name, desc) - @property + @hybrid_property def value(self): return self._value diff --git a/hydra_base/db/model/hydraconfig/validators.py b/hydra_base/db/model/hydraconfig/validators.py new file mode 100644 index 00000000..01e4d3a3 --- /dev/null +++ b/hydra_base/db/model/hydraconfig/validators.py @@ -0,0 +1,102 @@ +import json + +from abc import ABC, abstractmethod +from dataclasses import dataclass + +from hydra_base.exceptions import HydraError + + +class KeyValidator(ABC): + @dataclass + class Rule(): + name: str + description: str + value: type + + def __init__(self, rules, rules_spec): + self._rules = {} + for rule in rules: + if rule["name"] in self._rules: + raise HydraError(f"Rule {rule['name']} already exists") + + new_rule = KeyValidator.Rule(rule["name"], rule["description"], rule["value"]) + self._rules[new_rule.name] = new_rule + + if rules_spec is not None: + if isinstance(rules_spec, str): + rvi = json.loads(rules_spec) + for rule_name, rule_value in rvi.items(): + self.set_rule(rule_name, rule_value) + else: + raise TypeError(f"Invalid Rules specification: {rules_spec}") + + def set_rule(self, name, value): + rule = self._get_rule(name) + rule.value = value + + def clear_rule(self, name): + rule = self._get_rule(name) + rule.value = None + + def _get_rule(self, name): + if not (rule := self._rules.get(name)): + raise ValueError(f"No rule named '{name}' defined") + + return rule + + @property + def active_rules(self): + return {rule.name: rule for rule in self._rules.values() if rule.value is not None} + + @property + def rules(self): + return self._rules + + @abstractmethod + def validate(self): + pass + + +class ConfigKeyIntegerValidator(KeyValidator): + rules = [ + {"name": "max_value", + "description": "The maximum integer value of this key", + "value": None}, + {"name": "min_value", + "description": "The minimum integer value of this key", + "value": None} + ] + + def __init__(self, rules_spec=None): + super().__init__(self.__class__.rules, rules_spec) + + + def validate(self, value): + if max_value := self.active_rules.get("max_value"): + if value > max_value.value: + raise ValueError("over max") + if min_value := self.active_rules.get("min_value"): + if value < min_value.value: + raise ValueError("under min") + + +class ConfigKeyStringValidator(KeyValidator): + rules = [ + {"name": "max_length", + "description": "The maximum length of this key's string value", + "value": None}, + {"name": "min_length", + "description": "The minimum length of this key's string value", + "value": None}, + ] + + def __init__(self, rules_spec=None): + super().__init__(self.__class__.rules, rules_spec) + + def validate(self, value): + if max_length := self.active_rules.get("max_length"): + if len(value) > max_length.value: + raise ValueError("over max") + if min_length := self.active_rules.get("min_length"): + if len(value) < min_length.value: + raise ValueError("under min") diff --git a/tests/test_hydraconfig.py b/tests/test_hydraconfig.py index ee4cb763..a1d1a2b5 100644 --- a/tests/test_hydraconfig.py +++ b/tests/test_hydraconfig.py @@ -7,6 +7,74 @@ config_key_type_map, ConfigKey ) +from hydra_base.db.model.hydraconfig.validators import ( + ConfigKeyIntegerValidator, + ConfigKeyStringValidator +) + +class TestConfigKeyValidators(): + """ Standalone tests of Config Key validator classes """ + def test_create_validator(self): + """ + Do Validators register their subclass-specific rules? + """ + iv = ConfigKeyIntegerValidator() + assert len(iv.rules) == 2 + assert len(iv.active_rules) == 0 + + sv = ConfigKeyStringValidator() + assert len(sv.rules) == 2 + assert len(sv.active_rules) == 0 + + def test_validator_with_rules_spec(self): + # Integer Validator + too_low = 1 + too_high = 12 + valid_value = 7 + rules_spec = '{"min_value": 2, "max_value": 9}' + iv = ConfigKeyIntegerValidator(rules_spec) + assert len(iv.active_rules) == 2 + assert iv.validate(valid_value) is None + with pytest.raises(ValueError): + iv.validate(too_low) + with pytest.raises(ValueError): + iv.validate(too_high) + + # String Validator + too_short = "one" + too_long = "eleven" + valid_value = "five" + rules_spec = '{"min_length": 4, "max_length": 5}' + sv = ConfigKeyStringValidator(rules_spec) + assert len(sv.active_rules) == 2 + assert sv.validate(valid_value) is None + with pytest.raises(ValueError): + sv.validate(too_short) + with pytest.raises(ValueError): + sv.validate(too_long) + + def test_validate_integer_key(self): + key_value = 12 + max_value = 7 + iv = ConfigKeyIntegerValidator() + # Validation succeeds as no rules are active + assert iv.validate(key_value) is None + iv.set_rule("max_value", max_value) + # Validation of the same value now fails due to active rule + with pytest.raises(ValueError): + iv.validate(key_value) + + def test_validate_string_key(self): + key_value = "A string value" # len 14 + max_length = 7 + sv = ConfigKeyStringValidator() + # Validation succeeds as no rules are active + assert sv.validate(key_value) is None + sv.set_rule("max_length", max_length) + # Validation of the same value now fails due to active rule + with pytest.raises(ValueError): + sv.validate(key_value) + class TestHydraConfig(): def test_valid_config_key_type_map(self): @@ -52,6 +120,12 @@ def test_set_config_key_value(self, client): ret_value = client.get_config_key_value(key_name) assert isinstance(ret_value, int) assert ret_value == key_value + # Can existing key value be changed? + new_value = 77 + client.set_config_key_value(key_name, new_value) + new_ret_value = client.get_config_key_value(key_name) + assert isinstance(new_ret_value, int) + assert new_ret_value == new_value # Invalid int values should be rejected with pytest.raises(HydraError): From 231a9bd177c28ec6567d20ee4ed6ea2c67e82cf0 Mon Sep 17 00:00:00 2001 From: Paul Slavin Date: Thu, 21 Nov 2024 14:47:23 +0000 Subject: [PATCH 06/28] Validator ser/deser --- .../db/model/hydraconfig/hydraconfig.py | 11 +++++- hydra_base/db/model/hydraconfig/validators.py | 36 +++++++++++++++---- tests/test_hydraconfig.py | 11 ++++++ 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/hydra_base/db/model/hydraconfig/hydraconfig.py b/hydra_base/db/model/hydraconfig/hydraconfig.py index 76434782..45632d53 100644 --- a/hydra_base/db/model/hydraconfig/hydraconfig.py +++ b/hydra_base/db/model/hydraconfig/hydraconfig.py @@ -14,7 +14,8 @@ from sqlalchemy.orm import ( Mapped, - mapped_column + mapped_column, + reconstructor ) @@ -34,6 +35,7 @@ class ConfigKey(Base): name = Column(String(key_name_max_length), nullable=False, unique=True) description = Column(String(key_desc_max_length)) type = Column(String(key_type_tag_max_length)) + rules = Column(String(200)) __mapper_args__ = { @@ -62,6 +64,13 @@ def __init__(self, name, desc=None): if vcls := getattr(self.__class__, "validator_type", None): self.validator = vcls() + self.validator.key = self + + @reconstructor + def load_state(self): + if vcls := getattr(self.__class__, "validator_type", None): + self.validator = vcls(self.rules) + self.validator.key = self class HasValue: diff --git a/hydra_base/db/model/hydraconfig/validators.py b/hydra_base/db/model/hydraconfig/validators.py index 01e4d3a3..7f3008e9 100644 --- a/hydra_base/db/model/hydraconfig/validators.py +++ b/hydra_base/db/model/hydraconfig/validators.py @@ -2,6 +2,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass +from json import JSONDecodeError from hydra_base.exceptions import HydraError @@ -13,6 +14,10 @@ class Rule(): description: str value: type + class ValidatorEncoder(json.JSONEncoder): + def default(self, inst): + return {r.name: r.value for r in inst.rules} + def __init__(self, rules, rules_spec): self._rules = {} for rule in rules: @@ -24,7 +29,10 @@ def __init__(self, rules, rules_spec): if rules_spec is not None: if isinstance(rules_spec, str): - rvi = json.loads(rules_spec) + try: + rvi = json.loads(rules_spec) + except JSONDecodeError as e: + raise HydraError(f"Invalid rules_spec: {rules_spec}") from e for rule_name, rule_value in rvi.items(): self.set_rule(rule_name, rule_value) else: @@ -33,6 +41,8 @@ def __init__(self, rules, rules_spec): def set_rule(self, name, value): rule = self._get_rule(name) rule.value = value + if parent := getattr(self, "key", None): + parent.rules = self.as_json def clear_rule(self, name): rule = self._get_rule(name) @@ -50,7 +60,21 @@ def active_rules(self): @property def rules(self): - return self._rules + return self._rules.values() + + @property + def as_json(self): + return json.dumps(self, cls=self.ValidatorEncoder) + + @property + def key(self): + if hasattr(self, "_key"): + return self._key + + @key.setter + def key(self, key): + self._key = key + self._key.rules = self.as_json @abstractmethod def validate(self): @@ -58,7 +82,7 @@ def validate(self): class ConfigKeyIntegerValidator(KeyValidator): - rules = [ + rule_types = [ {"name": "max_value", "description": "The maximum integer value of this key", "value": None}, @@ -68,7 +92,7 @@ class ConfigKeyIntegerValidator(KeyValidator): ] def __init__(self, rules_spec=None): - super().__init__(self.__class__.rules, rules_spec) + super().__init__(self.__class__.rule_types, rules_spec) def validate(self, value): @@ -81,7 +105,7 @@ def validate(self, value): class ConfigKeyStringValidator(KeyValidator): - rules = [ + rule_types = [ {"name": "max_length", "description": "The maximum length of this key's string value", "value": None}, @@ -91,7 +115,7 @@ class ConfigKeyStringValidator(KeyValidator): ] def __init__(self, rules_spec=None): - super().__init__(self.__class__.rules, rules_spec) + super().__init__(self.__class__.rule_types, rules_spec) def validate(self, value): if max_length := self.active_rules.get("max_length"): diff --git a/tests/test_hydraconfig.py b/tests/test_hydraconfig.py index a1d1a2b5..5365bbc7 100644 --- a/tests/test_hydraconfig.py +++ b/tests/test_hydraconfig.py @@ -75,6 +75,17 @@ def test_validate_string_key(self): with pytest.raises(ValueError): sv.validate(key_value) + def test_serialise_validator(self): + min_value = 5 + max_value = 12 + initial_state = '{"max_value": null, "min_value": null}' + updated_rules = f'{{"max_value": {max_value}, "min_value": {min_value}}}' + iv = ConfigKeyIntegerValidator() + assert iv.as_json == initial_state + iv.set_rule("max_value", max_value) + iv.set_rule("min_value", min_value) + assert iv.as_json == updated_rules + class TestHydraConfig(): def test_valid_config_key_type_map(self): From 852c33ecb103f39085fea3e99a7e28063c514c72 Mon Sep 17 00:00:00 2001 From: Paul Slavin Date: Thu, 21 Nov 2024 17:10:32 +0000 Subject: [PATCH 07/28] Key validation; lib funcs and tests --- .../db/model/hydraconfig/hydraconfig.py | 6 +++ hydra_base/db/model/hydraconfig/validators.py | 11 ++++- hydra_base/lib/hydraconfig.py | 42 ++++++++++++++++++- 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/hydra_base/db/model/hydraconfig/hydraconfig.py b/hydra_base/db/model/hydraconfig/hydraconfig.py index 45632d53..20c3bad3 100644 --- a/hydra_base/db/model/hydraconfig/hydraconfig.py +++ b/hydra_base/db/model/hydraconfig/hydraconfig.py @@ -108,6 +108,9 @@ def value(self, val): except (TypeError, ValueError): raise HydraError(f"Config Key {self.name} requires an integer value, not {val}") + if validator := getattr(self, "validator", None): + validator.validate(val) + self._value = val @@ -128,6 +131,9 @@ def value(self): @value.setter def value(self, val): + if validator := getattr(self, "validator", None): + validator.validate(str(val)) + self._value = str(val) diff --git a/hydra_base/db/model/hydraconfig/validators.py b/hydra_base/db/model/hydraconfig/validators.py index 7f3008e9..396a0057 100644 --- a/hydra_base/db/model/hydraconfig/validators.py +++ b/hydra_base/db/model/hydraconfig/validators.py @@ -47,6 +47,8 @@ def set_rule(self, name, value): def clear_rule(self, name): rule = self._get_rule(name) rule.value = None + if parent := getattr(self, "key", None): + parent.rules = self.as_json def _get_rule(self, name): if not (rule := self._rules.get(name)): @@ -96,12 +98,17 @@ def __init__(self, rules_spec=None): def validate(self, value): + if parent := getattr(self, "key", None): + err_prefix = f"ConfigKey {parent.name}: " + else: + err_prefix = "" + if max_value := self.active_rules.get("max_value"): if value > max_value.value: - raise ValueError("over max") + raise ValueError(f"{err_prefix}value of {value} exceeds maximum of {max_value.value}") if min_value := self.active_rules.get("min_value"): if value < min_value.value: - raise ValueError("under min") + raise ValueError(f"{err_prefix}value of {value} beneath minimum of {min_value.value}") class ConfigKeyStringValidator(KeyValidator): diff --git a/hydra_base/lib/hydraconfig.py b/hydra_base/lib/hydraconfig.py index 9a117e74..cc5f17ac 100644 --- a/hydra_base/lib/hydraconfig.py +++ b/hydra_base/lib/hydraconfig.py @@ -38,15 +38,53 @@ def list_config_keys(like=None, **kwargs): keys = query.all() return [key.name for key in keys] -def set_config_key_value(key_name, value, **kwargs): +def config_key_set_value(key_name, value, **kwargs): key = db.DBSession.query(ConfigKey).filter(ConfigKey.name == key_name).one() key.value = value db.DBSession.flush() -def get_config_key_value(key_name, **kwargs): +def config_key_get_value(key_name, **kwargs): key = db.DBSession.query(ConfigKey).filter(ConfigKey.name == key_name).one() return key.value +""" Validation related functions """ + +def config_key_get_rule_types(key_name, **kwargs): + key = db.DBSession.query(ConfigKey).filter(ConfigKey.name == key_name).one() + if validator := getattr(key, "validator", None): + return [*validator.rules] + +def config_key_get_rule_description(key_name, rule_name, **kwargs): + pass + +def config_key_get_active_rules(key_name, **kwargs): + key = db.DBSession.query(ConfigKey).filter(ConfigKey.name == key_name).one() + if validator := getattr(key, "validator", None): + return {rule.name: rule.value for rule in validator.active_rules.values()} + else: + return {} + +def config_key_set_rule(key_name, rule_name, value, **kwargs): + key = db.DBSession.query(ConfigKey).filter(ConfigKey.name == key_name).one() + if validator := getattr(key, "validator", None): + validator.set_rule(rule_name, value) + +def config_key_clear_rule(key_name, rule_name, **kwargs): + key = db.DBSession.query(ConfigKey).filter(ConfigKey.name == key_name).one() + if validator := getattr(key, "validator", None): + validator.clear_rule(rule_name) + +def config_key_clear_all_rules(key_name, **kwargs): + rules = config_key_get_active_rules(key_name, **kwargs) + if rules is None or len(rules) == 0: + return 0 + + for rule_name in rules: + config_key_clear_rule(key_name, rule_name) + + return len(rules) + + """ Config Sets: Archived versions of complete configurations """ From 74d57ff9b00901b8286ffa82a83086d2e67c9b66 Mon Sep 17 00:00:00 2001 From: Paul Slavin Date: Fri, 22 Nov 2024 15:50:39 +0000 Subject: [PATCH 08/28] ConfigGroups; lib funcs and tests --- .../db/model/hydraconfig/hydraconfig.py | 37 ++++- hydra_base/lib/hydraconfig.py | 57 +++++-- tests/test_hydraconfig.py | 156 ++++++++++++++++-- 3 files changed, 222 insertions(+), 28 deletions(-) diff --git a/hydra_base/db/model/hydraconfig/hydraconfig.py b/hydra_base/db/model/hydraconfig/hydraconfig.py index 20c3bad3..081a69ee 100644 --- a/hydra_base/db/model/hydraconfig/hydraconfig.py +++ b/hydra_base/db/model/hydraconfig/hydraconfig.py @@ -4,6 +4,7 @@ import base64 import binascii +from hydra_base import db from hydra_base.db.model.base import * from hydra_base.exceptions import HydraError @@ -19,7 +20,7 @@ ) -__all__ = ["ConfigKey", "config_key_type_map"] +__all__ = ["ConfigKey", "config_key_type_map", "ConfigGroup", "ConfigGroupKeys"] config_key_type_map = {} @@ -33,7 +34,7 @@ class ConfigKey(Base): id = Column(Integer(), primary_key=True, nullable=False) name = Column(String(key_name_max_length), nullable=False, unique=True) - description = Column(String(key_desc_max_length)) + description = Column(String(key_desc_max_length), nullable=True, unique=False) type = Column(String(key_type_tag_max_length)) rules = Column(String(200)) @@ -72,6 +73,11 @@ def load_state(self): self.validator = vcls(self.rules) self.validator.key = self + @hybrid_property + def group(self): + group = db.DBSession.query(ConfigGroup).filter(ConfigGroup.id == self._group.group_id).one() + return group.name + class HasValue: value_max_length = 2000 @@ -200,3 +206,30 @@ class ConfigKey_Json(ConfigKey, HasValue): "polymorphic_identity": config_key_type } # Unvalidated json object + + +class ConfigGroup(Base): + __tablename__ = "tConfigGroup" + + group_name_max_length = 200 + group_desc_max_length = 2000 + + id = Column(Integer(), primary_key=True, nullable=False) + name = Column(String(group_name_max_length), nullable=False, unique=True) + description = Column(String(group_desc_max_length), nullable=True, unique=False) + + @hybrid_property + def keys(self): + member_key_ids = {k.key_id for k in self._keys} + member_keys = db.DBSession.query(ConfigKey).filter(ConfigKey.id.in_(member_key_ids)).all() + return [key.name for key in member_keys] + + +class ConfigGroupKeys(Base): + __tablename__ = "tConfigGroupKeys" + + group_id = Column(Integer(), ForeignKey("tConfigGroup.id"), primary_key=True, nullable=False) + key_id = Column(Integer(), ForeignKey("tConfigKey.id"), primary_key=True, nullable=False) + + keys = relationship("ConfigGroup", backref=backref("_keys", uselist=True, cascade="all, delete-orphan")) + group = relationship("ConfigKey", backref=backref("_group", uselist=False, cascade="all, delete-orphan")) diff --git a/hydra_base/lib/hydraconfig.py b/hydra_base/lib/hydraconfig.py index cc5f17ac..c4d3c535 100644 --- a/hydra_base/lib/hydraconfig.py +++ b/hydra_base/lib/hydraconfig.py @@ -11,9 +11,13 @@ from hydra_base.db.model.hydraconfig import ( ConfigKey, - config_key_type_map + config_key_type_map, + ConfigGroup, + ConfigGroupKeys ) +from sqlalchemy.exc import IntegrityError + """ Config Keys: Key:Value pairs of config settings """ @@ -22,8 +26,11 @@ def register_config_key(key_name, key_type, **kwargs): raise HydraError(f"Invalid ConfigKey type '{key_type}'") key = key_cls(name=key_name) - db.DBSession.add(key) - db.DBSession.flush() + try: + db.DBSession.add(key) + db.DBSession.flush() + except IntegrityError: + raise HydraError(f"ConfigKey with name '{key_name}' exists") return key @@ -106,17 +113,41 @@ def list_configset_versions(set_name): """ Config Groups: A named collection of Config Keys """ -def create_config_group(group_name): - pass +def create_config_group(group_name, group_desc=None, **kwargs): + group = ConfigGroup(name=group_name, description=group_desc) + try: + db.DBSession.add(group) + db.DBSession.flush() + except IntegrityError: + raise HydraError(f"ConfigGroup with name '{group_name}' exists") + +def delete_config_group(group_name, **kwargs): + group = db.DBSession.query(ConfigGroup).filter(ConfigGroup.name == group_name).one() + db.DBSession.delete(group) + db.DBSession.flush() -def delete_config_group(group_name): - pass +def list_config_groups(**kwargs): + groups = db.DBSession.query(ConfigGroup).all() + return groups -def list_config_groups(): - pass +def add_config_key_to_group(key_name, group_name, **kwargs): + key = db.DBSession.query(ConfigKey).filter(ConfigKey.name == key_name).one() + group = db.DBSession.query(ConfigGroup).filter(ConfigGroup.name == group_name).one() + gk = ConfigGroupKeys(group_id=group.id, key_id=key.id) + db.DBSession.add(gk) + db.DBSession.flush() -def add_config_key_to_group(key_name, group_name): - pass +def config_group_list_keys(group_name, **kwargs): + group = db.DBSession.query(ConfigGroup).filter(ConfigGroup.name == group_name).one() + return group.keys -def remove_config_key_from_group(key_name, group_name): - pass +def remove_config_key_from_group(key_name, group_name, **kwargs): + group = db.DBSession.query(ConfigGroup).filter(ConfigGroup.name == group_name).one() + key = db.DBSession.query(ConfigKey).filter(ConfigKey.name == key_name).one() + qfilter = { + ConfigGroupKeys.group_id == group.id, + ConfigGroupKeys.key_id == key.id + } + gk = db.DBSession.query(ConfigGroupKeys).filter(*qfilter).one() + db.DBSession.delete(gk) + db.DBSession.flush() diff --git a/tests/test_hydraconfig.py b/tests/test_hydraconfig.py index 5365bbc7..6c463c86 100644 --- a/tests/test_hydraconfig.py +++ b/tests/test_hydraconfig.py @@ -12,6 +12,16 @@ ConfigKeyStringValidator ) + +@pytest.fixture +def config_group(): + group_name = "Test ConfigKey Group" + group_desc = "Description of test group" + group = client.create_config_group(group_name, group_desc) + yield group + client.delete_config_group() + + class TestConfigKeyValidators(): """ Standalone tests of Config Key validator classes """ def test_create_validator(self): @@ -127,27 +137,27 @@ def test_set_config_key_value(self, client): key_name = "integer_value_test_key" key_value = 46 _ = client.register_config_key(key_name, "integer") - client.set_config_key_value(key_name, key_value) - ret_value = client.get_config_key_value(key_name) + client.config_key_set_value(key_name, key_value) + ret_value = client.config_key_get_value(key_name) assert isinstance(ret_value, int) assert ret_value == key_value # Can existing key value be changed? new_value = 77 - client.set_config_key_value(key_name, new_value) - new_ret_value = client.get_config_key_value(key_name) + client.config_key_set_value(key_name, new_value) + new_ret_value = client.config_key_get_value(key_name) assert isinstance(new_ret_value, int) assert new_ret_value == new_value # Invalid int values should be rejected with pytest.raises(HydraError): - client.set_config_key_value(key_name, math.nan) + client.config_key_set_value(key_name, math.nan) # ConfigKey_String key_name = "string_value_test_key" key_value = "A string value" _ = client.register_config_key(key_name, "string") - client.set_config_key_value(key_name, key_value) - ret_value = client.get_config_key_value(key_name) + client.config_key_set_value(key_name, key_value) + ret_value = client.config_key_get_value(key_name) assert isinstance(ret_value, str) assert ret_value == key_value @@ -156,14 +166,14 @@ def test_set_config_key_value(self, client): key_name = "boolean_value_test_key" key_value = True _ = client.register_config_key(key_name, "boolean") - client.set_config_key_value(key_name, key_value) - ret_value = client.get_config_key_value(key_name) + client.config_key_set_value(key_name, key_value) + ret_value = client.config_key_get_value(key_name) assert isinstance(ret_value, bool) assert ret_value == key_value # Invalid bool values should be rejected with pytest.raises(HydraError): - client.set_config_key_value(key_name, 'Y') + client.config_key_set_value(key_name, 'Y') # ConfigKey_Base64 key_name = "base64_value_test_key" @@ -171,12 +181,132 @@ def test_set_config_key_value(self, client): b64_bytes = base64.b64encode(raw_bytes) key_value = b64_bytes.decode("utf8") _ = client.register_config_key(key_name, "base64") - client.set_config_key_value(key_name, key_value) - ret_value = client.get_config_key_value(key_name) + client.config_key_set_value(key_name, key_value) + ret_value = client.config_key_get_value(key_name) assert isinstance(ret_value, str) ret_bytes = base64.b64decode(ret_value, validate=True) assert ret_bytes == raw_bytes # Non-b64 strings should be rejected with pytest.raises(HydraError): - client.set_config_key_value(key_name, "Not Base64") + client.config_key_set_value(key_name, "Not Base64") + + def test_integer_key_validation(self, client): + key_name = "integer_validation_test_key" + key_value = 46 + max_value = 47 + min_value = 12 + _ = client.register_config_key(key_name, "integer") + assert len(client.config_key_get_rule_types(key_name)) == 2 + assert len(client.config_key_get_active_rules(key_name)) == 0 + # Permissable as no rules are yet active + client.config_key_set_value(key_name, key_value) + # Set a max_value rule + client.config_key_set_rule(key_name, "max_value", max_value) + assert len(client.config_key_get_active_rules(key_name)) == 1 + # Now raises... + with pytest.raises(ValueError): + client.config_key_set_value(key_name, max_value+1) + # ...and value has remained unchanged + assert client.config_key_get_value(key_name) == key_value + # Set a min_value rule + client.config_key_set_rule(key_name, "min_value", min_value) + # Raises again... + with pytest.raises(ValueError): + client.config_key_set_value(key_name, min_value-1) + # ...and value has remained unchanged + assert client.config_key_get_value(key_name) == key_value + # Clear all rules... + num_cleared = client.config_key_clear_all_rules(key_name) + # ...two rules were cleared... + assert num_cleared == 2 + # ...and previously rejected values now accepted + client.config_key_set_value(key_name, max_value+1) + client.config_key_set_value(key_name, min_value-1) + + def test_string_key_validation(self, client): + key_name = "string_validation_test_key" + key_value = "string key value" # len 16 + min_length = 12 + max_length = 18 + _ = client.register_config_key(key_name, "string") + assert len(client.config_key_get_rule_types(key_name)) == 2 + assert len(client.config_key_get_active_rules(key_name)) == 0 + # Permissable as no rules are yet active + client.config_key_set_value(key_name, key_value) + # Set a max_length rule + client.config_key_set_rule(key_name, "max_length", max_length) + # Rule is now active... + assert len(client.config_key_get_active_rules(key_name)) == 1 + # ...so value exceeding max_length fails... + with pytest.raises(ValueError): + client.config_key_set_value(key_name, key_value+"extra text") + # ...and original value is unchanged + assert client.config_key_get_value(key_name) == key_value + # Set a min_length rule + client.config_key_set_rule(key_name, "min_length", min_length) + # Additional rule is now active... + assert len(client.config_key_get_active_rules(key_name)) == 2 + # ...so value shorter than min_length is rejected + with pytest.raises(ValueError): + client.config_key_set_value(key_name, key_value[:8]) + # ...and original value remains unchanged + assert client.config_key_get_value(key_name) == key_value + # Clear all rules + num_cleared = client.config_key_clear_all_rules(key_name) + assert num_cleared == 2 + # Previously rejected values may now be set + client.config_key_set_value(key_name, key_value+"extra text") + client.config_key_set_value(key_name, key_value[:8]) + + +class TestConfigKeyGroups(): + def test_create_config_group(self, client): + group_name = "Test ConfigKey Group" + group_desc = "Description of test group" + # Create a group and verify presence + client.create_config_group(group_name, group_desc) + groups = client.list_config_groups() + group_names = {g.name for g in groups} + assert group_name in group_names + # Verify unable to add group with same name + with pytest.raises(HydraError): + client.create_config_group(group_name, group_desc) + # Verify group can be deleted + client.delete_config_group(group_name) + groups = client.list_config_groups() + group_names = {g.name for g in groups} + assert group_name not in group_names + + def test_add_config_key_to_group(self, client): + group_name = "Membership Test ConfigKey Group" + group_desc = "Description of test group" + key_name = "group_membership_test_key" + key_value = 46 + _ = client.register_config_key(key_name, "integer") + client.config_key_set_value(key_name, key_value) + client.create_config_group(group_name, group_desc) + # Newly created group must be empty + new_group_keys = client.config_group_list_keys(group_name) + assert len(new_group_keys) == 0 + client.add_config_key_to_group(key_name, group_name) + group_keys = client.config_group_list_keys(group_name) + assert key_name in group_keys + # Now remove the key + client.remove_config_key_from_group(key_name, group_name) + group_keys = client.config_group_list_keys(group_name) + assert key_name not in group_keys + + def test_delete_populated_group(self, client): + group_name = "Populated ConfigKey Group" + group_desc = "Description of test group" + key_name = "populated_group_test_key" + key_value = 46 + _ = client.register_config_key(key_name, "integer") + client.config_key_set_value(key_name, key_value) + client.create_config_group(group_name, group_desc) + client.add_config_key_to_group(key_name, group_name) + client.delete_config_group(group_name) + # The member key and its value are unaffected by group deletion + assert key_name in client.list_config_keys() + assert key_value == client.config_key_get_value(key_name) From a273d55b2e44421767b517f2709ad5813a7c3c24 Mon Sep 17 00:00:00 2001 From: Paul Slavin Date: Fri, 22 Nov 2024 16:51:29 +0000 Subject: [PATCH 09/28] Begin ConfigSet --- hydra_base/util/configset.py | 17 +++++++++++++++++ tests/test_hydraconfig.py | 11 +++++++++++ 2 files changed, 28 insertions(+) create mode 100644 hydra_base/util/configset.py diff --git a/hydra_base/util/configset.py b/hydra_base/util/configset.py new file mode 100644 index 00000000..14afac26 --- /dev/null +++ b/hydra_base/util/configset.py @@ -0,0 +1,17 @@ +import hmac +import json + +from hydra_base import db +from hydra_base.db.model.hydraconfig import ConfigKey + + +config_set_secret_key = "dev_only_secret_key" + +class ConfigSet: + def __init__(self): + pass + + def serialise_all_keys(self): + keys = db.DBSession.query(ConfigKey).all() + configset = {key.name: {"value": key.value, "rules": key.rules} for key in keys} + return json.dumps(configset) diff --git a/tests/test_hydraconfig.py b/tests/test_hydraconfig.py index 6c463c86..9ca771a4 100644 --- a/tests/test_hydraconfig.py +++ b/tests/test_hydraconfig.py @@ -11,6 +11,7 @@ ConfigKeyIntegerValidator, ConfigKeyStringValidator ) +from hydra_base.util.configset import ConfigSet @pytest.fixture @@ -310,3 +311,13 @@ def test_delete_populated_group(self, client): # The member key and its value are unaffected by group deletion assert key_name in client.list_config_keys() assert key_value == client.config_key_get_value(key_name) + + +class TestConfigSets: + def test_create_config_set(self, client): + key_name = "configset_test_key" + key_value = 46 + _ = client.register_config_key(key_name, "integer") + client.config_key_set_value(key_name, key_value) + cs = ConfigSet() + css = cs.serialise_all_keys() From b89f84f4f631e342315396a2fc352560219a4ea5 Mon Sep 17 00:00:00 2001 From: Paul Slavin Date: Mon, 25 Nov 2024 17:53:00 +0000 Subject: [PATCH 10/28] ConfigSet save/load/hmac --- .../db/model/hydraconfig/hydraconfig.py | 5 +- hydra_base/lib/hydraconfig.py | 55 ++++++++++---- hydra_base/util/configset.py | 44 +++++++++-- tests/test_hydraconfig.py | 74 +++++++++++++++++-- 4 files changed, 148 insertions(+), 30 deletions(-) diff --git a/hydra_base/db/model/hydraconfig/hydraconfig.py b/hydra_base/db/model/hydraconfig/hydraconfig.py index 081a69ee..f06ee47a 100644 --- a/hydra_base/db/model/hydraconfig/hydraconfig.py +++ b/hydra_base/db/model/hydraconfig/hydraconfig.py @@ -75,8 +75,9 @@ def load_state(self): @hybrid_property def group(self): - group = db.DBSession.query(ConfigGroup).filter(ConfigGroup.id == self._group.group_id).one() - return group.name + if self._group: + group = db.DBSession.query(ConfigGroup).filter(ConfigGroup.id == self._group.group_id).one() + return group.name class HasValue: diff --git a/hydra_base/lib/hydraconfig.py b/hydra_base/lib/hydraconfig.py index c4d3c535..477bec1d 100644 --- a/hydra_base/lib/hydraconfig.py +++ b/hydra_base/lib/hydraconfig.py @@ -16,7 +16,10 @@ ConfigGroupKeys ) -from sqlalchemy.exc import IntegrityError +from sqlalchemy.exc import ( + IntegrityError, + NoResultFound +) """ Config Keys: Key:Value pairs of config settings """ @@ -34,8 +37,10 @@ def register_config_key(key_name, key_type, **kwargs): return key -def unregister_config_key(key): - pass +def unregister_config_key(key_name, **kwargs): + key = _get_config_key_by_name(key_name) + db.DBSession.delete(key) + db.DBSession.flush() def list_config_keys(like=None, **kwargs): query = db.DBSession.query(ConfigKey) @@ -46,18 +51,26 @@ def list_config_keys(like=None, **kwargs): return [key.name for key in keys] def config_key_set_value(key_name, value, **kwargs): - key = db.DBSession.query(ConfigKey).filter(ConfigKey.name == key_name).one() + key = _get_config_key_by_name(key_name) key.value = value db.DBSession.flush() def config_key_get_value(key_name, **kwargs): - key = db.DBSession.query(ConfigKey).filter(ConfigKey.name == key_name).one() + key = _get_config_key_by_name(key_name) return key.value +def _get_config_key_by_name(key_name): + try: + key = db.DBSession.query(ConfigKey).filter(ConfigKey.name == key_name).one() + except NoResultFound: + raise HydraError(f"No ConfigKey found with name: {key_name}") + + return key + """ Validation related functions """ def config_key_get_rule_types(key_name, **kwargs): - key = db.DBSession.query(ConfigKey).filter(ConfigKey.name == key_name).one() + key = _get_config_key_by_name(key_name) if validator := getattr(key, "validator", None): return [*validator.rules] @@ -65,19 +78,19 @@ def config_key_get_rule_description(key_name, rule_name, **kwargs): pass def config_key_get_active_rules(key_name, **kwargs): - key = db.DBSession.query(ConfigKey).filter(ConfigKey.name == key_name).one() + key = _get_config_key_by_name(key_name) if validator := getattr(key, "validator", None): return {rule.name: rule.value for rule in validator.active_rules.values()} else: return {} def config_key_set_rule(key_name, rule_name, value, **kwargs): - key = db.DBSession.query(ConfigKey).filter(ConfigKey.name == key_name).one() + key = _get_config_key_by_name(key_name) if validator := getattr(key, "validator", None): validator.set_rule(rule_name, value) def config_key_clear_rule(key_name, rule_name, **kwargs): - key = db.DBSession.query(ConfigKey).filter(ConfigKey.name == key_name).one() + key = _get_config_key_by_name(key_name) if validator := getattr(key, "validator", None): validator.clear_rule(rule_name) @@ -122,7 +135,7 @@ def create_config_group(group_name, group_desc=None, **kwargs): raise HydraError(f"ConfigGroup with name '{group_name}' exists") def delete_config_group(group_name, **kwargs): - group = db.DBSession.query(ConfigGroup).filter(ConfigGroup.name == group_name).one() + group = _get_config_group_by_name(group_name) db.DBSession.delete(group) db.DBSession.flush() @@ -130,20 +143,32 @@ def list_config_groups(**kwargs): groups = db.DBSession.query(ConfigGroup).all() return groups +def _get_config_group_by_name(group_name): + try: + group = db.DBSession.query(ConfigGroup).filter(ConfigGroup.name == group_name).one() + except NoResultFound: + raise HydraError(f"No ConfigGroup with name: {group_name}") + + return group + def add_config_key_to_group(key_name, group_name, **kwargs): - key = db.DBSession.query(ConfigKey).filter(ConfigKey.name == key_name).one() - group = db.DBSession.query(ConfigGroup).filter(ConfigGroup.name == group_name).one() + key = _get_config_key_by_name(key_name) + group = _get_config_group_by_name(group_name) gk = ConfigGroupKeys(group_id=group.id, key_id=key.id) db.DBSession.add(gk) db.DBSession.flush() def config_group_list_keys(group_name, **kwargs): - group = db.DBSession.query(ConfigGroup).filter(ConfigGroup.name == group_name).one() + group = _get_config_group_by_name(group_name) return group.keys +def config_key_get_group_name(key_name, **kwargs): + key = _get_config_key_by_name(key_name) + return key.group + def remove_config_key_from_group(key_name, group_name, **kwargs): - group = db.DBSession.query(ConfigGroup).filter(ConfigGroup.name == group_name).one() - key = db.DBSession.query(ConfigKey).filter(ConfigKey.name == key_name).one() + group = _get_config_group_by_name(group_name) + key = _get_config_key_by_name(key_name) qfilter = { ConfigGroupKeys.group_id == group.id, ConfigGroupKeys.key_id == key.id diff --git a/hydra_base/util/configset.py b/hydra_base/util/configset.py index 14afac26..e26a3c2e 100644 --- a/hydra_base/util/configset.py +++ b/hydra_base/util/configset.py @@ -1,3 +1,4 @@ +import datetime import hmac import json @@ -5,13 +6,44 @@ from hydra_base.db.model.hydraconfig import ConfigKey -config_set_secret_key = "dev_only_secret_key" +config_set_secret_key = b"dev_only_secret_key" class ConfigSet: - def __init__(self): - pass + mac_hash = "sha256" + def __init__(self, name, description=""): + if not name: + raise ValueError(f"ConfigSet requires a valid name, not: {name}") + self.name = name + self.desc = description - def serialise_all_keys(self): + def save_keys_to_configset(self): + configstate= { + "name": self.name, + "description": self.desc, + "timestamp": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "keys": self.all_keys_as_dict() + } + + digest = self.generate_mac(json.dumps(configstate)) + configstate["digest"] = digest + return configstate + + def load_configset(self, state): + if isinstance(state, str): + state = json.loads(state) + + loaded_digest = state.pop("digest") + calculated_digest = self.generate_mac(json.dumps(state)) + if loaded_digest != calculated_digest: + raise ValueError(f"ConfigSet digest is {calculated_digest} but claimed is {loaded_digest}") + + state["digest"] = calculated_digest + return state + + def all_keys_as_dict(self): keys = db.DBSession.query(ConfigKey).all() - configset = {key.name: {"value": key.value, "rules": key.rules} for key in keys} - return json.dumps(configset) + return {key.name: {"type": key.type, "value": key.value, "rules": key.rules} for key in keys} + + def generate_mac(self, state): + digest = hmac.digest(key=config_set_secret_key, msg=state.encode("utf8"), digest=ConfigSet.mac_hash) + return digest.hex() diff --git a/tests/test_hydraconfig.py b/tests/test_hydraconfig.py index 9ca771a4..ed28e373 100644 --- a/tests/test_hydraconfig.py +++ b/tests/test_hydraconfig.py @@ -1,6 +1,9 @@ import base64 +import copy import math import pytest +import random +import string from hydra_base.exceptions import HydraError from hydra_base.lib.hydraconfig import ( @@ -13,6 +16,30 @@ ) from hydra_base.util.configset import ConfigSet +# Util funcs + +def make_integer_key_with_value(client, key_name): + key_value = random.randint(2**7, 2**9) + key = client.register_config_key(key_name, "integer") + val_diff = random.randint(1, key_value//2) + min_value, max_value = key_value-val_diff, key_value+val_diff + client.config_key_set_rule(key_name, "min_value", min_value) + client.config_key_set_rule(key_name, "max_value", max_value) + client.config_key_set_value(key_name, key_value) + return key.name + + +def make_string_key_with_value(client, key_name): + key_len = random.randint(2**4, 2**6) + key_value = "".join(random.choices(string.ascii_lowercase, k=key_len)) + key = client.register_config_key(key_name, "string") + len_diff = random.randint(1, key_len//2) + min_length, max_length = key_len-len_diff, key_len+len_diff + client.config_key_set_rule(key_name, "min_length", min_length) + client.config_key_set_rule(key_name, "max_length", max_length) + client.config_key_set_value(key_name, key_value) + return key.name + @pytest.fixture def config_group(): @@ -22,6 +49,29 @@ def config_group(): yield group client.delete_config_group() +@pytest.fixture +def random_keys(client): + key_gen_funcs = { + "integer": make_integer_key_with_value, + "string": make_string_key_with_value + } + n_keys = 16 + key_prefixes = set() + key_names = [] + for idx in range(n_keys): + while True: + key_prefix = "".join(random.choices(string.ascii_lowercase, k=3)) + if key_prefix not in key_prefixes: + break + key_name = f"{key_prefix} test key" + key_func = key_gen_funcs[random.choice([*key_gen_funcs])] + key_names.append(key_func(client, key_name)) + + yield key_names + for key_name in key_names: + client.unregister_config_key(key_name) + + class TestConfigKeyValidators(): """ Standalone tests of Config Key validator classes """ @@ -290,6 +340,10 @@ def test_add_config_key_to_group(self, client): # Newly created group must be empty new_group_keys = client.config_group_list_keys(group_name) assert len(new_group_keys) == 0 + # Verify that groupless key reports no group + no_group_name = client.config_key_get_group_name(key_name) + assert no_group_name is None + # Add key to group and confirm membership client.add_config_key_to_group(key_name, group_name) group_keys = client.config_group_list_keys(group_name) assert key_name in group_keys @@ -314,10 +368,16 @@ def test_delete_populated_group(self, client): class TestConfigSets: - def test_create_config_set(self, client): - key_name = "configset_test_key" - key_value = 46 - _ = client.register_config_key(key_name, "integer") - client.config_key_set_value(key_name, key_value) - cs = ConfigSet() - css = cs.serialise_all_keys() + def test_create_config_set(self, client, random_keys): + cs = ConfigSet("Test Configset") + state = cs.save_keys_to_configset() + # Tamper with serialised set + modified = copy.deepcopy(state) + first_key = next(iter(modified["keys"])) + key_val = modified["keys"].pop(first_key) + modified["keys"]["modifed_name"] = key_val + # Naughtiness is detected... + with pytest.raises(ValueError): + cs.load_configset(modified) + # ...but original state can be loaded + loaded = cs.load_configset(state) From 8baa1310f80d74669264bd72fad8682ea681e0e6 Mon Sep 17 00:00:00 2001 From: Paul Slavin Date: Tue, 26 Nov 2024 17:23:21 +0000 Subject: [PATCH 11/28] ConfigSet apply to db; Updated tests --- hydra_base/util/configset.py | 71 ++++++++++++++++++++++- tests/test_hydraconfig.py | 107 +++++++++++++++++++++++++++-------- 2 files changed, 152 insertions(+), 26 deletions(-) diff --git a/hydra_base/util/configset.py b/hydra_base/util/configset.py index e26a3c2e..212bb55b 100644 --- a/hydra_base/util/configset.py +++ b/hydra_base/util/configset.py @@ -1,21 +1,38 @@ +""" + Utilities for the management of ConfigKeys +""" import datetime import hmac import json from hydra_base import db from hydra_base.db.model.hydraconfig import ConfigKey +from hydra_base.lib.hydraconfig import ( + register_config_key, + config_key_set_value, + config_key_set_rule +) config_set_secret_key = b"dev_only_secret_key" class ConfigSet: mac_hash = "sha256" + def __init__(self, name, description=""): - if not name: - raise ValueError(f"ConfigSet requires a valid name, not: {name}") self.name = name self.desc = description + @property + def name(self): + return self._name + + @name.setter + def name(self, name): + if not name: + raise ValueError(f"ConfigSet requires a valid name, not: {name}") + self._name = name + def save_keys_to_configset(self): configstate= { "name": self.name, @@ -28,7 +45,15 @@ def save_keys_to_configset(self): configstate["digest"] = digest return configstate - def load_configset(self, state): + def verify_configset(self, state): + """ + Verifies that the state argument contains a valid + hmac digest whose "message" corresponds to the contents + of the other fields in the state. + + Raises ValueError if the equivalent hmac calculated here + differs from that claimed by the input state. + """ if isinstance(state, str): state = json.loads(state) @@ -42,8 +67,48 @@ def load_configset(self, state): def all_keys_as_dict(self): keys = db.DBSession.query(ConfigKey).all() + if len(keys) == 0: + return {} return {key.name: {"type": key.type, "value": key.value, "rules": key.rules} for key in keys} def generate_mac(self, state): digest = hmac.digest(key=config_set_secret_key, msg=state.encode("utf8"), digest=ConfigSet.mac_hash) return digest.hex() + + def apply_configset_to_db(self, state): + """ + 1. Serialise existing state + 2. Verify integrity of new state + 3. Delete existing state + 4. Create new keys + 5. Set new validation rules + 6. Load new values + 7. Verify state loaded in 3-5 matches input state + 8. OK if so, else restore original state from 1 + """ + old_state = self.save_keys_to_configset() + new_state = self.verify_configset(state) + self._delete_all_keys() + self.load_keys_from_state(new_state) + # Verify update has succeeded and return old state if so + trial_state = self.save_keys_to_configset() + if trial_state["keys"] == new_state["keys"]: + return old_state + # Otherwise restore state to that before call + self._delete_all_keys() + self.load_keys_from_state(old_state) + # Returns None on failure to update + + def _delete_all_keys(self): + keys = db.DBSession.query(ConfigKey).all() + for key in keys: + db.DBSession.delete(key) + db.DBSession.flush() + + def load_keys_from_state(self, state): + for key_name, key in state["keys"].items(): + register_config_key(key_name, key["type"]) + rules = json.loads(key["rules"]) + for rule_name, rule_val in rules.items(): + config_key_set_rule(key_name, rule_name, rule_val) + config_key_set_value(key_name, key["value"]) diff --git a/tests/test_hydraconfig.py b/tests/test_hydraconfig.py index ed28e373..d497ca45 100644 --- a/tests/test_hydraconfig.py +++ b/tests/test_hydraconfig.py @@ -49,28 +49,43 @@ def config_group(): yield group client.delete_config_group() + @pytest.fixture def random_keys(client): - key_gen_funcs = { - "integer": make_integer_key_with_value, - "string": make_string_key_with_value - } - n_keys = 16 - key_prefixes = set() + """ + Returns a function which... + - Generates and registers n_keys ConfigKeys of random + types, including appropriate validator rule settings + and a key value which passes validation. + - Deletes these keys on return unless the optional + do_tidy argument is overwritten to be False. + """ key_names = [] - for idx in range(n_keys): - while True: - key_prefix = "".join(random.choices(string.ascii_lowercase, k=3)) - if key_prefix not in key_prefixes: - break - key_name = f"{key_prefix} test key" - key_func = key_gen_funcs[random.choice([*key_gen_funcs])] - key_names.append(key_func(client, key_name)) - - yield key_names - for key_name in key_names: - client.unregister_config_key(key_name) - + _do_tidy = True + def _random_keys(n_keys, do_tidy=True): + key_gen_funcs = { + "integer": make_integer_key_with_value, + "string": make_string_key_with_value + } + nonlocal _do_tidy + _do_tidy = do_tidy + key_prefixes = set() + for idx in range(n_keys): + while True: + key_prefix = "".join(random.choices(string.ascii_lowercase, k=3)) + if key_prefix not in key_prefixes: + key_prefixes.add(key_prefix) + break + key_name = f"{key_prefix} test key" + key_func = key_gen_funcs[random.choice([*key_gen_funcs])] + key_names.append(key_func(client, key_name)) + + return key_names + + yield _random_keys + if _do_tidy: + for key_name in key_names: + client.unregister_config_key(key_name) class TestConfigKeyValidators(): @@ -137,6 +152,10 @@ def test_validate_string_key(self): sv.validate(key_value) def test_serialise_validator(self): + """ + Does a validator have the correct initial state + and can it be serialised to the correct format? + """ min_value = 5 max_value = 12 initial_state = '{"max_value": null, "min_value": null}' @@ -178,6 +197,7 @@ def test_define_config_key(self, client): all_keys = client.list_config_keys() assert itk.name in all_keys + client.unregister_config_key("integer_test_key") def test_set_config_key_value(self, client): """ @@ -202,6 +222,7 @@ def test_set_config_key_value(self, client): # Invalid int values should be rejected with pytest.raises(HydraError): client.config_key_set_value(key_name, math.nan) + client.unregister_config_key(key_name) # ConfigKey_String key_name = "string_value_test_key" @@ -211,6 +232,7 @@ def test_set_config_key_value(self, client): ret_value = client.config_key_get_value(key_name) assert isinstance(ret_value, str) assert ret_value == key_value + client.unregister_config_key(key_name) # ConfigKey_Boolean @@ -225,6 +247,7 @@ def test_set_config_key_value(self, client): # Invalid bool values should be rejected with pytest.raises(HydraError): client.config_key_set_value(key_name, 'Y') + client.unregister_config_key(key_name) # ConfigKey_Base64 key_name = "base64_value_test_key" @@ -241,6 +264,7 @@ def test_set_config_key_value(self, client): # Non-b64 strings should be rejected with pytest.raises(HydraError): client.config_key_set_value(key_name, "Not Base64") + client.unregister_config_key(key_name) def test_integer_key_validation(self, client): key_name = "integer_validation_test_key" @@ -274,6 +298,7 @@ def test_integer_key_validation(self, client): # ...and previously rejected values now accepted client.config_key_set_value(key_name, max_value+1) client.config_key_set_value(key_name, min_value-1) + client.unregister_config_key(key_name) def test_string_key_validation(self, client): key_name = "string_validation_test_key" @@ -309,6 +334,7 @@ def test_string_key_validation(self, client): # Previously rejected values may now be set client.config_key_set_value(key_name, key_value+"extra text") client.config_key_set_value(key_name, key_value[:8]) + client.unregister_config_key(key_name) class TestConfigKeyGroups(): @@ -351,6 +377,7 @@ def test_add_config_key_to_group(self, client): client.remove_config_key_from_group(key_name, group_name) group_keys = client.config_group_list_keys(group_name) assert key_name not in group_keys + client.unregister_config_key(key_name) def test_delete_populated_group(self, client): group_name = "Populated ConfigKey Group" @@ -365,10 +392,16 @@ def test_delete_populated_group(self, client): # The member key and its value are unaffected by group deletion assert key_name in client.list_config_keys() assert key_value == client.config_key_get_value(key_name) + client.unregister_config_key(key_name) class TestConfigSets: - def test_create_config_set(self, client, random_keys): + def test_config_set_save_and_verify(self, random_keys): + """ + Can a ConfigSet be created, serialised and then + detect any modifications to the serialised version? + """ + _ = random_keys(16) cs = ConfigSet("Test Configset") state = cs.save_keys_to_configset() # Tamper with serialised set @@ -378,6 +411,34 @@ def test_create_config_set(self, client, random_keys): modified["keys"]["modifed_name"] = key_val # Naughtiness is detected... with pytest.raises(ValueError): - cs.load_configset(modified) - # ...but original state can be loaded - loaded = cs.load_configset(state) + cs.verify_configset(modified) + # ...but original unmodified state can be loaded + loaded = cs.verify_configset(state) + + def test_apply_configset_to_db(self, client, random_keys): + """ + 1. Serialises initial state + 2. Deletes this and replaces with temporary ConfigKeys + 3. Re-loads the initial state + 4. Confirms this returns the temporary state + 5. Confirms the final state is equal to initial state + """ + num_keys = 16 + key_names = random_keys(num_keys, do_tidy=False) + cs = ConfigSet("Configset") + # Save initial state + initial_state = cs.save_keys_to_configset() + # Manually delete initial keys + for key_name in key_names: + client.unregister_config_key(key_name) + # Generate new state + new_key_names = random_keys(num_keys, do_tidy=False) + # Save second state + second_state = cs.save_keys_to_configset() + # Reload the initial state + ret_state = cs.apply_configset_to_db(initial_state) + # The temporary key state was returned... + assert ret_state["keys"] == second_state["keys"] + # ...which means the initial key state was restored + final_state = cs.save_keys_to_configset() + assert final_state["keys"] == initial_state["keys"] From e3974ca727a108a930f07d1dfa02f08629def1d5 Mon Sep 17 00:00:00 2001 From: Paul Slavin Date: Wed, 27 Nov 2024 17:44:55 +0000 Subject: [PATCH 12/28] Lib funcs for ConfigSet export/apply --- hydra_base/lib/hydraconfig.py | 32 +++++++++--- tests/test_hydraconfig.py | 94 +++++++++++++++++++++++++++++++++-- 2 files changed, 115 insertions(+), 11 deletions(-) diff --git a/hydra_base/lib/hydraconfig.py b/hydra_base/lib/hydraconfig.py index 477bec1d..8e9334f4 100644 --- a/hydra_base/lib/hydraconfig.py +++ b/hydra_base/lib/hydraconfig.py @@ -1,6 +1,8 @@ """ Library functions for Hydra configuration """ +import json +import yaml from hydra_base import db from hydra_base.exceptions import ( @@ -108,14 +110,32 @@ def config_key_clear_all_rules(key_name, **kwargs): """ Config Sets: Archived versions of complete configurations """ -def create_configset(set_name): - pass +def export_config_as_json(name, description="", **kwargs): + from hydra_base.util.configset import ConfigSet + cs = ConfigSet(name, description=description) + state = cs.save_keys_to_configset() + return json.dumps(state) -def delete_configset(set_name): - pass +def export_config_as_yaml(name, description="", **kwargs): + from hydra_base.util.configset import ConfigSet + cs = ConfigSet(name, description=description) + state = cs.save_keys_to_configset() + return yaml.safe_dump(state) -def apply_configset(set_name): - pass +def apply_json_configset(json_src, **kwargs): + from hydra_base.util.configset import ConfigSet + if not isinstance(json_src, str): + raise ValueError(f"apply_json_configset requires a JSON encoded string argument") + + try: + state = json.loads(json_src) + except JSONDecodeError as e: + raise ValueError(f"Argument is not a valid JSON string: {e}") + + cs = ConfigSet(state["name"], description=state["description"]) + old_state = cs.apply_configset_to_db(state) + + return old_state def list_configsets(): pass diff --git a/tests/test_hydraconfig.py b/tests/test_hydraconfig.py index d497ca45..77bcae00 100644 --- a/tests/test_hydraconfig.py +++ b/tests/test_hydraconfig.py @@ -1,9 +1,12 @@ import base64 import copy +import inspect +import json import math import pytest import random import string +import yaml from hydra_base.exceptions import HydraError from hydra_base.lib.hydraconfig import ( @@ -60,34 +63,72 @@ def random_keys(client): - Deletes these keys on return unless the optional do_tidy argument is overwritten to be False. """ - key_names = [] + pending_tidy = [] _do_tidy = True def _random_keys(n_keys, do_tidy=True): key_gen_funcs = { "integer": make_integer_key_with_value, "string": make_string_key_with_value } - nonlocal _do_tidy + nonlocal _do_tidy, pending_tidy _do_tidy = do_tidy + prefix_len = 3 + existing_key_prefixes = set(k[:prefix_len] for k in pending_tidy) key_prefixes = set() + key_names = [] for idx in range(n_keys): while True: - key_prefix = "".join(random.choices(string.ascii_lowercase, k=3)) - if key_prefix not in key_prefixes: + key_prefix = "".join(random.choices(string.ascii_lowercase, k=prefix_len)) + if key_prefix not in key_prefixes | existing_key_prefixes: key_prefixes.add(key_prefix) break key_name = f"{key_prefix} test key" key_func = key_gen_funcs[random.choice([*key_gen_funcs])] key_names.append(key_func(client, key_name)) + if do_tidy: + pending_tidy += key_names return key_names yield _random_keys if _do_tidy: - for key_name in key_names: + for key_name in pending_tidy: client.unregister_config_key(key_name) +class TestFixtures(): + def test_fixture_reentrancy(self, random_keys, request): + """ + Verify random_keys fixture is reentrant wrt to + multiple calls to the returned func in the same + fixture scope. + """ + num_keys = 16 + # First fix func call returns new keys and has them pending deletion + keys0 = random_keys(num_keys) + assert len(keys0) == num_keys + rkpt0 = inspect.getclosurevars(random_keys).nonlocals["pending_tidy"] + assert len(rkpt0) == num_keys + for key in keys0: + assert key in rkpt0 + # Second fix func call returns new keys but do_tidy is False + # so pending deletion not expanded + keys1 = random_keys(num_keys, do_tidy=False) + assert len(keys1) == num_keys + rkpt1 = inspect.getclosurevars(random_keys).nonlocals["pending_tidy"] + assert len(rkpt1) == num_keys + for key in keys1: + assert key not in rkpt1 + # Third fix func call returns new keys and adds these to + # pending deletion + keys2 = random_keys(num_keys) + assert len(keys2) == num_keys + rkpt2 = inspect.getclosurevars(random_keys).nonlocals["pending_tidy"] + assert len(rkpt2) == 2*num_keys + for key in keys0 + keys2: + assert key in rkpt2 + + class TestConfigKeyValidators(): """ Standalone tests of Config Key validator classes """ def test_create_validator(self): @@ -431,6 +472,7 @@ def test_apply_configset_to_db(self, client, random_keys): # Manually delete initial keys for key_name in key_names: client.unregister_config_key(key_name) + key_names.clear() # Generate new state new_key_names = random_keys(num_keys, do_tidy=False) # Save second state @@ -442,3 +484,45 @@ def test_apply_configset_to_db(self, client, random_keys): # ...which means the initial key state was restored final_state = cs.save_keys_to_configset() assert final_state["keys"] == initial_state["keys"] + # Initial keys were restored by load so delete again + for key_name in final_state["keys"]: + client.unregister_config_key(key_name) + + def test_configset_api_json(self, client, random_keys): + num_keys = 16 + key_names = random_keys(num_keys) + cs_json = client.export_config_as_json("Configset API test keys", "ConfigSet API test desc") + cs = json.loads(cs_json) + assert len(cs["keys"]) == num_keys + for key_name in key_names: + assert key_name in cs["keys"] + + def test_configset_api_yaml(self, client, random_keys): + num_keys = 16 + key_names = random_keys(num_keys) + cs_yaml = client.export_config_as_yaml("Configset API test keys", "ConfigSet API test desc") + cs = yaml.safe_load(cs_yaml) + assert len(cs["keys"]) == num_keys + for key_name in key_names: + assert key_name in cs["keys"] + + def test_apply_json_configset(self, client, random_keys): + num_keys = 16 + # Create an initial state + key_names = random_keys(num_keys) + # Export this as json + cs_json = client.export_config_as_json("Configset API test keys", "ConfigSet API test desc") + # Then delete state + for key_name in key_names: + client.unregister_config_key(key_name) + # Confirm no loaded state + all_keys = client.list_config_keys() + assert len(all_keys) == 0 + # Apply the exported json configset + old_state = client.apply_json_configset(cs_json) + # Re-export the loaded state + applied_json = client.export_config_as_json("Applied keys") + # And confirm this is equal to initial state + orig_state = json.loads(cs_json) + applied_state = json.loads(applied_json) + assert orig_state["keys"] == applied_state["keys"] From 2bee0af20adc655371d7ac519f4328bf1cabe1ae Mon Sep 17 00:00:00 2001 From: Paul Slavin Date: Thu, 28 Nov 2024 17:32:17 +0000 Subject: [PATCH 13/28] Bool and description support; Update lib funcs --- .../db/model/hydraconfig/hydraconfig.py | 2 +- hydra_base/lib/hydraconfig.py | 33 ++++----- hydra_base/util/configset.py | 13 +++- requirements.txt | 28 +++---- tests/test_hydraconfig.py | 73 ++++++++++++++----- 5 files changed, 97 insertions(+), 52 deletions(-) diff --git a/hydra_base/db/model/hydraconfig/hydraconfig.py b/hydra_base/db/model/hydraconfig/hydraconfig.py index f06ee47a..bc104741 100644 --- a/hydra_base/db/model/hydraconfig/hydraconfig.py +++ b/hydra_base/db/model/hydraconfig/hydraconfig.py @@ -36,7 +36,7 @@ class ConfigKey(Base): name = Column(String(key_name_max_length), nullable=False, unique=True) description = Column(String(key_desc_max_length), nullable=True, unique=False) type = Column(String(key_type_tag_max_length)) - rules = Column(String(200)) + rules = Column(String(200), default='{}') __mapper_args__ = { diff --git a/hydra_base/lib/hydraconfig.py b/hydra_base/lib/hydraconfig.py index 8e9334f4..78ca5263 100644 --- a/hydra_base/lib/hydraconfig.py +++ b/hydra_base/lib/hydraconfig.py @@ -2,7 +2,6 @@ Library functions for Hydra configuration """ import json -import yaml from hydra_base import db from hydra_base.exceptions import ( @@ -26,11 +25,11 @@ """ Config Keys: Key:Value pairs of config settings """ -def register_config_key(key_name, key_type, **kwargs): +def register_config_key(key_name, key_type, description="", **kwargs): if not (key_cls := config_key_type_map.get(key_type, None)): raise HydraError(f"Invalid ConfigKey type '{key_type}'") - key = key_cls(name=key_name) + key = key_cls(name=key_name, desc=description) try: db.DBSession.add(key) db.DBSession.flush() @@ -61,6 +60,17 @@ def config_key_get_value(key_name, **kwargs): key = _get_config_key_by_name(key_name) return key.value +def config_key_set_description(key_name, description="", **kwargs): + key = _get_config_key_by_name(key_name) + if not description or not isinstance(description, str): + raise HydraError(f"Invalid description for {key_name}: '{description}'") + + key.description = description + +def config_key_get_description(key_name, **kwargs): + key = _get_config_key_by_name(key_name) + return key.description + def _get_config_key_by_name(key_name): try: key = db.DBSession.query(ConfigKey).filter(ConfigKey.name == key_name).one() @@ -116,16 +126,10 @@ def export_config_as_json(name, description="", **kwargs): state = cs.save_keys_to_configset() return json.dumps(state) -def export_config_as_yaml(name, description="", **kwargs): - from hydra_base.util.configset import ConfigSet - cs = ConfigSet(name, description=description) - state = cs.save_keys_to_configset() - return yaml.safe_dump(state) - -def apply_json_configset(json_src, **kwargs): +def apply_configset(json_src, **kwargs): from hydra_base.util.configset import ConfigSet if not isinstance(json_src, str): - raise ValueError(f"apply_json_configset requires a JSON encoded string argument") + raise ValueError(f"apply_configset requires a JSON encoded string argument") try: state = json.loads(json_src) @@ -137,13 +141,6 @@ def apply_json_configset(json_src, **kwargs): return old_state -def list_configsets(): - pass - -def list_configset_versions(set_name): - pass - - """ Config Groups: A named collection of Config Keys """ def create_config_group(group_name, group_desc=None, **kwargs): diff --git a/hydra_base/util/configset.py b/hydra_base/util/configset.py index 212bb55b..615e8c6f 100644 --- a/hydra_base/util/configset.py +++ b/hydra_base/util/configset.py @@ -10,7 +10,8 @@ from hydra_base.lib.hydraconfig import ( register_config_key, config_key_set_value, - config_key_set_rule + config_key_set_rule, + config_key_set_description ) @@ -69,7 +70,14 @@ def all_keys_as_dict(self): keys = db.DBSession.query(ConfigKey).all() if len(keys) == 0: return {} - return {key.name: {"type": key.type, "value": key.value, "rules": key.rules} for key in keys} + return { + key.name: { + "type": key.type, + "value": key.value, + "rules": key.rules, + "description": key.description + } for key in keys + } def generate_mac(self, state): digest = hmac.digest(key=config_set_secret_key, msg=state.encode("utf8"), digest=ConfigSet.mac_hash) @@ -112,3 +120,4 @@ def load_keys_from_state(self, state): for rule_name, rule_val in rules.items(): config_key_set_rule(key_name, rule_name, rule_val) config_key_set_value(key_name, key["value"]) + config_key_set_description(key_name, key["description"]) diff --git a/requirements.txt b/requirements.txt index c05adb5d..cd6c519b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,23 +1,23 @@ -sqlalchemy -psycopg2-binary -zope.sqlalchemy -pandas -numpy bcrypt -lxml -pymongo -mysqlclient -pudb -python-dateutil -cheroot beaker +cheroot click diskcache -pylibmc +fsspec +h5py +lxml mysqlclient +mysqlclient +numpy packaging +pandas +psycopg2-binary +pudb +pylibmc +pymongo +python-dateutil requests -fsspec -h5py s3fs +sqlalchemy tables +zope.sqlalchemy diff --git a/tests/test_hydraconfig.py b/tests/test_hydraconfig.py index 77bcae00..541a21a3 100644 --- a/tests/test_hydraconfig.py +++ b/tests/test_hydraconfig.py @@ -6,7 +6,6 @@ import pytest import random import string -import yaml from hydra_base.exceptions import HydraError from hydra_base.lib.hydraconfig import ( @@ -19,11 +18,13 @@ ) from hydra_base.util.configset import ConfigSet + # Util funcs def make_integer_key_with_value(client, key_name): key_value = random.randint(2**7, 2**9) - key = client.register_config_key(key_name, "integer") + key_description = generate_key_description(key_type="integer") + key = client.register_config_key(key_name, "integer", description=key_description) val_diff = random.randint(1, key_value//2) min_value, max_value = key_value-val_diff, key_value+val_diff client.config_key_set_rule(key_name, "min_value", min_value) @@ -35,7 +36,8 @@ def make_integer_key_with_value(client, key_name): def make_string_key_with_value(client, key_name): key_len = random.randint(2**4, 2**6) key_value = "".join(random.choices(string.ascii_lowercase, k=key_len)) - key = client.register_config_key(key_name, "string") + key_description = generate_key_description(key_type="string") + key = client.register_config_key(key_name, "string", description=key_description) len_diff = random.randint(1, key_len//2) min_length, max_length = key_len-len_diff, key_len+len_diff client.config_key_set_rule(key_name, "min_length", min_length) @@ -44,6 +46,47 @@ def make_string_key_with_value(client, key_name): return key.name +def make_boolean_key_with_value(client, key_name): + key_value = random.getrandbits(1) + key_description = generate_key_description(key_type="boolean") + key = client.register_config_key(key_name, "boolean", description=key_description) + client.config_key_set_value(key_name, key_value) + return key.name + + +def generate_key_description(src="\x80.", key_type=""): + dmap = { + "\x80": ["\x81 \x84 for a key of type \xC0"], + "\x81": ["An \x82", "A \x83", "The"], + "\x82": ["appropriate", "apt", "example", "illustrative"], + "\x83": ["fitting", "suitable", "relevant", "particular", "placeholder", "basic", "typical"], + "\x84": ["description", "comment", "overview", "explanation"] + } + maptop = 0x85 + out = [] + idx = -1 + while True: + idx += 1 + try: + c = src[idx] + except IndexError: + break + oc = ord(c) + if oc < 128: + out.append(c) + continue + else: + if oc < maptop: + mapline = dmap[c] + out.append(generate_key_description(src=random.choice(mapline), key_type=key_type)) + continue + if oc == 0xC0: + out.append(key_type) + continue + + return "".join(out) + + @pytest.fixture def config_group(): group_name = "Test ConfigKey Group" @@ -68,7 +111,8 @@ def random_keys(client): def _random_keys(n_keys, do_tidy=True): key_gen_funcs = { "integer": make_integer_key_with_value, - "string": make_string_key_with_value + "string": make_string_key_with_value, + "boolean": make_boolean_key_with_value } nonlocal _do_tidy, pending_tidy _do_tidy = do_tidy @@ -97,7 +141,7 @@ def _random_keys(n_keys, do_tidy=True): class TestFixtures(): - def test_fixture_reentrancy(self, random_keys, request): + def test_fixture_reentrancy(self, client, random_keys): """ Verify random_keys fixture is reentrant wrt to multiple calls to the returned func in the same @@ -119,6 +163,10 @@ def test_fixture_reentrancy(self, random_keys, request): assert len(rkpt1) == num_keys for key in keys1: assert key not in rkpt1 + # As a result of do_tidy=False, we have to do tidy + # or duplicates could occur + for key_name in keys1: + client.unregister_config_key(key_name) # Third fix func call returns new keys and adds these to # pending deletion keys2 = random_keys(num_keys) @@ -488,7 +536,7 @@ def test_apply_configset_to_db(self, client, random_keys): for key_name in final_state["keys"]: client.unregister_config_key(key_name) - def test_configset_api_json(self, client, random_keys): + def test_configset_api_export_json(self, client, random_keys): num_keys = 16 key_names = random_keys(num_keys) cs_json = client.export_config_as_json("Configset API test keys", "ConfigSet API test desc") @@ -497,16 +545,7 @@ def test_configset_api_json(self, client, random_keys): for key_name in key_names: assert key_name in cs["keys"] - def test_configset_api_yaml(self, client, random_keys): - num_keys = 16 - key_names = random_keys(num_keys) - cs_yaml = client.export_config_as_yaml("Configset API test keys", "ConfigSet API test desc") - cs = yaml.safe_load(cs_yaml) - assert len(cs["keys"]) == num_keys - for key_name in key_names: - assert key_name in cs["keys"] - - def test_apply_json_configset(self, client, random_keys): + def test_apply_configset(self, client, random_keys): num_keys = 16 # Create an initial state key_names = random_keys(num_keys) @@ -519,7 +558,7 @@ def test_apply_json_configset(self, client, random_keys): all_keys = client.list_config_keys() assert len(all_keys) == 0 # Apply the exported json configset - old_state = client.apply_json_configset(cs_json) + old_state = client.apply_configset(cs_json) # Re-export the loaded state applied_json = client.export_config_as_json("Applied keys") # And confirm this is equal to initial state From 0de5aeb2ef7dfb69a2ebd4519f290ca14cf3e1bf Mon Sep 17 00:00:00 2001 From: Paul Slavin Date: Mon, 20 Jan 2025 17:15:46 +0000 Subject: [PATCH 14/28] Add migrate_config_util --- hydra_base/util/migrate_config.py | 51 +++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 hydra_base/util/migrate_config.py diff --git a/hydra_base/util/migrate_config.py b/hydra_base/util/migrate_config.py new file mode 100644 index 00000000..14705e43 --- /dev/null +++ b/hydra_base/util/migrate_config.py @@ -0,0 +1,51 @@ +""" + Utilities to migrate from Hydra.ini config format + to DB-based config table via hydra_base.lib.hydraconfig +""" +import configparser +import os + + +def ini_to_configset(ini_filename): + db_config_schema = make_db_config_schema(ini_filename) + return db_config_schema + +def make_db_config_schema(ini_filename): + config = configparser.ConfigParser(allow_no_value=True) + config.read(ini_filename) + + # Values for "home_dir" and "hydra_base_dir" must be + # set to allow for interpolation into later values + home_dir = os.environ.get("HYDRA_HOME_DIR", '~') + hydra_base_dir = os.environ.get("HYDRA_BASE_DIR", os.getcwd()) + config.set("DEFAULT", "home_dir", os.path.expanduser(home_dir)) + config.set("DEFAULT", "hydra_base_dir", os.path.expanduser(hydra_base_dir)) + + db_config_schema = {} + for section in ["DEFAULT", *config.sections()]: + for key in config[section]: + try: + value = config[section][key] + except configparser.InterpolationSyntaxError: + value = config[section].get(key, raw=True) + key_name = f"{section}_{key}" + try: + value = int(value, 10) + key_type = "integer" + except ValueError: + key_type = "string" + + if key_name in db_config_schema: + raise ValueError(f"Duplicate key: {key_name}") + + db_config_schema[key_name] = { + "type": key_type, + "value": value + } + + return db_config_schema + +if __name__ == "__main__": + ini_file = "hydra_base/hydra.ini" + config = ini_to_configset(ini_file) + print(config) From d45999a512bc073d3aa5d4a9ee24b4a9cc5a3668 Mon Sep 17 00:00:00 2001 From: Paul Slavin Date: Wed, 22 Jan 2025 17:40:18 +0000 Subject: [PATCH 15/28] Enable startup without ini --- hydra_base/__init__.py | 6 --- hydra_base/config.py | 31 ++++++++++++- hydra_base/db/__init__.py | 7 +-- hydra_base/hydra_logging.py | 4 +- hydra_base/lib/cache.py | 7 ++- hydra_base/lib/hydraconfig.py | 2 +- hydra_base/util/migrate_config.py | 76 +++++++++++++++++++++++++++++-- tests/conftest.py | 19 +++++++- tests/templates/test_templates.py | 9 ++-- tests/test_data.py | 2 +- tests/test_hdf.py | 5 +- tests/test_hydraconfig.py | 14 +++++- 12 files changed, 153 insertions(+), 29 deletions(-) diff --git a/hydra_base/__init__.py b/hydra_base/__init__.py index 99fc214a..77209af0 100644 --- a/hydra_base/__init__.py +++ b/hydra_base/__init__.py @@ -48,12 +48,6 @@ if len(config.sysfiles) + len(config.repofiles) + len(config.userfiles) + len(config.sysfiles) == 0: log.critical("No config found. Please put your ini file into one of the files listed beside CONFIG above.") -if config.get("security", "max_login_attempts") is None: - """ Absence of max_login_attempts results in all users unable to log in, - so ensure this is defined in config, or fail. - """ - raise RuntimeError("Config 'security' section must define 'max_login_attempts'") - log.debug(" \n ") from .lib.attributes import * diff --git a/hydra_base/config.py b/hydra_base/config.py index da835465..f933b732 100644 --- a/hydra_base/config.py +++ b/hydra_base/config.py @@ -20,6 +20,8 @@ import glob import sys + + PYTHONVERSION = sys.version_info if PYTHONVERSION >= (3,2): import configparser as ConfigParser @@ -142,14 +144,41 @@ def read_values_from_environment(config, section_key, options_key): # print("Presente") config.set(section_key, options_key, env_value) +def read_env_db_config(): + return { + "hydra_db_server": os.environ.get("HYDRA_DB_SERVER"), + "hydra_db_name": os.environ.get("HYDRA_DB_NAME"), + "hydra_db_user": os.environ.get("HYDRA_DB_USER"), + "hydra_db_passwd": os.environ.get("HYDRA_DB_PASSWD"), + "hydra_db_autocreate": os.environ.get("HYDRA_DB_AUTOCREATE"), + "hydra_db_preping": os.environ.get("HYDRA_DB_PREPING") + } + +def read_env_startup_config(): + return { + "hydra_cachetype": os.environ.get("HYDRA_CACHETYPE"), + "hydra_log_confpath": os.environ.get("HYDRA_LOG_CONFPATH"), + "hydra_log_filedir": os.environ.get("HYDRA_LOG_FILEDIR") + } + +def get_startup_config(): + db_config = read_env_db_config() + db_config["url"] = f"mysql+mysqldb://{db_config['hydra_db_user']}:{db_config['hydra_db_passwd']}"\ + f"@{db_config['hydra_db_server']}/{db_config['hydra_db_name']}" + + db_config.update(read_env_startup_config()) + return db_config def get(section, option, default=None): + from hydra_base.lib.hydraconfig import ( + config_key_get_value + ) if CONFIG is None: load_config() try: - return CONFIG.get(section, option) + return config_key_get_value(f"{section}_{option}") except: return default diff --git a/hydra_base/db/__init__.py b/hydra_base/db/__init__.py index 57119e59..f1f34af6 100644 --- a/hydra_base/db/__init__.py +++ b/hydra_base/db/__init__.py @@ -81,12 +81,13 @@ def create_mysql_db(db_url): #Remove trailing whitespace and forwardslashes db_url = db_url.strip().strip('/') + db_config = config.get_startup_config() #Check this is a mysql URL if db_url.find('mysql') >= 0: #Get the DB name from config and check if it's in the URL - db_name = config.get('mysqld', 'db_name', 'hydradb') + db_name = db_config["hydra_db_name"] if db_url.find(db_name) >= 0: no_db_url = db_url.rsplit("/", 1)[0] else: @@ -104,7 +105,7 @@ def create_mysql_db(db_url): if db_url.find('charset') == -1: db_url = "{}?charset=utf8&use_unicode=1".format(db_url) - if config.get('mysqld', 'auto_create', 'Y') == 'Y': + if db_config.get("hydra_db_autocreate", 'Y') == 'Y': tmp_engine = create_engine(no_db_url) log.debug("Creating database {0} as it does not exist.".format(db_name)) with tmp_engine.connect() as conn: @@ -113,7 +114,7 @@ def create_mysql_db(db_url): def connect(db_url=None): if db_url is None: - db_url = config.get('mysqld', 'url') + db_url = config.get_startup_config()["url"] log.info("Connecting to database") if db_url.find('@') >= 0: diff --git a/hydra_base/hydra_logging.py b/hydra_base/hydra_logging.py index 3796c80b..4f9b64fc 100644 --- a/hydra_base/hydra_logging.py +++ b/hydra_base/hydra_logging.py @@ -57,7 +57,7 @@ def init(level=None): calling_file = os.path.split(calling_file)[1] log_file = "%s.log" % calling_file.split('.')[0] - log_base_path = config.get('logging_conf', 'log_file_dir', '.') + log_base_path = config.get_startup_config()["hydra_log_filedir"] if not os.path.isdir(log_base_path): os.makedirs(log_base_path) @@ -71,7 +71,7 @@ def init(level=None): use_default = False try: - config_file = os.path.expanduser(config.get('logging_conf', 'log_config_path', '.')) + config_file = os.path.expanduser(config.get_startup_config()["hydra_log_confpath"]) #check the config file exists... if os.path.isfile(config_file) and log_base_path is not None: logging.config.fileConfig(config_file) diff --git a/hydra_base/lib/cache.py b/hydra_base/lib/cache.py index 6722a247..bf8edcce 100644 --- a/hydra_base/lib/cache.py +++ b/hydra_base/lib/cache.py @@ -12,10 +12,12 @@ log = logging.getLogger(__name__) global cache -if hydraconfig.get('cache', 'type') != "memcached": +cache_type = hydraconfig.get_startup_config()["hydra_cachetype"] + +if cache_type != "memcached": import diskcache as dc cache = dc.Cache(tempfile.gettempdir()) -elif hydraconfig.get('cache', 'type') == 'memcached': +elif cache_type == 'memcached': try: import pylibmc cache = pylibmc.Client([hydraconfig.get('cache', 'host', '127.0.0.1')], binary=True) @@ -24,6 +26,7 @@ import diskcache as dc cache = dc.Cache(tempfile.gettempdir()) + def clear_cache(): if hasattr(cache, 'flush_all'): cache.flush_all() # memcache diff --git a/hydra_base/lib/hydraconfig.py b/hydra_base/lib/hydraconfig.py index 78ca5263..087e1623 100644 --- a/hydra_base/lib/hydraconfig.py +++ b/hydra_base/lib/hydraconfig.py @@ -62,7 +62,7 @@ def config_key_get_value(key_name, **kwargs): def config_key_set_description(key_name, description="", **kwargs): key = _get_config_key_by_name(key_name) - if not description or not isinstance(description, str): + if not isinstance(description, str): raise HydraError(f"Invalid description for {key_name}: '{description}'") key.description = description diff --git a/hydra_base/util/migrate_config.py b/hydra_base/util/migrate_config.py index 14705e43..d21b4353 100644 --- a/hydra_base/util/migrate_config.py +++ b/hydra_base/util/migrate_config.py @@ -4,6 +4,22 @@ """ import configparser import os +import transaction + +from pprint import pprint + +from hydra_base import db +from hydra_base.lib.hydraconfig import ( + register_config_key, + unregister_config_key, + list_config_keys, + config_key_set_value, + config_key_get_value +) + + +if not db.DBSession: + db.connect() def ini_to_configset(ini_filename): @@ -11,6 +27,7 @@ def ini_to_configset(ini_filename): return db_config_schema def make_db_config_schema(ini_filename): + exclude_sections = ("mysqld",) config = configparser.ConfigParser(allow_no_value=True) config.read(ini_filename) @@ -22,8 +39,29 @@ def make_db_config_schema(ini_filename): config.set("DEFAULT", "hydra_base_dir", os.path.expanduser(hydra_base_dir)) db_config_schema = {} - for section in ["DEFAULT", *config.sections()]: - for key in config[section]: + for key in config["DEFAULT"]: + try: + value = config["DEFAULT"][key] + except configparser.InterpolationSyntaxError: + value = config["DEFAULT"].get(key, raw=True) + try: + value = int(value, 10) + key_type = "integer" + except ValueError: + key_type = "string" + + if key in db_config_schema: + raise ValueError(f"Duplicate key: {key}") + + db_config_schema[key] = { + "type": key_type, + "value": value + } + + for section in config.sections(): + if section in exclude_sections: + continue + for key in config._sections[section].keys(): try: value = config[section][key] except configparser.InterpolationSyntaxError: @@ -45,7 +83,35 @@ def make_db_config_schema(ini_filename): return db_config_schema + +def make_config_from_schema(schema): + for name, key in schema.items(): + register_config_key(name, key["type"]) + config_key_set_value(name, key["value"]) + + transaction.commit() + + +def get_all_config_keys(): + keys = list_config_keys() + return {k: config_key_get_value(k) for k in keys} + + +def delete_all_config_keys(): + keys = list_config_keys() + for key in keys: + unregister_config_key(key) + + transaction.commit() + + if __name__ == "__main__": - ini_file = "hydra_base/hydra.ini" - config = ini_to_configset(ini_file) - print(config) + #ini_file = "hydra_base/hydra.ini" + #schema = ini_to_configset(ini_file) + #pprint(schema) + #config = make_config_from_schema(schema) + #keys = get_all_config_keys() + #pprint(keys) + #delete_all_config_keys() + keys = get_all_config_keys() + pprint(keys) diff --git a/tests/conftest.py b/tests/conftest.py index 4f9e68a0..2d2b378c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -122,6 +122,17 @@ def client(connection_type, testdb_uri): test_server=null_server) client.login('root', '') + from hydra_base.util.migrate_config import ( + ini_to_configset, + make_config_from_schema + ) + + ini_file = "hydra_base/hydra.ini" + schema = ini_to_configset(ini_file) + #pprint(schema) + config = make_config_from_schema(schema) + #keys = get_all_config_keys() + client.testutils = testing.TestUtil(client) pytest.root_user_id = 1 pytest.user_a = client.testutils.create_user("UserA") @@ -148,11 +159,15 @@ def drop_tables(db_url): @pytest.fixture() def network(client, project_id=None, num_nodes=10, new_proj=True, map_projection='EPSG:4326'): - return client.testutils.build_network(project_id, num_nodes, new_proj, map_projection) + return client.testutils.build_network(project_id=project_id, num_nodes=num_nodes, new_proj=new_proj, map_projection=map_projection) @pytest.fixture() def network_with_data(client, project_id=None, num_nodes=10, ret_full_net=True, new_proj=True, map_projection='EPSG:4326'): - return client.testutils.create_network_with_data(project_id, num_nodes, ret_full_net, new_proj, map_projection) + return client.testutils.create_network_with_data(project_id=project_id, + num_nodes=num_nodes, + ret_full_net=ret_full_net, + new_proj=new_proj, + map_projection=map_projection) @pytest.fixture() def network_with_child_scenario(client, project_id=None, num_nodes=10, ret_full_net=True, new_proj=True, map_projection='EPSG:4326'): diff --git a/tests/templates/test_templates.py b/tests/templates/test_templates.py index 32b9734a..15672564 100644 --- a/tests/templates/test_templates.py +++ b/tests/templates/test_templates.py @@ -143,6 +143,7 @@ class TestTemplates: """ Test for templates """ + """ def test_add_xml(self, template_json_object): new_tmpl = template_json_object @@ -168,7 +169,7 @@ def test_get_xml(self, client, template_json_object): assert db_template is not None - template_xsd_path = config.get('templates', 'template_xsd_path') + template_xsd_path = config.get('template_xsd_path') xmlschema_doc = etree.parse(template_xsd_path) xmlschema = etree.XMLSchema(xmlschema_doc) @@ -228,7 +229,7 @@ def test_get_dict(self, client, template_json_object): assert len(check_template_i.templatetypes) == 2 - + """ """ TEMPLATES Functions """ @@ -820,6 +821,7 @@ def test_remove_type_from_resource(self, client, mock_template, network_with_dat assert updated_node_j.types is None or str(result1_j.id) not in [str(x.type_id) for x in updated_node_j.types] + """ def test_create_template_from_network(self, client, network_with_data): network = network_with_data @@ -828,7 +830,7 @@ def test_create_template_from_network(self, client, network_with_data): assert net_template is not None - template_xsd_path = config.get('templates', 'template_xsd_path') + template_xsd_path = config.get("templates", "template_xsd_path") xmlschema_doc = etree.parse(template_xsd_path) xmlschema = etree.XMLSchema(xmlschema_doc) @@ -836,6 +838,7 @@ def test_create_template_from_network(self, client, network_with_data): xml_tree = etree.fromstring(net_template) xmlschema.assertValid(xml_tree) + """ def test_apply_template_to_network(self, client, mock_template, network_with_data): net_to_update = network_with_data diff --git a/tests/test_data.py b/tests/test_data.py index b87b3f6f..47a5ec38 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -251,7 +251,7 @@ def test_multiple_vals_at_time(self, client, network_with_data, seasonal_timeser val_a = json.loads(val_to_query.value) - dtformat = hb.config.get('DEFAULT', 'datetime_format', "%Y-%m-%dT%H:%M:%S.%f000Z") + dtformat = hb.config.get("datetime", "format", "%Y-%m-%dT%H:%M:%S.%f000Z") fmt = datetime.datetime.strftime qry_times = [ fmt(datetime.datetime(2000, 1, 10, 00, 00, 00), dtformat), diff --git a/tests/test_hdf.py b/tests/test_hdf.py index 681a5e9b..8e3e7ff9 100644 --- a/tests/test_hdf.py +++ b/tests/test_hdf.py @@ -109,16 +109,17 @@ def test_hdf_size(self, hdf, public_aws_file): @pytest.mark.requires_hdf def test_private_hdf_no_access(self, hdf, private_aws_file): """ - Do the reported properties of a dataset match expected values? + Does attempted access to a private HDF file without credentials result in an error? """ with pytest.raises(PermissionError): info = hdf.get_series_info(private_aws_file["path"], columns=private_aws_file["series_name"]) + @pytest.mark.skip @pytest.mark.requires_hdf @patch.dict('os.environ', {'AWS_ACCESS_KEY_ID': IAM_ACCESS_KEY, 'AWS_SECRET_ACCESS_KEY': IAM_SECRET_KEY}) def test_private_hdf_correct_access(self, private_aws_file): """ - Do the reported properties of a dataset match expected values? + Is access to a private HDF file with correct credentials permitted? """ hdf = HdfStorageAdapter() diff --git a/tests/test_hydraconfig.py b/tests/test_hydraconfig.py index 541a21a3..c2a420e2 100644 --- a/tests/test_hydraconfig.py +++ b/tests/test_hydraconfig.py @@ -137,7 +137,10 @@ def _random_keys(n_keys, do_tidy=True): yield _random_keys if _do_tidy: for key_name in pending_tidy: - client.unregister_config_key(key_name) + try: + client.unregister_config_key(key_name) + except HydraError: + pass class TestFixtures(): @@ -546,6 +549,12 @@ def test_configset_api_export_json(self, client, random_keys): assert key_name in cs["keys"] def test_apply_configset(self, client, random_keys): + # Backup pre-test config state + initial_keys = client.list_config_keys() + initial_state = client.export_config_as_json("Initial state", "Initial state desc") + for key_name in initial_keys: + client.unregister_config_key(key_name) + num_keys = 16 # Create an initial state key_names = random_keys(num_keys) @@ -565,3 +574,6 @@ def test_apply_configset(self, client, random_keys): orig_state = json.loads(cs_json) applied_state = json.loads(applied_json) assert orig_state["keys"] == applied_state["keys"] + + # Restore initial config state + _ = client.apply_configset(initial_state) From 03a477e931f871f8186f601fc164dfa0360b8988 Mon Sep 17 00:00:00 2001 From: Paul Slavin Date: Thu, 23 Jan 2025 11:35:57 +0000 Subject: [PATCH 16/28] Add startup env vars to CI --- .github/workflows/ci.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c12b4d8..382215c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,10 +10,19 @@ jobs: COV_MIN: 70 # Minimum acceptable coverage level TEST_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + HYDRA_DB_SERVER: 127.0.0.1 + HYDRA_DB_NAME: hydra_base_test + HYDRA_DB_USER: root + HYDRA_DB_PASSWD: root + HYDRA_DB_AUTOCREATE: Y + HYDRA_DB_PREPING: True + HYDRA_CACHETYPE: memcached + HYDRA_LOG_CONFPATH: hydra-base/logging.conf + HYDRA_LOG_FILEDIR: .hydra/log strategy: matrix: - py_version: ["3.8", "3.9", "3.10"] + py_version: ["3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 @@ -52,4 +61,4 @@ jobs: pip install -e . - name: Run pytests - run: pytest --db-backend=mysql --cov=hydra_base --cov-fail-under=$COV_MIN + run: pytest --db-backend=mysql --cov=hydra_base From 270434f925aed866f30d8cca991993cb4e8609e5 Mon Sep 17 00:00:00 2001 From: Paul Slavin Date: Thu, 23 Jan 2025 11:50:26 +0000 Subject: [PATCH 17/28] Add startup env vars to CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 382215c3..eb9ef85e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: strategy: matrix: - py_version: ["3.10", "3.11", "3.12"] + py_version: ["3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v3 From 75bdf6bb14c79a10c18b1cf287b3fd8e11d5f85d Mon Sep 17 00:00:00 2001 From: Paul Slavin Date: Thu, 23 Jan 2025 15:25:44 +0000 Subject: [PATCH 18/28] Add startup config for memcache host --- .github/workflows/ci.yml | 1 + hydra_base/config.py | 1 + hydra_base/lib/cache.py | 6 ++++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb9ef85e..baca7559 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,7 @@ jobs: HYDRA_DB_AUTOCREATE: Y HYDRA_DB_PREPING: True HYDRA_CACHETYPE: memcached + HYDRA_CACHEHOST: 127.0.0.1 HYDRA_LOG_CONFPATH: hydra-base/logging.conf HYDRA_LOG_FILEDIR: .hydra/log diff --git a/hydra_base/config.py b/hydra_base/config.py index f933b732..19370ca6 100644 --- a/hydra_base/config.py +++ b/hydra_base/config.py @@ -157,6 +157,7 @@ def read_env_db_config(): def read_env_startup_config(): return { "hydra_cachetype": os.environ.get("HYDRA_CACHETYPE"), + "hydra_cachehost": os.environ.get("HYDRA_CACHEHOST"), "hydra_log_confpath": os.environ.get("HYDRA_LOG_CONFPATH"), "hydra_log_filedir": os.environ.get("HYDRA_LOG_FILEDIR") } diff --git a/hydra_base/lib/cache.py b/hydra_base/lib/cache.py index bf8edcce..6d8f60df 100644 --- a/hydra_base/lib/cache.py +++ b/hydra_base/lib/cache.py @@ -12,7 +12,9 @@ log = logging.getLogger(__name__) global cache -cache_type = hydraconfig.get_startup_config()["hydra_cachetype"] +startup_config = hydraconfig.get_startup_config() +cache_type = startup_config["hydra_cachetype"] +cache_host = startup_config["hydra_cachehost"] if cache_type != "memcached": import diskcache as dc @@ -20,7 +22,7 @@ elif cache_type == 'memcached': try: import pylibmc - cache = pylibmc.Client([hydraconfig.get('cache', 'host', '127.0.0.1')], binary=True) + cache = pylibmc.Client([cache_host], binary=True) except ModuleNotFoundError: log.warning("Unable to find pylibmc. Defaulting to diskcache.") import diskcache as dc From 1c85157bf4cf51d39630c317f79ee9edff8948d1 Mon Sep 17 00:00:00 2001 From: Paul Slavin Date: Thu, 23 Jan 2025 19:05:16 +0000 Subject: [PATCH 19/28] Enable ConfigSet load on startup --- .github/workflows/ci.yml | 11 ++ default_configset.json | 1 + hydra_base/__init__.py | 13 -- hydra_base/config.py | 20 +++- hydra_base/db/model/project.py | 4 +- hydra_base/hydra.ini | 113 ------------------ hydra_base/lib/storage/hdfstorageadapter.py | 8 +- hydra_base/lib/storage/mongostorageadapter.py | 16 ++- hydra_base/util/configset.py | 3 + tests/conftest.py | 25 ++-- 10 files changed, 69 insertions(+), 145 deletions(-) create mode 100644 default_configset.json delete mode 100644 hydra_base/hydra.ini diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index baca7559..8498d678 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,17 @@ jobs: HYDRA_CACHEHOST: 127.0.0.1 HYDRA_LOG_CONFPATH: hydra-base/logging.conf HYDRA_LOG_FILEDIR: .hydra/log + HYDRA_MONGO_HOST: localhost + HYDRA_MONGO_PORT: 27017 + HYDRA_MONGO_DB_NAME: hydra + HYDRA_MONGO_USER: + HYDRA_MONGO_PASSWD: + HYDRA_MONGO_DATASETS: datasets + HYDRA_MONGO_THRESHOLD: 4096 + HYDRA_MONGO_DIRECT_LOCATION_TOKEN: mongo_direct + HYDRA_MONGO_VALUE_LOCATION_KEY: value_storage_location + HYDRA_DISABLE_HDF: False + HYDRA_HDF_FILESTORE: /tmp strategy: matrix: diff --git a/default_configset.json b/default_configset.json new file mode 100644 index 00000000..55a891c5 --- /dev/null +++ b/default_configset.json @@ -0,0 +1 @@ +{"name": "Default ConfigSet", "description": "The default Hydra startup state", "timestamp": "2025-01-23 15:57:19", "keys": {"log_level": {"type": "string", "value": "INFO", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_aux_dir": {"type": "string", "value": "/home/paul/.hydra", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "datetime_format": {"type": "string", "value": "%Y-%m-%dT%H:%M:%S.%f000Z", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "seasonal_key": {"type": "integer", "value": 9999, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "seasonal_year": {"type": "integer", "value": 1678, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "home_dir": {"type": "string", "value": "/home/paul", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_base_dir": {"type": "string", "value": "/home/paul/code/Hydra/hydra-base", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "db_instance": {"type": "string", "value": "MySQL", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "db_upper_bound": {"type": "integer", "value": 100, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "db_lower_bound": {"type": "integer", "value": 5, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "db_export_target": {"type": "string", "value": "/home/paul/.hydra/audit", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "db_purge_threshold": {"type": "integer", "value": 10000, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "db_compression_threshold": {"type": "integer", "value": 50000, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "sqlite_backup_dir": {"type": "string", "value": "/home/paul/.hydra/audit", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "sqlite_dbfile": {"type": "string", "value": "/home/paul/code/Hydra/hydra-base/HydraDB/hydra.db", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "sqlite_backup_url": {"type": "string", "value": "/home/paul/.hydra/audit/audit.db", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "mongodb_host": {"type": "string", "value": "localhost", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "mongodb_port": {"type": "integer", "value": 27017, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "mongodb_db_name": {"type": "string", "value": "hydra", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "mongodb_user": {"type": "string", "value": "", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "mongodb_passwd": {"type": "string", "value": "", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "mongodb_datasets": {"type": "string", "value": "datasets", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "mongodb_threshold": {"type": "integer", "value": 4096, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "mongodb_direct_location_token": {"type": "string", "value": "mongo_direct", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "mongodb_value_location_key": {"type": "string", "value": "value_storage_location", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "storage_hdf_disable_hdf": {"type": "string", "value": "False", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "storage_hdf_hdf_filestore": {"type": "string", "value": "/tmp", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_domain": {"type": "string", "value": "127.0.0.1", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_port": {"type": "integer", "value": 8080, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "hydra_server_path": {"type": "string", "value": "soap #deprecated", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_json_path": {"type": "string", "value": "json", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_http_path": {"type": "string", "value": "http", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_soap_path": {"type": "string", "value": "soap", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_url": {"type": "string", "value": "http://127.0.0.1:8080/soap #deprecated?wsdl", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_layout_xsd_path": {"type": "string", "value": "/home/paul/code/Hydra/hydra-base/static/resource_layout.xsd", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_domain": {"type": "string", "value": "http://127.0.0.1", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_port": {"type": "integer", "value": 8080, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "hydra_client_path": {"type": "string", "value": "json", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_json_path": {"type": "string", "value": "json", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_http_path": {"type": "string", "value": "http", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_soap_path": {"type": "string", "value": "soap", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_user": {"type": "string", "value": "root", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_password": {"type": "string", "value": "", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "filesys_img_src": {"type": "string", "value": "/home/paul/.hydra/images/", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "filesys_file_src": {"type": "string", "value": "/home/paul/.hydra/files/", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "plugin_default_directory": {"type": "string", "value": "/home/paul/.hydra/apps", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "plugin_queue_directory": {"type": "string", "value": "/home/paul/.hydra/apps/queue", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "plugin_upload_dir": {"type": "string", "value": "/home/paul/.hydra/apps/uploads", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "plugin_result_file": {"type": "string", "value": "/home/paul/.hydra/plugin_result/", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "plugin_plugin_xsd_path": {"type": "string", "value": "/home/paul/code/Hydra/hydra-base/static/plugin_input.xsd", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "unit_conversion_user_file": {"type": "string", "value": "/home/paul/code/Hydra/hydra-base/static/user_units.xml", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "unit_conversion_default_file": {"type": "string", "value": "/home/paul/code/Hydra/hydra-base/static/unit_definitions.xml", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "templates_template_xsd_path": {"type": "string", "value": "/home/paul/code/Hydra/hydra-base/static/template.xsd", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "logging_conf_log_config_path": {"type": "string", "value": "/home/paul/code/Hydra/hydra-base/logging.conf", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "logging_conf_log_file_dir": {"type": "string", "value": "/home/paul/.hydra/log", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "search_page_size": {"type": "integer", "value": 2000, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "polyvis_polyvis_url": {"type": "string", "value": "http://localhost:5000/", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "security_max_login_attempts": {"type": "integer", "value": 7, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "cache_type": {"type": "string", "value": "diskcache", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "limits_project_max_nest_depth": {"type": "integer", "value": 32, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}}, "digest": "af976b3f619d37be1c54220ce0dc4b1c2f162dbf017b02c260505b703baa0775"} \ No newline at end of file diff --git a/hydra_base/__init__.py b/hydra_base/__init__.py index 77209af0..f7094442 100644 --- a/hydra_base/__init__.py +++ b/hydra_base/__init__.py @@ -37,19 +37,6 @@ log.debug(" \n ") -log.debug("CONFIG localfiles %s found in %s", len(config.localfiles), config.localfile) - -log.debug("CONFIG repofiles %s found in %s", len(config.repofiles), config.repofile) - -log.debug("CONFIG userfiles %s found in %s", len(config.userfiles), config.userfile) - -log.debug("CONFIG sysfiles %s found in %s", len(config.sysfiles), config.sysfile) - -if len(config.sysfiles) + len(config.repofiles) + len(config.userfiles) + len(config.sysfiles) == 0: - log.critical("No config found. Please put your ini file into one of the files listed beside CONFIG above.") - -log.debug(" \n ") - from .lib.attributes import * from .lib.data import * from .lib.groups import * diff --git a/hydra_base/config.py b/hydra_base/config.py index 19370ca6..e77a9afd 100644 --- a/hydra_base/config.py +++ b/hydra_base/config.py @@ -67,6 +67,19 @@ def load_config(): global CONFIG logging.basicConfig(level='INFO') + from hydra_base.lib.hydraconfig import apply_configset, list_config_keys, config_key_get_value + from pprint import pprint + + with open("default_configset.json", 'r') as fp: + cs_json = fp.read() + + apply_configset(cs_json) + #keys = list_config_keys() + #kp = {k: config_key_get_value(k) for k in keys} + #pprint(kp) + + return + config = ConfigParser.ConfigParser(allow_no_value=True) modulepath = os.path.dirname(os.path.abspath(__file__)) @@ -175,8 +188,8 @@ def get(section, option, default=None): config_key_get_value ) - if CONFIG is None: - load_config() + #if CONFIG is None: + # load_config() try: return config_key_get_value(f"{section}_{option}") @@ -192,3 +205,6 @@ def getint(section, option, default=None): return CONFIG.getint(section, option) except: return default + + +load_config() diff --git a/hydra_base/db/model/project.py b/hydra_base/db/model/project.py index 07bc9d2d..8a759a08 100644 --- a/hydra_base/db/model/project.py +++ b/hydra_base/db/model/project.py @@ -25,7 +25,7 @@ from .attributes import Attr global project_cache_key -project_cache_key = config.get('cache', 'projectkey', 'userprojects') +project_cache_key = 'userprojects' __all__ = ['Project'] @@ -474,4 +474,4 @@ def get_hierarchy(self, user_id): if self.parent_id: project_hierarchy = project_hierarchy + self.parent.get_hierarchy(user_id) - return project_hierarchy \ No newline at end of file + return project_hierarchy diff --git a/hydra_base/hydra.ini b/hydra_base/hydra.ini deleted file mode 100644 index 2ee21f9a..00000000 --- a/hydra_base/hydra.ini +++ /dev/null @@ -1,113 +0,0 @@ -[DEFAULT] -#home_dir = ~ -log_level = INFO -hydra_aux_dir = %(home_dir)s/.hydra - -datetime_format = %Y-%m-%dT%H:%M:%S.%f000Z - -seasonal_key = 9999 -seasonal_year = 1678 - -[db] -instance = MySQL -upper_bound = 100 -lower_bound = 5 -export_target = %(hydra_aux_dir)s/audit -purge_threshold = 10000 -compression_threshold=50000 -#instance = SQLite - -[mysqld] -user = root -password = root -db_name = hydradb -server_name = 127.0.0.1 -#Y or N for True/False -auto_create = Y -pool_pre_ping=True -# Sqllite connection string -#url = sqlite:///%(hydra_aux_dir)s/hydra.db - -# Mysql connection string -#url = mysql+mysqldb://%(user)s:%(password)s@localhost/%(db_name)s -url = mysql+mysqldb://%(user)s:%(password)s@%(server_name)s/%(db_name)s - -[sqlite] -backup_dir = %(hydra_aux_dir)s/audit -dbfile = %(hydra_base_dir)s/HydraDB/hydra.db -backup_url = %(backup_dir)s/audit.db - -[mongodb] -host = localhost -port = 27017 -db_name = hydra -user = -passwd = -# collection for datasets -datasets = datasets -threshold = 4096 -direct_location_token = mongo_direct -value_location_key = value_storage_location - -[storage_hdf] -disable_hdf = False -hdf_filestore = /tmp - -[hydra_server] -domain = 127.0.0.1 -port = 8080 -path = soap #deprecated -json_path = json -http_path = http -soap_path = soap -#url = http://localhost:%()s?wsdl -url = http://%(domain)s:%(port)s/%(path)s?wsdl -layout_xsd_path = %(hydra_base_dir)s/static/resource_layout.xsd - -[hydra_client] -#url = http://ec2-54-229-95-247.eu-west-1.compute.amazonaws.com/hydra-server?wsdl -domain = http://127.0.0.1 -port = 8080 -path = json # deprecated -json_path = json -http_path = http -soap_path = soap -user = root -password = - -[filesys] -img_src = %(home_dir)s/.hydra/images/ -file_src = %(home_dir)s/.hydra/files/ - -[plugin] -default_directory = %(home_dir)s/.hydra/apps -queue_directory = %(default_directory)s/queue -upload_dir = %(default_directory)s/uploads -result_file = %(home_dir)s/.hydra/plugin_result/ -plugin_xsd_path = %(hydra_base_dir)s/static/plugin_input.xsd - -[unit_conversion] -user_file = %(hydra_base_dir)s/static/user_units.xml -default_file = %(hydra_base_dir)s/static/unit_definitions.xml - -[templates] -template_xsd_path = %(hydra_base_dir)s/static/template.xsd - -[logging_conf] -log_config_path = %(hydra_base_dir)s/logging.conf -log_file_dir = %(hydra_aux_dir)s/log - -[search] -page_size=2000 - -[polyvis] -POLYVIS_URL=http://localhost:5000/ - -[security] -max_login_attempts = 7 - -[cache] -type=diskcache - -[limits] -project_max_nest_depth = 32 diff --git a/hydra_base/lib/storage/hdfstorageadapter.py b/hydra_base/lib/storage/hdfstorageadapter.py index 35cb6280..faa4371f 100644 --- a/hydra_base/lib/storage/hdfstorageadapter.py +++ b/hydra_base/lib/storage/hdfstorageadapter.py @@ -87,8 +87,12 @@ def _get_anon(self): def get_hdf_config(config_key="storage_hdf", **kwargs): numeric = () boolean = ("disable_hdf", ) - hdf_keys = [k for k in config.CONFIG.options(config_key) if k not in config.CONFIG.defaults()] - hdf_items = {k: config.CONFIG.get(config_key, k) for k in hdf_keys} + + hdf_items = { + "disable_hdf": os.environ.get("HYDRA_DISABLE_HDF"), + "hdf_filestore": os.environ.get("HYDRA_HDF_FILESTORE") + } + for k in numeric: hdf_items[k] = int(hdf_items[k]) for k in boolean: diff --git a/hydra_base/lib/storage/mongostorageadapter.py b/hydra_base/lib/storage/mongostorageadapter.py index 629db9b5..ba088df2 100644 --- a/hydra_base/lib/storage/mongostorageadapter.py +++ b/hydra_base/lib/storage/mongostorageadapter.py @@ -1,4 +1,5 @@ import logging +import os from bson.objectid import ObjectId from pymongo import MongoClient @@ -37,8 +38,19 @@ def __init__(self): @staticmethod def get_mongo_config(config_key="mongodb"): numeric = ("threshold",) - mongo_keys = [k for k in config.CONFIG.options(config_key) if k not in config.CONFIG.defaults()] - mongo_items = {k: config.CONFIG.get(config_key, k) for k in mongo_keys} + + mongo_items = { + "host": os.environ.get("HYDRA_MONGO_HOST"), + "port": os.environ.get("HYDRA_MONGO_PORT"), + "user": os.environ.get("HYDRA_MONGO_USER"), + "passwd": os.environ.get("HYDRA_MONGO_PASSWD"), + "db_name": os.environ.get("HYDRA_MONGO_DB_NAME"), + "datasets": os.environ.get("HYDRA_MONGO_DATASETS"), + "threshold": os.environ.get("HYDRA_MONGO_THRESHOLD"), + "direct_location_token": os.environ.get("HYDRA_MONGO_DIRECT_LOCATION_TOKEN"), + "value_location_key": os.environ.get("HYDRA_MONGO_VALUE_LOCATION_KEY") + } + for k in numeric: mongo_items[k] = int(mongo_items[k]) diff --git a/hydra_base/util/configset.py b/hydra_base/util/configset.py index 615e8c6f..17ac8150 100644 --- a/hydra_base/util/configset.py +++ b/hydra_base/util/configset.py @@ -94,6 +94,8 @@ def apply_configset_to_db(self, state): 7. Verify state loaded in 3-5 matches input state 8. OK if so, else restore original state from 1 """ + if not db.DBSession: + db.connect() old_state = self.save_keys_to_configset() new_state = self.verify_configset(state) self._delete_all_keys() @@ -105,6 +107,7 @@ def apply_configset_to_db(self, state): # Otherwise restore state to that before call self._delete_all_keys() self.load_keys_from_state(old_state) + db.DBSession = None # Returns None on failure to update def _delete_all_keys(self): diff --git a/tests/conftest.py b/tests/conftest.py index 2d2b378c..52e1bb0d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,6 +20,11 @@ no_externaldb_opt = "--no-externaldb" externaldb_mark = "externaldb" requires_hdf_mark = "requires_hdf" +requires_replicaset_mark = "requires_replicaset" + +create_default_users_and_perms() +make_root_user() +create_default_units_and_dimensions() def pytest_addoption(parser): parser.addoption("--db-backend", action="store", default="sqlite", @@ -47,8 +52,10 @@ def pytest_collection_modifyitems(config, items): if externaldb_mark in item.keywords: item.add_marker(externaldb_skip) - conf_disabled = hydra_base.config.CONFIG.get("storage_hdf", "disable_hdf").lower() - hdf_disabled = True if conf_disabled in ("true", "yes") else False + truthlike = {"true", "yes", "y"} + + hdf_conf = hydra_base.config.get("storage_hdf", "disable_hdf").lower() + hdf_disabled = True if hdf_conf in truthlike else False hdf_skip = pytest.mark.skip(reason="Test not applicable when HDF support disabled") if hdf_disabled: for item in items: @@ -122,16 +129,12 @@ def client(connection_type, testdb_uri): test_server=null_server) client.login('root', '') - from hydra_base.util.migrate_config import ( - ini_to_configset, - make_config_from_schema - ) + from hydra_base.lib.hydraconfig import apply_configset + + with open("default_configset.json", 'r') as fp: + cs_json = fp.read() - ini_file = "hydra_base/hydra.ini" - schema = ini_to_configset(ini_file) - #pprint(schema) - config = make_config_from_schema(schema) - #keys = get_all_config_keys() + apply_configset(cs_json) client.testutils = testing.TestUtil(client) pytest.root_user_id = 1 From 181c72ac39b998bd2142a82b52a6506bd8ee57be Mon Sep 17 00:00:00 2001 From: Paul Slavin Date: Fri, 24 Jan 2025 16:28:32 +0000 Subject: [PATCH 20/28] Config value substitution; default ConfigSet; update conftest --- .github/workflows/ci.yml | 3 +- default_configset.json | 2 +- hydra_base/config.py | 71 ++++++++++++++++++++++++++++++++---- hydra_base/util/configset.py | 3 -- tests/conftest.py | 17 ++++++++- 5 files changed, 81 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8498d678..9ca3b1a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,8 +10,9 @@ jobs: COV_MIN: 70 # Minimum acceptable coverage level TEST_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + HYDRA_CONFIGSET: default_configset.json HYDRA_DB_SERVER: 127.0.0.1 - HYDRA_DB_NAME: hydra_base_test + HYDRA_DB_NAME: hydradb HYDRA_DB_USER: root HYDRA_DB_PASSWD: root HYDRA_DB_AUTOCREATE: Y diff --git a/default_configset.json b/default_configset.json index 55a891c5..405b7a1a 100644 --- a/default_configset.json +++ b/default_configset.json @@ -1 +1 @@ -{"name": "Default ConfigSet", "description": "The default Hydra startup state", "timestamp": "2025-01-23 15:57:19", "keys": {"log_level": {"type": "string", "value": "INFO", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_aux_dir": {"type": "string", "value": "/home/paul/.hydra", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "datetime_format": {"type": "string", "value": "%Y-%m-%dT%H:%M:%S.%f000Z", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "seasonal_key": {"type": "integer", "value": 9999, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "seasonal_year": {"type": "integer", "value": 1678, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "home_dir": {"type": "string", "value": "/home/paul", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_base_dir": {"type": "string", "value": "/home/paul/code/Hydra/hydra-base", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "db_instance": {"type": "string", "value": "MySQL", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "db_upper_bound": {"type": "integer", "value": 100, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "db_lower_bound": {"type": "integer", "value": 5, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "db_export_target": {"type": "string", "value": "/home/paul/.hydra/audit", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "db_purge_threshold": {"type": "integer", "value": 10000, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "db_compression_threshold": {"type": "integer", "value": 50000, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "sqlite_backup_dir": {"type": "string", "value": "/home/paul/.hydra/audit", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "sqlite_dbfile": {"type": "string", "value": "/home/paul/code/Hydra/hydra-base/HydraDB/hydra.db", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "sqlite_backup_url": {"type": "string", "value": "/home/paul/.hydra/audit/audit.db", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "mongodb_host": {"type": "string", "value": "localhost", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "mongodb_port": {"type": "integer", "value": 27017, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "mongodb_db_name": {"type": "string", "value": "hydra", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "mongodb_user": {"type": "string", "value": "", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "mongodb_passwd": {"type": "string", "value": "", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "mongodb_datasets": {"type": "string", "value": "datasets", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "mongodb_threshold": {"type": "integer", "value": 4096, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "mongodb_direct_location_token": {"type": "string", "value": "mongo_direct", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "mongodb_value_location_key": {"type": "string", "value": "value_storage_location", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "storage_hdf_disable_hdf": {"type": "string", "value": "False", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "storage_hdf_hdf_filestore": {"type": "string", "value": "/tmp", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_domain": {"type": "string", "value": "127.0.0.1", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_port": {"type": "integer", "value": 8080, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "hydra_server_path": {"type": "string", "value": "soap #deprecated", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_json_path": {"type": "string", "value": "json", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_http_path": {"type": "string", "value": "http", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_soap_path": {"type": "string", "value": "soap", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_url": {"type": "string", "value": "http://127.0.0.1:8080/soap #deprecated?wsdl", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_layout_xsd_path": {"type": "string", "value": "/home/paul/code/Hydra/hydra-base/static/resource_layout.xsd", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_domain": {"type": "string", "value": "http://127.0.0.1", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_port": {"type": "integer", "value": 8080, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "hydra_client_path": {"type": "string", "value": "json", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_json_path": {"type": "string", "value": "json", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_http_path": {"type": "string", "value": "http", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_soap_path": {"type": "string", "value": "soap", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_user": {"type": "string", "value": "root", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_password": {"type": "string", "value": "", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "filesys_img_src": {"type": "string", "value": "/home/paul/.hydra/images/", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "filesys_file_src": {"type": "string", "value": "/home/paul/.hydra/files/", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "plugin_default_directory": {"type": "string", "value": "/home/paul/.hydra/apps", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "plugin_queue_directory": {"type": "string", "value": "/home/paul/.hydra/apps/queue", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "plugin_upload_dir": {"type": "string", "value": "/home/paul/.hydra/apps/uploads", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "plugin_result_file": {"type": "string", "value": "/home/paul/.hydra/plugin_result/", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "plugin_plugin_xsd_path": {"type": "string", "value": "/home/paul/code/Hydra/hydra-base/static/plugin_input.xsd", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "unit_conversion_user_file": {"type": "string", "value": "/home/paul/code/Hydra/hydra-base/static/user_units.xml", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "unit_conversion_default_file": {"type": "string", "value": "/home/paul/code/Hydra/hydra-base/static/unit_definitions.xml", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "templates_template_xsd_path": {"type": "string", "value": "/home/paul/code/Hydra/hydra-base/static/template.xsd", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "logging_conf_log_config_path": {"type": "string", "value": "/home/paul/code/Hydra/hydra-base/logging.conf", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "logging_conf_log_file_dir": {"type": "string", "value": "/home/paul/.hydra/log", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "search_page_size": {"type": "integer", "value": 2000, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "polyvis_polyvis_url": {"type": "string", "value": "http://localhost:5000/", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "security_max_login_attempts": {"type": "integer", "value": 7, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "cache_type": {"type": "string", "value": "diskcache", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "limits_project_max_nest_depth": {"type": "integer", "value": 32, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}}, "digest": "af976b3f619d37be1c54220ce0dc4b1c2f162dbf017b02c260505b703baa0775"} \ No newline at end of file +{"name": "Default ConfigSet", "description": "The default Hydra startup state", "timestamp": "2025-01-24 15:15:28", "keys": {"log_level": {"type": "string", "value": "INFO", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_aux_dir": {"type": "string", "value": "__HOME_DIR__/.hydra", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "datetime_format": {"type": "string", "value": "%Y-%m-%dT%H:%M:%S.%f000Z", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "seasonal_key": {"type": "integer", "value": 9999, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "seasonal_year": {"type": "integer", "value": 1678, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "db_instance": {"type": "string", "value": "MySQL", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "db_upper_bound": {"type": "integer", "value": 100, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "db_lower_bound": {"type": "integer", "value": 5, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "db_export_target": {"type": "string", "value": "__HYDRA_AUX_DIR__/audit", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "db_purge_threshold": {"type": "integer", "value": 10000, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "db_compression_threshold": {"type": "integer", "value": 50000, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "sqlite_backup_dir": {"type": "string", "value": "__HYDRA_AUX_DIR__/audit", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "sqlite_dbfile": {"type": "string", "value": "__HYDRA_BASE_DIR__/HydraDB/hydra.db", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "sqlite_backup_url": {"type": "string", "value": "__SQLITE_BACKUP_DIR__/audit.db", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_domain": {"type": "string", "value": "127.0.0.1", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_port": {"type": "integer", "value": 8080, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "hydra_server_path": {"type": "string", "value": "soap", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_json_path": {"type": "string", "value": "json", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_http_path": {"type": "string", "value": "http", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_soap_path": {"type": "string", "value": "soap", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_url": {"type": "string", "value": "http://__HYDRA_SERVER_DOMAIN__:__HYDRA_SERVER_PORT__/__HYDRA_SERVER_PATH__?wsdl", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_layout_xsd_path": {"type": "string", "value": "__HYDRA_BASE_DIR__/static/resource_layout.xsd", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_domain": {"type": "string", "value": "http://127.0.0.1", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_port": {"type": "integer", "value": 8080, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "hydra_client_path": {"type": "string", "value": "json", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_json_path": {"type": "string", "value": "json", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_http_path": {"type": "string", "value": "http", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_soap_path": {"type": "string", "value": "soap", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_user": {"type": "string", "value": "root", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_password": {"type": "string", "value": "", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "filesys_img_src": {"type": "string", "value": "__HOME_DIR__/.hydra/images/", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "filesys_file_src": {"type": "string", "value": "__HOME_DIR__/.hydra/files/", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "unit_conversion_user_file": {"type": "string", "value": "__HYDRA_BASE_DIR__/static/user_units.xml", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "unit_conversion_default_file": {"type": "string", "value": "__HYDRA_BASE_DIR__/static/unit_definitions.xml", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "templates_template_xsd_path": {"type": "string", "value": "__HYDRA_BASE_DIR__/static/template.xsd", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "logging_conf_log_config_path": {"type": "string", "value": "__HYDRA_BASE_DIR__/logging.conf", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "logging_conf_log_file_dir": {"type": "string", "value": "__HYDRA_BASE_DIR__/log", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "search_page_size": {"type": "integer", "value": 2000, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "polyvis_polyvis_url": {"type": "string", "value": "http://localhost:5000/", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "security_max_login_attempts": {"type": "integer", "value": 7, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "cache_type": {"type": "string", "value": "diskcache", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "limits_project_max_nest_depth": {"type": "integer", "value": 32, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}}, "digest": "48050a66b5a9669ec6c41f0ffd09c22ec6b8fa6c32debd8bb3d3d7186d6220f2"} \ No newline at end of file diff --git a/hydra_base/config.py b/hydra_base/config.py index e77a9afd..c51b39c2 100644 --- a/hydra_base/config.py +++ b/hydra_base/config.py @@ -16,10 +16,12 @@ # You should have received a copy of the GNU Lesser General Public License # along with HydraPlatform. If not, see # -import os import glob +import os +import re import sys +from hydra_base import db PYTHONVERSION = sys.version_info @@ -67,13 +69,46 @@ def load_config(): global CONFIG logging.basicConfig(level='INFO') - from hydra_base.lib.hydraconfig import apply_configset, list_config_keys, config_key_get_value + from hydra_base.lib.hydraconfig import ( + apply_configset, + list_config_keys, + config_key_get_value, + config_key_set_value, + register_config_key + ) from pprint import pprint - with open("default_configset.json", 'r') as fp: - cs_json = fp.read() - - apply_configset(cs_json) + modulepath = os.path.dirname(os.path.abspath(__file__)) + home_dir = os.environ.get("HYDRA_HOME_DIR", '~') + hydra_base_dir = os.environ.get("HYDRA_BASE_DIR", modulepath) + configset = os.environ.get("HYDRA_CONFIGSET", "default_configset.json") + + + if not db.DBSession: + db.connect() + keys = list_config_keys() + if len(keys) == 0: + # No existing configset has been loaded + # Load set specified by env or default + # and register substitution keys + with open(configset, 'r') as fp: + cs_json = fp.read() + apply_configset(cs_json) + + try: + register_config_key("home_dir", "string") + config_key_set_value("home_dir", home_dir) + except Exception: + pass + + try: + register_config_key("hydra_base_dir", "string") + config_key_set_value("hydra_base_dir", hydra_base_dir) + except Exception: + pass + + CONFIG = True + #db.DBSession = None #keys = list_config_keys() #kp = {k: config_key_get_value(k) for k in keys} #pprint(kp) @@ -183,6 +218,22 @@ def get_startup_config(): db_config.update(read_env_startup_config()) return db_config +def make_value_substitutions(value): + if not isinstance(value, str): + return value + + p = r"__([a-zA-Z_]+)__" + tokens = re.findall(p, value) + for token in tokens: + try: + tkey = token.strip('_').lower() + tval = config_key_get_value(tkey) + value = value.replace(token, tval) + except Exception: + pass # Do not substitute invalid keys + + return value + def get(section, option, default=None): from hydra_base.lib.hydraconfig import ( config_key_get_value @@ -191,8 +242,13 @@ def get(section, option, default=None): #if CONFIG is None: # load_config() + #if option == "max_login_attempts": + # breakpoint() try: - return config_key_get_value(f"{section}_{option}") + value = config_key_get_value(f"{section}_{option}") + value = make_value_substitutions(value) + print(f"{section}_{option} = {value}") + return value except: return default @@ -207,4 +263,3 @@ def getint(section, option, default=None): return default -load_config() diff --git a/hydra_base/util/configset.py b/hydra_base/util/configset.py index 17ac8150..615e8c6f 100644 --- a/hydra_base/util/configset.py +++ b/hydra_base/util/configset.py @@ -94,8 +94,6 @@ def apply_configset_to_db(self, state): 7. Verify state loaded in 3-5 matches input state 8. OK if so, else restore original state from 1 """ - if not db.DBSession: - db.connect() old_state = self.save_keys_to_configset() new_state = self.verify_configset(state) self._delete_all_keys() @@ -107,7 +105,6 @@ def apply_configset_to_db(self, state): # Otherwise restore state to that before call self._delete_all_keys() self.load_keys_from_state(old_state) - db.DBSession = None # Returns None on failure to update def _delete_all_keys(self): diff --git a/tests/conftest.py b/tests/conftest.py index 52e1bb0d..939177e0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,6 +22,11 @@ requires_hdf_mark = "requires_hdf" requires_replicaset_mark = "requires_replicaset" +from hydra_base import db + +if not db.DBSession: + db.connect() + create_default_users_and_perms() make_root_user() create_default_units_and_dimensions() @@ -54,8 +59,11 @@ def pytest_collection_modifyitems(config, items): truthlike = {"true", "yes", "y"} - hdf_conf = hydra_base.config.get("storage_hdf", "disable_hdf").lower() - hdf_disabled = True if hdf_conf in truthlike else False + from hydra_base.lib.storage import HdfStorageAdapter + config = HdfStorageAdapter.get_hdf_config() + + hdf_disabled = config.get("disable_hdf") + #hdf_disabled = True if hdf_conf in truthlike else False hdf_skip = pytest.mark.skip(reason="Test not applicable when HDF support disabled") if hdf_disabled: for item in items: @@ -129,12 +137,17 @@ def client(connection_type, testdb_uri): test_server=null_server) client.login('root', '') + """ from hydra_base.lib.hydraconfig import apply_configset with open("default_configset.json", 'r') as fp: cs_json = fp.read() apply_configset(cs_json) + """ + from hydra_base.config import load_config + + load_config() client.testutils = testing.TestUtil(client) pytest.root_user_id = 1 From 995d0b4a022b49b7a189678f41c4df95336bfb1c Mon Sep 17 00:00:00 2001 From: Paul Slavin Date: Mon, 27 Jan 2025 17:14:43 +0000 Subject: [PATCH 21/28] New config.get format; CI patch for hydra_client --- .github/workflows/ci.yml | 6 +- hydra_base/config.py | 129 +++++++--------------------- hydra_base/db/__init__.py | 13 +-- hydra_base/db/audit.py | 11 +-- hydra_base/db/truncate.py | 27 +++--- hydra_base/hydra_logging.py | 4 +- hydra_base/lib/HydraTypes/Types.py | 4 +- hydra_base/lib/attributes.py | 2 +- hydra_base/lib/data.py | 2 +- hydra_base/lib/network.py | 4 +- hydra_base/lib/service.py | 12 +-- hydra_base/lib/static.py | 12 +-- hydra_base/lib/template/__init__.py | 2 +- hydra_base/lib/template/xml.py | 2 +- hydra_base/lib/users.py | 2 +- hydra_base/util/__init__.py | 4 +- hydra_base/util/hydra_dateutil.py | 8 +- hydra_base/util/testing.py | 2 +- tests/conftest.py | 2 +- tests/test_data.py | 2 +- 20 files changed, 97 insertions(+), 153 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ca3b1a8..0c9a8ad9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,11 @@ jobs: HYDRA_DB_USER: root HYDRA_DB_PASSWD: root HYDRA_DB_AUTOCREATE: Y - HYDRA_DB_PREPING: True + HYDRA_MYSQL_POOL_PREPING: True + HYDRA_MYSQL_POOL_SIZE: 10 + HYDRA_MYSQL_POOL_RECYCLE: 300 + HYDRA_MYSQL_POOL_TIMEOUT: 10 + HYDRA_MYSQL_MAX_OVERFLOW: 20 HYDRA_CACHETYPE: memcached HYDRA_CACHEHOST: 127.0.0.1 HYDRA_LOG_CONFPATH: hydra-base/logging.conf diff --git a/hydra_base/config.py b/hydra_base/config.py index c51b39c2..225a0cfa 100644 --- a/hydra_base/config.py +++ b/hydra_base/config.py @@ -115,82 +115,6 @@ def load_config(): return - config = ConfigParser.ConfigParser(allow_no_value=True) - - modulepath = os.path.dirname(os.path.abspath(__file__)) - - localfile = os.path.join(os.getcwd(), 'hydra.ini') - localfiles = glob.glob(localfile) - - repofile = os.path.join(modulepath, 'hydra.ini') - repofiles = glob.glob(repofile) - - if sys.platform.startswith("win"): - from hydra_base.util.windows import win_get_common_documents - userfile = os.path.join(os.path.expanduser('~'),'AppData','Local','hydra.ini') - userfiles = glob.glob(userfile) - - sysfile = os.path.join(win_get_common_documents(), 'Hydra','hydra.ini') - sysfiles = glob.glob(sysfile) - else: - userfile = os.path.join(os.path.expanduser('~'), '.hydra', 'hydra.ini') - userfiles = glob.glob(userfile) - - sysfile = os.path.join('etc','hydra','hydra.ini') - sysfiles = glob.glob(sysfile) - - - for ini_file in repofiles: - logging.debug("Repofile: %s"%ini_file) - config.read(ini_file) - for ini_file in sysfiles: - logging.debug("Sysfile: %s"%ini_file) - config.read(ini_file) - for ini_file in userfiles: - logging.debug("Userfile: %s"%ini_file) - config.read(ini_file) - for ini_file in localfiles: - logging.info("Localfile: %s"%ini_file) - config.read(ini_file) - - env_value = os.environ.get('HYDRA_CONFIG') - if env_value is not None: - if os.path.exists(env_value): - config.read(env_value) - else: - logging.warning('HYDRA_CONFIG set as %s but file does not exist', env_value) - - - try: - home_dir = config.get('DEFAULT', 'home_dir') - except: - home_dir = os.environ.get('HYDRA_HOME_DIR', '~') - config.set('DEFAULT', 'home_dir', os.path.expanduser(home_dir)) - - try: - hydra_base = config.get('DEFAULT', 'hydra_base_dir') - except: - hydra_base = os.environ.get('HYDRA_BASE_DIR', modulepath) - config.set('DEFAULT', 'hydra_base_dir', os.path.expanduser(hydra_base)) - - read_values_from_environment(config, 'mysqld', 'server_name') - - - CONFIG = config - - return config - -def read_values_from_environment(config, section_key, options_key): - ##################################### - # Settings for docker ENV variables # - ##################################### - env_var_name='HYDRA_DOCKER__' + section_key + '__' + options_key - - env_value = os.environ.get(env_var_name, '-') - if (env_value != '-'): - # Substitute the server_name with the end variable - # print("Presente") - config.set(section_key, options_key, env_value) def read_env_db_config(): return { @@ -199,7 +123,11 @@ def read_env_db_config(): "hydra_db_user": os.environ.get("HYDRA_DB_USER"), "hydra_db_passwd": os.environ.get("HYDRA_DB_PASSWD"), "hydra_db_autocreate": os.environ.get("HYDRA_DB_AUTOCREATE"), - "hydra_db_preping": os.environ.get("HYDRA_DB_PREPING") + "hydra_mysql_pool_preping": os.environ.get("HYDRA_MYSQL_POOL_PREPING"), + "hydra_mysql_pool_size": os.environ.get("HYDRA_MYSQL_POOL_SIZE"), + "hydra_mysql_pool_recycle": os.environ.get("HYDRA_MYSQL_POOL_RECYCLE"), + "hydra_mysql_pool_timeout": os.environ.get("HYDRA_MYSQL_POOL_TIMEOUT"), + "hydra_mysql_max_overflow": os.environ.get("HYDRA_MYSQL_MAX_OVERFLOW") } def read_env_startup_config(): @@ -234,32 +162,41 @@ def make_value_substitutions(value): return value -def get(section, option, default=None): +def get(*args, default=None): from hydra_base.lib.hydraconfig import ( config_key_get_value ) + """ + The section delineated below is a temporary + routine to allow calls from the hydra_client + module which use the old "section+option" + form of config.get to succeed. + This is required for tests to pass in CI + and should be removed on merge and update + of hydra_client. + """ + # Temporary CI adjustment begins + import inspect + sf = inspect.stack()[1] + mod = inspect.getmodule(sf[0]) + if mod.__name__.lower().startswith("hydra_client"): + if args[0].lower() == "default": + key = args[1] + else: + key = f"{args[0]}_{args[1]}" + if len(args) == 3: + default = args[2] + else: + key = args[0] + if len(args) == 2: + default = args[1] - #if CONFIG is None: - # load_config() + # Temporary CI adjustment ends - #if option == "max_login_attempts": - # breakpoint() try: - value = config_key_get_value(f"{section}_{option}") + value = config_key_get_value(key) value = make_value_substitutions(value) - print(f"{section}_{option} = {value}") + print(f"{key} = {value}") return value except: return default - -def getint(section, option, default=None): - - if CONFIG is None: - load_config() - - try: - return CONFIG.getint(section, option) - except: - return default - - diff --git a/hydra_base/db/__init__.py b/hydra_base/db/__init__.py index f1f34af6..02f8a1bd 100644 --- a/hydra_base/db/__init__.py +++ b/hydra_base/db/__init__.py @@ -113,8 +113,9 @@ def create_mysql_db(db_url): return db_url def connect(db_url=None): + db_config = config.get_startup_config() if db_url is None: - db_url = config.get_startup_config()["url"] + db_url = db_config["url"] log.info("Connecting to database") if db_url.find('@') >= 0: @@ -135,11 +136,11 @@ def connect(db_url=None): #These values MUST be smaller than the pool timeouts of the DB, otherwise the connection #will remain open on the client while it has been closed on the server, resulting in #an error - db_pool_size = int(config.get('mysqld', 'pool_size',10)) # 10 - db_pool_recycle = int(config.get('mysqld', 'pool_recycle', 300)) # 300 - db_max_overflow = int(config.get('mysqld', 'max_overflow', 20)) # 10 -> 30 - db_pool_timeout = int(config.get('mysqld', 'pool_timeout', 10)) - db_pool_pre_ping = True if config.get('mysqld', 'pool_pre_ping', 'Y').upper() == 'Y' else False + db_pool_size = int(db_config.get("hydra_mysql_pool_size" ,10)) # 10 + db_pool_recycle = int(db_config.get("hydra_mysql_pool_recycle", 300)) # 300 + db_max_overflow = int(db_config.get("hydra_mysql_max_overflow", 20)) # 10 -> 30 + db_pool_timeout = int(db_config.get("hydra_mysql_pool_timeout", 10)) + db_pool_pre_ping = True if db_config.get("hydra_mysql_pool_preping", "True").upper() == "TRUE" else False log.warning(f"db_pool_size: {db_pool_size} - pool_recycle: {db_pool_recycle} - max_overflow: {db_max_overflow} - pool_timeout: {db_pool_timeout} - pool_pre_ping: {db_pool_pre_ping}") diff --git a/hydra_base/db/audit.py b/hydra_base/db/audit.py index 860b014d..90c57243 100644 --- a/hydra_base/db/audit.py +++ b/hydra_base/db/audit.py @@ -34,8 +34,9 @@ from decimal import Decimal import os -engine_name = config.get('mysqld', 'url') -sqlite_engine = "sqlite:///%s"%(config.get('sqlite', 'backup_url')) +db_config = config.get_startup_config() +engine_name = db_config.get("url") +sqlite_engine = "sqlite:///%s"%(config.get("sqlite_backup_url")) def connect(): """ @@ -53,20 +54,20 @@ def create_sqlite_backup_db(audit_tables): #we always want to create a whole new DB, so delete the old one first #if it exists. try: - Popen("rm %s"%(config.get('sqlite', 'backup_url')), shell=True) + Popen("rm %s"%(config.get("sqlite_backup_url")), shell=True) logging.warn("Old sqlite backup DB removed") except Exception as e: logging.warn(e) try: - aux_dir = config.get('DEFAULT', 'hydra_aux_dir') + aux_dir = config.get("hydra_aux_dir") os.mkdir(aux_dir) logging.warn("%s created", aux_dir) except Exception as e: logging.warn(e) try: - backup_dir = config.get('db', 'export_target') + backup_dir = config.get("db_export_target") os.mkdir(backup_dir) logging.warn("%s created", backup_dir) except Exception as e: diff --git a/hydra_base/db/truncate.py b/hydra_base/db/truncate.py index 3e2da325..71430342 100644 --- a/hydra_base/db/truncate.py +++ b/hydra_base/db/truncate.py @@ -35,7 +35,8 @@ Base = declarative_base() -engine_name = config.get('mysqld', 'url') +db_config = config.get_startup_config() +engine_name = db_config.get("url") def connect_mysql(): """ @@ -46,10 +47,10 @@ def connect_mysql(): db = create_engine(engine_name) db.echo = True db.connect() - + return db -sqlite_engine = "sqlite:///%s"%(config.get('sqlite', 'backup_url')) +sqlite_engine = "sqlite:///%s"%(config.get("sqlite_backup_url")) def connect_sqlite(): @@ -69,7 +70,7 @@ def truncate_all_audit_tables(): # create a Session mysql_session = session1() - sqlite_session = session2() + sqlite_session = session2() mysql_metadata = MetaData(mysql_db) sqlite_metadata = MetaData(sqlite_db) @@ -93,7 +94,7 @@ def truncate_all_audit_tables(): continue export_table_to_sqlite(mysql_session, sqlite_session, sqlite_metadata, audit_table) truncate_table(mysql_session, table, audit_table) - + mysql_session.commit() sqlite_session.commit() logging.info("Truncation Complete") @@ -116,7 +117,7 @@ def truncate_table(session, table, audit_table): args = [] for arg in truncated_cols: args.append(arg == getattr(r, arg.name)) - + aud_id_col = audit_table.c['aud_id'] rs = session.query(aud_id_col).filter(and_(*args)).order_by(aud_id_col.desc())[3:] aud_ids = [str(r.aud_id) for r in rs] @@ -133,23 +134,23 @@ def export_table_to_csv(session, table, target=None): """ if target is None: - target_dir = os.path.join(config.get('db', 'export_target')) - target = os.path.join(config.get('db', 'export_target'), table.name) + target_dir = os.path.join(config.get("db_export_target")) + target = os.path.join(config.get("db_export_target"), table.name) if not os.path.exists(target_dir): os.mkdir(target_dir) if os.path.exists(target): target_file = open(target, 'r+') - + rs = session.query(table).all() - + entries_in_db = set() for r in rs: entries_in_db.add("%s"%(r.__repr__())) contents = set(target_file.read().split('\n')) - + new_data = entries_in_db.difference(contents) if len(new_data) > 0: @@ -180,10 +181,10 @@ def export_table_to_sqlite(mysql_session, sqlite_session, sqlite_metadata, audit entries_in_mysql_db = set(current_data) entries_in_sqlite_db = set(sqlite_data) - + new_data = list(entries_in_mysql_db.difference(entries_in_sqlite_db)) if len(new_data) > 0: - values = [] + values = [] for val in new_data: row = [] for i, v in enumerate(val): diff --git a/hydra_base/hydra_logging.py b/hydra_base/hydra_logging.py index 4f9b64fc..4f6a208b 100644 --- a/hydra_base/hydra_logging.py +++ b/hydra_base/hydra_logging.py @@ -26,7 +26,7 @@ def init(level=None): # if level is None: - # level = config.get('DEFAULT', 'log_level') + # level = config.get('log_level') # if os.name == "nt": # logging.addLevelName( logging.INFO, logging.getLevelName(logging.INFO)) @@ -38,7 +38,7 @@ def init(level=None): # return # if level is None: - # level = config.get('DEFAULT', 'log_level') + # level = config.get('log_level') # logging.addLevelName( logging.INFO, "\033[0;m%s\033[0;m" % logging.getLevelName(logging.INFO)) diff --git a/hydra_base/lib/HydraTypes/Types.py b/hydra_base/lib/HydraTypes/Types.py index abdf56e1..84e39af7 100644 --- a/hydra_base/lib/HydraTypes/Types.py +++ b/hydra_base/lib/HydraTypes/Types.py @@ -299,8 +299,8 @@ def fromDataset(cls, value, metadata=None): def validate(self): base_ts = pd.Timestamp("01-01-1970") #TODO: We need a more permanent solution to seasonal/repeating timeseries - seasonal_year = config.get('DEFAULT','seasonal_year', '1678') - seasonal_key = config.get('DEFAULT', 'seasonal_key', '9999') + seasonal_year = str(config.get("seasonal_year", "1678")) + seasonal_key = str(config.get("seasonal_key", "9999")) jd = json.loads(self.value, object_pairs_hook=collections.OrderedDict) for k,v in jd.items(): for date in (six.text_type(d) for d in v.keys()): diff --git a/hydra_base/lib/attributes.py b/hydra_base/lib/attributes.py index 2e3a6f87..53a691aa 100644 --- a/hydra_base/lib/attributes.py +++ b/hydra_base/lib/attributes.py @@ -349,7 +349,7 @@ def _reassign_scoped_attributes(attr_id, user_id): #first look up the hierarchy to see if there is an attribute scoped at a higher level. - max_levels = int(config.get("limits", "project_max_nest_depth", 32)) + max_levels = int(config.get("limits_project_max_nest_depth", 32)) attr_proj = db.DBSession.query(Project).filter(Project.id == attr_i.project_id).one() child_projects = attr_proj.get_child_projects(user_id=user_id, levels=max_levels) project_scope = {p["id"] for p in child_projects} | {attr_i.project_id} diff --git a/hydra_base/lib/data.py b/hydra_base/lib/data.py index a8a26b31..9e1ea8a8 100644 --- a/hydra_base/lib/data.py +++ b/hydra_base/lib/data.py @@ -262,7 +262,7 @@ def search_datasets(dataset_id=None, page_size)) if page_size is None: - page_size = config.get('SEARCH', 'page_size', 2000) + page_size = config.get("search_page_size", 2000) user_id = int(kwargs.get('user_id')) diff --git a/hydra_base/lib/network.py b/hydra_base/lib/network.py index 839abdc9..118c43db 100644 --- a/hydra_base/lib/network.py +++ b/hydra_base/lib/network.py @@ -197,7 +197,7 @@ def _bulk_add_resource_attrs(network_id, ref_key, resources, resource_name_map, ##the current user is validated, but some checks require admin permissions, ##so call as a user with all permissions - admin_id = config.get('DEFAULT', 'ALL_PERMISSION_USER', 1) + admin_id = config.get("ALL_PERMISSION_USER", 1) # template_lookup = {} #a lookup of all the templates used by the resource typeattr_lookup = {} # a lookup from type ID to a list of typeattrs @@ -898,7 +898,7 @@ def _get_all_templates(network_id, template_id): ##the current user is validated, but some checks require admin permissions, ##so call as a user with all permissions - admin_id = config.get('DEFAULT', 'ALL_PERMISSION_USER', 1) + admin_id = config.get("ALL_PERMISSION_USER", 1) for t in all_types: child_layout = None diff --git a/hydra_base/lib/service.py b/hydra_base/lib/service.py index 63044e23..2f16eb53 100644 --- a/hydra_base/lib/service.py +++ b/hydra_base/lib/service.py @@ -56,10 +56,10 @@ def login(username, password, **kwargs): hydra_session = session.Session( {}, #This is normally a request object, but in this case is empty - validate_key=config.get('COOKIES', 'VALIDATE_KEY', DEFAULT_VALIDATE_KEY), + validate_key=config.get("COOKIES_VALIDATE_KEY", DEFAULT_VALIDATE_KEY), type='file' if db.hydra_db_url.startswith('sqlite') else 'ext:sqla', cookie_expires=True, - data_dir=config.get('COOKIES', 'DATA_DIR', DEFAULT_DATA_DIR), + data_dir=config.get("COOKIES_DATA_DIR", DEFAULT_DATA_DIR), bind=db.engine, table=CACHE_TABLE ) @@ -85,10 +85,10 @@ def logout(session_id, **kwargs): hydra_session_object = session.SessionObject( {}, #This is normally a request object, but in this case is empty - validate_key=config.get('COOKIES', 'VALIDATE_KEY', DEFAULT_VALIDATE_KEY), + validate_key=config.get("COOKIES_VALIDATE_KEY", DEFAULT_VALIDATE_KEY), type='file' if db.hydra_db_url.startswith('sqlite') else 'ext:sqla', cookie_expires=True, - data_dir=config.get('COOKIES', 'DATA_DIR', DEFAULT_DATA_DIR), + data_dir=config.get("COOKIES_DATA_DIR", DEFAULT_DATA_DIR), bind=db.engine, table=CACHE_TABLE ) @@ -114,10 +114,10 @@ def get_session_user(session_id, **kwargs): hydra_session_object = session.SessionObject( {}, #This is normally a request object, but in this case is empty - validate_key=config.get('COOKIES', 'VALIDATE_KEY', DEFAULT_VALIDATE_KEY), + validate_key=config.get("COOKIES_VALIDATE_KEY", DEFAULT_VALIDATE_KEY), type='file' if db.hydra_db_url.startswith('sqlite') else 'ext:sqla', cookie_expires=True, - data_dir=config.get('COOKIES', 'DATA_DIR', DEFAULT_DATA_DIR), + data_dir=config.get("COOKIES_DATA_DIR", DEFAULT_DATA_DIR), bind=db.engine, table=CACHE_TABLE ) diff --git a/hydra_base/lib/static.py b/hydra_base/lib/static.py index f75c3565..1b50050a 100644 --- a/hydra_base/lib/static.py +++ b/hydra_base/lib/static.py @@ -25,7 +25,7 @@ log = logging.getLogger(__name__) def add_image(name, file,**kwargs): - path = config.get('filesys', 'img_src') + path = config.get("filesys_img_src") try: os.makedirs(path) except OSError: @@ -67,7 +67,7 @@ def add_image(name, file,**kwargs): return True def get_image(name,**kwargs): - path = config.get('filesys', 'img_src') + path = config.get("filesys_img_src") path = os.path.join(path, name) @@ -87,7 +87,7 @@ def get_image(name,**kwargs): return imageFile def remove_image(name,**kwargs): - path = config.get('filesys', 'img_src') + path = config.get("filesys_img_src") path = os.path.join(path, name) if(os.path.exists(path)): @@ -99,7 +99,7 @@ def remove_image(name,**kwargs): def add_file(resource_type, resource_id, name, file,**kwargs): - path = config.get('filesys', 'file_src') + path = config.get("filesys_file_src") path = os.path.join(path, resource_type) try: os.makedirs(path) @@ -148,7 +148,7 @@ def add_file(resource_type, resource_id, name, file,**kwargs): return True def get_file(resource_type, resource_id, name,**kwargs): - path = config.get('filesys', 'file_src') + path = config.get("filesys_file_src") path = os.path.join(path, resource_type, str(resource_id), name) @@ -170,7 +170,7 @@ def get_file(resource_type, resource_id, name,**kwargs): return file_to_send def remove_file(resource_type, resource_id, name,**kwargs): - path = config.get('filesys', 'file_src') + path = config.get("filesys_file_src") path = os.path.join(path, resource_type, str(resource_id), name) diff --git a/hydra_base/lib/template/__init__.py b/hydra_base/lib/template/__init__.py index 5ce859e9..1433c8c8 100644 --- a/hydra_base/lib/template/__init__.py +++ b/hydra_base/lib/template/__init__.py @@ -81,7 +81,7 @@ def _get_template_from_cache(template_id): now = datetime.datetime.now() #default the template timeout to a day -- they don't change often - timeout = datetime.timedelta(seconds=config.get('CACHE', 'CACHE_TIMEOUT', 86400)) + timeout = datetime.timedelta(seconds=config.get("CACHE_TIMEOUT", 86400)) cached_template = TEMPLATE_CACHE.get(template_id) if cached_template is None: diff --git a/hydra_base/lib/template/xml.py b/hydra_base/lib/template/xml.py index 4788850d..fd9c0e40 100644 --- a/hydra_base/lib/template/xml.py +++ b/hydra_base/lib/template/xml.py @@ -98,7 +98,7 @@ def import_template_xml(template_xml, allow_update=True, **kwargs): """ user_id = kwargs.get('user_id') - template_xsd_path = config.get('templates', 'template_xsd_path') + template_xsd_path = config.get("template_xsd_path") xmlschema_doc = etree.parse(template_xsd_path) xmlschema = etree.XMLSchema(xmlschema_doc) diff --git a/hydra_base/lib/users.py b/hydra_base/lib/users.py index a299f204..2432b6c3 100644 --- a/hydra_base/lib/users.py +++ b/hydra_base/lib/users.py @@ -208,7 +208,7 @@ def get_max_login_attempts(*args, **kwargs): a value of 0 will be returned and users will be unable to log in. """ - max_login_attempts = int(config.get("security", "max_login_attempts", 0)) + max_login_attempts = int(config.get("security_max_login_attempts", 0)) return max_login_attempts diff --git a/hydra_base/util/__init__.py b/hydra_base/util/__init__.py index 2661e417..f2a87da8 100644 --- a/hydra_base/util/__init__.py +++ b/hydra_base/util/__init__.py @@ -144,8 +144,8 @@ def get_val(dataset, timestamp=None): elif dataset.type == 'timeseries': #TODO: design a mechansim to retrieve this data if it's stored externally - seasonal_year = config.get('DEFAULT','seasonal_year', '1678') - seasonal_key = config.get('DEFAULT', 'seasonal_key', '9999') + seasonal_year = str(config.get("seasonal_year", 1678)) + seasonal_key = str(config.get("seasonal_key", 9999)) val = val.replace(seasonal_key, seasonal_year) timeseries = pd.read_json(val, convert_axes=True) diff --git a/hydra_base/util/hydra_dateutil.py b/hydra_base/util/hydra_dateutil.py index 050284a6..4a55f7af 100644 --- a/hydra_base/util/hydra_dateutil.py +++ b/hydra_base/util/hydra_dateutil.py @@ -162,7 +162,7 @@ def date_to_string(date, seasonal=False): recognised by Hydra as seasonal time stamp. """ - seasonal_key = config.get('DEFAULT', 'seasonal_key', '9999') + seasonal_key = str(config.get("seasonal_key", 9999)) if seasonal: FORMAT = seasonal_key+'-%m-%dT%H:%M:%S.%f' else: @@ -219,7 +219,7 @@ def guess_timefmt(datestr): if isinstance(datestr, float) or isinstance(datestr, int): return None - seasonal_key = str(config.get('DEFAULT', 'seasonal_key', '9999')) + seasonal_key = str(config.get("seasonal_key", "9999")) #replace 'T' with space to handle ISO times. if datestr.find('T') > 0: @@ -311,8 +311,8 @@ def reindex_timeseries(ts_string, new_timestamps): new_timestamps = new_timestamps_converted - seasonal_year = config.get('DEFAULT','seasonal_year', '1678') - seasonal_key = config.get('DEFAULT', 'seasonal_key', '9999') + seasonal_year = str(config.get("seasonal_year", 1678)) + seasonal_key = str(config.get("seasonal_key", 9999)) ts = ts_string.replace(seasonal_key, seasonal_year) diff --git a/hydra_base/util/testing.py b/hydra_base/util/testing.py index 8ac60b89..3a39c9c1 100644 --- a/hydra_base/util/testing.py +++ b/hydra_base/util/testing.py @@ -849,7 +849,7 @@ def create_timeseries(self, resource_attr, unit='m^3'): #with a resource attribute. #[[[1, 2, "hello"], [5, 4, 6]], [[10, 20, 30], [40, 50, 60]]] - fmt = hydra_base.config.get('DEFAULT', 'datetime_format', "%Y-%m-%dT%H:%M:%S.%f000Z") + fmt = hydra_base.config.get("datetime_format", "%Y-%m-%dT%H:%M:%S.%f000Z") t1 = datetime.datetime.now(datetime.timezone.utc) t2 = t1+datetime.timedelta(hours=1) diff --git a/tests/conftest.py b/tests/conftest.py index 939177e0..607cc016 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -96,7 +96,7 @@ def pytest_report_header(config): @pytest.fixture() def dateformat(): - return hydra_base.config.get('DEFAULT', 'datetime_format', "%Y-%m-%dT%H:%M:%S.%f000Z") + return hydra_base.config.get("datetime_format", "%Y-%m-%dT%H:%M:%S.%f000Z") @pytest.fixture(scope='module') def testdb_uri(db_backend): diff --git a/tests/test_data.py b/tests/test_data.py index 47a5ec38..53c9f349 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -251,7 +251,7 @@ def test_multiple_vals_at_time(self, client, network_with_data, seasonal_timeser val_a = json.loads(val_to_query.value) - dtformat = hb.config.get("datetime", "format", "%Y-%m-%dT%H:%M:%S.%f000Z") + dtformat = hb.config.get("datetime_format", "%Y-%m-%dT%H:%M:%S.%f000Z") fmt = datetime.datetime.strftime qry_times = [ fmt(datetime.datetime(2000, 1, 10, 00, 00, 00), dtformat), From 629940cb8b37d39a6b343ab714750e9ab2e08d6b Mon Sep 17 00:00:00 2001 From: Paul Slavin Date: Tue, 28 Jan 2025 11:33:15 +0000 Subject: [PATCH 22/28] Tidy config.py; Read config has from env --- .github/workflows/ci.yml | 1 + hydra_base/config.py | 58 +++------------- hydra_base/util/configset.py | 5 +- hydra_base/util/migrate_config.py | 26 ++----- tests/conftest.py | 12 +--- tests/templates/test_templates.py | 110 ------------------------------ 6 files changed, 22 insertions(+), 190 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c9a8ad9..96999b0f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,7 @@ jobs: HYDRA_MONGO_VALUE_LOCATION_KEY: value_storage_location HYDRA_DISABLE_HDF: False HYDRA_HDF_FILESTORE: /tmp + HYDRA_CONFIG_HASH_KEY: dev_only_secret_key strategy: matrix: diff --git a/hydra_base/config.py b/hydra_base/config.py index 225a0cfa..4b0f54ba 100644 --- a/hydra_base/config.py +++ b/hydra_base/config.py @@ -16,64 +16,26 @@ # You should have received a copy of the GNU Lesser General Public License # along with HydraPlatform. If not, see # -import glob +import logging import os import re import sys from hydra_base import db +from hydra_base.exceptions import HydraError -PYTHONVERSION = sys.version_info -if PYTHONVERSION >= (3,2): - import configparser as ConfigParser -else: - import ConfigParser -import logging - -global CONFIG CONFIG = None -global localfiles -global localfile -global repofile -global repofiles -global userfile -global userfiles -global sysfile -global sysfiles def load_config(): - """Load a config file. This function looks for a config (*.ini) file in the - following order:: - - (1) ./*.ini - (2) ~/.config/hydra/ - (3) /etc/hydra - (4) /path/to/hydra_base/*.ini - - (1) will override (2) will override (3) will override (4). Parameters not - defined in (1) will be taken from (2). Parameters not defined in (2) will - be taken from (3). (3) is the config folder that will be checked out from - the svn repository. (2) Will be be provided as soon as an installable - distribution is available. (1) will usually be written individually by - every user.""" - global localfiles - global localfile - global repofile - global repofiles - global userfile - global userfiles - global sysfile - global sysfiles global CONFIG logging.basicConfig(level='INFO') from hydra_base.lib.hydraconfig import ( apply_configset, - list_config_keys, - config_key_get_value, config_key_set_value, + list_config_keys, register_config_key ) from pprint import pprint @@ -108,11 +70,6 @@ def load_config(): pass CONFIG = True - #db.DBSession = None - #keys = list_config_keys() - #kp = {k: config_key_get_value(k) for k in keys} - #pprint(kp) - return @@ -130,14 +87,17 @@ def read_env_db_config(): "hydra_mysql_max_overflow": os.environ.get("HYDRA_MYSQL_MAX_OVERFLOW") } + def read_env_startup_config(): return { "hydra_cachetype": os.environ.get("HYDRA_CACHETYPE"), "hydra_cachehost": os.environ.get("HYDRA_CACHEHOST"), "hydra_log_confpath": os.environ.get("HYDRA_LOG_CONFPATH"), - "hydra_log_filedir": os.environ.get("HYDRA_LOG_FILEDIR") + "hydra_log_filedir": os.environ.get("HYDRA_LOG_FILEDIR"), + "hydra_config_hash_key": os.environ.get("HYDRA_CONFIG_HASH_KEY") } + def get_startup_config(): db_config = read_env_db_config() db_config["url"] = f"mysql+mysqldb://{db_config['hydra_db_user']}:{db_config['hydra_db_passwd']}"\ @@ -146,6 +106,7 @@ def get_startup_config(): db_config.update(read_env_startup_config()) return db_config + def make_value_substitutions(value): if not isinstance(value, str): return value @@ -157,11 +118,12 @@ def make_value_substitutions(value): tkey = token.strip('_').lower() tval = config_key_get_value(tkey) value = value.replace(token, tval) - except Exception: + except HydraError: pass # Do not substitute invalid keys return value + def get(*args, default=None): from hydra_base.lib.hydraconfig import ( config_key_get_value diff --git a/hydra_base/util/configset.py b/hydra_base/util/configset.py index 615e8c6f..e1ffa4c3 100644 --- a/hydra_base/util/configset.py +++ b/hydra_base/util/configset.py @@ -13,9 +13,10 @@ config_key_set_rule, config_key_set_description ) +from hydra_base.config import get_startup_config - -config_set_secret_key = b"dev_only_secret_key" +hash_key = get_startup_config()["hydra_config_hash_key"] +config_set_secret_key = bytes(hash_key, encoding="utf8") class ConfigSet: mac_hash = "sha256" diff --git a/hydra_base/util/migrate_config.py b/hydra_base/util/migrate_config.py index d21b4353..6703e5b3 100644 --- a/hydra_base/util/migrate_config.py +++ b/hydra_base/util/migrate_config.py @@ -6,15 +6,14 @@ import os import transaction -from pprint import pprint - from hydra_base import db from hydra_base.lib.hydraconfig import ( register_config_key, unregister_config_key, list_config_keys, config_key_set_value, - config_key_get_value + config_key_get_value, + export_config_as_json ) @@ -35,13 +34,13 @@ def make_db_config_schema(ini_filename): # set to allow for interpolation into later values home_dir = os.environ.get("HYDRA_HOME_DIR", '~') hydra_base_dir = os.environ.get("HYDRA_BASE_DIR", os.getcwd()) - config.set("DEFAULT", "home_dir", os.path.expanduser(home_dir)) - config.set("DEFAULT", "hydra_base_dir", os.path.expanduser(hydra_base_dir)) + #config.set("DEFAULT", "home_dir", os.path.expanduser(home_dir)) + #config.set("DEFAULT", "hydra_base_dir", os.path.expanduser(hydra_base_dir)) db_config_schema = {} for key in config["DEFAULT"]: try: - value = config["DEFAULT"][key] + value = config["DEFAULT"].get(key, raw=True) except configparser.InterpolationSyntaxError: value = config["DEFAULT"].get(key, raw=True) try: @@ -63,7 +62,8 @@ def make_db_config_schema(ini_filename): continue for key in config._sections[section].keys(): try: - value = config[section][key] + #value = config[section][key] + value = config[section].get(key, raw=True) except configparser.InterpolationSyntaxError: value = config[section].get(key, raw=True) key_name = f"{section}_{key}" @@ -103,15 +103,3 @@ def delete_all_config_keys(): unregister_config_key(key) transaction.commit() - - -if __name__ == "__main__": - #ini_file = "hydra_base/hydra.ini" - #schema = ini_to_configset(ini_file) - #pprint(schema) - #config = make_config_from_schema(schema) - #keys = get_all_config_keys() - #pprint(keys) - #delete_all_config_keys() - keys = get_all_config_keys() - pprint(keys) diff --git a/tests/conftest.py b/tests/conftest.py index 607cc016..03eb5cac 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,6 +15,7 @@ from hydra_base.util import testing from hydra_client.connection import JSONConnection, RemoteJSONConnection from hydra_base.lib.cache import clear_cache +from hydra_base.config import load_config no_externaldb_opt = "--no-externaldb" @@ -63,7 +64,6 @@ def pytest_collection_modifyitems(config, items): config = HdfStorageAdapter.get_hdf_config() hdf_disabled = config.get("disable_hdf") - #hdf_disabled = True if hdf_conf in truthlike else False hdf_skip = pytest.mark.skip(reason="Test not applicable when HDF support disabled") if hdf_disabled: for item in items: @@ -137,16 +137,6 @@ def client(connection_type, testdb_uri): test_server=null_server) client.login('root', '') - """ - from hydra_base.lib.hydraconfig import apply_configset - - with open("default_configset.json", 'r') as fp: - cs_json = fp.read() - - apply_configset(cs_json) - """ - from hydra_base.config import load_config - load_config() client.testutils = testing.TestUtil(client) diff --git a/tests/templates/test_templates.py b/tests/templates/test_templates.py index 15672564..63b5c78a 100644 --- a/tests/templates/test_templates.py +++ b/tests/templates/test_templates.py @@ -143,97 +143,6 @@ class TestTemplates: """ Test for templates """ - """ - def test_add_xml(self, template_json_object): - new_tmpl = template_json_object - - assert new_tmpl is not None, "Adding template from XML was not successful!" - - assert len(new_tmpl.templatetypes) == 2 - - for tt in new_tmpl.templatetypes: - if tt.name == 'Reservoir': - for ta in tt.typeattrs: - assert ta.data_type == 'scalar' - - assert tt.typeattrs[-1].properties is not None - assert eval(tt.typeattrs[-1].properties)['template_property'] == "Test property from template" - - return new_tmpl - - def test_get_xml(self, client, template_json_object): - xml_tmpl = template_json_object - - db_template = client.get_template_as_xml(xml_tmpl.id) - - - assert db_template is not None - - template_xsd_path = config.get('template_xsd_path') - xmlschema_doc = etree.parse(template_xsd_path) - - xmlschema = etree.XMLSchema(xmlschema_doc) - - xml_tree = etree.fromstring(db_template) - - xmlschema.assertValid(xml_tree) - - def test_get_dict(self, client, template_json_object): - - # Upload the xml file initally to avoid having to manage 2 template files - xml_tmpl = template_json_object - - template_dict = client.get_template_as_dict(xml_tmpl.id) - - # Error that there's already a template with this name. - with pytest.raises(HydraError): - client.import_template_dict(template_dict, allow_update=False) - - typename = template_dict['template']['templatetypes'][0]['name'] - - template_dict['template']['templatetypes'][0].name = typename + "_updated" - - # Finds a template with this name and updates it to mirror this dict. - # This includes deleting types if they're not in this dict. - # Changing the name of a type has this effect, as a new template does not have - # any reference to existing types in Hydra. - - updated_template = JSONObject(client.import_template_dict(template_dict)) - - type_names = [] - - for templatetype in updated_template.templatetypes: - type_names.append(templatetype.name) - - assert len(type_names) == 2 - assert typename + "_updated" in type_names and typename not in type_names - - # Now put it back to the original name so other tests will work - log.info("Reverting the type's name") - template_dict['template']['templatetypes'][0].name = typename - updated_template = JSONObject(client.import_template_dict(template_dict)) - - #just double-check that the JSON import works also - updated_template = JSONObject(client.import_template_json(json.dumps(template_dict))) - - type_names = [] - for templatetype in updated_template.templatetypes: - type_names.append(templatetype.name) - - assert len(type_names) == 2 - assert typename in type_names and typename + "_updated" not in type_names - - log.info("Checking to ensure Template has been updated correctly...") - # one final check to ensure that the type has been deleted - check_template_i = client.get_template(updated_template.id) - - assert len(check_template_i.templatetypes) == 2 - - """ - """ - TEMPLATES Functions - """ - def test_add_template(self, client, mock_template): link_attr_1 = client.testutils.create_attribute("link_attr_1", dimension='Pressure') @@ -821,25 +730,6 @@ def test_remove_type_from_resource(self, client, mock_template, network_with_dat assert updated_node_j.types is None or str(result1_j.id) not in [str(x.type_id) for x in updated_node_j.types] - """ - def test_create_template_from_network(self, client, network_with_data): - network = network_with_data - - net_template = client.get_network_as_xml_template(network.id) - - - assert net_template is not None - - template_xsd_path = config.get("templates", "template_xsd_path") - xmlschema_doc = etree.parse(template_xsd_path) - - xmlschema = etree.XMLSchema(xmlschema_doc) - - xml_tree = etree.fromstring(net_template) - - xmlschema.assertValid(xml_tree) - """ - def test_apply_template_to_network(self, client, mock_template, network_with_data): net_to_update = network_with_data template = mock_template From f65ad58f6565e98694dd72d5b6fced34405dcb16 Mon Sep 17 00:00:00 2001 From: Paul Slavin <3710230+pmslavin@users.noreply.github.com> Date: Thu, 2 Oct 2025 14:39:18 +0100 Subject: [PATCH 23/28] Resolve merge conflict in lib.cache --- hydra_base/lib/cache.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/hydra_base/lib/cache.py b/hydra_base/lib/cache.py index e3c6600b..92108bd6 100644 --- a/hydra_base/lib/cache.py +++ b/hydra_base/lib/cache.py @@ -12,16 +12,19 @@ log = logging.getLogger(__name__) global cache +startup_config = hydraconfig.get_startup_config() +cache_type = startup_config["hydra_cachetype"] +cache_host = startup_config["hydra_cachehost"] + def _init_diskcache(): log.info("Using diskcache for caching.") global cache import diskcache as dc cache = dc.Cache(tempfile.gettempdir()) -if hydraconfig.get('cache', 'type') != "memcached": +if cache_type != "memcached": _init_diskcache() - -elif hydraconfig.get('cache', 'type') == 'memcached': +elif cache_type == 'memcached': try: import pylibmc host = hydraconfig.get('cache', 'host', '127.0.0.1') From bd56dbed35b9483432cc410c7802ef7cdb91e6b1 Mon Sep 17 00:00:00 2001 From: Paul Slavin <3710230+pmslavin@users.noreply.github.com> Date: Thu, 2 Oct 2025 14:44:28 +0100 Subject: [PATCH 24/28] Resolve merge conflict in lib.cache --- hydra_base/lib/cache.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hydra_base/lib/cache.py b/hydra_base/lib/cache.py index 92108bd6..6858b3ae 100644 --- a/hydra_base/lib/cache.py +++ b/hydra_base/lib/cache.py @@ -27,9 +27,7 @@ def _init_diskcache(): elif cache_type == 'memcached': try: import pylibmc - host = hydraconfig.get('cache', 'host', '127.0.0.1') - port = hydraconfig.get('cache', 'port', 31211) - cache = pylibmc.Client([f"{host}:{port}"], binary=True) + cache = pylibmc.Client([f"{cache_host}:31211"], binary=True) # Check if Memcached server is reachable by setting a test key test_key = "__connection_test__" From dca82859fb1ec95674e508f1a0ad024f264b7d67 Mon Sep 17 00:00:00 2001 From: Paul Slavin <3710230+pmslavin@users.noreply.github.com> Date: Thu, 2 Oct 2025 14:57:01 +0100 Subject: [PATCH 25/28] Resolve merge conflict in test_templates --- tests/templates/test_templates.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/templates/test_templates.py b/tests/templates/test_templates.py index dc7ab714..55dfeb3d 100644 --- a/tests/templates/test_templates.py +++ b/tests/templates/test_templates.py @@ -35,15 +35,6 @@ def template(): return os.path.join(os.path.dirname(__file__), 'template.xml') -@pytest.fixture() -def template_json_object(client, template): - - with open(template) as fh: - file_contents = fh.read() - - return JSONObject(client.import_template_xml(file_contents)) - - @pytest.fixture() def mock_template(client): link_attr_1 = client.testutils.create_attribute("link_attr_1", dimension='Pressure') From 816fd5274dcfdaf17f536a2230f2eaac3a6c25a0 Mon Sep 17 00:00:00 2001 From: Paul Slavin <3710230+pmslavin@users.noreply.github.com> Date: Thu, 2 Oct 2025 15:06:05 +0100 Subject: [PATCH 26/28] Resolve merge conflict in lib.template --- hydra_base/lib/template/xml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hydra_base/lib/template/xml.py b/hydra_base/lib/template/xml.py index fd9c0e40..3f9d6edc 100644 --- a/hydra_base/lib/template/xml.py +++ b/hydra_base/lib/template/xml.py @@ -98,7 +98,7 @@ def import_template_xml(template_xml, allow_update=True, **kwargs): """ user_id = kwargs.get('user_id') - template_xsd_path = config.get("template_xsd_path") + template_xsd_path = config.get("templates_template_xsd_path") xmlschema_doc = etree.parse(template_xsd_path) xmlschema = etree.XMLSchema(xmlschema_doc) From 0c5197459fcf32cf9aefc04d33c794cebf1f3cc9 Mon Sep 17 00:00:00 2001 From: Paul Slavin <3710230+pmslavin@users.noreply.github.com> Date: Thu, 2 Oct 2025 15:06:51 +0100 Subject: [PATCH 27/28] Resolve merge conflict in test_templates --- tests/templates/test_templates.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/templates/test_templates.py b/tests/templates/test_templates.py index 55dfeb3d..17ae8b66 100644 --- a/tests/templates/test_templates.py +++ b/tests/templates/test_templates.py @@ -34,6 +34,13 @@ def template(): return os.path.join(os.path.dirname(__file__), 'template.xml') +@pytest.fixture() +def template_json_object(client, template): + + with open(template) as fh: + file_contents = fh.read() + + return JSONObject(client.import_template_xml(file_contents)) @pytest.fixture() def mock_template(client): From 96a05890994d45bca9a29ea58a034fc5a40b1726 Mon Sep 17 00:00:00 2001 From: Paul Slavin <3710230+pmslavin@users.noreply.github.com> Date: Thu, 2 Oct 2025 15:34:26 +0100 Subject: [PATCH 28/28] Resolve merge conflict in lib.template --- hydra_base/lib/template/xml.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hydra_base/lib/template/xml.py b/hydra_base/lib/template/xml.py index 3f9d6edc..2f331c69 100644 --- a/hydra_base/lib/template/xml.py +++ b/hydra_base/lib/template/xml.py @@ -18,6 +18,7 @@ import json import logging +import os from decimal import Decimal from lxml import etree @@ -98,7 +99,8 @@ def import_template_xml(template_xml, allow_update=True, **kwargs): """ user_id = kwargs.get('user_id') - template_xsd_path = config.get("templates_template_xsd_path") + hydra_base_dir = config.get("hydra_base_dir") + template_xsd_path = os.path.join(hydra_base_dir, "static/template.xsd") xmlschema_doc = etree.parse(template_xsd_path) xmlschema = etree.XMLSchema(xmlschema_doc)