diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18ece95a..bd80ab6d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,33 @@ jobs: COV_MIN: 70 # Minimum acceptable coverage level TEST_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + HYDRA_CONFIGSET: default_configset.json + HYDRA_DB_SERVER: 127.0.0.1 + HYDRA_DB_NAME: hydradb + HYDRA_DB_USER: root + HYDRA_DB_PASSWD: root + HYDRA_DB_AUTOCREATE: Y + HYDRA_MYSQL_POOL_PREPING: True + HYDRA_MYSQL_POOL_SIZE: 10 + HYDRA_MYSQL_POOL_RECYCLE: 300 + HYDRA_MYSQL_POOL_TIMEOUT: 10 + HYDRA_MYSQL_MAX_OVERFLOW: 20 + HYDRA_CACHETYPE: memcached + HYDRA_CACHEHOST: 127.0.0.1 + HYDRA_LOG_CONFPATH: hydra-base/logging.conf + HYDRA_LOG_FILEDIR: .hydra/log + HYDRA_MONGO_HOST: localhost + HYDRA_MONGO_PORT: 27017 + HYDRA_MONGO_DB_NAME: hydra + HYDRA_MONGO_USER: + HYDRA_MONGO_PASSWD: + HYDRA_MONGO_DATASETS: datasets + HYDRA_MONGO_THRESHOLD: 4096 + HYDRA_MONGO_DIRECT_LOCATION_TOKEN: mongo_direct + HYDRA_MONGO_VALUE_LOCATION_KEY: value_storage_location + HYDRA_DISABLE_HDF: False + HYDRA_HDF_FILESTORE: /tmp + HYDRA_CONFIG_HASH_KEY: dev_only_secret_key strategy: matrix: @@ -60,4 +87,4 @@ jobs: pip install -e . - name: Run pytests - run: pytest --db-backend=mysql --cov=hydra_base --cov-fail-under=$COV_MIN + run: pytest --db-backend=mysql --cov=hydra_base diff --git a/default_configset.json b/default_configset.json new file mode 100644 index 00000000..405b7a1a --- /dev/null +++ b/default_configset.json @@ -0,0 +1 @@ +{"name": "Default ConfigSet", "description": "The default Hydra startup state", "timestamp": "2025-01-24 15:15:28", "keys": {"log_level": {"type": "string", "value": "INFO", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_aux_dir": {"type": "string", "value": "__HOME_DIR__/.hydra", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "datetime_format": {"type": "string", "value": "%Y-%m-%dT%H:%M:%S.%f000Z", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "seasonal_key": {"type": "integer", "value": 9999, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "seasonal_year": {"type": "integer", "value": 1678, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "db_instance": {"type": "string", "value": "MySQL", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "db_upper_bound": {"type": "integer", "value": 100, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "db_lower_bound": {"type": "integer", "value": 5, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "db_export_target": {"type": "string", "value": "__HYDRA_AUX_DIR__/audit", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "db_purge_threshold": {"type": "integer", "value": 10000, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "db_compression_threshold": {"type": "integer", "value": 50000, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "sqlite_backup_dir": {"type": "string", "value": "__HYDRA_AUX_DIR__/audit", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "sqlite_dbfile": {"type": "string", "value": "__HYDRA_BASE_DIR__/HydraDB/hydra.db", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "sqlite_backup_url": {"type": "string", "value": "__SQLITE_BACKUP_DIR__/audit.db", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_domain": {"type": "string", "value": "127.0.0.1", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_port": {"type": "integer", "value": 8080, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "hydra_server_path": {"type": "string", "value": "soap", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_json_path": {"type": "string", "value": "json", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_http_path": {"type": "string", "value": "http", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_soap_path": {"type": "string", "value": "soap", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_url": {"type": "string", "value": "http://__HYDRA_SERVER_DOMAIN__:__HYDRA_SERVER_PORT__/__HYDRA_SERVER_PATH__?wsdl", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_layout_xsd_path": {"type": "string", "value": "__HYDRA_BASE_DIR__/static/resource_layout.xsd", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_domain": {"type": "string", "value": "http://127.0.0.1", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_port": {"type": "integer", "value": 8080, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "hydra_client_path": {"type": "string", "value": "json", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_json_path": {"type": "string", "value": "json", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_http_path": {"type": "string", "value": "http", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_soap_path": {"type": "string", "value": "soap", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_user": {"type": "string", "value": "root", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_password": {"type": "string", "value": "", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "filesys_img_src": {"type": "string", "value": "__HOME_DIR__/.hydra/images/", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "filesys_file_src": {"type": "string", "value": "__HOME_DIR__/.hydra/files/", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "unit_conversion_user_file": {"type": "string", "value": "__HYDRA_BASE_DIR__/static/user_units.xml", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "unit_conversion_default_file": {"type": "string", "value": "__HYDRA_BASE_DIR__/static/unit_definitions.xml", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "templates_template_xsd_path": {"type": "string", "value": "__HYDRA_BASE_DIR__/static/template.xsd", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "logging_conf_log_config_path": {"type": "string", "value": "__HYDRA_BASE_DIR__/logging.conf", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "logging_conf_log_file_dir": {"type": "string", "value": "__HYDRA_BASE_DIR__/log", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "search_page_size": {"type": "integer", "value": 2000, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "polyvis_polyvis_url": {"type": "string", "value": "http://localhost:5000/", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "security_max_login_attempts": {"type": "integer", "value": 7, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "cache_type": {"type": "string", "value": "diskcache", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "limits_project_max_nest_depth": {"type": "integer", "value": 32, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}}, "digest": "48050a66b5a9669ec6c41f0ffd09c22ec6b8fa6c32debd8bb3d3d7186d6220f2"} \ No newline at end of file diff --git a/hydra_base/__init__.py b/hydra_base/__init__.py index 859885ec..cebb6e94 100644 --- a/hydra_base/__init__.py +++ b/hydra_base/__init__.py @@ -45,25 +45,6 @@ log.debug(" \n ") -log.debug("CONFIG localfiles %s found in %s", len(config.localfiles), config.localfile) - -log.debug("CONFIG repofiles %s found in %s", len(config.repofiles), config.repofile) - -log.debug("CONFIG userfiles %s found in %s", len(config.userfiles), config.userfile) - -log.debug("CONFIG sysfiles %s found in %s", len(config.sysfiles), config.sysfile) - -if len(config.sysfiles) + len(config.repofiles) + len(config.userfiles) + len(config.sysfiles) == 0: - log.critical("No config found. Please put your ini file into one of the files listed beside CONFIG above.") - -if config.get("security", "max_login_attempts") is None: - """ Absence of max_login_attempts results in all users unable to log in, - so ensure this is defined in config, or fail. - """ - raise RuntimeError("Config 'security' section must define 'max_login_attempts'") - -log.debug(" \n ") - from .lib.attributes import * from .lib.data import * from .lib.groups import * @@ -80,3 +61,4 @@ from .lib.units import * from .lib.users import * from .lib.service import * +from .lib.hydraconfig import * diff --git a/hydra_base/config.py b/hydra_base/config.py index da835465..4b0f54ba 100644 --- a/hydra_base/config.py +++ b/hydra_base/config.py @@ -16,149 +16,149 @@ # You should have received a copy of the GNU Lesser General Public License # along with HydraPlatform. If not, see # +import logging import os -import glob +import re import sys -PYTHONVERSION = sys.version_info -if PYTHONVERSION >= (3,2): - import configparser as ConfigParser -else: - import ConfigParser -import logging +from hydra_base import db +from hydra_base.exceptions import HydraError + -global CONFIG CONFIG = None -global localfiles -global localfile -global repofile -global repofiles -global userfile -global userfiles -global sysfile -global sysfiles def load_config(): - """Load a config file. This function looks for a config (*.ini) file in the - following order:: - - (1) ./*.ini - (2) ~/.config/hydra/ - (3) /etc/hydra - (4) /path/to/hydra_base/*.ini - - (1) will override (2) will override (3) will override (4). Parameters not - defined in (1) will be taken from (2). Parameters not defined in (2) will - be taken from (3). (3) is the config folder that will be checked out from - the svn repository. (2) Will be be provided as soon as an installable - distribution is available. (1) will usually be written individually by - every user.""" - global localfiles - global localfile - global repofile - global repofiles - global userfile - global userfiles - global sysfile - global sysfiles global CONFIG logging.basicConfig(level='INFO') - config = ConfigParser.ConfigParser(allow_no_value=True) + from hydra_base.lib.hydraconfig import ( + apply_configset, + config_key_set_value, + list_config_keys, + register_config_key + ) + from pprint import pprint modulepath = os.path.dirname(os.path.abspath(__file__)) - - localfile = os.path.join(os.getcwd(), 'hydra.ini') - localfiles = glob.glob(localfile) - - repofile = os.path.join(modulepath, 'hydra.ini') - repofiles = glob.glob(repofile) - - if sys.platform.startswith("win"): - from hydra_base.util.windows import win_get_common_documents - userfile = os.path.join(os.path.expanduser('~'),'AppData','Local','hydra.ini') - userfiles = glob.glob(userfile) - - sysfile = os.path.join(win_get_common_documents(), 'Hydra','hydra.ini') - sysfiles = glob.glob(sysfile) - else: - userfile = os.path.join(os.path.expanduser('~'), '.hydra', 'hydra.ini') - userfiles = glob.glob(userfile) - - sysfile = os.path.join('etc','hydra','hydra.ini') - sysfiles = glob.glob(sysfile) - - - for ini_file in repofiles: - logging.debug("Repofile: %s"%ini_file) - config.read(ini_file) - for ini_file in sysfiles: - logging.debug("Sysfile: %s"%ini_file) - config.read(ini_file) - for ini_file in userfiles: - logging.debug("Userfile: %s"%ini_file) - config.read(ini_file) - for ini_file in localfiles: - logging.info("Localfile: %s"%ini_file) - config.read(ini_file) - - env_value = os.environ.get('HYDRA_CONFIG') - if env_value is not None: - if os.path.exists(env_value): - config.read(env_value) + home_dir = os.environ.get("HYDRA_HOME_DIR", '~') + hydra_base_dir = os.environ.get("HYDRA_BASE_DIR", modulepath) + configset = os.environ.get("HYDRA_CONFIGSET", "default_configset.json") + + + if not db.DBSession: + db.connect() + keys = list_config_keys() + if len(keys) == 0: + # No existing configset has been loaded + # Load set specified by env or default + # and register substitution keys + with open(configset, 'r') as fp: + cs_json = fp.read() + apply_configset(cs_json) + + try: + register_config_key("home_dir", "string") + config_key_set_value("home_dir", home_dir) + except Exception: + pass + + try: + register_config_key("hydra_base_dir", "string") + config_key_set_value("hydra_base_dir", hydra_base_dir) + except Exception: + pass + + CONFIG = True + return + + +def read_env_db_config(): + return { + "hydra_db_server": os.environ.get("HYDRA_DB_SERVER"), + "hydra_db_name": os.environ.get("HYDRA_DB_NAME"), + "hydra_db_user": os.environ.get("HYDRA_DB_USER"), + "hydra_db_passwd": os.environ.get("HYDRA_DB_PASSWD"), + "hydra_db_autocreate": os.environ.get("HYDRA_DB_AUTOCREATE"), + "hydra_mysql_pool_preping": os.environ.get("HYDRA_MYSQL_POOL_PREPING"), + "hydra_mysql_pool_size": os.environ.get("HYDRA_MYSQL_POOL_SIZE"), + "hydra_mysql_pool_recycle": os.environ.get("HYDRA_MYSQL_POOL_RECYCLE"), + "hydra_mysql_pool_timeout": os.environ.get("HYDRA_MYSQL_POOL_TIMEOUT"), + "hydra_mysql_max_overflow": os.environ.get("HYDRA_MYSQL_MAX_OVERFLOW") + } + + +def read_env_startup_config(): + return { + "hydra_cachetype": os.environ.get("HYDRA_CACHETYPE"), + "hydra_cachehost": os.environ.get("HYDRA_CACHEHOST"), + "hydra_log_confpath": os.environ.get("HYDRA_LOG_CONFPATH"), + "hydra_log_filedir": os.environ.get("HYDRA_LOG_FILEDIR"), + "hydra_config_hash_key": os.environ.get("HYDRA_CONFIG_HASH_KEY") + } + + +def get_startup_config(): + db_config = read_env_db_config() + db_config["url"] = f"mysql+mysqldb://{db_config['hydra_db_user']}:{db_config['hydra_db_passwd']}"\ + f"@{db_config['hydra_db_server']}/{db_config['hydra_db_name']}" + + db_config.update(read_env_startup_config()) + return db_config + + +def make_value_substitutions(value): + if not isinstance(value, str): + return value + + p = r"__([a-zA-Z_]+)__" + tokens = re.findall(p, value) + for token in tokens: + try: + tkey = token.strip('_').lower() + tval = config_key_get_value(tkey) + value = value.replace(token, tval) + except HydraError: + pass # Do not substitute invalid keys + + return value + + +def get(*args, default=None): + from hydra_base.lib.hydraconfig import ( + config_key_get_value + ) + """ + The section delineated below is a temporary + routine to allow calls from the hydra_client + module which use the old "section+option" + form of config.get to succeed. + This is required for tests to pass in CI + and should be removed on merge and update + of hydra_client. + """ + # Temporary CI adjustment begins + import inspect + sf = inspect.stack()[1] + mod = inspect.getmodule(sf[0]) + if mod.__name__.lower().startswith("hydra_client"): + if args[0].lower() == "default": + key = args[1] else: - logging.warning('HYDRA_CONFIG set as %s but file does not exist', env_value) - - - try: - home_dir = config.get('DEFAULT', 'home_dir') - except: - home_dir = os.environ.get('HYDRA_HOME_DIR', '~') - config.set('DEFAULT', 'home_dir', os.path.expanduser(home_dir)) - - try: - hydra_base = config.get('DEFAULT', 'hydra_base_dir') - except: - hydra_base = os.environ.get('HYDRA_BASE_DIR', modulepath) - config.set('DEFAULT', 'hydra_base_dir', os.path.expanduser(hydra_base)) - - read_values_from_environment(config, 'mysqld', 'server_name') - - - CONFIG = config - - return config - -def read_values_from_environment(config, section_key, options_key): - ##################################### - # Settings for docker ENV variables # - ##################################### - env_var_name='HYDRA_DOCKER__' + section_key + '__' + options_key - - env_value = os.environ.get(env_var_name, '-') - if (env_value != '-'): - # Substitute the server_name with the end variable - # print("Presente") - config.set(section_key, options_key, env_value) - - -def get(section, option, default=None): - - if CONFIG is None: - load_config() - - try: - return CONFIG.get(section, option) - except: - return default - -def getint(section, option, default=None): + key = f"{args[0]}_{args[1]}" + if len(args) == 3: + default = args[2] + else: + key = args[0] + if len(args) == 2: + default = args[1] - if CONFIG is None: - load_config() + # Temporary CI adjustment ends try: - return CONFIG.getint(section, option) + value = config_key_get_value(key) + value = make_value_substitutions(value) + print(f"{key} = {value}") + return value except: return default diff --git a/hydra_base/db/__init__.py b/hydra_base/db/__init__.py index 57119e59..02f8a1bd 100644 --- a/hydra_base/db/__init__.py +++ b/hydra_base/db/__init__.py @@ -81,12 +81,13 @@ def create_mysql_db(db_url): #Remove trailing whitespace and forwardslashes db_url = db_url.strip().strip('/') + db_config = config.get_startup_config() #Check this is a mysql URL if db_url.find('mysql') >= 0: #Get the DB name from config and check if it's in the URL - db_name = config.get('mysqld', 'db_name', 'hydradb') + db_name = db_config["hydra_db_name"] if db_url.find(db_name) >= 0: no_db_url = db_url.rsplit("/", 1)[0] else: @@ -104,7 +105,7 @@ def create_mysql_db(db_url): if db_url.find('charset') == -1: db_url = "{}?charset=utf8&use_unicode=1".format(db_url) - if config.get('mysqld', 'auto_create', 'Y') == 'Y': + if db_config.get("hydra_db_autocreate", 'Y') == 'Y': tmp_engine = create_engine(no_db_url) log.debug("Creating database {0} as it does not exist.".format(db_name)) with tmp_engine.connect() as conn: @@ -112,8 +113,9 @@ def create_mysql_db(db_url): return db_url def connect(db_url=None): + db_config = config.get_startup_config() if db_url is None: - db_url = config.get('mysqld', 'url') + db_url = db_config["url"] log.info("Connecting to database") if db_url.find('@') >= 0: @@ -134,11 +136,11 @@ def connect(db_url=None): #These values MUST be smaller than the pool timeouts of the DB, otherwise the connection #will remain open on the client while it has been closed on the server, resulting in #an error - db_pool_size = int(config.get('mysqld', 'pool_size',10)) # 10 - db_pool_recycle = int(config.get('mysqld', 'pool_recycle', 300)) # 300 - db_max_overflow = int(config.get('mysqld', 'max_overflow', 20)) # 10 -> 30 - db_pool_timeout = int(config.get('mysqld', 'pool_timeout', 10)) - db_pool_pre_ping = True if config.get('mysqld', 'pool_pre_ping', 'Y').upper() == 'Y' else False + db_pool_size = int(db_config.get("hydra_mysql_pool_size" ,10)) # 10 + db_pool_recycle = int(db_config.get("hydra_mysql_pool_recycle", 300)) # 300 + db_max_overflow = int(db_config.get("hydra_mysql_max_overflow", 20)) # 10 -> 30 + db_pool_timeout = int(db_config.get("hydra_mysql_pool_timeout", 10)) + db_pool_pre_ping = True if db_config.get("hydra_mysql_pool_preping", "True").upper() == "TRUE" else False log.warning(f"db_pool_size: {db_pool_size} - pool_recycle: {db_pool_recycle} - max_overflow: {db_max_overflow} - pool_timeout: {db_pool_timeout} - pool_pre_ping: {db_pool_pre_ping}") diff --git a/hydra_base/db/audit.py b/hydra_base/db/audit.py index 860b014d..90c57243 100644 --- a/hydra_base/db/audit.py +++ b/hydra_base/db/audit.py @@ -34,8 +34,9 @@ from decimal import Decimal import os -engine_name = config.get('mysqld', 'url') -sqlite_engine = "sqlite:///%s"%(config.get('sqlite', 'backup_url')) +db_config = config.get_startup_config() +engine_name = db_config.get("url") +sqlite_engine = "sqlite:///%s"%(config.get("sqlite_backup_url")) def connect(): """ @@ -53,20 +54,20 @@ def create_sqlite_backup_db(audit_tables): #we always want to create a whole new DB, so delete the old one first #if it exists. try: - Popen("rm %s"%(config.get('sqlite', 'backup_url')), shell=True) + Popen("rm %s"%(config.get("sqlite_backup_url")), shell=True) logging.warn("Old sqlite backup DB removed") except Exception as e: logging.warn(e) try: - aux_dir = config.get('DEFAULT', 'hydra_aux_dir') + aux_dir = config.get("hydra_aux_dir") os.mkdir(aux_dir) logging.warn("%s created", aux_dir) except Exception as e: logging.warn(e) try: - backup_dir = config.get('db', 'export_target') + backup_dir = config.get("db_export_target") os.mkdir(backup_dir) logging.warn("%s created", backup_dir) except Exception as e: diff --git a/hydra_base/db/model/hydraconfig/__init__.py b/hydra_base/db/model/hydraconfig/__init__.py new file mode 100644 index 00000000..4cf6bea0 --- /dev/null +++ b/hydra_base/db/model/hydraconfig/__init__.py @@ -0,0 +1 @@ +from .hydraconfig import * diff --git a/hydra_base/db/model/hydraconfig/hydraconfig.py b/hydra_base/db/model/hydraconfig/hydraconfig.py new file mode 100644 index 00000000..bc104741 --- /dev/null +++ b/hydra_base/db/model/hydraconfig/hydraconfig.py @@ -0,0 +1,236 @@ +""" + Types and definitions for Hydra config values +""" +import base64 +import binascii + +from hydra_base import db +from hydra_base.db.model.base import * +from hydra_base.exceptions import HydraError + +from hydra_base.db.model.hydraconfig.validators import ( + ConfigKeyIntegerValidator, + ConfigKeyStringValidator +) + +from sqlalchemy.orm import ( + Mapped, + mapped_column, + reconstructor +) + + +__all__ = ["ConfigKey", "config_key_type_map", "ConfigGroup", "ConfigGroupKeys"] + +config_key_type_map = {} + + +class ConfigKey(Base): + __tablename__ = "tConfigKey" + + key_name_max_length = 200 + key_type_tag_max_length = 40 + key_desc_max_length = 1000 + + id = Column(Integer(), primary_key=True, nullable=False) + name = Column(String(key_name_max_length), nullable=False, unique=True) + description = Column(String(key_desc_max_length), nullable=True, unique=False) + type = Column(String(key_type_tag_max_length)) + rules = Column(String(200), default='{}') + + + __mapper_args__ = { + "polymorphic_on": type, + "polymorphic_identity": "configkey" + } + + subclass_type_key = "config_key_type" + + @classmethod + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + keyattr = __class__.subclass_type_key + configkey_type = getattr(cls, keyattr, None) + if not configkey_type or not isinstance(configkey_type, str): + raise NotImplementedError(f"ConfigKey subclass {cls.__name__} does not define a '{keyattr}' attribute") + config_key_type_map[configkey_type] = cls + log.debug(f"Registered ConfigKey type '{cls.__name__}' with key '{configkey_type}'") + + def __init__(self, name, desc=None): + if not name: + raise ValueError(f"ConfigKey requires a valid name, not '{name}'") + + self.name = name + self.description = desc if desc else "" + + if vcls := getattr(self.__class__, "validator_type", None): + self.validator = vcls() + self.validator.key = self + + @reconstructor + def load_state(self): + if vcls := getattr(self.__class__, "validator_type", None): + self.validator = vcls(self.rules) + self.validator.key = self + + @hybrid_property + def group(self): + if self._group: + group = db.DBSession.query(ConfigGroup).filter(ConfigGroup.id == self._group.group_id).one() + return group.name + + +class HasValue: + value_max_length = 2000 + _value: Mapped[str] = mapped_column(String(value_max_length), nullable=True, use_existing_column=True) + + + +class ConfigKey_Integer(ConfigKey, HasValue): + config_key_type = "integer" + validator_type = ConfigKeyIntegerValidator + + __mapper_args__ = { + "polymorphic_identity": config_key_type + } + + + def __init__(self, name, desc=None): + super().__init__(name, desc) + + @hybrid_property + def value(self): + if self._value is None: + return None + + return int(self._value) + + @value.setter + def value(self, val): + if val is None: + return + + try: + _ = int(val) + except (TypeError, ValueError): + raise HydraError(f"Config Key {self.name} requires an integer value, not {val}") + + if validator := getattr(self, "validator", None): + validator.validate(val) + + self._value = val + + +class ConfigKey_String(ConfigKey, HasValue): + config_key_type = "string" + validator_type = ConfigKeyStringValidator + + __mapper_args__ = { + "polymorphic_identity": config_key_type + } + + def __init__(self, name, desc=None): + super().__init__(name, desc) + + @hybrid_property + def value(self): + return str(self._value) + + @value.setter + def value(self, val): + if validator := getattr(self, "validator", None): + validator.validate(str(val)) + + self._value = str(val) + + +class ConfigKey_Boolean(ConfigKey, HasValue): + config_key_type = "boolean" + + __mapper_args__ = { + "polymorphic_identity": config_key_type + } + + def __init__(self, name, desc=None): + super().__init__(name, desc) + + @hybrid_property + def value(self): + return self._value == "True" + + @value.setter + def value(self, val): + if val not in (True, False): + raise HydraError(f"Config Key {self.name} with Boolean type accepts only True or False value") + + self._value = "True" if val else "False" + + +class ConfigKey_Base64(ConfigKey, HasValue): + config_key_type = "base64" + + __mapper_args__ = { + "polymorphic_identity": config_key_type + } + + def __init__(self, name, desc=None): + super().__init__(name, desc) + + @hybrid_property + def value(self): + return self._value + + @value.setter + def value(self, val): + try: + _ = base64.b64decode(val, validate=True) + except (AttributeError, TypeError, binascii.Error) as e: + raise HydraError(f"Config Key {self.name} with Base64 " + f"type accepts only valid Base64 strings") from e + + self._value = val + + +class ConfigKey_Uri(ConfigKey, HasValue): + config_key_type = "uri" + + __mapper_args__ = { + "polymorphic_identity": config_key_type + } + # Includes paths with file:// scheme + + +class ConfigKey_Json(ConfigKey, HasValue): + config_key_type = "json" + + __mapper_args__ = { + "polymorphic_identity": config_key_type + } + # Unvalidated json object + + +class ConfigGroup(Base): + __tablename__ = "tConfigGroup" + + group_name_max_length = 200 + group_desc_max_length = 2000 + + id = Column(Integer(), primary_key=True, nullable=False) + name = Column(String(group_name_max_length), nullable=False, unique=True) + description = Column(String(group_desc_max_length), nullable=True, unique=False) + + @hybrid_property + def keys(self): + member_key_ids = {k.key_id for k in self._keys} + member_keys = db.DBSession.query(ConfigKey).filter(ConfigKey.id.in_(member_key_ids)).all() + return [key.name for key in member_keys] + + +class ConfigGroupKeys(Base): + __tablename__ = "tConfigGroupKeys" + + group_id = Column(Integer(), ForeignKey("tConfigGroup.id"), primary_key=True, nullable=False) + key_id = Column(Integer(), ForeignKey("tConfigKey.id"), primary_key=True, nullable=False) + + keys = relationship("ConfigGroup", backref=backref("_keys", uselist=True, cascade="all, delete-orphan")) + group = relationship("ConfigKey", backref=backref("_group", uselist=False, cascade="all, delete-orphan")) diff --git a/hydra_base/db/model/hydraconfig/validators.py b/hydra_base/db/model/hydraconfig/validators.py new file mode 100644 index 00000000..396a0057 --- /dev/null +++ b/hydra_base/db/model/hydraconfig/validators.py @@ -0,0 +1,133 @@ +import json + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from json import JSONDecodeError + +from hydra_base.exceptions import HydraError + + +class KeyValidator(ABC): + @dataclass + class Rule(): + name: str + description: str + value: type + + class ValidatorEncoder(json.JSONEncoder): + def default(self, inst): + return {r.name: r.value for r in inst.rules} + + def __init__(self, rules, rules_spec): + self._rules = {} + for rule in rules: + if rule["name"] in self._rules: + raise HydraError(f"Rule {rule['name']} already exists") + + new_rule = KeyValidator.Rule(rule["name"], rule["description"], rule["value"]) + self._rules[new_rule.name] = new_rule + + if rules_spec is not None: + if isinstance(rules_spec, str): + try: + rvi = json.loads(rules_spec) + except JSONDecodeError as e: + raise HydraError(f"Invalid rules_spec: {rules_spec}") from e + for rule_name, rule_value in rvi.items(): + self.set_rule(rule_name, rule_value) + else: + raise TypeError(f"Invalid Rules specification: {rules_spec}") + + def set_rule(self, name, value): + rule = self._get_rule(name) + rule.value = value + if parent := getattr(self, "key", None): + parent.rules = self.as_json + + def clear_rule(self, name): + rule = self._get_rule(name) + rule.value = None + if parent := getattr(self, "key", None): + parent.rules = self.as_json + + def _get_rule(self, name): + if not (rule := self._rules.get(name)): + raise ValueError(f"No rule named '{name}' defined") + + return rule + + @property + def active_rules(self): + return {rule.name: rule for rule in self._rules.values() if rule.value is not None} + + @property + def rules(self): + return self._rules.values() + + @property + def as_json(self): + return json.dumps(self, cls=self.ValidatorEncoder) + + @property + def key(self): + if hasattr(self, "_key"): + return self._key + + @key.setter + def key(self, key): + self._key = key + self._key.rules = self.as_json + + @abstractmethod + def validate(self): + pass + + +class ConfigKeyIntegerValidator(KeyValidator): + rule_types = [ + {"name": "max_value", + "description": "The maximum integer value of this key", + "value": None}, + {"name": "min_value", + "description": "The minimum integer value of this key", + "value": None} + ] + + def __init__(self, rules_spec=None): + super().__init__(self.__class__.rule_types, rules_spec) + + + def validate(self, value): + if parent := getattr(self, "key", None): + err_prefix = f"ConfigKey {parent.name}: " + else: + err_prefix = "" + + if max_value := self.active_rules.get("max_value"): + if value > max_value.value: + raise ValueError(f"{err_prefix}value of {value} exceeds maximum of {max_value.value}") + if min_value := self.active_rules.get("min_value"): + if value < min_value.value: + raise ValueError(f"{err_prefix}value of {value} beneath minimum of {min_value.value}") + + +class ConfigKeyStringValidator(KeyValidator): + rule_types = [ + {"name": "max_length", + "description": "The maximum length of this key's string value", + "value": None}, + {"name": "min_length", + "description": "The minimum length of this key's string value", + "value": None}, + ] + + def __init__(self, rules_spec=None): + super().__init__(self.__class__.rule_types, rules_spec) + + def validate(self, value): + if max_length := self.active_rules.get("max_length"): + if len(value) > max_length.value: + raise ValueError("over max") + if min_length := self.active_rules.get("min_length"): + if len(value) < min_length.value: + raise ValueError("under min") diff --git a/hydra_base/db/model/project.py b/hydra_base/db/model/project.py index 6dfd77d9..e252216b 100644 --- a/hydra_base/db/model/project.py +++ b/hydra_base/db/model/project.py @@ -25,7 +25,7 @@ from .attributes import Attr global project_cache_key -project_cache_key = config.get('cache', 'projectkey', 'userprojects') +project_cache_key = 'userprojects' __all__ = ['Project'] diff --git a/hydra_base/db/truncate.py b/hydra_base/db/truncate.py index 3e2da325..71430342 100644 --- a/hydra_base/db/truncate.py +++ b/hydra_base/db/truncate.py @@ -35,7 +35,8 @@ Base = declarative_base() -engine_name = config.get('mysqld', 'url') +db_config = config.get_startup_config() +engine_name = db_config.get("url") def connect_mysql(): """ @@ -46,10 +47,10 @@ def connect_mysql(): db = create_engine(engine_name) db.echo = True db.connect() - + return db -sqlite_engine = "sqlite:///%s"%(config.get('sqlite', 'backup_url')) +sqlite_engine = "sqlite:///%s"%(config.get("sqlite_backup_url")) def connect_sqlite(): @@ -69,7 +70,7 @@ def truncate_all_audit_tables(): # create a Session mysql_session = session1() - sqlite_session = session2() + sqlite_session = session2() mysql_metadata = MetaData(mysql_db) sqlite_metadata = MetaData(sqlite_db) @@ -93,7 +94,7 @@ def truncate_all_audit_tables(): continue export_table_to_sqlite(mysql_session, sqlite_session, sqlite_metadata, audit_table) truncate_table(mysql_session, table, audit_table) - + mysql_session.commit() sqlite_session.commit() logging.info("Truncation Complete") @@ -116,7 +117,7 @@ def truncate_table(session, table, audit_table): args = [] for arg in truncated_cols: args.append(arg == getattr(r, arg.name)) - + aud_id_col = audit_table.c['aud_id'] rs = session.query(aud_id_col).filter(and_(*args)).order_by(aud_id_col.desc())[3:] aud_ids = [str(r.aud_id) for r in rs] @@ -133,23 +134,23 @@ def export_table_to_csv(session, table, target=None): """ if target is None: - target_dir = os.path.join(config.get('db', 'export_target')) - target = os.path.join(config.get('db', 'export_target'), table.name) + target_dir = os.path.join(config.get("db_export_target")) + target = os.path.join(config.get("db_export_target"), table.name) if not os.path.exists(target_dir): os.mkdir(target_dir) if os.path.exists(target): target_file = open(target, 'r+') - + rs = session.query(table).all() - + entries_in_db = set() for r in rs: entries_in_db.add("%s"%(r.__repr__())) contents = set(target_file.read().split('\n')) - + new_data = entries_in_db.difference(contents) if len(new_data) > 0: @@ -180,10 +181,10 @@ def export_table_to_sqlite(mysql_session, sqlite_session, sqlite_metadata, audit entries_in_mysql_db = set(current_data) entries_in_sqlite_db = set(sqlite_data) - + new_data = list(entries_in_mysql_db.difference(entries_in_sqlite_db)) if len(new_data) > 0: - values = [] + values = [] for val in new_data: row = [] for i, v in enumerate(val): diff --git a/hydra_base/hydra.ini b/hydra_base/hydra.ini deleted file mode 100644 index 2ee21f9a..00000000 --- a/hydra_base/hydra.ini +++ /dev/null @@ -1,113 +0,0 @@ -[DEFAULT] -#home_dir = ~ -log_level = INFO -hydra_aux_dir = %(home_dir)s/.hydra - -datetime_format = %Y-%m-%dT%H:%M:%S.%f000Z - -seasonal_key = 9999 -seasonal_year = 1678 - -[db] -instance = MySQL -upper_bound = 100 -lower_bound = 5 -export_target = %(hydra_aux_dir)s/audit -purge_threshold = 10000 -compression_threshold=50000 -#instance = SQLite - -[mysqld] -user = root -password = root -db_name = hydradb -server_name = 127.0.0.1 -#Y or N for True/False -auto_create = Y -pool_pre_ping=True -# Sqllite connection string -#url = sqlite:///%(hydra_aux_dir)s/hydra.db - -# Mysql connection string -#url = mysql+mysqldb://%(user)s:%(password)s@localhost/%(db_name)s -url = mysql+mysqldb://%(user)s:%(password)s@%(server_name)s/%(db_name)s - -[sqlite] -backup_dir = %(hydra_aux_dir)s/audit -dbfile = %(hydra_base_dir)s/HydraDB/hydra.db -backup_url = %(backup_dir)s/audit.db - -[mongodb] -host = localhost -port = 27017 -db_name = hydra -user = -passwd = -# collection for datasets -datasets = datasets -threshold = 4096 -direct_location_token = mongo_direct -value_location_key = value_storage_location - -[storage_hdf] -disable_hdf = False -hdf_filestore = /tmp - -[hydra_server] -domain = 127.0.0.1 -port = 8080 -path = soap #deprecated -json_path = json -http_path = http -soap_path = soap -#url = http://localhost:%()s?wsdl -url = http://%(domain)s:%(port)s/%(path)s?wsdl -layout_xsd_path = %(hydra_base_dir)s/static/resource_layout.xsd - -[hydra_client] -#url = http://ec2-54-229-95-247.eu-west-1.compute.amazonaws.com/hydra-server?wsdl -domain = http://127.0.0.1 -port = 8080 -path = json # deprecated -json_path = json -http_path = http -soap_path = soap -user = root -password = - -[filesys] -img_src = %(home_dir)s/.hydra/images/ -file_src = %(home_dir)s/.hydra/files/ - -[plugin] -default_directory = %(home_dir)s/.hydra/apps -queue_directory = %(default_directory)s/queue -upload_dir = %(default_directory)s/uploads -result_file = %(home_dir)s/.hydra/plugin_result/ -plugin_xsd_path = %(hydra_base_dir)s/static/plugin_input.xsd - -[unit_conversion] -user_file = %(hydra_base_dir)s/static/user_units.xml -default_file = %(hydra_base_dir)s/static/unit_definitions.xml - -[templates] -template_xsd_path = %(hydra_base_dir)s/static/template.xsd - -[logging_conf] -log_config_path = %(hydra_base_dir)s/logging.conf -log_file_dir = %(hydra_aux_dir)s/log - -[search] -page_size=2000 - -[polyvis] -POLYVIS_URL=http://localhost:5000/ - -[security] -max_login_attempts = 7 - -[cache] -type=diskcache - -[limits] -project_max_nest_depth = 32 diff --git a/hydra_base/hydra_logging.py b/hydra_base/hydra_logging.py index 3796c80b..4f6a208b 100644 --- a/hydra_base/hydra_logging.py +++ b/hydra_base/hydra_logging.py @@ -26,7 +26,7 @@ def init(level=None): # if level is None: - # level = config.get('DEFAULT', 'log_level') + # level = config.get('log_level') # if os.name == "nt": # logging.addLevelName( logging.INFO, logging.getLevelName(logging.INFO)) @@ -38,7 +38,7 @@ def init(level=None): # return # if level is None: - # level = config.get('DEFAULT', 'log_level') + # level = config.get('log_level') # logging.addLevelName( logging.INFO, "\033[0;m%s\033[0;m" % logging.getLevelName(logging.INFO)) @@ -57,7 +57,7 @@ def init(level=None): calling_file = os.path.split(calling_file)[1] log_file = "%s.log" % calling_file.split('.')[0] - log_base_path = config.get('logging_conf', 'log_file_dir', '.') + log_base_path = config.get_startup_config()["hydra_log_filedir"] if not os.path.isdir(log_base_path): os.makedirs(log_base_path) @@ -71,7 +71,7 @@ def init(level=None): use_default = False try: - config_file = os.path.expanduser(config.get('logging_conf', 'log_config_path', '.')) + config_file = os.path.expanduser(config.get_startup_config()["hydra_log_confpath"]) #check the config file exists... if os.path.isfile(config_file) and log_base_path is not None: logging.config.fileConfig(config_file) diff --git a/hydra_base/lib/HydraTypes/Types.py b/hydra_base/lib/HydraTypes/Types.py index c3949ff7..27550bbd 100644 --- a/hydra_base/lib/HydraTypes/Types.py +++ b/hydra_base/lib/HydraTypes/Types.py @@ -298,8 +298,8 @@ def fromDataset(cls, value, metadata=None): def validate(self): base_ts = pd.Timestamp("01-01-1970") #TODO: We need a more permanent solution to seasonal/repeating timeseries - seasonal_year = config.get('DEFAULT','seasonal_year', '1678') - seasonal_key = config.get('DEFAULT', 'seasonal_key', '9999') + seasonal_year = str(config.get("seasonal_year", "1678")) + seasonal_key = str(config.get("seasonal_key", "9999")) jd = json.loads(self.value, object_pairs_hook=collections.OrderedDict) for k,v in jd.items(): for date in (six.text_type(d) for d in v.keys()): diff --git a/hydra_base/lib/attributes.py b/hydra_base/lib/attributes.py index 101dc6bb..e222a2b9 100644 --- a/hydra_base/lib/attributes.py +++ b/hydra_base/lib/attributes.py @@ -352,7 +352,7 @@ def _reassign_scoped_attributes(attr_id, user_id): #first look up the hierarchy to see if there is an attribute scoped at a higher level. - max_levels = int(config.get("limits", "project_max_nest_depth", 32)) + max_levels = int(config.get("limits_project_max_nest_depth", 32)) attr_proj = db.DBSession.query(Project).filter(Project.id == attr_i.project_id).one() child_projects = attr_proj.get_child_projects(user_id=user_id, levels=max_levels) project_scope = {p["id"] for p in child_projects} | {attr_i.project_id} diff --git a/hydra_base/lib/cache.py b/hydra_base/lib/cache.py index e3c6600b..6858b3ae 100644 --- a/hydra_base/lib/cache.py +++ b/hydra_base/lib/cache.py @@ -12,21 +12,22 @@ log = logging.getLogger(__name__) global cache +startup_config = hydraconfig.get_startup_config() +cache_type = startup_config["hydra_cachetype"] +cache_host = startup_config["hydra_cachehost"] + def _init_diskcache(): log.info("Using diskcache for caching.") global cache import diskcache as dc cache = dc.Cache(tempfile.gettempdir()) -if hydraconfig.get('cache', 'type') != "memcached": +if cache_type != "memcached": _init_diskcache() - -elif hydraconfig.get('cache', 'type') == 'memcached': +elif cache_type == 'memcached': try: import pylibmc - host = hydraconfig.get('cache', 'host', '127.0.0.1') - port = hydraconfig.get('cache', 'port', 31211) - cache = pylibmc.Client([f"{host}:{port}"], binary=True) + cache = pylibmc.Client([f"{cache_host}:31211"], binary=True) # Check if Memcached server is reachable by setting a test key test_key = "__connection_test__" diff --git a/hydra_base/lib/data.py b/hydra_base/lib/data.py index a8a26b31..9e1ea8a8 100644 --- a/hydra_base/lib/data.py +++ b/hydra_base/lib/data.py @@ -262,7 +262,7 @@ def search_datasets(dataset_id=None, page_size)) if page_size is None: - page_size = config.get('SEARCH', 'page_size', 2000) + page_size = config.get("search_page_size", 2000) user_id = int(kwargs.get('user_id')) diff --git a/hydra_base/lib/hydraconfig.py b/hydra_base/lib/hydraconfig.py new file mode 100644 index 00000000..087e1623 --- /dev/null +++ b/hydra_base/lib/hydraconfig.py @@ -0,0 +1,195 @@ +""" + Library functions for Hydra configuration +""" +import json + +from hydra_base import db +from hydra_base.exceptions import ( + HydraError, + ResourceNotFoundError, + PermissionError +) + +from hydra_base.db.model.hydraconfig import ( + ConfigKey, + config_key_type_map, + ConfigGroup, + ConfigGroupKeys +) + +from sqlalchemy.exc import ( + IntegrityError, + NoResultFound +) + + +""" Config Keys: Key:Value pairs of config settings """ + +def register_config_key(key_name, key_type, description="", **kwargs): + if not (key_cls := config_key_type_map.get(key_type, None)): + raise HydraError(f"Invalid ConfigKey type '{key_type}'") + + key = key_cls(name=key_name, desc=description) + try: + db.DBSession.add(key) + db.DBSession.flush() + except IntegrityError: + raise HydraError(f"ConfigKey with name '{key_name}' exists") + + return key + +def unregister_config_key(key_name, **kwargs): + key = _get_config_key_by_name(key_name) + db.DBSession.delete(key) + db.DBSession.flush() + +def list_config_keys(like=None, **kwargs): + query = db.DBSession.query(ConfigKey) + if like: + query = query.filter(ConfigKey.name.like(f"%{like}%")) + + keys = query.all() + return [key.name for key in keys] + +def config_key_set_value(key_name, value, **kwargs): + key = _get_config_key_by_name(key_name) + key.value = value + db.DBSession.flush() + +def config_key_get_value(key_name, **kwargs): + key = _get_config_key_by_name(key_name) + return key.value + +def config_key_set_description(key_name, description="", **kwargs): + key = _get_config_key_by_name(key_name) + if not isinstance(description, str): + raise HydraError(f"Invalid description for {key_name}: '{description}'") + + key.description = description + +def config_key_get_description(key_name, **kwargs): + key = _get_config_key_by_name(key_name) + return key.description + +def _get_config_key_by_name(key_name): + try: + key = db.DBSession.query(ConfigKey).filter(ConfigKey.name == key_name).one() + except NoResultFound: + raise HydraError(f"No ConfigKey found with name: {key_name}") + + return key + +""" Validation related functions """ + +def config_key_get_rule_types(key_name, **kwargs): + key = _get_config_key_by_name(key_name) + if validator := getattr(key, "validator", None): + return [*validator.rules] + +def config_key_get_rule_description(key_name, rule_name, **kwargs): + pass + +def config_key_get_active_rules(key_name, **kwargs): + key = _get_config_key_by_name(key_name) + if validator := getattr(key, "validator", None): + return {rule.name: rule.value for rule in validator.active_rules.values()} + else: + return {} + +def config_key_set_rule(key_name, rule_name, value, **kwargs): + key = _get_config_key_by_name(key_name) + if validator := getattr(key, "validator", None): + validator.set_rule(rule_name, value) + +def config_key_clear_rule(key_name, rule_name, **kwargs): + key = _get_config_key_by_name(key_name) + if validator := getattr(key, "validator", None): + validator.clear_rule(rule_name) + +def config_key_clear_all_rules(key_name, **kwargs): + rules = config_key_get_active_rules(key_name, **kwargs) + if rules is None or len(rules) == 0: + return 0 + + for rule_name in rules: + config_key_clear_rule(key_name, rule_name) + + return len(rules) + + + +""" Config Sets: Archived versions of complete configurations """ + +def export_config_as_json(name, description="", **kwargs): + from hydra_base.util.configset import ConfigSet + cs = ConfigSet(name, description=description) + state = cs.save_keys_to_configset() + return json.dumps(state) + +def apply_configset(json_src, **kwargs): + from hydra_base.util.configset import ConfigSet + if not isinstance(json_src, str): + raise ValueError(f"apply_configset requires a JSON encoded string argument") + + try: + state = json.loads(json_src) + except JSONDecodeError as e: + raise ValueError(f"Argument is not a valid JSON string: {e}") + + cs = ConfigSet(state["name"], description=state["description"]) + old_state = cs.apply_configset_to_db(state) + + return old_state + +""" Config Groups: A named collection of Config Keys """ + +def create_config_group(group_name, group_desc=None, **kwargs): + group = ConfigGroup(name=group_name, description=group_desc) + try: + db.DBSession.add(group) + db.DBSession.flush() + except IntegrityError: + raise HydraError(f"ConfigGroup with name '{group_name}' exists") + +def delete_config_group(group_name, **kwargs): + group = _get_config_group_by_name(group_name) + db.DBSession.delete(group) + db.DBSession.flush() + +def list_config_groups(**kwargs): + groups = db.DBSession.query(ConfigGroup).all() + return groups + +def _get_config_group_by_name(group_name): + try: + group = db.DBSession.query(ConfigGroup).filter(ConfigGroup.name == group_name).one() + except NoResultFound: + raise HydraError(f"No ConfigGroup with name: {group_name}") + + return group + +def add_config_key_to_group(key_name, group_name, **kwargs): + key = _get_config_key_by_name(key_name) + group = _get_config_group_by_name(group_name) + gk = ConfigGroupKeys(group_id=group.id, key_id=key.id) + db.DBSession.add(gk) + db.DBSession.flush() + +def config_group_list_keys(group_name, **kwargs): + group = _get_config_group_by_name(group_name) + return group.keys + +def config_key_get_group_name(key_name, **kwargs): + key = _get_config_key_by_name(key_name) + return key.group + +def remove_config_key_from_group(key_name, group_name, **kwargs): + group = _get_config_group_by_name(group_name) + key = _get_config_key_by_name(key_name) + qfilter = { + ConfigGroupKeys.group_id == group.id, + ConfigGroupKeys.key_id == key.id + } + gk = db.DBSession.query(ConfigGroupKeys).filter(*qfilter).one() + db.DBSession.delete(gk) + db.DBSession.flush() diff --git a/hydra_base/lib/network.py b/hydra_base/lib/network.py index c35d6c93..accc1d96 100644 --- a/hydra_base/lib/network.py +++ b/hydra_base/lib/network.py @@ -197,7 +197,7 @@ def _bulk_add_resource_attrs(network_id, ref_key, resources, resource_name_map, ##the current user is validated, but some checks require admin permissions, ##so call as a user with all permissions - admin_id = config.get('DEFAULT', 'ALL_PERMISSION_USER', 1) + admin_id = config.get("ALL_PERMISSION_USER", 1) # template_lookup = {} #a lookup of all the templates used by the resource typeattr_lookup = {} # a lookup from type ID to a list of typeattrs @@ -917,7 +917,7 @@ def _get_all_templates(network_id, template_id): ##the current user is validated, but some checks require admin permissions, ##so call as a user with all permissions - admin_id = config.get('DEFAULT', 'ALL_PERMISSION_USER', 1) + admin_id = config.get("ALL_PERMISSION_USER", 1) for t in all_types: child_layout = None diff --git a/hydra_base/lib/service.py b/hydra_base/lib/service.py index 63044e23..2f16eb53 100644 --- a/hydra_base/lib/service.py +++ b/hydra_base/lib/service.py @@ -56,10 +56,10 @@ def login(username, password, **kwargs): hydra_session = session.Session( {}, #This is normally a request object, but in this case is empty - validate_key=config.get('COOKIES', 'VALIDATE_KEY', DEFAULT_VALIDATE_KEY), + validate_key=config.get("COOKIES_VALIDATE_KEY", DEFAULT_VALIDATE_KEY), type='file' if db.hydra_db_url.startswith('sqlite') else 'ext:sqla', cookie_expires=True, - data_dir=config.get('COOKIES', 'DATA_DIR', DEFAULT_DATA_DIR), + data_dir=config.get("COOKIES_DATA_DIR", DEFAULT_DATA_DIR), bind=db.engine, table=CACHE_TABLE ) @@ -85,10 +85,10 @@ def logout(session_id, **kwargs): hydra_session_object = session.SessionObject( {}, #This is normally a request object, but in this case is empty - validate_key=config.get('COOKIES', 'VALIDATE_KEY', DEFAULT_VALIDATE_KEY), + validate_key=config.get("COOKIES_VALIDATE_KEY", DEFAULT_VALIDATE_KEY), type='file' if db.hydra_db_url.startswith('sqlite') else 'ext:sqla', cookie_expires=True, - data_dir=config.get('COOKIES', 'DATA_DIR', DEFAULT_DATA_DIR), + data_dir=config.get("COOKIES_DATA_DIR", DEFAULT_DATA_DIR), bind=db.engine, table=CACHE_TABLE ) @@ -114,10 +114,10 @@ def get_session_user(session_id, **kwargs): hydra_session_object = session.SessionObject( {}, #This is normally a request object, but in this case is empty - validate_key=config.get('COOKIES', 'VALIDATE_KEY', DEFAULT_VALIDATE_KEY), + validate_key=config.get("COOKIES_VALIDATE_KEY", DEFAULT_VALIDATE_KEY), type='file' if db.hydra_db_url.startswith('sqlite') else 'ext:sqla', cookie_expires=True, - data_dir=config.get('COOKIES', 'DATA_DIR', DEFAULT_DATA_DIR), + data_dir=config.get("COOKIES_DATA_DIR", DEFAULT_DATA_DIR), bind=db.engine, table=CACHE_TABLE ) diff --git a/hydra_base/lib/static.py b/hydra_base/lib/static.py index f75c3565..1b50050a 100644 --- a/hydra_base/lib/static.py +++ b/hydra_base/lib/static.py @@ -25,7 +25,7 @@ log = logging.getLogger(__name__) def add_image(name, file,**kwargs): - path = config.get('filesys', 'img_src') + path = config.get("filesys_img_src") try: os.makedirs(path) except OSError: @@ -67,7 +67,7 @@ def add_image(name, file,**kwargs): return True def get_image(name,**kwargs): - path = config.get('filesys', 'img_src') + path = config.get("filesys_img_src") path = os.path.join(path, name) @@ -87,7 +87,7 @@ def get_image(name,**kwargs): return imageFile def remove_image(name,**kwargs): - path = config.get('filesys', 'img_src') + path = config.get("filesys_img_src") path = os.path.join(path, name) if(os.path.exists(path)): @@ -99,7 +99,7 @@ def remove_image(name,**kwargs): def add_file(resource_type, resource_id, name, file,**kwargs): - path = config.get('filesys', 'file_src') + path = config.get("filesys_file_src") path = os.path.join(path, resource_type) try: os.makedirs(path) @@ -148,7 +148,7 @@ def add_file(resource_type, resource_id, name, file,**kwargs): return True def get_file(resource_type, resource_id, name,**kwargs): - path = config.get('filesys', 'file_src') + path = config.get("filesys_file_src") path = os.path.join(path, resource_type, str(resource_id), name) @@ -170,7 +170,7 @@ def get_file(resource_type, resource_id, name,**kwargs): return file_to_send def remove_file(resource_type, resource_id, name,**kwargs): - path = config.get('filesys', 'file_src') + path = config.get("filesys_file_src") path = os.path.join(path, resource_type, str(resource_id), name) diff --git a/hydra_base/lib/storage/hdfstorageadapter.py b/hydra_base/lib/storage/hdfstorageadapter.py index 35cb6280..faa4371f 100644 --- a/hydra_base/lib/storage/hdfstorageadapter.py +++ b/hydra_base/lib/storage/hdfstorageadapter.py @@ -87,8 +87,12 @@ def _get_anon(self): def get_hdf_config(config_key="storage_hdf", **kwargs): numeric = () boolean = ("disable_hdf", ) - hdf_keys = [k for k in config.CONFIG.options(config_key) if k not in config.CONFIG.defaults()] - hdf_items = {k: config.CONFIG.get(config_key, k) for k in hdf_keys} + + hdf_items = { + "disable_hdf": os.environ.get("HYDRA_DISABLE_HDF"), + "hdf_filestore": os.environ.get("HYDRA_HDF_FILESTORE") + } + for k in numeric: hdf_items[k] = int(hdf_items[k]) for k in boolean: diff --git a/hydra_base/lib/storage/mongostorageadapter.py b/hydra_base/lib/storage/mongostorageadapter.py index 629db9b5..ba088df2 100644 --- a/hydra_base/lib/storage/mongostorageadapter.py +++ b/hydra_base/lib/storage/mongostorageadapter.py @@ -1,4 +1,5 @@ import logging +import os from bson.objectid import ObjectId from pymongo import MongoClient @@ -37,8 +38,19 @@ def __init__(self): @staticmethod def get_mongo_config(config_key="mongodb"): numeric = ("threshold",) - mongo_keys = [k for k in config.CONFIG.options(config_key) if k not in config.CONFIG.defaults()] - mongo_items = {k: config.CONFIG.get(config_key, k) for k in mongo_keys} + + mongo_items = { + "host": os.environ.get("HYDRA_MONGO_HOST"), + "port": os.environ.get("HYDRA_MONGO_PORT"), + "user": os.environ.get("HYDRA_MONGO_USER"), + "passwd": os.environ.get("HYDRA_MONGO_PASSWD"), + "db_name": os.environ.get("HYDRA_MONGO_DB_NAME"), + "datasets": os.environ.get("HYDRA_MONGO_DATASETS"), + "threshold": os.environ.get("HYDRA_MONGO_THRESHOLD"), + "direct_location_token": os.environ.get("HYDRA_MONGO_DIRECT_LOCATION_TOKEN"), + "value_location_key": os.environ.get("HYDRA_MONGO_VALUE_LOCATION_KEY") + } + for k in numeric: mongo_items[k] = int(mongo_items[k]) diff --git a/hydra_base/lib/template/xml.py b/hydra_base/lib/template/xml.py index 4788850d..2f331c69 100644 --- a/hydra_base/lib/template/xml.py +++ b/hydra_base/lib/template/xml.py @@ -18,6 +18,7 @@ import json import logging +import os from decimal import Decimal from lxml import etree @@ -98,7 +99,8 @@ def import_template_xml(template_xml, allow_update=True, **kwargs): """ user_id = kwargs.get('user_id') - template_xsd_path = config.get('templates', 'template_xsd_path') + hydra_base_dir = config.get("hydra_base_dir") + template_xsd_path = os.path.join(hydra_base_dir, "static/template.xsd") xmlschema_doc = etree.parse(template_xsd_path) xmlschema = etree.XMLSchema(xmlschema_doc) diff --git a/hydra_base/lib/users.py b/hydra_base/lib/users.py index a299f204..2432b6c3 100644 --- a/hydra_base/lib/users.py +++ b/hydra_base/lib/users.py @@ -208,7 +208,7 @@ def get_max_login_attempts(*args, **kwargs): a value of 0 will be returned and users will be unable to log in. """ - max_login_attempts = int(config.get("security", "max_login_attempts", 0)) + max_login_attempts = int(config.get("security_max_login_attempts", 0)) return max_login_attempts diff --git a/hydra_base/util/__init__.py b/hydra_base/util/__init__.py index 2661e417..f2a87da8 100644 --- a/hydra_base/util/__init__.py +++ b/hydra_base/util/__init__.py @@ -144,8 +144,8 @@ def get_val(dataset, timestamp=None): elif dataset.type == 'timeseries': #TODO: design a mechansim to retrieve this data if it's stored externally - seasonal_year = config.get('DEFAULT','seasonal_year', '1678') - seasonal_key = config.get('DEFAULT', 'seasonal_key', '9999') + seasonal_year = str(config.get("seasonal_year", 1678)) + seasonal_key = str(config.get("seasonal_key", 9999)) val = val.replace(seasonal_key, seasonal_year) timeseries = pd.read_json(val, convert_axes=True) diff --git a/hydra_base/util/configset.py b/hydra_base/util/configset.py new file mode 100644 index 00000000..e1ffa4c3 --- /dev/null +++ b/hydra_base/util/configset.py @@ -0,0 +1,124 @@ +""" + Utilities for the management of ConfigKeys +""" +import datetime +import hmac +import json + +from hydra_base import db +from hydra_base.db.model.hydraconfig import ConfigKey +from hydra_base.lib.hydraconfig import ( + register_config_key, + config_key_set_value, + config_key_set_rule, + config_key_set_description +) +from hydra_base.config import get_startup_config + +hash_key = get_startup_config()["hydra_config_hash_key"] +config_set_secret_key = bytes(hash_key, encoding="utf8") + +class ConfigSet: + mac_hash = "sha256" + + def __init__(self, name, description=""): + self.name = name + self.desc = description + + @property + def name(self): + return self._name + + @name.setter + def name(self, name): + if not name: + raise ValueError(f"ConfigSet requires a valid name, not: {name}") + self._name = name + + def save_keys_to_configset(self): + configstate= { + "name": self.name, + "description": self.desc, + "timestamp": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "keys": self.all_keys_as_dict() + } + + digest = self.generate_mac(json.dumps(configstate)) + configstate["digest"] = digest + return configstate + + def verify_configset(self, state): + """ + Verifies that the state argument contains a valid + hmac digest whose "message" corresponds to the contents + of the other fields in the state. + + Raises ValueError if the equivalent hmac calculated here + differs from that claimed by the input state. + """ + if isinstance(state, str): + state = json.loads(state) + + loaded_digest = state.pop("digest") + calculated_digest = self.generate_mac(json.dumps(state)) + if loaded_digest != calculated_digest: + raise ValueError(f"ConfigSet digest is {calculated_digest} but claimed is {loaded_digest}") + + state["digest"] = calculated_digest + return state + + def all_keys_as_dict(self): + keys = db.DBSession.query(ConfigKey).all() + if len(keys) == 0: + return {} + return { + key.name: { + "type": key.type, + "value": key.value, + "rules": key.rules, + "description": key.description + } for key in keys + } + + def generate_mac(self, state): + digest = hmac.digest(key=config_set_secret_key, msg=state.encode("utf8"), digest=ConfigSet.mac_hash) + return digest.hex() + + def apply_configset_to_db(self, state): + """ + 1. Serialise existing state + 2. Verify integrity of new state + 3. Delete existing state + 4. Create new keys + 5. Set new validation rules + 6. Load new values + 7. Verify state loaded in 3-5 matches input state + 8. OK if so, else restore original state from 1 + """ + old_state = self.save_keys_to_configset() + new_state = self.verify_configset(state) + self._delete_all_keys() + self.load_keys_from_state(new_state) + # Verify update has succeeded and return old state if so + trial_state = self.save_keys_to_configset() + if trial_state["keys"] == new_state["keys"]: + return old_state + # Otherwise restore state to that before call + self._delete_all_keys() + self.load_keys_from_state(old_state) + # Returns None on failure to update + + def _delete_all_keys(self): + keys = db.DBSession.query(ConfigKey).all() + for key in keys: + db.DBSession.delete(key) + db.DBSession.flush() + + def load_keys_from_state(self, state): + for key_name, key in state["keys"].items(): + register_config_key(key_name, key["type"]) + rules = json.loads(key["rules"]) + for rule_name, rule_val in rules.items(): + config_key_set_rule(key_name, rule_name, rule_val) + config_key_set_value(key_name, key["value"]) + config_key_set_description(key_name, key["description"]) diff --git a/hydra_base/util/hydra_dateutil.py b/hydra_base/util/hydra_dateutil.py index 050284a6..4a55f7af 100644 --- a/hydra_base/util/hydra_dateutil.py +++ b/hydra_base/util/hydra_dateutil.py @@ -162,7 +162,7 @@ def date_to_string(date, seasonal=False): recognised by Hydra as seasonal time stamp. """ - seasonal_key = config.get('DEFAULT', 'seasonal_key', '9999') + seasonal_key = str(config.get("seasonal_key", 9999)) if seasonal: FORMAT = seasonal_key+'-%m-%dT%H:%M:%S.%f' else: @@ -219,7 +219,7 @@ def guess_timefmt(datestr): if isinstance(datestr, float) or isinstance(datestr, int): return None - seasonal_key = str(config.get('DEFAULT', 'seasonal_key', '9999')) + seasonal_key = str(config.get("seasonal_key", "9999")) #replace 'T' with space to handle ISO times. if datestr.find('T') > 0: @@ -311,8 +311,8 @@ def reindex_timeseries(ts_string, new_timestamps): new_timestamps = new_timestamps_converted - seasonal_year = config.get('DEFAULT','seasonal_year', '1678') - seasonal_key = config.get('DEFAULT', 'seasonal_key', '9999') + seasonal_year = str(config.get("seasonal_year", 1678)) + seasonal_key = str(config.get("seasonal_key", 9999)) ts = ts_string.replace(seasonal_key, seasonal_year) diff --git a/hydra_base/util/migrate_config.py b/hydra_base/util/migrate_config.py new file mode 100644 index 00000000..6703e5b3 --- /dev/null +++ b/hydra_base/util/migrate_config.py @@ -0,0 +1,105 @@ +""" + Utilities to migrate from Hydra.ini config format + to DB-based config table via hydra_base.lib.hydraconfig +""" +import configparser +import os +import transaction + +from hydra_base import db +from hydra_base.lib.hydraconfig import ( + register_config_key, + unregister_config_key, + list_config_keys, + config_key_set_value, + config_key_get_value, + export_config_as_json +) + + +if not db.DBSession: + db.connect() + + +def ini_to_configset(ini_filename): + db_config_schema = make_db_config_schema(ini_filename) + return db_config_schema + +def make_db_config_schema(ini_filename): + exclude_sections = ("mysqld",) + config = configparser.ConfigParser(allow_no_value=True) + config.read(ini_filename) + + # Values for "home_dir" and "hydra_base_dir" must be + # set to allow for interpolation into later values + home_dir = os.environ.get("HYDRA_HOME_DIR", '~') + hydra_base_dir = os.environ.get("HYDRA_BASE_DIR", os.getcwd()) + #config.set("DEFAULT", "home_dir", os.path.expanduser(home_dir)) + #config.set("DEFAULT", "hydra_base_dir", os.path.expanduser(hydra_base_dir)) + + db_config_schema = {} + for key in config["DEFAULT"]: + try: + value = config["DEFAULT"].get(key, raw=True) + except configparser.InterpolationSyntaxError: + value = config["DEFAULT"].get(key, raw=True) + try: + value = int(value, 10) + key_type = "integer" + except ValueError: + key_type = "string" + + if key in db_config_schema: + raise ValueError(f"Duplicate key: {key}") + + db_config_schema[key] = { + "type": key_type, + "value": value + } + + for section in config.sections(): + if section in exclude_sections: + continue + for key in config._sections[section].keys(): + try: + #value = config[section][key] + value = config[section].get(key, raw=True) + except configparser.InterpolationSyntaxError: + value = config[section].get(key, raw=True) + key_name = f"{section}_{key}" + try: + value = int(value, 10) + key_type = "integer" + except ValueError: + key_type = "string" + + if key_name in db_config_schema: + raise ValueError(f"Duplicate key: {key_name}") + + db_config_schema[key_name] = { + "type": key_type, + "value": value + } + + return db_config_schema + + +def make_config_from_schema(schema): + for name, key in schema.items(): + register_config_key(name, key["type"]) + config_key_set_value(name, key["value"]) + + transaction.commit() + + +def get_all_config_keys(): + keys = list_config_keys() + return {k: config_key_get_value(k) for k in keys} + + +def delete_all_config_keys(): + keys = list_config_keys() + for key in keys: + unregister_config_key(key) + + transaction.commit() diff --git a/hydra_base/util/testing.py b/hydra_base/util/testing.py index 215c33d9..44dea3ad 100644 --- a/hydra_base/util/testing.py +++ b/hydra_base/util/testing.py @@ -850,7 +850,7 @@ def create_timeseries(self, resource_attr, unit='m^3'): #with a resource attribute. #[[[1, 2, "hello"], [5, 4, 6]], [[10, 20, 30], [40, 50, 60]]] - fmt = hydra_base.config.get('DEFAULT', 'datetime_format', "%Y-%m-%dT%H:%M:%S.%f000Z") + fmt = hydra_base.config.get("datetime_format", "%Y-%m-%dT%H:%M:%S.%f000Z") t1 = datetime.datetime.now(datetime.timezone.utc) t2 = t1+datetime.timedelta(hours=1) diff --git a/requirements.txt b/requirements.txt index 988422a9..3a1a7c5b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,4 @@ bcrypt -lxml -pymongo -mysqlclient -pudb -python-dateutil -cheroot beaker cheroot click diff --git a/tests/conftest.py b/tests/conftest.py index e031b6bc..0bbd7cb4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,11 +15,22 @@ from hydra_base.util import testing from hydra_client.connection import JSONConnection, RemoteJSONConnection from hydra_base.lib.cache import clear_cache +from hydra_base.config import load_config no_externaldb_opt = "--no-externaldb" externaldb_mark = "externaldb" requires_hdf_mark = "requires_hdf" +requires_replicaset_mark = "requires_replicaset" + +from hydra_base import db + +if not db.DBSession: + db.connect() + +create_default_users_and_perms() +make_root_user() +create_default_units_and_dimensions() def pytest_addoption(parser): parser.addoption("--db-backend", action="store", default="sqlite", @@ -47,8 +58,12 @@ def pytest_collection_modifyitems(config, items): if externaldb_mark in item.keywords: item.add_marker(externaldb_skip) - conf_disabled = hydra_base.config.CONFIG.get("storage_hdf", "disable_hdf").lower() - hdf_disabled = True if conf_disabled in ("true", "yes") else False + truthlike = {"true", "yes", "y"} + + from hydra_base.lib.storage import HdfStorageAdapter + config = HdfStorageAdapter.get_hdf_config() + + hdf_disabled = config.get("disable_hdf") hdf_skip = pytest.mark.skip(reason="Test not applicable when HDF support disabled") if hdf_disabled: for item in items: @@ -81,7 +96,7 @@ def pytest_report_header(config): @pytest.fixture() def dateformat(): - return hydra_base.config.get('DEFAULT', 'datetime_format', "%Y-%m-%dT%H:%M:%S.%f000Z") + return hydra_base.config.get("datetime_format", "%Y-%m-%dT%H:%M:%S.%f000Z") @pytest.fixture(scope='module') def testdb_uri(db_backend): @@ -122,6 +137,8 @@ def client(connection_type, testdb_uri): test_server=null_server) client.login('root', '') + load_config() + client.testutils = testing.TestUtil(client) pytest.root_user_id = 1 pytest.user_a = client.testutils.create_user("UserA") @@ -148,11 +165,15 @@ def drop_tables(db_url): @pytest.fixture() def network(client, project_id=None, num_nodes=10, new_proj=True, map_projection='EPSG:4326'): - return client.testutils.build_network(project_id, num_nodes, new_proj, map_projection) + return client.testutils.build_network(project_id=project_id, num_nodes=num_nodes, new_proj=new_proj, map_projection=map_projection) @pytest.fixture() def network_with_data(client, project_id=None, num_nodes=10, ret_full_net=True, new_proj=True, map_projection='EPSG:4326'): - return client.testutils.create_network_with_data(project_id, num_nodes, ret_full_net, new_proj, map_projection) + return client.testutils.create_network_with_data(project_id=project_id, + num_nodes=num_nodes, + ret_full_net=ret_full_net, + new_proj=new_proj, + map_projection=map_projection) @pytest.fixture() def network_with_child_scenario(client, project_id=None, num_nodes=10, ret_full_net=True, new_proj=True, map_projection='EPSG:4326'): diff --git a/tests/templates/test_templates.py b/tests/templates/test_templates.py index ed3ea557..17ae8b66 100644 --- a/tests/templates/test_templates.py +++ b/tests/templates/test_templates.py @@ -34,7 +34,6 @@ def template(): return os.path.join(os.path.dirname(__file__), 'template.xml') - @pytest.fixture() def template_json_object(client, template): @@ -43,7 +42,6 @@ def template_json_object(client, template): return JSONObject(client.import_template_xml(file_contents)) - @pytest.fixture() def mock_template(client): link_attr_1 = client.testutils.create_attribute("link_attr_1", dimension='Pressure') @@ -141,94 +139,6 @@ class TestTemplates: """ Test for templates """ - def test_add_xml(self, template_json_object): - new_tmpl = template_json_object - - assert new_tmpl is not None, "Adding template from XML was not successful!" - - assert len(new_tmpl.templatetypes) == 2 - - for tt in new_tmpl.templatetypes: - if tt.name == 'Reservoir': - for ta in tt.typeattrs: - assert ta.data_type == 'scalar' - - assert tt.typeattrs[-1].properties is not None - assert eval(tt.typeattrs[-1].properties)['template_property'] == "Test property from template" - - def test_get_xml(self, client, template_json_object): - xml_tmpl = template_json_object - - db_template = client.get_template_as_xml(xml_tmpl.id) - - - assert db_template is not None - - template_xsd_path = config.get('templates', 'template_xsd_path') - xmlschema_doc = etree.parse(template_xsd_path) - - xmlschema = etree.XMLSchema(xmlschema_doc) - - xml_tree = etree.fromstring(db_template) - - xmlschema.assertValid(xml_tree) - - def test_get_dict(self, client, template_json_object): - - # Upload the xml file initally to avoid having to manage 2 template files - xml_tmpl = template_json_object - - template_dict = client.get_template_as_dict(xml_tmpl.id) - - # Error that there's already a template with this name. - with pytest.raises(HydraError): - client.import_template_dict(template_dict, allow_update=False) - - typename = template_dict['template']['templatetypes'][0]['name'] - - template_dict['template']['templatetypes'][0].name = typename + "_updated" - - # Finds a template with this name and updates it to mirror this dict. - # This includes deleting types if they're not in this dict. - # Changing the name of a type has this effect, as a new template does not have - # any reference to existing types in Hydra. - - updated_template = JSONObject(client.import_template_dict(template_dict)) - - type_names = [] - - for templatetype in updated_template.templatetypes: - type_names.append(templatetype.name) - - assert len(type_names) == 2 - assert typename + "_updated" in type_names and typename not in type_names - - # Now put it back to the original name so other tests will work - log.info("Reverting the type's name") - template_dict['template']['templatetypes'][0].name = typename - updated_template = JSONObject(client.import_template_dict(template_dict)) - - #just double-check that the JSON import works also - updated_template = JSONObject(client.import_template_json(json.dumps(template_dict))) - - type_names = [] - for templatetype in updated_template.templatetypes: - type_names.append(templatetype.name) - - assert len(type_names) == 2 - assert typename in type_names and typename + "_updated" not in type_names - - log.info("Checking to ensure Template has been updated correctly...") - # one final check to ensure that the type has been deleted - check_template_i = client.get_template(updated_template.id) - - assert len(check_template_i.templatetypes) == 2 - - - """ - TEMPLATES Functions - """ - def test_add_template(self, client, mock_template): link_attr_1 = client.testutils.create_attribute("link_attr_1", dimension='Pressure') @@ -857,23 +767,6 @@ def test_remove_type_from_resource(self, client, mock_template, network_with_dat assert updated_node_j.types is None or str(result1_j.id) not in [str(x.type_id) for x in updated_node_j.types] - def test_create_template_from_network(self, client, network_with_data): - network = network_with_data - - net_template = client.get_network_as_xml_template(network.id) - - - assert net_template is not None - - template_xsd_path = config.get('templates', 'template_xsd_path') - xmlschema_doc = etree.parse(template_xsd_path) - - xmlschema = etree.XMLSchema(xmlschema_doc) - - xml_tree = etree.fromstring(net_template) - - xmlschema.assertValid(xml_tree) - def test_apply_template_to_network(self, client, mock_template, network_with_data): net_to_update = network_with_data template = mock_template diff --git a/tests/test_data.py b/tests/test_data.py index b87b3f6f..53c9f349 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -251,7 +251,7 @@ def test_multiple_vals_at_time(self, client, network_with_data, seasonal_timeser val_a = json.loads(val_to_query.value) - dtformat = hb.config.get('DEFAULT', 'datetime_format', "%Y-%m-%dT%H:%M:%S.%f000Z") + dtformat = hb.config.get("datetime_format", "%Y-%m-%dT%H:%M:%S.%f000Z") fmt = datetime.datetime.strftime qry_times = [ fmt(datetime.datetime(2000, 1, 10, 00, 00, 00), dtformat), diff --git a/tests/test_hdf.py b/tests/test_hdf.py index 681a5e9b..8e3e7ff9 100644 --- a/tests/test_hdf.py +++ b/tests/test_hdf.py @@ -109,16 +109,17 @@ def test_hdf_size(self, hdf, public_aws_file): @pytest.mark.requires_hdf def test_private_hdf_no_access(self, hdf, private_aws_file): """ - Do the reported properties of a dataset match expected values? + Does attempted access to a private HDF file without credentials result in an error? """ with pytest.raises(PermissionError): info = hdf.get_series_info(private_aws_file["path"], columns=private_aws_file["series_name"]) + @pytest.mark.skip @pytest.mark.requires_hdf @patch.dict('os.environ', {'AWS_ACCESS_KEY_ID': IAM_ACCESS_KEY, 'AWS_SECRET_ACCESS_KEY': IAM_SECRET_KEY}) def test_private_hdf_correct_access(self, private_aws_file): """ - Do the reported properties of a dataset match expected values? + Is access to a private HDF file with correct credentials permitted? """ hdf = HdfStorageAdapter() diff --git a/tests/test_hydraconfig.py b/tests/test_hydraconfig.py new file mode 100644 index 00000000..c2a420e2 --- /dev/null +++ b/tests/test_hydraconfig.py @@ -0,0 +1,579 @@ +import base64 +import copy +import inspect +import json +import math +import pytest +import random +import string + +from hydra_base.exceptions import HydraError +from hydra_base.lib.hydraconfig import ( + config_key_type_map, + ConfigKey +) +from hydra_base.db.model.hydraconfig.validators import ( + ConfigKeyIntegerValidator, + ConfigKeyStringValidator +) +from hydra_base.util.configset import ConfigSet + + +# Util funcs + +def make_integer_key_with_value(client, key_name): + key_value = random.randint(2**7, 2**9) + key_description = generate_key_description(key_type="integer") + key = client.register_config_key(key_name, "integer", description=key_description) + val_diff = random.randint(1, key_value//2) + min_value, max_value = key_value-val_diff, key_value+val_diff + client.config_key_set_rule(key_name, "min_value", min_value) + client.config_key_set_rule(key_name, "max_value", max_value) + client.config_key_set_value(key_name, key_value) + return key.name + + +def make_string_key_with_value(client, key_name): + key_len = random.randint(2**4, 2**6) + key_value = "".join(random.choices(string.ascii_lowercase, k=key_len)) + key_description = generate_key_description(key_type="string") + key = client.register_config_key(key_name, "string", description=key_description) + len_diff = random.randint(1, key_len//2) + min_length, max_length = key_len-len_diff, key_len+len_diff + client.config_key_set_rule(key_name, "min_length", min_length) + client.config_key_set_rule(key_name, "max_length", max_length) + client.config_key_set_value(key_name, key_value) + return key.name + + +def make_boolean_key_with_value(client, key_name): + key_value = random.getrandbits(1) + key_description = generate_key_description(key_type="boolean") + key = client.register_config_key(key_name, "boolean", description=key_description) + client.config_key_set_value(key_name, key_value) + return key.name + + +def generate_key_description(src="\x80.", key_type=""): + dmap = { + "\x80": ["\x81 \x84 for a key of type \xC0"], + "\x81": ["An \x82", "A \x83", "The"], + "\x82": ["appropriate", "apt", "example", "illustrative"], + "\x83": ["fitting", "suitable", "relevant", "particular", "placeholder", "basic", "typical"], + "\x84": ["description", "comment", "overview", "explanation"] + } + maptop = 0x85 + out = [] + idx = -1 + while True: + idx += 1 + try: + c = src[idx] + except IndexError: + break + oc = ord(c) + if oc < 128: + out.append(c) + continue + else: + if oc < maptop: + mapline = dmap[c] + out.append(generate_key_description(src=random.choice(mapline), key_type=key_type)) + continue + if oc == 0xC0: + out.append(key_type) + continue + + return "".join(out) + + +@pytest.fixture +def config_group(): + group_name = "Test ConfigKey Group" + group_desc = "Description of test group" + group = client.create_config_group(group_name, group_desc) + yield group + client.delete_config_group() + + +@pytest.fixture +def random_keys(client): + """ + Returns a function which... + - Generates and registers n_keys ConfigKeys of random + types, including appropriate validator rule settings + and a key value which passes validation. + - Deletes these keys on return unless the optional + do_tidy argument is overwritten to be False. + """ + pending_tidy = [] + _do_tidy = True + def _random_keys(n_keys, do_tidy=True): + key_gen_funcs = { + "integer": make_integer_key_with_value, + "string": make_string_key_with_value, + "boolean": make_boolean_key_with_value + } + nonlocal _do_tidy, pending_tidy + _do_tidy = do_tidy + prefix_len = 3 + existing_key_prefixes = set(k[:prefix_len] for k in pending_tidy) + key_prefixes = set() + key_names = [] + for idx in range(n_keys): + while True: + key_prefix = "".join(random.choices(string.ascii_lowercase, k=prefix_len)) + if key_prefix not in key_prefixes | existing_key_prefixes: + key_prefixes.add(key_prefix) + break + key_name = f"{key_prefix} test key" + key_func = key_gen_funcs[random.choice([*key_gen_funcs])] + key_names.append(key_func(client, key_name)) + + if do_tidy: + pending_tidy += key_names + return key_names + + yield _random_keys + if _do_tidy: + for key_name in pending_tidy: + try: + client.unregister_config_key(key_name) + except HydraError: + pass + + +class TestFixtures(): + def test_fixture_reentrancy(self, client, random_keys): + """ + Verify random_keys fixture is reentrant wrt to + multiple calls to the returned func in the same + fixture scope. + """ + num_keys = 16 + # First fix func call returns new keys and has them pending deletion + keys0 = random_keys(num_keys) + assert len(keys0) == num_keys + rkpt0 = inspect.getclosurevars(random_keys).nonlocals["pending_tidy"] + assert len(rkpt0) == num_keys + for key in keys0: + assert key in rkpt0 + # Second fix func call returns new keys but do_tidy is False + # so pending deletion not expanded + keys1 = random_keys(num_keys, do_tidy=False) + assert len(keys1) == num_keys + rkpt1 = inspect.getclosurevars(random_keys).nonlocals["pending_tidy"] + assert len(rkpt1) == num_keys + for key in keys1: + assert key not in rkpt1 + # As a result of do_tidy=False, we have to do tidy + # or duplicates could occur + for key_name in keys1: + client.unregister_config_key(key_name) + # Third fix func call returns new keys and adds these to + # pending deletion + keys2 = random_keys(num_keys) + assert len(keys2) == num_keys + rkpt2 = inspect.getclosurevars(random_keys).nonlocals["pending_tidy"] + assert len(rkpt2) == 2*num_keys + for key in keys0 + keys2: + assert key in rkpt2 + + +class TestConfigKeyValidators(): + """ Standalone tests of Config Key validator classes """ + def test_create_validator(self): + """ + Do Validators register their subclass-specific rules? + """ + iv = ConfigKeyIntegerValidator() + assert len(iv.rules) == 2 + assert len(iv.active_rules) == 0 + + sv = ConfigKeyStringValidator() + assert len(sv.rules) == 2 + assert len(sv.active_rules) == 0 + + def test_validator_with_rules_spec(self): + # Integer Validator + too_low = 1 + too_high = 12 + valid_value = 7 + rules_spec = '{"min_value": 2, "max_value": 9}' + iv = ConfigKeyIntegerValidator(rules_spec) + assert len(iv.active_rules) == 2 + assert iv.validate(valid_value) is None + with pytest.raises(ValueError): + iv.validate(too_low) + with pytest.raises(ValueError): + iv.validate(too_high) + + # String Validator + too_short = "one" + too_long = "eleven" + valid_value = "five" + rules_spec = '{"min_length": 4, "max_length": 5}' + sv = ConfigKeyStringValidator(rules_spec) + assert len(sv.active_rules) == 2 + assert sv.validate(valid_value) is None + with pytest.raises(ValueError): + sv.validate(too_short) + with pytest.raises(ValueError): + sv.validate(too_long) + + def test_validate_integer_key(self): + key_value = 12 + max_value = 7 + iv = ConfigKeyIntegerValidator() + # Validation succeeds as no rules are active + assert iv.validate(key_value) is None + iv.set_rule("max_value", max_value) + # Validation of the same value now fails due to active rule + with pytest.raises(ValueError): + iv.validate(key_value) + + def test_validate_string_key(self): + key_value = "A string value" # len 14 + max_length = 7 + sv = ConfigKeyStringValidator() + # Validation succeeds as no rules are active + assert sv.validate(key_value) is None + sv.set_rule("max_length", max_length) + # Validation of the same value now fails due to active rule + with pytest.raises(ValueError): + sv.validate(key_value) + + def test_serialise_validator(self): + """ + Does a validator have the correct initial state + and can it be serialised to the correct format? + """ + min_value = 5 + max_value = 12 + initial_state = '{"max_value": null, "min_value": null}' + updated_rules = f'{{"max_value": {max_value}, "min_value": {min_value}}}' + iv = ConfigKeyIntegerValidator() + assert iv.as_json == initial_state + iv.set_rule("max_value", max_value) + iv.set_rule("min_value", min_value) + assert iv.as_json == updated_rules + + +class TestHydraConfig(): + def test_valid_config_key_type_map(self): + """ + Ensure that all basic config key subclasses are + registered and map to the correct type + """ + assert config_key_type_map is not None + + required_types = ("integer", "string", "boolean", "base64", "uri", "json") + for rtype in required_types: + assert rtype in config_key_type_map + assert issubclass(config_key_type_map[rtype], ConfigKey) + + def test_define_config_key(self, client): + """ + Can only keys with valid names and types be registered + and are these correctly retrieved? + """ + with pytest.raises(HydraError): + client.register_config_key("test_key_name", "invalid type") + + with pytest.raises(ValueError): + client.register_config_key("", "integer") + + itk = client.register_config_key("integer_test_key", "integer") + assert itk.name == "integer_test_key" + assert itk.type == "integer" + + all_keys = client.list_config_keys() + assert itk.name in all_keys + client.unregister_config_key("integer_test_key") + + def test_set_config_key_value(self, client): + """ + Can a key be assigned an appropriate value and + this value then retrieved as the correct type? + """ + # ConfigKey_Integer + key_name = "integer_value_test_key" + key_value = 46 + _ = client.register_config_key(key_name, "integer") + client.config_key_set_value(key_name, key_value) + ret_value = client.config_key_get_value(key_name) + assert isinstance(ret_value, int) + assert ret_value == key_value + # Can existing key value be changed? + new_value = 77 + client.config_key_set_value(key_name, new_value) + new_ret_value = client.config_key_get_value(key_name) + assert isinstance(new_ret_value, int) + assert new_ret_value == new_value + + # Invalid int values should be rejected + with pytest.raises(HydraError): + client.config_key_set_value(key_name, math.nan) + client.unregister_config_key(key_name) + + # ConfigKey_String + key_name = "string_value_test_key" + key_value = "A string value" + _ = client.register_config_key(key_name, "string") + client.config_key_set_value(key_name, key_value) + ret_value = client.config_key_get_value(key_name) + assert isinstance(ret_value, str) + assert ret_value == key_value + client.unregister_config_key(key_name) + + + # ConfigKey_Boolean + key_name = "boolean_value_test_key" + key_value = True + _ = client.register_config_key(key_name, "boolean") + client.config_key_set_value(key_name, key_value) + ret_value = client.config_key_get_value(key_name) + assert isinstance(ret_value, bool) + assert ret_value == key_value + + # Invalid bool values should be rejected + with pytest.raises(HydraError): + client.config_key_set_value(key_name, 'Y') + client.unregister_config_key(key_name) + + # ConfigKey_Base64 + key_name = "base64_value_test_key" + raw_bytes = bytes([73, 110, 112, 117, 116, 32, 118, 97, 108, 117, 101]) + b64_bytes = base64.b64encode(raw_bytes) + key_value = b64_bytes.decode("utf8") + _ = client.register_config_key(key_name, "base64") + client.config_key_set_value(key_name, key_value) + ret_value = client.config_key_get_value(key_name) + assert isinstance(ret_value, str) + ret_bytes = base64.b64decode(ret_value, validate=True) + assert ret_bytes == raw_bytes + + # Non-b64 strings should be rejected + with pytest.raises(HydraError): + client.config_key_set_value(key_name, "Not Base64") + client.unregister_config_key(key_name) + + def test_integer_key_validation(self, client): + key_name = "integer_validation_test_key" + key_value = 46 + max_value = 47 + min_value = 12 + _ = client.register_config_key(key_name, "integer") + assert len(client.config_key_get_rule_types(key_name)) == 2 + assert len(client.config_key_get_active_rules(key_name)) == 0 + # Permissable as no rules are yet active + client.config_key_set_value(key_name, key_value) + # Set a max_value rule + client.config_key_set_rule(key_name, "max_value", max_value) + assert len(client.config_key_get_active_rules(key_name)) == 1 + # Now raises... + with pytest.raises(ValueError): + client.config_key_set_value(key_name, max_value+1) + # ...and value has remained unchanged + assert client.config_key_get_value(key_name) == key_value + # Set a min_value rule + client.config_key_set_rule(key_name, "min_value", min_value) + # Raises again... + with pytest.raises(ValueError): + client.config_key_set_value(key_name, min_value-1) + # ...and value has remained unchanged + assert client.config_key_get_value(key_name) == key_value + # Clear all rules... + num_cleared = client.config_key_clear_all_rules(key_name) + # ...two rules were cleared... + assert num_cleared == 2 + # ...and previously rejected values now accepted + client.config_key_set_value(key_name, max_value+1) + client.config_key_set_value(key_name, min_value-1) + client.unregister_config_key(key_name) + + def test_string_key_validation(self, client): + key_name = "string_validation_test_key" + key_value = "string key value" # len 16 + min_length = 12 + max_length = 18 + _ = client.register_config_key(key_name, "string") + assert len(client.config_key_get_rule_types(key_name)) == 2 + assert len(client.config_key_get_active_rules(key_name)) == 0 + # Permissable as no rules are yet active + client.config_key_set_value(key_name, key_value) + # Set a max_length rule + client.config_key_set_rule(key_name, "max_length", max_length) + # Rule is now active... + assert len(client.config_key_get_active_rules(key_name)) == 1 + # ...so value exceeding max_length fails... + with pytest.raises(ValueError): + client.config_key_set_value(key_name, key_value+"extra text") + # ...and original value is unchanged + assert client.config_key_get_value(key_name) == key_value + # Set a min_length rule + client.config_key_set_rule(key_name, "min_length", min_length) + # Additional rule is now active... + assert len(client.config_key_get_active_rules(key_name)) == 2 + # ...so value shorter than min_length is rejected + with pytest.raises(ValueError): + client.config_key_set_value(key_name, key_value[:8]) + # ...and original value remains unchanged + assert client.config_key_get_value(key_name) == key_value + # Clear all rules + num_cleared = client.config_key_clear_all_rules(key_name) + assert num_cleared == 2 + # Previously rejected values may now be set + client.config_key_set_value(key_name, key_value+"extra text") + client.config_key_set_value(key_name, key_value[:8]) + client.unregister_config_key(key_name) + + +class TestConfigKeyGroups(): + def test_create_config_group(self, client): + group_name = "Test ConfigKey Group" + group_desc = "Description of test group" + # Create a group and verify presence + client.create_config_group(group_name, group_desc) + groups = client.list_config_groups() + group_names = {g.name for g in groups} + assert group_name in group_names + # Verify unable to add group with same name + with pytest.raises(HydraError): + client.create_config_group(group_name, group_desc) + # Verify group can be deleted + client.delete_config_group(group_name) + groups = client.list_config_groups() + group_names = {g.name for g in groups} + assert group_name not in group_names + + def test_add_config_key_to_group(self, client): + group_name = "Membership Test ConfigKey Group" + group_desc = "Description of test group" + key_name = "group_membership_test_key" + key_value = 46 + _ = client.register_config_key(key_name, "integer") + client.config_key_set_value(key_name, key_value) + client.create_config_group(group_name, group_desc) + # Newly created group must be empty + new_group_keys = client.config_group_list_keys(group_name) + assert len(new_group_keys) == 0 + # Verify that groupless key reports no group + no_group_name = client.config_key_get_group_name(key_name) + assert no_group_name is None + # Add key to group and confirm membership + client.add_config_key_to_group(key_name, group_name) + group_keys = client.config_group_list_keys(group_name) + assert key_name in group_keys + # Now remove the key + client.remove_config_key_from_group(key_name, group_name) + group_keys = client.config_group_list_keys(group_name) + assert key_name not in group_keys + client.unregister_config_key(key_name) + + def test_delete_populated_group(self, client): + group_name = "Populated ConfigKey Group" + group_desc = "Description of test group" + key_name = "populated_group_test_key" + key_value = 46 + _ = client.register_config_key(key_name, "integer") + client.config_key_set_value(key_name, key_value) + client.create_config_group(group_name, group_desc) + client.add_config_key_to_group(key_name, group_name) + client.delete_config_group(group_name) + # The member key and its value are unaffected by group deletion + assert key_name in client.list_config_keys() + assert key_value == client.config_key_get_value(key_name) + client.unregister_config_key(key_name) + + +class TestConfigSets: + def test_config_set_save_and_verify(self, random_keys): + """ + Can a ConfigSet be created, serialised and then + detect any modifications to the serialised version? + """ + _ = random_keys(16) + cs = ConfigSet("Test Configset") + state = cs.save_keys_to_configset() + # Tamper with serialised set + modified = copy.deepcopy(state) + first_key = next(iter(modified["keys"])) + key_val = modified["keys"].pop(first_key) + modified["keys"]["modifed_name"] = key_val + # Naughtiness is detected... + with pytest.raises(ValueError): + cs.verify_configset(modified) + # ...but original unmodified state can be loaded + loaded = cs.verify_configset(state) + + def test_apply_configset_to_db(self, client, random_keys): + """ + 1. Serialises initial state + 2. Deletes this and replaces with temporary ConfigKeys + 3. Re-loads the initial state + 4. Confirms this returns the temporary state + 5. Confirms the final state is equal to initial state + """ + num_keys = 16 + key_names = random_keys(num_keys, do_tidy=False) + cs = ConfigSet("Configset") + # Save initial state + initial_state = cs.save_keys_to_configset() + # Manually delete initial keys + for key_name in key_names: + client.unregister_config_key(key_name) + key_names.clear() + # Generate new state + new_key_names = random_keys(num_keys, do_tidy=False) + # Save second state + second_state = cs.save_keys_to_configset() + # Reload the initial state + ret_state = cs.apply_configset_to_db(initial_state) + # The temporary key state was returned... + assert ret_state["keys"] == second_state["keys"] + # ...which means the initial key state was restored + final_state = cs.save_keys_to_configset() + assert final_state["keys"] == initial_state["keys"] + # Initial keys were restored by load so delete again + for key_name in final_state["keys"]: + client.unregister_config_key(key_name) + + def test_configset_api_export_json(self, client, random_keys): + num_keys = 16 + key_names = random_keys(num_keys) + cs_json = client.export_config_as_json("Configset API test keys", "ConfigSet API test desc") + cs = json.loads(cs_json) + assert len(cs["keys"]) == num_keys + for key_name in key_names: + assert key_name in cs["keys"] + + def test_apply_configset(self, client, random_keys): + # Backup pre-test config state + initial_keys = client.list_config_keys() + initial_state = client.export_config_as_json("Initial state", "Initial state desc") + for key_name in initial_keys: + client.unregister_config_key(key_name) + + num_keys = 16 + # Create an initial state + key_names = random_keys(num_keys) + # Export this as json + cs_json = client.export_config_as_json("Configset API test keys", "ConfigSet API test desc") + # Then delete state + for key_name in key_names: + client.unregister_config_key(key_name) + # Confirm no loaded state + all_keys = client.list_config_keys() + assert len(all_keys) == 0 + # Apply the exported json configset + old_state = client.apply_configset(cs_json) + # Re-export the loaded state + applied_json = client.export_config_as_json("Applied keys") + # And confirm this is equal to initial state + orig_state = json.loads(cs_json) + applied_state = json.loads(applied_json) + assert orig_state["keys"] == applied_state["keys"] + + # Restore initial config state + _ = client.apply_configset(initial_state)