157 add a config db table

.github/workflows/ci.yml

@@ -10,6 +10,33 @@ jobs:
       COV_MIN: 70  # Minimum acceptable coverage level
       TEST_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      HYDRA_CONFIGSET: default_configset.json
+      HYDRA_DB_SERVER: 127.0.0.1
+      HYDRA_DB_NAME: hydradb
+      HYDRA_DB_USER: root
+      HYDRA_DB_PASSWD: root
+      HYDRA_DB_AUTOCREATE: Y
+      HYDRA_MYSQL_POOL_PREPING: True
+      HYDRA_MYSQL_POOL_SIZE: 10
+      HYDRA_MYSQL_POOL_RECYCLE: 300
+      HYDRA_MYSQL_POOL_TIMEOUT: 10
+      HYDRA_MYSQL_MAX_OVERFLOW: 20
+      HYDRA_CACHETYPE: memcached
+      HYDRA_CACHEHOST: 127.0.0.1
+      HYDRA_LOG_CONFPATH: hydra-base/logging.conf
+      HYDRA_LOG_FILEDIR: .hydra/log
+      HYDRA_MONGO_HOST: localhost
+      HYDRA_MONGO_PORT: 27017
+      HYDRA_MONGO_DB_NAME: hydra
+      HYDRA_MONGO_USER:
+      HYDRA_MONGO_PASSWD:
+      HYDRA_MONGO_DATASETS: datasets
+      HYDRA_MONGO_THRESHOLD: 4096
+      HYDRA_MONGO_DIRECT_LOCATION_TOKEN: mongo_direct
+      HYDRA_MONGO_VALUE_LOCATION_KEY: value_storage_location
+      HYDRA_DISABLE_HDF: False
+      HYDRA_HDF_FILESTORE: /tmp
+      HYDRA_CONFIG_HASH_KEY: dev_only_secret_key
 
     strategy:
       matrix:
@@ -60,4 +87,4 @@ jobs:
              pip install -e .
 
       - name: Run pytests
-        run: pytest --db-backend=mysql --cov=hydra_base --cov-fail-under=$COV_MIN
+        run: pytest --db-backend=mysql --cov=hydra_base

default_configset.json

@@ -0,0 +1 @@
+{"name": "Default ConfigSet", "description": "The default Hydra startup state", "timestamp": "2025-01-24 15:15:28", "keys": {"log_level": {"type": "string", "value": "INFO", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_aux_dir": {"type": "string", "value": "__HOME_DIR__/.hydra", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "datetime_format": {"type": "string", "value": "%Y-%m-%dT%H:%M:%S.%f000Z", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "seasonal_key": {"type": "integer", "value": 9999, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "seasonal_year": {"type": "integer", "value": 1678, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "db_instance": {"type": "string", "value": "MySQL", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "db_upper_bound": {"type": "integer", "value": 100, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "db_lower_bound": {"type": "integer", "value": 5, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "db_export_target": {"type": "string", "value": "__HYDRA_AUX_DIR__/audit", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "db_purge_threshold": {"type": "integer", "value": 10000, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "db_compression_threshold": {"type": "integer", "value": 50000, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "sqlite_backup_dir": {"type": "string", "value": "__HYDRA_AUX_DIR__/audit", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "sqlite_dbfile": {"type": "string", "value": "__HYDRA_BASE_DIR__/HydraDB/hydra.db", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "sqlite_backup_url": {"type": "string", "value": "__SQLITE_BACKUP_DIR__/audit.db", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_domain": {"type": "string", "value": "127.0.0.1", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_port": {"type": "integer", "value": 8080, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "hydra_server_path": {"type": "string", "value": "soap", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_json_path": {"type": "string", "value": "json", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_http_path": {"type": "string", "value": "http", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_soap_path": {"type": "string", "value": "soap", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_url": {"type": "string", "value": "http://__HYDRA_SERVER_DOMAIN__:__HYDRA_SERVER_PORT__/__HYDRA_SERVER_PATH__?wsdl", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_server_layout_xsd_path": {"type": "string", "value": "__HYDRA_BASE_DIR__/static/resource_layout.xsd", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_domain": {"type": "string", "value": "http://127.0.0.1", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_port": {"type": "integer", "value": 8080, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "hydra_client_path": {"type": "string", "value": "json", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_json_path": {"type": "string", "value": "json", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_http_path": {"type": "string", "value": "http", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_soap_path": {"type": "string", "value": "soap", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_user": {"type": "string", "value": "root", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "hydra_client_password": {"type": "string", "value": "", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "filesys_img_src": {"type": "string", "value": "__HOME_DIR__/.hydra/images/", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "filesys_file_src": {"type": "string", "value": "__HOME_DIR__/.hydra/files/", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "unit_conversion_user_file": {"type": "string", "value": "__HYDRA_BASE_DIR__/static/user_units.xml", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "unit_conversion_default_file": {"type": "string", "value": "__HYDRA_BASE_DIR__/static/unit_definitions.xml", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "templates_template_xsd_path": {"type": "string", "value": "__HYDRA_BASE_DIR__/static/template.xsd", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "logging_conf_log_config_path": {"type": "string", "value": "__HYDRA_BASE_DIR__/logging.conf", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "logging_conf_log_file_dir": {"type": "string", "value": "__HYDRA_BASE_DIR__/log", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "search_page_size": {"type": "integer", "value": 2000, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "polyvis_polyvis_url": {"type": "string", "value": "http://localhost:5000/", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "security_max_login_attempts": {"type": "integer", "value": 7, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}, "cache_type": {"type": "string", "value": "diskcache", "rules": "{\"max_length\": null, \"min_length\": null}", "description": ""}, "limits_project_max_nest_depth": {"type": "integer", "value": 32, "rules": "{\"max_value\": null, \"min_value\": null}", "description": ""}}, "digest": "48050a66b5a9669ec6c41f0ffd09c22ec6b8fa6c32debd8bb3d3d7186d6220f2"}

hydra_base/__init__.py

@@ -45,25 +45,6 @@
 
 log.debug(" \n ")
 
-log.debug("CONFIG localfiles %s found in %s", len(config.localfiles), config.localfile)
-
-log.debug("CONFIG repofiles %s found in %s", len(config.repofiles), config.repofile)
-
-log.debug("CONFIG userfiles %s found in %s", len(config.userfiles), config.userfile)
-
-log.debug("CONFIG sysfiles %s found in %s", len(config.sysfiles), config.sysfile)
-
-if len(config.sysfiles) + len(config.repofiles) + len(config.userfiles) + len(config.sysfiles) == 0:
-    log.critical("No config found. Please put your ini file into one of the files listed beside CONFIG above.")
-
-if config.get("security", "max_login_attempts") is None:
-    """  Absence of max_login_attempts results in all users unable to log in,
-         so ensure this is defined in config, or fail.
-    """
-    raise RuntimeError("Config 'security' section must define 'max_login_attempts'")
-
-log.debug(" \n ")
-
 from .lib.attributes import *
 from .lib.data import *
 from .lib.groups import *
@@ -80,3 +61,4 @@
 from .lib.units import *
 from .lib.users import *
 from .lib.service import *
+from .lib.hydraconfig import *

hydra_base/config.py

@@ -16,149 +16,149 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with HydraPlatform.  If not, see <http://www.gnu.org/licenses/>
 #
+import logging
 import os
-import glob
+import re
 import sys
 
-PYTHONVERSION = sys.version_info
-if PYTHONVERSION >= (3,2):
-    import configparser as ConfigParser
-else:
-    import ConfigParser
-import logging
+from hydra_base import db
+from hydra_base.exceptions import HydraError
+
 
-global CONFIG
 CONFIG = None
 
-global localfiles
-global localfile
-global repofile
-global repofiles
-global userfile
-global userfiles
-global sysfile
-global sysfiles
 
 def load_config():
-    """Load a config file. This function looks for a config (*.ini) file in the
-    following order::
-
-        (1) ./*.ini
-        (2) ~/.config/hydra/
-        (3) /etc/hydra
-        (4) /path/to/hydra_base/*.ini
-
-    (1) will override (2) will override (3) will override (4). Parameters not
-    defined in (1) will be taken from (2). Parameters not defined in (2) will
-    be taken from (3).  (3) is the config folder that will be checked out from
-    the svn repository.  (2) Will be be provided as soon as an installable
-    distribution is available. (1) will usually be written individually by
-    every user."""
-    global localfiles
-    global localfile
-    global repofile
-    global repofiles
-    global userfile
-    global userfiles
-    global sysfile
-    global sysfiles
     global CONFIG
     logging.basicConfig(level='INFO')
 
-    config = ConfigParser.ConfigParser(allow_no_value=True)
+    from hydra_base.lib.hydraconfig import (
+        apply_configset,
+        config_key_set_value,
+        list_config_keys,
+        register_config_key
+    )
+    from pprint import pprint
 
     modulepath = os.path.dirname(os.path.abspath(__file__))
-
-    localfile = os.path.join(os.getcwd(), 'hydra.ini')
-    localfiles = glob.glob(localfile)
-
-    repofile = os.path.join(modulepath, 'hydra.ini')
-    repofiles = glob.glob(repofile)
-
-    if sys.platform.startswith("win"):
-        from hydra_base.util.windows import win_get_common_documents
-        userfile = os.path.join(os.path.expanduser('~'),'AppData','Local','hydra.ini')
-        userfiles = glob.glob(userfile)
-
-        sysfile = os.path.join(win_get_common_documents(), 'Hydra','hydra.ini')
-        sysfiles = glob.glob(sysfile)
-    else:
-        userfile = os.path.join(os.path.expanduser('~'), '.hydra', 'hydra.ini')
-        userfiles = glob.glob(userfile)
-
-        sysfile = os.path.join('etc','hydra','hydra.ini')
-        sysfiles = glob.glob(sysfile)
-
-
-    for ini_file in repofiles:
-        logging.debug("Repofile: %s"%ini_file)
-        config.read(ini_file)
-    for ini_file in sysfiles:
-        logging.debug("Sysfile: %s"%ini_file)
-        config.read(ini_file)
-    for ini_file in userfiles:
-        logging.debug("Userfile: %s"%ini_file)
-        config.read(ini_file)
-    for ini_file in localfiles:
-        logging.info("Localfile: %s"%ini_file)
-        config.read(ini_file)
-
-    env_value = os.environ.get('HYDRA_CONFIG')
-    if env_value is not None:
-        if os.path.exists(env_value):
-            config.read(env_value)
+    home_dir = os.environ.get("HYDRA_HOME_DIR", '~')
+    hydra_base_dir = os.environ.get("HYDRA_BASE_DIR", modulepath)
+    configset = os.environ.get("HYDRA_CONFIGSET", "default_configset.json")
+
+
+    if not db.DBSession:
+        db.connect()
+    keys = list_config_keys()
+    if len(keys) == 0:
+        # No existing configset has been loaded
+        # Load set specified by env or default
+        # and register substitution keys
+        with open(configset, 'r') as fp:
+            cs_json = fp.read()
+        apply_configset(cs_json)
+
+        try:
+            register_config_key("home_dir", "string")
+            config_key_set_value("home_dir", home_dir)
+        except Exception:
+            pass
+
+        try:
+            register_config_key("hydra_base_dir", "string")
+            config_key_set_value("hydra_base_dir", hydra_base_dir)
+        except Exception:
+            pass
+
+    CONFIG = True
+    return
+
+
+def read_env_db_config():
+    return {
+      "hydra_db_server": os.environ.get("HYDRA_DB_SERVER"),
+      "hydra_db_name": os.environ.get("HYDRA_DB_NAME"),
+      "hydra_db_user": os.environ.get("HYDRA_DB_USER"),
+      "hydra_db_passwd": os.environ.get("HYDRA_DB_PASSWD"),
+      "hydra_db_autocreate": os.environ.get("HYDRA_DB_AUTOCREATE"),
+      "hydra_mysql_pool_preping": os.environ.get("HYDRA_MYSQL_POOL_PREPING"),
+      "hydra_mysql_pool_size": os.environ.get("HYDRA_MYSQL_POOL_SIZE"),
+      "hydra_mysql_pool_recycle": os.environ.get("HYDRA_MYSQL_POOL_RECYCLE"),
+      "hydra_mysql_pool_timeout": os.environ.get("HYDRA_MYSQL_POOL_TIMEOUT"),
+      "hydra_mysql_max_overflow": os.environ.get("HYDRA_MYSQL_MAX_OVERFLOW")
+    }
+
+
+def read_env_startup_config():
+    return {
+      "hydra_cachetype": os.environ.get("HYDRA_CACHETYPE"),
+      "hydra_cachehost": os.environ.get("HYDRA_CACHEHOST"),
+      "hydra_log_confpath": os.environ.get("HYDRA_LOG_CONFPATH"),
+      "hydra_log_filedir": os.environ.get("HYDRA_LOG_FILEDIR"),
+      "hydra_config_hash_key": os.environ.get("HYDRA_CONFIG_HASH_KEY")
+    }
+
+
+def get_startup_config():
+    db_config = read_env_db_config()
+    db_config["url"] = f"mysql+mysqldb://{db_config['hydra_db_user']}:{db_config['hydra_db_passwd']}"\
+                       f"@{db_config['hydra_db_server']}/{db_config['hydra_db_name']}"
+
+    db_config.update(read_env_startup_config())
+    return db_config
+
+
+def make_value_substitutions(value):
+    if not isinstance(value, str):
+        return value
+
+    p = r"__([a-zA-Z_]+)__"
+    tokens = re.findall(p, value)
+    for token in tokens:
+        try:
+            tkey = token.strip('_').lower()
+            tval = config_key_get_value(tkey)
+            value = value.replace(token, tval)
+        except HydraError:
+            pass  # Do not substitute invalid keys
+
+    return value
+
+
+def get(*args, default=None):
+    from hydra_base.lib.hydraconfig import (
+        config_key_get_value
+    )
+    """
+      The section delineated below is a temporary
+      routine to allow calls from the hydra_client
+      module which use the old "section+option"
+      form of config.get to succeed.
+      This is required for tests to pass in CI
+      and should be removed on merge and update
+      of hydra_client.
+    """
+    # Temporary CI adjustment begins
+    import inspect
+    sf = inspect.stack()[1]
+    mod = inspect.getmodule(sf[0])
+    if mod.__name__.lower().startswith("hydra_client"):
+        if args[0].lower() == "default":
+            key = args[1]
         else:
-            logging.warning('HYDRA_CONFIG set as %s but file does not exist', env_value)
-
-
-    try:
-        home_dir = config.get('DEFAULT', 'home_dir')
-    except:
-        home_dir = os.environ.get('HYDRA_HOME_DIR', '~')
-    config.set('DEFAULT', 'home_dir', os.path.expanduser(home_dir))
-
-    try:
-        hydra_base = config.get('DEFAULT', 'hydra_base_dir')
-    except:
-        hydra_base = os.environ.get('HYDRA_BASE_DIR', modulepath)
-    config.set('DEFAULT', 'hydra_base_dir', os.path.expanduser(hydra_base))
-
-    read_values_from_environment(config, 'mysqld', 'server_name')
-
-
-    CONFIG = config
-
-    return config
-
-def read_values_from_environment(config, section_key, options_key):
-    #####################################
-    # Settings for docker ENV variables #
-    #####################################
-    env_var_name='HYDRA_DOCKER__' + section_key + '__' + options_key
-
-    env_value = os.environ.get(env_var_name, '-')
-    if (env_value != '-'):
-        # Substitute the server_name with the end variable
-        # print("Presente")
-        config.set(section_key, options_key, env_value)
-
-
-def get(section, option, default=None):
-
-    if CONFIG is None:
-        load_config()
-
-    try:
-        return CONFIG.get(section, option)
-    except:
-        return default
-
-def getint(section, option, default=None):
+            key = f"{args[0]}_{args[1]}"
+        if len(args) == 3:
+            default = args[2]
+    else:
+        key = args[0]
+        if len(args) == 2:
+            default = args[1]
 
-    if CONFIG is None:
-        load_config()
+    # Temporary CI adjustment ends
 
     try:
-        return CONFIG.getint(section, option)
+        value = config_key_get_value(key)
+        value = make_value_substitutions(value)
+        print(f"{key} = {value}")
+        return value
     except:
         return default