Update the template module to use diskcache instead of a global variable

hydra_base/db/model/project.py

@@ -242,10 +242,18 @@ def get_owners(self):
     """
     @classmethod
     def get_cache(cls, user_id=None):
-        if user_id is None:
-            return cache.get(project_cache_key, {})
-        else:
-            return cache.get(project_cache_key, {}).get(user_id, {})
+        try:
+            if user_id is None:
+                return cache.get(project_cache_key, {})
+            else:
+                return cache.get(project_cache_key, {}).get(user_id, {})
+        except Exception as e:
+            log.exception(e)
+            err_value = cache.get(project_cache_key, {})
+            if type(err_value) is dict:
+                err_value = err_value.get(user_id)
+            log.warning(f"Error to get project cache: {project_cache_key}, user_id={user_id}, err_value={err_value}")
+            return {}
 
     @classmethod
     def set_cache(cls, data):
@@ -474,4 +482,4 @@ def get_hierarchy(self, user_id):
         if self.parent_id:
             project_hierarchy = project_hierarchy + self.parent.get_hierarchy(user_id)
 
-        return project_hierarchy
+        return project_hierarchy

hydra_base/lib/attributes.py

@@ -942,7 +942,7 @@ def add_resource_attributes(resource_attributes, **kwargs):
         mark_changed(db.DBSession())
 
         cache.set(f'network_resource_attributes_{network_id}',
-            cache.get(f'network_resource_attributes_{network_id}') + [JSONObject(obj) for obj in objs], expire=60*60)
+            cache.get(f'network_resource_attributes_{network_id}') + [JSONObject(obj) for obj in objs], 60*60)
 
     db.DBSession.flush()
 
@@ -1144,7 +1144,7 @@ def get_all_network_resourceattributes(network_id, template_id=None, return_orm=
             ra_j.attr = JSONObject(ra.attr)
             network_attributes.append(ra_j)
 
-    cache.set(f'network_resource_attributes_{network_id}', network_attributes, expire=60*60)
+    cache.set(f'network_resource_attributes_{network_id}', network_attributes, 60*60)
 
     return network_attributes
 

hydra_base/lib/cache.py

@@ -5,33 +5,47 @@
     config, in which case use that.
 """
 import logging
-
+import datetime
 from hydra_base import config as hydraconfig
 import tempfile
 
 log = logging.getLogger(__name__)
 global cache
 
-if hydraconfig.get('cache', 'type') != "memcached":
+def _init_diskcache():
     log.info("Using diskcache for caching.")
+    global cache
     import diskcache as dc
     cache = dc.Cache(tempfile.gettempdir())
+
+if hydraconfig.get('cache', 'type') != "memcached":
+    _init_diskcache()
+
 elif hydraconfig.get('cache', 'type') == 'memcached':
-    log.info("Using memcached for caching.")
     try:
         import pylibmc
         host = hydraconfig.get('cache', 'host', '127.0.0.1')
         port = hydraconfig.get('cache', 'port', 31211)
         cache = pylibmc.Client([f"{host}:{port}"], binary=True)
-        log.info(f"Memcached client initialized with host: {host}:{port}")
-    except ModuleNotFoundError:
-        log.warning("Unable to find pylibmc. Defaulting to diskcache.")
-        import diskcache as dc
-        cache = dc.Cache(tempfile.gettempdir())
-    except Exception as e:
-        log.error(f"Error initializing memcached: {e}.  Defaulting to diskcache.")
-        import diskcache as dc
-        cache = dc.Cache(tempfile.gettempdir())
+
+        # Check if Memcached server is reachable by setting a test key
+        test_key = "__connection_test__"
+        #pick a unique key based on the time
+        test_value = datetime.datetime.toordinal(datetime.datetime.now())
+        try:
+            cache.set(test_key, test_value, 1)
+            cache.get(test_key)
+            log.info("Connected to memcached server.")
+        except (pylibmc.HostLookupError, pylibmc.ConnectionError):
+            raise ConnectionError("Memcached server not responding.")
+
+    except (ModuleNotFoundError, ConnectionError) as e:
+        if isinstance(e, ModuleNotFoundError):
+            log.warning("Unable to find pylibmc. Defaulting to diskcache.")
+        else:
+            log.warning("Memcached server not reachable. Defaulting to diskcache.")
+
+        _init_diskcache()
 
 def clear_cache():
     if hasattr(cache, 'flush_all'):

hydra_base/lib/template/__init__.py

@@ -61,69 +61,24 @@
     validate_resourcescenario,
     validate_network)
 
-log = logging.getLogger(__name__)
-
-#A mapping from template ID to a template object
-#The cache is a dict of lists of length 2.
-#list[0] = the last update time of the template
-#list[1] = the template JSONObect
-global TEMPLATE_CACHE
-TEMPLATE_CACHE = {}
-
-def _get_template_from_cache(template_id):
-    """
-        Get the template JSONObect from the cache, if it's not expired.
-        If an expired template is found, it id deleted.
-    """
-
-    global TEMPLATE_CACHE
-
-    now = datetime.datetime.now()
 
-    #default the template timeout to a day -- they don't change often
-    timeout = datetime.timedelta(seconds=config.get('CACHE', 'CACHE_TIMEOUT', 86400))
+from hydra_base.lib.cache import cache
 
-    cached_template = TEMPLATE_CACHE.get(template_id)
-    if cached_template is None:
-        return None
+log = logging.getLogger(__name__)
 
-    if cached_template[0] + timeout > now:
-        log.info("Returning cached template %s", template_id)
-        return cached_template[1]
-    else:
-        _remove_template_from_cache(template_id)
-        log.info("Found an expired template. Deleting.")
-
-    return None
+global CACHE_KEY
+CACHE_KEY = 'template'
 
 def _save_template_to_cache(template):
-    """
-        Save a template to the memory cache. save as a list:
-        [0] = the current time
-        [1] = the template
-    """
-    global TEMPLATE_CACHE
-
-    now = datetime.datetime.now()
-
-    TEMPLATE_CACHE[template.id] = [now, template]
-
-    return TEMPLATE_CACHE
+    cache.set(f"{CACHE_KEY}_{template.id}", template)
 
 def _remove_template_from_cache(template_id):
     """
         If a template is in the cache, remove it.
     """
-    global TEMPLATE_CACHE
-    if TEMPLATE_CACHE.get(template_id):
-        del TEMPLATE_CACHE[template_id]
-
+    cache.delete(f"{CACHE_KEY}_{template_id}")
     log.info("Template %s removed from cache.", template_id)
 
-def clear_cache():
-    global TEMPLATE_CACHE
-    TEMPLATE_CACHE = {}
-
 def parse_json_typeattr(type_i, typeattr_j, attribute_j, default_dataset_j, user_id=None):
     dimension_i = None
     if attribute_j.dimension_id is not None:
@@ -784,13 +739,16 @@ def get_template(template_id, **kwargs):
     """
         Get a specific resource template, by ID.
     """
-
-    tmpl_j = _get_template_from_cache(template_id)
+    log.info("Getting template %s", template_id)
+    tmpl_j = cache.get(f"{CACHE_KEY}_{template_id}")
 
     if tmpl_j is not None:
+        log.info("Returning cached template")
         return tmpl_j
 
     try:
+        log.info("Building template")
+
         tmpl_i = db.DBSession.query(Template).filter(
             Template.id == template_id).one()
 
@@ -802,6 +760,9 @@ def get_template(template_id, **kwargs):
         #ignore the messing around we've been doing to the ORM objects
         #db.DBSession.expunge(tmpl_i)
 
+        _save_template_to_cache(tmpl_j)
+
+
         return tmpl_j
     except NoResultFound:
         raise HydraError("Template %s not found"%template_id)
@@ -833,6 +794,11 @@ def add_templatetype(templatetype, **kwargs):
 
     _remove_template_from_cache(type_i.template_id)
 
+    #remove any child templates from the cache too
+    child_types = db.DBSession.query(TemplateType).filter(TemplateType.parent_id == type_i.id).all()
+    for child_type in child_types:
+        _remove_template_from_cache(child_type.template_id)
+
     db.DBSession.flush()
 
     return type_i
@@ -1230,7 +1196,7 @@ def clone_templatetype(type_id, name=None, **kwargs):
         if col.name in ['id', 'cr_date', 'updated_at']:
             continue
         setattr(cloned_templatetype, col.name, getattr(parent_templatetype, col.name))
-    
+
     if name is not None:
         base_name = name
     else:
@@ -1246,7 +1212,7 @@ def clone_templatetype(type_id, name=None, **kwargs):
     ).first() is not None:
         clone_name = f"{base_name} {counter}"
         counter += 1
-    
+
     cloned_templatetype.name = clone_name
 
     typeattrs = db.DBSession.query(TypeAttr).filter(
@@ -1258,7 +1224,7 @@ def clone_templatetype(type_id, name=None, **kwargs):
                 continue
             setattr(cloned_typeattr, col.name, getattr(typeattr, col.name))
         cloned_templatetype.typeattrs.append(cloned_typeattr)
-    
+
     db.DBSession.add(cloned_templatetype)
     db.DBSession.flush()  # Flush to get the ID
 

tests/conftest.py

@@ -129,11 +129,11 @@ def client(connection_type, testdb_uri):
     pytest.user_c = client.testutils.create_user("UserC", role='developer')
     pytest.user_d = client.testutils.create_user("UserD", role='developer')
     yield client
-    #???
-    hydra_base.lib.template.clear_cache()
+
     hydra_base.db.close_session()
 
     clear_cache() # clear the user project cache
+
     try:
         drop_tables(testdb_uri)
     except Exception as err: