From 595601af55c31c11276282d0022b0d99b745b0ec Mon Sep 17 00:00:00 2001 From: Stephen Knox Date: Tue, 11 Mar 2025 08:37:54 +0000 Subject: [PATCH 1/4] Update the template module to use diskcache instead of a global variable --- hydra_base/lib/cache.py | 33 +++++++++++--- hydra_base/lib/template/__init__.py | 67 ++++++----------------------- 2 files changed, 41 insertions(+), 59 deletions(-) diff --git a/hydra_base/lib/cache.py b/hydra_base/lib/cache.py index 6722a247..3068fa53 100644 --- a/hydra_base/lib/cache.py +++ b/hydra_base/lib/cache.py @@ -5,24 +5,45 @@ config, in which case use that. """ import logging - +import datetime from hydra_base import config as hydraconfig import tempfile log = logging.getLogger(__name__) global cache -if hydraconfig.get('cache', 'type') != "memcached": +def _init_diskcache(): + global cache import diskcache as dc cache = dc.Cache(tempfile.gettempdir()) + +if hydraconfig.get('cache', 'type') != "memcached": + _init_diskcache() + elif hydraconfig.get('cache', 'type') == 'memcached': + try: import pylibmc cache = pylibmc.Client([hydraconfig.get('cache', 'host', '127.0.0.1')], binary=True) - except ModuleNotFoundError: - log.warning("Unable to find pylibmc. Defaulting to diskcache.") - import diskcache as dc - cache = dc.Cache(tempfile.gettempdir()) + + # Check if Memcached server is reachable by setting a test key + test_key = "__connection_test__" + #pick a unique key based on the time + test_value = datetime.datetime.toordinal(datetime.datetime.now()) + try: + cache.set(test_key, test_value, time=1) + cache.get(test_key) + log.info("Connected to memcached server.") + except pylibmc.HostLookupError: + raise ConnectionError("Memcached server not responding.") + + except (ModuleNotFoundError, ConnectionError) as e: + if isinstance(e, ModuleNotFoundError): + log.warning("Unable to find pylibmc. Defaulting to diskcache.") + else: + log.warning("Memcached server not reachable. Defaulting to diskcache.") + + _init_diskcache() def clear_cache(): if hasattr(cache, 'flush_all'): diff --git a/hydra_base/lib/template/__init__.py b/hydra_base/lib/template/__init__.py index 5ce859e9..f5d6c728 100644 --- a/hydra_base/lib/template/__init__.py +++ b/hydra_base/lib/template/__init__.py @@ -61,69 +61,24 @@ validate_resourcescenario, validate_network) -log = logging.getLogger(__name__) - -#A mapping from template ID to a template object -#The cache is a dict of lists of length 2. -#list[0] = the last update time of the template -#list[1] = the template JSONObect -global TEMPLATE_CACHE -TEMPLATE_CACHE = {} - -def _get_template_from_cache(template_id): - """ - Get the template JSONObect from the cache, if it's not expired. - If an expired template is found, it id deleted. - """ - - global TEMPLATE_CACHE - now = datetime.datetime.now() +from hydra_base.lib.cache import cache - #default the template timeout to a day -- they don't change often - timeout = datetime.timedelta(seconds=config.get('CACHE', 'CACHE_TIMEOUT', 86400)) - - cached_template = TEMPLATE_CACHE.get(template_id) - if cached_template is None: - return None - - if cached_template[0] + timeout > now: - log.info("Returning cached template %s", template_id) - return cached_template[1] - else: - _remove_template_from_cache(template_id) - log.info("Found an expired template. Deleting.") +log = logging.getLogger(__name__) - return None +global CACHE_KEY +CACHE_KEY = 'template' def _save_template_to_cache(template): - """ - Save a template to the memory cache. save as a list: - [0] = the current time - [1] = the template - """ - global TEMPLATE_CACHE - - now = datetime.datetime.now() - - TEMPLATE_CACHE[template.id] = [now, template] - - return TEMPLATE_CACHE + cache.set(f"{CACHE_KEY}_{template.id}", template) def _remove_template_from_cache(template_id): """ If a template is in the cache, remove it. """ - global TEMPLATE_CACHE - if TEMPLATE_CACHE.get(template_id): - del TEMPLATE_CACHE[template_id] - + cache.delete(f"{CACHE_KEY}_{template_id}") log.info("Template %s removed from cache.", template_id) -def clear_cache(): - global TEMPLATE_CACHE - TEMPLATE_CACHE = {} - def parse_json_typeattr(type_i, typeattr_j, attribute_j, default_dataset_j, user_id=None): dimension_i = None if attribute_j.dimension_id is not None: @@ -784,13 +739,16 @@ def get_template(template_id, **kwargs): """ Get a specific resource template, by ID. """ - - tmpl_j = _get_template_from_cache(template_id) + log.info("Getting template %s", template_id) + tmpl_j = cache.get(f"{CACHE_KEY}_{template_id}") if tmpl_j is not None: + log.info("Returning cached template") return tmpl_j try: + log.info("Building template") + tmpl_i = db.DBSession.query(Template).filter( Template.id == template_id).one() @@ -802,6 +760,9 @@ def get_template(template_id, **kwargs): #ignore the messing around we've been doing to the ORM objects #db.DBSession.expunge(tmpl_i) + _save_template_to_cache(tmpl_j) + + return tmpl_j except NoResultFound: raise HydraError("Template %s not found"%template_id) From 8449db6518bebe5868e9db98fa7645626b468a00 Mon Sep 17 00:00:00 2001 From: Stephen Knox Date: Mon, 8 Sep 2025 13:01:13 +0100 Subject: [PATCH 2/4] If there is an issue getting a value from memcache, then catch the exceptiopn and return an empty dict --- hydra_base/db/model/project.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/hydra_base/db/model/project.py b/hydra_base/db/model/project.py index 07bc9d2d..e2a628d1 100644 --- a/hydra_base/db/model/project.py +++ b/hydra_base/db/model/project.py @@ -242,10 +242,18 @@ def get_owners(self): """ @classmethod def get_cache(cls, user_id=None): - if user_id is None: - return cache.get(project_cache_key, {}) - else: - return cache.get(project_cache_key, {}).get(user_id, {}) + try: + if user_id is None: + return cache.get(project_cache_key, {}) + else: + return cache.get(project_cache_key, {}).get(user_id, {}) + except Exception as e: + log.exception(e) + err_value = cache.get(project_cache_key, {}) + if type(err_value) is dict: + err_value = err_value.get(user_id) + log.warning(f"Error to get project cache: {project_cache_key}, user_id={user_id}, err_value={err_value}") + return {} @classmethod def set_cache(cls, data): @@ -474,4 +482,4 @@ def get_hierarchy(self, user_id): if self.parent_id: project_hierarchy = project_hierarchy + self.parent.get_hierarchy(user_id) - return project_hierarchy \ No newline at end of file + return project_hierarchy From d9fda62ea6da919403a01e28d5f7985b75798022 Mon Sep 17 00:00:00 2001 From: Stephen Knox Date: Mon, 8 Sep 2025 19:03:36 +0100 Subject: [PATCH 3/4] Connect correctly to the cache, and remove child templates from the cache when a parent template is updated --- hydra_base/lib/attributes.py | 4 ++-- hydra_base/lib/cache.py | 2 +- hydra_base/lib/template/__init__.py | 11 ++++++++--- tests/conftest.py | 4 ++-- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/hydra_base/lib/attributes.py b/hydra_base/lib/attributes.py index 046d46bc..2f3ba2f5 100644 --- a/hydra_base/lib/attributes.py +++ b/hydra_base/lib/attributes.py @@ -942,7 +942,7 @@ def add_resource_attributes(resource_attributes, **kwargs): mark_changed(db.DBSession()) cache.set(f'network_resource_attributes_{network_id}', - cache.get(f'network_resource_attributes_{network_id}') + [JSONObject(obj) for obj in objs], expire=60*60) + cache.get(f'network_resource_attributes_{network_id}') + [JSONObject(obj) for obj in objs], time=60*60) db.DBSession.flush() @@ -1144,7 +1144,7 @@ def get_all_network_resourceattributes(network_id, template_id=None, return_orm= ra_j.attr = JSONObject(ra.attr) network_attributes.append(ra_j) - cache.set(f'network_resource_attributes_{network_id}', network_attributes, expire=60*60) + cache.set(f'network_resource_attributes_{network_id}', network_attributes, time=60*60) return network_attributes diff --git a/hydra_base/lib/cache.py b/hydra_base/lib/cache.py index ecc816d4..c126007c 100644 --- a/hydra_base/lib/cache.py +++ b/hydra_base/lib/cache.py @@ -36,7 +36,7 @@ def _init_diskcache(): cache.set(test_key, test_value, time=1) cache.get(test_key) log.info("Connected to memcached server.") - except pylibmc.HostLookupError: + except (pylibmc.HostLookupError, pylibmc.ConnectionError): raise ConnectionError("Memcached server not responding.") except (ModuleNotFoundError, ConnectionError) as e: diff --git a/hydra_base/lib/template/__init__.py b/hydra_base/lib/template/__init__.py index 9d19c3db..57756468 100644 --- a/hydra_base/lib/template/__init__.py +++ b/hydra_base/lib/template/__init__.py @@ -794,6 +794,11 @@ def add_templatetype(templatetype, **kwargs): _remove_template_from_cache(type_i.template_id) + #remove any child templates from the cache too + child_types = db.DBSession.query(TemplateType).filter(TemplateType.parent_id == type_i.id).all() + for child_type in child_types: + _remove_template_from_cache(child_type.template_id) + db.DBSession.flush() return type_i @@ -1191,7 +1196,7 @@ def clone_templatetype(type_id, name=None, **kwargs): if col.name in ['id', 'cr_date', 'updated_at']: continue setattr(cloned_templatetype, col.name, getattr(parent_templatetype, col.name)) - + if name is not None: base_name = name else: @@ -1207,7 +1212,7 @@ def clone_templatetype(type_id, name=None, **kwargs): ).first() is not None: clone_name = f"{base_name} {counter}" counter += 1 - + cloned_templatetype.name = clone_name typeattrs = db.DBSession.query(TypeAttr).filter( @@ -1219,7 +1224,7 @@ def clone_templatetype(type_id, name=None, **kwargs): continue setattr(cloned_typeattr, col.name, getattr(typeattr, col.name)) cloned_templatetype.typeattrs.append(cloned_typeattr) - + db.DBSession.add(cloned_templatetype) db.DBSession.flush() # Flush to get the ID diff --git a/tests/conftest.py b/tests/conftest.py index 4f9e68a0..e031b6bc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -129,11 +129,11 @@ def client(connection_type, testdb_uri): pytest.user_c = client.testutils.create_user("UserC", role='developer') pytest.user_d = client.testutils.create_user("UserD", role='developer') yield client - #??? - hydra_base.lib.template.clear_cache() + hydra_base.db.close_session() clear_cache() # clear the user project cache + try: drop_tables(testdb_uri) except Exception as err: From 951aa8d4cbb21d10e530875ea6401e9bd7eaf761 Mon Sep 17 00:00:00 2001 From: Stephen Knox Date: Mon, 8 Sep 2025 19:48:42 +0100 Subject: [PATCH 4/4] Set expiry properly to work with both memcached and diskcacke --- hydra_base/lib/attributes.py | 4 ++-- hydra_base/lib/cache.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hydra_base/lib/attributes.py b/hydra_base/lib/attributes.py index 2f3ba2f5..101dc6bb 100644 --- a/hydra_base/lib/attributes.py +++ b/hydra_base/lib/attributes.py @@ -942,7 +942,7 @@ def add_resource_attributes(resource_attributes, **kwargs): mark_changed(db.DBSession()) cache.set(f'network_resource_attributes_{network_id}', - cache.get(f'network_resource_attributes_{network_id}') + [JSONObject(obj) for obj in objs], time=60*60) + cache.get(f'network_resource_attributes_{network_id}') + [JSONObject(obj) for obj in objs], 60*60) db.DBSession.flush() @@ -1144,7 +1144,7 @@ def get_all_network_resourceattributes(network_id, template_id=None, return_orm= ra_j.attr = JSONObject(ra.attr) network_attributes.append(ra_j) - cache.set(f'network_resource_attributes_{network_id}', network_attributes, time=60*60) + cache.set(f'network_resource_attributes_{network_id}', network_attributes, 60*60) return network_attributes diff --git a/hydra_base/lib/cache.py b/hydra_base/lib/cache.py index c126007c..b19fef26 100644 --- a/hydra_base/lib/cache.py +++ b/hydra_base/lib/cache.py @@ -33,7 +33,7 @@ def _init_diskcache(): #pick a unique key based on the time test_value = datetime.datetime.toordinal(datetime.datetime.now()) try: - cache.set(test_key, test_value, time=1) + cache.set(test_key, test_value, 1) cache.get(test_key) log.info("Connected to memcached server.") except (pylibmc.HostLookupError, pylibmc.ConnectionError):