diff --git a/hydra_base/db/model/project.py b/hydra_base/db/model/project.py index 07bc9d2d..e2a628d1 100644 --- a/hydra_base/db/model/project.py +++ b/hydra_base/db/model/project.py @@ -242,10 +242,18 @@ def get_owners(self): """ @classmethod def get_cache(cls, user_id=None): - if user_id is None: - return cache.get(project_cache_key, {}) - else: - return cache.get(project_cache_key, {}).get(user_id, {}) + try: + if user_id is None: + return cache.get(project_cache_key, {}) + else: + return cache.get(project_cache_key, {}).get(user_id, {}) + except Exception as e: + log.exception(e) + err_value = cache.get(project_cache_key, {}) + if type(err_value) is dict: + err_value = err_value.get(user_id) + log.warning(f"Error to get project cache: {project_cache_key}, user_id={user_id}, err_value={err_value}") + return {} @classmethod def set_cache(cls, data): @@ -474,4 +482,4 @@ def get_hierarchy(self, user_id): if self.parent_id: project_hierarchy = project_hierarchy + self.parent.get_hierarchy(user_id) - return project_hierarchy \ No newline at end of file + return project_hierarchy diff --git a/hydra_base/lib/attributes.py b/hydra_base/lib/attributes.py index 046d46bc..101dc6bb 100644 --- a/hydra_base/lib/attributes.py +++ b/hydra_base/lib/attributes.py @@ -942,7 +942,7 @@ def add_resource_attributes(resource_attributes, **kwargs): mark_changed(db.DBSession()) cache.set(f'network_resource_attributes_{network_id}', - cache.get(f'network_resource_attributes_{network_id}') + [JSONObject(obj) for obj in objs], expire=60*60) + cache.get(f'network_resource_attributes_{network_id}') + [JSONObject(obj) for obj in objs], 60*60) db.DBSession.flush() @@ -1144,7 +1144,7 @@ def get_all_network_resourceattributes(network_id, template_id=None, return_orm= ra_j.attr = JSONObject(ra.attr) network_attributes.append(ra_j) - cache.set(f'network_resource_attributes_{network_id}', network_attributes, expire=60*60) + cache.set(f'network_resource_attributes_{network_id}', network_attributes, 60*60) return network_attributes diff --git a/hydra_base/lib/cache.py b/hydra_base/lib/cache.py index 5f1982af..b19fef26 100644 --- a/hydra_base/lib/cache.py +++ b/hydra_base/lib/cache.py @@ -5,33 +5,47 @@ config, in which case use that. """ import logging - +import datetime from hydra_base import config as hydraconfig import tempfile log = logging.getLogger(__name__) global cache -if hydraconfig.get('cache', 'type') != "memcached": +def _init_diskcache(): log.info("Using diskcache for caching.") + global cache import diskcache as dc cache = dc.Cache(tempfile.gettempdir()) + +if hydraconfig.get('cache', 'type') != "memcached": + _init_diskcache() + elif hydraconfig.get('cache', 'type') == 'memcached': - log.info("Using memcached for caching.") try: import pylibmc host = hydraconfig.get('cache', 'host', '127.0.0.1') port = hydraconfig.get('cache', 'port', 31211) cache = pylibmc.Client([f"{host}:{port}"], binary=True) - log.info(f"Memcached client initialized with host: {host}:{port}") - except ModuleNotFoundError: - log.warning("Unable to find pylibmc. Defaulting to diskcache.") - import diskcache as dc - cache = dc.Cache(tempfile.gettempdir()) - except Exception as e: - log.error(f"Error initializing memcached: {e}. Defaulting to diskcache.") - import diskcache as dc - cache = dc.Cache(tempfile.gettempdir()) + + # Check if Memcached server is reachable by setting a test key + test_key = "__connection_test__" + #pick a unique key based on the time + test_value = datetime.datetime.toordinal(datetime.datetime.now()) + try: + cache.set(test_key, test_value, 1) + cache.get(test_key) + log.info("Connected to memcached server.") + except (pylibmc.HostLookupError, pylibmc.ConnectionError): + raise ConnectionError("Memcached server not responding.") + + except (ModuleNotFoundError, ConnectionError) as e: + if isinstance(e, ModuleNotFoundError): + log.warning("Unable to find pylibmc. Defaulting to diskcache.") + else: + log.warning("Memcached server not reachable. Defaulting to diskcache.") + + _init_diskcache() def clear_cache(): if hasattr(cache, 'flush_all'): diff --git a/hydra_base/lib/template/__init__.py b/hydra_base/lib/template/__init__.py index 4d20227f..57756468 100644 --- a/hydra_base/lib/template/__init__.py +++ b/hydra_base/lib/template/__init__.py @@ -61,69 +61,24 @@ validate_resourcescenario, validate_network) -log = logging.getLogger(__name__) - -#A mapping from template ID to a template object -#The cache is a dict of lists of length 2. -#list[0] = the last update time of the template -#list[1] = the template JSONObect -global TEMPLATE_CACHE -TEMPLATE_CACHE = {} - -def _get_template_from_cache(template_id): - """ - Get the template JSONObect from the cache, if it's not expired. - If an expired template is found, it id deleted. - """ - - global TEMPLATE_CACHE - - now = datetime.datetime.now() - #default the template timeout to a day -- they don't change often - timeout = datetime.timedelta(seconds=config.get('CACHE', 'CACHE_TIMEOUT', 86400)) +from hydra_base.lib.cache import cache - cached_template = TEMPLATE_CACHE.get(template_id) - if cached_template is None: - return None +log = logging.getLogger(__name__) - if cached_template[0] + timeout > now: - log.info("Returning cached template %s", template_id) - return cached_template[1] - else: - _remove_template_from_cache(template_id) - log.info("Found an expired template. Deleting.") - - return None +global CACHE_KEY +CACHE_KEY = 'template' def _save_template_to_cache(template): - """ - Save a template to the memory cache. save as a list: - [0] = the current time - [1] = the template - """ - global TEMPLATE_CACHE - - now = datetime.datetime.now() - - TEMPLATE_CACHE[template.id] = [now, template] - - return TEMPLATE_CACHE + cache.set(f"{CACHE_KEY}_{template.id}", template) def _remove_template_from_cache(template_id): """ If a template is in the cache, remove it. """ - global TEMPLATE_CACHE - if TEMPLATE_CACHE.get(template_id): - del TEMPLATE_CACHE[template_id] - + cache.delete(f"{CACHE_KEY}_{template_id}") log.info("Template %s removed from cache.", template_id) -def clear_cache(): - global TEMPLATE_CACHE - TEMPLATE_CACHE = {} - def parse_json_typeattr(type_i, typeattr_j, attribute_j, default_dataset_j, user_id=None): dimension_i = None if attribute_j.dimension_id is not None: @@ -784,13 +739,16 @@ def get_template(template_id, **kwargs): """ Get a specific resource template, by ID. """ - - tmpl_j = _get_template_from_cache(template_id) + log.info("Getting template %s", template_id) + tmpl_j = cache.get(f"{CACHE_KEY}_{template_id}") if tmpl_j is not None: + log.info("Returning cached template") return tmpl_j try: + log.info("Building template") + tmpl_i = db.DBSession.query(Template).filter( Template.id == template_id).one() @@ -802,6 +760,9 @@ def get_template(template_id, **kwargs): #ignore the messing around we've been doing to the ORM objects #db.DBSession.expunge(tmpl_i) + _save_template_to_cache(tmpl_j) + + return tmpl_j except NoResultFound: raise HydraError("Template %s not found"%template_id) @@ -833,6 +794,11 @@ def add_templatetype(templatetype, **kwargs): _remove_template_from_cache(type_i.template_id) + #remove any child templates from the cache too + child_types = db.DBSession.query(TemplateType).filter(TemplateType.parent_id == type_i.id).all() + for child_type in child_types: + _remove_template_from_cache(child_type.template_id) + db.DBSession.flush() return type_i @@ -1230,7 +1196,7 @@ def clone_templatetype(type_id, name=None, **kwargs): if col.name in ['id', 'cr_date', 'updated_at']: continue setattr(cloned_templatetype, col.name, getattr(parent_templatetype, col.name)) - + if name is not None: base_name = name else: @@ -1246,7 +1212,7 @@ def clone_templatetype(type_id, name=None, **kwargs): ).first() is not None: clone_name = f"{base_name} {counter}" counter += 1 - + cloned_templatetype.name = clone_name typeattrs = db.DBSession.query(TypeAttr).filter( @@ -1258,7 +1224,7 @@ def clone_templatetype(type_id, name=None, **kwargs): continue setattr(cloned_typeattr, col.name, getattr(typeattr, col.name)) cloned_templatetype.typeattrs.append(cloned_typeattr) - + db.DBSession.add(cloned_templatetype) db.DBSession.flush() # Flush to get the ID diff --git a/tests/conftest.py b/tests/conftest.py index 4f9e68a0..e031b6bc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -129,11 +129,11 @@ def client(connection_type, testdb_uri): pytest.user_c = client.testutils.create_user("UserC", role='developer') pytest.user_d = client.testutils.create_user("UserD", role='developer') yield client - #??? - hydra_base.lib.template.clear_cache() + hydra_base.db.close_session() clear_cache() # clear the user project cache + try: drop_tables(testdb_uri) except Exception as err: