From 85040440ea231dfb2c0b4e8fcb3d2e90b8c102f5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 12 Aug 2026 01:38:16 +0000 Subject: [PATCH] chore: update charm libraries --- .../data_platform_libs/v0/data_interfaces.py | 3478 ++++++++++++++--- lib/charms/data_platform_libs/v0/s3.py | 7 +- .../grafana_k8s/v0/grafana_dashboard.py | 1039 +++-- lib/charms/loki_k8s/v1/loki_push_api.py | 676 ++-- .../observability_libs/v0/juju_topology.py | 14 +- .../prometheus_k8s/v0/prometheus_scrape.py | 1223 ++---- lib/charms/redis_k8s/v0/redis.py | 24 +- lib/charms/saml_integrator/v0/saml.py | 22 +- lib/charms/smtp_integrator/v0/smtp.py | 241 +- lib/charms/traefik_k8s/v2/ingress.py | 234 +- 10 files changed, 4555 insertions(+), 2403 deletions(-) diff --git a/lib/charms/data_platform_libs/v0/data_interfaces.py b/lib/charms/data_platform_libs/v0/data_interfaces.py index aaed2e528..f4a65fdd7 100644 --- a/lib/charms/data_platform_libs/v0/data_interfaces.py +++ b/lib/charms/data_platform_libs/v0/data_interfaces.py @@ -16,7 +16,7 @@ This library contains the Requires and Provides classes for handling the relation between an application and multiple managed application supported by the data-team: -MySQL, Postgresql, MongoDB, Redis, and Kafka. +MySQL, Postgresql, MongoDB, Redis, Kafka, and Karapace. ### Database (MySQL, Postgresql, MongoDB, and Redis) @@ -34,6 +34,7 @@ from charms.data_platform_libs.v0.data_interfaces import ( DatabaseCreatedEvent, DatabaseRequires, + DatabaseEntityCreatedEvent, ) class ApplicationCharm(CharmBase): @@ -45,6 +46,7 @@ def __init__(self, *args): # Charm events defined in the database requires charm library. self.database = DatabaseRequires(self, relation_name="database", database_name="database") self.framework.observe(self.database.on.database_created, self._on_database_created) + self.framework.observe(self.database.on.database_entity_created, self._on_database_entity_created) def _on_database_created(self, event: DatabaseCreatedEvent) -> None: # Handle the created database @@ -61,12 +63,17 @@ def _on_database_created(self, event: DatabaseCreatedEvent) -> None: # Set active status self.unit.status = ActiveStatus("received database credentials") + + def _on_database_entity_created(self, event: DatabaseEntityCreatedEvent) -> None: + # Handle the created entity + ... ``` As shown above, the library provides some custom events to handle specific situations, which are listed below: - database_created: event emitted when the requested database is created. +- database_entity_created: event emitted when the requested entity is created. - endpoints_changed: event emitted when the read/write endpoints of the database have changed. - read_only_endpoints_changed: event emitted when the read-only endpoints of the database have changed. Event is not triggered if read/write endpoints changed too. @@ -141,7 +148,6 @@ def _on_cluster2_database_created(self, event: DatabaseCreatedEvent) -> None: event.endpoints, ) ... - ``` When it's needed to check whether a plugin (extension) is enabled on the PostgreSQL @@ -154,7 +160,6 @@ def _on_cluster2_database_created(self, event: DatabaseCreatedEvent) -> None: charm: charm-binary-python-packages: - psycopg[binary] - ``` ### Provider Charm @@ -187,6 +192,7 @@ def _on_database_requested(self, event: DatabaseRequestedEvent) -> None: self.provided_database.set_credentials(event.relation.id, username, password) # set other variables for the relation event.set_tls("False") ``` + As shown above, the library provides a custom event (database_requested) to handle the situation when an application charm requests a new database to be created. It's preferred to subscribe to this event instead of relation changed event to avoid @@ -207,6 +213,7 @@ def _on_database_requested(self, event: DatabaseRequestedEvent) -> None: BootstrapServerChangedEvent, KafkaRequires, TopicCreatedEvent, + TopicEntityCreatedEvent, ) class ApplicationCharm(CharmBase): @@ -220,6 +227,9 @@ def __init__(self, *args): self.framework.observe( self.kafka.on.topic_created, self._on_kafka_topic_created ) + self.framework.observe( + self.kafka.on.topic_entity_created, self._on_kafka_topic_entity_created + ) def _on_kafka_bootstrap_server_changed(self, event: BootstrapServerChangedEvent): # Event triggered when a bootstrap server was changed for this application @@ -238,6 +248,9 @@ def _on_kafka_topic_created(self, event: TopicCreatedEvent): zookeeper_uris = event.zookeeper_uris ... + def _on_kafka_topic_entity_created(self, event: TopicEntityCreatedEvent): + # Event triggered when an entity was created for this application + ... ``` As shown above, the library provides some custom events to handle specific situations, @@ -268,6 +281,7 @@ def __init__(self, *args): # Charm events defined in the Kafka Provides charm library. self.kafka_provider = KafkaProvides(self, relation_name="kafka_client") self.framework.observe(self.kafka_provider.on.topic_requested, self._on_topic_requested) + self.framework.observe(self.kafka_provider.on.topic_entity_requested, self._on_entity_requested) # Kafka generic helper self.kafka = KafkaHelper() @@ -283,12 +297,114 @@ def _on_topic_requested(self, event: TopicRequestedEvent): self.kafka_provider.set_tls(relation_id, "False") self.kafka_provider.set_zookeeper_uris(relation_id, ...) + def _on_entity_requested(self, event: EntityRequestedEvent): + # Handle the on_topic_entity_requested event. + ... ``` As shown above, the library provides a custom event (topic_requested) to handle the situation when an application charm requests a new topic to be created. It is preferred to subscribe to this event instead of relation changed event to avoid creating a new topic when other information other than a topic name is exchanged in the relation databag. + +### Karapace + +This library is the interface to use and interact with the Karapace charm. This library contains +custom events that add convenience to manage Karapace, and provides methods to consume the +application related data. + +#### Requirer Charm + +```python + +from charms.data_platform_libs.v0.data_interfaces import ( + EndpointsChangedEvent, + KarapaceRequires, + SubjectAllowedEvent, +) + +class ApplicationCharm(CharmBase): + + def __init__(self, *args): + super().__init__(*args) + self.karapace = KarapaceRequires(self, relation_name="karapace_client", subject="test-subject") + self.framework.observe( + self.karapace.on.server_changed, self._on_karapace_server_changed + ) + self.framework.observe( + self.karapace.on.subject_allowed, self._on_karapace_subject_allowed + ) + self.framework.observe( + self.karapace.on.subject_entity_created, self._on_subject_entity_created + ) + + + def _on_karapace_server_changed(self, event: EndpointsChangedEvent): + # Event triggered when a server endpoint was changed for this application + new_server = event.endpoints + ... + + def _on_karapace_subject_allowed(self, event: SubjectAllowedEvent): + # Event triggered when a subject was allowed for this application + username = event.username + password = event.password + tls = event.tls + endpoints = event.endpoints + ... + + def _on_subject_entity_created(self, event: SubjectEntityCreatedEvent): + # Event triggered when a subject entity was created this application + entity_name = event.entity_name + entity_password = event.entity_password + ... +``` + +As shown above, the library provides some custom events to handle specific situations, +which are listed below: + +- subject_allowed: event emitted when the requested subject is allowed. +- server_changed: event emitted when the server endpoints have changed. + +#### Provider Charm + +Following the previous example, this is an example of the provider charm. + +```python +class SampleCharm(CharmBase): + +from charms.data_platform_libs.v0.data_interfaces import ( + KarapaceProvides, + SubjectRequestedEvent, +) + + def __init__(self, *args): + super().__init__(*args) + + # Default charm events. + self.framework.observe(self.on.start, self._on_start) + + # Charm events defined in the Karapace Provides charm library. + self.karapace_provider = KarapaceProvides(self, relation_name="karapace_client") + self.framework.observe(self.karapace_provider.on.subject_requested, self._on_subject_requested) + # Karapace generic helper + self.karapace = KarapaceHelper() + + def _on_subject_requested(self, event: SubjectRequestedEvent): + # Handle the on_subject_requested event. + + subject = event.subject + relation_id = event.relation.id + # set connection info in the databag relation + self.karapace_provider.set_endpoint(relation_id, self.karapace.get_endpoint()) + self.karapace_provider.set_credentials(relation_id, username=username, password=password) + self.karapace_provider.set_tls(relation_id, "False") +``` + +As shown above, the library provides a custom event (subject_requested) to handle +the situation when an application charm requests a new subject to be created. +It is preferred to subscribe to this event instead of relation changed event to avoid +creating a new subject when other information other than a subject name is +exchanged in the relation databag. """ import copy @@ -296,21 +412,32 @@ def _on_topic_requested(self, event: TopicRequestedEvent): import logging from abc import ABC, abstractmethod from collections import UserDict, namedtuple +from dataclasses import asdict, dataclass from datetime import datetime from enum import Enum +from os import PathLike +from pathlib import Path from typing import ( Callable, Dict, + Final, ItemsView, KeysView, List, Optional, Set, Tuple, + TypedDict, Union, ValuesView, + overload, ) +try: + from cryptography.fernet import Fernet, InvalidToken +except ImportError: + Fernet = None + InvalidToken = None from ops import JujuVersion, Model, Secret, SecretInfo, SecretNotFoundError from ops.charm import ( CharmBase, @@ -320,7 +447,7 @@ def _on_topic_requested(self, event: TopicRequestedEvent): RelationEvent, SecretChangedEvent, ) -from ops.framework import EventSource, Object +from ops.framework import EventSource, Handle, Object from ops.model import Application, ModelError, Relation, Unit # The unique Charmhub library identifier, never change it @@ -331,7 +458,7 @@ def _on_topic_requested(self, event: TopicRequestedEvent): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 39 +LIBPATCH = 59 PYDEPS = ["ops>=2.0.0"] @@ -349,9 +476,15 @@ def _on_topic_requested(self, event: TopicRequestedEvent): changed - keys that still exist but have new values deleted - key that were deleted""" +OptionalPathLike = Optional[Union[PathLike, str]] + +ENTITY_USER = "USER" +ENTITY_GROUP = "GROUP" PROV_SECRET_PREFIX = "secret-" +PROV_SECRET_FIELDS = "provided-secrets" REQ_SECRET_FIELDS = "requested-secrets" +STATUS_FIELD = "status" GROUP_MAPPING_FIELD = "secret_group_mapping" GROUP_SEPARATOR = "@" @@ -361,6 +494,10 @@ def _on_topic_requested(self, event: TopicRequestedEvent): "owner_no_refresh": "ERROR secret owner cannot use --refresh", } +CROSS_MODEL_RELATION_CONSUMER_SECRETS = [ + "mtls-cert", +] + ############################################################################## # Exceptions @@ -391,6 +528,10 @@ class IllegalOperationError(DataInterfacesError): """To be used when an operation is not allowed to be performed.""" +class PrematureDataAccessError(DataInterfacesError): + """To be raised when the Relation Data may be accessed (written) before protocol init complete.""" + + ############################################################################## # Global helpers / utilities ############################################################################## @@ -575,12 +716,46 @@ class SecretGroup(str): """Secret groups specific type.""" +@dataclass +class RelationStatus: + """Base data class for status propagation on charm relations.""" + + code: int + message: str + resolution: str + + @property + def is_informational(self) -> bool: + """Is this an informational status?""" + return self.code // 1000 == 1 + + @property + def is_transitory(self) -> bool: + """Is this a transitory status?""" + return self.code // 1000 == 4 + + @property + def is_fatal(self) -> bool: + """Is this a fatal status, requiring removing the relation?""" + return self.code // 1000 == 5 + + +class RelationStatusDict(TypedDict): + """Base type for dict representation of `RelationStatus` dataclass.""" + + code: int + message: str + resolution: str + + class SecretGroupsAggregate(str): """Secret groups with option to extend with additional constants.""" def __init__(self): self.USER = SecretGroup("user") self.TLS = SecretGroup("tls") + self.MTLS = SecretGroup("mtls") + self.ENTITY = SecretGroup("entity") self.EXTRA = SecretGroup("extra") def __setattr__(self, name, value): @@ -605,7 +780,7 @@ def get_group(self, group: str) -> Optional[SecretGroup]: class CachedSecret: """Locally cache a secret. - The data structure is precisely re-using/simulating as in the actual Secret Storage + The data structure is precisely reusing/simulating as in the actual Secret Storage """ KNOWN_MODEL_ERRORS = [MODEL_ERRORS["no_label_and_uri"], MODEL_ERRORS["owner_no_refresh"]] @@ -676,6 +851,11 @@ def _legacy_compat_find_secret_by_old_label(self) -> None: self._secret_meta = self._model.get_secret(label=label) except SecretNotFoundError: pass + except ModelError as e: + # Permission denied can be raised if the secret exists but is not yet granted to us. + if "permission denied" in str(e): + return + raise else: if label != self.label: self.current_label = label @@ -710,6 +890,8 @@ def _legacy_migration_to_new_label_if_needed(self) -> None: except ModelError as err: if MODEL_ERRORS["not_leader"] not in str(err): raise + if "permission denied" not in str(err): + raise self.current_label = None ########################################################################## @@ -947,7 +1129,7 @@ def get(self, key: str, default: Optional[str] = None) -> Optional[str]: class Data(ABC): - """Base relation data mainpulation (abstract) class.""" + """Base relation data manipulation (abstract) class.""" SCOPE = Scope.APP @@ -956,10 +1138,16 @@ class Data(ABC): "username": SECRET_GROUPS.USER, "password": SECRET_GROUPS.USER, "uris": SECRET_GROUPS.USER, + "read-only-uris": SECRET_GROUPS.USER, "tls": SECRET_GROUPS.TLS, "tls-ca": SECRET_GROUPS.TLS, + "mtls-cert": SECRET_GROUPS.MTLS, + "entity-name": SECRET_GROUPS.ENTITY, + "entity-password": SECRET_GROUPS.ENTITY, } + SECRET_FIELDS = [] + def __init__( self, model: Model, @@ -973,15 +1161,13 @@ def __init__( self.component = self.local_app if self.SCOPE == Scope.APP else self.local_unit self.secrets = SecretCache(self._model, self.component) self.data_component = None + self._local_secret_fields = [] + self._remote_secret_fields = list(self.SECRET_FIELDS) @property def relations(self) -> List[Relation]: """The list of Relation instances associated with this relation_name.""" - return [ - relation - for relation in self._model.relations[self.relation_name] - if self._is_relation_active(relation) - ] + return self._model.relations[self.relation_name] @property def secrets_enabled(self): @@ -995,111 +1181,323 @@ def secret_label_map(self): """Exposing secret-label map via a property -- could be overridden in descendants!""" return self.SECRET_LABEL_MAP + @property + def local_secret_fields(self) -> Optional[List[str]]: + """Local access to secrets field, in case they are being used.""" + if self.secrets_enabled: + return self._local_secret_fields + + @property + def remote_secret_fields(self) -> Optional[List[str]]: + """Local access to secrets field, in case they are being used.""" + if self.secrets_enabled: + return self._remote_secret_fields + + @property + def my_secret_groups(self) -> Optional[List[SecretGroup]]: + """Local access to secrets field, in case they are being used.""" + if self.secrets_enabled: + return [ + self.SECRET_LABEL_MAP[field] + for field in self._local_secret_fields + if field in self.SECRET_LABEL_MAP + ] + # Mandatory overrides for internal/helper methods - @abstractmethod + @juju_secrets_only def _get_relation_secret( self, relation_id: int, group_mapping: SecretGroup, relation_name: Optional[str] = None ) -> Optional[CachedSecret]: """Retrieve a Juju Secret that's been stored in the relation databag.""" - raise NotImplementedError + if not relation_name: + relation_name = self.relation_name + + label = self._generate_secret_label(relation_name, relation_id, group_mapping) + if secret := self.secrets.get(label): + return secret + relation = self._model.get_relation(relation_name, relation_id) + if not relation: + return + + if secret_uri := self.get_secret_uri(relation, group_mapping): + return self.secrets.get(label, secret_uri) + + # Mandatory overrides for requirer and peer, implemented for Provider + # Requirer uses local component and switched keys + # _local_secret_fields -> PROV_SECRET_FIELDS + # _remote_secret_fields -> REQ_SECRET_FIELDS + # provider uses remote component and + # _local_secret_fields -> REQ_SECRET_FIELDS + # _remote_secret_fields -> PROV_SECRET_FIELDS @abstractmethod + def _load_secrets_from_databag(self, relation: Relation) -> None: + """Load secrets from the databag.""" + raise NotImplementedError + + def _get_encryption_key(self, relation: Relation) -> Optional[str]: + """Fetch the encryption key from the encryption secret if available.""" + if not (encryption_secret := relation.data[relation.app].get("encryption-secret")): + return None + + # get the encryption secret created on provider side + secret = self._model.get_secret(id=encryption_secret) + return secret.get_content().get("encryption-key") + def _fetch_specific_relation_data( self, relation: Relation, fields: Optional[List[str]] ) -> Dict[str, str]: - """Fetch data available (directily or indirectly -- i.e. secrets) from the relation.""" - raise NotImplementedError + """Fetch data available (directily or indirectly -- i.e. secrets) from the relation (remote app data).""" + if not relation.app: + return {} + self._load_secrets_from_databag(relation) + return self._fetch_relation_data_with_secrets( + relation.app, self.remote_secret_fields, relation, fields + ) - @abstractmethod def _fetch_my_specific_relation_data( self, relation: Relation, fields: Optional[List[str]] - ) -> Dict[str, str]: - """Fetch data available (directily or indirectly -- i.e. secrets) from the relation for owner/this_app.""" - raise NotImplementedError + ) -> dict: + """Fetch our own relation data.""" + # load secrets + self._load_secrets_from_databag(relation) + return self._fetch_relation_data_with_secrets( + self.local_app, + self.local_secret_fields, + relation, + fields, + ) - @abstractmethod def _update_relation_data(self, relation: Relation, data: Dict[str, str]) -> None: - """Update data available (directily or indirectly -- i.e. secrets) from the relation for owner/this_app.""" - raise NotImplementedError - - @abstractmethod - def _delete_relation_data(self, relation: Relation, fields: List[str]) -> None: - """Delete data available (directily or indirectly -- i.e. secrets) from the relation for owner/this_app.""" - raise NotImplementedError - - # Optional overrides - - def _legacy_apply_on_fetch(self) -> None: - """This function should provide a list of compatibility functions to be applied when fetching (legacy) data.""" - pass + """Set values for fields not caring whether it's a secret or not.""" + self._load_secrets_from_databag(relation) - def _legacy_apply_on_update(self, fields: List[str]) -> None: - """This function should provide a list of compatibility functions to be applied when writing data. + _, normal_fields = self._process_secret_fields( + relation, + self.local_secret_fields, + list(data), + self._add_or_update_relation_secrets, + data=data, + ) - Since data may be at a legacy version, migration may be mandatory. - """ - pass + normal_content = {k: v for k, v in data.items() if k in normal_fields} + self._update_relation_data_without_secrets(self.local_app, relation, normal_content) - def _legacy_apply_on_delete(self, fields: List[str]) -> None: - """This function should provide a list of compatibility functions to be applied when deleting (legacy) data.""" - pass + def _add_or_update_relation_secrets( + self, + relation: Relation, + group: SecretGroup, + secret_fields: Set[str], + data: Dict[str, str], + uri_to_databag=True, + ) -> bool: + """Update contents for Secret group. If the Secret doesn't exist, create it.""" + if self._get_relation_secret(relation.id, group): + return self._update_relation_secret(relation, group, secret_fields, data) - # Internal helper methods + return self._add_relation_secret(relation, group, secret_fields, data, uri_to_databag) - @staticmethod - def _is_relation_active(relation: Relation): - """Whether the relation is active based on contained data.""" - try: - _ = repr(relation.data) - return True - except (RuntimeError, ModelError): + @juju_secrets_only + def _add_relation_secret( + self, + relation: Relation, + group_mapping: SecretGroup, + secret_fields: Set[str], + data: Dict[str, str], + uri_to_databag=True, + ) -> bool: + """Add a new Juju Secret that will be registered in the relation databag.""" + if uri_to_databag and self.get_secret_uri(relation, group_mapping): + logging.error("Secret for relation %s already exists, not adding again", relation.id) return False - @staticmethod - def _is_secret_field(field: str) -> bool: - """Is the field in question a secret reference (URI) field or not?""" - return field.startswith(PROV_SECRET_PREFIX) + content = self._content_for_secret_group(data, secret_fields, group_mapping) - @staticmethod - def _generate_secret_label( - relation_name: str, relation_id: int, group_mapping: SecretGroup - ) -> str: - """Generate unique group_mappings for secrets within a relation context.""" - return f"{relation_name}.{relation_id}.{group_mapping}.secret" + label = self._generate_secret_label(self.relation_name, relation.id, group_mapping) + secret = self.secrets.add(label, content, relation) - def _generate_secret_field_name(self, group_mapping: SecretGroup) -> str: - """Generate unique group_mappings for secrets within a relation context.""" - return f"{PROV_SECRET_PREFIX}{group_mapping}" + if uri_to_databag: + # According to lint we may not have a Secret ID + if not secret.meta or not secret.meta.id: + logging.error("Secret is missing Secret ID") + raise SecretError("Secret added but is missing Secret ID") - def _relation_from_secret_label(self, secret_label: str) -> Optional[Relation]: - """Retrieve the relation that belongs to a secret label.""" - contents = secret_label.split(".") + self.set_secret_uri(relation, group_mapping, secret.meta.id) - if not (contents and len(contents) >= 3): - return + # Return the content that was added + return True - contents.pop() # ".secret" at the end - contents.pop() # Group mapping - relation_id = contents.pop() - try: - relation_id = int(relation_id) - except ValueError: - return + @juju_secrets_only + def _update_relation_secret( + self, + relation: Relation, + group_mapping: SecretGroup, + secret_fields: Set[str], + data: Dict[str, str], + ) -> bool: + """Update the contents of an existing Juju Secret, referred in the relation databag.""" + secret = self._get_relation_secret(relation.id, group_mapping) - # In case '.' character appeared in relation name - relation_name = ".".join(contents) + if not secret: + logging.error("Can't update secret for relation %s", relation.id) + return False - try: - return self.get_relation(relation_name, relation_id) - except ModelError: - return + content = self._content_for_secret_group(data, secret_fields, group_mapping) - def _group_secret_fields(self, secret_fields: List[str]) -> Dict[SecretGroup, List[str]]: - """Helper function to arrange secret mappings under their group. + old_content = secret.get_content() + full_content = copy.deepcopy(old_content) + full_content.update(content) + secret.set_content(full_content) - NOTE: All unrecognized items end up in the 'extra' secret bucket. - Make sure only secret fields are passed! + # Return True on success + return True + + @juju_secrets_only + def _delete_relation_secret( + self, relation: Relation, group: SecretGroup, secret_fields: List[str], fields: List[str] + ) -> bool: + """Update the contents of an existing Juju Secret, referred in the relation databag.""" + secret = self._get_relation_secret(relation.id, group) + + if not secret: + logging.error("Can't delete secret for relation %s", str(relation.id)) + return False + + old_content = secret.get_content() + new_content = copy.deepcopy(old_content) + for field in fields: + try: + new_content.pop(field) + except KeyError: + logging.debug( + "Non-existing secret was attempted to be removed %s, %s", + str(relation.id), + str(field), + ) + return False + + # Remove secret from the relation if it's fully gone + if not new_content: + field = self._generate_secret_field_name(group) + try: + relation.data[self.component].pop(field) + except KeyError: + pass + label = self._generate_secret_label(self.relation_name, relation.id, group) + self.secrets.remove(label) + else: + secret.set_content(new_content) + + # Return the content that was removed + return True + + def _delete_relation_data(self, relation: Relation, fields: List[str]) -> None: + """Delete data available (directily or indirectly -- i.e. secrets) from the relation for owner/this_app.""" + if relation.app: + self._load_secrets_from_databag(relation) + + _, normal_fields = self._process_secret_fields( + relation, self.local_secret_fields, fields, self._delete_relation_secret, fields=fields + ) + self._delete_relation_data_without_secrets(self.local_app, relation, list(normal_fields)) + + def _register_secret_to_relation( + self, relation_name: str, relation_id: int, secret_id: str, group: SecretGroup + ): + """Fetch secrets and apply local label on them. + + [MAGIC HERE] + If we fetch a secret using get_secret(id=, label=), + then will be "stuck" on the Secret object, whenever it may + appear (i.e. as an event attribute, or fetched manually) on future occasions. + + This will allow us to uniquely identify the secret on Provider side (typically on + 'secret-changed' events), and map it to the corresponding relation. + """ + label = self._generate_secret_label(relation_name, relation_id, group) + + # Fetching the Secret's meta information ensuring that it's locally getting registered with + CachedSecret(self._model, self.component, label, secret_id).meta + + def _register_secrets_to_relation(self, relation: Relation, params_name_list: List[str]): + """Make sure that secrets of the provided list are locally 'registered' from the databag. + + More on 'locally registered' magic is described in _register_secret_to_relation() method + """ + if not relation.app: + return + + for group in SECRET_GROUPS.groups(): + secret_field = self._generate_secret_field_name(group) + if secret_field in params_name_list and ( + secret_uri := self.get_secret_uri(relation, group) + ): + self._register_secret_to_relation(relation.name, relation.id, secret_uri, group) + + # Optional overrides + + def _legacy_apply_on_fetch(self) -> None: + """This function should provide a list of compatibility functions to be applied when fetching (legacy) data.""" + pass + + def _legacy_apply_on_update(self, fields: List[str]) -> None: + """This function should provide a list of compatibility functions to be applied when writing data. + + Since data may be at a legacy version, migration may be mandatory. + """ + pass + + def _legacy_apply_on_delete(self, fields: List[str]) -> None: + """This function should provide a list of compatibility functions to be applied when deleting (legacy) data.""" + pass + + # Internal helper methods + + @staticmethod + def _is_secret_field(field: str) -> bool: + """Is the field in question a secret reference (URI) field or not?""" + return field.startswith(PROV_SECRET_PREFIX) + + @staticmethod + def _generate_secret_label( + relation_name: str, relation_id: int, group_mapping: SecretGroup + ) -> str: + """Generate unique group_mappings for secrets within a relation context.""" + return f"{relation_name}.{relation_id}.{group_mapping}.secret" + + def _generate_secret_field_name(self, group_mapping: SecretGroup) -> str: + """Generate unique group_mappings for secrets within a relation context.""" + return f"{PROV_SECRET_PREFIX}{group_mapping}" + + def _relation_from_secret_label(self, secret_label: str) -> Optional[Relation]: + """Retrieve the relation that belongs to a secret label.""" + contents = secret_label.split(".") + + if not (contents and len(contents) >= 3): + return + + contents.pop() # ".secret" at the end + contents.pop() # Group mapping + relation_id = contents.pop() + try: + relation_id = int(relation_id) + except ValueError: + return + + # In case '.' character appeared in relation name + relation_name = ".".join(contents) + + try: + return self.get_relation(relation_name, relation_id) + except ModelError: + return + + def _group_secret_fields(self, secret_fields: List[str]) -> Dict[SecretGroup, List[str]]: + """Helper function to arrange secret mappings under their group. + + NOTE: All unrecognized items end up in the 'extra' secret bucket. + Make sure only secret fields are passed! """ secret_fieldnames_grouped = {} for key in secret_fields: @@ -1173,7 +1571,6 @@ def _process_secret_fields( and (self.local_unit == self._model.unit and self.local_unit.is_leader()) and set(req_secret_fields) & set(relation.data[self.component]) ) - normal_fields = set(impacted_rel_fields) if req_secret_fields and self.secrets_enabled and not fallback_to_databag: normal_fields = normal_fields - set(req_secret_fields) @@ -1207,11 +1604,48 @@ def _fetch_relation_data_without_secrets( return {} if fields: - return { + relation_data = { k: relation.data[component][k] for k in fields if k in relation.data[component] } else: - return dict(relation.data[component]) + relation_data = dict(relation.data[component]) + + try: + remote_model_uuid = relation.remote_model.uuid + except (FileNotFoundError, ModelError, RuntimeError) as e: + # access to remote model added in Juju 3.6.2, fails with 2.9 + logger.warning("Access to remote model failed: %s", e) + remote_model_uuid = "" + + # if not a cross-model relation we can return data as-is + if remote_model_uuid == "" or self._model.uuid == remote_model_uuid: + return relation_data + + if not (encryption_key := self._get_encryption_key(relation)): + return relation_data + + if not Fernet: + logger.warning( + "Cryptography module not installed. Could not decrypt sensitive field in cross-model relation" + ) + return relation_data + + # still here means sensitive data needs to be decrypted + for key, value in relation_data.items(): + if key in CROSS_MODEL_RELATION_CONSUMER_SECRETS: + try: + f = Fernet(encryption_key) + decrypted_value = f.decrypt(value.encode()).decode() + relation_data[key] = decrypted_value + except ( + AttributeError, + InvalidToken, + TypeError, + ValueError, + ): # pyright: ignore [reportGeneralTypeIssues] + logger.warning("Could not decrypt sensitive field in cross-model relation") + + return relation_data def _fetch_relation_data_with_secrets( self, @@ -1258,8 +1692,40 @@ def _update_relation_data_without_secrets( if component not in relation.data or relation.data[component] is None: return - if relation: - relation.data[component].update(data) + if not relation: + return + + # ensure no sensitive information is stored in relation data + encryption_key = self._get_encryption_key(relation) + + try: + remote_model_uuid = relation.remote_model.uuid + except (FileNotFoundError, ModelError, RuntimeError): + # access to remote model added in Juju 3.6.2, fails with 2.9 + remote_model_uuid = "" + + if encryption_key and remote_model_uuid != "" and self._model.uuid != remote_model_uuid: + for key, value in data.items(): + try: + f = Fernet(encryption_key) # pyright: ignore [reportOptionalCall] + if key in CROSS_MODEL_RELATION_CONSUMER_SECRETS: + encrypted_value = f.encrypt(value.encode()).decode() + data[key] = encrypted_value + except ( + AttributeError, + InvalidToken, + ValueError, + ): # pyright: ignore [reportGeneralTypeIssues] + logger.warning("Could not encrypt sensitive field in cross-model relation") + data[key] = "" + except TypeError: + # "TypeError: 'NoneType' object is not callable" raised when Fernet is `None` + logger.warning( + "Cryptography module not installed. Could not decrypt sensitive field in cross-model relation" + ) + data[key] = "" + + relation.data[component].update(data) def _delete_relation_data_without_secrets( self, component: Union[Application, Unit], relation: Relation, fields: List[str] @@ -1300,7 +1766,14 @@ def get_relation(self, relation_name, relation_id) -> Relation: def get_secret_uri(self, relation: Relation, group: SecretGroup) -> Optional[str]: """Get the secret URI for the corresponding group.""" secret_field = self._generate_secret_field_name(group) - return relation.data[self.component].get(secret_field) + # if the secret is not managed by this component, + # we need to fetch it from the other side + + # Fix for the linter + if self.my_secret_groups is None: + raise DataInterfacesError("Secrets are not enabled for this component") + component = self.component if group in self.my_secret_groups else relation.app + return relation.data[component].get(secret_field) def set_secret_uri(self, relation: Relation, group: SecretGroup, secret_uri: str) -> None: """Set the secret URI for the corresponding group.""" @@ -1429,6 +1902,32 @@ def __init__(self, charm: CharmBase, relation_data: Data, unique_key: str = ""): self._on_relation_changed_event, ) + self.framework.observe( + self.charm.on[relation_data.relation_name].relation_created, + self._on_relation_created_event, + ) + + self.framework.observe( + charm.on.secret_changed, + self._on_secret_changed_event, + ) + + # Event handlers + + def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: + """Event emitted when the relation is created.""" + pass + + @abstractmethod + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation data has changed.""" + raise NotImplementedError + + @abstractmethod + def _on_secret_changed_event(self, event: SecretChangedEvent) -> None: + """Event emitted when the relation data has changed.""" + raise NotImplementedError + def _diff(self, event: RelationChangedEvent) -> Diff: """Retrieves the diff of the data in the relation changed databag. @@ -1441,11 +1940,6 @@ def _diff(self, event: RelationChangedEvent) -> Diff: """ return diff(event, self.relation_data.data_component) - @abstractmethod - def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: - """Event emitted when the relation data has changed.""" - raise NotImplementedError - # Base ProviderData and RequiresData @@ -1453,239 +1947,211 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: class ProviderData(Data): """Base provides-side of the data products relation.""" + RESOURCE_FIELD = "database" + def __init__( self, model: Model, relation_name: str, + status_schema_path: OptionalPathLike = None, ) -> None: super().__init__(model, relation_name) self.data_component = self.local_app + self._local_secret_fields = [] + self._remote_secret_fields = list(self.SECRET_FIELDS) + self._status_schema = ( + {} if not status_schema_path else self._load_status_schema(Path(status_schema_path)) + ) - # Private methods handling secrets + def _load_status_schema(self, schema_path: Path) -> Dict[int, RelationStatus]: + """Load JSON schema defining status codes and their details. - @juju_secrets_only - def _add_relation_secret( - self, - relation: Relation, - group_mapping: SecretGroup, - secret_fields: Set[str], - data: Dict[str, str], - uri_to_databag=True, - ) -> bool: - """Add a new Juju Secret that will be registered in the relation databag.""" - if uri_to_databag and self.get_secret_uri(relation, group_mapping): - logging.error("Secret for relation %s already exists, not adding again", relation.id) - return False + Args: + schema_path: JSON schema file path. - content = self._content_for_secret_group(data, secret_fields, group_mapping) + Raises: + FileNotFoundError: If the provided path is invalid/inaccessible. - label = self._generate_secret_label(self.relation_name, relation.id, group_mapping) - secret = self.secrets.add(label, content, relation) + Returns: + dict[int, RelationStatusDict]: Mapping of status code to RelationStatus data objects. + """ + if not schema_path.exists(): + raise FileNotFoundError(f"Can't locate status schema file: {schema_path}") - # According to lint we may not have a Secret ID - if uri_to_databag and secret.meta and secret.meta.id: - self.set_secret_uri(relation, group_mapping, secret.meta.id) + content = json.load(open(schema_path, "r")) - # Return the content that was added - return True + return {s["code"]: RelationStatus(**s) for s in content.get("statuses", [])} - @juju_secrets_only - def _update_relation_secret( - self, - relation: Relation, - group_mapping: SecretGroup, - secret_fields: Set[str], - data: Dict[str, str], - ) -> bool: - """Update the contents of an existing Juju Secret, referred in the relation databag.""" - secret = self._get_relation_secret(relation.id, group_mapping) + def _update_relation_data(self, relation: Relation, data: Dict[str, str]) -> None: + """Set values for fields not caring whether it's a secret or not.""" + keys = set(data.keys()) + if self.fetch_relation_field(relation.id, self.RESOURCE_FIELD) is None and ( + keys - {"endpoints", "read-only-endpoints", "replset", "encryption-secret"} + ): + raise PrematureDataAccessError( + "Premature access to relation data, update is forbidden before the connection is initialized." + ) + super()._update_relation_data(relation, data) - if not secret: - logging.error("Can't update secret for relation %s", relation.id) - return False + # Public methods - "native" - content = self._content_for_secret_group(data, secret_fields, group_mapping) + def set_credentials(self, relation_id: int, username: str, password: str) -> None: + """Set credentials. - old_content = secret.get_content() - full_content = copy.deepcopy(old_content) - full_content.update(content) - secret.set_content(full_content) + This function writes in the application data bag, therefore, + only the leader unit can call it. - # Return True on success - return True + Args: + relation_id: the identifier for a particular relation. + username: user that was created. + password: password of the created user. + """ + self.update_relation_data(relation_id, {"username": username, "password": password}) - def _add_or_update_relation_secrets( - self, - relation: Relation, - group: SecretGroup, - secret_fields: Set[str], - data: Dict[str, str], - uri_to_databag=True, - ) -> bool: - """Update contents for Secret group. If the Secret doesn't exist, create it.""" - if self._get_relation_secret(relation.id, group): - return self._update_relation_secret(relation, group, secret_fields, data) - else: - return self._add_relation_secret(relation, group, secret_fields, data, uri_to_databag) - - @juju_secrets_only - def _delete_relation_secret( - self, relation: Relation, group: SecretGroup, secret_fields: List[str], fields: List[str] - ) -> bool: - """Update the contents of an existing Juju Secret, referred in the relation databag.""" - secret = self._get_relation_secret(relation.id, group) - - if not secret: - logging.error("Can't delete secret for relation %s", str(relation.id)) - return False - - old_content = secret.get_content() - new_content = copy.deepcopy(old_content) - for field in fields: - try: - new_content.pop(field) - except KeyError: - logging.debug( - "Non-existing secret was attempted to be removed %s, %s", - str(relation.id), - str(field), - ) - return False + def set_entity_credentials( + self, relation_id: int, entity_name: str, entity_password: Optional[str] = None + ) -> None: + """Set entity credentials. - # Remove secret from the relation if it's fully gone - if not new_content: - field = self._generate_secret_field_name(group) - try: - relation.data[self.component].pop(field) - except KeyError: - pass - label = self._generate_secret_label(self.relation_name, relation.id, group) - self.secrets.remove(label) - else: - secret.set_content(new_content) + This function writes in the application data bag, therefore, + only the leader unit can call it. - # Return the content that was removed - return True + Args: + relation_id: the identifier for a particular relation. + entity_name: name of the created entity + entity_password: password of the created entity. + """ + self.update_relation_data( + relation_id, + {"entity-name": entity_name, "entity-password": entity_password}, + ) - # Mandatory internal overrides + def set_tls(self, relation_id: int, tls: str) -> None: + """Set whether TLS is enabled. - @juju_secrets_only - def _get_relation_secret( - self, relation_id: int, group_mapping: SecretGroup, relation_name: Optional[str] = None - ) -> Optional[CachedSecret]: - """Retrieve a Juju Secret that's been stored in the relation databag.""" - if not relation_name: - relation_name = self.relation_name + Args: + relation_id: the identifier for a particular relation. + tls: whether tls is enabled (True or False). + """ + self.update_relation_data(relation_id, {"tls": tls}) - label = self._generate_secret_label(relation_name, relation_id, group_mapping) - if secret := self.secrets.get(label): - return secret + def set_tls_ca(self, relation_id: int, tls_ca: str) -> None: + """Set the TLS CA in the application relation databag. - relation = self._model.get_relation(relation_name, relation_id) - if not relation: - return + Args: + relation_id: the identifier for a particular relation. + tls_ca: TLS certification authority. + """ + self.update_relation_data(relation_id, {"tls-ca": tls_ca}) - if secret_uri := self.get_secret_uri(relation, group_mapping): - return self.secrets.get(label, secret_uri) + @leader_only + def get_statuses(self, relation_id: int) -> Dict[int, RelationStatus]: + """Return all currently active statuses on this relation. Can only be called on leader units. - def _fetch_specific_relation_data( - self, relation: Relation, fields: Optional[List[str]] - ) -> Dict[str, str]: - """Fetching relation data for Provider. + Args: + relation_id (int): the identifier for a particular relation. - NOTE: Since all secret fields are in the Provider side of the databag, we don't need to worry about that + Returns: + Dict[int, RelationStatus]: A mapping of status code to RelationStatus instances. """ - if not relation.app: - return {} - - return self._fetch_relation_data_without_secrets(relation.app, relation, fields) - - def _fetch_my_specific_relation_data( - self, relation: Relation, fields: Optional[List[str]] - ) -> dict: - """Fetching our own relation data.""" - secret_fields = None - if relation.app: - secret_fields = get_encoded_list(relation, relation.app, REQ_SECRET_FIELDS) + raw = self.fetch_my_relation_field(relation_id, STATUS_FIELD) or "[]" - return self._fetch_relation_data_with_secrets( - self.local_app, - secret_fields, - relation, - fields, - ) + return {item["code"]: RelationStatus(**item) for item in json.loads(raw)} - def _update_relation_data(self, relation: Relation, data: Dict[str, str]) -> None: - """Set values for fields not caring whether it's a secret or not.""" - req_secret_fields = [] - if relation.app: - req_secret_fields = get_encoded_list(relation, relation.app, REQ_SECRET_FIELDS) + @overload + def raise_status(self, relation_id: int, status: int) -> None: ... - _, normal_fields = self._process_secret_fields( - relation, - req_secret_fields, - list(data), - self._add_or_update_relation_secrets, - data=data, - ) + @overload + def raise_status(self, relation_id: int, status: RelationStatusDict) -> None: ... - normal_content = {k: v for k, v in data.items() if k in normal_fields} - self._update_relation_data_without_secrets(self.local_app, relation, normal_content) + @overload + def raise_status(self, relation_id: int, status: RelationStatus) -> None: ... - def _delete_relation_data(self, relation: Relation, fields: List[str]) -> None: - """Delete fields from the Relation not caring whether it's a secret or not.""" - req_secret_fields = [] - if relation.app: - req_secret_fields = get_encoded_list(relation, relation.app, REQ_SECRET_FIELDS) + def raise_status( + self, relation_id: int, status: Union[RelationStatus, RelationStatusDict, int] + ) -> None: + """Raise a status on the relation. Can only be called on leader units. - _, normal_fields = self._process_secret_fields( - relation, req_secret_fields, fields, self._delete_relation_secret, fields=fields - ) - self._delete_relation_data_without_secrets(self.local_app, relation, list(normal_fields)) + Args: + relation_id (int): the identifier for a particular relation. + status (RelationStatus | RelationStatusDict | int): A representation of the status being raised, + which could be either a RelationStatus, an appropriate dict, or the numeric status code. - # Public methods - "native" + Raises: + ValueError: If the status provided is not correctly formatted. + """ + if isinstance(status, int): + # we expect the status schema to be defined in this case. + if status not in self._status_schema: + raise KeyError(f"Status code [{status}] not defined.") + _status = self._status_schema[status] + elif isinstance(status, dict): + _status = RelationStatus(**status) + elif isinstance(status, RelationStatus): + _status = status + else: + raise ValueError( + "The status should be either a RelationStatus, an appropriate dict, or the numeric status code." + ) - def set_credentials(self, relation_id: int, username: str, password: str) -> None: - """Set credentials. + statuses = self.get_statuses(relation_id) + statuses.update({_status.code: _status}) + serialized = json.dumps([asdict(statuses[k]) for k in sorted(statuses)]) + self.update_relation_data(relation_id, {STATUS_FIELD: serialized}) - This function writes in the application data bag, therefore, - only the leader unit can call it. + def resolve_status(self, relation_id: int, status_code: int) -> None: + """Set a previously raised status as resolved. Args: - relation_id: the identifier for a particular relation. - username: user that was created. - password: password of the created user. + relation_id (int): the identifier for a particular relation. + status_code (int): the numeric code of the resolved status. """ - self.update_relation_data(relation_id, {"username": username, "password": password}) - - def set_tls(self, relation_id: int, tls: str) -> None: - """Set whether TLS is enabled. + statuses = self.get_statuses(relation_id) + if status_code not in statuses: + logger.error(f"Status [{status_code}] has never been raised before.") + return - Args: - relation_id: the identifier for a particular relation. - tls: whether tls is enabled (True or False). - """ - self.update_relation_data(relation_id, {"tls": tls}) + statuses.pop(status_code) + serialized = json.dumps([asdict(statuses[k]) for k in sorted(statuses)]) + self.update_relation_data(relation_id, {STATUS_FIELD: serialized}) - def set_tls_ca(self, relation_id: int, tls_ca: str) -> None: - """Set the TLS CA in the application relation databag. + def clear_statuses(self, relation_id: int) -> None: + """Clear all previously raised statuses. Args: - relation_id: the identifier for a particular relation. - tls_ca: TLS certification authority. + relation_id (int): the identifier for a particular relation. """ - self.update_relation_data(relation_id, {"tls-ca": tls_ca}) + self.delete_relation_data(relation_id, [STATUS_FIELD]) # Public functions -- inherited fetch_my_relation_data = leader_only(Data.fetch_my_relation_data) fetch_my_relation_field = leader_only(Data.fetch_my_relation_field) + def _load_secrets_from_databag(self, relation: Relation) -> None: + """Load secrets from the databag.""" + requested_secrets = get_encoded_list(relation, relation.app, REQ_SECRET_FIELDS) + provided_secrets = get_encoded_list(relation, relation.app, PROV_SECRET_FIELDS) + if requested_secrets is not None: + self._local_secret_fields = requested_secrets + + if provided_secrets is not None: + self._remote_secret_fields = provided_secrets + class RequirerData(Data): """Requirer-side of the relation.""" - SECRET_FIELDS = ["username", "password", "tls", "tls-ca", "uris"] + SECRET_FIELDS = [ + "username", + "password", + "tls", + "tls-ca", + "uris", + "read-only-uris", + "entity-name", + "entity-password", + ] def __init__( self, @@ -1693,75 +2159,115 @@ def __init__( relation_name: str, extra_user_roles: Optional[str] = None, additional_secret_fields: Optional[List[str]] = [], + extra_group_roles: Optional[str] = None, + entity_type: Optional[str] = None, + entity_permissions: Optional[str] = None, + requested_entity_secret: Optional[str] = None, + requested_entity_name: Optional[str] = None, + requested_entity_password: Optional[str] = None, + prefix_matching: Optional[str] = None, ): """Manager of base client relations.""" super().__init__(model, relation_name) self.extra_user_roles = extra_user_roles - self._secret_fields = list(self.SECRET_FIELDS) - if additional_secret_fields: - self._secret_fields += additional_secret_fields - self.data_component = self.local_unit + self.extra_group_roles = extra_group_roles + self.entity_type = entity_type + self.entity_permissions = entity_permissions + self.requested_entity_secret = requested_entity_secret + self.requested_entity_name = requested_entity_name + self.requested_entity_password = requested_entity_password + self.prefix_matching = prefix_matching - @property - def secret_fields(self) -> Optional[List[str]]: - """Local access to secrets field, in case they are being used.""" - if self.secrets_enabled: - return self._secret_fields + if ( + self.requested_entity_secret or self.requested_entity_name + ) and not self.secrets_enabled: + raise SecretsUnavailableError("Secrets unavailable on current Juju version") - # Internal helper functions + if self.requested_entity_secret and ( + self.requested_entity_name or self.requested_entity_password + ): + raise IllegalOperationError("Unable to use provided and automated entity name secret") - def _register_secret_to_relation( - self, relation_name: str, relation_id: int, secret_id: str, group: SecretGroup - ): - """Fetch secrets and apply local label on them. + if self.requested_entity_password and not self.requested_entity_name: + raise IllegalOperationError("Unable to set entity password without an entity name") - [MAGIC HERE] - If we fetch a secret using get_secret(id=, label=), - then will be "stuck" on the Secret object, whenever it may - appear (i.e. as an event attribute, or fetched manually) on future occasions. + self._validate_entity_type() + self._validate_entity_permissions() - This will allow us to uniquely identify the secret on Provider side (typically on - 'secret-changed' events), and map it to the corresponding relation. - """ - label = self._generate_secret_label(relation_name, relation_id, group) + self._remote_secret_fields = list(self.SECRET_FIELDS) + self._local_secret_fields = [ + field + for field in self.SECRET_LABEL_MAP.keys() + if field not in self._remote_secret_fields + and not ( + field in CROSS_MODEL_RELATION_CONSUMER_SECRETS and self.is_cross_model_relation + ) + ] + if additional_secret_fields: + self._remote_secret_fields += additional_secret_fields + self.data_component = self.local_unit - # Fetching the Secret's meta information ensuring that it's locally getting registered with - CachedSecret(self._model, self.component, label, secret_id).meta + # Internal functions + @property + def is_cross_model_relation(self) -> bool: + """Determines whether the relation is a cross-model relation or not.""" + if len(self.relations) == 0: + return False - def _register_secrets_to_relation(self, relation: Relation, params_name_list: List[str]): - """Make sure that secrets of the provided list are locally 'registered' from the databag. + try: + remote_model_uuid = self.relations[0].remote_model.uuid + except (FileNotFoundError, ModelError, RuntimeError): + # access to remote model added in Juju 3.6.2, fails with 2.9 + return False - More on 'locally registered' magic is described in _register_secret_to_relation() method - """ - if not relation.app: - return + if self._model.uuid != remote_model_uuid: + return True - for group in SECRET_GROUPS.groups(): - secret_field = self._generate_secret_field_name(group) - if secret_field in params_name_list and ( - secret_uri := self.get_secret_uri(relation, group) - ): - self._register_secret_to_relation(relation.name, relation.id, secret_uri, group) + return False def _is_resource_created_for_relation(self, relation: Relation) -> bool: if not relation.app: return False - data = self.fetch_relation_data([relation.id], ["username", "password"]).get( - relation.id, {} + data = self.fetch_relation_data( + [relation.id], + ["username", "password", "entity-name", "entity-password"], + ).get(relation.id, {}) + + return any( + [ + all(bool(data.get(field)) for field in ("username", "password")), + all(bool(data.get(field)) for field in ("entity-name",)), + ] ) - return bool(data.get("username")) and bool(data.get("password")) - # Public functions + def _validate_entity_type(self) -> None: + """Validates the consistency of the provided entity-type and its extra roles.""" + if self.entity_type and self.entity_type not in {ENTITY_USER, ENTITY_GROUP}: + raise ValueError("Invalid entity-type. Possible values are USER and GROUP") - def get_secret_uri(self, relation: Relation, group: SecretGroup) -> Optional[str]: - """Getting relation secret URI for the corresponding Secret Group.""" - secret_field = self._generate_secret_field_name(group) - return relation.data[relation.app].get(secret_field) + if self.entity_type == ENTITY_USER and self.extra_group_roles: + raise ValueError("Inconsistent entity information. Use extra_user_roles instead") + + if self.entity_type == ENTITY_GROUP and self.extra_user_roles: + raise ValueError("Inconsistent entity information. Use extra_group_roles instead") - def set_secret_uri(self, relation: Relation, group: SecretGroup, uri: str) -> None: - """Setting relation secret URI is not possible for a Requirer.""" - raise NotImplementedError("Requirer can not change the relation secret URI.") + def _validate_entity_permissions(self) -> None: + """Validates whether the provided entity permissions follow the right JSON format.""" + if not self.entity_permissions: + return + + accepted_keys = {"resource_name", "resource_type", "privileges"} + + try: + permissions = json.loads(self.entity_permissions) + for permission in permissions: + if permission.keys() != accepted_keys: + raise ValueError("Invalid entity permissions format. See accepted keys") + except json.decoder.JSONDecodeError: + raise ValueError("Invalid entity permissions format. It must be JSON format") + + # Public functions def is_resource_created(self, relation_id: Optional[int] = None) -> bool: """Check if the resource has been created. @@ -1796,62 +2302,69 @@ def is_resource_created(self, relation_id: Optional[int] = None) -> bool: else False ) - # Mandatory internal overrides + # Public functions -- inherited - @juju_secrets_only - def _get_relation_secret( - self, relation_id: int, group: SecretGroup, relation_name: Optional[str] = None - ) -> Optional[CachedSecret]: - """Retrieve a Juju Secret that's been stored in the relation databag.""" - if not relation_name: - relation_name = self.relation_name + fetch_my_relation_data = leader_only(Data.fetch_my_relation_data) + fetch_my_relation_field = leader_only(Data.fetch_my_relation_field) - label = self._generate_secret_label(relation_name, relation_id, group) - return self.secrets.get(label) + def _load_secrets_from_databag(self, relation: Relation) -> None: + """Load secrets from the databag.""" + requested_secrets = get_encoded_list(relation, self.local_unit, REQ_SECRET_FIELDS) + provided_secrets = get_encoded_list(relation, self.local_unit, PROV_SECRET_FIELDS) + if requested_secrets: + self._remote_secret_fields = requested_secrets - def _fetch_specific_relation_data( - self, relation, fields: Optional[List[str]] = None - ) -> Dict[str, str]: - """Fetching Requirer data -- that may include secrets.""" - if not relation.app: - return {} - return self._fetch_relation_data_with_secrets( - relation.app, self.secret_fields, relation, fields - ) + if provided_secrets: + self._local_secret_fields = provided_secrets - def _fetch_my_specific_relation_data(self, relation, fields: Optional[List[str]]) -> dict: - """Fetching our own relation data.""" - return self._fetch_relation_data_without_secrets(self.local_app, relation, fields) - def _update_relation_data(self, relation: Relation, data: dict) -> None: - """Updates a set of key-value pairs in the relation. +class StatusEventBase(RelationEvent): + """Base class for relation status change events.""" - This function writes in the application data bag, therefore, - only the leader unit can call it. + def __init__( + self, + handle: Handle, + relation: Relation, + status: RelationStatus, + app: Optional[Application] = None, + unit: Optional[Unit] = None, + ): + super().__init__(handle, relation, app=app, unit=unit) + self.status = status - Args: - relation: the particular relation. - data: dict containing the key-value pairs - that should be updated in the relation. - """ - return self._update_relation_data_without_secrets(self.local_app, relation, data) + def snapshot(self) -> dict: + """Return a snapshot of the event.""" + return super().snapshot() | {"status": json.dumps(asdict(self.status))} - def _delete_relation_data(self, relation: Relation, fields: List[str]) -> None: - """Deletes a set of fields from the relation. + def restore(self, snapshot: dict): + """Restore the event from a snapshot.""" + super().restore(snapshot) + self.status = RelationStatus(**json.loads(snapshot["status"])) - This function writes in the application data bag, therefore, - only the leader unit can call it. + @property + def active_statuses(self) -> List[RelationStatus]: + """Returns a list of all currently active statuses on this relation.""" + if not self.relation.app: + return [] - Args: - relation: the particular relation. - fields: list containing the field names that should be removed from the relation. - """ - return self._delete_relation_data_without_secrets(self.local_app, relation, fields) + raw = json.loads(self.relation.data[self.relation.app].get(STATUS_FIELD, "[]")) - # Public functions -- inherited + return [RelationStatus(**item) for item in raw] - fetch_my_relation_data = leader_only(Data.fetch_my_relation_data) - fetch_my_relation_field = leader_only(Data.fetch_my_relation_field) + +class StatusRaisedEvent(StatusEventBase): + """Event emitted on the requirer when a new status is being raised by the provider on relation.""" + + +class StatusResolvedEvent(StatusEventBase): + """Event emitted on the requirer when a status is marked as resolved by the provider on relation.""" + + +class RequirerCharmEvents(CharmEvents): + """Base events for data requirer charms.""" + + status_raised = EventSource(StatusRaisedEvent) + status_resolved = EventSource(StatusResolvedEvent) class RequirerEventHandlers(EventHandlers): @@ -1861,13 +2374,24 @@ def __init__(self, charm: CharmBase, relation_data: RequirerData, unique_key: st """Manager of base client relations.""" super().__init__(charm, relation_data, unique_key) - self.framework.observe( - self.charm.on[relation_data.relation_name].relation_created, - self._on_relation_created_event, - ) - self.framework.observe( - charm.on.secret_changed, - self._on_secret_changed_event, + def _main_credentials_shared(self, diff: Diff) -> bool: + """Whether the relation data-bag contains username / password keys.""" + user_secret = self.relation_data._generate_secret_field_name(SECRET_GROUPS.USER) + return any( + [ + user_secret in diff.added, + "username" in diff.added and "password" in diff.added, + ] + ) + + def _entity_credentials_shared(self, diff: Diff) -> bool: + """Whether the relation data-bag contains rolename / password keys.""" + entity_secret = self.relation_data._generate_secret_field_name(SECRET_GROUPS.ENTITY) + return any( + [ + entity_secret in diff.added, + "entity-name" in diff.added, + ] ) # Event handlers @@ -1877,18 +2401,148 @@ def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: if not self.relation_data.local_unit.is_leader(): return - if self.relation_data.secret_fields: # pyright: ignore [reportAttributeAccessIssue] + if self.relation_data.remote_secret_fields: + if self.relation_data.SCOPE == Scope.APP: + set_encoded_field( + event.relation, + self.relation_data.local_app, + REQ_SECRET_FIELDS, + self.relation_data.remote_secret_fields, + ) + set_encoded_field( event.relation, - self.relation_data.component, + self.relation_data.local_unit, REQ_SECRET_FIELDS, - self.relation_data.secret_fields, # pyright: ignore [reportAttributeAccessIssue] + self.relation_data.remote_secret_fields, ) - @abstractmethod - def _on_secret_changed_event(self, event: RelationChangedEvent) -> None: + if self.relation_data.local_secret_fields: + if self.relation_data.SCOPE == Scope.APP: + set_encoded_field( + event.relation, + self.relation_data.local_app, + PROV_SECRET_FIELDS, + self.relation_data.local_secret_fields, + ) + set_encoded_field( + event.relation, + self.relation_data.local_unit, + PROV_SECRET_FIELDS, + self.relation_data.local_secret_fields, + ) + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation has changed.""" + # Retrieve old statuses from "data" + old_data = get_encoded_dict(event.relation, self.relation_data.local_unit, "data") or {} + old_statuses = json.loads(old_data.get(STATUS_FIELD, "[]")) + previous_codes = {status.get("code") for status in old_statuses} + + # Compute current statuses + current_statuses = json.loads( + self.relation_data.fetch_relation_field(event.relation.id, STATUS_FIELD) or "[]" + ) + current_codes = {status.get("code") for status in current_statuses} + + # Detect changes + raised = current_codes - previous_codes + resolved = previous_codes - current_codes + + for status_code in raised: + logger.debug(f"Status [{status_code}] raised") + _status = next(s for s in current_statuses if s["code"] == status_code) + _status_instance = RelationStatus(**_status) + getattr(self.on, "status_raised").emit( + event.relation, + status=_status_instance, + app=event.app, + unit=event.unit, + ) + + for status_code in resolved: + logger.debug(f"Status [{status_code}] resolved") + _status = next(s for s in old_statuses if s["code"] == status_code) + _status_instance = RelationStatus(**_status) + getattr(self.on, "status_resolved").emit( + event.relation, + status=_status_instance, + app=event.app, + unit=event.unit, + ) + + +class ProviderEventHandlers(EventHandlers): + """Provider-side of the relation.""" + + def __init__(self, charm: CharmBase, relation_data: ProviderData, unique_key: str = ""): + """Manager of base client relations.""" + super().__init__(charm, relation_data, unique_key) + + @staticmethod + def _validate_entity_consistency(event: RelationEvent, diff: Diff) -> None: + """Validates that entity information is not changed after relation is established. + + - When entity-type changes, backwards compatibility is broken. + - When extra-user-roles changes, role membership checks become incredibly complex. + - When extra-group-roles changes, role membership checks become incredibly complex. + """ + if not isinstance(event, RelationChangedEvent): + return + + for key in ["entity-type", "extra-user-roles", "extra-group-roles"]: + if key in diff.changed: + raise ValueError(f"Cannot change {key} after relation has already been created") + + # Event handlers + def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: + """Event emitted when a relation is created.""" + if not self.relation_data.local_unit.is_leader(): + return + + try: + remote_model_uuid = event.relation.remote_model.uuid + except (FileNotFoundError, ModelError, RuntimeError): + # access to remote model added in Juju 3.6.2, fails with 2.9 + return + + if self.model.uuid == remote_model_uuid: + return + + if not Fernet: + logger.warning( + "Cryptography module not installed. Could not decrypt sensitive field in cross-model relation" + ) + return + + # in cross-model relations, generate an encryption key and share it with the requirer as a secret + event_data = {} + secret_label = f"{self.model.uuid}-{event.relation.id}-encryption-secret" + + try: + # check if secret was already created to avoid duplicates + secret = self.charm.model.get_secret(label=secret_label) + except SecretNotFoundError: + encryption_key = Fernet.generate_key() # pyright: ignore [reportOptionalMemberAccess] + content = {"encryption-key": encryption_key.decode()} + secret = self.charm.app.add_secret(content, label=secret_label) + + secret.grant(event.relation) + if not secret.id: + raise SecretError("Encryption secret is missing secred id") + event_data["encryption-secret"] = secret.id + + self.relation_data.update_relation_data(event.relation.id, event_data) + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: """Event emitted when the relation data has changed.""" - raise NotImplementedError + requested_secrets = get_encoded_list(event.relation, event.relation.app, REQ_SECRET_FIELDS) + provided_secrets = get_encoded_list(event.relation, event.relation.app, PROV_SECRET_FIELDS) + if requested_secrets is not None: + self.relation_data._local_secret_fields = requested_secrets + + if provided_secrets is not None: + self.relation_data._remote_secret_fields = provided_secrets ################################################################################ @@ -1907,7 +2561,6 @@ def __init__( self, model, relation_name: str, - extra_user_roles: Optional[str] = None, additional_secret_fields: Optional[List[str]] = [], additional_secret_group_mapping: Dict[str, str] = {}, secret_field_name: Optional[str] = None, @@ -1915,10 +2568,9 @@ def __init__( ): RequirerData.__init__( self, - model, - relation_name, - extra_user_roles, - additional_secret_fields, + model=model, + relation_name=relation_name, + additional_secret_fields=additional_secret_fields, ) self.secret_field_name = secret_field_name if secret_field_name else self.SECRET_FIELD_NAME self.deleted_label = deleted_label @@ -1939,7 +2591,7 @@ def __init__( secret_group = SECRET_GROUPS.get_group(group) internal_field = self._field_to_internal_name(field, secret_group) self._secret_label_map.setdefault(group, []).append(internal_field) - self._secret_fields.append(internal_field) + self._remote_secret_fields.append(internal_field) @property def scope(self) -> Optional[Scope]: @@ -1957,10 +2609,10 @@ def secret_label_map(self) -> Dict[str, str]: @property def static_secret_fields(self) -> List[str]: """Re-definition of the property in a way that dynamically extended list is retrieved.""" - return self._secret_fields + return self._remote_secret_fields @property - def secret_fields(self) -> List[str]: + def local_secret_fields(self) -> List[str]: """Re-definition of the property in a way that dynamically extended list is retrieved.""" return ( self.static_secret_fields if self.static_secret_fields else self.current_secret_fields @@ -1978,7 +2630,12 @@ def current_secret_fields(self) -> List[str]: relation = self._model.relations[self.relation_name][0] fields = [] - ignores = [SECRET_GROUPS.get_group("user"), SECRET_GROUPS.get_group("tls")] + ignores = [ + SECRET_GROUPS.get_group("user"), + SECRET_GROUPS.get_group("tls"), + SECRET_GROUPS.get_group("mtls"), + SECRET_GROUPS.get_group("entity"), + ] for group in SECRET_GROUPS.groups(): if group in ignores: continue @@ -2087,11 +2744,11 @@ def _content_for_secret_group( ) -> Dict[str, str]: """Select : pairs from input, that belong to this particular Secret group.""" if group_mapping == SECRET_GROUPS.EXTRA: - return {k: v for k, v in content.items() if k in self.secret_fields} + return {k: v for k, v in content.items() if k in self.local_secret_fields} return { self._internal_name_to_field(k)[0]: v for k, v in content.items() - if k in self.secret_fields + if k in self.local_secret_fields } def valid_field_pattern(self, field: str, full_field: str) -> bool: @@ -2106,6 +2763,16 @@ def valid_field_pattern(self, field: str, full_field: str) -> bool: return False return True + def _load_secrets_from_databag(self, relation: Relation) -> None: + """Load secrets from the databag.""" + requested_secrets = get_encoded_list(relation, self.component, REQ_SECRET_FIELDS) + provided_secrets = get_encoded_list(relation, self.component, PROV_SECRET_FIELDS) + if requested_secrets: + self._remote_secret_fields = requested_secrets + + if provided_secrets: + self._local_secret_fields = provided_secrets + ########################################################################## # Backwards compatibility / Upgrades ########################################################################## @@ -2161,7 +2828,7 @@ def _legacy_compat_check_deleted_label(self, relation, fields) -> None: if current_data is not None: # Check if the secret we wanna delete actually exists # Given the "deleted label", here we can't rely on the default mechanism (i.e. 'key not found') - if non_existent := (set(fields) & set(self.secret_fields)) - set( + if non_existent := (set(fields) & set(self.local_secret_fields)) - set( current_data.get(relation.id, []) ): logger.debug( @@ -2211,10 +2878,10 @@ def _legacy_migration_remove_secret_from_databag(self, relation, fields: List[st Practically what happens here is to remove stuff from the databag that is to be stored in secrets. """ - if not self.secret_fields: + if not self.local_secret_fields: return - secret_fields_passed = set(self.secret_fields) & set(fields) + secret_fields_passed = set(self.local_secret_fields) & set(fields) for field in secret_fields_passed: if self._fetch_relation_data_without_secrets(self.component, relation, [field]): self._delete_relation_data_without_secrets(self.component, relation, [field]) @@ -2326,15 +2993,17 @@ def _fetch_my_specific_relation_data( ) -> Dict[str, str]: """Fetch data available (directily or indirectly -- i.e. secrets) from the relation for owner/this_app.""" return self._fetch_relation_data_with_secrets( - self.component, self.secret_fields, relation, fields + self.component, self.local_secret_fields, relation, fields ) @either_static_or_dynamic_secrets def _update_relation_data(self, relation: Relation, data: Dict[str, str]) -> None: """Update data available (directily or indirectly -- i.e. secrets) from the relation for owner/this_app.""" + self._load_secrets_from_databag(relation) + _, normal_fields = self._process_secret_fields( relation, - self.secret_fields, + self.local_secret_fields, list(data), self._add_or_update_relation_secrets, data=data, @@ -2347,18 +3016,22 @@ def _update_relation_data(self, relation: Relation, data: Dict[str, str]) -> Non @either_static_or_dynamic_secrets def _delete_relation_data(self, relation: Relation, fields: List[str]) -> None: """Delete data available (directily or indirectly -- i.e. secrets) from the relation for owner/this_app.""" - if self.secret_fields and self.deleted_label: - + self._load_secrets_from_databag(relation) + if self.local_secret_fields and self.deleted_label: _, normal_fields = self._process_secret_fields( relation, - self.secret_fields, + self.local_secret_fields, fields, self._update_relation_secret, - data={field: self.deleted_label for field in fields}, + data=dict.fromkeys(fields, self.deleted_label), ) else: _, normal_fields = self._process_secret_fields( - relation, self.secret_fields, fields, self._delete_relation_secret, fields=fields + relation, + self.local_secret_fields, + fields, + self._delete_relation_secret, + fields=fields, ) self._delete_relation_data_without_secrets(self.component, relation, list(normal_fields)) @@ -2414,7 +3087,6 @@ def __init__( self, charm, relation_name: str, - extra_user_roles: Optional[str] = None, additional_secret_fields: Optional[List[str]] = [], additional_secret_group_mapping: Dict[str, str] = {}, secret_field_name: Optional[str] = None, @@ -2425,7 +3097,6 @@ def __init__( self, charm.model, relation_name, - extra_user_roles, additional_secret_fields, additional_secret_group_mapping, secret_field_name, @@ -2450,7 +3121,6 @@ def __init__( self, charm, relation_name: str, - extra_user_roles: Optional[str] = None, additional_secret_fields: Optional[List[str]] = [], additional_secret_group_mapping: Dict[str, str] = {}, secret_field_name: Optional[str] = None, @@ -2461,7 +3131,6 @@ def __init__( self, charm.model, relation_name, - extra_user_roles, additional_secret_fields, additional_secret_group_mapping, secret_field_name, @@ -2504,7 +3173,6 @@ def __init__( unit: Unit, charm: CharmBase, relation_name: str, - extra_user_roles: Optional[str] = None, additional_secret_fields: Optional[List[str]] = [], additional_secret_group_mapping: Dict[str, str] = {}, secret_field_name: Optional[str] = None, @@ -2515,7 +3183,6 @@ def __init__( unit, charm.model, relation_name, - extra_user_roles, additional_secret_fields, additional_secret_group_mapping, secret_field_name, @@ -2525,24 +3192,12 @@ def __init__( ################################################################################ -# Cross-charm Relatoins Data Handling and Evenets +# Cross-charm Relations Data Handling and Events ################################################################################ # Generic events -class ExtraRoleEvent(RelationEvent): - """Base class for data events.""" - - @property - def extra_user_roles(self) -> Optional[str]: - """Returns the extra user roles that were requested.""" - if not self.relation.app: - return None - - return self.relation.data[self.relation.app].get("extra-user-roles") - - class RelationEventWithSecret(RelationEvent): """Base class for Relation Events that need to handle secrets.""" @@ -2574,6 +3229,76 @@ def secrets_enabled(self): return JujuVersion.from_environ().has_secrets +class EntityProvidesEvent(RelationEvent): + """Base class for data events.""" + + @property + def extra_user_roles(self) -> Optional[str]: + """Returns the extra user roles that were requested.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("extra-user-roles") + + @property + def extra_group_roles(self) -> Optional[str]: + """Returns the extra group roles that were requested.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("extra-group-roles") + + @property + def entity_type(self) -> Optional[str]: + """Returns the entity_type that were requested.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("entity-type") + + @property + def entity_permissions(self) -> Optional[str]: + """Returns the entity_permissions that were requested.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("entity-permissions") + + +class EntityRequiresEvent(RelationEventWithSecret): + """Base class for authentication fields for events. + + The amount of logic added here is not ideal -- but this was the only way to preserve + the interface when moving to Juju Secrets + """ + + @property + def entity_name(self) -> Optional[str]: + """Returns the name for the created entity.""" + if not self.relation.app: + return None + + if self.secrets_enabled: + secret = self._get_secret("entity") + if secret: + return secret.get("entity-name") + + return self.relation.data[self.relation.app].get("entity-name") + + @property + def entity_password(self) -> Optional[str]: + """Returns the password for the created entity.""" + if not self.relation.app: + return None + + if self.secrets_enabled: + secret = self._get_secret("entity") + if secret: + return secret.get("entity-password") + + return self.relation.data[self.relation.app].get("entity-password") + + class AuthenticationEvent(RelationEventWithSecret): """Base class for authentication fields for events. @@ -2649,9 +3374,17 @@ def database(self) -> Optional[str]: return self.relation.data[self.relation.app].get("database") -class DatabaseRequestedEvent(DatabaseProvidesEvent, ExtraRoleEvent): +class DatabaseRequestedEvent(DatabaseProvidesEvent): """Event emitted when a new database is requested for use on this relation.""" + @property + def extra_user_roles(self) -> Optional[str]: + """Returns the extra user roles that were requested.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("extra-user-roles") + @property def external_node_connectivity(self) -> bool: """Returns the requested external_node_connectivity field.""" @@ -2663,6 +3396,37 @@ def external_node_connectivity(self) -> bool: == "true" ) + @property + def requested_entity_secret_content(self) -> Optional[Dict[str, Optional[str]]]: + """Returns the content of the requested entity secret.""" + names = None + if secret_uri := self.relation.data.get(self.relation.app, {}).get( + "requested-entity-secret" + ): + secret = self.framework.model.get_secret(id=secret_uri) + if content := secret.get_content(refresh=True): + if "entity-name" in content: + names = {content["entity-name"]: content.get("password")} + else: + logger.warning("Invalid requested-entity-secret: no entity name") + return names + + @property + def prefix_matching(self) -> Optional[str]: + """Returns the prefix matching strategy that were requested.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("prefix-matching") + + +class DatabaseEntityRequestedEvent(DatabaseProvidesEvent, EntityProvidesEvent): + """Event emitted when a new entity is requested for use on this relation.""" + + +class DatabaseEntityPermissionsChangedEvent(DatabaseProvidesEvent, EntityProvidesEvent): + """Event emitted when existing entity permissions are changed on this relation.""" + class DatabaseProvidesEvents(CharmEvents): """Database events. @@ -2671,6 +3435,8 @@ class DatabaseProvidesEvents(CharmEvents): """ database_requested = EventSource(DatabaseRequestedEvent) + database_entity_requested = EventSource(DatabaseEntityRequestedEvent) + database_entity_permissions_changed = EventSource(DatabaseEntityPermissionsChangedEvent) class DatabaseRequiresEvent(RelationEventWithSecret): @@ -2735,6 +3501,19 @@ def uris(self) -> Optional[str]: return self.relation.data[self.relation.app].get("uris") + @property + def read_only_uris(self) -> Optional[str]: + """Returns the readonly connection URIs.""" + if not self.relation.app: + return None + + if self.secrets_enabled: + secret = self._get_secret("user") + if secret: + return secret.get("read-only-uris") + + return self.relation.data[self.relation.app].get("read-only-uris") + @property def version(self) -> Optional[str]: """Returns the version of the database. @@ -2746,11 +3525,25 @@ def version(self) -> Optional[str]: return self.relation.data[self.relation.app].get("version") + @property + def prefix_databases(self) -> Optional[List[str]]: + """Returns a list of databases matching a prefix.""" + if not self.relation.app: + return None + + if prefixed_databases := self.relation.data[self.relation.app].get("prefix-databases"): + return prefixed_databases.split(",") + return [] + class DatabaseCreatedEvent(AuthenticationEvent, DatabaseRequiresEvent): """Event emitted when a new database is created for use on this relation.""" +class DatabaseEntityCreatedEvent(EntityRequiresEvent, DatabaseRequiresEvent): + """Event emitted when a new entity is created for use on this relation.""" + + class DatabaseEndpointsChangedEvent(AuthenticationEvent, DatabaseRequiresEvent): """Event emitted when the read/write endpoints are changed.""" @@ -2759,15 +3552,21 @@ class DatabaseReadOnlyEndpointsChangedEvent(AuthenticationEvent, DatabaseRequire """Event emitted when the read only endpoints are changed.""" -class DatabaseRequiresEvents(CharmEvents): +class DatabasePrefixDatabasesChangedEvent(AuthenticationEvent, DatabaseRequiresEvent): + """Event emitted when the prefix databases are changed.""" + + +class DatabaseRequiresEvents(RequirerCharmEvents): """Database events. This class defines the events that the database can emit. """ database_created = EventSource(DatabaseCreatedEvent) + database_entity_created = EventSource(DatabaseEntityCreatedEvent) endpoints_changed = EventSource(DatabaseEndpointsChangedEvent) read_only_endpoints_changed = EventSource(DatabaseReadOnlyEndpointsChangedEvent) + prefix_databases_changed = EventSource(DatabasePrefixDatabasesChangedEvent) # Database Provider and Requires @@ -2776,8 +3575,10 @@ class DatabaseRequiresEvents(CharmEvents): class DatabaseProviderData(ProviderData): """Provider-side data of the database relations.""" - def __init__(self, model: Model, relation_name: str) -> None: - super().__init__(model, relation_name) + def __init__( + self, model: Model, relation_name: str, status_schema_path: OptionalPathLike = None + ) -> None: + super().__init__(model, relation_name, status_schema_path=status_schema_path) def set_database(self, relation_id: int, database_name: str) -> None: """Set database name. @@ -2791,6 +3592,18 @@ def set_database(self, relation_id: int, database_name: str) -> None: """ self.update_relation_data(relation_id, {"database": database_name}) + def set_prefix_databases(self, relation_id: int, databases: List[str]) -> None: + """Set a coma separated list of databases matching a prefix. + + This function writes in the application data bag, therefore, + only the leader unit can call it. + + Args: + relation_id: the identifier for a particular relation. + databases: list of database names matching the requested prefix. + """ + self.update_relation_data(relation_id, {"prefix-databases": ",".join(sorted(databases))}) + def set_endpoints(self, relation_id: int, connection_strings: str) -> None: """Set database primary connections. @@ -2841,6 +3654,15 @@ def set_uris(self, relation_id: int, uris: str) -> None: """ self.update_relation_data(relation_id, {"uris": uris}) + def set_read_only_uris(self, relation_id: int, uris: str) -> None: + """Set the database readonly connection URIs in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + uris: connection URIs. + """ + self.update_relation_data(relation_id, {"read-only-uris": uris}) + def set_version(self, relation_id: int, version: str) -> None: """Set the database version in the application relation databag. @@ -2859,7 +3681,7 @@ def set_subordinated(self, relation_id: int) -> None: self.update_relation_data(relation_id, {"subordinated": "true"}) -class DatabaseProviderEventHandlers(EventHandlers): +class DatabaseProviderEventHandlers(ProviderEventHandlers): """Provider-side of the database relation handlers.""" on = DatabaseProvidesEvents() # pyright: ignore [reportAssignmentType] @@ -2874,25 +3696,65 @@ def __init__( def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: """Event emitted when the relation has changed.""" + super()._on_relation_changed_event(event) # Leader only if not self.relation_data.local_unit.is_leader(): return + # Check which data has changed to emit customs events. diff = self._diff(event) - # Emit a database requested event if the setup key (database name and optional - # extra user roles) was added to the relation databag by the application. - if "database" in diff.added: + # Validate entity information is not dynamically changed + self._validate_entity_consistency(event, diff) + + # Emit a database requested event if the setup key (database name) + # was added to the relation databag, but the entity-type key was not. + if "database" in diff.added and "entity-type" not in diff.added: getattr(self.on, "database_requested").emit( event.relation, app=event.app, unit=event.unit ) + # To avoid unnecessary application restarts do not trigger other events. + return + + # Emit an entity requested event if the setup key (database name) + # was added to the relation databag, in addition to the entity-type key. + if "database" in diff.added and "entity-type" in diff.added: + getattr(self.on, "database_entity_requested").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + + # Emit a permissions changed event if the setup key (database name) + # was added to the relation databag, and the entity-permissions key changed. + if ( + "database" not in diff.added + and "entity-type" not in diff.added + and ("entity-permissions" in diff.added or "entity-permissions" in diff.changed) + ): + getattr(self.on, "database_entity_permissions_changed").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + + def _on_secret_changed_event(self, event: SecretChangedEvent) -> None: + """Event emitted when the secret has changed.""" + pass + class DatabaseProvides(DatabaseProviderData, DatabaseProviderEventHandlers): """Provider-side of the database relations.""" - def __init__(self, charm: CharmBase, relation_name: str) -> None: - DatabaseProviderData.__init__(self, charm.model, relation_name) + def __init__( + self, charm: CharmBase, relation_name: str, status_schema_path: OptionalPathLike = None + ) -> None: + DatabaseProviderData.__init__( + self, charm.model, relation_name, status_schema_path=status_schema_path + ) DatabaseProviderEventHandlers.__init__(self, charm, self) @@ -2908,9 +3770,28 @@ def __init__( relations_aliases: Optional[List[str]] = None, additional_secret_fields: Optional[List[str]] = [], external_node_connectivity: bool = False, + extra_group_roles: Optional[str] = None, + entity_type: Optional[str] = None, + entity_permissions: Optional[str] = None, + requested_entity_secret: Optional[str] = None, + requested_entity_name: Optional[str] = None, + requested_entity_password: Optional[str] = None, + prefix_matching: Optional[str] = None, ): """Manager of database client relations.""" - super().__init__(model, relation_name, extra_user_roles, additional_secret_fields) + super().__init__( + model, + relation_name, + extra_user_roles, + additional_secret_fields, + extra_group_roles, + entity_type, + entity_permissions, + requested_entity_secret, + requested_entity_name, + requested_entity_password, + prefix_matching, + ) self.database = database_name self.relations_aliases = relations_aliases self.external_node_connectivity = external_node_connectivity @@ -2993,14 +3874,26 @@ def __init__( if self.relation_data.relations_aliases: for relation_alias in self.relation_data.relations_aliases: - self.on.define_event(f"{relation_alias}_database_created", DatabaseCreatedEvent) self.on.define_event( - f"{relation_alias}_endpoints_changed", DatabaseEndpointsChangedEvent + f"{relation_alias}_database_created", + DatabaseCreatedEvent, + ) + self.on.define_event( + f"{relation_alias}_database_entity_created", + DatabaseEntityCreatedEvent, + ) + self.on.define_event( + f"{relation_alias}_endpoints_changed", + DatabaseEndpointsChangedEvent, ) self.on.define_event( f"{relation_alias}_read_only_endpoints_changed", DatabaseReadOnlyEndpointsChangedEvent, ) + self.on.define_event( + f"{relation_alias}_prefix_databases_changed", + DatabasePrefixDatabasesChangedEvent, + ) def _on_secret_changed_event(self, event: SecretChangedEvent): """Event notifying about a new value of a secret.""" @@ -3085,6 +3978,32 @@ def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: if self.relation_data.extra_user_roles: event_data["extra-user-roles"] = self.relation_data.extra_user_roles + if self.relation_data.extra_group_roles: + event_data["extra-group-roles"] = self.relation_data.extra_group_roles + if self.relation_data.entity_type: + event_data["entity-type"] = self.relation_data.entity_type + if self.relation_data.entity_permissions: + event_data["entity-permissions"] = self.relation_data.entity_permissions + if self.relation_data.requested_entity_secret: + event_data["requested-entity-secret"] = self.relation_data.requested_entity_secret + if self.relation_data.prefix_matching: + event_data["prefix-matching"] = self.relation_data.prefix_matching + + # Create helper secret if needed + if ( + self.relation_data.requested_entity_name + and not self.relation_data.requested_entity_secret + ): + content = {"entity-name": self.relation_data.requested_entity_name} + if self.relation_data.requested_entity_password: + content["password"] = self.relation_data.requested_entity_password + secret = self.charm.app.add_secret( + content, label=f"{self.model.uuid}-{event.relation.id}-requested-entity" + ) + secret.grant(event.relation) + if not secret.id: + raise SecretError("Secret helper missing Id") + event_data["requested-entity-secret"] = secret.id # set external-node-connectivity field if self.relation_data.external_node_connectivity: @@ -3092,8 +4011,22 @@ def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: self.relation_data.update_relation_data(event.relation.id, event_data) + def _clear_helper_secret(self, event: RelationChangedEvent, app_databag: Dict) -> None: + """Remove helper secret if set.""" + if ( + self.relation_data.local_unit.is_leader() + and self.relation_data.requested_entity_name + and (secret_uri := app_databag.get("requested-entity-secret")) + ): + try: + secret = self.framework.model.get_secret(id=secret_uri) + secret.remove_all_revisions() + except ModelError: + logger.debug("Unable to remove helper secret") + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: """Event emitted when the database relation has changed.""" + super()._on_relation_changed_event(event) is_subordinate = False remote_unit_data = None for key in event.relation.data.keys(): @@ -3103,10 +4036,7 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: is_subordinate = event.relation.data[key].get("subordinated") == "true" if is_subordinate: - if not remote_unit_data: - return - - if remote_unit_data.get("state") != "ready": + if not remote_unit_data or remote_unit_data.get("state") != "ready": return # Check which data has changed to emit customs events. @@ -3116,12 +4046,13 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: if any(newval for newval in diff.added if self.relation_data._is_secret_field(newval)): self.relation_data._register_secrets_to_relation(event.relation, diff.added) + app_databag = get_encoded_dict(event.relation, event.app, "data") + if app_databag is None: + app_databag = {} + # Check if the database is created # (the database charm shared the credentials). - secret_field_user = self.relation_data._generate_secret_field_name(SECRET_GROUPS.USER) - if ( - "username" in diff.added and "password" in diff.added - ) or secret_field_user in diff.added: + if self._main_credentials_shared(diff) and "entity-type" not in app_databag: # Emit the default event (the one without an alias). logger.info("database created at %s", datetime.now()) getattr(self.on, "database_created").emit( @@ -3130,38 +4061,41 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: # Emit the aliased event (if any). self._emit_aliased_event(event, "database_created") + self._clear_helper_secret(event, app_databag) - # To avoid unnecessary application restarts do not trigger - # “endpoints_changed“ event if “database_created“ is triggered. + # To avoid unnecessary application restarts do not trigger other events. return - # Emit an endpoints changed event if the database - # added or changed this info in the relation databag. - if "endpoints" in diff.added or "endpoints" in diff.changed: + if self._entity_credentials_shared(diff) and "entity-type" in app_databag: # Emit the default event (the one without an alias). - logger.info("endpoints changed on %s", datetime.now()) - getattr(self.on, "endpoints_changed").emit( + logger.info("entity created at %s", datetime.now()) + getattr(self.on, "database_entity_created").emit( event.relation, app=event.app, unit=event.unit ) # Emit the aliased event (if any). - self._emit_aliased_event(event, "endpoints_changed") + self._emit_aliased_event(event, "database_entity_created") + self._clear_helper_secret(event, app_databag) - # To avoid unnecessary application restarts do not trigger - # “read_only_endpoints_changed“ event if “endpoints_changed“ is triggered. + # To avoid unnecessary application restarts do not trigger other events. return - # Emit a read only endpoints changed event if the database - # added or changed this info in the relation databag. - if "read-only-endpoints" in diff.added or "read-only-endpoints" in diff.changed: - # Emit the default event (the one without an alias). - logger.info("read-only-endpoints changed on %s", datetime.now()) - getattr(self.on, "read_only_endpoints_changed").emit( - event.relation, app=event.app, unit=event.unit - ) - - # Emit the aliased event (if any). - self._emit_aliased_event(event, "read_only_endpoints_changed") + for key, event_name in [ + ("endpoints", "endpoints_changed"), + ("read-only-endpoints", "read_only_endpoints_changed"), + ("prefix-databases", "prefix_databases_changed"), + ]: + # Emit a change event if the key changed. + if key in diff.added or key in diff.changed: + # Emit the default event (the one without an alias). + logger.info("%s changed on %s", key, datetime.now()) + getattr(self.on, event_name).emit(event.relation, app=event.app, unit=event.unit) + + # Emit the aliased event (if any). + self._emit_aliased_event(event, event_name) + + # To avoid unnecessary application restarts do not trigger other events. + return class DatabaseRequires(DatabaseRequirerData, DatabaseRequirerEventHandlers): @@ -3176,6 +4110,13 @@ def __init__( relations_aliases: Optional[List[str]] = None, additional_secret_fields: Optional[List[str]] = [], external_node_connectivity: bool = False, + extra_group_roles: Optional[str] = None, + entity_type: Optional[str] = None, + entity_permissions: Optional[str] = None, + requested_entity_secret: Optional[str] = None, + requested_entity_name: Optional[str] = None, + requested_entity_password: Optional[str] = None, + prefix_matching: Optional[str] = None, ): DatabaseRequirerData.__init__( self, @@ -3186,6 +4127,13 @@ def __init__( relations_aliases, additional_secret_fields, external_node_connectivity, + extra_group_roles, + entity_type, + entity_permissions, + requested_entity_secret, + requested_entity_name, + requested_entity_password, + prefix_matching, ) DatabaseRequirerEventHandlers.__init__(self, charm, self) @@ -3197,7 +4145,7 @@ def __init__( # Kafka Events -class KafkaProvidesEvent(RelationEvent): +class KafkaProvidesEvent(RelationEventWithSecret): """Base class for Kafka events.""" @property @@ -3216,18 +4164,71 @@ def consumer_group_prefix(self) -> Optional[str]: return self.relation.data[self.relation.app].get("consumer-group-prefix") + @property + def mtls_cert(self) -> Optional[str]: + """Returns TLS cert of the client.""" + if not self.relation.app: + return None -class TopicRequestedEvent(KafkaProvidesEvent, ExtraRoleEvent): - """Event emitted when a new topic is requested for use on this relation.""" + if not self.secrets_enabled: + raise SecretsUnavailableError("Secrets unavailable on current Juju version") + secret_field = f"{PROV_SECRET_PREFIX}{SECRET_GROUPS.MTLS}" + if secret_uri := self.relation.data[self.app].get(secret_field): + secret = self.framework.model.get_secret(id=secret_uri) + content = secret.get_content(refresh=True) + if content: + return content.get("mtls-cert") -class KafkaProvidesEvents(CharmEvents): - """Kafka events. + +class KafkaClientMtlsCertUpdatedEvent(KafkaProvidesEvent): + """Event emitted when the mtls relation is updated.""" + + def __init__(self, handle, relation, old_mtls_cert: Optional[str] = None, app=None, unit=None): + super().__init__(handle, relation, app, unit) + + self.old_mtls_cert = old_mtls_cert + + def snapshot(self): + """Return a snapshot of the event.""" + return super().snapshot() | {"old_mtls_cert": self.old_mtls_cert} + + def restore(self, snapshot): + """Restore the event from a snapshot.""" + super().restore(snapshot) + self.old_mtls_cert = snapshot["old_mtls_cert"] + + +class TopicRequestedEvent(KafkaProvidesEvent): + """Event emitted when a new topic is requested for use on this relation.""" + + @property + def extra_user_roles(self) -> Optional[str]: + """Returns the extra user roles that were requested.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("extra-user-roles") + + +class TopicEntityRequestedEvent(KafkaProvidesEvent, EntityProvidesEvent): + """Event emitted when a new entity is requested for use on this relation.""" + + +class TopicEntityPermissionsChangedEvent(KafkaProvidesEvent, EntityProvidesEvent): + """Event emitted when existing entity permissions are changed on this relation.""" + + +class KafkaProvidesEvents(CharmEvents): + """Kafka events. This class defines the events that the Kafka can emit. """ topic_requested = EventSource(TopicRequestedEvent) + topic_entity_requested = EventSource(TopicEntityRequestedEvent) + topic_entity_permissions_changed = EventSource(TopicEntityPermissionsChangedEvent) + mtls_cert_updated = EventSource(KafkaClientMtlsCertUpdatedEvent) class KafkaRequiresEvent(RelationEvent): @@ -3270,17 +4271,22 @@ class TopicCreatedEvent(AuthenticationEvent, KafkaRequiresEvent): """Event emitted when a new topic is created for use on this relation.""" +class TopicEntityCreatedEvent(EntityRequiresEvent, KafkaRequiresEvent): + """Event emitted when a new entity is created for use on this relation.""" + + class BootstrapServerChangedEvent(AuthenticationEvent, KafkaRequiresEvent): """Event emitted when the bootstrap server is changed.""" -class KafkaRequiresEvents(CharmEvents): +class KafkaRequiresEvents(RequirerCharmEvents): """Kafka events. This class defines the events that the Kafka can emit. """ topic_created = EventSource(TopicCreatedEvent) + topic_entity_created = EventSource(TopicEntityCreatedEvent) bootstrap_server_changed = EventSource(BootstrapServerChangedEvent) @@ -3290,8 +4296,12 @@ class KafkaRequiresEvents(CharmEvents): class KafkaProviderData(ProviderData): """Provider-side of the Kafka relation.""" - def __init__(self, model: Model, relation_name: str) -> None: - super().__init__(model, relation_name) + RESOURCE_FIELD = "topic" + + def __init__( + self, model: Model, relation_name: str, status_schema_path: OptionalPathLike = None + ) -> None: + super().__init__(model, relation_name, status_schema_path=status_schema_path) def set_topic(self, relation_id: int, topic: str) -> None: """Set topic name in the application relation databag. @@ -3330,7 +4340,7 @@ def set_zookeeper_uris(self, relation_id: int, zookeeper_uris: str) -> None: self.update_relation_data(relation_id, {"zookeeper-uris": zookeeper_uris}) -class KafkaProviderEventHandlers(EventHandlers): +class KafkaProviderEventHandlers(ProviderEventHandlers): """Provider-side of the Kafka relation.""" on = KafkaProvidesEvents() # pyright: ignore [reportAssignmentType] @@ -3342,6 +4352,14 @@ def __init__(self, charm: CharmBase, relation_data: KafkaProviderData) -> None: def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: """Event emitted when the relation has changed.""" + super()._on_relation_changed_event(event) + + new_data_keys = list(event.relation.data[event.app].keys()) + if any(newval for newval in new_data_keys if self.relation_data._is_secret_field(newval)): + self.relation_data._register_secrets_to_relation(event.relation, new_data_keys) + + getattr(self.on, "mtls_cert_updated").emit(event.relation, app=event.app, unit=event.unit) + # Leader only if not self.relation_data.local_unit.is_leader(): return @@ -3349,19 +4367,88 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: # Check which data has changed to emit customs events. diff = self._diff(event) - # Emit a topic requested event if the setup key (topic name and optional - # extra user roles) was added to the relation databag by the application. - if "topic" in diff.added: + # Validate entity information is not dynamically changed + self._validate_entity_consistency(event, diff) + + # Emit a topic requested event if the setup key (topic name) + # was added to the relation databag, but the entity-type key was not. + if "topic" in diff.added and "entity-type" not in diff.added: getattr(self.on, "topic_requested").emit( event.relation, app=event.app, unit=event.unit ) + # To avoid unnecessary application restarts do not trigger other events. + return + + # Emit an entity requested event if the setup key (topic name) + # was added to the relation databag, in addition to the entity-type key. + if "topic" in diff.added and "entity-type" in diff.added: + getattr(self.on, "topic_entity_requested").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + + # Emit a permissions changed event if the setup key (topic name) + # was added to the relation databag, and the entity-permissions key changed. + if ( + "topic" not in diff.added + and "entity-type" not in diff.added + and ("entity-permissions" in diff.added or "entity-permissions" in diff.changed) + ): + getattr(self.on, "topic_entity_permissions_changed").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + + def _on_secret_changed_event(self, event: SecretChangedEvent): + """Event notifying about a new value of a secret.""" + if not event.secret.label: + return + + relation = self.relation_data._relation_from_secret_label(event.secret.label) + if not relation: + logging.info( + f"Received secret {event.secret.label} but couldn't parse, seems irrelevant" + ) + return + + if relation.app == self.charm.app: + logging.info("Secret changed event ignored for Secret Owner") + + if relation.name != self.relation_data.relation_name: + logger.debug( + "Ignoring secret-changed from endpoint %s (expected %s)", + relation.name, + self.relation_data.relation_name, + ) + return + + remote_unit = None + for unit in relation.units: + if unit.app != self.charm.app: + remote_unit = unit + + old_mtls_cert = event.secret.get_content().get("mtls-cert") + # mtls-cert is the only secret that can be updated + logger.info("mtls-cert updated") + getattr(self.on, "mtls_cert_updated").emit( + relation, app=relation.app, unit=remote_unit, old_mtls_cert=old_mtls_cert + ) + class KafkaProvides(KafkaProviderData, KafkaProviderEventHandlers): """Provider-side of the Kafka relation.""" - def __init__(self, charm: CharmBase, relation_name: str) -> None: - KafkaProviderData.__init__(self, charm.model, relation_name) + def __init__( + self, charm: CharmBase, relation_name: str, status_schema_path: OptionalPathLike = None + ) -> None: + KafkaProviderData.__init__( + self, charm.model, relation_name, status_schema_path=status_schema_path + ) KafkaProviderEventHandlers.__init__(self, charm, self) @@ -3376,11 +4463,29 @@ def __init__( extra_user_roles: Optional[str] = None, consumer_group_prefix: Optional[str] = None, additional_secret_fields: Optional[List[str]] = [], + mtls_cert: Optional[str] = None, + extra_group_roles: Optional[str] = None, + entity_type: Optional[str] = None, + entity_permissions: Optional[str] = None, ): """Manager of Kafka client relations.""" - super().__init__(model, relation_name, extra_user_roles, additional_secret_fields) + super().__init__( + model, + relation_name, + extra_user_roles, + additional_secret_fields, + extra_group_roles, + entity_type, + entity_permissions, + ) self.topic = topic self.consumer_group_prefix = consumer_group_prefix or "" + self.mtls_cert = mtls_cert + + @staticmethod + def is_topic_value_acceptable(topic_value: str) -> bool: + """Check whether the given Kafka topic value is acceptable.""" + return "*" not in topic_value[:3] @property def topic(self): @@ -3389,11 +4494,19 @@ def topic(self): @topic.setter def topic(self, value): - # Avoid wildcards - if value == "*": - raise ValueError(f"Error on topic '{value}', cannot be a wildcard.") + if not self.is_topic_value_acceptable(value): + raise ValueError(f"Error on topic '{value}', unacceptable value.") self._topic = value + def set_mtls_cert(self, relation_id: int, mtls_cert: str) -> None: + """Set the mtls cert in the application relation databag / secret. + + Args: + relation_id: the identifier for a particular relation. + mtls_cert: mtls cert. + """ + self.update_relation_data(relation_id, {"mtls-cert": mtls_cert}) + class KafkaRequirerEventHandlers(RequirerEventHandlers): """Requires-side of the Kafka relation.""" @@ -3415,12 +4528,21 @@ def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: # Sets topic, extra user roles, and "consumer-group-prefix" in the relation relation_data = {"topic": self.relation_data.topic} - if self.relation_data.extra_user_roles: - relation_data["extra-user-roles"] = self.relation_data.extra_user_roles + if self.relation_data.mtls_cert: + relation_data["mtls-cert"] = self.relation_data.mtls_cert if self.relation_data.consumer_group_prefix: relation_data["consumer-group-prefix"] = self.relation_data.consumer_group_prefix + if self.relation_data.extra_user_roles: + relation_data["extra-user-roles"] = self.relation_data.extra_user_roles + if self.relation_data.extra_group_roles: + relation_data["extra-group-roles"] = self.relation_data.extra_group_roles + if self.relation_data.entity_type: + relation_data["entity-type"] = self.relation_data.entity_type + if self.relation_data.entity_permissions: + relation_data["entity-permissions"] = self.relation_data.entity_permissions + self.relation_data.update_relation_data(event.relation.id, relation_data) def _on_secret_changed_event(self, event: SecretChangedEvent): @@ -3429,311 +4551,1421 @@ def _on_secret_changed_event(self, event: SecretChangedEvent): def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: """Event emitted when the Kafka relation has changed.""" + super()._on_relation_changed_event(event) + + # Check which data has changed to emit customs events. + diff = self._diff(event) + + # send request again if encryption secret was added from provider side + if "encryption-secret" in diff.added and self.relation_data.local_unit.is_leader(): + relation_data = { + "topic": self.relation_data.topic, + "encryption-secret": event.relation.data[event.relation.app].get( + "encryption-secret" + ), + } + + if self.relation_data.mtls_cert: + relation_data["mtls-cert"] = self.relation_data.mtls_cert + + if self.relation_data.consumer_group_prefix: + relation_data["consumer-group-prefix"] = self.relation_data.consumer_group_prefix + + if self.relation_data.extra_user_roles: + relation_data["extra-user-roles"] = self.relation_data.extra_user_roles + if self.relation_data.extra_group_roles: + relation_data["extra-group-roles"] = self.relation_data.extra_group_roles + if self.relation_data.entity_type: + relation_data["entity-type"] = self.relation_data.entity_type + if self.relation_data.entity_permissions: + relation_data["entity-permissions"] = self.relation_data.entity_permissions + + self.relation_data.update_relation_data(event.relation.id, relation_data) + return + + # Check if the topic is created + # (the Kafka charm shared the credentials). + + # Register all new secrets with their labels + if any(newval for newval in diff.added if self.relation_data._is_secret_field(newval)): + self.relation_data._register_secrets_to_relation(event.relation, diff.added) + + app_databag = get_encoded_dict(event.relation, event.app, "data") + if app_databag is None: + app_databag = {} + + if self._main_credentials_shared(diff) and "entity-type" not in app_databag: + # Emit the default event (the one without an alias). + logger.info("topic created at %s", datetime.now()) + getattr(self.on, "topic_created").emit(event.relation, app=event.app, unit=event.unit) + + # To avoid unnecessary application restarts do not trigger other events. + return + + if self._entity_credentials_shared(diff) and "entity-type" in app_databag: + # Emit the default event (the one without an alias). + logger.info("entity created at %s", datetime.now()) + getattr(self.on, "topic_entity_created").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + + # Emit an endpoints (bootstrap-server) changed event if the Kafka endpoints + # added or changed this info in the relation databag. + if "endpoints" in diff.added or "endpoints" in diff.changed: + # Emit the default event (the one without an alias). + logger.info("endpoints changed on %s", datetime.now()) + getattr(self.on, "bootstrap_server_changed").emit( + event.relation, app=event.app, unit=event.unit + ) # here check if this is the right design + + # To avoid unnecessary application restarts do not trigger other events. + return + + +class KafkaRequires(KafkaRequirerData, KafkaRequirerEventHandlers): + """Provider-side of the Kafka relation.""" + + def __init__( + self, + charm: CharmBase, + relation_name: str, + topic: str, + extra_user_roles: Optional[str] = None, + consumer_group_prefix: Optional[str] = None, + additional_secret_fields: Optional[List[str]] = [], + mtls_cert: Optional[str] = None, + extra_group_roles: Optional[str] = None, + entity_type: Optional[str] = None, + entity_permissions: Optional[str] = None, + ) -> None: + KafkaRequirerData.__init__( + self, + charm.model, + relation_name, + topic, + extra_user_roles=extra_user_roles, + consumer_group_prefix=consumer_group_prefix, + additional_secret_fields=additional_secret_fields, + mtls_cert=mtls_cert, + extra_group_roles=extra_group_roles, + entity_type=entity_type, + entity_permissions=entity_permissions, + ) + KafkaRequirerEventHandlers.__init__(self, charm, self) + + +# Karapace related events + + +class KarapaceProvidesEvent(RelationEvent): + """Base class for Karapace events.""" + + @property + def subject(self) -> Optional[str]: + """Returns the subject that was requested.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("subject") + + +class SubjectRequestedEvent(KarapaceProvidesEvent): + """Event emitted when a new subject is requested for use on this relation.""" + + @property + def extra_user_roles(self) -> Optional[str]: + """Returns the extra user roles that were requested.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("extra-user-roles") + + +class SubjectEntityRequestedEvent(KarapaceProvidesEvent, EntityProvidesEvent): + """Event emitted when a new entity is requested for use on this relation.""" + + +class SubjectEntityPermissionsChangedEvent(KarapaceProvidesEvent, EntityProvidesEvent): + """Event emitted when existing entity permissions are changed on this relation.""" + + +class KarapaceProvidesEvents(CharmEvents): + """Karapace events. + + This class defines the events that the Karapace can emit. + """ + + subject_requested = EventSource(SubjectRequestedEvent) + subject_entity_requested = EventSource(SubjectEntityRequestedEvent) + subject_entity_permissions_changed = EventSource(SubjectEntityPermissionsChangedEvent) + + +class KarapaceRequiresEvent(RelationEvent): + """Base class for Karapace events.""" + + @property + def subject(self) -> Optional[str]: + """Returns the subject.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("subject") + + @property + def endpoints(self) -> Optional[str]: + """Returns a comma-separated list of broker uris.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("endpoints") + + +class SubjectAllowedEvent(AuthenticationEvent, KarapaceRequiresEvent): + """Event emitted when a new subject ACL is created for use on this relation.""" + + +class SubjectEntityCreatedEvent(EntityRequiresEvent, KarapaceRequiresEvent): + """Event emitted when a new entity is created for use on this relation.""" + + +class EndpointsChangedEvent(AuthenticationEvent, KarapaceRequiresEvent): + """Event emitted when the endpoints are changed.""" + + +class KarapaceRequiresEvents(RequirerCharmEvents): + """Karapace events. + + This class defines the events that Karapace can emit. + """ + + subject_allowed = EventSource(SubjectAllowedEvent) + subject_entity_created = EventSource(SubjectEntityCreatedEvent) + server_changed = EventSource(EndpointsChangedEvent) + + +# Karapace Provides and Requires + + +class KarapaceProviderData(ProviderData): + """Provider-side of the Karapace relation.""" + + RESOURCE_FIELD = "subject" + + def __init__( + self, model: Model, relation_name: str, status_schema_path: OptionalPathLike = None + ) -> None: + super().__init__(model, relation_name, status_schema_path=status_schema_path) + + def set_subject(self, relation_id: int, subject: str) -> None: + """Set subject name in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + subject: the subject name. + """ + self.update_relation_data(relation_id, {"subject": subject}) + + def set_endpoint(self, relation_id: int, endpoint: str) -> None: + """Set the endpoint in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + endpoint: the server address. + """ + self.update_relation_data(relation_id, {"endpoints": endpoint}) + + +class KarapaceProviderEventHandlers(ProviderEventHandlers): + """Provider-side of the Karapace relation.""" + + on = KarapaceProvidesEvents() # pyright: ignore [reportAssignmentType] + + def __init__(self, charm: CharmBase, relation_data: KarapaceProviderData) -> None: + super().__init__(charm, relation_data) + # Just to keep lint quiet, can't resolve inheritance. The same happened in super().__init__() above + self.relation_data = relation_data + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation has changed.""" + super()._on_relation_changed_event(event) + + # Leader only + if not self.relation_data.local_unit.is_leader(): + return + + # Check which data has changed to emit customs events. + diff = self._diff(event) + + # Validate entity information is not dynamically changed + self._validate_entity_consistency(event, diff) + + # Emit a subject requested event if the setup key (subject name) + # was added to the relation databag, but the entity-type key was not. + if "subject" in diff.added and "entity-type" not in diff.added: + getattr(self.on, "subject_requested").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + + # Emit an entity requested event if the setup key (subject name) + # was added to the relation databag, in addition to the entity-type key. + if "subject" in diff.added and "entity-type" in diff.added: + getattr(self.on, "subject_entity_requested").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + + # Emit a permissions changed event if the setup key (subject name) + # was added to the relation databag, and the entity-permissions key changed. + if ( + "subject" not in diff.added + and "entity-type" not in diff.added + and ("entity-permissions" in diff.added or "entity-permissions" in diff.changed) + ): + getattr(self.on, "subject_entity_permissions_changed").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + + def _on_secret_changed_event(self, event: SecretChangedEvent): + """Event notifying about a new value of a secret.""" + pass + + +class KarapaceProvides(KarapaceProviderData, KarapaceProviderEventHandlers): + """Provider-side of the Karapace relation.""" + + def __init__( + self, charm: CharmBase, relation_name: str, status_schema_path: OptionalPathLike = None + ) -> None: + KarapaceProviderData.__init__( + self, charm.model, relation_name, status_schema_path=status_schema_path + ) + KarapaceProviderEventHandlers.__init__(self, charm, self) + + +class KarapaceRequirerData(RequirerData): + """Requirer-side of the Karapace relation.""" + + def __init__( + self, + model: Model, + relation_name: str, + subject: str, + extra_user_roles: Optional[str] = None, + additional_secret_fields: Optional[List[str]] = [], + extra_group_roles: Optional[str] = None, + entity_type: Optional[str] = None, + entity_permissions: Optional[str] = None, + ): + """Manager of Karapace client relations.""" + super().__init__( + model, + relation_name, + extra_user_roles, + additional_secret_fields, + extra_group_roles, + entity_type, + entity_permissions, + ) + self.subject = subject + + @property + def subject(self): + """Topic to use in Karapace.""" + return self._subject + + @subject.setter + def subject(self, value): + # Avoid wildcards + if value == "*": + raise ValueError(f"Error on subject '{value}', cannot be a wildcard.") + self._subject = value + + +class KarapaceRequirerEventHandlers(RequirerEventHandlers): + """Requires-side of the Karapace relation.""" + + on = KarapaceRequiresEvents() # pyright: ignore [reportAssignmentType] + + def __init__(self, charm: CharmBase, relation_data: KarapaceRequirerData) -> None: + super().__init__(charm, relation_data) + # Just to keep lint quiet, can't resolve inheritance. The same happened in super().__init__() above + self.relation_data = relation_data + + def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: + """Event emitted when the Karapace relation is created.""" + super()._on_relation_created_event(event) + + if not self.relation_data.local_unit.is_leader(): + return + + # Sets subject and extra user roles + relation_data = {"subject": self.relation_data.subject} + + if self.relation_data.extra_user_roles: + relation_data["extra-user-roles"] = self.relation_data.extra_user_roles + if self.relation_data.extra_group_roles: + relation_data["extra-group-roles"] = self.relation_data.extra_group_roles + if self.relation_data.entity_type: + relation_data["entity-type"] = self.relation_data.entity_type + if self.relation_data.entity_permissions: + relation_data["entity-permissions"] = self.relation_data.entity_permissions + + self.relation_data.update_relation_data(event.relation.id, relation_data) + + def _on_secret_changed_event(self, event: SecretChangedEvent): + """Event notifying about a new value of a secret.""" + pass + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the Karapace relation has changed.""" + super()._on_relation_changed_event(event) + + # Check which data has changed to emit customs events. + diff = self._diff(event) + + # Check if the subject ACLs are created + # (the Karapace charm shared the credentials). + + # Register all new secrets with their labels + if any(newval for newval in diff.added if self.relation_data._is_secret_field(newval)): + self.relation_data._register_secrets_to_relation(event.relation, diff.added) + + app_databag = get_encoded_dict(event.relation, event.app, "data") + if app_databag is None: + app_databag = {} + + if self._main_credentials_shared(diff) and "entity-type" not in app_databag: + # Emit the default event (the one without an alias). + logger.info("subject ACL created at %s", datetime.now()) + getattr(self.on, "subject_allowed").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + + if self._entity_credentials_shared(diff) and "entity-type" in app_databag: + # Emit the default event (the one without an alias). + logger.info("entity created at %s", datetime.now()) + getattr(self.on, "subject_entity_created").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + + # Emit an endpoints changed event if the Karapace endpoints added or changed + # this info in the relation databag. + if "endpoints" in diff.added or "endpoints" in diff.changed: + # Emit the default event (the one without an alias). + logger.info("endpoints changed on %s", datetime.now()) + getattr(self.on, "server_changed").emit( + event.relation, app=event.app, unit=event.unit + ) # here check if this is the right design + + # To avoid unnecessary application restarts do not trigger other events. + return + + +class KarapaceRequires(KarapaceRequirerData, KarapaceRequirerEventHandlers): + """Provider-side of the Karapace relation.""" + + def __init__( + self, + charm: CharmBase, + relation_name: str, + subject: str, + extra_user_roles: Optional[str] = None, + additional_secret_fields: Optional[List[str]] = [], + extra_group_roles: Optional[str] = None, + entity_type: Optional[str] = None, + entity_permissions: Optional[str] = None, + ) -> None: + KarapaceRequirerData.__init__( + self, + charm.model, + relation_name, + subject, + extra_user_roles, + additional_secret_fields, + extra_group_roles, + entity_type, + entity_permissions, + ) + KarapaceRequirerEventHandlers.__init__(self, charm, self) + + +# Kafka Connect Events + + +class KafkaConnectProvidesEvent(RelationEvent): + """Base class for Kafka Connect Provider events.""" + + @property + def plugin_url(self) -> Optional[str]: + """Returns the REST endpoint URL which serves the connector plugin.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("plugin-url") + + +class IntegrationRequestedEvent(KafkaConnectProvidesEvent): + """Event emitted when a new integrator boots up and is ready to serve the connector plugin.""" + + +class KafkaConnectProvidesEvents(CharmEvents): + """Kafka Connect Provider Events.""" + + integration_requested = EventSource(IntegrationRequestedEvent) + + +class KafkaConnectRequiresEvent(AuthenticationEvent): + """Base class for Kafka Connect Requirer events.""" + + @property + def plugin_url(self) -> Optional[str]: + """Returns the REST endpoint URL which serves the connector plugin.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("plugin-url") + + +class IntegrationCreatedEvent(KafkaConnectRequiresEvent): + """Event emitted when the credentials are created for this integrator.""" + + +class IntegrationEndpointsChangedEvent(KafkaConnectRequiresEvent): + """Event emitted when Kafka Connect REST endpoints change.""" + + +class KafkaConnectRequiresEvents(RequirerCharmEvents): + """Kafka Connect Requirer Events.""" + + integration_created = EventSource(IntegrationCreatedEvent) + integration_endpoints_changed = EventSource(IntegrationEndpointsChangedEvent) + + +class KafkaConnectProviderData(ProviderData): + """Provider-side of the Kafka Connect relation.""" + + RESOURCE_FIELD = "plugin-url" + + def __init__( + self, model: Model, relation_name: str, status_schema_path: OptionalPathLike = None + ) -> None: + super().__init__(model, relation_name, status_schema_path=status_schema_path) + + def set_endpoints(self, relation_id: int, endpoints: str) -> None: + """Sets REST endpoints of the Kafka Connect service.""" + self.update_relation_data(relation_id, {"endpoints": endpoints}) + + +class KafkaConnectProviderEventHandlers(EventHandlers): + """Provider-side implementation of the Kafka Connect event handlers.""" + + on = KafkaConnectProvidesEvents() # pyright: ignore [reportAssignmentType] + + def __init__(self, charm: CharmBase, relation_data: KafkaConnectProviderData) -> None: + super().__init__(charm, relation_data) + self.relation_data = relation_data + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation has changed.""" + # Leader only + if not self.relation_data.local_unit.is_leader(): + return + + # Check which data has changed to emit customs events. + diff = self._diff(event) + + if "plugin-url" in diff.added: + getattr(self.on, "integration_requested").emit( + event.relation, app=event.app, unit=event.unit + ) + + def _on_secret_changed_event(self, event: SecretChangedEvent): + """Event notifying about a new value of a secret.""" + pass + + +class KafkaConnectProvides(KafkaConnectProviderData, KafkaConnectProviderEventHandlers): + """Provider-side implementation of the Kafka Connect relation.""" + + def __init__( + self, charm: CharmBase, relation_name: str, status_schema_path: OptionalPathLike = None + ) -> None: + KafkaConnectProviderData.__init__( + self, charm.model, relation_name, status_schema_path=status_schema_path + ) + KafkaConnectProviderEventHandlers.__init__(self, charm, self) + + +# Sentinel value passed from Kafka Connect requirer side when it does not need to serve any plugins. +PLUGIN_URL_NOT_REQUIRED: Final[str] = "NOT-REQUIRED" + + +class KafkaConnectRequirerData(RequirerData): + """Requirer-side of the Kafka Connect relation.""" + + def __init__( + self, + model: Model, + relation_name: str, + plugin_url: str, + extra_user_roles: Optional[str] = None, + additional_secret_fields: Optional[List[str]] = [], + ): + """Manager of Kafka client relations.""" + super().__init__( + model, + relation_name, + extra_user_roles=extra_user_roles, + additional_secret_fields=additional_secret_fields, + ) + self.plugin_url = plugin_url + + @property + def plugin_url(self): + """The REST endpoint URL which serves the connector plugin.""" + return self._plugin_url + + @plugin_url.setter + def plugin_url(self, value): + self._plugin_url = value + + +class KafkaConnectRequirerEventHandlers(RequirerEventHandlers): + """Requirer-side of the Kafka Connect relation.""" + + on = KafkaConnectRequiresEvents() # pyright: ignore [reportAssignmentType] + + def __init__(self, charm: CharmBase, relation_data: KafkaConnectRequirerData) -> None: + super().__init__(charm, relation_data) + self.relation_data = relation_data + + def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: + """Event emitted when the Kafka Connect relation is created.""" + super()._on_relation_created_event(event) + + if not self.relation_data.local_unit.is_leader(): + return + + relation_data = {"plugin-url": self.relation_data.plugin_url} + self.relation_data.update_relation_data(event.relation.id, relation_data) + + def _on_secret_changed_event(self, event: SecretChangedEvent): + """Event notifying about a new value of a secret.""" + pass + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the Kafka Connect relation has changed.""" + super()._on_relation_changed_event(event) + + # Check which data has changed to emit customs events. + diff = self._diff(event) + + # Register all new secrets with their labels + if any(newval for newval in diff.added if self.relation_data._is_secret_field(newval)): + self.relation_data._register_secrets_to_relation(event.relation, diff.added) + + if self._main_credentials_shared(diff): + logger.info("integration created at %s", datetime.now()) + getattr(self.on, "integration_created").emit( + event.relation, app=event.app, unit=event.unit + ) + return + + # Emit an endpoints changed event if the provider added or + # changed this info in the relation databag. + if "endpoints" in diff.added or "endpoints" in diff.changed: + # Emit the default event (the one without an alias). + logger.info("endpoints changed on %s", datetime.now()) + getattr(self.on, "integration_endpoints_changed").emit( + event.relation, app=event.app, unit=event.unit + ) + return + + +class KafkaConnectRequires(KafkaConnectRequirerData, KafkaConnectRequirerEventHandlers): + """Requirer-side implementation of the Kafka Connect relation.""" + + def __init__( + self, + charm: CharmBase, + relation_name: str, + plugin_url: str, + extra_user_roles: Optional[str] = None, + additional_secret_fields: Optional[List[str]] = [], + ) -> None: + KafkaConnectRequirerData.__init__( + self, + charm.model, + relation_name, + plugin_url, + extra_user_roles=extra_user_roles, + additional_secret_fields=additional_secret_fields, + ) + KafkaConnectRequirerEventHandlers.__init__(self, charm, self) + + +# Opensearch related events + + +class OpenSearchProvidesEvent(RelationEvent): + """Base class for OpenSearch events.""" + + @property + def index(self) -> Optional[str]: + """Returns the index that was requested.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("index") + + +class IndexRequestedEvent(OpenSearchProvidesEvent): + """Event emitted when a new index is requested for use on this relation.""" + + @property + def extra_user_roles(self) -> Optional[str]: + """Returns the extra user roles that were requested.""" + if not self.relation.app: + return None + + return self.relation.data[self.relation.app].get("extra-user-roles") + + +class IndexEntityRequestedEvent(OpenSearchProvidesEvent, EntityProvidesEvent): + """Event emitted when a new entity is requested for use on this relation.""" + + +class IndexEntityPermissionsChangedEvent(OpenSearchProvidesEvent, EntityProvidesEvent): + """Event emitted when existing entity permissions are changed on this relation.""" + + +class OpenSearchProvidesEvents(CharmEvents): + """OpenSearch events. + + This class defines the events that OpenSearch can emit. + """ + + index_requested = EventSource(IndexRequestedEvent) + index_entity_requested = EventSource(IndexEntityRequestedEvent) + index_entity_permissions_changed = EventSource(IndexEntityPermissionsChangedEvent) + + +class OpenSearchRequiresEvent(DatabaseRequiresEvent): + """Base class for OpenSearch requirer events.""" + + +class IndexCreatedEvent(AuthenticationEvent, OpenSearchRequiresEvent): + """Event emitted when a new index is created for use on this relation.""" + + +class IndexEntityCreatedEvent(EntityRequiresEvent, OpenSearchRequiresEvent): + """Event emitted when a new index is created for use on this relation.""" + + +class OpenSearchRequiresEvents(RequirerCharmEvents): + """OpenSearch events. + + This class defines the events that the opensearch requirer can emit. + """ + + index_created = EventSource(IndexCreatedEvent) + index_entity_created = EventSource(IndexEntityCreatedEvent) + endpoints_changed = EventSource(DatabaseEndpointsChangedEvent) + authentication_updated = EventSource(AuthenticationEvent) + + +# OpenSearch Provides and Requires Objects + + +class OpenSearchProvidesData(ProviderData): + """Provider-side of the OpenSearch relation.""" + + RESOURCE_FIELD = "index" + + def __init__( + self, model: Model, relation_name: str, status_schema_path: OptionalPathLike = None + ) -> None: + super().__init__(model, relation_name, status_schema_path=status_schema_path) + + def set_index(self, relation_id: int, index: str) -> None: + """Set the index in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + index: the index as it is _created_ on the provider charm. This needn't match the + requested index, and can be used to present a different index name if, for example, + the requested index is invalid. + """ + self.update_relation_data(relation_id, {"index": index}) + + def set_endpoints(self, relation_id: int, endpoints: str) -> None: + """Set the endpoints in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + endpoints: the endpoint addresses for opensearch nodes. + """ + self.update_relation_data(relation_id, {"endpoints": endpoints}) + + def set_version(self, relation_id: int, version: str) -> None: + """Set the opensearch version in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + version: database version. + """ + self.update_relation_data(relation_id, {"version": version}) + + +class OpenSearchProvidesEventHandlers(ProviderEventHandlers): + """Provider-side of the OpenSearch relation.""" + + on = OpenSearchProvidesEvents() # pyright: ignore[reportAssignmentType] + + def __init__(self, charm: CharmBase, relation_data: OpenSearchProvidesData) -> None: + super().__init__(charm, relation_data) + # Just to keep lint quiet, can't resolve inheritance. The same happened in super().__init__() above + self.relation_data = relation_data + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the relation has changed.""" + super()._on_relation_changed_event(event) + + # Leader only + if not self.relation_data.local_unit.is_leader(): + return + + # Check which data has changed to emit customs events. + diff = self._diff(event) + + # Validate entity information is not dynamically changed + self._validate_entity_consistency(event, diff) + + # Emit an index requested event if the setup key (index name) + # was added to the relation databag, but the entity-type key was not. + if "index" in diff.added and "entity-type" not in diff.added: + getattr(self.on, "index_requested").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + + # Emit an entity requested event if the setup key (index name) + # was added to the relation databag, in addition to the entity-type key. + if "index" in diff.added and "entity-type" in diff.added: + getattr(self.on, "index_entity_requested").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + + # Emit a permissions changed event if the setup key (index name) + # was added to the relation databag, and the entity-permissions key changed. + if ( + "index" not in diff.added + and "entity-type" not in diff.added + and ("entity-permissions" in diff.added or "entity-permissions" in diff.changed) + ): + getattr(self.on, "index_entity_permissions_changed").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + + def _on_secret_changed_event(self, event: SecretChangedEvent) -> None: + """Event emitted when the relation data has changed.""" + pass + + +class OpenSearchProvides(OpenSearchProvidesData, OpenSearchProvidesEventHandlers): + """Provider-side of the OpenSearch relation.""" + + def __init__( + self, charm: CharmBase, relation_name: str, status_schema_path: OptionalPathLike = None + ) -> None: + OpenSearchProvidesData.__init__( + self, charm.model, relation_name, status_schema_path=status_schema_path + ) + OpenSearchProvidesEventHandlers.__init__(self, charm, self) + + +class OpenSearchRequiresData(RequirerData): + """Requires data side of the OpenSearch relation.""" + + def __init__( + self, + model: Model, + relation_name: str, + index: str, + extra_user_roles: Optional[str] = None, + additional_secret_fields: Optional[List[str]] = [], + extra_group_roles: Optional[str] = None, + entity_type: Optional[str] = None, + entity_permissions: Optional[str] = None, + ): + """Manager of OpenSearch client relations.""" + super().__init__( + model, + relation_name, + extra_user_roles, + additional_secret_fields, + extra_group_roles, + entity_type, + entity_permissions, + ) + self.index = index + + +class OpenSearchRequiresEventHandlers(RequirerEventHandlers): + """Requires events side of the OpenSearch relation.""" + + on = OpenSearchRequiresEvents() # pyright: ignore[reportAssignmentType] + + def __init__(self, charm: CharmBase, relation_data: OpenSearchRequiresData) -> None: + super().__init__(charm, relation_data) + # Just to keep lint quiet, can't resolve inheritance. The same happened in super().__init__() above + self.relation_data = relation_data + + def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: + """Event emitted when the OpenSearch relation is created.""" + super()._on_relation_created_event(event) + + if not self.relation_data.local_unit.is_leader(): + return + + # Sets both index and extra user roles in the relation if the roles are provided. + # Otherwise, sets only the index. + data = {"index": self.relation_data.index} + + if self.relation_data.extra_user_roles: + data["extra-user-roles"] = self.relation_data.extra_user_roles + if self.relation_data.extra_group_roles: + data["extra-group-roles"] = self.relation_data.extra_group_roles + if self.relation_data.entity_type: + data["entity-type"] = self.relation_data.entity_type + if self.relation_data.entity_permissions: + data["entity-permissions"] = self.relation_data.entity_permissions + + self.relation_data.update_relation_data(event.relation.id, data) + + def _on_secret_changed_event(self, event: SecretChangedEvent): + """Event notifying about a new value of a secret.""" + if not event.secret.label: + return + + relation = self.relation_data._relation_from_secret_label(event.secret.label) + if not relation: + logging.info( + f"Received secret {event.secret.label} but couldn't parse, seems irrelevant" + ) + return + + if relation.name != self.relation_data.relation_name: + logger.debug( + "Ignoring secret-changed from endpoint %s (expected %s)", + relation.name, + self.relation_data.relation_name, + ) + return + + if relation.app == self.charm.app: + logging.info("Secret changed event ignored for Secret Owner") + + remote_unit = None + for unit in relation.units: + if unit.app != self.charm.app: + remote_unit = unit + + logger.info("authentication updated") + getattr(self.on, "authentication_updated").emit( + relation, app=relation.app, unit=remote_unit + ) + + def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: + """Event emitted when the OpenSearch relation has changed. + + This event triggers individual custom events depending on the changing relation. + """ + super()._on_relation_changed_event(event) + # Check which data has changed to emit customs events. diff = self._diff(event) - # Check if the topic is created - # (the Kafka charm shared the credentials). - # Register all new secrets with their labels if any(newval for newval in diff.added if self.relation_data._is_secret_field(newval)): self.relation_data._register_secrets_to_relation(event.relation, diff.added) secret_field_user = self.relation_data._generate_secret_field_name(SECRET_GROUPS.USER) - if ( - "username" in diff.added and "password" in diff.added - ) or secret_field_user in diff.added: + secret_field_tls = self.relation_data._generate_secret_field_name(SECRET_GROUPS.TLS) + updates = {"username", "password", "tls", "tls-ca", secret_field_user, secret_field_tls} + if len(set(diff._asdict().keys()) - updates) < len(diff): + logger.info("authentication updated at: %s", datetime.now()) + getattr(self.on, "authentication_updated").emit( + event.relation, app=event.app, unit=event.unit + ) + + app_databag = get_encoded_dict(event.relation, event.app, "data") + if app_databag is None: + app_databag = {} + + # Check if the index is created + # (the OpenSearch charm shares the credentials). + if self._main_credentials_shared(diff) and "entity-type" not in app_databag: # Emit the default event (the one without an alias). - logger.info("topic created at %s", datetime.now()) - getattr(self.on, "topic_created").emit(event.relation, app=event.app, unit=event.unit) + logger.info("index created at: %s", datetime.now()) + getattr(self.on, "index_created").emit(event.relation, app=event.app, unit=event.unit) - # To avoid unnecessary application restarts do not trigger - # “endpoints_changed“ event if “topic_created“ is triggered. + # To avoid unnecessary application restarts do not trigger other events. return - # Emit an endpoints (bootstrap-server) changed event if the Kafka endpoints + if self._entity_credentials_shared(diff) and "entity-type" in app_databag: + # Emit the default event (the one without an alias). + logger.info("entity created at: %s", datetime.now()) + getattr(self.on, "index_entity_created").emit( + event.relation, app=event.app, unit=event.unit + ) + + # To avoid unnecessary application restarts do not trigger other events. + return + + # Emit a endpoints changed event if the OpenSearch application # added or changed this info in the relation databag. if "endpoints" in diff.added or "endpoints" in diff.changed: # Emit the default event (the one without an alias). logger.info("endpoints changed on %s", datetime.now()) - getattr(self.on, "bootstrap_server_changed").emit( + getattr(self.on, "endpoints_changed").emit( event.relation, app=event.app, unit=event.unit - ) # here check if this is the right design + ) + + # To avoid unnecessary application restarts do not trigger other events. return -class KafkaRequires(KafkaRequirerData, KafkaRequirerEventHandlers): - """Provider-side of the Kafka relation.""" +class OpenSearchRequires(OpenSearchRequiresData, OpenSearchRequiresEventHandlers): + """Requires-side of the OpenSearch relation.""" def __init__( self, charm: CharmBase, relation_name: str, - topic: str, + index: str, extra_user_roles: Optional[str] = None, - consumer_group_prefix: Optional[str] = None, additional_secret_fields: Optional[List[str]] = [], + extra_group_roles: Optional[str] = None, + entity_type: Optional[str] = None, + entity_permissions: Optional[str] = None, ) -> None: - KafkaRequirerData.__init__( + OpenSearchRequiresData.__init__( self, charm.model, relation_name, - topic, + index, extra_user_roles, - consumer_group_prefix, additional_secret_fields, + extra_group_roles, + entity_type, + entity_permissions, ) - KafkaRequirerEventHandlers.__init__(self, charm, self) + OpenSearchRequiresEventHandlers.__init__(self, charm, self) -# Opensearch related events +# Etcd related events -class OpenSearchProvidesEvent(RelationEvent): - """Base class for OpenSearch events.""" +class EtcdProviderEvent(RelationEventWithSecret): + """Base class for Etcd events.""" @property - def index(self) -> Optional[str]: + def prefix(self) -> Optional[str]: """Returns the index that was requested.""" if not self.relation.app: return None - return self.relation.data[self.relation.app].get("index") + return self.relation.data[self.relation.app].get("prefix") + @property + def mtls_cert(self) -> Optional[str]: + """Returns TLS cert of the client.""" + if not self.relation.app: + return None -class IndexRequestedEvent(OpenSearchProvidesEvent, ExtraRoleEvent): - """Event emitted when a new index is requested for use on this relation.""" + if not self.secrets_enabled: + raise SecretsUnavailableError("Secrets unavailable on current Juju version") + secret_field = f"{PROV_SECRET_PREFIX}{SECRET_GROUPS.MTLS}" + if secret_uri := self.relation.data[self.app].get(secret_field): + secret = self.framework.model.get_secret(id=secret_uri) + content = secret.get_content(refresh=True) + if content: + return content.get("mtls-cert") -class OpenSearchProvidesEvents(CharmEvents): - """OpenSearch events. - This class defines the events that OpenSearch can emit. - """ +class MTLSCertUpdatedEvent(EtcdProviderEvent): + """Event emitted when the mtls relation is updated.""" - index_requested = EventSource(IndexRequestedEvent) + def __init__(self, handle, relation, old_mtls_cert: Optional[str] = None, app=None, unit=None): + super().__init__(handle, relation, app, unit) + self.old_mtls_cert = old_mtls_cert -class OpenSearchRequiresEvent(DatabaseRequiresEvent): - """Base class for OpenSearch requirer events.""" + def snapshot(self): + """Return a snapshot of the event.""" + return super().snapshot() | {"old_mtls_cert": self.old_mtls_cert} + def restore(self, snapshot): + """Restore the event from a snapshot.""" + super().restore(snapshot) + self.old_mtls_cert = snapshot["old_mtls_cert"] -class IndexCreatedEvent(AuthenticationEvent, OpenSearchRequiresEvent): - """Event emitted when a new index is created for use on this relation.""" +class EtcdProviderEvents(CharmEvents): + """Etcd events. -class OpenSearchRequiresEvents(CharmEvents): - """OpenSearch events. + This class defines the events that Etcd can emit. + """ - This class defines the events that the opensearch requirer can emit. + mtls_cert_updated = EventSource(MTLSCertUpdatedEvent) + + +class EtcdReadyEvent(AuthenticationEvent, DatabaseRequiresEvent): + """Event emitted when the etcd relation is ready to be consumed.""" + + +class EtcdRequirerEvents(RequirerCharmEvents): + """Etcd events. + + This class defines the events that the etcd requirer can emit. """ - index_created = EventSource(IndexCreatedEvent) endpoints_changed = EventSource(DatabaseEndpointsChangedEvent) - authentication_updated = EventSource(AuthenticationEvent) + etcd_ready = EventSource(EtcdReadyEvent) -# OpenSearch Provides and Requires Objects +# Etcd Provides and Requires Objects -class OpenSearchProvidesData(ProviderData): - """Provider-side of the OpenSearch relation.""" +class EtcdProviderData(ProviderData): + """Provider-side of the Etcd relation.""" - def __init__(self, model: Model, relation_name: str) -> None: - super().__init__(model, relation_name) + RESOURCE_FIELD = "prefix" - def set_index(self, relation_id: int, index: str) -> None: - """Set the index in the application relation databag. + def __init__( + self, model: Model, relation_name: str, status_schema_path: OptionalPathLike = None + ) -> None: + super().__init__(model, relation_name, status_schema_path=status_schema_path) + + def set_uris(self, relation_id: int, uris: str) -> None: + """Set the database connection URIs in the application relation databag. Args: relation_id: the identifier for a particular relation. - index: the index as it is _created_ on the provider charm. This needn't match the - requested index, and can be used to present a different index name if, for example, - the requested index is invalid. + uris: connection URIs. """ - self.update_relation_data(relation_id, {"index": index}) + self.update_relation_data(relation_id, {"uris": uris}) def set_endpoints(self, relation_id: int, endpoints: str) -> None: """Set the endpoints in the application relation databag. Args: relation_id: the identifier for a particular relation. - endpoints: the endpoint addresses for opensearch nodes. + endpoints: the endpoint addresses for etcd nodes "ip:port" format. """ self.update_relation_data(relation_id, {"endpoints": endpoints}) def set_version(self, relation_id: int, version: str) -> None: - """Set the opensearch version in the application relation databag. + """Set the etcd version in the application relation databag. Args: relation_id: the identifier for a particular relation. - version: database version. + version: etcd API version. """ self.update_relation_data(relation_id, {"version": version}) + def set_tls_ca(self, relation_id: int, tls_ca: str) -> None: + """Set the TLS CA in the application relation databag. + + Args: + relation_id: the identifier for a particular relation. + tls_ca: TLS certification authority. + """ + self.update_relation_data(relation_id, {"tls-ca": tls_ca, "tls": "True"}) -class OpenSearchProvidesEventHandlers(EventHandlers): - """Provider-side of the OpenSearch relation.""" - on = OpenSearchProvidesEvents() # pyright: ignore[reportAssignmentType] +class EtcdProviderEventHandlers(ProviderEventHandlers): + """Provider-side of the Etcd relation.""" - def __init__(self, charm: CharmBase, relation_data: OpenSearchProvidesData) -> None: + on = EtcdProviderEvents() # pyright: ignore[reportAssignmentType] + + def __init__(self, charm: CharmBase, relation_data: EtcdProviderData) -> None: super().__init__(charm, relation_data) # Just to keep lint quiet, can't resolve inheritance. The same happened in super().__init__() above self.relation_data = relation_data def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: """Event emitted when the relation has changed.""" - # Leader only - if not self.relation_data.local_unit.is_leader(): - return + super()._on_relation_changed_event(event) + # register all new secrets with their labels + new_data_keys = list(event.relation.data[event.app].keys()) + if any(newval for newval in new_data_keys if self.relation_data._is_secret_field(newval)): + self.relation_data._register_secrets_to_relation(event.relation, new_data_keys) + # Check which data has changed to emit customs events. diff = self._diff(event) - # Emit an index requested event if the setup key (index name and optional extra user roles) - # have been added to the relation databag by the application. - if "index" in diff.added: - getattr(self.on, "index_requested").emit( - event.relation, app=event.app, unit=event.unit + # Validate entity information is not dynamically changed + self._validate_entity_consistency(event, diff) + + getattr(self.on, "mtls_cert_updated").emit(event.relation, app=event.app, unit=event.unit) + return + + def _on_secret_changed_event(self, event: SecretChangedEvent): + """Event notifying about a new value of a secret.""" + if not event.secret.label: + return + + relation = self.relation_data._relation_from_secret_label(event.secret.label) + if not relation: + logging.info( + f"Received secret {event.secret.label} but couldn't parse, seems irrelevant" ) + return + if relation.name != self.relation_data.relation_name: + logger.debug( + "Ignoring secret-changed from endpoint %s (expected %s)", + relation.name, + self.relation_data.relation_name, + ) + return -class OpenSearchProvides(OpenSearchProvidesData, OpenSearchProvidesEventHandlers): - """Provider-side of the OpenSearch relation.""" + if relation.app == self.charm.app: + logging.info("Secret changed event ignored for Secret Owner") - def __init__(self, charm: CharmBase, relation_name: str) -> None: - OpenSearchProvidesData.__init__(self, charm.model, relation_name) - OpenSearchProvidesEventHandlers.__init__(self, charm, self) + remote_unit = None + for unit in relation.units: + if unit.app != self.charm.app: + remote_unit = unit + + old_mtls_cert = event.secret.get_content().get("mtls-cert") + # mtls-cert is the only secret that can be updated + logger.info("mtls-cert updated") + getattr(self.on, "mtls_cert_updated").emit( + relation, app=relation.app, unit=remote_unit, old_mtls_cert=old_mtls_cert + ) -class OpenSearchRequiresData(RequirerData): - """Requires data side of the OpenSearch relation.""" +class EtcdProvides(EtcdProviderData, EtcdProviderEventHandlers): + """Provider-side of the Etcd relation.""" + + def __init__( + self, charm: CharmBase, relation_name: str, status_schema_path: OptionalPathLike = None + ) -> None: + EtcdProviderData.__init__( + self, charm.model, relation_name, status_schema_path=status_schema_path + ) + EtcdProviderEventHandlers.__init__(self, charm, self) + if not self.secrets_enabled: + raise SecretsUnavailableError("Secrets unavailable on current Juju version") + + +class EtcdRequirerData(RequirerData): + """Requires data side of the Etcd relation.""" def __init__( self, model: Model, relation_name: str, - index: str, + prefix: str, + mtls_cert: Optional[str], extra_user_roles: Optional[str] = None, additional_secret_fields: Optional[List[str]] = [], + extra_group_roles: Optional[str] = None, + entity_type: Optional[str] = None, + entity_permissions: Optional[str] = None, ): - """Manager of OpenSearch client relations.""" - super().__init__(model, relation_name, extra_user_roles, additional_secret_fields) - self.index = index + """Manager of Etcd client relations.""" + super().__init__( + model, + relation_name, + extra_user_roles, + additional_secret_fields, + extra_group_roles, + entity_type, + entity_permissions, + ) + self.prefix = prefix + self.mtls_cert = mtls_cert + def set_mtls_cert(self, relation_id: int, mtls_cert: str) -> None: + """Set the mtls cert in the application relation databag / secret. -class OpenSearchRequiresEventHandlers(RequirerEventHandlers): - """Requires events side of the OpenSearch relation.""" + Args: + relation_id: the identifier for a particular relation. + mtls_cert: mtls cert. + """ + self.update_relation_data(relation_id, {"mtls-cert": mtls_cert}) - on = OpenSearchRequiresEvents() # pyright: ignore[reportAssignmentType] - def __init__(self, charm: CharmBase, relation_data: OpenSearchRequiresData) -> None: +class EtcdRequirerEventHandlers(RequirerEventHandlers): + """Requires events side of the Etcd relation.""" + + on = EtcdRequirerEvents() # pyright: ignore[reportAssignmentType] + + def __init__(self, charm: CharmBase, relation_data: EtcdRequirerData) -> None: super().__init__(charm, relation_data) # Just to keep lint quiet, can't resolve inheritance. The same happened in super().__init__() above self.relation_data = relation_data def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: - """Event emitted when the OpenSearch relation is created.""" + """Event emitted when the Etcd relation is created.""" super()._on_relation_created_event(event) - if not self.relation_data.local_unit.is_leader(): - return - - # Sets both index and extra user roles in the relation if the roles are provided. - # Otherwise, sets only the index. - data = {"index": self.relation_data.index} - if self.relation_data.extra_user_roles: - data["extra-user-roles"] = self.relation_data.extra_user_roles - - self.relation_data.update_relation_data(event.relation.id, data) - - def _on_secret_changed_event(self, event: SecretChangedEvent): - """Event notifying about a new value of a secret.""" - if not event.secret.label: - return - - relation = self.relation_data._relation_from_secret_label(event.secret.label) - if not relation: - logging.info( - f"Received secret {event.secret.label} but couldn't parse, seems irrelevant" - ) - return - - if relation.app == self.charm.app: - logging.info("Secret changed event ignored for Secret Owner") - - remote_unit = None - for unit in relation.units: - if unit.app != self.charm.app: - remote_unit = unit + payload = { + "prefix": self.relation_data.prefix, + } + if self.relation_data.mtls_cert: + payload["mtls-cert"] = self.relation_data.mtls_cert - logger.info("authentication updated") - getattr(self.on, "authentication_updated").emit( - relation, app=relation.app, unit=remote_unit + self.relation_data.update_relation_data( + event.relation.id, + payload, ) def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: - """Event emitted when the OpenSearch relation has changed. + """Event emitted when the Etcd relation has changed. This event triggers individual custom events depending on the changing relation. """ + super()._on_relation_changed_event(event) + # Check which data has changed to emit customs events. diff = self._diff(event) + # send request again if encryption secret was added from provider side + if "encryption-secret" in diff.added and self.relation_data.local_unit.is_leader(): + payload = { + "prefix": self.relation_data.prefix, + "encryption-secret": event.relation.data[event.relation.app].get( + "encryption-secret" + ), + } + if self.relation_data.mtls_cert: + payload["mtls-cert"] = self.relation_data.mtls_cert + + self.relation_data.update_relation_data( + event.relation.id, + payload, + ) + return + # Register all new secrets with their labels if any(newval for newval in diff.added if self.relation_data._is_secret_field(newval)): self.relation_data._register_secrets_to_relation(event.relation, diff.added) secret_field_user = self.relation_data._generate_secret_field_name(SECRET_GROUPS.USER) secret_field_tls = self.relation_data._generate_secret_field_name(SECRET_GROUPS.TLS) - updates = {"username", "password", "tls", "tls-ca", secret_field_user, secret_field_tls} - if len(set(diff._asdict().keys()) - updates) < len(diff): - logger.info("authentication updated at: %s", datetime.now()) - getattr(self.on, "authentication_updated").emit( + + # Emit a endpoints changed event if the etcd application added or changed this info + # in the relation databag. + if "endpoints" in diff.added or "endpoints" in diff.changed: + # Emit the default event (the one without an alias). + logger.info("endpoints changed on %s", datetime.now()) + getattr(self.on, "endpoints_changed").emit( event.relation, app=event.app, unit=event.unit ) - # Check if the index is created - # (the OpenSearch charm shares the credentials). if ( - "username" in diff.added and "password" in diff.added - ) or secret_field_user in diff.added: + secret_field_tls in diff.added + or secret_field_tls in diff.changed + or secret_field_user in diff.added + or secret_field_user in diff.changed + or "username" in diff.added + or "username" in diff.changed + ): # Emit the default event (the one without an alias). - logger.info("index created at: %s", datetime.now()) - getattr(self.on, "index_created").emit(event.relation, app=event.app, unit=event.unit) + logger.info("etcd ready on %s", datetime.now()) + getattr(self.on, "etcd_ready").emit(event.relation, app=event.app, unit=event.unit) + + def _on_secret_changed_event(self, event: SecretChangedEvent): + """Event notifying about a new value of a secret.""" + if not event.secret.label: + return - # To avoid unnecessary application restarts do not trigger - # “endpoints_changed“ event if “index_created“ is triggered. + relation = self.relation_data._relation_from_secret_label(event.secret.label) + if not relation: + logging.info( + f"Received secret {event.secret.label} but couldn't parse, seems irrelevant" + ) return - # Emit a endpoints changed event if the OpenSearch application added or changed this info - # in the relation databag. - if "endpoints" in diff.added or "endpoints" in diff.changed: - # Emit the default event (the one without an alias). - logger.info("endpoints changed on %s", datetime.now()) - getattr(self.on, "endpoints_changed").emit( - event.relation, app=event.app, unit=event.unit - ) # here check if this is the right design + if relation.app == self.charm.app: + logging.info("Secret changed event ignored for Secret Owner") + + if relation.name != self.relation_data.relation_name: + logger.debug( + "Ignoring secret-changed from endpoint %s (expected %s)", + relation.name, + self.relation_data.relation_name, + ) return + remote_unit = None + for unit in relation.units: + if unit.app != self.charm.app: + remote_unit = unit -class OpenSearchRequires(OpenSearchRequiresData, OpenSearchRequiresEventHandlers): - """Requires-side of the OpenSearch relation.""" + # secret-user or secret-tls updated + logger.info("etcd_ready updated") + getattr(self.on, "etcd_ready").emit(relation, app=relation.app, unit=remote_unit) + + +class EtcdRequires(EtcdRequirerData, EtcdRequirerEventHandlers): + """Requires-side of the Etcd relation.""" def __init__( self, charm: CharmBase, relation_name: str, - index: str, + prefix: str, + mtls_cert: Optional[str], extra_user_roles: Optional[str] = None, additional_secret_fields: Optional[List[str]] = [], + extra_group_roles: Optional[str] = None, + entity_type: Optional[str] = None, + entity_permissions: Optional[str] = None, ) -> None: - OpenSearchRequiresData.__init__( + EtcdRequirerData.__init__( self, charm.model, relation_name, - index, + prefix, + mtls_cert, extra_user_roles, additional_secret_fields, + extra_group_roles, + entity_type, + entity_permissions, ) - OpenSearchRequiresEventHandlers.__init__(self, charm, self) + EtcdRequirerEventHandlers.__init__(self, charm, self) + if not self.secrets_enabled: + raise SecretsUnavailableError("Secrets unavailable on current Juju version") diff --git a/lib/charms/data_platform_libs/v0/s3.py b/lib/charms/data_platform_libs/v0/s3.py index f5614aaf6..dbf4d5bb7 100644 --- a/lib/charms/data_platform_libs/v0/s3.py +++ b/lib/charms/data_platform_libs/v0/s3.py @@ -110,6 +110,7 @@ def _on_credential_gone(self, event: CredentialsGoneEvent): ``` """ + import json import logging from collections import namedtuple @@ -137,7 +138,7 @@ def _on_credential_gone(self, event: CredentialsGoneEvent): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 5 +LIBPATCH = 6 logger = logging.getLogger(__name__) @@ -354,7 +355,7 @@ def update_connection_info(self, relation_id: int, connection_data: dict) -> Non updated_connection_data[configuration_option] = configuration_value relation.data[self.local_app].update(updated_connection_data) - logger.debug(f"Updated S3 connection info: {updated_connection_data}") + logger.debug("Updated S3 connection info.") @property def relations(self) -> List[Relation]: @@ -721,7 +722,7 @@ def update_connection_info(self, relation_id: int, connection_data: dict) -> Non updated_connection_data[configuration_option] = configuration_value relation.data[self.local_app].update(updated_connection_data) - logger.debug(f"Updated S3 credentials: {updated_connection_data}") + logger.debug("Updated S3 credentials.") def _load_relation_data(self, raw_relation_data: RelationDataContent) -> Dict[str, str]: """Loads relation data from the relation data bag. diff --git a/lib/charms/grafana_k8s/v0/grafana_dashboard.py b/lib/charms/grafana_k8s/v0/grafana_dashboard.py index dfc32ddcb..2e7ce257f 100644 --- a/lib/charms/grafana_k8s/v0/grafana_dashboard.py +++ b/lib/charms/grafana_k8s/v0/grafana_dashboard.py @@ -157,7 +157,7 @@ def __init__(self, *args): self._on_dashboards_changed, ) -Dashboards can be retrieved the :meth:`dashboards`: +Dashboards can be retrieved via the `dashboards` method: It will be returned in the format of: @@ -175,7 +175,6 @@ def __init__(self, *args): The consuming charm should decompress the dashboard. """ -import base64 import hashlib import json import logging @@ -185,11 +184,12 @@ def __init__(self, *args): import re import subprocess import tempfile -import uuid -from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, Union +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional, Tuple import yaml +from cosl import DashboardPath40UID, LZMABase64 +from cosl.types import type_convert_stored from ops.charm import ( CharmBase, HookEvent, @@ -204,8 +204,6 @@ def __init__(self, *args): EventSource, Object, ObjectEvents, - StoredDict, - StoredList, StoredState, ) from ops.model import Relation @@ -219,7 +217,9 @@ def __init__(self, *args): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 36 +LIBPATCH = 51 + +PYDEPS = ["cosl >= 0.0.50"] logger = logging.getLogger(__name__) @@ -392,6 +392,13 @@ def __init__(self, *args): } +def _data_hash(data: Any) -> str: + """Deterministic hash of a template dict for use as a stable relation data key.""" + return hashlib.shake_128( + json.dumps(data, sort_keys=True).encode() + ).digest(8).hex() + + class RelationNotFoundError(Exception): """Raised if there is no relation with the given name.""" @@ -415,8 +422,7 @@ def __init__( self.expected_relation_interface = expected_relation_interface self.actual_relation_interface = actual_relation_interface self.message = ( - "The '{}' relation has '{}' as " - "interface rather than the expected '{}'".format( + "The '{}' relation has '{}' as " "interface rather than the expected '{}'".format( relation_name, actual_relation_interface, expected_relation_interface ) ) @@ -544,369 +550,506 @@ def _validate_relation_by_interface_and_direction( raise Exception("Unexpected RelationDirection: {}".format(expected_relation_role)) -def _encode_dashboard_content(content: Union[str, bytes]) -> str: - if isinstance(content, str): - content = bytes(content, "utf-8") +class CharmedDashboard: + """A helper class for handling dashboards on the requirer (Grafana) side.""" - return base64.b64encode(lzma.compress(content)).decode("utf-8") + @classmethod + def _convert_dashboard_fields(cls, content: str, inject_dropdowns: bool = True) -> str: + """Make sure values are present for Juju topology. + Inserts Juju topology variables and selectors into the template, as well as + a variable for Prometheus. + """ + dict_content = json.loads(content) + datasources = {} + existing_templates = False + + template_dropdowns = ( + TOPOLOGY_TEMPLATE_DROPDOWNS + DATASOURCE_TEMPLATE_DROPDOWNS # type: ignore + if inject_dropdowns + else DATASOURCE_TEMPLATE_DROPDOWNS + ) -def _decode_dashboard_content(encoded_content: str) -> str: - return lzma.decompress(base64.b64decode(encoded_content.encode("utf-8"))).decode() + # If the dashboard has __inputs, get the names to replace them. These are stripped + # from reactive dashboards in GrafanaDashboardAggregator, but charm authors in + # newer charms may import them directly from the marketplace + if "__inputs" in dict_content: + for field in dict_content["__inputs"]: + if "type" in field and field["type"] == "datasource": + datasources[field["name"]] = field["pluginName"].lower() + del dict_content["__inputs"] + + # If no existing template variables exist, just insert our own + if "templating" not in dict_content: + dict_content["templating"] = {"list": list(template_dropdowns)} # type: ignore + else: + # Otherwise, set a flag so we can go back later + existing_templates = True + for template_value in dict_content["templating"]["list"]: + # Build a list of `datasource_name`: `datasource_type` mappings + # The "query" field is actually "prometheus", "loki", "influxdb", etc + if "type" in template_value and template_value["type"] == "datasource": + datasources[template_value["name"]] = template_value["query"].lower() + + # Put our own variables in the template + # We only want to inject our own dropdowns IFF they are NOT + # already in the template coming over relation data. + # We'll store all dropdowns in the template from the provider + # in a set. We'll add our own if they are not in this set. + existing_names = { + item.get("name") + for item in dict_content["templating"]["list"] + } + + for d in template_dropdowns: # type: ignore + if d.get("name") not in existing_names: + dict_content["templating"]["list"].insert(0, d) + existing_names.add(d.get("name")) + + dict_content = cls._replace_template_fields(dict_content, datasources, existing_templates) + return json.dumps(dict_content) + @classmethod + def _replace_template_fields( # noqa: C901 + cls, dict_content: dict, datasources: dict, existing_templates: bool + ) -> dict: + """Make templated fields get cleaned up afterwards. -def _convert_dashboard_fields(content: str, inject_dropdowns: bool = True) -> str: - """Make sure values are present for Juju topology. + If existing datasource variables are present, try to substitute them. + """ + replacements = {"loki": "${lokids}", "prometheus": "${prometheusds}"} + used_replacements = [] # type: List[str] + + # If any existing datasources match types we know, or we didn't find + # any templating variables at all, template them. + if datasources or not existing_templates: + panels = dict_content.get("panels", {}) + if panels: + dict_content["panels"] = cls._template_panels( + panels, replacements, used_replacements, existing_templates, datasources + ) - Inserts Juju topology variables and selectors into the template, as well as - a variable for Prometheus. - """ - dict_content = json.loads(content) - datasources = {} - existing_templates = False - - template_dropdowns = ( - TOPOLOGY_TEMPLATE_DROPDOWNS + DATASOURCE_TEMPLATE_DROPDOWNS # type: ignore - if inject_dropdowns - else DATASOURCE_TEMPLATE_DROPDOWNS - ) + # Find panels nested under rows + rows = dict_content.get("rows", {}) + if rows: + for row_idx, row in enumerate(rows): + if "panels" in row.keys(): + rows[row_idx]["panels"] = cls._template_panels( + row["panels"], + replacements, + used_replacements, + existing_templates, + datasources, + ) + + dict_content["rows"] = rows + + # Finally, go back and pop off the templates we stubbed out + deletions = [] + for tmpl in dict_content["templating"]["list"]: + if tmpl["name"] and tmpl["name"] in used_replacements: + # it might happen that existing template var name is the same as the one we insert (i.e prometheusds or lokids) + # in that case, we want to pop the existing one only. + if tmpl not in DATASOURCE_TEMPLATE_DROPDOWNS: + deletions.append(tmpl) + + for d in deletions: + dict_content["templating"]["list"].remove(d) + + return dict_content + + @classmethod + def _template_panels( + cls, + panels: dict, + replacements: dict, + used_replacements: list, + existing_templates: bool, + datasources: dict, + ) -> dict: + """Iterate through a `panels` object and template it appropriately.""" + # Go through all the panels. If they have a datasource set, AND it's one + # that we can convert to ${lokids} or ${prometheusds}, by stripping off the + # ${} templating and comparing the name to the list we built, replace it, + # otherwise, leave it alone. + # + for panel in panels: + if "datasource" not in panel or not panel.get("datasource"): + continue + if not existing_templates: + datasource = panel.get("datasource") + if isinstance(datasource, str): + if "loki" in datasource: + panel["datasource"] = "${lokids}" + elif "grafana" in datasource: + continue + else: + panel["datasource"] = "${prometheusds}" + elif isinstance(datasource, dict): + # In dashboards exported by Grafana 9, datasource type is dict + dstype = datasource.get("type", "") + if dstype == "loki": + panel["datasource"]["uid"] = "${lokids}" + elif dstype == "prometheus": + panel["datasource"]["uid"] = "${prometheusds}" + else: + logger.debug("Unrecognized datasource type '%s'; skipping", dstype) + continue + else: + logger.error("Unknown datasource format: skipping") + continue + else: + if isinstance(panel["datasource"], str): + if panel["datasource"].lower() in replacements.values(): + # Already a known template variable + continue + # Strip out variable characters and maybe braces + ds = re.sub(r"(\$|\{|\})", "", panel["datasource"]) + + if ds not in datasources.keys(): + # Unknown, non-templated datasource, potentially a Grafana builtin + continue + + replacement = replacements.get(datasources[ds], "") + if replacement: + used_replacements.append(ds) + panel["datasource"] = replacement or panel["datasource"] + elif isinstance(panel["datasource"], dict): + dstype = panel["datasource"].get("type", "") + if panel["datasource"].get("uid", "").lower() in replacements.values(): + # Already a known template variable + continue + # Strip out variable characters and maybe braces + ds = re.sub(r"(\$|\{|\})", "", panel["datasource"].get("uid", "")) + + if ds not in datasources.keys(): + # Unknown, non-templated datasource, potentially a Grafana builtin + continue + + replacement = replacements.get(datasources[ds], "") + if replacement: + used_replacements.append(ds) + panel["datasource"]["uid"] = replacement + else: + logger.error("Unknown datasource format: skipping") + continue + return panels - # If the dashboard has __inputs, get the names to replace them. These are stripped - # from reactive dashboards in GrafanaDashboardAggregator, but charm authors in - # newer charms may import them directly from the marketplace - if "__inputs" in dict_content: - for field in dict_content["__inputs"]: - if "type" in field and field["type"] == "datasource": - datasources[field["name"]] = field["pluginName"].lower() - del dict_content["__inputs"] - - # If no existing template variables exist, just insert our own - if "templating" not in dict_content: - dict_content["templating"] = {"list": list(template_dropdowns)} # type: ignore - else: - # Otherwise, set a flag so we can go back later - existing_templates = True - for template_value in dict_content["templating"]["list"]: - # Build a list of `datasource_name`: `datasource_type` mappings - # The "query" field is actually "prometheus", "loki", "influxdb", etc - if "type" in template_value and template_value["type"] == "datasource": - datasources[template_value["name"]] = template_value["query"].lower() + @classmethod + def _inject_labels(cls, content: str, topology: dict, transformer: "CosTool") -> str: + """Inject Juju topology into panel expressions via CosTool. - # Put our own variables in the template - for d in template_dropdowns: # type: ignore - if d not in dict_content["templating"]["list"]: - dict_content["templating"]["list"].insert(0, d) + A dashboard will have a structure approximating: + { + "__inputs": [], + "templating": { + "list": [ + { + "name": "prometheusds", + "type": "prometheus" + } + ] + }, + "panels": [ + { + "foo": "bar", + "targets": [ + { + "some": "field", + "expr": "up{job="foo"}" + }, + { + "some_other": "field", + "expr": "sum(http_requests_total{instance="$foo"}[5m])} + } + ], + "datasource": "${someds}" + } + ] + } - dict_content = _replace_template_fields(dict_content, datasources, existing_templates) - return json.dumps(dict_content) + `templating` is used elsewhere in this library, but the structure is not rigid. It is + not guaranteed that a panel will actually have any targets (it could be a "spacer" with + no datasource, hence no expression). It could have only one target. It could have multiple + targets. It could have multiple targets of which only one has an `expr` to evaluate. We need + to try to handle all of these concisely. + `cos-tool` (`github.com/canonical/cos-tool` as a Go module in general) + does not know "Grafana-isms", such as using `[$_variable]` to modify the query from the user + interface, so we add placeholders (as `5y`, since it must parse, but a dashboard looking for + five years for a panel query would be unusual). -def _replace_template_fields( # noqa: C901 - dict_content: dict, datasources: dict, existing_templates: bool -) -> dict: - """Make templated fields get cleaned up afterwards. + Args: + content: dashboard content as a string + topology: a dict containing topology values + transformer: a 'CosTool' instance + Returns: + dashboard content with replaced values. + """ + dict_content = json.loads(content) - If existing datasource variables are present, try to substitute them. - """ - replacements = {"loki": "${lokids}", "prometheus": "${prometheusds}"} - used_replacements = [] # type: List[str] - - # If any existing datasources match types we know, or we didn't find - # any templating variables at all, template them. - if datasources or not existing_templates: - panels = dict_content.get("panels", {}) - if panels: - dict_content["panels"] = _template_panels( - panels, replacements, used_replacements, existing_templates, datasources - ) + if "panels" not in dict_content.keys(): + return json.dumps(dict_content) - # Find panels nested under rows - rows = dict_content.get("rows", {}) - if rows: - for row_idx, row in enumerate(rows): - if "panels" in row.keys(): - rows[row_idx]["panels"] = _template_panels( - row["panels"], - replacements, - used_replacements, - existing_templates, - datasources, - ) - - dict_content["rows"] = rows - - # Finally, go back and pop off the templates we stubbed out - deletions = [] - for tmpl in dict_content["templating"]["list"]: - if tmpl["name"] and tmpl["name"] in used_replacements: - deletions.append(tmpl) - - for d in deletions: - dict_content["templating"]["list"].remove(d) - - return dict_content - - -def _template_panels( - panels: dict, - replacements: dict, - used_replacements: list, - existing_templates: bool, - datasources: dict, -) -> dict: - """Iterate through a `panels` object and template it appropriately.""" - # Go through all the panels. If they have a datasource set, AND it's one - # that we can convert to ${lokids} or ${prometheusds}, by stripping off the - # ${} templating and comparing the name to the list we built, replace it, - # otherwise, leave it alone. - # - for panel in panels: - if "datasource" not in panel or not panel.get("datasource"): - continue - if not existing_templates: - datasource = panel.get("datasource") - if isinstance(datasource, str): - if "loki" in datasource: - panel["datasource"] = "${lokids}" - elif "grafana" in datasource: - continue - else: - panel["datasource"] = "${prometheusds}" - elif isinstance(datasource, dict): - # In dashboards exported by Grafana 9, datasource type is dict - dstype = datasource.get("type", "") - if dstype == "loki": - panel["datasource"]["uid"] = "${lokids}" - elif dstype == "prometheus": - panel["datasource"]["uid"] = "${prometheusds}" - else: - logger.debug("Unrecognized datasource type '%s'; skipping", dstype) - continue - else: - logger.error("Unknown datasource format: skipping") + # Go through all the panels and inject topology labels + # Panels may have more than one 'target' where the expressions live, so that must be + # accounted for. Additionally, `promql-transform` does not necessarily gracefully handle + # expressions with range queries including variables. Exclude these. + # + # It is not a certainty that the `datasource` field will necessarily reflect the type, so + # operate on all fields. + panels = dict_content["panels"] + topology_with_prefix = {"juju_{}".format(k): v for k, v in topology.items()} + + # We need to use an index so we can insert the changed element back later + for panel_idx, panel in enumerate(panels): + if not isinstance(panel, dict): continue - else: - if isinstance(panel["datasource"], str): - if panel["datasource"].lower() in replacements.values(): - # Already a known template variable - continue - # Strip out variable characters and maybe braces - ds = re.sub(r"(\$|\{|\})", "", panel["datasource"]) - if ds not in datasources.keys(): - # Unknown, non-templated datasource, potentially a Grafana builtin - continue + # Use the index to insert it back in the same location + panels[panel_idx] = cls._modify_panel(panel, topology_with_prefix, transformer) - replacement = replacements.get(datasources[ds], "") - if replacement: - used_replacements.append(ds) - panel["datasource"] = replacement or panel["datasource"] - elif isinstance(panel["datasource"], dict): - dstype = panel["datasource"].get("type", "") - if panel["datasource"].get("uid", "").lower() in replacements.values(): - # Already a known template variable - continue - # Strip out variable characters and maybe braces - ds = re.sub(r"(\$|\{|\})", "", panel["datasource"].get("uid", "")) + return json.dumps(dict_content) - if ds not in datasources.keys(): - # Unknown, non-templated datasource, potentially a Grafana builtin - continue + @classmethod + def _modify_panel(cls, panel: dict, topology: dict, transformer: "CosTool") -> dict: + """Inject Juju topology into panel expressions via CosTool. - replacement = replacements.get(datasources[ds], "") - if replacement: - used_replacements.append(ds) - panel["datasource"]["uid"] = replacement - else: - logger.error("Unknown datasource format: skipping") - continue - return panels + Args: + panel: a dashboard panel as a dict + topology: a dict containing topology values + transformer: a 'CosTool' instance + Returns: + the panel with injected values + """ + if "targets" not in panel.keys(): + return panel + # Pre-compile a regular expression to grab values from inside of [] + range_re = re.compile(r"\[(?P.*?)\]") + # Do the same for any offsets + offset_re = re.compile(r"offset\s+(?P-?\s*[$\w]+)") -def _inject_labels(content: str, topology: dict, transformer: "CosTool") -> str: - """Inject Juju topology into panel expressions via CosTool. + known_datasources = {"${prometheusds}": "promql", "${lokids}": "logql"} - A dashboard will have a structure approximating: - { - "__inputs": [], - "templating": { - "list": [ - { - "name": "prometheusds", - "type": "prometheus" - } - ] - }, - "panels": [ - { - "foo": "bar", - "targets": [ - { - "some": "field", - "expr": "up{job="foo"}" - }, - { - "some_other": "field", - "expr": "sum(http_requests_total{instance="$foo"}[5m])} - } - ], - "datasource": "${someds}" - } - ] - } + targets = panel["targets"] - `templating` is used elsewhere in this library, but the structure is not rigid. It is - not guaranteed that a panel will actually have any targets (it could be a "spacer" with - no datasource, hence no expression). It could have only one target. It could have multiple - targets. It could have multiple targets of which only one has an `expr` to evaluate. We need - to try to handle all of these concisely. + # We need to use an index so we can insert the changed element back later + for idx, target in enumerate(targets): + # If there's no expression, we don't need to do anything + if "expr" not in target.keys(): + continue + expr = target["expr"] - `cos-tool` (`github.com/canonical/cos-tool` as a Go module in general) - does not know "Grafana-isms", such as using `[$_variable]` to modify the query from the user - interface, so we add placeholders (as `5y`, since it must parse, but a dashboard looking for - five years for a panel query would be unusual). + if "datasource" not in panel.keys(): + continue - Args: - content: dashboard content as a string - topology: a dict containing topology values - transformer: a 'CosTool' instance - Returns: - dashboard content with replaced values. - """ - dict_content = json.loads(content) + if isinstance(panel["datasource"], str): + if panel["datasource"] not in known_datasources: + continue + querytype = known_datasources[panel["datasource"]] + elif isinstance(panel["datasource"], dict): + if panel["datasource"]["uid"] not in known_datasources: + continue + querytype = known_datasources[panel["datasource"]["uid"]] + else: + logger.error("Unknown datasource format: skipping") + continue - if "panels" not in dict_content.keys(): - return json.dumps(dict_content) + # Capture all values inside `[]` into a list which we'll iterate over later to + # put them back in-order. Then apply the regex again and replace everything with + # `[5y]` so promql/parser will take it. + # + # Then do it again for offsets + range_values = [m.group("value") for m in range_re.finditer(expr)] + expr = range_re.sub(r"[5y]", expr) + + offset_values = [m.group("value") for m in offset_re.finditer(expr)] + expr = offset_re.sub(r"offset 5y", expr) + # Retrieve the new expression (which may be unchanged if there were no label + # matchers in the expression, or if tt was unable to be parsed like logql. It's + # virtually impossible to tell from any datasource "name" in a panel what the + # actual type is without re-implementing a complete dashboard parser, but no + # harm will some from passing invalid promql -- we'll just get the original back. + # + replacement = transformer.inject_label_matchers(expr, topology, querytype) - # Go through all the panels and inject topology labels - # Panels may have more than one 'target' where the expressions live, so that must be - # accounted for. Additionally, `promql-transform` does not necessarily gracefully handle - # expressions with range queries including variables. Exclude these. - # - # It is not a certainty that the `datasource` field will necessarily reflect the type, so - # operate on all fields. - panels = dict_content["panels"] - topology_with_prefix = {"juju_{}".format(k): v for k, v in topology.items()} + if replacement == target["expr"]: + # promql-transform caught an error. Move on + continue + + # Go back and substitute values in [] which were pulled out + # Enumerate with an index... again. The same regex is ok, since it will still match + # `[(.*?)]`, which includes `[5y]`, our placeholder + for i, match in enumerate(range_re.finditer(replacement)): + # Replace one-by-one, starting from the left. We build the string back with + # `str.replace(string_to_replace, replacement_value, count)`. Limit the count + # to one, since we are going through one-by-one through the list we saved earlier + # in `range_values`. + replacement = replacement.replace( + "[{}]".format(match.group("value")), + "[{}]".format(range_values[i]), + 1, + ) - # We need to use an index so we can insert the changed element back later - for panel_idx, panel in enumerate(panels): - if not isinstance(panel, dict): - continue + for i, match in enumerate(offset_re.finditer(replacement)): + # Replace one-by-one, starting from the left. We build the string back with + # `str.replace(string_to_replace, replacement_value, count)`. Limit the count + # to one, since we are going through one-by-one through the list we saved earlier + # in `range_values`. + replacement = replacement.replace( + "offset {}".format(match.group("value")), + "offset {}".format(offset_values[i]), + 1, + ) - # Use the index to insert it back in the same location - panels[panel_idx] = _modify_panel(panel, topology_with_prefix, transformer) + # Use the index to insert it back in the same location + targets[idx]["expr"] = replacement - return json.dumps(dict_content) + panel["targets"] = targets + return panel + @classmethod + def _content_to_dashboard_object( + cls, + *, + charm_name, + content: str, + juju_topology: dict, + inject_dropdowns: bool = True, + dashboard_alt_uid: Optional[str] = None, + ) -> Dict: + """Helper method for keeping a consistent stored state schema for the dashboard and some metadata. -def _modify_panel(panel: dict, topology: dict, transformer: "CosTool") -> dict: - """Inject Juju topology into panel expressions via CosTool. + Args: + charm_name: Charm name (although the aggregator passes the app name). + content: The compressed dashboard. + juju_topology: This is not actually used in the dashboards, but is present to provide a secondary + salt to ensure uniqueness in the dict keys in case individual charm units provide dashboards. + inject_dropdowns: Whether to auto-render topology dropdowns. + dashboard_alt_uid: Alternative uid used for dashboards added programmatically. + """ + ret = { + "charm": charm_name, + "content": content, + "juju_topology": juju_topology if inject_dropdowns else {}, + "inject_dropdowns": inject_dropdowns, + } - Args: - panel: a dashboard panel as a dict - topology: a dict containing topology values - transformer: a 'CosTool' instance - Returns: - the panel with injected values - """ - if "targets" not in panel.keys(): - return panel + if dashboard_alt_uid is not None: + ret["dashboard_alt_uid"] = dashboard_alt_uid - # Pre-compile a regular expression to grab values from inside of [] - range_re = re.compile(r"\[(?P.*?)\]") - # Do the same for any offsets - offset_re = re.compile(r"offset\s+(?P-?\s*[$\w]+)") + return ret - known_datasources = {"${prometheusds}": "promql", "${lokids}": "logql"} + @classmethod + def _generate_alt_uid(cls, charm_name: str, key: str) -> str: + """Generate alternative uid for dashboards. - targets = panel["targets"] + Args: + charm_name: The name of the charm (not app; from metadata). + key: A string used (along with charm.meta.name) to build the hash uid. - # We need to use an index so we can insert the changed element back later - for idx, target in enumerate(targets): - # If there's no expression, we don't need to do anything - if "expr" not in target.keys(): - continue - expr = target["expr"] + Returns: A hash string. + """ + raw_dashboard_alt_uid = "{}-{}".format(charm_name, key) + return hashlib.shake_256(raw_dashboard_alt_uid.encode("utf-8")).hexdigest(8) - if "datasource" not in panel.keys(): - continue + @classmethod + def _replace_uid( + cls, *, dashboard_dict: dict, dashboard_path: Path, charm_dir: Path, charm_name: str + ): + # If we're running this from within an aggregator (such as grafana agent), then the uid was + # already rendered there, so we do not want to overwrite it with a uid generated from aggregator's info. + # We overwrite the uid only if it's not a valid "Path40" uid. + original_uid = dashboard_dict.get("uid", "") + + if DashboardPath40UID.is_valid(original_uid): + logger.debug( + "Processed dashboard '%s': kept original uid '%s'", dashboard_path, original_uid + ) + return - if isinstance(panel["datasource"], str): - if panel["datasource"] not in known_datasources: - continue - querytype = known_datasources[panel["datasource"]] - elif isinstance(panel["datasource"], dict): - if panel["datasource"]["uid"] not in known_datasources: - continue - querytype = known_datasources[panel["datasource"]["uid"]] + try: + rel_path = str( + dashboard_path.relative_to(charm_dir) + if dashboard_path.is_absolute() + else dashboard_path + ) + except ValueError: + uid = DashboardPath40UID.generate(charm_name, str(dashboard_path)) else: - logger.error("Unknown datasource format: skipping") - continue + uid = DashboardPath40UID.generate(charm_name, rel_path) - # Capture all values inside `[]` into a list which we'll iterate over later to - # put them back in-order. Then apply the regex again and replace everything with - # `[5y]` so promql/parser will take it. - # - # Then do it again for offsets - range_values = [m.group("value") for m in range_re.finditer(expr)] - expr = range_re.sub(r"[5y]", expr) - - offset_values = [m.group("value") for m in offset_re.finditer(expr)] - expr = offset_re.sub(r"offset 5y", expr) - # Retrieve the new expression (which may be unchanged if there were no label - # matchers in the expression, or if tt was unable to be parsed like logql. It's - # virtually impossible to tell from any datasource "name" in a panel what the - # actual type is without re-implementing a complete dashboard parser, but no - # harm will some from passing invalid promql -- we'll just get the original back. - # - replacement = transformer.inject_label_matchers(expr, topology, querytype) - - if replacement == target["expr"]: - # promql-tranform caught an error. Move on - continue - - # Go back and substitute values in [] which were pulled out - # Enumerate with an index... again. The same regex is ok, since it will still match - # `[(.*?)]`, which includes `[5y]`, our placeholder - for i, match in enumerate(range_re.finditer(replacement)): - # Replace one-by-one, starting from the left. We build the string back with - # `str.replace(string_to_replace, replacement_value, count)`. Limit the count - # to one, since we are going through one-by-one through the list we saved earlier - # in `range_values`. - replacement = replacement.replace( - "[{}]".format(match.group("value")), - "[{}]".format(range_values[i]), - 1, - ) - for i, match in enumerate(offset_re.finditer(replacement)): - # Replace one-by-one, starting from the left. We build the string back with - # `str.replace(string_to_replace, replacement_value, count)`. Limit the count - # to one, since we are going through one-by-one through the list we saved earlier - # in `range_values`. - replacement = replacement.replace( - "offset {}".format(match.group("value")), - "offset {}".format(offset_values[i]), - 1, + logger.debug( + "Processed dashboard '%s': replaced original uid '%s' with '%s'", + dashboard_path, + original_uid, + uid, + ) + dashboard_dict["uid"] = uid + + @classmethod + def _add_tags(cls, dashboard_dict: dict, charm_name: str): + tags: List[str] = dashboard_dict.get("tags", []) + if not any(tag.startswith("charm: ") for tag in tags): + tags.append(f"charm: {charm_name}") + dashboard_dict["tags"] = tags + + @classmethod + def load_dashboards_from_dir( + cls, + *, + dashboards_path: Path, + charm_name: str, + charm_dir: Path, + inject_dropdowns: bool, + juju_topology: dict, + path_filter: Callable[[Path], bool] = lambda p: True, + ) -> dict: + """Load dashboards files from directory into a mapping from "dashboard id" to a so-called "dashboard object".""" + + # Path.glob uses fnmatch on the backend, which is pretty limited, so use a + # custom function for the filter + def _is_dashboard(p: Path) -> bool: + return ( + p.is_file() + and p.name.endswith((".json", ".json.tmpl", ".tmpl")) + and path_filter(p) ) - # Use the index to insert it back in the same location - targets[idx]["expr"] = replacement + dashboard_templates = {} + + for path in filter(_is_dashboard, Path(dashboards_path).glob("**/*")): + try: + dashboard_dict = json.loads(path.read_bytes()) + except json.JSONDecodeError as e: + logger.error("Failed to load dashboard '%s': %s", path, e) + continue + if type(dashboard_dict) is not dict: + logger.error( + "Invalid dashboard '%s': expected dict, got %s", path, type(dashboard_dict) + ) + + cls._replace_uid( + dashboard_dict=dashboard_dict, + dashboard_path=path, + charm_dir=charm_dir, + charm_name=charm_name, + ) - panel["targets"] = targets - return panel + cls._add_tags(dashboard_dict=dashboard_dict, charm_name=charm_name) + id = "file:{}".format(path.stem) + dashboard_templates[id] = cls._content_to_dashboard_object( + charm_name=charm_name, + content=LZMABase64.compress(json.dumps(dashboard_dict)), + dashboard_alt_uid=cls._generate_alt_uid(charm_name, id), + inject_dropdowns=inject_dropdowns, + juju_topology=juju_topology, + ) -def _type_convert_stored(obj): - """Convert Stored* to their appropriate types, recursively.""" - if isinstance(obj, StoredList): - return list(map(_type_convert_stored, obj)) - if isinstance(obj, StoredDict): - rdict = {} # type: Dict[Any, Any] - for k in obj.keys(): - rdict[k] = _type_convert_stored(obj[k]) - return rdict - return obj + return dashboard_templates class GrafanaDashboardsChanged(EventBase): @@ -1075,16 +1218,19 @@ def add_dashboard(self, content: str, inject_dropdowns: bool = True) -> None: # that the stored state is there when this unit becomes leader. stored_dashboard_templates: Any = self._stored.dashboard_templates # pyright: ignore - encoded_dashboard = _encode_dashboard_content(content) + encoded_dashboard = LZMABase64.compress(content) # Use as id the first chars of the encoded dashboard, so that # it is predictable across units. id = "prog:{}".format(encoded_dashboard[-24:-16]) - stored_dashboard_templates[id] = self._content_to_dashboard_object( - encoded_dashboard, inject_dropdowns + stored_dashboard_templates[id] = CharmedDashboard._content_to_dashboard_object( + charm_name=self._charm.meta.name, + content=encoded_dashboard, + dashboard_alt_uid=CharmedDashboard._generate_alt_uid(self._charm.meta.name, id), + inject_dropdowns=inject_dropdowns, + juju_topology=self._juju_topology, ) - stored_dashboard_templates[id]["dashboard_alt_uid"] = self._generate_alt_uid(id) if self._charm.unit.is_leader(): for dashboard_relation in self._charm.model.relations[self._relation_name]: @@ -1111,6 +1257,10 @@ def update_dashboards(self) -> None: for dashboard_relation in self._charm.model.relations[self._relation_name]: self._upset_dashboards_on_relation(dashboard_relation) + def reload_dashboards(self, inject_dropdowns: bool = True) -> None: + """Reloads dashboards and updates all relations.""" + self._update_all_dashboards_from_dir(inject_dropdowns=inject_dropdowns) + def _update_all_dashboards_from_dir( self, _: Optional[HookEvent] = None, inject_dropdowns: bool = True ) -> None: @@ -1127,38 +1277,22 @@ def _update_all_dashboards_from_dir( if dashboard_id.startswith("file:"): del stored_dashboard_templates[dashboard_id] - # Path.glob uses fnmatch on the backend, which is pretty limited, so use a - # custom function for the filter - def _is_dashboard(p: Path) -> bool: - return p.is_file() and p.name.endswith((".json", ".json.tmpl", ".tmpl")) - - for path in filter(_is_dashboard, Path(self._dashboards_path).glob("*")): - # path = Path(path) - id = "file:{}".format(path.stem) - stored_dashboard_templates[id] = self._content_to_dashboard_object( - _encode_dashboard_content(path.read_bytes()), inject_dropdowns + stored_dashboard_templates.update( + CharmedDashboard.load_dashboards_from_dir( + dashboards_path=Path(self._dashboards_path), + charm_name=self._charm.meta.name, + charm_dir=self._charm.charm_dir, + inject_dropdowns=inject_dropdowns, + juju_topology=self._juju_topology, ) - stored_dashboard_templates[id]["dashboard_alt_uid"] = self._generate_alt_uid(id) - - self._stored.dashboard_templates = stored_dashboard_templates + ) if self._charm.unit.is_leader(): for dashboard_relation in self._charm.model.relations[self._relation_name]: self._upset_dashboards_on_relation(dashboard_relation) - def _generate_alt_uid(self, key: str) -> str: - """Generate alternative uid for dashboards. - - Args: - key: A string used (along with charm.meta.name) to build the hash uid. - - Returns: A hash string. - """ - raw_dashboard_alt_uid = "{}-{}".format(self._charm.meta.name, key) - return hashlib.shake_256(raw_dashboard_alt_uid.encode("utf-8")).hexdigest(8) - def _reinitialize_dashboard_data(self, inject_dropdowns: bool = True) -> None: - """Triggers a reload of dashboard outside of an eventing workflow. + """Triggers a reload of dashboard outside an eventing workflow. Args: inject_dropdowns: a :bool: used to indicate whether topology dropdowns should be added @@ -1222,26 +1356,33 @@ def _on_grafana_dashboard_relation_changed(self, event: RelationChangedEvent) -> def _upset_dashboards_on_relation(self, relation: Relation) -> None: """Update the dashboards in the relation data bucket.""" + new_templates = type_convert_stored(self._stored.dashboard_templates) # pyright: ignore + + # Check if the templates have actually changed before updating. + # This avoids generating a new UUID on every event, which would cause + # unnecessary relation-changed events on the consumer side. + # See: https://github.com/canonical/opentelemetry-collector-operator/issues/331 + existing_data_str = relation.data[self._charm.app].get("dashboards", "{}") + try: + existing_data = json.loads(existing_data_str) + existing_templates = existing_data.get("templates", {}) + except json.JSONDecodeError: + existing_templates = {} + + if new_templates == existing_templates: + return # No change in templates, don't update the databag + # It's completely ridiculous to add a UUID, but if we don't have some - # pseudo-random value, this never makes it across 'juju set-state' + # pseudo-random value, this never makes it across 'juju set-state'. + # Use a deterministic hash of the templates so the value is stable when + # templates haven't changed, avoiding spurious relation-changed events. stored_data = { - "templates": _type_convert_stored(self._stored.dashboard_templates), # pyright: ignore - "uuid": str(uuid.uuid4()), + "templates": new_templates, + "uuid": _data_hash(new_templates), } relation.data[self._charm.app]["dashboards"] = json.dumps(stored_data) - def _content_to_dashboard_object(self, content: str, inject_dropdowns: bool = True) -> Dict: - return { - "charm": self._charm.meta.name, - "content": content, - "juju_topology": self._juju_topology if inject_dropdowns else {}, - "inject_dropdowns": inject_dropdowns, - } - - # This is not actually used in the dashboards, but is present to provide a secondary - # salt to ensure uniqueness in the dict keys in case individual charm units provide - # dashboards @property def _juju_topology(self) -> Dict: return { @@ -1306,7 +1447,7 @@ def __init__( super().__init__(charm, relation_name) self._charm = charm self._relation_name = relation_name - self._tranformer = CosTool(self._charm) + self._transformer = CosTool(self._charm) self._stored.set_default(dashboards={}) # type: ignore @@ -1436,21 +1577,21 @@ def _render_dashboards_and_signal_changed(self, relation: Relation) -> bool: # error = None topology = template.get("juju_topology", {}) try: - content = _decode_dashboard_content(template["content"]) + content = LZMABase64.decompress(template["content"]) inject_dropdowns = template.get("inject_dropdowns", True) content = self._manage_dashboard_uid(content, template) - content = _convert_dashboard_fields(content, inject_dropdowns) + content = CharmedDashboard._convert_dashboard_fields(content, inject_dropdowns) if topology: - content = _inject_labels(content, topology, self._tranformer) + content = CharmedDashboard._inject_labels(content, topology, self._transformer) - content = _encode_dashboard_content(content) + content = LZMABase64.compress(content) except lzma.LZMAError as e: error = str(e) relation_has_invalid_dashboards = True except json.JSONDecodeError as e: error = str(e.msg) - logger.warning("Invalid JSON in Grafana dashboard: {}".format(fname)) + logger.warning("Invalid JSON in Grafana dashboard '{}': {}".format(fname, error)) continue # Prepend the relation name and ID to the dashboard ID to avoid clashes with @@ -1502,11 +1643,11 @@ def _render_dashboards_and_signal_changed(self, relation: Relation) -> bool: # stored_data = rendered_dashboards currently_stored_data = self._get_stored_dashboards(relation.id) - coerced_data = _type_convert_stored(currently_stored_data) if currently_stored_data else {} + coerced_data = type_convert_stored(currently_stored_data) if currently_stored_data else {} if not coerced_data == stored_data: stored_dashboards = self.get_peer_data("dashboards") - stored_dashboards[relation.id] = stored_data + stored_dashboards[str(relation.id)] = stored_data self.set_peer_data("dashboards", stored_dashboards) return True return None # type: ignore @@ -1529,29 +1670,60 @@ def _remove_all_dashboards_for_relation(self, relation: Relation) -> None: self.on.dashboards_changed.emit() # pyright: ignore def _to_external_object(self, relation_id, dashboard): + decompressed = LZMABase64.decompress(dashboard["content"]) + as_dict = json.loads(decompressed) + + dashboard_title = as_dict.get("title", "") + dashboard_uid = as_dict.get("uid", "") + + try: + dashboard_version = int(as_dict["version"]) + except (KeyError, ValueError): + logger.warning("Dashboard '%s' (uid '%s') is missing a '.version' field or is invalid (must be integer); using '0' as fallback", dashboard_title, dashboard_uid) + dashboard_version = 0 + return { "id": dashboard["original_id"], "relation_id": relation_id, "charm": dashboard["template"]["charm"], - "content": _decode_dashboard_content(dashboard["content"]), + "content": decompressed, + "dashboard_uid": dashboard_uid, + "dashboard_version": dashboard_version, + "dashboard_title": dashboard_title, } @property def dashboards(self) -> List[Dict]: """Get a list of known dashboards across all instances of the monitored relation. + Filters out dashboards with the same uid, keeping only the one with the highest version. + When more than one dashboard have the same uid and version, keep the first one when + sorted by (relation_id, content) in reverse lexicographic order (highest relid first). + Returns: a list of known dashboards. The JSON of each of the dashboards is available in the `content` field of the corresponding `dict`. """ - dashboards = [] + d: Dict[str, dict] = {} for _, (relation_id, dashboards_for_relation) in enumerate( self.get_peer_data("dashboards").items() ): for dashboard in dashboards_for_relation: - dashboards.append(self._to_external_object(relation_id, dashboard)) + obj = self._to_external_object(relation_id, dashboard) + + key = obj.get("dashboard_uid") + if key is None or str(key).strip() == "": + # At this point, we assume that a `.uid` is present so we do not render a fallback identifier here. Instead, we omit it. + logger.error("dashboard '%s' from relation id '%s' is missing a '.uid' field; omitted", obj["dashboard_title"], obj["relation_id"]) + continue + + if key in d: + d[key] = max(d[key], obj, key=lambda o: (o["dashboard_version"], o["relation_id"], o["content"])) + logger.warning("deduplicate dashboard '%s' (uid '%s') - kept version '%s' from relation id '%s'", d[key]["dashboard_title"], d[key]["dashboard_uid"], d[key]["dashboard_version"], d[key]["relation_id"]) + else: + d[key] = obj - return dashboards + return list(d.values()) def _get_stored_dashboards(self, relation_id: int) -> list: """Pull stored dashboards out of the peer data bucket.""" @@ -1566,11 +1738,21 @@ def _set_default_data(self) -> None: def set_peer_data(self, key: str, data: Any) -> None: """Put information into the peer data bucket instead of `StoredState`.""" - self._charm.peers.data[self._charm.app][key] = json.dumps(data) # type: ignore[attr-defined] + peers = self._charm.peers # type: ignore[attr-defined] + if not peers or not peers.data: + logger.info("set_peer_data: no peer relation. Is the charm being installed/removed?") + return + peers.data[self._charm.app][key] = json.dumps(data, sort_keys=True) # type: ignore[attr-defined] def get_peer_data(self, key: str) -> Any: """Retrieve information from the peer data bucket instead of `StoredState`.""" - data = self._charm.peers.data[self._charm.app].get(key, "") # type: ignore[attr-defined] + peers = self._charm.peers # type: ignore[attr-defined] + if not peers or not peers.data: + logger.warning( + "get_peer_data: no peer relation. Is the charm being installed/removed?" + ) + return {} + data = peers.data[self._charm.app].get(key, "") return json.loads(data) if data else {} @@ -1662,8 +1844,11 @@ def _upset_dashboards_on_event(self, event: RelationEvent) -> None: return for id in dashboards: - self._stored.dashboard_templates[id] = self._content_to_dashboard_object( # type: ignore - dashboards[id], event + self._stored.dashboard_templates[id] = CharmedDashboard._content_to_dashboard_object( # type: ignore + charm_name=event.app.name, + content=dashboards[id], + inject_dropdowns=True, + juju_topology=self._hybrid_topology(event), ) self._stored.id_mappings[event.app.name] = dashboards # type: ignore @@ -1671,19 +1856,35 @@ def _upset_dashboards_on_event(self, event: RelationEvent) -> None: def _update_remote_grafana(self, _: Optional[RelationEvent] = None) -> None: """Push dashboards to the downstream Grafana relation.""" - # It's still ridiculous to add a UUID here, but needed - stored_data = { - "templates": _type_convert_stored(self._stored.dashboard_templates), # pyright: ignore - "uuid": str(uuid.uuid4()), - } + new_templates = type_convert_stored(self._stored.dashboard_templates) # pyright: ignore if self._charm.unit.is_leader(): for grafana_relation in self.model.relations[self._grafana_relation]: + # Check if the templates have actually changed before updating. + # This avoids generating a new UUID on every event, which would cause + # unnecessary relation-changed events on the consumer side. + existing_data_str = grafana_relation.data[self._charm.app].get("dashboards", "{}") + try: + existing_data = json.loads(existing_data_str) + existing_templates = existing_data.get("templates", {}) + except json.JSONDecodeError: + existing_templates = {} + + if new_templates == existing_templates: + continue # No change in templates, don't update the databag + + # It's still ridiculous to add a UUID here, but needed. + # Use a deterministic hash of the templates so the value is stable when + # templates haven't changed, avoiding spurious relation-changed events. + stored_data = { + "templates": new_templates, + "uuid": _data_hash(new_templates), + } grafana_relation.data[self._charm.app]["dashboards"] = json.dumps(stored_data) def remove_dashboards(self, event: RelationBrokenEvent) -> None: """Remove a dashboard if the relation is broken.""" - app_ids = _type_convert_stored(self._stored.id_mappings.get(event.app.name, "")) # type: ignore + app_ids = type_convert_stored(self._stored.id_mappings.get(event.app.name, "")) # type: ignore if not app_ids: logger.info("Could not look up stored dashboards for %s", event.app.name) # type: ignore @@ -1693,9 +1894,10 @@ def remove_dashboards(self, event: RelationBrokenEvent) -> None: for id in app_ids: del self._stored.dashboard_templates[id] # type: ignore + remaining_templates = type_convert_stored(self._stored.dashboard_templates) # pyright: ignore stored_data = { - "templates": _type_convert_stored(self._stored.dashboard_templates), # pyright: ignore - "uuid": str(uuid.uuid4()), + "templates": remaining_templates, + "uuid": _data_hash(remaining_templates), } if self._charm.unit.is_leader(): @@ -1824,7 +2026,7 @@ def _handle_reactive_dashboards(self, event: RelationEvent) -> Optional[Dict]: from jinja2 import DebugUndefined, Template - content = _encode_dashboard_content( + content = LZMABase64.compress( Template(dash, undefined=DebugUndefined).render(datasource=r"${prometheusds}") # type: ignore ) id = "prog:{}".format(content[-24:-16]) @@ -1855,32 +2057,20 @@ def _maybe_get_builtin_dashboards(self, event: RelationEvent) -> Dict: ) if dashboards_path: - - def is_dashboard(p: Path) -> bool: - return p.is_file() and p.name.endswith((".json", ".json.tmpl", ".tmpl")) - - for path in filter(is_dashboard, Path(dashboards_path).glob("*")): - # path = Path(path) - if event.app.name in path.name: # type: ignore - id = "file:{}".format(path.stem) - builtins[id] = self._content_to_dashboard_object( - _encode_dashboard_content(path.read_bytes()), event - ) + builtins.update( + CharmedDashboard.load_dashboards_from_dir( + dashboards_path=Path(dashboards_path), + charm_name=event.app.name, + charm_dir=self._charm.charm_dir, + inject_dropdowns=True, + juju_topology=self._hybrid_topology(event), + path_filter=lambda path: event.app.name in path.name, + ) + ) return builtins - def _content_to_dashboard_object(self, content: str, event: RelationEvent) -> Dict: - return { - "charm": event.app.name, # type: ignore - "content": content, - "juju_topology": self._juju_topology(event), - "inject_dropdowns": True, - } - - # This is not actually used in the dashboards, but is present to provide a secondary - # salt to ensure uniqueness in the dict keys in case individual charm units provide - # dashboards - def _juju_topology(self, event: RelationEvent) -> Dict: + def _hybrid_topology(self, event: RelationEvent) -> Dict: return { "model": self._charm.model.name, "model_uuid": self._charm.model.uuid, @@ -1952,11 +2142,11 @@ def validate_alert_rules(self, rules: dict) -> Tuple[bool, str]: # - alert: OtherAlert # expr: up transformed_rules = {"groups": []} # type: ignore - for rule in rules["groups"]: - transformed = {"name": str(uuid.uuid4()), "rules": [rule]} + for i, rule in enumerate(rules["groups"]): + transformed = {"name": f"group_{i}", "rules": [rule]} transformed_rules["groups"].append(transformed) - rule_path.write_text(yaml.dump(transformed_rules)) + rule_path.write_text(yaml.safe_dump(transformed_rules, sort_keys=True)) # databag-order: ignore args = [str(self.path), "validate", str(rule_path)] # noinspection PyBroadException @@ -1999,12 +2189,9 @@ def _get_tool_path(self) -> Optional[Path]: arch = "amd64" if arch == "x86_64" else arch res = "cos-tool-{}".format(arch) try: - path = Path(res).resolve() - path.chmod(0o777) + path = Path(res).resolve(strict=True) return path - except NotImplementedError: - logger.debug("System lacks support for chmod") - except FileNotFoundError: + except (FileNotFoundError, OSError): logger.debug('Could not locate cos-tool at: "{}"'.format(res)) return None diff --git a/lib/charms/loki_k8s/v1/loki_push_api.py b/lib/charms/loki_k8s/v1/loki_push_api.py index 7f8372c47..e0052229b 100644 --- a/lib/charms/loki_k8s/v1/loki_push_api.py +++ b/lib/charms/loki_k8s/v1/loki_push_api.py @@ -79,7 +79,6 @@ def __init__(self, *args): external_url = urlparse(self._external_url) self.loki_provider = LokiPushApiProvider( self, - address=external_url.hostname or self.hostname, port=external_url.port or 80, scheme=external_url.scheme, path=f"{external_url.path}/loki/api/v1/push", @@ -96,6 +95,7 @@ def __init__(self, *args): 1. Set the URL of the Loki Push API in the relation application data bag; the URL must be unique to all instances (e.g. using a load balancer). + The default URL is the FQDN, but this can be overridden by calling `update_endpoint()`. 2. Set the Promtail binary URL (`promtail_binary_zip_url`) so clients that use `LogProxyConsumer` object could download and configure it. @@ -190,10 +190,7 @@ def __init__(self, *args): with its clients. If provided, this relation name must match a required relation in metadata.yaml with the `loki_push_api` interface. - This argument is not required if your metadata.yaml has precisely one - required relation in metadata.yaml with the `loki_push_api` interface, as the - lib will automatically resolve the relation name inspecting the using the - meta information of the charm + If not provided, the relation name defaults to `logging`. Any time the relation between a Loki provider charm and a Loki consumer charm is established, a `LokiPushApiEndpointJoined` event is fired. In the consumer side @@ -409,7 +406,7 @@ def __init__(self, *args): This directory must reside at the top level in the `src` folder of the consumer charm. Each file in this directory is assumed to be a single alert rule -in YAML format. The file name must have the `.rule` extension. +in YAML format. The file name must have one of the following extensions: `.yaml`, `.yml`, `.rule`, or `.rules`. The format of this alert rule conforms to the [Loki docs](https://grafana.com/docs/loki/latest/rules/#alerting-rules). @@ -480,28 +477,48 @@ def _alert_rules_error(self, event): Units of consumer charm send their alert rules over app relation data using the `alert_rules` key. + +## Charm logging +The `charms.loki_k8s.v0.charm_logging` library can be used in conjunction with this one to configure python's +logging module to forward all logs to Loki via the loki-push-api interface. + +```python +from lib.charms.loki_k8s.v0.charm_logging import log_charm +from lib.charms.loki_k8s.v1.loki_push_api import charm_logging_config, LokiPushApiConsumer + +@log_charm(logging_endpoint="my_endpoints", server_cert="cert_path") +class MyCharm(...): + _cert_path = "/path/to/cert/on/charm/container.crt" + def __init__(self, ...): + self.logging = LokiPushApiConsumer(...) + self.my_endpoints, self.cert_path = charm_logging_config( + self.logging, self._cert_path) +``` + +Do this, and all charm logs will be forwarded to Loki as soon as a relation is formed. """ +import copy import json import logging import os import platform import re import socket -import subprocess -import tempfile -import typing +import warnings from copy import deepcopy from gzip import GzipFile from hashlib import sha256 from io import BytesIO from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union, cast from urllib import request from urllib.error import URLError import yaml -from cosl import JujuTopology +from cosl import CosTool, JujuTopology +from cosl.rules import AlertRules +from cosl.types import OfficialRuleFileFormat from ops.charm import ( CharmBase, HookEvent, @@ -514,7 +531,7 @@ def _alert_rules_error(self, event): RelationRole, WorkloadEvent, ) -from ops.framework import EventBase, EventSource, Object, ObjectEvents +from ops.framework import BoundEvent, EventBase, EventSource, Object, ObjectEvents from ops.jujuversion import JujuVersion from ops.model import Container, ModelError, Relation from ops.pebble import APIError, ChangeError, Layer, PathError, ProtocolError @@ -527,7 +544,7 @@ def _alert_rules_error(self, event): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 12 +LIBPATCH = 31 PYDEPS = ["cosl"] @@ -577,7 +594,11 @@ def _alert_rules_error(self, event): GRPC_LISTEN_PORT_START = 9095 # odd start port -class RelationNotFoundError(ValueError): +class LokiPushApiError(Exception): + """Base class for errors raised by this module.""" + + +class RelationNotFoundError(LokiPushApiError): """Raised if there is no relation with the given name.""" def __init__(self, relation_name: str): @@ -587,7 +608,7 @@ def __init__(self, relation_name: str): super().__init__(self.message) -class RelationInterfaceMismatchError(Exception): +class RelationInterfaceMismatchError(LokiPushApiError): """Raised if the relation with the given name has a different interface.""" def __init__( @@ -607,7 +628,7 @@ def __init__( super().__init__(self.message) -class RelationRoleMismatchError(Exception): +class RelationRoleMismatchError(LokiPushApiError): """Raised if the relation with the given name has a different direction.""" def __init__( @@ -697,273 +718,6 @@ def __init__( super().__init__(self.message) -def _is_official_alert_rule_format(rules_dict: dict) -> bool: - """Are alert rules in the upstream format as supported by Loki. - - Alert rules in dictionary format are in "official" form if they - contain a "groups" key, since this implies they contain a list of - alert rule groups. - - Args: - rules_dict: a set of alert rules in Python dictionary format - - Returns: - True if alert rules are in official Loki file format. - """ - return "groups" in rules_dict - - -def _is_single_alert_rule_format(rules_dict: dict) -> bool: - """Are alert rules in single rule format. - - The Loki charm library supports reading of alert rules in a - custom format that consists of a single alert rule per file. This - does not conform to the official Loki alert rule file format - which requires that each alert rules file consists of a list of - alert rule groups and each group consists of a list of alert - rules. - - Alert rules in dictionary form are considered to be in single rule - format if in the least it contains two keys corresponding to the - alert rule name and alert expression. - - Returns: - True if alert rule is in single rule file format. - """ - # one alert rule per file - return set(rules_dict) >= {"alert", "expr"} - - -class AlertRules: - """Utility class for amalgamating Loki alert rule files and injecting juju topology. - - An `AlertRules` object supports aggregating alert rules from files and directories in both - official and single rule file formats using the `add_path()` method. All the alert rules - read are annotated with Juju topology labels and amalgamated into a single data structure - in the form of a Python dictionary using the `as_dict()` method. Such a dictionary can be - easily dumped into JSON format and exchanged over relation data. The dictionary can also - be dumped into YAML format and written directly into an alert rules file that is read by - Loki. Note that multiple `AlertRules` objects must not be written into the same file, - since Loki allows only a single list of alert rule groups per alert rules file. - - The official Loki format is a YAML file conforming to the Loki documentation - (https://grafana.com/docs/loki/latest/api/#list-rule-groups). - The custom single rule format is a subsection of the official YAML, having a single alert - rule, effectively "one alert per file". - """ - - # This class uses the following terminology for the various parts of a rule file: - # - alert rules file: the entire groups[] yaml, including the "groups:" key. - # - alert groups (plural): the list of groups[] (a list, i.e. no "groups:" key) - it is a list - # of dictionaries that have the "name" and "rules" keys. - # - alert group (singular): a single dictionary that has the "name" and "rules" keys. - # - alert rules (plural): all the alerts in a given alert group - a list of dictionaries with - # the "alert" and "expr" keys. - # - alert rule (singular): a single dictionary that has the "alert" and "expr" keys. - - def __init__(self, topology: Optional[JujuTopology] = None): - """Build and alert rule object. - - Args: - topology: a `JujuTopology` instance that is used to annotate all alert rules. - """ - self.topology = topology - self.tool = CosTool(None) - self.alert_groups = [] # type: List[dict] - - def _from_file(self, root_path: Path, file_path: Path) -> List[dict]: - """Read a rules file from path, injecting juju topology. - - Args: - root_path: full path to the root rules folder (used only for generating group name) - file_path: full path to a *.rule file. - - Returns: - A list of dictionaries representing the rules file, if file is valid (the structure is - formed by `yaml.safe_load` of the file); an empty list otherwise. - """ - with file_path.open() as rf: - # Load a list of rules from file then add labels and filters - try: - rule_file = yaml.safe_load(rf) or {} - - except Exception as e: - logger.error("Failed to read alert rules from %s: %s", file_path.name, e) - return [] - - if _is_official_alert_rule_format(rule_file): - alert_groups = rule_file["groups"] - elif _is_single_alert_rule_format(rule_file): - # convert to list of alert groups - # group name is made up from the file name - alert_groups = [{"name": file_path.stem, "rules": [rule_file]}] - else: - # invalid/unsupported - reason = "file is empty" if not rule_file else "unexpected file structure" - logger.error("Invalid rules file (%s): %s", reason, file_path.name) - return [] - - # update rules with additional metadata - for alert_group in alert_groups: - # update group name with topology and sub-path - alert_group["name"] = self._group_name( - str(root_path), - str(file_path), - alert_group["name"], - ) - - # add "juju_" topology labels - for alert_rule in alert_group["rules"]: - if "labels" not in alert_rule: - alert_rule["labels"] = {} - - if self.topology: - # only insert labels that do not already exist - for label, val in self.topology.label_matcher_dict.items(): - if label not in alert_rule["labels"]: - alert_rule["labels"][label] = val - - # insert juju topology filters into a prometheus alert rule - # logql doesn't like empty matchers, so add a job matcher which hits - # any string as a "wildcard" which the topology labels will - # filter down - alert_rule["expr"] = self.tool.inject_label_matchers( - re.sub(r"%%juju_topology%%", r'job=~".+"', alert_rule["expr"]), - self.topology.label_matcher_dict, - ) - - return alert_groups - - def _group_name( - self, - root_path: typing.Union[Path, str], - file_path: typing.Union[Path, str], - group_name: str, - ) -> str: - """Generate group name from path and topology. - - The group name is made up of the relative path between the root dir_path, the file path, - and topology identifier. - - Args: - root_path: path to the root rules dir. - file_path: path to rule file. - group_name: original group name to keep as part of the new augmented group name - - Returns: - New group name, augmented by juju topology and relative path. - """ - file_path = Path(file_path) if not isinstance(file_path, Path) else file_path - root_path = Path(root_path) if not isinstance(root_path, Path) else root_path - rel_path = file_path.parent.relative_to(root_path.as_posix()) - - # We should account for both absolute paths and Windows paths. Convert it to a POSIX - # string, strip off any leading /, then join it - - path_str = "" - if not rel_path == Path("."): - # Get rid of leading / and optionally drive letters so they don't muck up - # the template later, since Path.parts returns them. The 'if relpath.is_absolute ...' - # isn't even needed since re.sub doesn't throw exceptions if it doesn't match, so it's - # optional, but it makes it clear what we're doing. - - # Note that Path doesn't actually care whether the path is valid just to instantiate - # the object, so we can happily strip that stuff out to make templating nicer - rel_path = Path( - re.sub(r"^([A-Za-z]+:)?/", "", rel_path.as_posix()) - if rel_path.is_absolute() - else str(rel_path) - ) - - # Get rid of relative path characters in the middle which both os.path and pathlib - # leave hanging around. We could use path.resolve(), but that would lead to very - # long template strings when rules come from pods and/or other deeply nested charm - # paths - path_str = "_".join(filter(lambda x: x not in ["..", "/"], rel_path.parts)) - - # Generate group name: - # - name, from juju topology - # - suffix, from the relative path of the rule file; - group_name_parts = [self.topology.identifier] if self.topology else [] - group_name_parts.extend([path_str, group_name, "alerts"]) - # filter to remove empty strings - return "_".join(filter(lambda x: x, group_name_parts)) - - @classmethod - def _multi_suffix_glob( - cls, dir_path: Path, suffixes: List[str], recursive: bool = True - ) -> list: - """Helper function for getting all files in a directory that have a matching suffix. - - Args: - dir_path: path to the directory to glob from. - suffixes: list of suffixes to include in the glob (items should begin with a period). - recursive: a flag indicating whether a glob is recursive (nested) or not. - - Returns: - List of files in `dir_path` that have one of the suffixes specified in `suffixes`. - """ - all_files_in_dir = dir_path.glob("**/*" if recursive else "*") - return list(filter(lambda f: f.is_file() and f.suffix in suffixes, all_files_in_dir)) - - def _from_dir(self, dir_path: Path, recursive: bool) -> List[dict]: - """Read all rule files in a directory. - - All rules from files for the same directory are loaded into a single - group. The generated name of this group includes juju topology. - By default, only the top directory is scanned; for nested scanning, pass `recursive=True`. - - Args: - dir_path: directory containing *.rule files (alert rules without groups). - recursive: flag indicating whether to scan for rule files recursively. - - Returns: - a list of dictionaries representing prometheus alert rule groups, each dictionary - representing an alert group (structure determined by `yaml.safe_load`). - """ - alert_groups = [] # type: List[dict] - - # Gather all alerts into a list of groups - for file_path in self._multi_suffix_glob(dir_path, [".rule", ".rules"], recursive): - alert_groups_from_file = self._from_file(dir_path, file_path) - if alert_groups_from_file: - logger.debug("Reading alert rule from %s", file_path) - alert_groups.extend(alert_groups_from_file) - - return alert_groups - - def add_path(self, path_str: str, *, recursive: bool = False): - """Add rules from a dir path. - - All rules from files are aggregated into a data structure representing a single rule file. - All group names are augmented with juju topology. - - Args: - path_str: either a rules file or a dir of rules files. - recursive: whether to read files recursively or not (no impact if `path` is a file). - - Raises: - InvalidAlertRulePathError: if the provided path is invalid. - """ - path = Path(path_str) # type: Path - if path.is_dir(): - self.alert_groups.extend(self._from_dir(path, recursive)) - elif path.is_file(): - self.alert_groups.extend(self._from_file(path.parent, path)) - else: - logger.debug("The alerts file does not exist: %s", path) - - def as_dict(self) -> dict: - """Return standard alert rules file in dict representation. - - Returns: - a dictionary containing a single list of alert rule groups. - The list of alert rule groups is provided as value of the - "groups" dictionary key. - """ - return {"groups": self.alert_groups} if self.alert_groups else {} - - def _resolve_dir_against_charm_path(charm: CharmBase, *path_elements: str) -> str: """Resolve the provided path items against the directory of the main file. @@ -1129,7 +883,7 @@ def __init__( *, port: Union[str, int] = 3100, scheme: str = "http", - address: str = "localhost", + address: str = "", path: str = "loki/api/v1/push", ): """A Loki service provider. @@ -1144,7 +898,9 @@ def __init__( other charms that consume metrics endpoints. port: an optional port of the Loki service (default is "3100"). scheme: an optional scheme of the Loki API URL (default is "http"). - address: an optional address of the Loki service (default is "localhost"). + address: DEPRECATED. This argument is ignored and will be removed in v2. + It is kept for backward compatibility. + Use `update_endpoint()` instead. path: an optional path of the Loki API URL (default is "loki/api/v1/push") Raises: @@ -1160,14 +916,23 @@ def __init__( _validate_relation_by_interface_and_direction( charm, relation_name, RELATION_INTERFACE_NAME, RelationRole.provides ) + + if address != "": + warnings.warn( + "The 'address' parameter is deprecated and will be removed in v2. " + "Use 'update_endpoint()' instead.", + DeprecationWarning, + stacklevel=2, + ) + super().__init__(charm, relation_name) self._charm = charm self._relation_name = relation_name - self._tool = CosTool(self) + self._tool = CosTool("logql") self.port = int(port) self.scheme = scheme - self.address = address self.path = path + self._custom_url = None events = self._charm.on[relation_name] self.framework.observe(self._charm.on.upgrade_charm, self._on_lifecycle_event) @@ -1281,6 +1046,14 @@ def _process_logging_relation_changed(self, relation: Relation) -> bool: """ relation.data[self._charm.unit]["public_address"] = socket.getfqdn() or "" self.update_endpoint(relation=relation) + + # Ensure promtail binary URL is set in app data. This is normally done on + # relation_joined, but charms using the reconcile pattern may miss that event + # if the workload container is not yet ready when the relation is first established. + if self._charm.unit.is_leader(): + if not relation.data[self._charm.app].get("promtail_binary_zip_url"): + relation.data[self._charm.app].update(self._promtail_binary_url) + return self._should_update_alert_rules(relation) @property @@ -1305,6 +1078,11 @@ def update_endpoint(self, url: str = "", relation: Optional[Relation] = None) -> host address change because the charmed operator becomes connected to an Ingress after the `logging` relation is established. + To make this library reconciler-friendly, the endpoint URL was made sticky i.e., once the + endpoint is updated with a custom URL, using the public method, it cannot be unset. Users + of this method should set the "url" arg to an internal URL if the charms ingress is no + longer available. + Args: url: An optional url value to update relation data. relation: An optional instance of `class:ops.model.Relation` to update. @@ -1318,7 +1096,10 @@ def update_endpoint(self, url: str = "", relation: Optional[Relation] = None) -> else: relations_list = [relation] - endpoint = self._endpoint(url or self._url) + if url: + self._custom_url = url + + endpoint = self._endpoint(self._custom_url or self._url) for relation in relations_list: relation.data[self._charm.unit].update({"endpoint": json.dumps(endpoint)}) @@ -1331,7 +1112,7 @@ def _url(self) -> str: Return url to loki, including port number, but without the endpoint subpath. """ - return "http://{}:{}".format(socket.getfqdn(), self.port) + return f"{self.scheme}://{socket.getfqdn()}:{self.port}" def _endpoint(self, url) -> dict: """Get Loki push API endpoint for a given url. @@ -1393,7 +1174,6 @@ def alerts(self) -> dict: # noqa: C901 try: metadata = json.loads(relation.data[relation.app]["metadata"]) identifier = JujuTopology.from_dict(metadata).identifier - alerts[identifier] = self._tool.apply_label_matchers(alert_rules) # type: ignore except KeyError as e: logger.debug( @@ -1408,15 +1188,64 @@ def alerts(self) -> dict: # noqa: C901 ) continue - _, errmsg = self._tool.validate_alert_rules(alert_rules) + # Topology labels are already injected by _inject_alert_expr_labels using + # alert_expression_dict, which intentionally excludes juju_charm and juju_unit. + # Don't call apply_label_matchers here as it would re-inject juju_charm. + alerts[identifier] = alert_rules + + _, errmsg = self._tool.validate_alert_rules(cast(OfficialRuleFileFormat, alert_rules)) if errmsg: - relation.data[self._charm.app]["event"] = json.dumps({"errors": errmsg}) + logger.error(f"Invalid alert rule file: {errmsg}") + if alerts[identifier]: + del alerts[identifier] + if self._charm.unit.is_leader(): + relation.data[self._charm.app]["event"] = json.dumps({"errors": errmsg}) continue + if self._charm.unit.is_leader(): + event_data = json.loads(relation.data[self._charm.app].get("event", "{}")) + event_data.pop("errors", None) + relation.data[self._charm.app]["event"] = json.dumps(event_data) alerts[identifier] = alert_rules return alerts + def has_invalid_alert_rules(self) -> bool: + """Check whether any relation reported invalid alert rules. + + Validation errors, written to relation app data by the :attr:`alerts` + property, are read back to determine whether the relation currently + carries invalid alert rules. Non-leader units never write the app data + that holds these errors, so they always report no errors. + + Returns: + True if any related consumer reported alert rule validation + errors, False otherwise. + """ + if not self._charm.unit.is_leader(): + return False + + for relation in self._charm.model.relations.get(self._relation_name, []): + app_data = relation.data.get(self._charm.app) + if not app_data: + continue + + event_raw = app_data.get("event", "{}") + try: + event_data = json.loads(event_raw) + except (json.JSONDecodeError, TypeError): + continue + + if error_msg := event_data.get("errors"): + logger.error( + "Alert rule validation error on relation %s: %s", + relation.id, + error_msg, + ) + return True + + return False + def _get_identifier_by_alert_rules( self, rules: dict ) -> Tuple[Union[str, None], Union[JujuTopology, None]]: @@ -1493,10 +1322,13 @@ def _inject_alert_expr_labels(self, rules: Dict[str, Any]) -> Dict[str, Any]: charm_name=labels.get("juju_charm", ""), ) - # Inject topology and put it back in the list + # Inject topology and put it back in the list. + # Use alert_expression_dict (excludes juju_charm) instead of + # label_matcher_dict because subordinate charms (e.g. otelcol) + # label logs with their own charm name, not the principal's. rule["expr"] = self._tool.inject_label_matchers( re.sub(r"%%juju_topology%%,?", "", rule["expr"]), - topology.label_matcher_dict, + topology.alert_expression_dict, ) except KeyError: # Some required JujuTopology key is missing. Just move on. @@ -1520,10 +1352,15 @@ def __init__( alert_rules_path: str = DEFAULT_ALERT_RULES_RELATIVE_PATH, recursive: bool = False, skip_alert_topology_labeling: bool = False, + *, + forward_alert_rules: bool = True, + extra_alert_labels: Dict = {}, ): super().__init__(charm, relation_name) self._charm = charm self._relation_name = relation_name + self._forward_alert_rules = forward_alert_rules + self._extra_alert_labels = extra_alert_labels self.topology = JujuTopology.from_charm(charm) try: @@ -1539,16 +1376,33 @@ def __init__( self._recursive = recursive + @staticmethod + def _inject_extra_labels_to_alert_rules(rules: Dict, extra_alert_labels: Dict) -> Dict: + """Return a copy of the rules dict with extra labels injected.""" + result = copy.deepcopy(rules) + for group in result.get("groups", []): + for rule in group.get("rules", []): + rule.setdefault("labels", {}).update(extra_alert_labels) + return result + def _handle_alert_rules(self, relation): if not self._charm.unit.is_leader(): return alert_rules = ( - AlertRules(None) if self._skip_alert_topology_labeling else AlertRules(self.topology) + AlertRules(query_type="logql") + if self._skip_alert_topology_labeling + else AlertRules(query_type="logql", topology=self.topology) ) - alert_rules.add_path(self._alert_rules_path, recursive=self._recursive) + if self._forward_alert_rules: + alert_rules.add_path(self._alert_rules_path, recursive=self._recursive) alert_rules_as_dict = alert_rules.as_dict() + if self._extra_alert_labels: + alert_rules_as_dict = ConsumerBase._inject_extra_labels_to_alert_rules( + alert_rules_as_dict, self._extra_alert_labels + ) + relation.data[self._charm.app]["metadata"] = json.dumps(self.topology.as_dict()) relation.data[self._charm.app]["alert_rules"] = json.dumps( alert_rules_as_dict, @@ -1560,24 +1414,41 @@ def loki_endpoints(self) -> List[dict]: """Fetch Loki Push API endpoints sent from LokiPushApiProvider through relation data. Returns: - A list of dictionaries with Loki Push API endpoints, for instance: + A list of unique dictionaries with Loki Push API endpoints, for instance: [ {"url": "http://loki1:3100/loki/api/v1/push"}, {"url": "http://loki2:3100/loki/api/v1/push"}, ] """ - endpoints = [] # type: list + endpoints = [] + seen_urls = set() for relation in self._charm.model.relations[self._relation_name]: for unit in relation.units: if unit.app == self._charm.app: - # This is a peer unit continue - endpoint = relation.data[unit].get("endpoint") - if endpoint: - deserialized_endpoint = json.loads(endpoint) - endpoints.append(deserialized_endpoint) + if not (endpoint := relation.data[unit].get("endpoint")): + continue + + deserialized_endpoint = json.loads(endpoint) + url = deserialized_endpoint.get("url") + + # Deduplicate by URL. + # With loki-k8s we have ingress-per-unit, so in that case + # we do want to collect the URLs of all the units. + # With loki-coordinator-k8s, even when the coordinator + # is scaled, we want to advertise only one URL. + # Without deduplication, we'd end up with the same + # tls config section in the promtail config file, in which + # case promtail immediately exits with the following error: + # [promtail] level=error ts= msg="error creating promtail" error="failed to create client manager: duplicate client configs are not allowed, found duplicate for name: " + + if not url or url in seen_urls: + continue + + seen_urls.add(url) + endpoints.append(deserialized_endpoint) return endpoints @@ -1594,6 +1465,10 @@ def __init__( alert_rules_path: str = DEFAULT_ALERT_RULES_RELATIVE_PATH, recursive: bool = True, skip_alert_topology_labeling: bool = False, + *, + refresh_event: Optional[Union[BoundEvent, List[BoundEvent]]] = None, + forward_alert_rules: bool = True, + extra_alert_labels: Dict = {}, ): """Construct a Loki charm client. @@ -1619,6 +1494,10 @@ def __init__( alert_rules_path: a string indicating a path where alert rules can be found recursive: Whether to scan for rule files recursively. skip_alert_topology_labeling: whether to skip the alert topology labeling. + forward_alert_rules: a boolean flag to toggle forwarding of charmed alert rules. + extra_alert_labels: Dict of extra labels to inject alert rules with. + refresh_event: an optional bound event or list of bound events which + will be observed to re-set scrape job data (IP address and others) Raises: RelationNotFoundError: If there is no relation in the charm's metadata.yaml @@ -1644,14 +1523,27 @@ def __init__( charm, relation_name, RELATION_INTERFACE_NAME, RelationRole.requires ) super().__init__( - charm, relation_name, alert_rules_path, recursive, skip_alert_topology_labeling + charm, + relation_name, + alert_rules_path, + recursive, + skip_alert_topology_labeling, + forward_alert_rules=forward_alert_rules, + extra_alert_labels=extra_alert_labels, ) events = self._charm.on[relation_name] self.framework.observe(self._charm.on.upgrade_charm, self._on_lifecycle_event) + self.framework.observe(self._charm.on.config_changed, self._on_lifecycle_event) self.framework.observe(events.relation_joined, self._on_logging_relation_joined) self.framework.observe(events.relation_changed, self._on_logging_relation_changed) self.framework.observe(events.relation_departed, self._on_logging_relation_departed) + if refresh_event: + if not isinstance(refresh_event, list): + refresh_event = [refresh_event] + for ev in refresh_event: + self.framework.observe(ev, self._on_lifecycle_event) + def _on_lifecycle_event(self, _: HookEvent): """Update require relation data on charm upgrades and other lifecycle events. @@ -1711,8 +1603,11 @@ def _on_logging_relation_changed(self, event: RelationEvent): self.on.loki_push_api_endpoint_joined.emit() - def _reinitialize_alert_rules(self): + def reload_alerts(self) -> None: """Reloads alert rules and updates all relations.""" + self._reinitialize_alert_rules() + + def _reinitialize_alert_rules(self): for relation in self._charm.model.relations[self._relation_name]: self._handle_alert_rules(relation) @@ -1850,7 +1745,7 @@ def __init__( self._promtails_ports = self._generate_promtails_ports(logs_scheme) # architecture used for promtail binary - arch = platform.processor() + arch = platform.machine() if arch in ["x86_64", "amd64"]: self._arch = "amd64" elif arch in ["aarch64", "arm64", "armv8b", "armv8l"]: @@ -2444,6 +2339,7 @@ def _build_log_target( "juju_model_uuid": topology._model_uuid, "juju_application": topology._application, "juju_unit": topology._unit, + "job": f"juju_{topology.identifier}", }, } ) @@ -2527,10 +2423,17 @@ def __init__( alert_rules_path: str = DEFAULT_ALERT_RULES_RELATIVE_PATH, recursive: bool = True, skip_alert_topology_labeling: bool = False, + refresh_event: Optional[Union[BoundEvent, List[BoundEvent]]] = None, + forward_alert_rules: bool = True, ): _PebbleLogClient.check_juju_version() super().__init__( - charm, relation_name, alert_rules_path, recursive, skip_alert_topology_labeling + charm, + relation_name, + alert_rules_path, + recursive, + skip_alert_topology_labeling, + forward_alert_rules=forward_alert_rules, ) self._charm = charm self._relation_name = relation_name @@ -2541,6 +2444,12 @@ def __init__( self.framework.observe(on.relation_departed, self._update_logging) self.framework.observe(on.relation_broken, self._update_logging) + if refresh_event: + if not isinstance(refresh_event, list): + refresh_event = [refresh_event] + for ev in refresh_event: + self.framework.observe(ev, self._update_logging) + for container_name in self._charm.meta.containers.keys(): snake_case_container_name = container_name.replace("-", "_") self.framework.observe( @@ -2555,7 +2464,7 @@ def _on_pebble_ready(self, event: PebbleReadyEvent): self._update_endpoints(event.workload, loki_endpoints) - def _update_logging(self, _): + def _update_logging(self, event: RelationEvent): """Update the log forwarding to match the active Loki endpoints.""" if not (loki_endpoints := self._retrieve_endpoints_from_relation()): logger.warning("No Loki endpoints available") @@ -2566,6 +2475,8 @@ def _update_logging(self, _): self._update_endpoints(container, loki_endpoints) # else: `_update_endpoints` will be called on pebble-ready anyway. + self._handle_alert_rules(event.relation) + def _retrieve_endpoints_from_relation(self) -> dict: loki_endpoints = {} @@ -2635,118 +2546,47 @@ def _fetch_endpoints(self, relation: Relation) -> Dict[str, str]: return endpoints -class CosTool: - """Uses cos-tool to inject label matchers into alert rule expressions and validate rules.""" +def charm_logging_config( + endpoint_requirer: LokiPushApiConsumer, cert_path: Optional[Union[Path, str]] +) -> Tuple[Optional[List[str]], Optional[str]]: + """Utility function to determine the charm_logging config you will likely want. - _path = None - _disabled = False + If no endpoint is provided: + disable charm logging. + If https endpoint is provided but cert_path is not found on disk: + disable charm logging. + If https endpoint is provided and cert_path is None: + ERROR + Else: + proceed with charm logging (with or without tls, as appropriate) - def __init__(self, charm): - self._charm = charm + Args: + endpoint_requirer: an instance of LokiPushApiConsumer. + cert_path: a path where a cert is stored. - @property - def path(self): - """Lazy lookup of the path of cos-tool.""" - if self._disabled: - return None - if not self._path: - self._path = self._get_tool_path() - if not self._path: - logger.debug("Skipping injection of juju topology as label matchers") - self._disabled = True - return self._path - - def apply_label_matchers(self, rules) -> dict: - """Will apply label matchers to the expression of all alerts in all supplied groups.""" - if not self.path: - return rules - for group in rules["groups"]: - rules_in_group = group.get("rules", []) - for rule in rules_in_group: - topology = {} - # if the user for some reason has provided juju_unit, we'll need to honor it - # in most cases, however, this will be empty - for label in [ - "juju_model", - "juju_model_uuid", - "juju_application", - "juju_charm", - "juju_unit", - ]: - if label in rule["labels"]: - topology[label] = rule["labels"][label] - - rule["expr"] = self.inject_label_matchers(rule["expr"], topology) - return rules + Returns: + A tuple with (optionally) the values of the endpoints and the certificate path. - def validate_alert_rules(self, rules: dict) -> Tuple[bool, str]: - """Will validate correctness of alert rules, returning a boolean and any errors.""" - if not self.path: - logger.debug("`cos-tool` unavailable. Not validating alert correctness.") - return True, "" - - with tempfile.TemporaryDirectory() as tmpdir: - rule_path = Path(tmpdir + "/validate_rule.yaml") - - # Smash "our" rules format into what upstream actually uses, which is more like: - # - # groups: - # - name: foo - # rules: - # - alert: SomeAlert - # expr: up - # - alert: OtherAlert - # expr: up - transformed_rules = {"groups": []} # type: ignore - for rule in rules["groups"]: - transformed_rules["groups"].append(rule) - - rule_path.write_text(yaml.dump(transformed_rules)) - args = [str(self.path), "--format", "logql", "validate", str(rule_path)] - # noinspection PyBroadException - try: - self._exec(args) - return True, "" - except subprocess.CalledProcessError as e: - logger.debug("Validating the rules failed: %s", e.output) - return False, ", ".join([line for line in e.output if "error validating" in line]) - - def inject_label_matchers(self, expression, topology) -> str: - """Add label matchers to an expression.""" - if not topology: - return expression - if not self.path: - logger.debug("`cos-tool` unavailable. Leaving expression unchanged: %s", expression) - return expression - args = [str(self.path), "--format", "logql", "transform"] - args.extend( - ["--label-matcher={}={}".format(key, value) for key, value in topology.items()] - ) + Raises: + LokiPushApiError: if some endpoint are http and others https. + """ + endpoints = [ep["url"] for ep in endpoint_requirer.loki_endpoints] + if not endpoints: + return None, None - args.extend(["{}".format(expression)]) - # noinspection PyBroadException - try: - return self._exec(args) - except subprocess.CalledProcessError as e: - logger.debug('Applying the expression failed: "%s", falling back to the original', e) - print('Applying the expression failed: "{}", falling back to the original'.format(e)) - return expression - - def _get_tool_path(self) -> Optional[Path]: - arch = platform.processor() - arch = "amd64" if arch == "x86_64" else arch - res = "cos-tool-{}".format(arch) - try: - path = Path(res).resolve() - path.chmod(0o777) - return path - except NotImplementedError: - logger.debug("System lacks support for chmod") - except FileNotFoundError: - logger.debug('Could not locate cos-tool at: "{}"'.format(res)) - return None - - def _exec(self, cmd) -> str: - result = subprocess.run(cmd, check=True, stdout=subprocess.PIPE) - output = result.stdout.decode("utf-8").strip() - return output + https = tuple(endpoint.startswith("https://") for endpoint in endpoints) + + if all(https): # all endpoints are https + if cert_path is None: + raise LokiPushApiError("Cannot send logs to https endpoints without a certificate.") + if not Path(cert_path).exists(): + # if endpoints is https BUT we don't have a server_cert yet: + # disable charm logging until we do to prevent tls errors + return None, None + return endpoints, str(cert_path) + + if all(not x for x in https): # all endpoints are http + return endpoints, None + + # if there's a disagreement, that's very weird: + raise LokiPushApiError("Some endpoints are http, some others are https. That's not good.") diff --git a/lib/charms/observability_libs/v0/juju_topology.py b/lib/charms/observability_libs/v0/juju_topology.py index a79e5d434..55969d643 100644 --- a/lib/charms/observability_libs/v0/juju_topology.py +++ b/lib/charms/observability_libs/v0/juju_topology.py @@ -67,6 +67,8 @@ ``` """ + +import warnings from collections import OrderedDict from typing import Dict, List, Optional from uuid import UUID @@ -75,7 +77,7 @@ LIBID = "bced1658f20f49d28b88f61f83c2d232" LIBAPI = 0 -LIBPATCH = 6 +LIBPATCH = 7 class InvalidUUIDError(Exception): @@ -119,6 +121,13 @@ def __init__( unit: a unit name as a string charm_name: name of charm as a string """ + warnings.warn( + """ + observability_libs.v0.juju_topology is deprecated. Please import the + library from `cosl` instead: https://github.com/canonical/cos-lib + """, + DeprecationWarning, + ) if not self.is_valid_uuid(model_uuid): raise InvalidUUIDError(model_uuid) @@ -215,7 +224,8 @@ def as_dict( if remapped_keys: ret = OrderedDict( - (remapped_keys.get(k), v) if remapped_keys.get(k) else (k, v) for k, v in ret.items() # type: ignore + (remapped_keys.get(k), v) if remapped_keys.get(k) else (k, v) + for k, v in ret.items() # type: ignore ) return ret diff --git a/lib/charms/prometheus_k8s/v0/prometheus_scrape.py b/lib/charms/prometheus_k8s/v0/prometheus_scrape.py index e3d35c6f3..358c08de4 100644 --- a/lib/charms/prometheus_k8s/v0/prometheus_scrape.py +++ b/lib/charms/prometheus_k8s/v0/prometheus_scrape.py @@ -335,12 +335,13 @@ def _on_scrape_targets_changed(self, event): import tempfile from collections import defaultdict from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Callable, Dict, List, Literal, Optional, Tuple, Union from urllib.parse import urlparse import yaml -from cosl import JujuTopology -from cosl.rules import AlertRules +from cosl import CosTool, JujuTopology +from cosl.rules import AlertRules, generic_alert_groups +from cosl.types import OfficialRuleFileFormat from ops.charm import CharmBase, RelationRole from ops.framework import ( BoundEvent, @@ -350,7 +351,6 @@ def _on_scrape_targets_changed(self, event): ObjectEvents, StoredDict, StoredList, - StoredState, ) from ops.model import Relation @@ -362,9 +362,10 @@ def _on_scrape_targets_changed(self, event): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 47 +LIBPATCH = 65 -PYDEPS = ["cosl"] +# Version 0.0.53 needed for cosl.rules.generic_alert_groups +PYDEPS = ["cosl>=0.0.53"] logger = logging.getLogger(__name__) @@ -399,6 +400,14 @@ def _on_scrape_targets_changed(self, event): DEFAULT_ALERT_RULES_RELATIVE_PATH = "./src/prometheus_alert_rules" +FallbackScrapeProtocol = Literal[ + "PrometheusProto", + "OpenMetricsText0.0.1", + "OpenMetricsText1.0.0", + "PrometheusText0.0.4", + "PrometheusText1.0.0", +] + class PrometheusConfig: """A namespace for utility functions for manipulating the prometheus config dict.""" @@ -461,22 +470,156 @@ def prefix_job_names(scrape_configs: List[dict], prefix: str) -> List[dict]: return modified_scrape_configs + @staticmethod + def _build_host_to_unit( + hosts: Dict[str, Tuple[str, str, str]], + topology: Optional[JujuTopology], + ) -> Dict[str, str]: + """Build a reverse lookup dict: {address: unit_name, fqdn: unit_name, ...}. + + Maps each known unit identifier (IP address and/or FQDN) to its unit name, + so that non-wildcard targets can be matched whether specified as IP or FQDN. + + Returns an empty dict when ``topology`` is None, since matching only serves + the purpose of injecting ``juju_unit`` labels. + + The set subtraction ``{addr, fqdn} - {""}`` drops empty strings (absent FQDN, + e.g. when external_url is set) and deduplicates when addr == fqdn (non-IP + bind address). + """ + if not topology: + return {} + return { + identifier: unit_name + for unit_name, (addr, _, fqdn) in hosts.items() + for identifier in {addr, fqdn} - {""} + } + + @staticmethod + def _classify_targets(targets: List[str]) -> Tuple[List[str], List[str]]: + """Split a list of targets into wildcard and non-wildcard targets. + + Returns: + A ``(wildcard_targets, non_wildcard_targets)`` tuple. + """ + wildcard_targets = [] + non_wildcard_targets = [] + wildcard_re = re.compile(r"\*(?:(:\d+))?") + for target in targets: + if wildcard_re.match(target): + wildcard_targets.append(target) + else: + non_wildcard_targets.append(target) + return wildcard_targets, non_wildcard_targets + + @staticmethod + def _match_non_wildcard_targets( + targets: List[str], + host_to_unit: Dict[str, str], + ) -> Tuple[Dict[str, List[str]], List[str]]: + """Match non-wildcard targets against known unit addresses. + + Parses the host portion of each target (handling IPv6 bracket notation) and + looks it up in ``host_to_unit``. + + Returns: + A ``(matched_by_unit, unmatched_targets)`` tuple where ``matched_by_unit`` + maps each matched unit name to the list of targets belonging to it, and + ``unmatched_targets`` contains targets with no unit match. + """ + matched_by_unit: Dict[str, List[str]] = {} + unmatched_targets: List[str] = [] + for target in targets: + # urlparse correctly handles IPv6 (e.g. [::1]:9093), host:port, and + # bare hostnames — unlike a naive split(":")[0]. + parsed = urlparse(f"//{target}") + target_host = parsed.hostname or target.split(":", 1)[0] + matched_unit = host_to_unit.get(target_host) + if matched_unit: + matched_by_unit.setdefault(matched_unit, []).append(target) + else: + unmatched_targets.append(target) + return matched_by_unit, unmatched_targets + + @staticmethod + def _build_per_unit_job( + job: dict, + static_config: dict, + targets: List[str], + unit_name: str, + unit_path: str, + topology: Optional[JujuTopology], + ) -> dict: + """Build a single per-unit scrape job with topology labels and relabeling rules. + + Used for both wildcard and matched non-wildcard targets to avoid duplication. + + Args: + job: the original scrape job dict to base the new job on. + static_config: the original static_config dict to copy labels from. + targets: the resolved target addresses for this unit. + unit_name: the Juju unit name (e.g. "alertmanager/0"). + unit_path: path prefix to prepend to the metrics path (from external URL, may be ""). + topology: optional topology for adding Juju labels. + + Returns: + A new scrape job dict for this unit. + """ + unit_num = unit_name.split("/")[-1] + new_static = static_config.copy() + new_static["targets"] = targets + new_job = job.copy() + new_job["job_name"] = new_job.get("job_name", "unnamed-job") + "-" + unit_num + new_job["metrics_path"] = unit_path + (new_job.get("metrics_path") or "/metrics") + if topology: + new_static["labels"] = { + **topology.label_matcher_dict, + "juju_unit": unit_name, + **new_static.get("labels", {}), + } + # Instance relabeling for topology should be last in order. + new_job["relabel_configs"] = new_job.get("relabel_configs", []) + [ + PrometheusConfig.topology_relabel_config_wildcard + ] + new_job["static_configs"] = [new_static] + return new_job + @staticmethod def expand_wildcard_targets_into_individual_jobs( scrape_jobs: List[dict], - hosts: Dict[str, Tuple[str, str]], + hosts: Dict[str, Tuple[str, str, str]], topology: Optional[JujuTopology] = None, ) -> List[dict]: """Extract wildcard hosts from the given scrape_configs list into separate jobs. + For wildcard targets (e.g. "*:9093"), one job per unit is created. When + ``topology`` is provided, the ``juju_unit`` label is injected into each + per-unit job; without ``topology`` the per-unit jobs are created but no + topology labels are added. + + For non-wildcard targets (fully qualified hostnames/IPs), the host portion of + each target is matched against the known unit addresses in ``hosts``. Targets + whose address matches a known unit are expanded into a per-unit job (with + ``juju_unit`` when ``topology`` is provided), mirroring the wildcard behaviour. + Targets with no match (e.g. external services) are kept in a single job without + ``juju_unit``, preserving the previous behaviour. + Args: scrape_jobs: list of scrape jobs. - hosts: a dictionary mapping host names to host address for - all units of the relation for which this job configuration - must be constructed. + hosts: a dictionary mapping unit names to ``(address, path, fqdn)`` tuples for + all units of the relation for which this job configuration must be + constructed. topology: optional arg for adding topology labels to scrape targets. + When ``None``, wildcard targets are still expanded into per-unit jobs but + no ``juju_unit`` or topology labels are added. Non-wildcard target matching + is skipped entirely (all non-wildcard targets are kept in a single job), + since matching only serves the purpose of injecting ``juju_unit`` labels. """ - # hosts = self._relation_hosts(relation) + # Build a reverse lookup: {address: unit_name, fqdn: unit_name, ...} + # so that non-wildcard targets can be matched whether specified as IP or FQDN. + # The set subtraction {addr, fqdn} - {""} drops empty strings (absent FQDN) + # and deduplicates when addr == fqdn (non-IP bind address). + host_to_unit = PrometheusConfig._build_host_to_unit(hosts, topology) modified_scrape_jobs = [] for job in scrape_jobs: @@ -484,84 +627,66 @@ def expand_wildcard_targets_into_individual_jobs( if not static_configs: continue - # When a single unit specified more than one wildcard target, then they are expanded - # into a static_config per target - non_wildcard_static_configs = [] + # Accumulates non-wildcard targets that could not be matched to any known unit. + # These are kept in a single job with topology-only labels (no juju_unit): + # fully-qualified targets that predate this feature are unaffected. + unmatched_static_configs = [] for static_config in static_configs: targets = static_config.get("targets") if not targets: continue - # All non-wildcard targets remain in the same static_config - non_wildcard_targets = [] - - # All wildcard targets are extracted to a job per unit. If multiple wildcard - # targets are specified, they remain in the same static_config (per unit). - wildcard_targets = [] - - for target in targets: - match = re.compile(r"\*(?:(:\d+))?").match(target) - if match: - # This is a wildcard target. - # Need to expand into separate jobs and remove it from this job here - wildcard_targets.append(target) - else: - # This is not a wildcard target. Copy it over into its own static_config. - non_wildcard_targets.append(target) + wildcard_targets, non_wildcard_targets = PrometheusConfig._classify_targets( + targets + ) - # All non-wildcard targets remain in the same static_config + # Non-wildcard targets: try to match each target's host against known unit + # addresses. Matched targets get a per-unit job with juju_unit; unmatched + # targets get topology-only labels with no per-unit expansion. if non_wildcard_targets: - non_wildcard_static_config = static_config.copy() - non_wildcard_static_config["targets"] = non_wildcard_targets - - if topology: - # When non-wildcard targets (aka fully qualified hostnames) are specified, - # there is no reliable way to determine the name (Juju topology unit name) - # for such a target. Therefore labeling with Juju topology, excluding the - # unit name. - non_wildcard_static_config["labels"] = { - **topology.label_matcher_dict, - **non_wildcard_static_config.get("labels", {}), - } - - non_wildcard_static_configs.append(non_wildcard_static_config) - - # Extract wildcard targets into individual jobs - if wildcard_targets: - for unit_name, (unit_hostname, unit_path) in hosts.items(): - modified_job = job.copy() - modified_job["static_configs"] = [static_config.copy()] - modified_static_config = modified_job["static_configs"][0] - modified_static_config["targets"] = [ - target.replace("*", unit_hostname) for target in wildcard_targets - ] - - unit_num = unit_name.split("/")[-1] - job_name = modified_job.get("job_name", "unnamed-job") + "-" + unit_num - modified_job["job_name"] = job_name - modified_job["metrics_path"] = unit_path + ( - job.get("metrics_path") or "/metrics" + matched_by_unit, unmatched_targets = ( + PrometheusConfig._match_non_wildcard_targets( + non_wildcard_targets, host_to_unit ) + ) + # Unmatched targets: no unit mapping found — kept with topology-only + # labels and no per-unit expansion (juju_unit is not added). + if unmatched_targets: + unmatched_static_config = static_config.copy() + unmatched_static_config["targets"] = unmatched_targets if topology: - # Add topology labels - modified_static_config["labels"] = { + unmatched_static_config["labels"] = { **topology.label_matcher_dict, - **{"juju_unit": unit_name}, - **modified_static_config.get("labels", {}), + **unmatched_static_config.get("labels", {}), } + unmatched_static_configs.append(unmatched_static_config) + + # Matched targets: one per-unit job with juju_unit label. + for unit_name, unit_targets_list in matched_by_unit.items(): + _, unit_path, _ = hosts.get(unit_name, ("", "", "")) + modified_scrape_jobs.append( + PrometheusConfig._build_per_unit_job( + job, static_config, unit_targets_list, unit_name, unit_path, topology + ) + ) - # Instance relabeling for topology should be last in order. - modified_job["relabel_configs"] = modified_job.get( - "relabel_configs", [] - ) + [PrometheusConfig.topology_relabel_config_wildcard] - - modified_scrape_jobs.append(modified_job) + # Wildcard targets: one per-unit job per host, replacing "*" with the unit address. + if wildcard_targets: + for unit_name, (unit_hostname, unit_path, _unit_fqdn) in hosts.items(): + resolved_targets = [ + target.replace("*", unit_hostname) for target in wildcard_targets + ] + modified_scrape_jobs.append( + PrometheusConfig._build_per_unit_job( + job, static_config, resolved_targets, unit_name, unit_path, topology + ) + ) - if non_wildcard_static_configs: + if unmatched_static_configs: modified_job = job.copy() - modified_job["static_configs"] = non_wildcard_static_configs + modified_job["static_configs"] = unmatched_static_configs modified_job["metrics_path"] = modified_job.get("metrics_path") or "/metrics" if topology: @@ -719,7 +844,7 @@ def _type_convert_stored(obj): if isinstance(obj, StoredList): return list(map(_type_convert_stored, obj)) if isinstance(obj, StoredDict): - rdict = {} # type: Dict[Any, Any] + rdict = {} for k in obj.keys(): rdict[k] = _type_convert_stored(obj[k]) return rdict @@ -796,43 +921,6 @@ def __init__( super().__init__(self.message) -def _is_official_alert_rule_format(rules_dict: dict) -> bool: - """Are alert rules in the upstream format as supported by Prometheus. - - Alert rules in dictionary format are in "official" form if they - contain a "groups" key, since this implies they contain a list of - alert rule groups. - - Args: - rules_dict: a set of alert rules in Python dictionary format - - Returns: - True if alert rules are in official Prometheus file format. - """ - return "groups" in rules_dict - - -def _is_single_alert_rule_format(rules_dict: dict) -> bool: - """Are alert rules in single rule format. - - The Prometheus charm library supports reading of alert rules in a - custom format that consists of a single alert rule per file. This - does not conform to the official Prometheus alert rule file format - which requires that each alert rules file consists of a list of - alert rule groups and each group consists of a list of alert - rules. - - Alert rules in dictionary form are considered to be in single rule - format if in the least it contains two keys corresponding to the - alert rule name and alert expression. - - Returns: - True if alert rule is in single rule file format. - """ - # one alert rule per file - return set(rules_dict) >= {"alert", "expr"} - - class TargetsChangedEvent(EventBase): """Event emitted when Prometheus scrape targets change.""" @@ -860,7 +948,12 @@ class MetricsEndpointConsumer(Object): on = MonitoringEvents() # pyright: ignore - def __init__(self, charm: CharmBase, relation_name: str = DEFAULT_RELATION_NAME): + def __init__( + self, + charm: CharmBase, + relation_name: str = DEFAULT_RELATION_NAME, + fallback_scrape_protocol: Optional[FallbackScrapeProtocol] = None, + ): """A Prometheus based Monitoring service. Args: @@ -871,6 +964,17 @@ def __init__(self, charm: CharmBase, relation_name: str = DEFAULT_RELATION_NAME) It is strongly advised not to change the default, so that people deploying your charm will have a consistent experience with all other charms that consume metrics endpoints. + fallback_scrape_protocol: an optional fallback protocol to use when the + Content-Type header of a scrape response is missing or invalid. Supported + values: "PrometheusProto", "OpenMetricsText0.0.1", "OpenMetricsText1.0.0", + "PrometheusText0.0.4", "PrometheusText1.0.0". Ref: + https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config. + This had to be added after we bumped to Prometheus workload major version 3. Starting in major 3, + Prometheus no longer defaults to the Prometheus text format (PrometheusText0.0.4) + when the Content-Type header is missing or invalid, and instead fails the scrape with an error. + This parameter should only be used by MetricsEndpointConsumers that use Prometheus 3 and above, as setting + this key in the scrape configs of Prometheus 2 will result in the error: + "field fallback_scrape_protocol not found in type config.ScrapeConfig". Raises: RelationNotFoundError: If there is no relation in the charm's metadata.yaml @@ -889,12 +993,17 @@ def __init__(self, charm: CharmBase, relation_name: str = DEFAULT_RELATION_NAME) super().__init__(charm, relation_name) self._charm = charm self._relation_name = relation_name - self._tool = CosTool(self._charm) + self._fallback_scrape_protocol = fallback_scrape_protocol + self._tool = CosTool("promql") events = self._charm.on[relation_name] self.framework.observe(events.relation_changed, self._on_metrics_provider_relation_changed) self.framework.observe( events.relation_departed, self._on_metrics_provider_relation_departed ) + self.framework.observe( + events.relation_broken, self._on_metrics_provider_relation_departed + ) + def _on_metrics_provider_relation_changed(self, event): """Handle changes with related metrics providers. @@ -944,13 +1053,18 @@ def jobs(self) -> list: # Therefore we need to dedupe here and after all jobs are collected. static_scrape_jobs = _dedupe_job_names(static_scrape_jobs) try: - self._tool.validate_scrape_jobs(static_scrape_jobs) + _validate_scrape_jobs(static_scrape_jobs) except subprocess.CalledProcessError as e: + logger.error(f"Invalid scrape job file: {e}") if self._charm.unit.is_leader(): data = json.loads(relation.data[self._charm.app].get("event", "{}")) data["scrape_job_errors"] = str(e) relation.data[self._charm.app]["event"] = json.dumps(data) else: + if self._charm.unit.is_leader(): + data = json.loads(relation.data[self._charm.app].get("event", "{}")) + data.pop("scrape_job_errors", None) + relation.data[self._charm.app]["event"] = json.dumps(data) scrape_jobs.extend(static_scrape_jobs) scrape_jobs = _dedupe_job_names(scrape_jobs) @@ -999,7 +1113,7 @@ def alerts(self) -> dict: A dictionary mapping the Juju topology identifier of the source charm to its list of alert rule groups. """ - alerts = {} # type: Dict[str, dict] # mapping b/w juju identifiers and alert rule files + alerts: Dict[str, OfficialRuleFileFormat] = {} for relation in self._charm.model.relations[self._relation_name]: if not relation.units or not relation.app: continue @@ -1037,6 +1151,7 @@ def alerts(self) -> dict: _, errmsg = self._tool.validate_alert_rules(alert_rules) if errmsg: + logger.error(f"Invalid alert rule file: {errmsg}") if alerts[identifier]: del alerts[identifier] if self._charm.unit.is_leader(): @@ -1044,11 +1159,15 @@ def alerts(self) -> dict: data["errors"] = errmsg relation.data[self._charm.app]["event"] = json.dumps(data) continue + if self._charm.unit.is_leader(): + data = json.loads(relation.data[self._charm.app].get("event", "{}")) + data.pop("errors", None) + relation.data[self._charm.app]["event"] = json.dumps(data) return alerts def _get_identifier_by_alert_rules( - self, rules: dict + self, rules: OfficialRuleFileFormat ) -> Tuple[Union[str, None], Union[JujuTopology, None]]: """Determine an appropriate dict key for alert rules. @@ -1068,7 +1187,9 @@ def _get_identifier_by_alert_rules( # Construct an ID based on what's in the alert rules if they have labels for group in rules["groups"]: try: - labels = group["rules"][0]["labels"] + labels = group["rules"][0].get("labels") + if not labels: + continue topology = JujuTopology( # Don't try to safely get required constructor fields. There's already # a handler for KeyErrors @@ -1095,7 +1216,7 @@ def _get_identifier_by_alert_rules( return None, None - def _inject_alert_expr_labels(self, rules: Dict[str, Any]) -> Dict[str, Any]: + def _inject_alert_expr_labels(self, rules: OfficialRuleFileFormat) -> OfficialRuleFileFormat: """Iterate through alert rules and inject topology into expressions. Args: @@ -1182,21 +1303,43 @@ def _static_scrape_config(self, relation) -> list: # For https scrape targets we still do not render a `tls_config` section because certs # are expected to be made available by the charm via the `update-ca-certificates` mechanism. + + if self._fallback_scrape_protocol: + for job in scrape_configs: + job["fallback_scrape_protocol"] = self._fallback_scrape_protocol + return scrape_configs - def _relation_hosts(self, relation: Relation) -> Dict[str, Tuple[str, str]]: - """Returns a mapping from unit names to (address, path) tuples, for the given relation.""" + def _relation_hosts(self, relation: Relation) -> Dict[str, Tuple[str, str, str]]: + """Returns a mapping from unit names to (address, path, fqdn) tuples. + + Args: + relation: the relation to read unit data from. + + Returns: + A dict mapping each unit name to a ``(address, path, fqdn)`` tuple. The + ``fqdn`` element may be an empty string when the FQDN is not known. When + present, it may either be distinct from, or equal to ``address``. For + example, when the unit address itself is already a hostname. + """ hosts = {} for unit in relation.units: + if not (unit_databag := relation.data.get(unit)): + continue + + unit_path = unit_databag.get("prometheus_scrape_unit_path", "") # TODO deprecate and remove unit.name - unit_name = relation.data[unit].get("prometheus_scrape_unit_name") or unit.name + unit_name = unit_databag.get("prometheus_scrape_unit_name") or unit.name # TODO deprecate and remove "prometheus_scrape_host" - unit_address = relation.data[unit].get( - "prometheus_scrape_unit_address" - ) or relation.data[unit].get("prometheus_scrape_host") - unit_path = relation.data[unit].get("prometheus_scrape_unit_path", "") - if unit_name and unit_address: - hosts.update({unit_name: (unit_address, unit_path)}) + unit_address = unit_databag.get("prometheus_scrape_unit_address") or unit_databag.get( + "prometheus_scrape_host" + ) + unit_fqdn = unit_databag.get("prometheus_scrape_unit_fqdn", "") + + if not (unit_name and unit_address): + continue + + hosts.update({unit_name: (unit_address, unit_path, unit_fqdn)}) return hosts @@ -1220,6 +1363,102 @@ def _target_parts(self, target) -> list: return parts + def has_invalid_scrape_jobs(self) -> bool: + """Check whether any relation reported invalid scrape jobs. + + Validation errors, written to relation app data by this consumer + (see :meth:`jobs`), are read back to determine whether the relationship + currently carries an invalid scrape job. + + Returns: + True if any related metrics provider reported scrape job validation + errors, False otherwise. + """ + return self._has_relation_error("scrape_job_errors", "Scrape job validation error") + + def has_invalid_alert_rules(self) -> bool: + """Check whether any relation reported invalid alert rules. + + Validation errors, written to relation app data by this consumer + (see :attr:`alerts`), are read back to determine whether the relationship + currently carries invalid alert rules. + + Returns: + True if any related metrics provider reported alert rule validation + errors, False otherwise. + """ + return self._has_relation_error("errors", "Alert rule validation error") + + def _has_relation_error(self, error_key: str, error_label: str) -> bool: + """Check whether any relation reported the given validation error. + + Args: + error_key: the relation app data key that holds the validation error, + i.e. "scrape_job_errors" or "errors". + error_label: a human readable description of the validation error + type, used for logging, e.g. "Scrape job validation error". + + Returns: + True if any related metrics provider reported the validation error, + False otherwise. + """ + if not self._charm.unit.is_leader(): + return False + + for relation in self._charm.model.relations.get(self._relation_name, []): + app_data = relation.data.get(self._charm.app) + if not app_data: + continue + + event_raw = app_data.get("event", "{}") + try: + event_data = json.loads(event_raw) + except (json.JSONDecodeError, TypeError): + continue + + if error_msg := event_data.get(error_key): + logger.error("%s on relation %s: %s", error_label, relation.id, error_msg) + return True + + return False + + +def _validate_scrape_jobs(jobs: list) -> bool: + """Validate scrape jobs using cos-tool. + + Args: + jobs: A list of Prometheus scrape job dicts to validate. + + Returns: + True if validation passed or cos-tool is unavailable. + + Raises: + subprocess.CalledProcessError: if cos-tool rejects the scrape jobs. + """ + arch = platform.machine() + arch = "amd64" if arch == "x86_64" else arch + cos_tool_path = Path("cos-tool-{}".format(arch)) + try: + cos_tool_path = cos_tool_path.resolve(strict=True) + except (FileNotFoundError, OSError): + logger.debug("cos-tool unavailable. Not validating scrape jobs.") + return True + + conf = {"scrape_configs": jobs} + with tempfile.NamedTemporaryFile(suffix=".yaml", mode="w", delete=False) as tmpfile: + tmpfile.write(yaml.safe_dump(conf)) + tmpfile_name = tmpfile.name + try: + subprocess.run( + [str(cos_tool_path), "validate-config", tmpfile_name], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + finally: + Path(tmpfile_name).unlink(missing_ok=True) + return True + def _dedupe_job_names(jobs: List[dict]): """Deduplicate a list of dicts by appending a hash to the value of the 'job_name' key. @@ -1309,6 +1548,8 @@ def __init__( refresh_event: Optional[Union[BoundEvent, List[BoundEvent]]] = None, external_url: str = "", lookaside_jobs_callable: Optional[Callable] = None, + *, + forward_alert_rules: bool = True, ): """Construct a metrics provider for a Prometheus charm. @@ -1411,6 +1652,7 @@ def __init__( files. Defaults to "./prometheus_alert_rules", resolved relative to the directory hosting the charm entry file. The alert rules are automatically updated on charm upgrade. + forward_alert_rules: a boolean flag to toggle forwarding of charmed alert rules. refresh_event: an optional bound event or list of bound events which will be observed to re-set scrape job data (IP address and others) external_url: an optional argument that represents an external url that @@ -1449,6 +1691,7 @@ def __init__( self._charm = charm self._alert_rules_path = alert_rules_path + self._forward_alert_rules = forward_alert_rules self._relation_name = relation_name # sanitize job configurations to the supported subset of parameters jobs = [] if jobs is None else jobs @@ -1492,8 +1735,21 @@ def __init__( for ev in refresh_event: self.framework.observe(ev, self.set_scrape_job_spec) + # Always re-evaluate the unit address on `update_status`, regardless of the charm type + # (sidecar/pebble or podspec) or any user-provided `refresh_event`. On Kubernetes a pod + # can be rescheduled (e.g. node reboot/maintenance) and come back with a new IP without + # re-emitting `relation_joined`/`pebble_ready`. Without this, the stale address lingers in + # relation data and the consumer keeps scraping a dead IP until the relation is recreated. + # `update_status` fires periodically, so the address self-heals within one hook interval. + # See https://github.com/canonical/opentelemetry-collector-k8s-operator/issues/270 + self.framework.observe(self._charm.on.update_status, self._set_unit_ip) + def _on_relation_changed(self, event): """Check for alert rule messages in the relation data before moving on.""" + # Refresh the unit address on every `relation_changed`. This reacts faster than waiting for + # the next `update_status` when the pod IP changes, and is safe to call repeatedly. + self._set_unit_ip() + if self._charm.unit.is_leader(): ev = json.loads(event.relation.data[event.app].get("event", "{}")) @@ -1530,7 +1786,12 @@ def set_scrape_job_spec(self, _=None): return alert_rules = AlertRules(query_type="promql", topology=self.topology) - alert_rules.add_path(self._alert_rules_path, recursive=True) + if self._forward_alert_rules: + alert_rules.add_path(self._alert_rules_path, recursive=True) + alert_rules.add( + copy.deepcopy(generic_alert_groups.application_rules), + group_name_prefix=self.topology.identifier, + ) alert_rules_as_dict = alert_rules.as_dict() for relation in self._charm.model.relations[self._relation_name]: @@ -1562,18 +1823,22 @@ def _set_unit_ip(self, _=None): parsed = urlparse(self.external_url) unit_address = parsed.hostname path = parsed.path + unit_fqdn = "" elif self._is_valid_unit_address(unit_ip): unit_address = unit_ip + unit_fqdn = socket.getfqdn() path = "" else: unit_address = socket.getfqdn() + unit_fqdn = unit_address path = "" - relation.data[self._charm.unit]["prometheus_scrape_unit_address"] = unit_address - relation.data[self._charm.unit]["prometheus_scrape_unit_path"] = path - relation.data[self._charm.unit]["prometheus_scrape_unit_name"] = str( - self._charm.model.unit.name - ) + relation.data[self._charm.unit].update({ + "prometheus_scrape_unit_address": unit_address, + "prometheus_scrape_unit_path": path, + "prometheus_scrape_unit_name": str(self._charm.model.unit.name), + "prometheus_scrape_unit_fqdn": unit_fqdn, + }) def _is_valid_unit_address(self, address: str) -> bool: """Validate a unit address. @@ -1661,6 +1926,7 @@ def __init__( events.relation_changed, self._charm.on.leader_elected, self._charm.on.upgrade_charm, + self._charm.on.config_changed, ] for event_source in event_sources: @@ -1685,694 +1951,3 @@ def _update_relation_data(self, _): alert_rules_as_dict, sort_keys=True, # sort, to prevent unnecessary relation_changed events ) - - -class MetricsEndpointAggregator(Object): - """Aggregate metrics from multiple scrape targets. - - `MetricsEndpointAggregator` collects scrape target information from one - or more related charms and forwards this to a `MetricsEndpointConsumer` - charm, which may be in a different Juju model. However, it is - essential that `MetricsEndpointAggregator` itself resides in the same - model as its scrape targets, as this is currently the only way to - ensure in Juju that the `MetricsEndpointAggregator` will be able to - determine the model name and uuid of the scrape targets. - - `MetricsEndpointAggregator` should be used in place of - `MetricsEndpointProvider` in the following two use cases: - - 1. Integrating one or more scrape targets that do not support the - `prometheus_scrape` interface. - - 2. Integrating one or more scrape targets through cross model - relations. Although the [Scrape Config Operator](https://charmhub.io/cos-configuration-k8s) - may also be used for the purpose of supporting cross model - relations. - - Using `MetricsEndpointAggregator` to build a Prometheus charm client - only requires instantiating it. Instantiating - `MetricsEndpointAggregator` is similar to `MetricsEndpointProvider` except - that it requires specifying the names of three relations: the - relation with scrape targets, the relation for alert rules, and - that with the Prometheus charms. For example - - ```python - self._aggregator = MetricsEndpointAggregator( - self, - { - "prometheus": "monitoring", - "scrape_target": "prometheus-target", - "alert_rules": "prometheus-rules" - } - ) - ``` - - `MetricsEndpointAggregator` assumes that each unit of a scrape target - sets in its unit-level relation data two entries with keys - "hostname" and "port". If it is required to integrate with charms - that do not honor these assumptions, it is always possible to - derive from `MetricsEndpointAggregator` overriding the `_get_targets()` - method, which is responsible for aggregating the unit name, host - address ("hostname") and port of the scrape target. - `MetricsEndpointAggregator` also assumes that each unit of a - scrape target sets in its unit-level relation data a key named - "groups". The value of this key is expected to be the string - representation of list of Prometheus Alert rules in YAML format. - An example of a single such alert rule is - - ```yaml - - alert: HighRequestLatency - expr: job:request_latency_seconds:mean5m{job="myjob"} > 0.5 - for: 10m - labels: - severity: page - annotations: - summary: High request latency - ``` - - Once again if it is required to integrate with charms that do not - honour these assumptions about alert rules then an object derived - from `MetricsEndpointAggregator` may be used by overriding the - `_get_alert_rules()` method. - - `MetricsEndpointAggregator` ensures that Prometheus scrape job - specifications and alert rules are annotated with Juju topology - information, just like `MetricsEndpointProvider` and - `MetricsEndpointConsumer` do. - - By default, `MetricsEndpointAggregator` ensures that Prometheus - "instance" labels refer to Juju topology. This ensures that - instance labels are stable over unit recreation. While it is not - advisable to change this option, if required it can be done by - setting the "relabel_instance" keyword argument to `False` when - constructing an aggregator object. - """ - - _stored = StoredState() - - def __init__( - self, - charm, - relation_names: Optional[dict] = None, - relabel_instance=True, - resolve_addresses=False, - ): - """Construct a `MetricsEndpointAggregator`. - - Args: - charm: a `CharmBase` object that manages this - `MetricsEndpointAggregator` object. Typically, this is - `self` in the instantiating class. - relation_names: a dictionary with three keys. The value - of the "scrape_target" and "alert_rules" keys are - the relation names over which scrape job and alert rule - information is gathered by this `MetricsEndpointAggregator`. - And the value of the "prometheus" key is the name of - the relation with a `MetricsEndpointConsumer` such as - the Prometheus charm. - relabel_instance: A boolean flag indicating if Prometheus - scrape job "instance" labels must refer to Juju Topology. - resolve_addresses: A boolean flag indiccating if the aggregator - should attempt to perform DNS lookups of targets and append - a `dns_name` label - """ - self._charm = charm - - relation_names = relation_names or {} - - self._prometheus_relation = relation_names.get( - "prometheus", "downstream-prometheus-scrape" - ) - self._target_relation = relation_names.get("scrape_target", "prometheus-target") - self._alert_rules_relation = relation_names.get("alert_rules", "prometheus-rules") - - super().__init__(charm, self._prometheus_relation) - self._stored.set_default(jobs=[], alert_rules=[]) - - self._relabel_instance = relabel_instance - self._resolve_addresses = resolve_addresses - - # manage Prometheus charm relation events - prometheus_events = self._charm.on[self._prometheus_relation] - self.framework.observe(prometheus_events.relation_joined, self._set_prometheus_data) - - # manage list of Prometheus scrape jobs from related scrape targets - target_events = self._charm.on[self._target_relation] - self.framework.observe(target_events.relation_changed, self._on_prometheus_targets_changed) - self.framework.observe( - target_events.relation_departed, self._on_prometheus_targets_departed - ) - - # manage alert rules for Prometheus from related scrape targets - alert_rule_events = self._charm.on[self._alert_rules_relation] - self.framework.observe(alert_rule_events.relation_changed, self._on_alert_rules_changed) - self.framework.observe(alert_rule_events.relation_departed, self._on_alert_rules_departed) - - def _set_prometheus_data(self, event): - """Ensure every new Prometheus instances is updated. - - Any time a new Prometheus unit joins the relation with - `MetricsEndpointAggregator`, that Prometheus unit is provided - with the complete set of existing scrape jobs and alert rules. - """ - if not self._charm.unit.is_leader(): - return - - jobs = [] + _type_convert_stored( - self._stored.jobs # pyright: ignore - ) # list of scrape jobs, one per relation - for relation in self.model.relations[self._target_relation]: - targets = self._get_targets(relation) - if targets and relation.app: - jobs.append(self._static_scrape_job(targets, relation.app.name)) - - groups = [] + _type_convert_stored( - self._stored.alert_rules # pyright: ignore - ) # list of alert rule groups - for relation in self.model.relations[self._alert_rules_relation]: - unit_rules = self._get_alert_rules(relation) - if unit_rules and relation.app: - appname = relation.app.name - rules = self._label_alert_rules(unit_rules, appname) - group = {"name": self.group_name(appname), "rules": rules} - groups.append(group) - - event.relation.data[self._charm.app]["scrape_jobs"] = json.dumps(jobs) - event.relation.data[self._charm.app]["alert_rules"] = json.dumps({"groups": groups}) - - def _on_prometheus_targets_changed(self, event): - """Update scrape jobs in response to scrape target changes. - - When there is any change in relation data with any scrape - target, the Prometheus scrape job, for that specific target is - updated. - """ - targets = self._get_targets(event.relation) - if not targets: - return - - # new scrape job for the relation that has changed - self.set_target_job_data(targets, event.relation.app.name) - - def set_target_job_data(self, targets: dict, app_name: str, **kwargs) -> None: - """Update scrape jobs in response to scrape target changes. - - When there is any change in relation data with any scrape - target, the Prometheus scrape job, for that specific target is - updated. Additionally, if this method is called manually, do the - same. - - Args: - targets: a `dict` containing target information - app_name: a `str` identifying the application - kwargs: a `dict` of the extra arguments passed to the function - """ - if not self._charm.unit.is_leader(): - return - - # new scrape job for the relation that has changed - updated_job = self._static_scrape_job(targets, app_name, **kwargs) - - for relation in self.model.relations[self._prometheus_relation]: - jobs = json.loads(relation.data[self._charm.app].get("scrape_jobs", "[]")) - # list of scrape jobs that have not changed - jobs = [job for job in jobs if updated_job["job_name"] != job["job_name"]] - jobs.append(updated_job) - relation.data[self._charm.app]["scrape_jobs"] = json.dumps(jobs) - - if not _type_convert_stored(self._stored.jobs) == jobs: # pyright: ignore - self._stored.jobs = jobs - - def _on_prometheus_targets_departed(self, event): - """Remove scrape jobs when a target departs. - - Any time a scrape target departs, any Prometheus scrape job - associated with that specific scrape target is removed. - """ - job_name = self._job_name(event.relation.app.name) - unit_name = event.unit.name - self.remove_prometheus_jobs(job_name, unit_name) - - def remove_prometheus_jobs(self, job_name: str, unit_name: Optional[str] = ""): - """Given a job name and unit name, remove scrape jobs associated. - - The `unit_name` parameter is used for automatic, relation data bag-based - generation, where the unit name in labels can be used to ensure that jobs with - similar names (which are generated via the app name when scanning relation data - bags) are not accidentally removed, as their unit name labels will differ. - For NRPE, the job name is calculated from an ID sent via the NRPE relation, and is - sufficient to uniquely identify the target. - """ - if not self._charm.unit.is_leader(): - return - - for relation in self.model.relations[self._prometheus_relation]: - jobs = json.loads(relation.data[self._charm.app].get("scrape_jobs", "[]")) - if not jobs: - continue - - changed_job = [j for j in jobs if j.get("job_name") == job_name] - if not changed_job: - continue - changed_job = changed_job[0] - - # list of scrape jobs that have not changed - jobs = [job for job in jobs if job.get("job_name") != job_name] - - # list of scrape jobs for units of the same application that still exist - configs_kept = [ - config - for config in changed_job["static_configs"] # type: ignore - if config.get("labels", {}).get("juju_unit") != unit_name - ] - - if configs_kept: - changed_job["static_configs"] = configs_kept # type: ignore - jobs.append(changed_job) - - relation.data[self._charm.app]["scrape_jobs"] = json.dumps(jobs) - - if not _type_convert_stored(self._stored.jobs) == jobs: # pyright: ignore - self._stored.jobs = jobs - - def _job_name(self, appname) -> str: - """Construct a scrape job name. - - Each relation has its own unique scrape job name. All units in - the relation are scraped as part of the same scrape job. - - Args: - appname: string name of a related application. - - Returns: - a string Prometheus scrape job name for the application. - """ - return "juju_{}_{}_{}_prometheus_scrape".format( - self.model.name, self.model.uuid[:7], appname - ) - - def _get_targets(self, relation) -> dict: - """Fetch scrape targets for a relation. - - Scrape target information is returned for each unit in the - relation. This information contains the unit name, network - hostname (or address) for that unit, and port on which a - metrics endpoint is exposed in that unit. - - Args: - relation: an `ops.model.Relation` object for which scrape - targets are required. - - Returns: - a dictionary whose keys are names of the units in the - relation. There values associated with each key is itself - a dictionary of the form - ``` - {"hostname": hostname, "port": port} - ``` - """ - targets = {} - for unit in relation.units: - port = relation.data[unit].get("port", 80) - hostname = relation.data[unit].get("hostname") - if hostname: - targets.update({unit.name: {"hostname": hostname, "port": port}}) - - return targets - - def _static_scrape_job(self, targets, application_name, **kwargs) -> dict: - """Construct a static scrape job for an application. - - Args: - targets: a dictionary providing hostname and port for all - scrape target. The keys of this dictionary are unit - names. Values corresponding to these keys are - themselves a dictionary with keys "hostname" and - "port". - application_name: a string name of the application for - which this static scrape job is being constructed. - kwargs: a `dict` of the extra arguments passed to the function - - Returns: - A dictionary corresponding to a Prometheus static scrape - job configuration for one application. The returned - dictionary may be transformed into YAML and appended to - the list of any existing list of Prometheus static configs. - """ - juju_model = self.model.name - juju_model_uuid = self.model.uuid - - job = { - "job_name": self._job_name(application_name), - "static_configs": [ - { - "targets": ["{}:{}".format(target["hostname"], target["port"])], - "labels": { - "juju_model": juju_model, - "juju_model_uuid": juju_model_uuid, - "juju_application": application_name, - "juju_unit": unit_name, - "host": target["hostname"], - # Expanding this will merge the dicts and replace the - # topology labels if any were present/found - **self._static_config_extra_labels(target), - }, - } - for unit_name, target in targets.items() - ], - "relabel_configs": self._relabel_configs + kwargs.get("relabel_configs", []), - } - job.update(kwargs.get("updates", {})) - - return job - - def _static_config_extra_labels(self, target: Dict[str, str]) -> Dict[str, str]: - """Build a list of extra static config parameters, if specified.""" - extra_info = {} - - if self._resolve_addresses: - try: - dns_name = socket.gethostbyaddr(target["hostname"])[0] - except OSError: - logger.debug("Could not perform DNS lookup for %s", target["hostname"]) - dns_name = target["hostname"] - extra_info["dns_name"] = dns_name - - return extra_info - - @property - def _relabel_configs(self) -> list: - """Create Juju topology relabeling configuration. - - Using Juju topology for instance labels ensures that these - labels are stable across unit recreation. - - Returns: - a list of Prometheus relabeling configurations. Each item in - this list is one relabel configuration. - """ - return ( - [ - { - "source_labels": [ - "juju_model", - "juju_model_uuid", - "juju_application", - "juju_unit", - ], - "separator": "_", - "target_label": "instance", - "regex": "(.*)", - } - ] - if self._relabel_instance - else [] - ) - - def _on_alert_rules_changed(self, event): - """Update alert rules in response to scrape target changes. - - When there is any change in alert rule relation data for any - scrape target, the list of alert rules for that specific - target is updated. - """ - unit_rules = self._get_alert_rules(event.relation) - if not unit_rules: - return - - app_name = event.relation.app.name - self.set_alert_rule_data(app_name, unit_rules) - - def set_alert_rule_data(self, name: str, unit_rules: dict, label_rules: bool = True) -> None: - """Update alert rule data. - - The unit rules should be a dict, which is has additional Juju topology labels added. For - rules generated by the NRPE exporter, they are pre-labeled so lookups can be performed. - """ - if not self._charm.unit.is_leader(): - return - - if label_rules: - rules = self._label_alert_rules(unit_rules, name) - else: - rules = [unit_rules] - updated_group = {"name": self.group_name(name), "rules": rules} - - for relation in self.model.relations[self._prometheus_relation]: - alert_rules = json.loads(relation.data[self._charm.app].get("alert_rules", "{}")) - groups = alert_rules.get("groups", []) - # list of alert rule groups that have not changed - for group in groups: - if group["name"] == updated_group["name"]: - group["rules"] = [r for r in group["rules"] if r not in updated_group["rules"]] - group["rules"].extend(updated_group["rules"]) - - if updated_group["name"] not in [g["name"] for g in groups]: - groups.append(updated_group) - relation.data[self._charm.app]["alert_rules"] = json.dumps({"groups": groups}) - - if not _type_convert_stored(self._stored.alert_rules) == groups: # pyright: ignore - self._stored.alert_rules = groups - - def _on_alert_rules_departed(self, event): - """Remove alert rules for departed targets. - - Any time a scrape target departs any alert rules associated - with that specific scrape target is removed. - """ - group_name = self.group_name(event.relation.app.name) - unit_name = event.unit.name - self.remove_alert_rules(group_name, unit_name) - - def remove_alert_rules(self, group_name: str, unit_name: str) -> None: - """Remove an alert rule group from relation data.""" - if not self._charm.unit.is_leader(): - return - - for relation in self.model.relations[self._prometheus_relation]: - alert_rules = json.loads(relation.data[self._charm.app].get("alert_rules", "{}")) - if not alert_rules: - continue - - groups = alert_rules.get("groups", []) - if not groups: - continue - - changed_group = [group for group in groups if group["name"] == group_name] - if not changed_group: - continue - changed_group = changed_group[0] - - # list of alert rule groups that have not changed - groups = [group for group in groups if group["name"] != group_name] - - # list of alert rules not associated with departing unit - rules_kept = [ - rule - for rule in changed_group.get("rules") # type: ignore - if rule.get("labels").get("juju_unit") != unit_name - ] - - if rules_kept: - changed_group["rules"] = rules_kept # type: ignore - groups.append(changed_group) - - relation.data[self._charm.app]["alert_rules"] = ( - json.dumps({"groups": groups}) if groups else "{}" - ) - - if not _type_convert_stored(self._stored.alert_rules) == groups: # pyright: ignore - self._stored.alert_rules = groups - - def _get_alert_rules(self, relation) -> dict: - """Fetch alert rules for a relation. - - Each unit of the related scrape target may have its own - associated alert rules. Alert rules for all units are returned - indexed by unit name. - - Args: - relation: an `ops.model.Relation` object for which alert - rules are required. - - Returns: - a dictionary whose keys are names of the units in the - relation. There values associated with each key is a list - of alert rules. Each rule is in dictionary format. The - structure "rule dictionary" corresponds to single - Prometheus alert rule. - """ - rules = {} - for unit in relation.units: - unit_rules = yaml.safe_load(relation.data[unit].get("groups", "")) - if unit_rules: - rules.update({unit.name: unit_rules}) - - return rules - - def group_name(self, unit_name: str) -> str: - """Construct name for an alert rule group. - - Each unit in a relation may define its own alert rules. All - rules, for all units in a relation are grouped together and - given a single alert rule group name. - - Args: - unit_name: string name of a related application. - - Returns: - a string Prometheus alert rules group name for the unit. - """ - unit_name = re.sub(r"/", "_", unit_name) - return "juju_{}_{}_{}_alert_rules".format(self.model.name, self.model.uuid[:7], unit_name) - - def _label_alert_rules(self, unit_rules, app_name: str) -> list: - """Apply juju topology labels to alert rules. - - Args: - unit_rules: a list of alert rules, where each rule is in - dictionary format. - app_name: a string name of the application to which the - alert rules belong. - - Returns: - a list of alert rules with Juju topology labels. - """ - labeled_rules = [] - for unit_name, rules in unit_rules.items(): - for rule in rules: - # the new JujuTopology removed this, so build it up by hand - matchers = { - "juju_{}".format(k): v - for k, v in JujuTopology(self.model.name, self.model.uuid, app_name, unit_name) - .as_dict(excluded_keys=["charm_name"]) - .items() - } - rule["labels"].update(matchers.items()) - labeled_rules.append(rule) - - return labeled_rules - - -class CosTool: - """Uses cos-tool to inject label matchers into alert rule expressions and validate rules.""" - - _path = None - _disabled = False - - def __init__(self, charm): - self._charm = charm - - @property - def path(self): - """Lazy lookup of the path of cos-tool.""" - if self._disabled: - return None - if not self._path: - self._path = self._get_tool_path() - if not self._path: - logger.debug("Skipping injection of juju topology as label matchers") - self._disabled = True - return self._path - - def apply_label_matchers(self, rules) -> dict: - """Will apply label matchers to the expression of all alerts in all supplied groups.""" - if not self.path: - return rules - for group in rules["groups"]: - rules_in_group = group.get("rules", []) - for rule in rules_in_group: - topology = {} - # if the user for some reason has provided juju_unit, we'll need to honor it - # in most cases, however, this will be empty - for label in [ - "juju_model", - "juju_model_uuid", - "juju_application", - "juju_charm", - "juju_unit", - ]: - if label in rule["labels"]: - topology[label] = rule["labels"][label] - - rule["expr"] = self.inject_label_matchers(rule["expr"], topology) - return rules - - def validate_alert_rules(self, rules: dict) -> Tuple[bool, str]: - """Will validate correctness of alert rules, returning a boolean and any errors.""" - if not self.path: - logger.debug("`cos-tool` unavailable. Not validating alert correctness.") - return True, "" - - with tempfile.TemporaryDirectory() as tmpdir: - rule_path = Path(tmpdir + "/validate_rule.yaml") - rule_path.write_text(yaml.dump(rules)) - - args = [str(self.path), "validate", str(rule_path)] - # noinspection PyBroadException - try: - self._exec(args) - return True, "" - except subprocess.CalledProcessError as e: - logger.debug("Validating the rules failed: %s", e.output) - return False, ", ".join( - [ - line - for line in e.output.decode("utf8").splitlines() - if "error validating" in line - ] - ) - - def validate_scrape_jobs(self, jobs: list) -> bool: - """Validate scrape jobs using cos-tool.""" - if not self.path: - logger.debug("`cos-tool` unavailable. Not validating scrape jobs.") - return True - conf = {"scrape_configs": jobs} - with tempfile.NamedTemporaryFile() as tmpfile: - with open(tmpfile.name, "w") as f: - f.write(yaml.safe_dump(conf)) - try: - self._exec([str(self.path), "validate-config", tmpfile.name]) - except subprocess.CalledProcessError as e: - logger.error("Validating scrape jobs failed: {}".format(e.output)) - raise - return True - - def inject_label_matchers(self, expression, topology) -> str: - """Add label matchers to an expression.""" - if not topology: - return expression - if not self.path: - logger.debug("`cos-tool` unavailable. Leaving expression unchanged: %s", expression) - return expression - args = [str(self.path), "transform"] - args.extend( - ["--label-matcher={}={}".format(key, value) for key, value in topology.items()] - ) - - args.extend(["{}".format(expression)]) - # noinspection PyBroadException - try: - return self._exec(args) - except subprocess.CalledProcessError as e: - logger.debug('Applying the expression failed: "%s", falling back to the original', e) - return expression - - def _get_tool_path(self) -> Optional[Path]: - arch = platform.machine() - arch = "amd64" if arch == "x86_64" else arch - res = "cos-tool-{}".format(arch) - try: - path = Path(res).resolve() - path.chmod(0o777) - return path - except NotImplementedError: - logger.debug("System lacks support for chmod") - except FileNotFoundError: - logger.debug('Could not locate cos-tool at: "{}"'.format(res)) - return None - - def _exec(self, cmd) -> str: - result = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - return result.stdout.decode("utf-8").strip() diff --git a/lib/charms/redis_k8s/v0/redis.py b/lib/charms/redis_k8s/v0/redis.py index a73150737..e28b14c2d 100644 --- a/lib/charms/redis_k8s/v0/redis.py +++ b/lib/charms/redis_k8s/v0/redis.py @@ -39,7 +39,7 @@ # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version. -LIBPATCH = 6 +LIBPATCH = 7 logger = logging.getLogger(__name__) @@ -78,6 +78,18 @@ def _on_relation_broken(self, event): # Trigger an event that our charm can react to. self.charm.on.redis_relation_updated.emit() + @property + def app_data(self) -> Optional[Dict[str, str]]: + """Retrieve the app data. + + Returns: + Dict: dict containing the app data. + """ + relation = self.model.get_relation(self.relation_name) + if not relation: + return None + return relation.data[relation.app] + @property def relation_data(self) -> Optional[Dict[str, str]]: """Retrieve the relation data. @@ -98,10 +110,16 @@ def url(self) -> Optional[str]: Returns: str: the Redis URL. """ - relation_data = self.relation_data - if not relation_data: + if not (relation_data := self.relation_data): return None + redis_host = relation_data.get("hostname") + + if app_data := self.app_data: + try: + redis_host = self.app_data.get("leader-host", redis_host) + except KeyError: + pass redis_port = relation_data.get("port") return f"redis://{redis_host}:{redis_port}" diff --git a/lib/charms/saml_integrator/v0/saml.py b/lib/charms/saml_integrator/v0/saml.py index 8fd1610e3..28b808c88 100644 --- a/lib/charms/saml_integrator/v0/saml.py +++ b/lib/charms/saml_integrator/v0/saml.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2024 Canonical Ltd. +# Copyright 2025 Canonical Ltd. # Licensed under the Apache2.0. See LICENSE file in charm source for details. """Library to manage the relation data for the SAML Integrator charm. @@ -68,15 +68,15 @@ class method `from_relation_data`. # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 10 +LIBPATCH = 11 # pylint: disable=wrong-import-position +# ruff: noqa: E402 import re import typing import ops -from pydantic import AnyHttpUrl, BaseModel, Field -from pydantic.tools import parse_obj_as +from pydantic import AnyHttpUrl, BaseModel, Field, TypeAdapter DEFAULT_RELATION_NAME = "saml" @@ -92,9 +92,9 @@ class SamlEndpoint(BaseModel): """ name: str = Field(..., min_length=1) - url: typing.Optional[AnyHttpUrl] + url: typing.Optional[AnyHttpUrl] = None binding: str = Field(..., min_length=1) - response_url: typing.Optional[AnyHttpUrl] + response_url: typing.Optional[AnyHttpUrl] = None def to_relation_data(self) -> typing.Dict[str, str]: """Convert an instance of SamlEndpoint to the relation representation. @@ -139,13 +139,13 @@ def from_relation_data(cls, relation_data: typing.Dict[str, str]) -> "SamlEndpoi return cls( name=name, url=( - parse_obj_as(AnyHttpUrl, relation_data[f"{prefix}url"]) + TypeAdapter(AnyHttpUrl).validate_python(relation_data[f"{prefix}url"]) if relation_data[f"{prefix}url"] else None ), binding=relation_data[f"{prefix}binding"], response_url=( - parse_obj_as(AnyHttpUrl, relation_data[f"{prefix}response_url"]) + TypeAdapter(AnyHttpUrl).validate_python(relation_data[f"{prefix}response_url"]) if f"{prefix}response_url" in relation_data else None ), @@ -208,7 +208,7 @@ def from_relation_data(cls, relation_data: ops.RelationDataContent) -> "SamlRela return cls( entity_id=relation_data.get("entity_id"), # type: ignore metadata_url=( - parse_obj_as(AnyHttpUrl, relation_data.get("metadata_url")) + TypeAdapter(AnyHttpUrl).validate_python(relation_data.get("metadata_url")) if relation_data.get("metadata_url") else None ), # type: ignore @@ -231,7 +231,7 @@ class SamlDataAvailableEvent(ops.RelationEvent): @property def saml_relation_data(self) -> SamlRelationData: """Get a SamlRelationData for the relation data.""" - assert self.relation.app + assert self.relation.app # noqa: S101 return SamlRelationData.from_relation_data(self.relation.data[self.relation.app]) @property @@ -294,7 +294,7 @@ def _on_relation_changed(self, event: ops.RelationChangedEvent) -> None: Args: event: event triggering this handler. """ - assert event.relation.app + assert event.relation.app # noqa: S101 if event.relation.data[event.relation.app]: self.on.saml_data_available.emit(event.relation, app=event.app, unit=event.unit) diff --git a/lib/charms/smtp_integrator/v0/smtp.py b/lib/charms/smtp_integrator/v0/smtp.py index 2816965ef..8ab7f285e 100644 --- a/lib/charms/smtp_integrator/v0/smtp.py +++ b/lib/charms/smtp_integrator/v0/smtp.py @@ -1,4 +1,4 @@ -# Copyright 2024 Canonical Ltd. +# Copyright 2025 Canonical Ltd. # Licensed under the Apache2.0. See LICENSE file in charm source for details. """Library to manage the integration with the SMTP Integrator charm. @@ -68,27 +68,61 @@ def _on_config_changed(self, _) -> None: # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 11 +LIBPATCH = 21 -PYDEPS = ["pydantic>=2"] +PYDEPS = ["pydantic>=1.10,<3", "email-validator>=2"] # pylint: disable=wrong-import-position import itertools +import json import logging import typing from ast import literal_eval from enum import Enum -from typing import Dict, Optional +from typing import Any, Callable, Dict, List, Optional, TypeVar, cast import ops -from pydantic import BaseModel, Field, ValidationError +from pydantic import BaseModel, EmailStr, Field, ValidationError logger = logging.getLogger(__name__) +_F = TypeVar("_F", bound=Callable[..., Any]) + +try: + # Pydantic v2 + from pydantic import field_validator as _pyd_field_validator + + _PYDANTIC_V2 = True +except ImportError: + _pyd_field_validator = None # type: ignore[assignment] + _PYDANTIC_V2 = False + +# Pydantic v1 field validation decorator (v2 uses field_validator) +from pydantic import validator as _pyd_validator # type: ignore[attr-defined] + DEFAULT_RELATION_NAME = "smtp" LEGACY_RELATION_NAME = "smtp-legacy" +def recipients_validator() -> Callable[[_F], _F]: + """Return the correct recipients validator decorator for pydantic v1/v2. + + Returns: + A decorator to validate/normalize the recipients field before EmailStr validation. + """ + if _PYDANTIC_V2: + return cast(Any, _pyd_field_validator)("recipients", mode="before") + return cast(Any, _pyd_validator)("recipients", pre=True) + + +class SmtpError(Exception): + """Common ancestor for Smtp related exceptions.""" + + +class SecretError(SmtpError): + """Common ancestor for Secrets related exceptions.""" + + class TransportSecurity(str, Enum): """Represent the transport security values. @@ -130,10 +164,12 @@ class SmtpRelationData(BaseModel): transport_security: The security protocol to use for the outgoing SMTP relay. domain: The domain used by the emails sent from SMTP relay. skip_ssl_verify: Specifies if certificate trust verification is skipped in the SMTP relay. + smtp_sender: Optional sender email address for outgoing notifications. + recipients: List of recipient email addresses for notifications. """ host: str = Field(..., min_length=1) - port: int = Field(None, ge=1, le=65536) + port: int = Field(..., ge=1, le=65536) user: Optional[str] = None password: Optional[str] = None password_id: Optional[str] = None @@ -141,6 +177,14 @@ class SmtpRelationData(BaseModel): transport_security: TransportSecurity domain: Optional[str] = None skip_ssl_verify: Optional[bool] = False + smtp_sender: Optional[EmailStr] = None + recipients: List[EmailStr] = Field(default_factory=list) + + @recipients_validator() + @classmethod + def _recipients_str_to_list(cls, value: Any) -> Any: + """Convert recipients input to list[str] before EmailStr validation.""" + return parse_recipients(value) def to_relation_data(self) -> Dict[str, str]: """Convert an instance of SmtpRelationData to the relation representation. @@ -162,7 +206,18 @@ def to_relation_data(self) -> Dict[str, str]: if self.password: result["password"] = self.password if self.password_id: + if "password" in result: + logger.warning("password field exists along with password_id field, removing.") + del result["password"] result["password_id"] = self.password_id + + if self.smtp_sender: + result["smtp_sender"] = str(self.smtp_sender) + + if self.recipients: + recipients = list(self.recipients) + result["recipients"] = json.dumps([str(r) for r in recipients]) + return result @@ -179,6 +234,8 @@ class SmtpDataAvailableEvent(ops.RelationEvent): transport_security: The security protocol to use for the outgoing SMTP relay. domain: The domain used by the emails sent from SMTP relay. skip_ssl_verify: Specifies if certificate trust verification is skipped in the SMTP relay. + smtp_sender: Optional sender email address for outgoing notifications. + recipients: List of recipient email addresses for notifications. """ @property @@ -237,6 +294,27 @@ def skip_ssl_verify(self) -> bool: typing.cast(str, self.relation.data[self.relation.app].get("skip_ssl_verify")) ) + @property + def smtp_sender(self) -> Optional[str]: + """Fetch the SMTP sender from the relation. + + Returns: + smtp_sender: Optional sender email address for outgoing notifications. + """ + assert self.relation.app + return self.relation.data[self.relation.app].get("smtp_sender") + + @property + def recipients(self) -> List[str]: + """Fetch the SMTP recipients from the relation. + + Returns: + recipients: list of recipient email addresses for notifications. + """ + assert self.relation.app + raw = self.relation.data[self.relation.app].get("recipients") + return parse_recipients(raw) + class SmtpRequiresEvents(ops.CharmEvents): """SMTP events. @@ -270,6 +348,7 @@ def __init__(self, charm: ops.CharmBase, relation_name: str = DEFAULT_RELATION_N self.charm = charm self.relation_name = relation_name self.framework.observe(charm.on[relation_name].relation_changed, self._on_relation_changed) + self.framework.observe(charm.on.secret_changed, self._on_secret_changed) def get_relation_data(self) -> Optional[SmtpRelationData]: """Retrieve the relation data. @@ -278,9 +357,11 @@ def get_relation_data(self) -> Optional[SmtpRelationData]: SmtpRelationData: the relation data. """ relation = self.model.get_relation(self.relation_name) - return self._get_relation_data_from_relation(relation) if relation else None + return self.get_relation_data_from_relation(relation) if relation else None - def _get_relation_data_from_relation(self, relation: ops.Relation) -> SmtpRelationData: + def get_relation_data_from_relation( + self, relation: ops.Relation + ) -> Optional[SmtpRelationData]: """Retrieve the relation data. Args: @@ -288,20 +369,32 @@ def _get_relation_data_from_relation(self, relation: ops.Relation) -> SmtpRelati Returns: SmtpRelationData: the relation data. + + Raises: + SecretError: if the secret can't be read. """ assert relation.app - relation_data = relation.data[relation.app] - return SmtpRelationData( - host=typing.cast(str, relation_data.get("host")), - port=typing.cast(int, relation_data.get("port")), - user=relation_data.get("user"), - password=relation_data.get("password"), - password_id=relation_data.get("password_id"), - auth_type=AuthType(relation_data.get("auth_type")), - transport_security=TransportSecurity(relation_data.get("transport_security")), - domain=relation_data.get("domain"), - skip_ssl_verify=typing.cast(bool, relation_data.get("skip_ssl_verify")), - ) + raw_relation_data = relation.data[relation.app] + if not raw_relation_data: + return None + + data: Dict[str, Any] = dict(raw_relation_data) + + password = data.get("password") + if password is None and data.get("password_id"): + try: + password = ( + self.model.get_secret(id=data["password_id"]) + .get_content(refresh=True) + .get("password") + ) + except ops.model.ModelError as exc: + raise SecretError(f"Could not consume secret {data.get('password_id')}") from exc + + # normalize recipients + data["recipients"] = parse_recipients(data.get("recipients")) + + return SmtpRelationData(**{**data, "password": password}) def _is_relation_data_valid(self, relation: ops.Relation) -> bool: """Validate the relation data. @@ -313,7 +406,7 @@ def _is_relation_data_valid(self, relation: ops.Relation) -> bool: true: if the relation data is valid. """ try: - _ = self._get_relation_data_from_relation(relation) + _ = self.get_relation_data_from_relation(relation) return True except ValidationError as ex: error_fields = set( @@ -324,7 +417,7 @@ def _is_relation_data_valid(self, relation: ops.Relation) -> bool: return False def _on_relation_changed(self, event: ops.RelationChangedEvent) -> None: - """Event emitted when the relation has changed. + """Handle the relation changed event. Args: event: event triggering this handler. @@ -339,6 +432,43 @@ def _on_relation_changed(self, event: ops.RelationChangedEvent) -> None: if self._is_relation_data_valid(event.relation): self.on.smtp_data_available.emit(event.relation, app=event.app, unit=event.unit) + @staticmethod + def _secret_uri_equal(left: str, right: str) -> bool: + """Check if two juju secret URIs are equal.""" + left_without_protocol = left.removeprefix("secret://") + left_without_protocol = left_without_protocol.removeprefix("secret:") + right_without_protocol = right.removeprefix("secret://") + right_without_protocol = right_without_protocol.removeprefix("secret:") + # If they are both fully qualified secret URLs, compare them directly + if "/" in left_without_protocol and "/" in right_without_protocol: + return left_without_protocol == right_without_protocol + # Otherwise, compare only the secret ID part and ignore the potential model UUID + left_id = left_without_protocol.split("/")[-1] + right_id = right_without_protocol.split("/")[-1] + return left_id == right_id + + def _on_secret_changed(self, event: ops.SecretChangedEvent) -> None: + """Handle the relation secret event.""" + changed_secret_uri = event.secret.id + if changed_secret_uri is None: + return + for relation in self.charm.model.relations[self.relation_name]: + if relation.app is None: + continue + relation_data = relation.data[relation.app] + password_id = relation_data.get("password_id") + if not password_id: + continue + try: + secret = self.model.get_secret(id=password_id) + except ops.ModelError: + continue + secret_uri = secret.id + if secret_uri is None: + continue + if self._secret_uri_equal(changed_secret_uri, secret_uri): + self.on.smtp_data_available.emit(relation, app=relation.app, unit=None) + class SmtpProvides(ops.Object): """Provider side of the SMTP relation.""" @@ -361,9 +491,68 @@ def update_relation_data(self, relation: ops.Relation, smtp_data: SmtpRelationDa relation: the relation for which to update the data. smtp_data: a SmtpRelationData instance wrapping the data to be updated. """ - relation_data = smtp_data.to_relation_data() - if relation_data["auth_type"] == AuthType.NONE.value: + new_data = smtp_data.to_relation_data() + if new_data["auth_type"] == AuthType.NONE.value: logger.warning('Insecure setting: auth_type has a value "none"') - if relation_data["transport_security"] == TransportSecurity.NONE.value: + if new_data["transport_security"] == TransportSecurity.NONE.value: logger.warning('Insecure setting: transport_security has value "none"') - relation.data[self.charm.model.app].update(relation_data) + relation_data = relation.data[self.charm.model.app] + if dict(relation_data) != dict(new_data): + logger.info("update data in relation id:%s", relation.id) + relation_data.clear() + relation_data.update(new_data) + + +def parse_recipients(raw: Any) -> list[str]: + """Normalize SMTP recipient input into a list of email strings. + + The function produces a normalized list[str] so that downstream validation (EmailStr) + can be applied consistently. + + Args: + raw: Recipient input as received from relation data, charm config, + May be None, str, or list. + + Accepted input forms: + - None or empty string + - list of stripped string values + - JSON list string + - Comma-separated string + - Single address string + + Returns: + A list of recipient strings. The email correctness is validated later by EmailStr. + + Raises: + TypeError: If raw is not None, str or list. + ValueError: If a JSON-encoded value does not decode to a list. + """ + if raw is None: + return [] + + if isinstance(raw, list): + return [str(x).strip() for x in raw if str(x).strip()] + + if not isinstance(raw, str): + raise TypeError("recipients must be a string, list, or None") + + s = raw.strip() + if not s: + return [] + + # JSON list string + if s.startswith("["): + loaded = json.loads(s) + if not isinstance(loaded, list): + raise ValueError("recipients JSON must decode to a list") + return [str(x).strip() for x in loaded if str(x).strip()] + + # JSON without a bracelet: '"a@x.com", "b@y.com"' + if '"' in s and "," in s: + loaded = json.loads(f"[{s}]") + if not isinstance(loaded, list): + raise ValueError("recipients must decode to a list") + return [str(x).strip() for x in loaded if str(x).strip()] + + # comma-separated or single + return [p.strip() for p in s.split(",") if p.strip()] diff --git a/lib/charms/traefik_k8s/v2/ingress.py b/lib/charms/traefik_k8s/v2/ingress.py index bb7ac5ed5..f93b8632e 100644 --- a/lib/charms/traefik_k8s/v2/ingress.py +++ b/lib/charms/traefik_k8s/v2/ingress.py @@ -50,6 +50,7 @@ def _on_ingress_ready(self, event: IngressPerAppReadyEvent): def _on_ingress_revoked(self, event: IngressPerAppRevokedEvent): logger.info("This app no longer has ingress") """ + import ipaddress import json import logging @@ -57,9 +58,21 @@ def _on_ingress_revoked(self, event: IngressPerAppRevokedEvent): import typing from dataclasses import dataclass from functools import partial -from typing import Any, Callable, Dict, List, MutableMapping, Optional, Sequence, Tuple, Union +from typing import ( + Any, + Callable, + Dict, + List, + MutableMapping, + Optional, + Sequence, + Tuple, + Union, + cast, +) import pydantic +from ops import EventBase from ops.charm import CharmBase, RelationBrokenEvent, RelationEvent from ops.framework import EventSource, Object, ObjectEvents, StoredState from ops.model import ModelError, Relation, Unit @@ -73,7 +86,7 @@ def _on_ingress_revoked(self, event: IngressPerAppRevokedEvent): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 14 +LIBPATCH = 21 PYDEPS = ["pydantic"] @@ -84,7 +97,7 @@ def _on_ingress_revoked(self, event: IngressPerAppRevokedEvent): BUILTIN_JUJU_KEYS = {"ingress-address", "private-address", "egress-subnets"} PYDANTIC_IS_V1 = int(pydantic.version.VERSION.split(".")[0]) < 2 -if PYDANTIC_IS_V1: +if PYDANTIC_IS_V1: # noqa from pydantic import validator input_validator = partial(validator, pre=True) @@ -100,8 +113,10 @@ class Config: _NEST_UNDER = None + # Annotating -> "DatabagModel" as the return type here doesn't sit well with pyright + # We are disabling this line for now and come back to it later. @classmethod - def load(cls, databag: MutableMapping): + def load(cls, databag: MutableMapping): # type: ignore[no-untyped-def] """Load this model from a Juju databag.""" if cls._NEST_UNDER: return cls.parse_obj(json.loads(databag[cls._NEST_UNDER])) @@ -125,7 +140,7 @@ def load(cls, databag: MutableMapping): log.debug(msg, exc_info=True) raise DataValidationError(msg) from e - def dump(self, databag: Optional[MutableMapping] = None, clear: bool = True): + def dump(self, databag: Optional[MutableMapping] = None, clear: bool = True) -> Any: """Write the contents of this model to Juju databag. :param databag: the databag to write the data to. @@ -141,7 +156,7 @@ def dump(self, databag: Optional[MutableMapping] = None, clear: bool = True): databag[self._NEST_UNDER] = self.json(by_alias=True, exclude_defaults=True) return databag - for key, value in self.dict(by_alias=True, exclude_defaults=True).items(): # type: ignore + for key, value in self.dict(by_alias=True, exclude_defaults=True).items(): # type: ignore # noqa databag[key] = json.dumps(value) return databag @@ -149,9 +164,9 @@ def dump(self, databag: Optional[MutableMapping] = None, clear: bool = True): else: from pydantic import ConfigDict, field_validator - input_validator = partial(field_validator, mode="before") + input_validator = partial(field_validator, mode="before") # type: ignore - class DatabagModel(BaseModel): + class DatabagModel(BaseModel): # type: ignore """Base databag model.""" model_config = ConfigDict( @@ -165,8 +180,10 @@ class DatabagModel(BaseModel): ) # type: ignore """Pydantic config.""" + # Annotating -> "DatabagModel" as the return type here doesn't sit well with pyright + # We are disabling this line for now and come back to it later. @classmethod - def load(cls, databag: MutableMapping): + def load(cls, databag: MutableMapping): # type: ignore[no-untyped-def] """Load this model from a Juju databag.""" nest_under = cls.model_config.get("_NEST_UNDER") if nest_under: @@ -191,7 +208,7 @@ def load(cls, databag: MutableMapping): log.debug(msg, exc_info=True) raise DataValidationError(msg) from e - def dump(self, databag: Optional[MutableMapping] = None, clear: bool = True): + def dump(self, databag: Optional[MutableMapping] = None, clear: bool = True) -> Any: """Write the contents of this model to Juju databag. :param databag: the databag to write the data to. @@ -211,7 +228,11 @@ def dump(self, databag: Optional[MutableMapping] = None, clear: bool = True): ) return databag - dct = self.model_dump(mode="json", by_alias=True, exclude_defaults=True) # type: ignore + dct = self.model_dump( + mode="json", + by_alias=True, + exclude_defaults=True, # type: ignore + ) databag.update({k: json.dumps(v) for k, v in dct.items()}) return databag @@ -226,7 +247,7 @@ class IngressUrl(BaseModel): class IngressProviderAppData(DatabagModel): """Ingress application databag schema.""" - ingress: IngressUrl + ingress: Optional[IngressUrl] = None class ProviderSchema(BaseModel): @@ -235,12 +256,33 @@ class ProviderSchema(BaseModel): app: IngressProviderAppData +class IngressHealthCheck(BaseModel): + """HealthCheck schema for Ingress.""" + + path: str = Field(description="The health check endpoint path (required).") + scheme: Optional[str] = Field( + default=None, description="Replaces the server URL scheme for the health check endpoint." + ) + hostname: Optional[str] = Field( + default=None, description="Hostname to be set in the health check request." + ) + port: Optional[int] = Field( + default=None, description="Replaces the server URL port for the health check endpoint." + ) + interval: str = Field(default="30s", description="Frequency of the health check calls.") + timeout: str = Field(default="5s", description="Maximum duration for a health check request.") + + class IngressRequirerAppData(DatabagModel): """Ingress requirer application databag model.""" model: str = Field(description="The model the application is in.") name: str = Field(description="the name of the app requesting ingress.") port: int = Field(description="The port the app wishes to be exposed.") + is_port_open: bool = Field(default=False, description="Whether the port is open.") + healthcheck_params: Optional[IngressHealthCheck] = Field( + default=None, description="Optional health check configuration for ingress." + ) # fields on top of vanilla 'ingress' interface: strip_prefix: Optional[bool] = Field( @@ -258,15 +300,17 @@ class IngressRequirerAppData(DatabagModel): default="http", description="What scheme to use in the generated ingress url" ) + # pydantic wants 'cls' as first arg @input_validator("scheme") - def validate_scheme(cls, scheme): # noqa: N805 # pydantic wants 'cls' as first arg + def validate_scheme(cls, scheme: str) -> str: # noqa: N805 """Validate scheme arg.""" if scheme not in {"http", "https", "h2c"}: raise ValueError("invalid scheme: should be one of `http|https|h2c`") return scheme + # pydantic wants 'cls' as first arg @input_validator("port") - def validate_port(cls, port): # noqa: N805 # pydantic wants 'cls' as first arg + def validate_port(cls, port: int) -> int: # noqa: N805 """Validate port.""" assert isinstance(port, int), type(port) assert 0 < port < 65535, "port out of TCP range" @@ -283,14 +327,16 @@ class IngressRequirerUnitData(DatabagModel): "IP can only be None if the IP information can't be retrieved from juju.", ) + # pydantic wants 'cls' as first arg @input_validator("host") - def validate_host(cls, host): # noqa: N805 # pydantic wants 'cls' as first arg + def validate_host(cls, host: str) -> str: # noqa: N805 """Validate host.""" assert isinstance(host, str), type(host) return host + # pydantic wants 'cls' as first arg @input_validator("ip") - def validate_ip(cls, ip): # noqa: N805 # pydantic wants 'cls' as first arg + def validate_ip(cls, ip: str) -> Optional[str]: # noqa: N805 """Validate ip.""" if ip is None: return None @@ -340,8 +386,6 @@ def __init__(self, charm: CharmBase, relation_name: str = DEFAULT_RELATION_NAME) observe = self.framework.observe rel_events = charm.on[relation_name] - observe(rel_events.relation_created, self._handle_relation) - observe(rel_events.relation_joined, self._handle_relation) observe(rel_events.relation_changed, self._handle_relation) observe(rel_events.relation_departed, self._handle_relation) observe(rel_events.relation_broken, self._handle_relation_broken) @@ -349,19 +393,19 @@ def __init__(self, charm: CharmBase, relation_name: str = DEFAULT_RELATION_NAME) observe(charm.on.upgrade_charm, self._handle_upgrade_or_leader) # type: ignore @property - def relations(self): + def relations(self) -> List[Relation]: """The list of Relation instances associated with this endpoint.""" return list(self.charm.model.relations[self.relation_name]) - def _handle_relation(self, event): + def _handle_relation(self, event: RelationEvent) -> None: """Subclasses should implement this method to handle a relation update.""" pass - def _handle_relation_broken(self, event): + def _handle_relation_broken(self, event: RelationEvent) -> None: """Subclasses should implement this method to handle a relation breaking.""" pass - def _handle_upgrade_or_leader(self, event): + def _handle_upgrade_or_leader(self, event: EventBase) -> None: """Subclasses should implement this method to handle upgrades or leadership change.""" pass @@ -371,10 +415,10 @@ class _IPAEvent(RelationEvent): __optional_kwargs__: Dict[str, Any] = {} @classmethod - def __attrs__(cls): + def __attrs__(cls): # type: ignore return cls.__args__ + tuple(cls.__optional_kwargs__.keys()) - def __init__(self, handle, relation, *args, **kwargs): + def __init__(self, handle, relation, *args, **kwargs): # type: ignore super().__init__(handle, relation) if not len(self.__args__) == len(args): @@ -386,7 +430,7 @@ def __init__(self, handle, relation, *args, **kwargs): obj = kwargs.get(attr, default) setattr(self, attr, obj) - def snapshot(self): + def snapshot(self) -> Dict[str, Any]: dct = super().snapshot() for attr in self.__attrs__(): obj = getattr(self, attr) @@ -401,7 +445,7 @@ def snapshot(self): return dct - def restore(self, snapshot) -> None: + def restore(self, snapshot: Any) -> None: super().restore(snapshot) for attr, obj in snapshot.items(): setattr(self, attr, obj) @@ -425,11 +469,16 @@ class IngressPerAppDataRemovedEvent(RelationEvent): """Event representing that ingress data has been removed for an app.""" +class IngressPerAppEndpointsUpdatedEvent(RelationEvent): + """Event representing that the proxied endpoints have been updated.""" + + class IngressPerAppProviderEvents(ObjectEvents): """Container for IPA Provider events.""" data_provided = EventSource(IngressPerAppDataProvidedEvent) data_removed = EventSource(IngressPerAppDataRemovedEvent) + endpoints_updated = EventSource(IngressPerAppEndpointsUpdatedEvent) @dataclass @@ -459,7 +508,7 @@ def __init__( """ super().__init__(charm, relation_name) - def _handle_relation(self, event): + def _handle_relation(self, event: RelationEvent) -> None: # created, joined or changed: if remote side has sent the required data: # notify listeners. if self.is_ready(event.relation): @@ -476,10 +525,10 @@ def _handle_relation(self, event): data.app.redirect_https or False, ) - def _handle_relation_broken(self, event): - self.on.data_removed.emit(event.relation) # type: ignore + def _handle_relation_broken(self, event: RelationEvent) -> None: + self.on.data_removed.emit(event.relation, event.relation.app) # type: ignore - def wipe_ingress_data(self, relation: Relation): + def wipe_ingress_data(self, relation: Relation) -> None: """Clear ingress data from relation.""" assert self.unit.is_leader(), "only leaders can do this" try: @@ -492,9 +541,10 @@ def wipe_ingress_data(self, relation: Relation): ) return del relation.data[self.app]["ingress"] + self.on.endpoints_updated.emit(relation=relation, app=relation.app) def _get_requirer_units_data(self, relation: Relation) -> List["IngressRequirerUnitData"]: - """Fetch and validate the requirer's app databag.""" + """Fetch and validate the requirer's unit databag.""" out: List["IngressRequirerUnitData"] = [] unit: Unit @@ -502,7 +552,7 @@ def _get_requirer_units_data(self, relation: Relation) -> List["IngressRequirerU databag = relation.data[unit] try: data = IngressRequirerUnitData.load(databag) - out.append(data) + out.append(cast(IngressRequirerUnitData, data)) except pydantic.ValidationError: log.info(f"failed to validate remote unit data for {unit}") raise @@ -516,7 +566,7 @@ def _get_requirer_app_data(relation: Relation) -> "IngressRequirerAppData": raise NotReadyError(relation) databag = relation.data[app] - return IngressRequirerAppData.load(databag) + return cast(IngressRequirerAppData, IngressRequirerAppData.load(databag)) def get_data(self, relation: Relation) -> IngressRequirerData: """Fetch the remote (requirer) app and units' databags.""" @@ -525,9 +575,11 @@ def get_data(self, relation: Relation) -> IngressRequirerData: self._get_requirer_app_data(relation), self._get_requirer_units_data(relation) ) except (pydantic.ValidationError, DataValidationError) as e: - raise DataValidationError("failed to validate ingress requirer data") from e + raise DataValidationError( + "failed to validate ingress requirer data: %s" % str(e) + ) from e - def is_ready(self, relation: Optional[Relation] = None): + def is_ready(self, relation: Optional[Relation] = None) -> bool: """The Provider is ready if the requirer has sent valid data.""" if not relation: return any(map(self.is_ready, self.relations)) @@ -535,7 +587,7 @@ def is_ready(self, relation: Optional[Relation] = None): try: self.get_data(relation) except (DataValidationError, NotReadyError) as e: - log.debug("Provider not ready; validation error encountered: %s" % str(e)) + log.info("Provider not ready; validation error encountered: %s" % str(e)) return False return True @@ -555,10 +607,23 @@ def _published_url(self, relation: Relation) -> Optional["IngressProviderAppData return IngressProviderAppData.load(databag) - def publish_url(self, relation: Relation, url: str): + def publish_url(self, relation: Relation, url: str) -> None: """Publish to the app databag the ingress url.""" ingress_url = {"url": url} - IngressProviderAppData(ingress=ingress_url).dump(relation.data[self.app]) # type: ignore + try: + IngressProviderAppData(ingress=ingress_url).dump(relation.data[self.app]) # type: ignore + self.on.endpoints_updated.emit(relation=relation, app=relation.app) + except pydantic.ValidationError as e: + # If we cannot validate the url as valid, publish an empty databag and log the error. + log.error(f"Failed to validate ingress url '{url}' - got ValidationError {e}") + log.error( + ( + f"url was not published to ingress relation for {relation.app}." + f"This error is likely due to an error or misconfiguration of the" + "charm calling this library." + ) + ) + IngressProviderAppData(ingress=None).dump(relation.data[self.app]) # type: ignore @property def proxied_endpoints(self) -> Dict[str, Dict[str, str]]: @@ -581,7 +646,8 @@ def proxied_endpoints(self) -> Dict[str, Dict[str, str]]: for ingress_relation in self.relations: if not ingress_relation.app: log.warning( - f"no app in relation {ingress_relation} when fetching proxied endpoints: skipping" + f"no app in relation {ingress_relation} when fetching proxied endpoints:" + "skipping" ) continue try: @@ -596,10 +662,13 @@ def proxied_endpoints(self) -> Dict[str, Dict[str, str]]: if not ingress_data: log.warning(f"relation {ingress_relation} not ready yet: try again in some time.") continue + + # Validation above means ingress cannot be None, but type checker doesn't know that. + ingress = cast(IngressProviderAppData, ingress_data.ingress) if PYDANTIC_IS_V1: - results[ingress_relation.app.name] = ingress_data.ingress.dict() + results[ingress_relation.app.name] = ingress.dict() else: - results[ingress_relation.app.name] = ingress_data.ingress.model_dump(mode="json") + results[ingress_relation.app.name] = ingress.model_dump(mode="json") return results @@ -630,6 +699,7 @@ class IngressPerAppRequirer(_IngressPerAppBase): # used to prevent spurious urls to be sent out if the event we're currently # handling is a relation-broken one. _stored = StoredState() + _auto_data: Optional[Tuple[Optional[str], Optional[str], int]] def __init__( self, @@ -642,8 +712,10 @@ def __init__( strip_prefix: bool = False, redirect_https: bool = False, # fixme: this is horrible UX. - # shall we switch to manually calling provide_ingress_requirements with all args when ready? + # shall we switch to manually calling provide_ingress_requirements with all args when + # ready? scheme: Union[Callable[[], str], str] = lambda: "http", + healthcheck_params: Optional[Dict[str, Any]] = None, ): """Constructor for IngressRequirer. @@ -653,23 +725,40 @@ def __init__( All request args must be given as keyword args. Args: - charm: the charm that is instantiating the library. - relation_name: the name of the relation endpoint to bind to (defaults to `ingress`); - relation must be of interface type `ingress` and have "limit: 1") + charm: The charm that is instantiating the library. + relation_name: The name of the relation endpoint to bind to (defaults to "ingress"); + the relation must be of interface type "ingress" and have a limit of 1. host: Hostname to be used by the ingress provider to address the requiring application; if unspecified, the default Kubernetes service name will be used. ip: Alternative addressing method other than host to be used by the ingress provider; - if unspecified, binding address from juju network API will be used. - strip_prefix: configure Traefik to strip the path prefix. - redirect_https: redirect incoming requests to HTTPS. - scheme: callable returning the scheme to use when constructing the ingress url. - Or a string, if the scheme is known and stable at charm-init-time. + if unspecified, the binding address from the Juju network API will be used. + healthcheck_params: Optional dictionary containing health check + configuration parameters conforming to the IngressHealthCheck schema. + The dictionary must include: + - "path" (str): The health check endpoint path (required). + It may also include: + - "scheme" (Optional[str]): Replaces the server URL scheme for the health check + endpoint. + - "hostname" (Optional[str]): Hostname to be set in the health check request. + - "port" (Optional[int]): Replaces the server URL port for the health check + endpoint. + - "interval" (str): Frequency of the health check calls + (defaults to "30s" if omitted). + - "timeout" (str): Maximum duration for a health check request + (defaults to "5s" if omitted). + If provided, "path" is required while "interval" and "timeout" will use Traefik's + defaults when not specified. + strip_prefix: Configure Traefik to strip the path prefix. + redirect_https: Redirect incoming requests to HTTPS. + scheme: Either a callable that returns the scheme to use when constructing the ingress + URL, or a string if the scheme is known and stable at charm initialization. Request Args: port: the port of the service """ super().__init__(charm, relation_name) self.charm: CharmBase = charm + self.healthcheck_params = healthcheck_params self.relation_name = relation_name self._strip_prefix = strip_prefix self._redirect_https = redirect_https @@ -684,7 +773,7 @@ def __init__( else: self._auto_data = None - def _handle_relation(self, event): + def _handle_relation(self, event: RelationEvent) -> None: # created, joined or changed: if we have auto data: publish it self._publish_auto_data() if self.is_ready(): @@ -698,15 +787,15 @@ def _handle_relation(self, event): self._stored.current_url = new_url # type: ignore self.on.ready.emit(event.relation, new_url) # type: ignore - def _handle_relation_broken(self, event): + def _handle_relation_broken(self, event: RelationEvent) -> None: self._stored.current_url = None # type: ignore - self.on.revoked.emit(event.relation) # type: ignore + self.on.revoked.emit(relation=event.relation, app=event.relation.app) # type: ignore - def _handle_upgrade_or_leader(self, event): + def _handle_upgrade_or_leader(self, event: EventBase) -> None: """On upgrade/leadership change: ensure we publish the data we have.""" self._publish_auto_data() - def is_ready(self): + def is_ready(self) -> bool: """The Requirer is ready if the Provider has sent valid data.""" try: return bool(self._get_url_from_relation_data()) @@ -714,7 +803,7 @@ def is_ready(self): log.debug("Requirer not ready; validation error encountered: %s" % str(e)) return False - def _publish_auto_data(self): + def _publish_auto_data(self) -> None: if self._auto_data: host, ip, port = self._auto_data self.provide_ingress_requirements(host=host, ip=ip, port=port) @@ -726,7 +815,7 @@ def provide_ingress_requirements( host: Optional[str] = None, ip: Optional[str] = None, port: int, - ): + ) -> None: """Publishes the data that Traefik needs to provide ingress. Args: @@ -747,7 +836,7 @@ def _provide_ingress_requirements( ip: Optional[str], port: int, relation: Relation, - ): + ) -> None: if self.unit.is_leader(): self._publish_app_data(scheme, port, relation) @@ -758,7 +847,7 @@ def _publish_unit_data( host: Optional[str], ip: Optional[str], relation: Relation, - ): + ) -> None: if not host: host = socket.getfqdn() @@ -785,7 +874,7 @@ def _publish_app_data( scheme: Optional[str], port: int, relation: Relation, - ): + ) -> None: # assumes leadership! app_databag = relation.data[self.app] @@ -794,13 +883,20 @@ def _publish_app_data( scheme = self._get_scheme() try: - IngressRequirerAppData( # type: ignore # pyright does not like aliases + # Ignore pyright errors since pyright does not like aliases. + IngressRequirerAppData( # type: ignore model=self.model.name, name=self.app.name, scheme=scheme, port=port, - strip_prefix=self._strip_prefix, # type: ignore # pyright does not like aliases - redirect_https=self._redirect_https, # type: ignore # pyright does not like aliases + is_port_open=port in {open_port.port for open_port in self.unit.opened_ports()}, + strip_prefix=self._strip_prefix, # type: ignore + redirect_https=self._redirect_https, # type: ignore + healthcheck_params=( + IngressHealthCheck(**self.healthcheck_params) + if self.healthcheck_params + else None + ), ).dump(app_databag) except pydantic.ValidationError as e: msg = "failed to validate app data" @@ -808,12 +904,12 @@ def _publish_app_data( raise DataValidationError(msg) from e @property - def relation(self): + def relation(self) -> Optional[Relation]: """The established Relation instance, or None.""" return self.relations[0] if self.relations else None def _get_url_from_relation_data(self) -> Optional[str]: - """The full ingress URL to reach the current unit. + """The full ingress URL to reach the charm application. Returns None if the URL isn't available yet. """ @@ -834,11 +930,15 @@ def _get_url_from_relation_data(self) -> Optional[str]: if not databag: # not ready yet return None - return str(IngressProviderAppData.load(databag).ingress.url) + ingress = cast(IngressProviderAppData, IngressProviderAppData.load(databag)).ingress + if ingress is None: + return None + + return str(ingress.url) @property def url(self) -> Optional[str]: - """The full ingress URL to reach the current unit. + """The full ingress URL to reach the charm application. Returns None if the URL isn't available yet. """