From e0af759fb487217f8c9f81c42cf1dfeb32cc375c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 13 Aug 2026 01:34:12 +0000 Subject: [PATCH] chore: update charm libraries --- .../data_platform_libs/v0/data_interfaces.py | 201 ++++++++++++- .../grafana_k8s/v0/grafana_dashboard.py | 74 +++-- lib/charms/loki_k8s/v0/loki_push_api.py | 19 +- .../prometheus_k8s/v0/prometheus_scrape.py | 267 +++++++++--------- 4 files changed, 404 insertions(+), 157 deletions(-) diff --git a/lib/charms/data_platform_libs/v0/data_interfaces.py b/lib/charms/data_platform_libs/v0/data_interfaces.py index 5be1d931..f4a65fdd 100644 --- a/lib/charms/data_platform_libs/v0/data_interfaces.py +++ b/lib/charms/data_platform_libs/v0/data_interfaces.py @@ -433,6 +433,11 @@ def _on_subject_requested(self, event: SubjectRequestedEvent): overload, ) +try: + from cryptography.fernet import Fernet, InvalidToken +except ImportError: + Fernet = None + InvalidToken = None from ops import JujuVersion, Model, Secret, SecretInfo, SecretNotFoundError from ops.charm import ( CharmBase, @@ -453,7 +458,7 @@ def _on_subject_requested(self, event: SubjectRequestedEvent): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 58 +LIBPATCH = 59 PYDEPS = ["ops>=2.0.0"] @@ -489,6 +494,10 @@ def _on_subject_requested(self, event: SubjectRequestedEvent): "owner_no_refresh": "ERROR secret owner cannot use --refresh", } +CROSS_MODEL_RELATION_CONSUMER_SECRETS = [ + "mtls-cert", +] + ############################################################################## # Exceptions @@ -1227,6 +1236,15 @@ def _load_secrets_from_databag(self, relation: Relation) -> None: """Load secrets from the databag.""" raise NotImplementedError + def _get_encryption_key(self, relation: Relation) -> Optional[str]: + """Fetch the encryption key from the encryption secret if available.""" + if not (encryption_secret := relation.data[relation.app].get("encryption-secret")): + return None + + # get the encryption secret created on provider side + secret = self._model.get_secret(id=encryption_secret) + return secret.get_content().get("encryption-key") + def _fetch_specific_relation_data( self, relation: Relation, fields: Optional[List[str]] ) -> Dict[str, str]: @@ -1586,11 +1604,48 @@ def _fetch_relation_data_without_secrets( return {} if fields: - return { + relation_data = { k: relation.data[component][k] for k in fields if k in relation.data[component] } else: - return dict(relation.data[component]) + relation_data = dict(relation.data[component]) + + try: + remote_model_uuid = relation.remote_model.uuid + except (FileNotFoundError, ModelError, RuntimeError) as e: + # access to remote model added in Juju 3.6.2, fails with 2.9 + logger.warning("Access to remote model failed: %s", e) + remote_model_uuid = "" + + # if not a cross-model relation we can return data as-is + if remote_model_uuid == "" or self._model.uuid == remote_model_uuid: + return relation_data + + if not (encryption_key := self._get_encryption_key(relation)): + return relation_data + + if not Fernet: + logger.warning( + "Cryptography module not installed. Could not decrypt sensitive field in cross-model relation" + ) + return relation_data + + # still here means sensitive data needs to be decrypted + for key, value in relation_data.items(): + if key in CROSS_MODEL_RELATION_CONSUMER_SECRETS: + try: + f = Fernet(encryption_key) + decrypted_value = f.decrypt(value.encode()).decode() + relation_data[key] = decrypted_value + except ( + AttributeError, + InvalidToken, + TypeError, + ValueError, + ): # pyright: ignore [reportGeneralTypeIssues] + logger.warning("Could not decrypt sensitive field in cross-model relation") + + return relation_data def _fetch_relation_data_with_secrets( self, @@ -1637,8 +1692,40 @@ def _update_relation_data_without_secrets( if component not in relation.data or relation.data[component] is None: return - if relation: - relation.data[component].update(data) + if not relation: + return + + # ensure no sensitive information is stored in relation data + encryption_key = self._get_encryption_key(relation) + + try: + remote_model_uuid = relation.remote_model.uuid + except (FileNotFoundError, ModelError, RuntimeError): + # access to remote model added in Juju 3.6.2, fails with 2.9 + remote_model_uuid = "" + + if encryption_key and remote_model_uuid != "" and self._model.uuid != remote_model_uuid: + for key, value in data.items(): + try: + f = Fernet(encryption_key) # pyright: ignore [reportOptionalCall] + if key in CROSS_MODEL_RELATION_CONSUMER_SECRETS: + encrypted_value = f.encrypt(value.encode()).decode() + data[key] = encrypted_value + except ( + AttributeError, + InvalidToken, + ValueError, + ): # pyright: ignore [reportGeneralTypeIssues] + logger.warning("Could not encrypt sensitive field in cross-model relation") + data[key] = "" + except TypeError: + # "TypeError: 'NoneType' object is not callable" raised when Fernet is `None` + logger.warning( + "Cryptography module not installed. Could not decrypt sensitive field in cross-model relation" + ) + data[key] = "" + + relation.data[component].update(data) def _delete_relation_data_without_secrets( self, component: Union[Application, Unit], relation: Relation, fields: List[str] @@ -1899,7 +1986,7 @@ def _update_relation_data(self, relation: Relation, data: Dict[str, str]) -> Non """Set values for fields not caring whether it's a secret or not.""" keys = set(data.keys()) if self.fetch_relation_field(relation.id, self.RESOURCE_FIELD) is None and ( - keys - {"endpoints", "read-only-endpoints", "replset"} + keys - {"endpoints", "read-only-endpoints", "replset", "encryption-secret"} ): raise PrematureDataAccessError( "Premature access to relation data, update is forbidden before the connection is initialized." @@ -2112,12 +2199,31 @@ def __init__( field for field in self.SECRET_LABEL_MAP.keys() if field not in self._remote_secret_fields + and not ( + field in CROSS_MODEL_RELATION_CONSUMER_SECRETS and self.is_cross_model_relation + ) ] if additional_secret_fields: self._remote_secret_fields += additional_secret_fields self.data_component = self.local_unit # Internal functions + @property + def is_cross_model_relation(self) -> bool: + """Determines whether the relation is a cross-model relation or not.""" + if len(self.relations) == 0: + return False + + try: + remote_model_uuid = self.relations[0].remote_model.uuid + except (FileNotFoundError, ModelError, RuntimeError): + # access to remote model added in Juju 3.6.2, fails with 2.9 + return False + + if self._model.uuid != remote_model_uuid: + return True + + return False def _is_resource_created_for_relation(self, relation: Relation) -> bool: if not relation.app: @@ -2389,6 +2495,44 @@ def _validate_entity_consistency(event: RelationEvent, diff: Diff) -> None: raise ValueError(f"Cannot change {key} after relation has already been created") # Event handlers + def _on_relation_created_event(self, event: RelationCreatedEvent) -> None: + """Event emitted when a relation is created.""" + if not self.relation_data.local_unit.is_leader(): + return + + try: + remote_model_uuid = event.relation.remote_model.uuid + except (FileNotFoundError, ModelError, RuntimeError): + # access to remote model added in Juju 3.6.2, fails with 2.9 + return + + if self.model.uuid == remote_model_uuid: + return + + if not Fernet: + logger.warning( + "Cryptography module not installed. Could not decrypt sensitive field in cross-model relation" + ) + return + + # in cross-model relations, generate an encryption key and share it with the requirer as a secret + event_data = {} + secret_label = f"{self.model.uuid}-{event.relation.id}-encryption-secret" + + try: + # check if secret was already created to avoid duplicates + secret = self.charm.model.get_secret(label=secret_label) + except SecretNotFoundError: + encryption_key = Fernet.generate_key() # pyright: ignore [reportOptionalMemberAccess] + content = {"encryption-key": encryption_key.decode()} + secret = self.charm.app.add_secret(content, label=secret_label) + + secret.grant(event.relation) + if not secret.id: + raise SecretError("Encryption secret is missing secred id") + event_data["encryption-secret"] = secret.id + + self.relation_data.update_relation_data(event.relation.id, event_data) def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: """Event emitted when the relation data has changed.""" @@ -4412,6 +4556,33 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: # Check which data has changed to emit customs events. diff = self._diff(event) + # send request again if encryption secret was added from provider side + if "encryption-secret" in diff.added and self.relation_data.local_unit.is_leader(): + relation_data = { + "topic": self.relation_data.topic, + "encryption-secret": event.relation.data[event.relation.app].get( + "encryption-secret" + ), + } + + if self.relation_data.mtls_cert: + relation_data["mtls-cert"] = self.relation_data.mtls_cert + + if self.relation_data.consumer_group_prefix: + relation_data["consumer-group-prefix"] = self.relation_data.consumer_group_prefix + + if self.relation_data.extra_user_roles: + relation_data["extra-user-roles"] = self.relation_data.extra_user_roles + if self.relation_data.extra_group_roles: + relation_data["extra-group-roles"] = self.relation_data.extra_group_roles + if self.relation_data.entity_type: + relation_data["entity-type"] = self.relation_data.entity_type + if self.relation_data.entity_permissions: + relation_data["entity-permissions"] = self.relation_data.entity_permissions + + self.relation_data.update_relation_data(event.relation.id, relation_data) + return + # Check if the topic is created # (the Kafka charm shared the credentials). @@ -5689,6 +5860,24 @@ def _on_relation_changed_event(self, event: RelationChangedEvent) -> None: # Check which data has changed to emit customs events. diff = self._diff(event) + + # send request again if encryption secret was added from provider side + if "encryption-secret" in diff.added and self.relation_data.local_unit.is_leader(): + payload = { + "prefix": self.relation_data.prefix, + "encryption-secret": event.relation.data[event.relation.app].get( + "encryption-secret" + ), + } + if self.relation_data.mtls_cert: + payload["mtls-cert"] = self.relation_data.mtls_cert + + self.relation_data.update_relation_data( + event.relation.id, + payload, + ) + return + # Register all new secrets with their labels if any(newval for newval in diff.added if self.relation_data._is_secret_field(newval)): self.relation_data._register_secrets_to_relation(event.relation, diff.added) diff --git a/lib/charms/grafana_k8s/v0/grafana_dashboard.py b/lib/charms/grafana_k8s/v0/grafana_dashboard.py index 9886fc2b..2e7ce257 100644 --- a/lib/charms/grafana_k8s/v0/grafana_dashboard.py +++ b/lib/charms/grafana_k8s/v0/grafana_dashboard.py @@ -184,7 +184,7 @@ def __init__(self, *args): import re import subprocess import tempfile -import uuid + from pathlib import Path from typing import Any, Callable, Dict, List, Optional, Tuple import yaml @@ -217,7 +217,7 @@ def __init__(self, *args): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 49 +LIBPATCH = 51 PYDEPS = ["cosl >= 0.0.50"] @@ -392,6 +392,13 @@ def __init__(self, *args): } +def _data_hash(data: Any) -> str: + """Deterministic hash of a template dict for use as a stable relation data key.""" + return hashlib.shake_128( + json.dumps(data, sort_keys=True).encode() + ).digest(8).hex() + + class RelationNotFoundError(Exception): """Raised if there is no relation with the given name.""" @@ -1349,11 +1356,29 @@ def _on_grafana_dashboard_relation_changed(self, event: RelationChangedEvent) -> def _upset_dashboards_on_relation(self, relation: Relation) -> None: """Update the dashboards in the relation data bucket.""" + new_templates = type_convert_stored(self._stored.dashboard_templates) # pyright: ignore + + # Check if the templates have actually changed before updating. + # This avoids generating a new UUID on every event, which would cause + # unnecessary relation-changed events on the consumer side. + # See: https://github.com/canonical/opentelemetry-collector-operator/issues/331 + existing_data_str = relation.data[self._charm.app].get("dashboards", "{}") + try: + existing_data = json.loads(existing_data_str) + existing_templates = existing_data.get("templates", {}) + except json.JSONDecodeError: + existing_templates = {} + + if new_templates == existing_templates: + return # No change in templates, don't update the databag + # It's completely ridiculous to add a UUID, but if we don't have some - # pseudo-random value, this never makes it across 'juju set-state' + # pseudo-random value, this never makes it across 'juju set-state'. + # Use a deterministic hash of the templates so the value is stable when + # templates haven't changed, avoiding spurious relation-changed events. stored_data = { - "templates": type_convert_stored(self._stored.dashboard_templates), # pyright: ignore - "uuid": str(uuid.uuid4()), + "templates": new_templates, + "uuid": _data_hash(new_templates), } relation.data[self._charm.app]["dashboards"] = json.dumps(stored_data) @@ -1717,7 +1742,7 @@ def set_peer_data(self, key: str, data: Any) -> None: if not peers or not peers.data: logger.info("set_peer_data: no peer relation. Is the charm being installed/removed?") return - peers.data[self._charm.app][key] = json.dumps(data) # type: ignore[attr-defined] + peers.data[self._charm.app][key] = json.dumps(data, sort_keys=True) # type: ignore[attr-defined] def get_peer_data(self, key: str) -> Any: """Retrieve information from the peer data bucket instead of `StoredState`.""" @@ -1831,14 +1856,30 @@ def _upset_dashboards_on_event(self, event: RelationEvent) -> None: def _update_remote_grafana(self, _: Optional[RelationEvent] = None) -> None: """Push dashboards to the downstream Grafana relation.""" - # It's still ridiculous to add a UUID here, but needed - stored_data = { - "templates": type_convert_stored(self._stored.dashboard_templates), # pyright: ignore - "uuid": str(uuid.uuid4()), - } + new_templates = type_convert_stored(self._stored.dashboard_templates) # pyright: ignore if self._charm.unit.is_leader(): for grafana_relation in self.model.relations[self._grafana_relation]: + # Check if the templates have actually changed before updating. + # This avoids generating a new UUID on every event, which would cause + # unnecessary relation-changed events on the consumer side. + existing_data_str = grafana_relation.data[self._charm.app].get("dashboards", "{}") + try: + existing_data = json.loads(existing_data_str) + existing_templates = existing_data.get("templates", {}) + except json.JSONDecodeError: + existing_templates = {} + + if new_templates == existing_templates: + continue # No change in templates, don't update the databag + + # It's still ridiculous to add a UUID here, but needed. + # Use a deterministic hash of the templates so the value is stable when + # templates haven't changed, avoiding spurious relation-changed events. + stored_data = { + "templates": new_templates, + "uuid": _data_hash(new_templates), + } grafana_relation.data[self._charm.app]["dashboards"] = json.dumps(stored_data) def remove_dashboards(self, event: RelationBrokenEvent) -> None: @@ -1853,9 +1894,10 @@ def remove_dashboards(self, event: RelationBrokenEvent) -> None: for id in app_ids: del self._stored.dashboard_templates[id] # type: ignore + remaining_templates = type_convert_stored(self._stored.dashboard_templates) # pyright: ignore stored_data = { - "templates": type_convert_stored(self._stored.dashboard_templates), # pyright: ignore - "uuid": str(uuid.uuid4()), + "templates": remaining_templates, + "uuid": _data_hash(remaining_templates), } if self._charm.unit.is_leader(): @@ -2100,11 +2142,11 @@ def validate_alert_rules(self, rules: dict) -> Tuple[bool, str]: # - alert: OtherAlert # expr: up transformed_rules = {"groups": []} # type: ignore - for rule in rules["groups"]: - transformed = {"name": str(uuid.uuid4()), "rules": [rule]} + for i, rule in enumerate(rules["groups"]): + transformed = {"name": f"group_{i}", "rules": [rule]} transformed_rules["groups"].append(transformed) - rule_path.write_text(yaml.dump(transformed_rules)) + rule_path.write_text(yaml.safe_dump(transformed_rules, sort_keys=True)) # databag-order: ignore args = [str(self.path), "validate", str(rule_path)] # noinspection PyBroadException diff --git a/lib/charms/loki_k8s/v0/loki_push_api.py b/lib/charms/loki_k8s/v0/loki_push_api.py index 4b08894f..b63d10ac 100644 --- a/lib/charms/loki_k8s/v0/loki_push_api.py +++ b/lib/charms/loki_k8s/v0/loki_push_api.py @@ -458,7 +458,7 @@ def _alert_rules_error(self, event): from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, Union, cast from urllib import request -from urllib.error import HTTPError +from urllib.error import URLError import yaml from cosl import JujuTopology @@ -485,7 +485,7 @@ def _alert_rules_error(self, event): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 32 +LIBPATCH = 35 PYDEPS = ["cosl"] @@ -847,7 +847,16 @@ def _multi_suffix_glob( List of files in `dir_path` that have one of the suffixes specified in `suffixes`. """ all_files_in_dir = dir_path.glob("**/*" if recursive else "*") - return list(filter(lambda f: f.is_file() and f.suffix in suffixes, all_files_in_dir)) + all_files = {p for p in all_files_in_dir if p.is_file()} + matched = {p for p in all_files if p.suffix in suffixes} + ignored = all_files - matched + if ignored: + logger.info( + "Ignoring files with unrecognized suffix (expected one of %s): %s", + suffixes, + ", ".join(str(p) for p in sorted(ignored)), + ) + return sorted(matched) def _from_dir(self, dir_path: Path, recursive: bool) -> List[dict]: """Read all rule files in a directory. @@ -1812,7 +1821,7 @@ def __init__( self.insecure_skip_verify = insecure_skip_verify # architecture used for promtail binary - arch = platform.processor() + arch = platform.machine() self._arch = "amd64" if arch == "x86_64" else arch events = self._charm.on[relation_name] @@ -2332,7 +2341,7 @@ def _setup_promtail(self) -> None: if not self._is_promtail_installed(promtail_binaries[self._arch]): try: self._obtain_promtail(promtail_binaries[self._arch]) - except HTTPError as e: + except URLError as e: msg = "Promtail binary couldn't be downloaded - {}".format(str(e)) logger.warning(msg) self.on.promtail_digest_error.emit(msg) diff --git a/lib/charms/prometheus_k8s/v0/prometheus_scrape.py b/lib/charms/prometheus_k8s/v0/prometheus_scrape.py index ff52245c..358c08de 100644 --- a/lib/charms/prometheus_k8s/v0/prometheus_scrape.py +++ b/lib/charms/prometheus_k8s/v0/prometheus_scrape.py @@ -335,12 +335,13 @@ def _on_scrape_targets_changed(self, event): import tempfile from collections import defaultdict from pathlib import Path -from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union +from typing import Callable, Dict, List, Literal, Optional, Tuple, Union from urllib.parse import urlparse import yaml -from cosl import JujuTopology +from cosl import CosTool, JujuTopology from cosl.rules import AlertRules, generic_alert_groups +from cosl.types import OfficialRuleFileFormat from ops.charm import CharmBase, RelationRole from ops.framework import ( BoundEvent, @@ -361,7 +362,7 @@ def _on_scrape_targets_changed(self, event): # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 58 +LIBPATCH = 65 # Version 0.0.53 needed for cosl.rules.generic_alert_groups PYDEPS = ["cosl>=0.0.53"] @@ -843,7 +844,7 @@ def _type_convert_stored(obj): if isinstance(obj, StoredList): return list(map(_type_convert_stored, obj)) if isinstance(obj, StoredDict): - rdict = {} # type: Dict[Any, Any] + rdict = {} for k in obj.keys(): rdict[k] = _type_convert_stored(obj[k]) return rdict @@ -993,12 +994,16 @@ def __init__( self._charm = charm self._relation_name = relation_name self._fallback_scrape_protocol = fallback_scrape_protocol - self._tool = CosTool(self._charm) + self._tool = CosTool("promql") events = self._charm.on[relation_name] self.framework.observe(events.relation_changed, self._on_metrics_provider_relation_changed) self.framework.observe( events.relation_departed, self._on_metrics_provider_relation_departed ) + self.framework.observe( + events.relation_broken, self._on_metrics_provider_relation_departed + ) + def _on_metrics_provider_relation_changed(self, event): """Handle changes with related metrics providers. @@ -1048,13 +1053,18 @@ def jobs(self) -> list: # Therefore we need to dedupe here and after all jobs are collected. static_scrape_jobs = _dedupe_job_names(static_scrape_jobs) try: - self._tool.validate_scrape_jobs(static_scrape_jobs) + _validate_scrape_jobs(static_scrape_jobs) except subprocess.CalledProcessError as e: + logger.error(f"Invalid scrape job file: {e}") if self._charm.unit.is_leader(): data = json.loads(relation.data[self._charm.app].get("event", "{}")) data["scrape_job_errors"] = str(e) relation.data[self._charm.app]["event"] = json.dumps(data) else: + if self._charm.unit.is_leader(): + data = json.loads(relation.data[self._charm.app].get("event", "{}")) + data.pop("scrape_job_errors", None) + relation.data[self._charm.app]["event"] = json.dumps(data) scrape_jobs.extend(static_scrape_jobs) scrape_jobs = _dedupe_job_names(scrape_jobs) @@ -1103,7 +1113,7 @@ def alerts(self) -> dict: A dictionary mapping the Juju topology identifier of the source charm to its list of alert rule groups. """ - alerts = {} # type: Dict[str, dict] # mapping b/w juju identifiers and alert rule files + alerts: Dict[str, OfficialRuleFileFormat] = {} for relation in self._charm.model.relations[self._relation_name]: if not relation.units or not relation.app: continue @@ -1141,6 +1151,7 @@ def alerts(self) -> dict: _, errmsg = self._tool.validate_alert_rules(alert_rules) if errmsg: + logger.error(f"Invalid alert rule file: {errmsg}") if alerts[identifier]: del alerts[identifier] if self._charm.unit.is_leader(): @@ -1148,11 +1159,15 @@ def alerts(self) -> dict: data["errors"] = errmsg relation.data[self._charm.app]["event"] = json.dumps(data) continue + if self._charm.unit.is_leader(): + data = json.loads(relation.data[self._charm.app].get("event", "{}")) + data.pop("errors", None) + relation.data[self._charm.app]["event"] = json.dumps(data) return alerts def _get_identifier_by_alert_rules( - self, rules: dict + self, rules: OfficialRuleFileFormat ) -> Tuple[Union[str, None], Union[JujuTopology, None]]: """Determine an appropriate dict key for alert rules. @@ -1172,7 +1187,9 @@ def _get_identifier_by_alert_rules( # Construct an ID based on what's in the alert rules if they have labels for group in rules["groups"]: try: - labels = group["rules"][0]["labels"] + labels = group["rules"][0].get("labels") + if not labels: + continue topology = JujuTopology( # Don't try to safely get required constructor fields. There's already # a handler for KeyErrors @@ -1199,7 +1216,7 @@ def _get_identifier_by_alert_rules( return None, None - def _inject_alert_expr_labels(self, rules: Dict[str, Any]) -> Dict[str, Any]: + def _inject_alert_expr_labels(self, rules: OfficialRuleFileFormat) -> OfficialRuleFileFormat: """Iterate through alert rules and inject topology into expressions. Args: @@ -1346,6 +1363,102 @@ def _target_parts(self, target) -> list: return parts + def has_invalid_scrape_jobs(self) -> bool: + """Check whether any relation reported invalid scrape jobs. + + Validation errors, written to relation app data by this consumer + (see :meth:`jobs`), are read back to determine whether the relationship + currently carries an invalid scrape job. + + Returns: + True if any related metrics provider reported scrape job validation + errors, False otherwise. + """ + return self._has_relation_error("scrape_job_errors", "Scrape job validation error") + + def has_invalid_alert_rules(self) -> bool: + """Check whether any relation reported invalid alert rules. + + Validation errors, written to relation app data by this consumer + (see :attr:`alerts`), are read back to determine whether the relationship + currently carries invalid alert rules. + + Returns: + True if any related metrics provider reported alert rule validation + errors, False otherwise. + """ + return self._has_relation_error("errors", "Alert rule validation error") + + def _has_relation_error(self, error_key: str, error_label: str) -> bool: + """Check whether any relation reported the given validation error. + + Args: + error_key: the relation app data key that holds the validation error, + i.e. "scrape_job_errors" or "errors". + error_label: a human readable description of the validation error + type, used for logging, e.g. "Scrape job validation error". + + Returns: + True if any related metrics provider reported the validation error, + False otherwise. + """ + if not self._charm.unit.is_leader(): + return False + + for relation in self._charm.model.relations.get(self._relation_name, []): + app_data = relation.data.get(self._charm.app) + if not app_data: + continue + + event_raw = app_data.get("event", "{}") + try: + event_data = json.loads(event_raw) + except (json.JSONDecodeError, TypeError): + continue + + if error_msg := event_data.get(error_key): + logger.error("%s on relation %s: %s", error_label, relation.id, error_msg) + return True + + return False + + +def _validate_scrape_jobs(jobs: list) -> bool: + """Validate scrape jobs using cos-tool. + + Args: + jobs: A list of Prometheus scrape job dicts to validate. + + Returns: + True if validation passed or cos-tool is unavailable. + + Raises: + subprocess.CalledProcessError: if cos-tool rejects the scrape jobs. + """ + arch = platform.machine() + arch = "amd64" if arch == "x86_64" else arch + cos_tool_path = Path("cos-tool-{}".format(arch)) + try: + cos_tool_path = cos_tool_path.resolve(strict=True) + except (FileNotFoundError, OSError): + logger.debug("cos-tool unavailable. Not validating scrape jobs.") + return True + + conf = {"scrape_configs": jobs} + with tempfile.NamedTemporaryFile(suffix=".yaml", mode="w", delete=False) as tmpfile: + tmpfile.write(yaml.safe_dump(conf)) + tmpfile_name = tmpfile.name + try: + subprocess.run( + [str(cos_tool_path), "validate-config", tmpfile_name], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + finally: + Path(tmpfile_name).unlink(missing_ok=True) + return True + def _dedupe_job_names(jobs: List[dict]): """Deduplicate a list of dicts by appending a hash to the value of the 'job_name' key. @@ -1622,8 +1735,21 @@ def __init__( for ev in refresh_event: self.framework.observe(ev, self.set_scrape_job_spec) + # Always re-evaluate the unit address on `update_status`, regardless of the charm type + # (sidecar/pebble or podspec) or any user-provided `refresh_event`. On Kubernetes a pod + # can be rescheduled (e.g. node reboot/maintenance) and come back with a new IP without + # re-emitting `relation_joined`/`pebble_ready`. Without this, the stale address lingers in + # relation data and the consumer keeps scraping a dead IP until the relation is recreated. + # `update_status` fires periodically, so the address self-heals within one hook interval. + # See https://github.com/canonical/opentelemetry-collector-k8s-operator/issues/270 + self.framework.observe(self._charm.on.update_status, self._set_unit_ip) + def _on_relation_changed(self, event): """Check for alert rule messages in the relation data before moving on.""" + # Refresh the unit address on every `relation_changed`. This reacts faster than waiting for + # the next `update_status` when the pod IP changes, and is safe to call repeatedly. + self._set_unit_ip() + if self._charm.unit.is_leader(): ev = json.loads(event.relation.data[event.app].get("event", "{}")) @@ -1800,6 +1926,7 @@ def __init__( events.relation_changed, self._charm.on.leader_elected, self._charm.on.upgrade_charm, + self._charm.on.config_changed, ] for event_source in event_sources: @@ -1824,123 +1951,3 @@ def _update_relation_data(self, _): alert_rules_as_dict, sort_keys=True, # sort, to prevent unnecessary relation_changed events ) - -class CosTool: - """Uses cos-tool to inject label matchers into alert rule expressions and validate rules.""" - - _path = None - _disabled = False - - def __init__(self, charm): - self._charm = charm - - @property - def path(self): - """Lazy lookup of the path of cos-tool.""" - if self._disabled: - return None - if not self._path: - self._path = self._get_tool_path() - if not self._path: - logger.debug("Skipping injection of juju topology as label matchers") - self._disabled = True - return self._path - - def apply_label_matchers(self, rules) -> dict: - """Will apply label matchers to the expression of all alerts in all supplied groups.""" - if not self.path: - return rules - for group in rules["groups"]: - rules_in_group = group.get("rules", []) - for rule in rules_in_group: - topology = {} - # if the user for some reason has provided juju_unit, we'll need to honor it - # in most cases, however, this will be empty - for label in [ - "juju_model", - "juju_model_uuid", - "juju_application", - "juju_charm", - "juju_unit", - ]: - if label in rule["labels"]: - topology[label] = rule["labels"][label] - - rule["expr"] = self.inject_label_matchers(rule["expr"], topology) - return rules - - def validate_alert_rules(self, rules: dict) -> Tuple[bool, str]: - """Will validate correctness of alert rules, returning a boolean and any errors.""" - if not self.path: - logger.debug("`cos-tool` unavailable. Not validating alert correctness.") - return True, "" - - with tempfile.TemporaryDirectory() as tmpdir: - rule_path = Path(tmpdir + "/validate_rule.yaml") - rule_path.write_text(yaml.dump(rules)) - - args = [str(self.path), "validate", str(rule_path)] - # noinspection PyBroadException - try: - self._exec(args) - return True, "" - except subprocess.CalledProcessError as e: - logger.debug("Validating the rules failed: %s", e.output.decode("utf8")) - return False, ", ".join( - [ - line - for line in e.output.decode("utf8").splitlines() - if "error validating" in line - ] - ) - - def validate_scrape_jobs(self, jobs: list) -> bool: - """Validate scrape jobs using cos-tool.""" - if not self.path: - logger.debug("`cos-tool` unavailable. Not validating scrape jobs.") - return True - conf = {"scrape_configs": jobs} - with tempfile.NamedTemporaryFile() as tmpfile: - with open(tmpfile.name, "w") as f: - f.write(yaml.safe_dump(conf)) - try: - self._exec([str(self.path), "validate-config", tmpfile.name]) - except subprocess.CalledProcessError as e: - logger.error("Validating scrape jobs failed: {}".format(e.output)) - raise - return True - - def inject_label_matchers(self, expression, topology) -> str: - """Add label matchers to an expression.""" - if not topology: - return expression - if not self.path: - logger.debug("`cos-tool` unavailable. Leaving expression unchanged: %s", expression) - return expression - args = [str(self.path), "transform"] - args.extend( - ["--label-matcher={}={}".format(key, value) for key, value in topology.items()] - ) - - args.extend(["{}".format(expression)]) - # noinspection PyBroadException - try: - return self._exec(args) - except subprocess.CalledProcessError as e: - logger.debug('Applying the expression failed: "%s", falling back to the original', e) - return expression - - def _get_tool_path(self) -> Optional[Path]: - arch = platform.machine() - arch = "amd64" if arch == "x86_64" else arch - res = "cos-tool-{}".format(arch) - try: - path = Path(res).resolve(strict=True) - return path - except (FileNotFoundError, OSError): - logger.debug('Could not locate cos-tool at: "{}"'.format(res)) - return None - - def _exec(self, cmd) -> str: - result = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - return result.stdout.decode("utf-8").strip()