From 42c51e75c47e1675034ebe9c8c3dffcd63d35281 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Sun, 22 Jun 2025 16:30:05 +0800 Subject: [PATCH 01/15] Initial charm --- .custom_wordlist.txt | 4 + .github/pull_request_template.yaml | 6 +- .github/workflows/integration_test.yaml | 6 +- .github/workflows/test.yaml | 1 + .licenserc.yaml | 7 +- .woke.yaml | 7 + README.md | 77 +- charmcraft.yaml | 108 +- .../explanation/charm-architecture.md | 122 -- docs-template/explanation/security.md | 26 - docs-template/how-to/back-up-restore.md | 9 - docs-template/how-to/contribute.md | 15 - docs-template/how-to/integrate-with-cos.md | 7 - docs-template/how-to/upgrade.md | 10 - docs-template/index.md | 57 - docs-template/reference/actions.md | 5 - docs-template/reference/integrations.md | 16 - docs-template/reference/metrics.md | 24 - {docs-template => docs}/changelog.md | 0 docs/explanation/charm-architecture.md | 110 + docs/explanation/security.md | 20 + docs/how-to/contribute.md | 15 + docs/how-to/integrate-with-cos.md | 69 + docs/how-to/upgrade.md | 4 + docs/index.md | 50 + .../reference/configurations.md | 2 +- docs/reference/integrations.md | 34 + docs/reference/metrics.md | 35 + docs/tutorial.md | 117 ++ lib/charms/grafana_agent/v0/cos_agent.py | 1427 +++++++++++++ lib/charms/operator_libs_linux/v0/apt.py | 1779 +++++++++++++++++ lib/charms/operator_libs_linux/v1/systemd.py | 288 +++ pyproject.toml | 4 +- requirements.txt | 5 +- src/charm.py | 208 +- src/chrony.py | 401 ++++ src/grafana_dashboards/chrony.json | 1212 +++++++++++ src/prometheus_alert_rules/chrony.rule | 45 + tests/conftest.py | 20 +- tests/integration/__init__.py | 2 + tests/integration/conftest.py | 198 ++ tests/integration/requirements.txt | 2 + tests/integration/test_charm.py | 74 +- tests/unit/__init__.py | 2 + tests/unit/conftest.py | 112 ++ tests/unit/requirements.txt | 3 + tests/unit/test_base.py | 75 - tests/unit/test_charm.py | 183 ++ tox.ini | 16 +- 49 files changed, 6419 insertions(+), 600 deletions(-) create mode 100644 .custom_wordlist.txt create mode 100644 .woke.yaml delete mode 100644 docs-template/explanation/charm-architecture.md delete mode 100644 docs-template/explanation/security.md delete mode 100644 docs-template/how-to/back-up-restore.md delete mode 100644 docs-template/how-to/contribute.md delete mode 100644 docs-template/how-to/integrate-with-cos.md delete mode 100644 docs-template/how-to/upgrade.md delete mode 100644 docs-template/index.md delete mode 100644 docs-template/reference/actions.md delete mode 100644 docs-template/reference/integrations.md delete mode 100644 docs-template/reference/metrics.md rename {docs-template => docs}/changelog.md (100%) create mode 100644 docs/explanation/charm-architecture.md create mode 100644 docs/explanation/security.md create mode 100644 docs/how-to/contribute.md create mode 100644 docs/how-to/integrate-with-cos.md create mode 100644 docs/how-to/upgrade.md create mode 100644 docs/index.md rename {docs-template => docs}/reference/configurations.md (69%) create mode 100644 docs/reference/integrations.md create mode 100644 docs/reference/metrics.md create mode 100644 docs/tutorial.md create mode 100644 lib/charms/grafana_agent/v0/cos_agent.py create mode 100644 lib/charms/operator_libs_linux/v0/apt.py create mode 100644 lib/charms/operator_libs_linux/v1/systemd.py create mode 100644 src/chrony.py create mode 100644 src/grafana_dashboards/chrony.json create mode 100644 src/prometheus_alert_rules/chrony.rule create mode 100644 tests/integration/conftest.py create mode 100644 tests/integration/requirements.txt create mode 100644 tests/unit/conftest.py create mode 100644 tests/unit/requirements.txt delete mode 100644 tests/unit/test_base.py create mode 100644 tests/unit/test_charm.py diff --git a/.custom_wordlist.txt b/.custom_wordlist.txt new file mode 100644 index 0000000..68a0460 --- /dev/null +++ b/.custom_wordlist.txt @@ -0,0 +1,4 @@ +chrony +Chrony +PPMs +reachability \ No newline at end of file diff --git a/.github/pull_request_template.yaml b/.github/pull_request_template.yaml index 3633e3d..bea9d6a 100644 --- a/.github/pull_request_template.yaml +++ b/.github/pull_request_template.yaml @@ -8,15 +8,15 @@ Applicable spec: -### Juju Events Changes +### Juju events changes -### Module Changes +### Module changes -### Library Changes +### Library changes diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 19a5606..fc0ee36 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -8,11 +8,7 @@ jobs: uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main secrets: inherit with: - load-test-enabled: false - load-test-run-args: "-e LOAD_TEST_HOST=localhost" - trivy-fs-enabled: true - trivy-image-config: "trivy.yaml" self-hosted-runner: true self-hosted-runner-label: "edge" juju-channel: '3/stable' - channel: '1.32-strict/stable' + provider: 'lxd' diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bd1426c..21f4024 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -10,3 +10,4 @@ jobs: with: self-hosted-runner: true self-hosted-runner-label: "edge" + vale-style-check: true diff --git a/.licenserc.yaml b/.licenserc.yaml index 2994aef..3315013 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -6,8 +6,8 @@ header: Copyright [year] [owner] See LICENSE file for licensing details. pattern: | - Copyright \d{4} Canonical Ltd. - See LICENSE file for licensing details. + Copyright \d{4} Canonical Ltd. + See LICENSE file for licensing details. paths: - '**' paths-ignore: @@ -23,4 +23,7 @@ header: - 'trivy.yaml' - 'pyproject.toml' - 'zap_rules.tsv' + - 'lib/**' + - 'src/grafana_dashboards/**' + - 'src/prometheus_alert_rules/**' comment: on-failure diff --git a/.woke.yaml b/.woke.yaml new file mode 100644 index 0000000..1a76403 --- /dev/null +++ b/.woke.yaml @@ -0,0 +1,7 @@ +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +ignore_files: + # Ignore apt charm library as it uses non compliant terminology: + # man-in-the-middle. + - lib/charms/operator_libs_linux/v0/apt.py diff --git a/README.md b/README.md index 206e585..7c7f28c 100644 --- a/README.md +++ b/README.md @@ -1,41 +1,28 @@ - - -# platform-engineering-charm-template - +A [Juju](https://juju.is/) [subordinate charm](https://documentation.ubuntu.com/juju/latest/reference/charm/#subordinate) +that deploys and configures Chrony as an NTP client. -Describe your charm in 1-2 sentences. Include the software that the charm deploys (if applicable), and the substrate (VM/K8s). +Like any Juju charm, it supports one-line deployment, configuration, +integration, scaling, and more. Specifically, the Chrony client charm +can: -Like any Juju charm, this charm supports one-line deployment, configuration, integration, scaling, and more. For Charmed {Name}, this includes: -* list or summary of app-specific features +* Install Chrony as an NTP client replacing the system default NTP client +* Configure time sources +* Integrate with COS for time tracking status observability -For information about how to deploy, integrate, and manage this charm, see the Official [platform-engineering-charm-template Documentation](external link). +For information on deploying, integrating, and managing this charm, see +the official [Chrony Client documentation](https://charmhub.io/chrony-client). ## Get started - +To begin, refer to the [Getting Started](./docs/tutorial.md) tutorial +for step-by-step instructions. ### Basic operations -## (Optional) Integrations - - ## Learn more - -* [Read more]() -* [Developer documentation]() -* [Official webpage]() -* [Troubleshooting]() +* [Read more](https://charmhub.io/chrony-client) +* [Developer documentation](https://chrony-project.org/documentation.html) +* [Official webpage](https://chrony-project.org/) +* [Troubleshooting](https://matrix.to/#/#charmhub-charmdev:ubuntu.com) ## Project and community -* [Issues]() -* [Contributing]() -* [Matrix]() -* [Launchpad]() - -## (Optional) Licensing and trademark +* [Issues](https://github.com/canonical/chrony-client-operator/issues) +* [Contributing](./CONTRIBUTING.md) +* [Matrix](https://matrix.to/#/#charmhub-charmdev:ubuntu.com) diff --git a/charmcraft.yaml b/charmcraft.yaml index 9ad69f6..8ad9abf 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -1,65 +1,67 @@ # Copyright 2025 Canonical Ltd. # See LICENSE file for licensing details. -# This file configures Charmcraft. -# See https://canonical-charmcraft.readthedocs-hosted.com/stable/howto/manage-charmcraft/ -# for guidance. -type: charm -name: is-charms-template -title: Charm Template -summary: A very short one-line summary of the charm. +name: chrony-client +title: Chrony Client Charm +summary: Chrony NTP client charm. links: - documentation: https://discourse.charmhub.io - issues: https://github.com/canonical/is-charms-template-repo/issues - source: https://github.com/canonical/is-charms-template-repo + issues: https://github.com/canonical/chrony-client-operator/issues + source: https://github.com/canonical/chrony-client-operator contact: https://launchpad.net/~canonical-is-devops description: | - A single sentence that says what the charm is, concisely and memorably. - - A paragraph of one to three short sentences, that describe what the charm does. - - A third paragraph that explains what need the charm meets. - - Finally, a paragraph that describes whom the charm is useful for. - -bases: - - build-on: - - name: ubuntu - channel: "22.04" - run-on: - - name: ubuntu - channel: "22.04" - -# The containers and resources metadata apply to Kubernetes charms only. -# Remove them if not required. + A [Juju](https://juju.is/) [charm](https://juju.is/docs/olm/charmed-operators) + for deploying and managing the [Chrony](https://chrony-project.org) NTP server + in your systems. + + This charm simplifies the configuration and maintenance of `chrony` as a + NTP client. -# Your workload’s containers. -containers: - httpbin: - resource: httpbin-image - -# This field populates the Resources tab on Charmhub. -resources: - # An OCI image resource for each container listed above. - # You may remove this if your charm will run without a workload sidecar container. - httpbin-image: - type: oci-image - description: OCI image for httpbin - # The upstream-source field is ignored by Juju. It is included here as a reference - # so the integration testing suite knows which image to deploy during testing. This field - # is also used by the 'canonical/charming-actions' Github action for automated releasing. - upstream-source: kennethreitz/httpbin +type: charm +subordinate: true -# See https://canonical-charmcraft.readthedocs-hosted.com/stable/reference/files/config-yaml-file/ -# for guidance. config: options: - # An example config option to customise the log level of the workload - log-level: - description: | - Configures the log level of gunicorn. - - Acceptable values are: "info", "debug", "warning", "error" and "critical" - default: "info" + sources: + description: >- + Time sources for the Chrony server are specified as a comma-separated list of URLs. + Currently, the Chrony charm supports using only another NTP server as the time source. + The format for NTP source URLs is as follows: `ntp://host[:port][?options]`, where the options are the same as + Chrony pool configuration settings (https://chrony-project.org/doc/4.5/chrony.conf.html). + For NTP sources that support NTS, the URL format is: `nts://host[:nts-ke-port][?options]`. + The options for NTS URLs are the same as those for NTP sources. + Here are some examples of supported sources: + `ntp://ntp.example.com` + `ntp://ntp.example.com:1234` + `ntp://ntp.example.com?iburst=true&maxsources=2` + `nts://ntp.example.com` type: string + default: |- + ntp://ntp.ubuntu.com?iburst=true&maxsources=4, + ntp://0.ubuntu.pool.ntp.org?iburst=true&maxsources=1, + ntp://1.ubuntu.pool.ntp.org?iburst=true&maxsources=1, + ntp://2.ubuntu.pool.ntp.org?iburst=true&maxsources=2 + +requires: + juju-info: + interface: juju-info + scope: container + +provides: + cos-agent: + interface: cos_agent + +platforms: + ubuntu@24.04:amd64: + ubuntu@22.04:amd64: +parts: + charm: + build-snaps: + - rustup + override-build: | + rustup default stable + craftctl default + build-packages: + - libffi-dev + - libssl-dev + - pkg-config diff --git a/docs-template/explanation/charm-architecture.md b/docs-template/explanation/charm-architecture.md deleted file mode 100644 index 7aeec6b..0000000 --- a/docs-template/explanation/charm-architecture.md +++ /dev/null @@ -1,122 +0,0 @@ -# Charm architecture - -Add overview material here: - -1. What kind of application is it? What kind of software does it use? -2. Describe Pebble services. - - - -## High-level overview of deployment - -The following diagram shows a typical deployment of the charm. - - - - -## Charm architecture - -The following diagram shows the architecture of the charm: - - - -### Containers - -Configuration files for the containers can be found in the respective directories that define the rock. - - - -## OCI images - -We use [Rockcraft](https://canonical-rockcraft.readthedocs-hosted.com/en/latest/) to build OCI Images for . -The images are defined in [ rock](link to rock). -They are published to [Charmhub](https://charmhub.io/), the official repository of charms. - -> See more: [How to publish your charm on Charmhub](https://canonical-charmcraft.readthedocs-hosted.com/en/stable/howto/manage-charms/#publish-a-charm-on-charmhub) - -## Metrics - - - -## Juju events - -For this charm, the following Juju events are observed: - - - -> See more in the Juju docs: [Hook](https://documentation.ubuntu.com/juju/latest/user/reference/hook/) - -## Charm code overview - -The `src/charm.py` is the default entry point for a charm and has the Python class which inherits -from CharmBase. CharmBase is the base class from which all charms are formed, defined -by [Ops](https://ops.readthedocs.io/en/latest/index.html) (Python framework for developing charms). - -> See more in the Juju docs: [Charm](https://documentation.ubuntu.com/juju/latest/user/reference/charm/) - -The `__init__` method guarantees that the charm observes all events relevant to its operation and handles them. - -Take, for example, when a configuration is changed by using the CLI. - -1. User runs the configuration command: - -```bash -juju config -``` - -2. A `config-changed` event is emitted. -3. In the `__init__` method is defined how to handle this event like this: - -```python -self.framework.observe(self.on.config_changed, self._on_config_changed) -``` - -4. The method `_on_config_changed`, for its turn, will take the necessary actions such as waiting for all the relations to be ready and then configuring the containers. diff --git a/docs-template/explanation/security.md b/docs-template/explanation/security.md deleted file mode 100644 index af25027..0000000 --- a/docs-template/explanation/security.md +++ /dev/null @@ -1,26 +0,0 @@ -# Security overview - - \ No newline at end of file diff --git a/docs-template/how-to/back-up-restore.md b/docs-template/how-to/back-up-restore.md deleted file mode 100644 index 81fb56f..0000000 --- a/docs-template/how-to/back-up-restore.md +++ /dev/null @@ -1,9 +0,0 @@ -# How to back up and restore - - \ No newline at end of file diff --git a/docs-template/how-to/contribute.md b/docs-template/how-to/contribute.md deleted file mode 100644 index c842f82..0000000 --- a/docs-template/how-to/contribute.md +++ /dev/null @@ -1,15 +0,0 @@ -# How to contribute - - - -Our documentation is hosted on the [Charmhub forum](link-to-charmhub-overview-page) to enable collaboration. -Please use the "Help us improve this documentation" links on each documentation page to either -directly change something you see that's wrong, ask a question, or make a suggestion about a potential -change via the comments section. - -Our documentation is also available alongside the [source code on GitHub](link-to-github-repo). -You may open a pull request with your documentation changes, or you can -[file a bug](link-to-issues) to provide constructive feedback or suggestions. - -See [CONTRIBUTING.md](link-to-contributing-md) -for information on contributing to the source code. \ No newline at end of file diff --git a/docs-template/how-to/integrate-with-cos.md b/docs-template/how-to/integrate-with-cos.md deleted file mode 100644 index 7f4f9de..0000000 --- a/docs-template/how-to/integrate-with-cos.md +++ /dev/null @@ -1,7 +0,0 @@ -# Integrate with COS - - \ No newline at end of file diff --git a/docs-template/how-to/upgrade.md b/docs-template/how-to/upgrade.md deleted file mode 100644 index 7e15ae1..0000000 --- a/docs-template/how-to/upgrade.md +++ /dev/null @@ -1,10 +0,0 @@ -# How to upgrade - - \ No newline at end of file diff --git a/docs-template/index.md b/docs-template/index.md deleted file mode 100644 index b9db875..0000000 --- a/docs-template/index.md +++ /dev/null @@ -1,57 +0,0 @@ -# Operator - - - -A [Juju](https://juju.is/) [charm](https://documentation.ubuntu.com/juju/3.6/reference/charm/) deploying and managing on -Kubernetes. - - - -Like any Juju charm, this charm supports one-line deployment, configuration, integration, scaling, and more. -For , this includes: -* list or summary of app-specific features - -The charm allows for deployment on many different Kubernetes platforms, from [MicroK8s](https://microk8s.io/) to -[Charmed Kubernetes](https://ubuntu.com/kubernetes) to public cloud Kubernetes offerings. - - - -This charm will make operating simple and straightforward for DevOps or SRE teams through Juju's clean interface. - -## In this documentation - -| | | -|--|--| -| [Tutorials](link to tutorial)
Get started - a hands-on introduction to using the charm for new users
| [How-to guides](link to how-to guide)
Step-by-step guides covering key operations and common tasks | -| [Reference](link to reference)
Technical information - specifications, APIs, architecture | [Explanation](link to explanation)
Concepts - discussion and clarification of key topics | - -## Contributing to this documentation - -Documentation is an important part of this project, and we take the same open-source approach -to the documentation as the code. As such, we welcome community contributions, suggestions, and -constructive feedback on our documentation. -See [How to contribute](link to contribute page) for more information. - - -If there's a particular area of documentation that you'd like to see that's missing, please -[file a bug](link to issues page). - -## Project and community - -The Operator is a member of the Ubuntu family. It's an open-source project that warmly welcomes community -projects, contributions, suggestions, fixes, and constructive feedback. - -- [Code of conduct](https://ubuntu.com/community/code-of-conduct) -- [Get support](https://discourse.charmhub.io/) -- [Join our online chat](https://matrix.to/#/#charmhub-charmdev:ubuntu.com) -- [Contribute](link to Contribute page) - -Thinking about using the Operator for your next project? -[Get in touch](https://matrix.to/#/#charmhub-charmdev:ubuntu.com)! - -# Contents - -1. [Tutorial](link to tutorial) -1. [How-to](link to how-to) -1. [Reference](link to reference) -1. [Explanation](link to explanation) diff --git a/docs-template/reference/actions.md b/docs-template/reference/actions.md deleted file mode 100644 index 5fac2eb..0000000 --- a/docs-template/reference/actions.md +++ /dev/null @@ -1,5 +0,0 @@ -# Actions - -See [Actions](link to actions page). - -> Read more about actions in the Juju docs: [Action](https://documentation.ubuntu.com/juju/latest/user/reference/action/) diff --git a/docs-template/reference/integrations.md b/docs-template/reference/integrations.md deleted file mode 100644 index 0d4d824..0000000 --- a/docs-template/reference/integrations.md +++ /dev/null @@ -1,16 +0,0 @@ -# Integrations - - - -### Integration example - -_Interface_: -_Supported charms_: - -Description here. - -Example integrate command: - -``` -juju integrate : -``` diff --git a/docs-template/reference/metrics.md b/docs-template/reference/metrics.md deleted file mode 100644 index 400de84..0000000 --- a/docs-template/reference/metrics.md +++ /dev/null @@ -1,24 +0,0 @@ -## Metrics - - \ No newline at end of file diff --git a/docs-template/changelog.md b/docs/changelog.md similarity index 100% rename from docs-template/changelog.md rename to docs/changelog.md diff --git a/docs/explanation/charm-architecture.md b/docs/explanation/charm-architecture.md new file mode 100644 index 0000000..8a2e9f9 --- /dev/null +++ b/docs/explanation/charm-architecture.md @@ -0,0 +1,110 @@ +# Charm architecture + +At its core, the Chrony client charm is a simple Python program that +installs and configures `chrony` and `chrony_exporter`. + +The Chrony client charm is a subordinate charm which is a charm designed +to be deployed adjacent to another charm and to augment the +functionality of that charm. In this case, it helps to set up Chrony as +a NTP client. + + +## High-level overview of Chrony client charm deployment + + +The following diagram shows a typical deployment of the Chrony client +charm in a VM environment. The principal charm here can be any +non-subordinate machine charm. + +```mermaid +C4Context + title Component diagram for Chrony client charm + + System_Boundary(vm, "VM machine") { + Container(principal, "Principal charm") + Container_Boundary(chrony-client, "Chrony client charm") { + Component(chrony, "Chrony") + Component(chrony-exporter, "Chrony exporter") + } + Container_Boundary(grafana-agent-charm, "Grafana agent charm") { + Component(grafana-agent, "Grafana agent") + } + Rel(chrony-exporter, grafana-agent, "Prometheus metrics") + UpdateRelStyle(chrony-exporter, grafana-agent, $offsetX="-50", $offsetY="10") + } +``` + +## Metrics + +See [metrics](../reference/metrics.md) for more information. + +## Juju events + +Juju events allow progression of the charm through its lifecycle and +encapsulate part of the execution context of a charm. Below is a list of +observed events for the Chrony client charm and how the charm reacts +to each event. For more information about the charm’s lifecycle in +general, refer to the charm’s lifecycle [documentation](https://canonical-juju.readthedocs-hosted.com/en/latest/user/reference/hook/). + +### `install` + +The `install` event is emitted once per unit at the beginning of a +charm’s lifecycle. The charm will install `chrony` and `chrony_exporter` +during this event. See the documentation on the [`install` event](https://documentation.ubuntu.com/juju/latest/reference/hook/index.html#install). + +### `upgrade-charm` + +The `upgrade-charm` hook always runs once immediately after the charm +directory contents have been changed by an unforced charm upgrade +operation and may run after a forced upgrade, but it will not run +following a forced upgrade from an existing error state. During this +event, the Chrony client charm will upgrade the installed `chrony` or +`chrony_exporter`. See the documentation on the [`upgrade-charm` event](https://documentation.ubuntu.com/juju/latest/reference/hook/index.html#hook-upgrade-charm). + +### `config-changed` + +The `config-changed` hook always runs once immediately after the initial +install, after leader-elected hooks, and after the `upgrade-charm` hook. +It also runs whenever application configuration changes. During this +event, the Chrony client charm will update the configuration of `chrony` +and may restart the `chrony` service if the configuration has changed. +See the documentation on the [`config-changed` event](https://documentation.ubuntu.com/juju/latest/reference/hook/index.html#config-changed). + +### `remove` +The remove event is emitted only once per unit: when the Juju controller +is ready to remove the unit completely. All necessary steps for handling +removal should be handled there. During this event, the Chrony client +charm remove some installed packages and reset the chrony configuration +back to default. See the documentation on the [`remove` event](https://documentation.ubuntu.com/juju/latest/reference/hook/index.html#remove). + +## Charm code overview + +The `src/charm.py` is the default entry point for a charm and has the +`ChronyClientCharm` Python class which inherits from `CharmBase`. +`CharmBase` is the base class from which all charms are formed, defined +by [Ops](https://ops.readthedocs.io/en/latest/index.html) (Python +framework for developing charms). + +> See more in the Juju docs: [Charm](https://documentation.ubuntu.com/juju/latest/user/reference/charm/) + +The `__init__` method guarantees that the charm observes all events +relevant to its operation and handles them. + +Take, for example, when a configuration is changed by using the CLI. + +1. User runs the configuration command: + +```bash +juju config chrony-client sources=ntp://0.pool.ntp.org +``` + +2. A `config-changed` event is emitted. +3. In the `__init__` method is defined how to handle this event like this: + +```python +self.framework.observe(self.on.config_changed, self._on_config_changed) +``` + +4. The method `_on_config_changed`, for its turn, will take the + necessary actions such as waiting for all the relations to be ready + and then configuring the containers. diff --git a/docs/explanation/security.md b/docs/explanation/security.md new file mode 100644 index 0000000..01a698f --- /dev/null +++ b/docs/explanation/security.md @@ -0,0 +1,20 @@ +# Security overview + + + +## Risks + +The Chrony client charm is a simple charm with a minimal attack surface. +The Chrony service is configured as a pure NTP client, and the Chrony +client charm does not expose any ports. The Chrony exporter only listens +on localhost. + +## Security patches + +`chrony` is installed from the Ubuntu archive, and security patches are +delivered through Ubuntu archive updates. Use Ubuntu Pro for faster +security responses. Learn more about [Ubuntu Pro in Juju charms](https://charmhub.io/ubuntu-advantage). + +`chrony_exporter` is installed from the Platform Engineering team’s +PPA (`ppa:canonical-is-devops/chrony-charm`) and maintained by the +Platform Engineering team. diff --git a/docs/how-to/contribute.md b/docs/how-to/contribute.md new file mode 100644 index 0000000..6daf95b --- /dev/null +++ b/docs/how-to/contribute.md @@ -0,0 +1,15 @@ +# How to contribute + +Our documentation is hosted on the [Charmhub forum](https://charmhub.io/chrony-client) +to enable collaboration. Please use the "Help us improve this documentation" +links on each documentation page to either directly change something you +see that's wrong, ask a question, or make a suggestion about a potential +change via the comments section. + +Our documentation is also available alongside the [source code on GitHub](https://github.com/canonical/chrony-client-operator). +You may open a pull request with your documentation changes, or you can +[file a bug](https://github.com/canonical/chrony-client-operator/issues) +to provide constructive feedback or suggestions. + +See [CONTRIBUTING.md](https://github.com/canonical/chrony-client-operator/blob/main/CONTRIBUTING.md) +for information on contributing to the source code. \ No newline at end of file diff --git a/docs/how-to/integrate-with-cos.md b/docs/how-to/integrate-with-cos.md new file mode 100644 index 0000000..2768a9a --- /dev/null +++ b/docs/how-to/integrate-with-cos.md @@ -0,0 +1,69 @@ + +# Integrate with COS + + +## Prerequisites + +The COS integration for the Chrony client charm is provided by +the [Grafana Agent charm](https://charmhub.io/grafana-agent). Before +integrating the COS charms, you must first integrate the Chrony client +charm with the Grafana Agent charm. Because the Grafana Agent charm is +also a subordinate charm, you cannot directly relate it to the Chrony +client charm. Instead, integrate the Grafana Agent charm with a +principal charm first, then relate it to the Chrony client charm. + +Assuming you have already integrated the Chrony client charm with the +Ubuntu charm as the principal charm: + +```bash +juju deploy chrony-client +juju relate chrony-client:juju-info ubuntu +``` + +Use the Grafana Agent charm’s `juju-info` interface to relate it to the +principal charm: + +```bash +juju relate grafana-agent:juju-info ubuntu +``` + +Then relate the Chrony client charm to the Grafana Agent charm. + + +## Integrate with the Prometheus K8s operator + + +Deploy and relate +the [`prometheus-k8s`](https://charmhub.io/prometheus-k8s) charm with the +Grafana Agent charm through the `send-remote-write` relation using the +`prometheus_remote_write` interface. The Grafana Agent will push the +Prometheus metrics collected from the Chrony exporter to the Prometheus +charm. + +Because the Prometheus charm is a Kubernetes charm, you must establish a +cross-model relation. For more information on cross-model relations and +how to add one, see [the cross-model relation documentation](https://documentation.ubuntu.com/juju/latest/reference/relation/#cross-model). + +```bash +juju consume cos-juju-controller:cos-juju-user/cos-model.receive-remote-write +juju relate grafana-agent:send-remote-write receive-remote-write +``` + + +## Integrate with the Grafana K8s operator + + +Deploy and relate the [`grafana-k8s`](https://charmhub.io/grafana-k8s) +charm with the Grafana Agent charm through the +`grafana-dashboards-provider` relation using the `grafana_dashboard` +interface. The Grafana Agent will relay the dashboards provided by the +Chrony client charm to the Grafana charm. + +As with the Prometheus charm, the Grafana charm is a Kubernetes charm, +so you must establish a cross-model relation. For details, +see [the cross-model relation documentation](https://documentation.ubuntu.com/juju/latest/reference/relation/#cross-model). + +```bash +juju consume cos-juju-controller:cos-juju-user/cos-model.grafana-dashboard +juju relate grafana-agent:grafana-dashboards-provider grafana-dashboard +``` diff --git a/docs/how-to/upgrade.md b/docs/how-to/upgrade.md new file mode 100644 index 0000000..b9419b1 --- /dev/null +++ b/docs/how-to/upgrade.md @@ -0,0 +1,4 @@ +# How to upgrade + +You can use the [`juju refresh` command](https://documentation.ubuntu.com/juju/latest/reference/juju-cli/list-of-juju-cli-commands/refresh/) +to upgrade the Chrony client charm. No additional operations are needed. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..56bc205 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,50 @@ +# Chrony client operator + +A [Juju](https://juju.is/) [subordinate charm](https://documentation.ubuntu.com/juju/latest/reference/charm/#subordinate) +install and managing Chrony as a NTP client. + +Like any Juju charm, it supports one-line deployment, configuration, +integration, scaling, and more. Specifically, the Chrony client charm +can: + +* Install Chrony as an NTP client +* Configure time sources +* Integrate with COS for time tracking status observability + +The Chrony client charm allows for deployment on many different machine +platforms, from [MAAS](https://maas.io/) to [Charmed OpenStack](https://ubuntu.com/openstack) +to public cloud offerings. + +This charm will make managing Chrony as NTP client simple and +straightforward for DevOps or SRE teams through Juju's clean interface. + +## In this documentation + +| | | +|---------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------| +| [Tutorials](./tutorial.md)
Get started - a hands-on introduction to using the charm for new users
| [How-to guides](./how-to)
Step-by-step guides covering key operations and common tasks | +| [Reference](./reference)
Technical information - specifications, APIs, architecture | [Explanation](./explanation)
Concepts - discussion and clarification of key topics | + +## Contributing to this documentation + +Documentation is an important part of this project, and we take the same open-source approach +to the documentation as the code. As such, we welcome community contributions, suggestions, and +constructive feedback on our documentation. +See [How to contribute](./how-to/contribute.md) for more information. + + +If there's a particular area of documentation that you'd like to see that's missing, please +[file a bug](https://github.com/canonical/chrony-client-operator/issues). + +## Project and community + +The Chrony client charm is a member of the Ubuntu family. It's an open-source project that warmly welcomes community +projects, contributions, suggestions, fixes, and constructive feedback. + +- [Code of conduct](https://ubuntu.com/community/code-of-conduct) +- [Get support](https://discourse.charmhub.io/) +- [Join our online chat](https://matrix.to/#/#charmhub-charmdev:ubuntu.com) +- [Contribute](https://github.com/canonical/chrony-client-operator/blob/main/CONTRIBUTING.md) + +Thinking about using the Chrony client charm for your next project? +[Get in touch](https://matrix.to/#/#charmhub-charmdev:ubuntu.com)! diff --git a/docs-template/reference/configurations.md b/docs/reference/configurations.md similarity index 69% rename from docs-template/reference/configurations.md rename to docs/reference/configurations.md index 2577516..047baf4 100644 --- a/docs-template/reference/configurations.md +++ b/docs/reference/configurations.md @@ -1,5 +1,5 @@ # Configurations -See [Configurations](link to configurations page). +See [Configurations](https://charmhub.io/chrony-client/configurations). > Read more about configurations in the Juju docs: [Configuration](https://documentation.ubuntu.com/juju/latest/user/reference/configuration/) diff --git a/docs/reference/integrations.md b/docs/reference/integrations.md new file mode 100644 index 0000000..a1ff86f --- /dev/null +++ b/docs/reference/integrations.md @@ -0,0 +1,34 @@ +# Integrations + + + +### `cos-agent` + +_Interface_: `cos_agent` +_Supported charms_: [Grafana agent](https://charmhub.io/grafana-agent) + +Chrony client charm uses the `cos-agent` relation to provide COS-related +information, such as Prometheus metrics, Grafana dashboards, and +Prometheus alerts, to the Grafana agent. + +Example `cos-agent` integrate command: + +``` +juju integrate chrony-client:cos-agent grafana-agent +``` + +### `juju-info` + +_Interface_: `juju-info` +_Supported charms_: every charm + +The `juju-info` interface is a special and implicit relationship that +works with any charm. It is mainly useful for subordinate charms +, in this case Chrony client charm, that can add functionality to any +existing machine without the host charm being aware of it. + +Example `juju-info` integrate command: + +``` +juju integrate chrony-client:juju-info ubuntu +``` diff --git a/docs/reference/metrics.md b/docs/reference/metrics.md new file mode 100644 index 0000000..6975cf7 --- /dev/null +++ b/docs/reference/metrics.md @@ -0,0 +1,35 @@ +## Metrics + +- **`chrony_serverstats_authenticated_ntp_packets_total`**: The number of received NTP requests that were authenticated (with a symmetric key or NTS). +- **`chrony_serverstats_client_log_records_dropped_total`**: The number of client log records dropped by the server to limit the memory use. +- **`chrony_serverstats_command_packets_dropped_total`**: The number of command requests dropped by the server due to rate limiting. +- **`chrony_serverstats_command_packets_received_total`**: The number of command requests received by the server. +- **`chrony_serverstats_interleaved_ntp_packets_total`**: The number of received NTP requests that were detected to be in the interleaved mode. +- **`chrony_serverstats_ntp_packets_dropped_total`**: The number of NTP requests dropped by the server due to rate limiting. +- **`chrony_serverstats_ntp_packets_received_total`**: The number of valid NTP requests received by the server. +- **`chrony_serverstats_ntp_timestamp_span_seconds`**: The interval (in seconds) covered by the currently held NTP timestamps. +- **`chrony_serverstats_ntp_timestamps_held`**: The number of pairs of receive and transmit timestamps that the server is currently holding in memory for clients using the interleaved mode. +- **`chrony_serverstats_nts_ke_connections_accepted_total`**: The number of NTS-KE connections accepted by the server. +- **`chrony_serverstats_nts_ke_connections_dropped_total`**: The number of NTS-KE connections dropped by the server due to rate limiting. +- **`chrony_sources_last_sample_age_seconds`**: Chrony sources last good sample age in seconds +- **`chrony_sources_last_sample_error_margin_seconds`**: Chrony sources last sample margin of error in seconds +- **`chrony_sources_last_sample_offset_seconds`**: Chrony sources last sample offset in seconds +- **`chrony_sources_polling_interval_seconds`**: Chrony sources polling interval in seconds +- **`chrony_sources_reachability_ratio`**: Chrony sources ratio of packet reachability +- **`chrony_sources_reachability_success`**: Chrony sources last poll reachability success +- **`chrony_sources_state_info`**: Chrony sources state info +- **`chrony_sources_stratum`**: Chrony sources stratum +- **`chrony_tracking_frequency_ppms`**: Rate by which the system's clock would be wrong if chronyd was not correcting it, in PPMs +- **`chrony_tracking_info`**: Chrony tracking info +- **`chrony_tracking_last_offset_seconds`**: Chrony tracking last offset in seconds +- **`chrony_tracking_reference_timestamp_seconds`**: Chrony tracking Reference timestamp +- **`chrony_tracking_remote_reference`**: Chrony tracking is connected to a remote source +- **`chrony_tracking_residual_frequency_ppms`**: For the currently selected reference source, the difference between the frequency it suggests and the one currently in use, in PPMs +- **`chrony_tracking_rms_offset_seconds`**: Chrony tracking long-term average of the offset +- **`chrony_tracking_root_delay_seconds`**: This is the total of the network path delays to the stratum-1 computer from which the computer is ultimately synchronised +- **`chrony_tracking_root_dispersion_seconds`**: Chrony tracking total of all measurement errors to the NTP root +- **`chrony_tracking_skew_ppms`**: The estimated error bound on the frequency, in PPMs +- **`chrony_tracking_stratum`**: Chrony tracking client stratum +- **`chrony_tracking_system_time_seconds`**: Chrony tracking System time +- **`chrony_tracking_update_interval_seconds`**: The time elapsed since the last measurement from the reference source was processed, in seconds +- **`chrony_up`**: Whether the chrony server is up. diff --git a/docs/tutorial.md b/docs/tutorial.md new file mode 100644 index 0000000..5ac947d --- /dev/null +++ b/docs/tutorial.md @@ -0,0 +1,117 @@ + +# Deploy the Chrony client charm + + +The Chrony client charm installs and configures Chrony as the NTP client +on target systems. It also provides observability into the target +system’s time-tracking status. This tutorial will walk you through each +step of deploying the Chrony client charm. + +## What you'll need + + +- A working station, e.g., a laptop, with AMD64 architecture. +- Juju 3 installed and bootstrapped to a LXD controller. You can + accomplish this process by using a Multipass VM as outlined in this + guide: [Set up your test environment](https://canonical-juju.readthedocs-hosted.com/en/latest/user/howto/manage-your-deployment/manage-your-deployment-environment/#set-things-up) + + +## What you'll do + +- Deploy the [Ubuntu charm](https://charmhub.io/ubuntu) +- [Deploy and Chrony client charm on the Ubuntu charm](#deploy-the-chrony-client-charm-on-the-ubuntu-charm) +- [Configure time sources] + +## Set up the environment + +To be able to work inside the Multipass VM first you need to log in with +the following command: + +```bash +multipass shell my-juju-vm +``` + +[note] +If you're working locally, you don't need to do this step. +[/note] + +To manage resources effectively and to separate this tutorial's workload +from your usual work, create a new model in the MicroK8s controller +using the following command: + +```bash +juju add-model chrony-client-tutorial +``` + + +## Deploy Ubuntu charm + + +As the Chrony client charm is +a [subordinate charm](https://documentation.ubuntu.com/juju/latest/reference/charm/#subordinate), +it requires a principal charm to be deployed on. The Chrony client charm +can be deployed with any charm. In this tutorial, we will choose +the [Ubuntu charm](https://charmhub.io/ubuntu). + +```bash +juju deploy ubuntu --base ubuntu@24.04 +``` + + +## Deploy the Chrony client charm on the Ubuntu charm + + +The following commands deploy the Chrony client charm and integrate it +with the Ubuntu charm to create a principal-subordinate relation. + +```bash +juju deploy chrony-client --channel latest/edge --base ubuntu@24.04 +juju integrate chrony-client:juju-info ubuntu +``` + +Run `juju status` to see the current status of the deployment. The +output should be similar to the following: + +``` +Model Controller Cloud/Region Version SLA Timestamp +chrony-client-tutorial lxd localhost/localhost 3.6.2 unsupported 13:45:19+08:00 + +App Version Status Scale Charm Channel Rev Exposed Message +chrony-client active 1 chrony-client 1 no +ubuntu 24.04 active 1 ubuntu latest/stable 26 no + +Unit Workload Agent Machine Public address Ports Message +ubuntu/0* active idle 0 10.212.71.96 + chrony-client/0* active idle 10.212.71.96 + +Machine State Address Inst id Base AZ Message +0 started 10.212.71.96 juju-2cbd10-0 ubuntu@24.04 Running +``` + +The deployment finishes when the status shows "active" for both the +Ubuntu and Chrony client charms. + +## Configure time sources + +By default, the Chrony client charm uses Ubuntu’s NTP servers +(ntp.ubuntu.com) as its time source. You can configure the charm to use +different time sources, for example, switching to the NTP Pool servers, +by running the following command: + +```bash +juju config chrony-client sources='ntp://0.pool.ntp.org?iburst=true&maxsources=4, +ntp://1.pool.ntp.org?iburst=true&maxsources=4, +ntp://2.pool.ntp.org?iburst=true&maxsources=4, +ntp://3.pool.ntp.org?iburst=true&maxsources=4' +``` + +The charm should reach the active state when the configuration is +successful. It will end up in the blocked state if the configuration +value is invalid. + +## Clean up the environment + +Congratulations! You have successfully deployed the Chrony client charm. + +You can clean up your environment by following this guide: +[Tear down your test environment](https://canonical-juju.readthedocs-hosted.com/en/3.6/user/howto/manage-your-deployment/manage-your-deployment-environment/#tear-things-down) diff --git a/lib/charms/grafana_agent/v0/cos_agent.py b/lib/charms/grafana_agent/v0/cos_agent.py new file mode 100644 index 0000000..64c2d77 --- /dev/null +++ b/lib/charms/grafana_agent/v0/cos_agent.py @@ -0,0 +1,1427 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +r"""## Overview. + +This library can be used to manage the cos_agent relation interface: + +- `COSAgentProvider`: Use in machine charms that need to have a workload's metrics + or logs scraped, or forward rule files or dashboards to Prometheus, Loki or Grafana through + the Grafana Agent machine charm. + NOTE: Be sure to add `limit: 1` in your charm for the cos-agent relation. That is the only + way we currently have to prevent two different grafana agent apps deployed on the same VM. + +- `COSAgentConsumer`: Used in the Grafana Agent machine charm to manage the requirer side of + the `cos_agent` interface. + + +## COSAgentProvider Library Usage + +Grafana Agent machine Charmed Operator interacts with its clients using the cos_agent library. +Charms seeking to send telemetry, must do so using the `COSAgentProvider` object from +this charm library. + +Using the `COSAgentProvider` object only requires instantiating it, +typically in the `__init__` method of your charm (the one which sends telemetry). + + +```python + def __init__( + self, + charm: CharmType, + relation_name: str = DEFAULT_RELATION_NAME, + metrics_endpoints: Optional[List[_MetricsEndpointDict]] = None, + metrics_rules_dir: str = "./src/prometheus_alert_rules", + logs_rules_dir: str = "./src/loki_alert_rules", + recurse_rules_dirs: bool = False, + log_slots: Optional[List[str]] = None, + dashboard_dirs: Optional[List[str]] = None, + refresh_events: Optional[List] = None, + tracing_protocols: Optional[List[str]] = None, + scrape_configs: Optional[Union[List[Dict], Callable]] = None, + ): +``` + +### Parameters + +- `charm`: The instance of the charm that instantiates `COSAgentProvider`, typically `self`. + +- `relation_name`: If your charmed operator uses a relation name other than `cos-agent` to use + the `cos_agent` interface, this is where you have to specify that. + +- `metrics_endpoints`: In this parameter you can specify the metrics endpoints that Grafana Agent + machine Charmed Operator will scrape. The configs of this list will be merged with the configs + from `scrape_configs`. + +- `metrics_rules_dir`: The directory in which the Charmed Operator stores its metrics alert rules + files. + +- `logs_rules_dir`: The directory in which the Charmed Operator stores its logs alert rules files. + +- `recurse_rules_dirs`: This parameters set whether Grafana Agent machine Charmed Operator has to + search alert rules files recursively in the previous two directories or not. + +- `log_slots`: Snap slots to connect to for scraping logs in the form ["snap-name:slot", ...]. + +- `dashboard_dirs`: List of directories where the dashboards are stored in the Charmed Operator. + +- `refresh_events`: List of events on which to refresh relation data. + +- `tracing_protocols`: List of requested tracing protocols that the charm requires to send traces. + +- `scrape_configs`: List of standard scrape_configs dicts or a callable that returns the list in + case the configs need to be generated dynamically. The contents of this list will be merged + with the configs from `metrics_endpoints`. + + +### Example 1 - Minimal instrumentation: + +In order to use this object the following should be in the `charm.py` file. + +```python +from charms.grafana_agent.v0.cos_agent import COSAgentProvider +... +class TelemetryProviderCharm(CharmBase): + def __init__(self, *args): + ... + self._grafana_agent = COSAgentProvider(self) +``` + +### Example 2 - Full instrumentation: + +In order to use this object the following should be in the `charm.py` file. + +```python +from charms.grafana_agent.v0.cos_agent import COSAgentProvider +... +class TelemetryProviderCharm(CharmBase): + def __init__(self, *args): + ... + self._grafana_agent = COSAgentProvider( + self, + relation_name="custom-cos-agent", + metrics_endpoints=[ + # specify "path" and "port" to scrape from localhost + {"path": "/metrics", "port": 9000}, + {"path": "/metrics", "port": 9001}, + {"path": "/metrics", "port": 9002}, + ], + metrics_rules_dir="./src/alert_rules/prometheus", + logs_rules_dir="./src/alert_rules/loki", + recursive_rules_dir=True, + log_slots=["my-app:slot"], + dashboard_dirs=["./src/dashboards_1", "./src/dashboards_2"], + refresh_events=["update-status", "upgrade-charm"], + tracing_protocols=["otlp_http", "otlp_grpc"], + scrape_configs=[ + { + "job_name": "custom_job", + "metrics_path": "/metrics", + "authorization": {"credentials": "bearer-token"}, + "static_configs": [ + { + "targets": ["localhost:9003"]}, + "labels": {"key": "value"}, + }, + ], + }, + ] + ) +``` + +### Example 3 - Dynamic scrape configs generation: + +Pass a function to the `scrape_configs` to decouple the generation of the configs +from the instantiation of the COSAgentProvider object. + +```python +from charms.grafana_agent.v0.cos_agent import COSAgentProvider +... + +class TelemetryProviderCharm(CharmBase): + def generate_scrape_configs(self): + return [ + { + "job_name": "custom", + "metrics_path": "/metrics", + "static_configs": [{"targets": ["localhost:9000"]}], + }, + ] + + def __init__(self, *args): + ... + self._grafana_agent = COSAgentProvider( + self, + scrape_configs=self.generate_scrape_configs, + ) +``` + +## COSAgentConsumer Library Usage + +This object may be used by any Charmed Operator which gathers telemetry data by +implementing the consumer side of the `cos_agent` interface. +For instance Grafana Agent machine Charmed Operator. + +For this purpose the charm needs to instantiate the `COSAgentConsumer` object with one mandatory +and two optional arguments. + +### Parameters + +- `charm`: A reference to the parent (Grafana Agent machine) charm. + +- `relation_name`: The name of the relation that the charm uses to interact + with its clients that provides telemetry data using the `COSAgentProvider` object. + + If provided, this relation name must match a provided relation in metadata.yaml with the + `cos_agent` interface. + The default value of this argument is "cos-agent". + +- `refresh_events`: List of events on which to refresh relation data. + + +### Example 1 - Minimal instrumentation: + +In order to use this object the following should be in the `charm.py` file. + +```python +from charms.grafana_agent.v0.cos_agent import COSAgentConsumer +... +class GrafanaAgentMachineCharm(GrafanaAgentCharm) + def __init__(self, *args): + ... + self._cos = COSAgentRequirer(self) +``` + + +### Example 2 - Full instrumentation: + +In order to use this object the following should be in the `charm.py` file. + +```python +from charms.grafana_agent.v0.cos_agent import COSAgentConsumer +... +class GrafanaAgentMachineCharm(GrafanaAgentCharm) + def __init__(self, *args): + ... + self._cos = COSAgentRequirer( + self, + relation_name="cos-agent-consumer", + refresh_events=["update-status", "upgrade-charm"], + ) +``` +""" + +import enum +import json +import logging +import socket +from collections import namedtuple +from itertools import chain +from pathlib import Path +from typing import ( + TYPE_CHECKING, + Any, + Callable, + ClassVar, + Dict, + List, + Literal, + MutableMapping, + Optional, + Set, + Tuple, + Union, +) + +import pydantic +from cosl import DashboardPath40UID, JujuTopology, LZMABase64 +from cosl.rules import AlertRules, generic_alert_groups +from ops.charm import RelationChangedEvent +from ops.framework import EventBase, EventSource, Object, ObjectEvents +from ops.model import ModelError, Relation +from ops.testing import CharmType + +if TYPE_CHECKING: + try: + from typing import TypedDict + + class _MetricsEndpointDict(TypedDict): + path: str + port: int + + except ModuleNotFoundError: + _MetricsEndpointDict = Dict # pyright: ignore + +LIBID = "dc15fa84cef84ce58155fb84f6c6213a" +LIBAPI = 0 +LIBPATCH = 20 + +PYDEPS = ["cosl >= 0.0.50", "pydantic"] + +DEFAULT_RELATION_NAME = "cos-agent" +DEFAULT_PEER_RELATION_NAME = "peers" +DEFAULT_SCRAPE_CONFIG = { + "static_configs": [{"targets": ["localhost:80"]}], + "metrics_path": "/metrics", +} + +logger = logging.getLogger(__name__) +SnapEndpoint = namedtuple("SnapEndpoint", "owner, name") + +# Note: MutableMapping is imported from the typing module and not collections.abc +# because subscripting collections.abc.MutableMapping was added in python 3.9, but +# most of our charms are based on 20.04, which has python 3.8. + +_RawDatabag = MutableMapping[str, str] + + +class TransportProtocolType(str, enum.Enum): + """Receiver Type.""" + + http = "http" + grpc = "grpc" + + +receiver_protocol_to_transport_protocol = { + "zipkin": TransportProtocolType.http, + "kafka": TransportProtocolType.http, + "tempo_http": TransportProtocolType.http, + "tempo_grpc": TransportProtocolType.grpc, + "otlp_grpc": TransportProtocolType.grpc, + "otlp_http": TransportProtocolType.http, + "jaeger_thrift_http": TransportProtocolType.http, +} + +_tracing_receivers_ports = { + # OTLP receiver: see + # https://github.com/open-telemetry/opentelemetry-collector/tree/v0.96.0/receiver/otlpreceiver + "otlp_http": 4318, + "otlp_grpc": 4317, + # Jaeger receiver: see + # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/v0.96.0/receiver/jaegerreceiver + "jaeger_grpc": 14250, + "jaeger_thrift_http": 14268, + # Zipkin receiver: see + # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/v0.96.0/receiver/zipkinreceiver + "zipkin": 9411, +} + +ReceiverProtocol = Literal["otlp_grpc", "otlp_http", "zipkin", "jaeger_thrift_http", "jaeger_grpc"] + + +class TracingError(Exception): + """Base class for custom errors raised by tracing.""" + + +class NotReadyError(TracingError): + """Raised by the provider wrapper if a requirer hasn't published the required data (yet).""" + + +class ProtocolNotFoundError(TracingError): + """Raised if the user doesn't receive an endpoint for a protocol it requested.""" + + +class ProtocolNotRequestedError(ProtocolNotFoundError): + """Raised if the user attempts to obtain an endpoint for a protocol it did not request.""" + + +class DataValidationError(TracingError): + """Raised when data validation fails on IPU relation data.""" + + +class AmbiguousRelationUsageError(TracingError): + """Raised when one wrongly assumes that there can only be one relation on an endpoint.""" + + +# TODO we want to eventually use `DatabagModel` from cosl but it likely needs a move to common package first +if int(pydantic.version.VERSION.split(".")[0]) < 2: # type: ignore + + class DatabagModel(pydantic.BaseModel): # type: ignore + """Base databag model.""" + + class Config: + """Pydantic config.""" + + # ignore any extra fields in the databag + extra = "ignore" + """Ignore any extra fields in the databag.""" + allow_population_by_field_name = True + """Allow instantiating this class by field name (instead of forcing alias).""" + + _NEST_UNDER = None + + @classmethod + def load(cls, databag: MutableMapping): + """Load this model from a Juju databag.""" + if cls._NEST_UNDER: + return cls.parse_obj(json.loads(databag[cls._NEST_UNDER])) + + try: + data = { + k: json.loads(v) + for k, v in databag.items() + # Don't attempt to parse model-external values + if k in {f.alias for f in cls.__fields__.values()} + } + except json.JSONDecodeError as e: + msg = f"invalid databag contents: expecting json. {databag}" + logger.error(msg) + raise DataValidationError(msg) from e + + try: + return cls.parse_raw(json.dumps(data)) # type: ignore + except pydantic.ValidationError as e: + msg = f"failed to validate databag: {databag}" + logger.debug(msg, exc_info=True) + raise DataValidationError(msg) from e + + def dump(self, databag: Optional[MutableMapping] = None, clear: bool = True): + """Write the contents of this model to Juju databag. + + :param databag: the databag to write the data to. + :param clear: ensure the databag is cleared before writing it. + """ + if clear and databag: + databag.clear() + + if databag is None: + databag = {} + + if self._NEST_UNDER: + databag[self._NEST_UNDER] = self.json(by_alias=True) + return databag + + dct = self.dict() + for key, field in self.__fields__.items(): # type: ignore + value = dct[key] + databag[field.alias or key] = json.dumps(value) + + return databag + +else: + from pydantic import ConfigDict + + class DatabagModel(pydantic.BaseModel): + """Base databag model.""" + + model_config = ConfigDict( + # ignore any extra fields in the databag + extra="ignore", + # Allow instantiating this class by field name (instead of forcing alias). + populate_by_name=True, + # Custom config key: whether to nest the whole datastructure (as json) + # under a field or spread it out at the toplevel. + _NEST_UNDER=None, # type: ignore + arbitrary_types_allowed=True, + ) + """Pydantic config.""" + + @classmethod + def load(cls, databag: MutableMapping): + """Load this model from a Juju databag.""" + nest_under = cls.model_config.get("_NEST_UNDER") # type: ignore + if nest_under: + return cls.model_validate(json.loads(databag[nest_under])) # type: ignore + + try: + data = { + k: json.loads(v) + for k, v in databag.items() + # Don't attempt to parse model-external values + if k in {(f.alias or n) for n, f in cls.__fields__.items()} + } + except json.JSONDecodeError as e: + msg = f"invalid databag contents: expecting json. {databag}" + logger.error(msg) + raise DataValidationError(msg) from e + + try: + return cls.model_validate_json(json.dumps(data)) # type: ignore + except pydantic.ValidationError as e: + msg = f"failed to validate databag: {databag}" + logger.debug(msg, exc_info=True) + raise DataValidationError(msg) from e + + def dump(self, databag: Optional[MutableMapping] = None, clear: bool = True): + """Write the contents of this model to Juju databag. + + :param databag: the databag to write the data to. + :param clear: ensure the databag is cleared before writing it. + """ + if clear and databag: + databag.clear() + + if databag is None: + databag = {} + nest_under = self.model_config.get("_NEST_UNDER") + if nest_under: + databag[nest_under] = self.model_dump_json( # type: ignore + by_alias=True, + # skip keys whose values are default + exclude_defaults=True, + ) + return databag + + dct = self.model_dump() # type: ignore + for key, field in self.model_fields.items(): # type: ignore + value = dct[key] + if value == field.default: + continue + databag[field.alias or key] = json.dumps(value) + + return databag + + +class CosAgentProviderUnitData(DatabagModel): + """Unit databag model for `cos-agent` relation.""" + + # The following entries are the same for all units of the same principal. + # Note that the same grafana agent subordinate may be related to several apps. + # this needs to make its way to the gagent leader + metrics_alert_rules: dict + log_alert_rules: dict + dashboards: List[str] + # subordinate is no longer used but we should keep it until we bump the library to ensure + # we don't break compatibility. + subordinate: Optional[bool] = None + + # The following entries may vary across units of the same principal app. + # this data does not need to be forwarded to the gagent leader + metrics_scrape_jobs: List[Dict] + log_slots: List[str] + + # Requested tracing protocols. + tracing_protocols: Optional[List[str]] = None + + # when this whole datastructure is dumped into a databag, it will be nested under this key. + # while not strictly necessary (we could have it 'flattened out' into the databag), + # this simplifies working with the model. + KEY: ClassVar[str] = "config" + + +class CosAgentPeersUnitData(DatabagModel): + """Unit databag model for `peers` cos-agent machine charm peer relation.""" + + # We need the principal unit name and relation metadata to be able to render identifiers + # (e.g. topology) on the leader side, after all the data moves into peer data (the grafana + # agent leader can only see its own principal, because it is a subordinate charm). + unit_name: str + relation_id: str + relation_name: str + + # The only data that is forwarded to the leader is data that needs to go into the app databags + # of the outgoing o11y relations. + metrics_alert_rules: Optional[dict] + log_alert_rules: Optional[dict] + dashboards: Optional[List[str]] + + # when this whole datastructure is dumped into a databag, it will be nested under this key. + # while not strictly necessary (we could have it 'flattened out' into the databag), + # this simplifies working with the model. + KEY: ClassVar[str] = "config" + + @property + def app_name(self) -> str: + """Parse out the app name from the unit name. + + TODO: Switch to using `model_post_init` when pydantic v2 is released? + https://github.com/pydantic/pydantic/issues/1729#issuecomment-1300576214 + """ + return self.unit_name.split("/")[0] + + +if int(pydantic.version.VERSION.split(".")[0]) < 2: # type: ignore + + class ProtocolType(pydantic.BaseModel): # type: ignore + """Protocol Type.""" + + class Config: + """Pydantic config.""" + + use_enum_values = True + """Allow serializing enum values.""" + + name: str = pydantic.Field( + ..., + description="Receiver protocol name. What protocols are supported (and what they are called) " + "may differ per provider.", + examples=["otlp_grpc", "otlp_http", "tempo_http"], + ) + + type: TransportProtocolType = pydantic.Field( + ..., + description="The transport protocol used by this receiver.", + examples=["http", "grpc"], + ) + +else: + + class ProtocolType(pydantic.BaseModel): + """Protocol Type.""" + + model_config = pydantic.ConfigDict( + # Allow serializing enum values. + use_enum_values=True + ) + """Pydantic config.""" + + name: str = pydantic.Field( + ..., + description="Receiver protocol name. What protocols are supported (and what they are called) " + "may differ per provider.", + examples=["otlp_grpc", "otlp_http", "tempo_http"], + ) + + type: TransportProtocolType = pydantic.Field( + ..., + description="The transport protocol used by this receiver.", + examples=["http", "grpc"], + ) + + +class Receiver(pydantic.BaseModel): + """Specification of an active receiver.""" + + protocol: ProtocolType = pydantic.Field(..., description="Receiver protocol name and type.") + url: Optional[str] = pydantic.Field( + ..., + description="""URL at which the receiver is reachable. If there's an ingress, it would be the external URL. + Otherwise, it would be the service's fqdn or internal IP. + If the protocol type is grpc, the url will not contain a scheme.""", + examples=[ + "http://traefik_address:2331", + "https://traefik_address:2331", + "http://tempo_public_ip:2331", + "https://tempo_public_ip:2331", + "tempo_public_ip:2331", + ], + ) + + +class CosAgentRequirerUnitData(DatabagModel): # noqa: D101 + """Application databag model for the COS-agent requirer.""" + + receivers: List[Receiver] = pydantic.Field( + ..., + description="List of all receivers enabled on the tracing provider.", + ) + + +class COSAgentProvider(Object): + """Integration endpoint wrapper for the provider side of the cos_agent interface.""" + + def __init__( + self, + charm: CharmType, + relation_name: str = DEFAULT_RELATION_NAME, + metrics_endpoints: Optional[List["_MetricsEndpointDict"]] = None, + metrics_rules_dir: str = "./src/prometheus_alert_rules", + logs_rules_dir: str = "./src/loki_alert_rules", + recurse_rules_dirs: bool = False, + log_slots: Optional[List[str]] = None, + dashboard_dirs: Optional[List[str]] = None, + refresh_events: Optional[List] = None, + tracing_protocols: Optional[List[str]] = None, + *, + scrape_configs: Optional[Union[List[dict], Callable]] = None, + ): + """Create a COSAgentProvider instance. + + Args: + charm: The `CharmBase` instance that is instantiating this object. + relation_name: The name of the relation to communicate over. + metrics_endpoints: List of endpoints in the form [{"path": path, "port": port}, ...]. + This argument is a simplified form of the `scrape_configs`. + The contents of this list will be merged with the contents of `scrape_configs`. + metrics_rules_dir: Directory where the metrics rules are stored. + logs_rules_dir: Directory where the logs rules are stored. + recurse_rules_dirs: Whether to recurse into rule paths. + log_slots: Snap slots to connect to for scraping logs + in the form ["snap-name:slot", ...]. + dashboard_dirs: Directory where the dashboards are stored. + refresh_events: List of events on which to refresh relation data. + tracing_protocols: List of protocols that the charm will be using for sending traces. + scrape_configs: List of standard scrape_configs dicts or a callable + that returns the list in case the configs need to be generated dynamically. + The contents of this list will be merged with the contents of `metrics_endpoints`. + """ + super().__init__(charm, relation_name) + dashboard_dirs = dashboard_dirs or ["./src/grafana_dashboards"] + + self._charm = charm + self._relation_name = relation_name + self._metrics_endpoints = metrics_endpoints or [] + self._scrape_configs = scrape_configs or [] + self._metrics_rules = metrics_rules_dir + self._logs_rules = logs_rules_dir + self._recursive = recurse_rules_dirs + self._log_slots = log_slots or [] + self._dashboard_dirs = dashboard_dirs + self._refresh_events = refresh_events or [self._charm.on.config_changed] + self._tracing_protocols = tracing_protocols + self._is_single_endpoint = charm.meta.relations[relation_name].limit == 1 + + events = self._charm.on[relation_name] + self.framework.observe(events.relation_joined, self._on_refresh) + self.framework.observe(events.relation_changed, self._on_refresh) + for event in self._refresh_events: + self.framework.observe(event, self._on_refresh) + + def _on_refresh(self, event): + """Trigger the class to update relation data.""" + relations = self._charm.model.relations[self._relation_name] + + for relation in relations: + # Before a principal is related to the grafana-agent subordinate, we'd get + # ModelError: ERROR cannot read relation settings: unit "zk/2": settings not found + # Add a guard to make sure it doesn't happen. + if relation.data and self._charm.unit in relation.data: + # Subordinate relations can communicate only over unit data. + try: + data = CosAgentProviderUnitData( + metrics_alert_rules=self._metrics_alert_rules, + log_alert_rules=self._log_alert_rules, + dashboards=self._dashboards, + metrics_scrape_jobs=self._scrape_jobs, + log_slots=self._log_slots, + tracing_protocols=self._tracing_protocols, + ) + relation.data[self._charm.unit][data.KEY] = data.json() + except ( + pydantic.ValidationError, + json.decoder.JSONDecodeError, + ) as e: + logger.error("Invalid relation data provided: %s", e) + + @property + def _scrape_jobs(self) -> List[Dict]: + """Return a prometheus_scrape-like data structure for jobs. + + https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config + """ + if callable(self._scrape_configs): + scrape_configs = self._scrape_configs() + else: + # Create a copy of the user scrape_configs, since we will mutate this object + scrape_configs = self._scrape_configs.copy() + + # Convert "metrics_endpoints" to standard scrape_configs, and add them in + for endpoint in self._metrics_endpoints: + scrape_configs.append( + { + "metrics_path": endpoint["path"], + "static_configs": [{"targets": [f"localhost:{endpoint['port']}"]}], + } + ) + + scrape_configs = scrape_configs or [DEFAULT_SCRAPE_CONFIG] + + # Augment job name to include the app name and a unique id (index) + for idx, scrape_config in enumerate(scrape_configs): + scrape_config["job_name"] = "_".join( + [self._charm.app.name, str(idx), scrape_config.get("job_name", "default")] + ) + + return scrape_configs + + @property + def _metrics_alert_rules(self) -> Dict: + """Use (for now) the prometheus_scrape AlertRules to initialize this.""" + alert_rules = AlertRules( + query_type="promql", topology=JujuTopology.from_charm(self._charm) + ) + alert_rules.add_path(self._metrics_rules, recursive=self._recursive) + alert_rules.add( + generic_alert_groups.application_rules, + group_name_prefix=JujuTopology.from_charm(self._charm).identifier, + ) + return alert_rules.as_dict() + + @property + def _log_alert_rules(self) -> Dict: + """Use (for now) the loki_push_api AlertRules to initialize this.""" + alert_rules = AlertRules(query_type="logql", topology=JujuTopology.from_charm(self._charm)) + alert_rules.add_path(self._logs_rules, recursive=self._recursive) + return alert_rules.as_dict() + + @property + def _dashboards(self) -> List[str]: + dashboards: List[str] = [] + for d in self._dashboard_dirs: + for path in Path(d).glob("*"): + with open(path, "rt") as fp: + dashboard = json.load(fp) + rel_path = str( + path.relative_to(self._charm.charm_dir) if path.is_absolute() else path + ) + # COSAgentProvider is somewhat analogous to GrafanaDashboardProvider. We need to overwrite the uid here + # because there is currently no other way to communicate the dashboard path separately. + # https://github.com/canonical/grafana-k8s-operator/pull/363 + dashboard["uid"] = DashboardPath40UID.generate(self._charm.meta.name, rel_path) + + # Add tags + tags: List[str] = dashboard.get("tags", []) + if not any(tag.startswith("charm: ") for tag in tags): + tags.append(f"charm: {self._charm.meta.name}") + dashboard["tags"] = tags + + dashboards.append(LZMABase64.compress(json.dumps(dashboard))) + return dashboards + + @property + def relations(self) -> List[Relation]: + """The tracing relations associated with this endpoint.""" + return self._charm.model.relations[self._relation_name] + + @property + def _relation(self) -> Optional[Relation]: + """If this wraps a single endpoint, the relation bound to it, if any.""" + if not self._is_single_endpoint: + objname = type(self).__name__ + raise AmbiguousRelationUsageError( + f"This {objname} wraps a {self._relation_name} endpoint that has " + "limit != 1. We can't determine what relation, of the possibly many, you are " + f"referring to. Please pass a relation instance while calling {objname}, " + "or set limit=1 in the charm metadata." + ) + relations = self.relations + return relations[0] if relations else None + + def is_ready(self, relation: Optional[Relation] = None): + """Is this endpoint ready?""" + relation = relation or self._relation + if not relation: + logger.debug(f"no relation on {self._relation_name!r}: tracing not ready") + return False + if relation.data is None: + logger.error(f"relation data is None for {relation}") + return False + if not relation.app: + logger.error(f"{relation} event received but there is no relation.app") + return False + try: + unit = next(iter(relation.units), None) + if not unit: + return False + databag = dict(relation.data[unit]) + CosAgentRequirerUnitData.load(databag) + + except (json.JSONDecodeError, pydantic.ValidationError, DataValidationError): + logger.info(f"failed validating relation data for {relation}") + return False + return True + + def get_all_endpoints( + self, relation: Optional[Relation] = None + ) -> Optional[CosAgentRequirerUnitData]: + """Unmarshalled relation data.""" + relation = relation or self._relation + if not relation or not self.is_ready(relation): + return None + unit = next(iter(relation.units), None) + if not unit: + return None + return CosAgentRequirerUnitData.load(relation.data[unit]) # type: ignore + + def _get_tracing_endpoint( + self, relation: Optional[Relation], protocol: ReceiverProtocol + ) -> str: + """Return a tracing endpoint URL if it is available or raise a ProtocolNotFoundError.""" + unit_data = self.get_all_endpoints(relation) + if not unit_data: + # we didn't find the protocol because the remote end didn't publish any data yet + # it might also mean that grafana-agent doesn't have a relation to the tracing backend + raise ProtocolNotFoundError(protocol) + receivers: List[Receiver] = [i for i in unit_data.receivers if i.protocol.name == protocol] + if not receivers: + # we didn't find the protocol because grafana-agent didn't return us the protocol that we requested + # the caller might want to verify that we did indeed request this protocol + raise ProtocolNotFoundError(protocol) + if len(receivers) > 1: + logger.warning( + f"too many receivers with protocol={protocol!r}; using first one. Found: {receivers}" + ) + + receiver = receivers[0] + if not receiver.url: + # grafana-agent isn't connected to the tracing backend yet + raise ProtocolNotFoundError(protocol) + return receiver.url + + def get_tracing_endpoint( + self, protocol: ReceiverProtocol, relation: Optional[Relation] = None + ) -> str: + """Receiver endpoint for the given protocol. + + It could happen that this function gets called before the provider publishes the endpoints. + In such a scenario, if a non-leader unit calls this function, a permission denied exception will be raised due to + restricted access. To prevent this, this function needs to be guarded by the `is_ready` check. + + Raises: + ProtocolNotRequestedError: + If the charm unit is the leader unit and attempts to obtain an endpoint for a protocol it did not request. + ProtocolNotFoundError: + If the charm attempts to obtain an endpoint when grafana-agent isn't related to a tracing backend. + """ + try: + return self._get_tracing_endpoint(relation or self._relation, protocol=protocol) + except ProtocolNotFoundError: + # let's see if we didn't find it because we didn't request the endpoint + requested_protocols = set() + relations = [relation] if relation else self.relations + for relation in relations: + try: + databag = CosAgentProviderUnitData.load(relation.data[self._charm.unit]) + except DataValidationError: + continue + + if databag.tracing_protocols: + requested_protocols.update(databag.tracing_protocols) + + if protocol not in requested_protocols: + raise ProtocolNotRequestedError(protocol, relation) + + raise + + +class COSAgentDataChanged(EventBase): + """Event emitted by `COSAgentRequirer` when relation data changes.""" + + +class COSAgentValidationError(EventBase): + """Event emitted by `COSAgentRequirer` when there is an error in the relation data.""" + + def __init__(self, handle, message: str = ""): + super().__init__(handle) + self.message = message + + def snapshot(self) -> Dict: + """Save COSAgentValidationError source information.""" + return {"message": self.message} + + def restore(self, snapshot): + """Restore COSAgentValidationError source information.""" + self.message = snapshot["message"] + + +class COSAgentRequirerEvents(ObjectEvents): + """`COSAgentRequirer` events.""" + + data_changed = EventSource(COSAgentDataChanged) + validation_error = EventSource(COSAgentValidationError) + + +class COSAgentRequirer(Object): + """Integration endpoint wrapper for the Requirer side of the cos_agent interface.""" + + on = COSAgentRequirerEvents() # pyright: ignore + + def __init__( + self, + charm: CharmType, + *, + relation_name: str = DEFAULT_RELATION_NAME, + peer_relation_name: str = DEFAULT_PEER_RELATION_NAME, + refresh_events: Optional[List[str]] = None, + ): + """Create a COSAgentRequirer instance. + + Args: + charm: The `CharmBase` instance that is instantiating this object. + relation_name: The name of the relation to communicate over. + peer_relation_name: The name of the peer relation to communicate over. + refresh_events: List of events on which to refresh relation data. + """ + super().__init__(charm, relation_name) + self._charm = charm + self._relation_name = relation_name + self._peer_relation_name = peer_relation_name + self._refresh_events = refresh_events or [self._charm.on.config_changed] + + events = self._charm.on[relation_name] + self.framework.observe( + events.relation_joined, self._on_relation_data_changed + ) # TODO: do we need this? + self.framework.observe(events.relation_changed, self._on_relation_data_changed) + self.framework.observe(events.relation_departed, self._on_relation_departed) + + for event in self._refresh_events: + self.framework.observe(event, self.trigger_refresh) # pyright: ignore + + # Peer relation events + # A peer relation is needed as it is the only mechanism for exchanging data across + # subordinate units. + # self.framework.observe( + # self.on[self._peer_relation_name].relation_joined, self._on_peer_relation_joined + # ) + peer_events = self._charm.on[peer_relation_name] + self.framework.observe(peer_events.relation_changed, self._on_peer_relation_changed) + + @property + def peer_relation(self) -> Optional["Relation"]: + """Helper function for obtaining the peer relation object. + + Returns: peer relation object + (NOTE: would return None if called too early, e.g. during install). + """ + return self.model.get_relation(self._peer_relation_name) + + def _on_peer_relation_changed(self, _): + # Peer data is used for forwarding data from principal units to the grafana agent + # subordinate leader, for updating the app data of the outgoing o11y relations. + if self._charm.unit.is_leader(): + self.on.data_changed.emit() # pyright: ignore + + def _on_relation_departed(self, event): + """Remove provider's (principal's) alert rules and dashboards from peer data when the cos-agent relation to the principal is removed.""" + if not self.peer_relation: + event.defer() + return + # empty the departing unit's alert rules and dashboards from peer data + data = CosAgentPeersUnitData( + unit_name=event.unit.name, + relation_id=str(event.relation.id), + relation_name=event.relation.name, + metrics_alert_rules={}, + log_alert_rules={}, + dashboards=[], + ) + self.peer_relation.data[self._charm.unit][ + f"{CosAgentPeersUnitData.KEY}-{event.unit.name}" + ] = data.json() + + self.on.data_changed.emit() # pyright: ignore + + def _on_relation_data_changed(self, event: RelationChangedEvent): + # Peer data is the only means of communication between subordinate units. + if not self.peer_relation: + event.defer() + return + + cos_agent_relation = event.relation + if not event.unit or not cos_agent_relation.data.get(event.unit): + return + principal_unit = event.unit + + # Coherence check + units = cos_agent_relation.units + if len(units) > 1: + # should never happen + raise ValueError( + f"unexpected error: subordinate relation {cos_agent_relation} " + f"should have exactly one unit" + ) + + if not (raw := cos_agent_relation.data[principal_unit].get(CosAgentProviderUnitData.KEY)): + return + + if not (provider_data := self._validated_provider_data(raw)): + return + + # write enabled receivers to cos-agent relation + try: + self.update_tracing_receivers() + except ModelError: + raise + + # Copy data from the cos_agent relation to the peer relation, so the leader could + # follow up. + # Save the originating unit name, so it could be used for topology later on by the leader. + data = CosAgentPeersUnitData( # peer relation databag model + unit_name=event.unit.name, + relation_id=str(event.relation.id), + relation_name=event.relation.name, + metrics_alert_rules=provider_data.metrics_alert_rules, + log_alert_rules=provider_data.log_alert_rules, + dashboards=provider_data.dashboards, + ) + self.peer_relation.data[self._charm.unit][ + f"{CosAgentPeersUnitData.KEY}-{event.unit.name}" + ] = data.json() + + # We can't easily tell if the data that was changed is limited to only the data + # that goes into peer relation (in which case, if this is not a leader unit, we wouldn't + # need to emit `on.data_changed`), so we're emitting `on.data_changed` either way. + self.on.data_changed.emit() # pyright: ignore + + def update_tracing_receivers(self): + """Updates the list of exposed tracing receivers in all relations.""" + try: + for relation in self._charm.model.relations[self._relation_name]: + CosAgentRequirerUnitData( + receivers=[ + Receiver( + # if tracing isn't ready, we don't want the wrong receiver URLs present in the databag. + # however, because of the backwards compatibility requirements, we need to still provide + # the protocols list so that the charm with older cos_agent version doesn't error its hooks. + # before this change was added, the charm with old cos_agent version threw exceptions with + # connections to grafana-agent timing out. After the change, the charm will fail validating + # databag contents (as it expects a string in URL) but that won't cause any errors as + # tracing endpoints are the only content in the grafana-agent's side of the databag. + url=f"{self._get_tracing_receiver_url(protocol)}" + if self._charm.tracing.is_ready() # type: ignore + else None, + protocol=ProtocolType( + name=protocol, + type=receiver_protocol_to_transport_protocol[protocol], + ), + ) + for protocol in self.requested_tracing_protocols() + ], + ).dump(relation.data[self._charm.unit]) + + except ModelError as e: + # args are bytes + msg = e.args[0] + if isinstance(msg, bytes): + if msg.startswith( + b"ERROR cannot read relation application settings: permission denied" + ): + logger.error( + f"encountered error {e} while attempting to update_relation_data." + f"The relation must be gone." + ) + return + raise + + def _validated_provider_data(self, raw) -> Optional[CosAgentProviderUnitData]: + try: + return CosAgentProviderUnitData(**json.loads(raw)) + except (pydantic.ValidationError, json.decoder.JSONDecodeError) as e: + self.on.validation_error.emit(message=str(e)) # pyright: ignore + return None + + def trigger_refresh(self, _): + """Trigger a refresh of relation data.""" + # FIXME: Figure out what we should do here + self.on.data_changed.emit() # pyright: ignore + + def _get_requested_protocols(self, relation: Relation): + # Coherence check + units = relation.units + if len(units) > 1: + # should never happen + raise ValueError( + f"unexpected error: subordinate relation {relation} should have exactly one unit" + ) + + unit = next(iter(units), None) + + if not unit: + return None + + if not (raw := relation.data[unit].get(CosAgentProviderUnitData.KEY)): + return None + + if not (provider_data := self._validated_provider_data(raw)): + return None + + return provider_data.tracing_protocols + + def requested_tracing_protocols(self): + """All receiver protocols that have been requested by our related apps.""" + requested_protocols = set() + for relation in self._charm.model.relations[self._relation_name]: + try: + protocols = self._get_requested_protocols(relation) + except NotReadyError: + continue + if protocols: + requested_protocols.update(protocols) + return requested_protocols + + def _get_tracing_receiver_url(self, protocol: str): + scheme = "http" + try: + if self._charm.cert.enabled: # type: ignore + scheme = "https" + # not only Grafana Agent can implement cos_agent. If the charm doesn't have the `cert` attribute + # using our cert_handler, it won't have the `enabled` parameter. In this case, we pass and assume http. + except AttributeError: + pass + # the assumption is that a subordinate charm will always be accessible to its principal charm under its fqdn + if receiver_protocol_to_transport_protocol[protocol] == TransportProtocolType.grpc: + return f"{socket.getfqdn()}:{_tracing_receivers_ports[protocol]}" + return f"{scheme}://{socket.getfqdn()}:{_tracing_receivers_ports[protocol]}" + + @property + def _remote_data(self) -> List[Tuple[CosAgentProviderUnitData, JujuTopology]]: + """Return a list of remote data from each of the related units. + + Assumes that the relation is of type subordinate. + Relies on the fact that, for subordinate relations, the only remote unit visible to + *this unit* is the principal unit that this unit is attached to. + """ + all_data = [] + + for relation in self._charm.model.relations[self._relation_name]: + if not relation.units: + continue + unit = next(iter(relation.units)) + if not (raw := relation.data[unit].get(CosAgentProviderUnitData.KEY)): + continue + if not (provider_data := self._validated_provider_data(raw)): + continue + + topology = JujuTopology( + model=self._charm.model.name, + model_uuid=self._charm.model.uuid, + application=unit.name.name, + unit=unit.name, + ) + + all_data.append((provider_data, topology)) + + return all_data + + def _gather_peer_data(self) -> List[CosAgentPeersUnitData]: + """Collect data from the peers. + + Returns a trimmed-down list of CosAgentPeersUnitData. + """ + relation = self.peer_relation + + # Ensure that whatever context we're running this in, we take the necessary precautions: + if not relation or not relation.data or not relation.app: + return [] + + # Iterate over all peer unit data and only collect every principal once. + peer_data: List[CosAgentPeersUnitData] = [] + app_names: Set[str] = set() + + for unit in chain((self._charm.unit,), relation.units): + if not relation.data.get(unit): + continue + + for unit_name in relation.data.get(unit): # pyright: ignore + if not unit_name.startswith(CosAgentPeersUnitData.KEY): + continue + raw = relation.data[unit].get(unit_name) + if raw is None: + continue + data = CosAgentPeersUnitData(**json.loads(raw)) + # Have we already seen this principal app? + if (app_name := data.app_name) in app_names: + continue + peer_data.append(data) + app_names.add(app_name) + + return peer_data + + @property + def metrics_alerts(self) -> Dict[str, Any]: + """Fetch metrics alerts.""" + alert_rules = {} + + seen_apps: List[str] = [] + for data in self._gather_peer_data(): + if rules := data.metrics_alert_rules: + app_name = data.app_name + if app_name in seen_apps: + continue # dedup! + seen_apps.append(app_name) + # This is only used for naming the file, so be as specific as we can be + identifier = JujuTopology( + model=self._charm.model.name, + model_uuid=self._charm.model.uuid, + application=app_name, + # For the topology unit, we could use `data.principal_unit_name`, but that unit + # name may not be very stable: `_gather_peer_data` de-duplicates by app name so + # the exact unit name that turns up first in the iterator may vary from time to + # time. So using the grafana-agent unit name instead. + unit=self._charm.unit.name, + ).identifier + + alert_rules[identifier] = rules + + return alert_rules + + @property + def metrics_jobs(self) -> List[Dict]: + """Parse the relation data contents and extract the metrics jobs.""" + scrape_jobs = [] + for data, topology in self._remote_data: + for job in data.metrics_scrape_jobs: + # In #220, relation schema changed from a simplified dict to the standard + # `scrape_configs`. + # This is to ensure backwards compatibility with Providers older than v0.5. + if "path" in job and "port" in job and "job_name" in job: + job = { + "job_name": job["job_name"], + "metrics_path": job["path"], + "static_configs": [{"targets": [f"localhost:{job['port']}"]}], + # We include insecure_skip_verify because we are always scraping localhost. + # Even if we have the certs for the scrape targets, we'd rather specify the scrape + # jobs with localhost rather than the SAN DNS the cert was issued for. + "tls_config": {"insecure_skip_verify": True}, + } + + # Apply labels to the scrape jobs + for static_config in job.get("static_configs", []): + topo_as_dict = topology.as_dict(excluded_keys=["charm_name"]) + static_config["labels"] = { + # Be sure to keep labels from static_config + **static_config.get("labels", {}), + # TODO: We should add a new method in juju_topology.py + # that like `as_dict` method, returns the keys with juju_ prefix + # https://github.com/canonical/cos-lib/issues/18 + **{ + "juju_{}".format(key): value + for key, value in topo_as_dict.items() + if value + }, + } + + scrape_jobs.append(job) + + return scrape_jobs + + @property + def snap_log_endpoints(self) -> List[SnapEndpoint]: + """Fetch logging endpoints exposed by related snaps.""" + endpoints = [] + endpoints_with_topology = self.snap_log_endpoints_with_topology + for endpoint, _ in endpoints_with_topology: + endpoints.append(endpoint) + + return endpoints + + @property + def snap_log_endpoints_with_topology(self) -> List[Tuple[SnapEndpoint, JujuTopology]]: + """Fetch logging endpoints and charm topology for each related snap.""" + plugs = [] + for data, topology in self._remote_data: + targets = data.log_slots + if targets: + for target in targets: + if target in plugs: + logger.warning( + f"plug {target} already listed. " + "The same snap is being passed from multiple " + "endpoints; this should not happen." + ) + else: + plugs.append((target, topology)) + + endpoints = [] + for plug, topology in plugs: + if ":" not in plug: + logger.error(f"invalid plug definition received: {plug}. Ignoring...") + else: + endpoint = SnapEndpoint(*plug.split(":")) + endpoints.append((endpoint, topology)) + + return endpoints + + @property + def logs_alerts(self) -> Dict[str, Any]: + """Fetch log alerts.""" + alert_rules = {} + seen_apps: List[str] = [] + + for data in self._gather_peer_data(): + if rules := data.log_alert_rules: + # This is only used for naming the file, so be as specific as we can be + app_name = data.app_name + if app_name in seen_apps: + continue # dedup! + seen_apps.append(app_name) + + identifier = JujuTopology( + model=self._charm.model.name, + model_uuid=self._charm.model.uuid, + application=app_name, + # For the topology unit, we could use `data.unit_name`, but that unit + # name may not be very stable: `_gather_peer_data` de-duplicates by app name so + # the exact unit name that turns up first in the iterator may vary from time to + # time. So using the grafana-agent unit name instead. + unit=self._charm.unit.name, + ).identifier + + alert_rules[identifier] = rules + + return alert_rules + + @property + def dashboards(self) -> List[Dict[str, str]]: + """Fetch dashboards as encoded content. + + Dashboards are assumed not to vary across units of the same primary. + """ + dashboards: List[Dict[str, Any]] = [] + + seen_apps: List[str] = [] + for data in self._gather_peer_data(): + app_name = data.app_name + if app_name in seen_apps: + continue # dedup! + seen_apps.append(app_name) + + for encoded_dashboard in data.dashboards or (): + content = json.loads(LZMABase64.decompress(encoded_dashboard)) + + title = content.get("title", "no_title") + + dashboards.append( + { + "relation_id": data.relation_id, + # We have the remote charm name - use it for the identifier + "charm": f"{data.relation_name}-{app_name}", + "content": content, + "title": title, + } + ) + + return dashboards + + +def charm_tracing_config( + endpoint_requirer: COSAgentProvider, cert_path: Optional[Union[Path, str]] +) -> Tuple[Optional[str], Optional[str]]: + """Utility function to determine the charm_tracing config you will likely want. + + If no endpoint is provided: + disable charm tracing. + If https endpoint is provided but cert_path is not found on disk: + disable charm tracing. + If https endpoint is provided and cert_path is None: + raise TracingError + Else: + proceed with charm tracing (with or without tls, as appropriate) + + Usage: + >>> from lib.charms.tempo_coordinator_k8s.v0.charm_tracing import trace_charm + >>> from lib.charms.tempo_coordinator_k8s.v0.tracing import charm_tracing_config + >>> @trace_charm(tracing_endpoint="my_endpoint", cert_path="cert_path") + >>> class MyCharm(...): + >>> _cert_path = "/path/to/cert/on/charm/container.crt" + >>> def __init__(self, ...): + >>> self.tracing = TracingEndpointRequirer(...) + >>> self.my_endpoint, self.cert_path = charm_tracing_config( + ... self.tracing, self._cert_path) + """ + if not endpoint_requirer.is_ready(): + return None, None + + try: + endpoint = endpoint_requirer.get_tracing_endpoint("otlp_http") + except ProtocolNotFoundError: + logger.warn( + "Endpoint for tracing wasn't provided as tracing backend isn't ready yet. If grafana-agent isn't connected to a tracing backend, integrate it. Otherwise this issue should resolve itself in a few events." + ) + return None, None + + if not endpoint: + return None, None + + is_https = endpoint.startswith("https://") + + if is_https: + if cert_path is None: + raise TracingError("Cannot send traces to an https endpoint without a certificate.") + if not Path(cert_path).exists(): + # if endpoint is https BUT we don't have a server_cert yet: + # disable charm tracing until we do to prevent tls errors + return None, None + return endpoint, str(cert_path) + return endpoint, None diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py new file mode 100644 index 0000000..27ab939 --- /dev/null +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -0,0 +1,1779 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Abstractions for the system's Debian/Ubuntu package information and repositories. + +This module contains abstractions and wrappers around Debian/Ubuntu-style repositories and +packages, in order to easily provide an idiomatic and Pythonic mechanism for adding packages and/or +repositories to systems for use in machine charms. + +A sane default configuration is attainable through nothing more than instantiation of the +appropriate classes. `DebianPackage` objects provide information about the architecture, version, +name, and status of a package. + +`DebianPackage` will try to look up a package either from `dpkg -L` or from `apt-cache` when +provided with a string indicating the package name. If it cannot be located, `PackageNotFoundError` +will be returned, as `apt` and `dpkg` otherwise return `100` for all errors, and a meaningful error +message if the package is not known is desirable. + +To install packages with convenience methods: + +```python +try: + # Run `apt-get update` + apt.update() + apt.add_package("zsh") + apt.add_package(["vim", "htop", "wget"]) +except PackageNotFoundError: + logger.error("a specified package not found in package cache or on system") +except PackageError as e: + logger.error("could not install package. Reason: %s", e.message) +```` + +To find details of a specific package: + +```python +try: + vim = apt.DebianPackage.from_system("vim") + + # To find from the apt cache only + # apt.DebianPackage.from_apt_cache("vim") + + # To find from installed packages only + # apt.DebianPackage.from_installed_package("vim") + + vim.ensure(PackageState.Latest) + logger.info("updated vim to version: %s", vim.fullversion) +except PackageNotFoundError: + logger.error("a specified package not found in package cache or on system") +except PackageError as e: + logger.error("could not install package. Reason: %s", e.message) +``` + + +`RepositoryMapping` will return a dict-like object containing enabled system repositories +and their properties (available groups, baseuri. gpg key). This class can add, disable, or +manipulate repositories. Items can be retrieved as `DebianRepository` objects. + +In order add a new repository with explicit details for fields, a new `DebianRepository` can +be added to `RepositoryMapping` + +`RepositoryMapping` provides an abstraction around the existing repositories on the system, +and can be accessed and iterated over like any `Mapping` object, to retrieve values by key, +iterate, or perform other operations. + +Keys are constructed as `{repo_type}-{}-{release}` in order to uniquely identify a repository. + +Repositories can be added with explicit values through a Python constructor. + +Example: +```python +repositories = apt.RepositoryMapping() + +if "deb-example.com-focal" not in repositories: + repositories.add(DebianRepository(enabled=True, repotype="deb", + uri="https://example.com", release="focal", groups=["universe"])) +``` + +Alternatively, any valid `sources.list` line may be used to construct a new +`DebianRepository`. + +Example: +```python +repositories = apt.RepositoryMapping() + +if "deb-us.archive.ubuntu.com-xenial" not in repositories: + line = "deb http://us.archive.ubuntu.com/ubuntu xenial main restricted" + repo = DebianRepository.from_repo_line(line) + repositories.add(repo) +``` +""" + +from __future__ import annotations + +import fileinput +import glob +import logging +import os +import re +import subprocess +import typing +from enum import Enum +from subprocess import PIPE, CalledProcessError, check_output +from typing import Any, Iterable, Iterator, Literal, Mapping +from urllib.parse import urlparse + +logger = logging.getLogger(__name__) + +# The unique Charmhub library identifier, never change it +LIBID = "7c3dbc9c2ad44a47bd6fcb25caa270e5" + +# Increment this major API version when introducing breaking changes +LIBAPI = 0 + +# Increment this PATCH version before using `charmcraft publish-lib` or reset +# to 0 if you are raising the major API version +LIBPATCH = 17 + + +VALID_SOURCE_TYPES = ("deb", "deb-src") +OPTIONS_MATCHER = re.compile(r"\[.*?\]") +_GPG_KEY_DIR = "/etc/apt/trusted.gpg.d/" + + +class Error(Exception): + """Base class of most errors raised by this library.""" + + def __repr__(self): + """Represent the Error.""" + return f"<{type(self).__module__}.{type(self).__name__} {self.args}>" + + @property + def name(self): + """Return a string representation of the model plus class.""" + return f"<{type(self).__module__}.{type(self).__name__}>" + + @property + def message(self): + """Return the message passed as an argument.""" + return self.args[0] + + +class PackageError(Error): + """Raised when there's an error installing or removing a package.""" + + +class PackageNotFoundError(Error): + """Raised when a requested package is not known to the system.""" + + +class PackageState(Enum): + """A class to represent possible package states.""" + + Present = "present" + Absent = "absent" + Latest = "latest" + Available = "available" + + +class DebianPackage: + """Represents a traditional Debian package and its utility functions. + + `DebianPackage` wraps information and functionality around a known package, whether installed + or available. The version, epoch, name, and architecture can be easily queried and compared + against other `DebianPackage` objects to determine the latest version or to install a specific + version. + + The representation of this object as a string mimics the output from `dpkg` for familiarity. + + Installation and removal of packages is handled through the `state` property or `ensure` + method, with the following options: + + apt.PackageState.Absent + apt.PackageState.Available + apt.PackageState.Present + apt.PackageState.Latest + + When `DebianPackage` is initialized, the state of a given `DebianPackage` object will be set to + `Available`, `Present`, or `Latest`, with `Absent` implemented as a convenience for removal + (though it operates essentially the same as `Available`). + """ + + def __init__( + self, name: str, version: str, epoch: str, arch: str, state: PackageState + ) -> None: + self._name = name + self._arch = arch + self._state = state + self._version = Version(version, epoch) + + def __eq__(self, other: object) -> bool: + """Equality for comparison. + + Args: + other: a `DebianPackage` object for comparison + + Returns: + A boolean reflecting equality + """ + return isinstance(other, self.__class__) and ( + self._name, + self._version.number, + ) == (other._name, other._version.number) + + def __hash__(self): + """Return a hash of this package.""" + return hash((self._name, self._version.number)) + + def __repr__(self): + """Represent the package.""" + return f"<{self.__module__}.{type(self).__name__}: {self.__dict__}>" + + def __str__(self): + """Return a human-readable representation of the package.""" + return ( + f"<{type(self).__name__}: {self._name}-{self._version}.{self._arch} -- {self._state}>" + ) + + @staticmethod + def _apt( + command: str, + package_names: str | list[str], + optargs: list[str] | None = None, + ) -> None: + """Wrap package management commands for Debian/Ubuntu systems. + + Args: + command: the command given to `apt-get` + package_names: a package name or list of package names to operate on + optargs: an (Optional) list of additional arguments + + Raises: + PackageError if an error is encountered + """ + optargs = optargs if optargs is not None else [] + if isinstance(package_names, str): + package_names = [package_names] + _cmd = ["apt-get", "-y", *optargs, command, *package_names] + try: + env = os.environ.copy() + env["DEBIAN_FRONTEND"] = "noninteractive" + subprocess.run(_cmd, capture_output=True, check=True, text=True, env=env) + except CalledProcessError as e: + raise PackageError( + f"Could not {command} package(s) {package_names}: {e.stderr}" + ) from None + + def _add(self) -> None: + """Add a package to the system.""" + self._apt( + "install", + f"{self.name}={self.version}", + optargs=["--option=Dpkg::Options::=--force-confold"], + ) + + def _remove(self) -> None: + """Remove a package from the system. Implementation-specific.""" + return self._apt("remove", f"{self.name}={self.version}") + + @property + def name(self) -> str: + """Returns the name of the package.""" + return self._name + + def ensure(self, state: PackageState): + """Ensure that a package is in a given state. + + Args: + state: a `PackageState` to reconcile the package to + + Raises: + PackageError from the underlying call to apt + """ + if self._state is not state: + if state not in (PackageState.Present, PackageState.Latest): + self._remove() + else: + self._add() + self._state = state + + @property + def present(self) -> bool: + """Returns whether or not a package is present.""" + return self._state in (PackageState.Present, PackageState.Latest) + + @property + def latest(self) -> bool: + """Returns whether the package is the most recent version.""" + return self._state is PackageState.Latest + + @property + def state(self) -> PackageState: + """Returns the current package state.""" + return self._state + + @state.setter + def state(self, state: PackageState) -> None: + """Set the package state to a given value. + + Args: + state: a `PackageState` to reconcile the package to + + Raises: + PackageError from the underlying call to apt + """ + if state in (PackageState.Latest, PackageState.Present): + self._add() + else: + self._remove() + self._state = state + + @property + def version(self) -> Version: + """Returns the version for a package.""" + return self._version + + @property + def epoch(self) -> str: + """Returns the epoch for a package. May be unset.""" + return self._version.epoch + + @property + def arch(self) -> str: + """Returns the architecture for a package.""" + return self._arch + + @property + def fullversion(self) -> str: + """Returns the name+epoch for a package.""" + return f"{self._version}.{self._arch}" + + @staticmethod + def _get_epoch_from_version(version: str) -> tuple[str, str]: + """Pull the epoch, if any, out of a version string.""" + epoch_matcher = re.compile(r"^((?P\d+):)?(?P.*)") + result = epoch_matcher.search(version) + assert result is not None + matches = result.groupdict() + return matches.get("epoch", ""), matches["version"] + + @classmethod + def from_system( + cls, package: str, version: str | None = "", arch: str | None = "" + ) -> DebianPackage: + """Locates a package, either on the system or known to apt, and serializes the information. + + Args: + package: a string representing the package + version: an optional string if a specific version is requested + arch: an optional architecture, defaulting to `dpkg --print-architecture`. If an + architecture is not specified, this will be used for selection. + + """ + try: + return DebianPackage.from_installed_package(package, version, arch) + except PackageNotFoundError: + logger.debug( + "package '%s' is not currently installed or has the wrong architecture.", package + ) + + # Ok, try `apt-cache ...` + try: + return DebianPackage.from_apt_cache(package, version, arch) + except (PackageNotFoundError, PackageError): + # If we get here, it's not known to the systems. + # This seems unnecessary, but virtually all `apt` commands have a return code of `100`, + # and providing meaningful error messages without this is ugly. + arch_str = f".{arch}" if arch else "" + raise PackageNotFoundError( + f"Package '{package}{arch_str}' " + "could not be found on the system or in the apt cache!" + ) from None + + @classmethod + def from_installed_package( + cls, package: str, version: str | None = "", arch: str | None = "" + ) -> DebianPackage: + """Check whether the package is already installed and return an instance. + + Args: + package: a string representing the package + version: an optional string if a specific version is requested + arch: an optional architecture, defaulting to `dpkg --print-architecture`. + If an architecture is not specified, this will be used for selection. + """ + system_arch = check_output( + ["dpkg", "--print-architecture"], universal_newlines=True + ).strip() + arch = arch if arch else system_arch + + # Regexps are a really terrible way to do this. Thanks dpkg + output = "" + try: + output = check_output(["dpkg", "-l", package], stderr=PIPE, universal_newlines=True) + except CalledProcessError: + raise PackageNotFoundError(f"Package is not installed: {package}") from None + + # Pop off the output from `dpkg -l' because there's no flag to + # omit it` + lines = str(output).splitlines()[5:] + + dpkg_matcher = re.compile( + r""" + ^(?P\w+?)\s+ + (?P.*?)(?P:\w+?)?\s+ + (?P.*?)\s+ + (?P\w+?)\s+ + (?P.*) + """, + re.VERBOSE, + ) + + for line in lines: + result = dpkg_matcher.search(line) + if result is None: + logger.warning("dpkg matcher could not parse line: %s", line) + continue + matches = result.groupdict() + package_status = matches["package_status"] + + if not package_status.endswith("i"): + logger.debug( + "package '%s' in dpkg output but not installed, status: '%s'", + package, + package_status, + ) + break + + epoch, split_version = DebianPackage._get_epoch_from_version(matches["version"]) + pkg = DebianPackage( + name=matches["package_name"], + version=split_version, + epoch=epoch, + arch=matches["arch"], + state=PackageState.Present, + ) + if (pkg.arch == "all" or pkg.arch == arch) and ( + version == "" or str(pkg.version) == version + ): + return pkg + + # If we didn't find it, fail through + raise PackageNotFoundError(f"Package {package}.{arch} is not installed!") + + @classmethod + def from_apt_cache( + cls, package: str, version: str | None = "", arch: str | None = "" + ) -> DebianPackage: + """Check whether the package is already installed and return an instance. + + Args: + package: a string representing the package + version: an optional string if a specific version is requested + arch: an optional architecture, defaulting to `dpkg --print-architecture`. + If an architecture is not specified, this will be used for selection. + """ + system_arch = check_output( + ["dpkg", "--print-architecture"], universal_newlines=True + ).strip() + arch = arch if arch else system_arch + + # Regexps are a really terrible way to do this. Thanks dpkg + keys = ("Package", "Architecture", "Version") + + try: + output = check_output( + ["apt-cache", "show", package], stderr=PIPE, universal_newlines=True + ) + except CalledProcessError as e: + raise PackageError(f"Could not list packages in apt-cache: {e.stderr}") from None + + pkg_groups = output.strip().split("\n\n") + keys = ("Package", "Architecture", "Version") + + for pkg_raw in pkg_groups: + lines = str(pkg_raw).splitlines() + vals: dict[str, str] = {} + for line in lines: + if line.startswith(keys): + items = line.split(":", 1) + vals[items[0]] = items[1].strip() + else: + continue + + epoch, split_version = DebianPackage._get_epoch_from_version(vals["Version"]) + pkg = DebianPackage( + name=vals["Package"], + version=split_version, + epoch=epoch, + arch=vals["Architecture"], + state=PackageState.Available, + ) + + if (pkg.arch == "all" or pkg.arch == arch) and ( + version == "" or str(pkg.version) == version + ): + return pkg + + # If we didn't find it, fail through + raise PackageNotFoundError(f"Package {package}.{arch} is not in the apt cache!") + + +class Version: + """An abstraction around package versions. + + This seems like it should be strictly unnecessary, except that `apt_pkg` is not usable inside a + venv, and wedging version comparisons into `DebianPackage` would overcomplicate it. + + This class implements the algorithm found here: + https://www.debian.org/doc/debian-policy/ch-controlfields.html#version + """ + + def __init__(self, version: str, epoch: str): + self._version = version + self._epoch = epoch or "" + + def __repr__(self): + """Represent the package.""" + return f"<{self.__module__}.{type(self).__name__}: {self.__dict__}>" + + def __str__(self): + """Return human-readable representation of the package.""" + epoch = f"{self._epoch}:" if self._epoch else "" + return f"{epoch}{self._version}" + + @property + def epoch(self): + """Returns the epoch for a package. May be empty.""" + return self._epoch + + @property + def number(self) -> str: + """Returns the version number for a package.""" + return self._version + + def _get_parts(self, version: str) -> tuple[str, str]: + """Separate the version into component upstream and Debian pieces.""" + try: + version.rindex("-") + except ValueError: + # No hyphens means no Debian version + return version, "0" + + upstream, debian = version.rsplit("-", 1) + return upstream, debian + + def _listify(self, revision: str) -> list[str | int]: + """Split a revision string into a list. + + This list is comprised of alternating between strings and numbers, + padded on either end to always be "str, int, str, int..." and + always be of even length. This allows us to trivially implement the + comparison algorithm described. + """ + result: list[str | int] = [] + while revision: + rev_1, remains = self._get_alphas(revision) + rev_2, remains = self._get_digits(remains) + result.extend([rev_1, rev_2]) + revision = remains + return result + + def _get_alphas(self, revision: str) -> tuple[str, str]: + """Return a tuple of the first non-digit characters of a revision.""" + # get the index of the first digit + for i, char in enumerate(revision): + if char.isdigit(): + if i == 0: + return "", revision + return revision[0:i], revision[i:] + # string is entirely alphas + return revision, "" + + def _get_digits(self, revision: str) -> tuple[int, str]: + """Return a tuple of the first integer characters of a revision.""" + # If the string is empty, return (0,'') + if not revision: + return 0, "" + # get the index of the first non-digit + for i, char in enumerate(revision): + if not char.isdigit(): + if i == 0: + return 0, revision + return int(revision[0:i]), revision[i:] + # string is entirely digits + return int(revision), "" + + def _dstringcmp(self, a: str, b: str) -> Literal[-1, 0, 1]: + """Debian package version string section lexical sort algorithm. + + The lexical comparison is a comparison of ASCII values modified so + that all the letters sort earlier than all the non-letters and so that + a tilde sorts before anything, even the end of a part. + """ + if a == b: + return 0 + try: + for i, char in enumerate(a): + if char == b[i]: + continue + # "a tilde sorts before anything, even the end of a part" + # (emptyness) + if char == "~": + return -1 + if b[i] == "~": + return 1 + # "all the letters sort earlier than all the non-letters" + if char.isalpha() and not b[i].isalpha(): + return -1 + if not char.isalpha() and b[i].isalpha(): + return 1 + # otherwise lexical sort + if ord(char) > ord(b[i]): + return 1 + if ord(char) < ord(b[i]): + return -1 + except IndexError: + # a is longer than b but otherwise equal, greater unless there are tildes + # FIXME: type checker thinks "char" is possibly unbound as it's a loop variable + # but it won't be since the IndexError can only occur inside the loop + # -- I'd like to refactor away this `try ... except` anyway + if char == "~": # pyright: ignore[reportPossiblyUnboundVariable] + return -1 + return 1 + # if we get here, a is shorter than b but otherwise equal, so check for tildes... + if b[len(a)] == "~": + return 1 + return -1 + + def _compare_revision_strings(self, first: str, second: str) -> Literal[-1, 0, 1]: + """Compare two debian revision strings.""" + if first == second: + return 0 + + # listify pads results so that we will always be comparing ints to ints + # and strings to strings (at least until we fall off the end of a list) + first_list = self._listify(first) + second_list = self._listify(second) + if first_list == second_list: + return 0 + try: + for i, item in enumerate(first_list): + # explicitly raise IndexError if we've fallen off the edge of list2 + if i >= len(second_list): + raise IndexError + other = second_list[i] + # if the items are equal, next + if item == other: + continue + # numeric comparison + if isinstance(item, int): + assert isinstance(other, int) + if item > other: + return 1 + if item < other: + return -1 + else: + # string comparison + assert isinstance(other, str) + return self._dstringcmp(item, other) + except IndexError: + # rev1 is longer than rev2 but otherwise equal, hence greater + # ...except for goddamn tildes + # FIXME: bug?? we return 1 in both cases + # FIXME: first_list[len(second_list)] should be a string + # why are we indexing to 0 twice? + if first_list[len(second_list)][0][0] == "~": # type: ignore + return 1 + return 1 + # rev1 is shorter than rev2 but otherwise equal, hence lesser + # ...except for goddamn tildes + # FIXME: bug?? we return -1 in both cases + # FIXME: first_list[len(second_list)] should be a string, why are we indexing to 0 twice? + if second_list[len(first_list)][0][0] == "~": # type: ignore + return -1 + return -1 + + def _compare_version(self, other: Version) -> Literal[-1, 0, 1]: + if (self.number, self.epoch) == (other.number, other.epoch): + return 0 + + if self.epoch < other.epoch: + return -1 + if self.epoch > other.epoch: + return 1 + + # If none of these are true, follow the algorithm + upstream_version, debian_version = self._get_parts(self.number) + other_upstream_version, other_debian_version = self._get_parts(other.number) + + upstream_cmp = self._compare_revision_strings(upstream_version, other_upstream_version) + if upstream_cmp != 0: + return upstream_cmp + + debian_cmp = self._compare_revision_strings(debian_version, other_debian_version) + if debian_cmp != 0: + return debian_cmp + + return 0 + + def __lt__(self, other: Version) -> bool: + """Less than magic method impl.""" + return self._compare_version(other) < 0 + + def __eq__(self, other: object) -> bool: + """Equality magic method impl.""" + if not isinstance(other, Version): + return False + return self._compare_version(other) == 0 + + def __gt__(self, other: Version) -> bool: + """Greater than magic method impl.""" + return self._compare_version(other) > 0 + + def __le__(self, other: Version) -> bool: + """Less than or equal to magic method impl.""" + return self.__eq__(other) or self.__lt__(other) + + def __ge__(self, other: Version) -> bool: + """Greater than or equal to magic method impl.""" + return self.__gt__(other) or self.__eq__(other) + + def __ne__(self, other: object) -> bool: + """Not equal to magic method impl.""" + return not self.__eq__(other) + + +@typing.overload +def add_package( + package_names: str, + version: str | None = "", + arch: str | None = "", + update_cache: bool = False, +) -> DebianPackage: ... +@typing.overload +def add_package( + package_names: list[str], + version: str | None = "", + arch: str | None = "", + update_cache: bool = False, +) -> DebianPackage | list[DebianPackage]: ... +def add_package( + package_names: str | list[str], + version: str | None = "", + arch: str | None = "", + update_cache: bool = False, +) -> DebianPackage | list[DebianPackage]: + """Add a package or list of packages to the system. + + Args: + package_names: single package name, or list of package names + name: the name(s) of the package(s) + version: an (Optional) version as a string. Defaults to the latest known + arch: an optional architecture for the package + update_cache: whether or not to run `apt-get update` prior to operating + + Raises: + TypeError if no package name is given, or explicit version is set for multiple packages + PackageNotFoundError if the package is not in the cache. + PackageError if packages fail to install + """ + cache_refreshed = False + if update_cache: + update() + cache_refreshed = True + + package_names = [package_names] if isinstance(package_names, str) else package_names + if not package_names: + raise TypeError("Expected at least one package name to add, received zero!") + + if len(package_names) != 1 and version: + raise TypeError( + "Explicit version should not be set if more than one package is being added!" + ) + + succeeded: list[DebianPackage] = [] + retry: list[str] = [] + failed: list[str] = [] + + for p in package_names: + pkg, _ = _add(p, version, arch) + if isinstance(pkg, DebianPackage): + succeeded.append(pkg) + elif cache_refreshed: + logger.warning("failed to locate and install/update '%s'", pkg) + failed.append(p) + else: + logger.warning("failed to locate and install/update '%s', will retry later", pkg) + retry.append(p) + + if retry: + logger.info("updating the apt-cache and retrying installation of failed packages.") + update() + + for p in retry: + pkg, _ = _add(p, version, arch) + if isinstance(pkg, DebianPackage): + succeeded.append(pkg) + else: + failed.append(p) + + if failed: + raise PackageError(f"Failed to install packages: {', '.join(failed)}") + + return succeeded[0] if len(succeeded) == 1 else succeeded + + +def _add( + name: str, + version: str | None = "", + arch: str | None = "", +) -> tuple[DebianPackage, Literal[True]] | tuple[str, Literal[False]]: + """Add a package to the system. + + Args: + name: the name(s) of the package(s) + version: an (Optional) version as a string. Defaults to the latest known + arch: an optional architecture for the package + + Returns: a tuple of `DebianPackage` if found, or a :str: if it is not, and + a boolean indicating success + """ + try: + pkg = DebianPackage.from_system(name, version, arch) + pkg.ensure(state=PackageState.Present) + return pkg, True + except PackageNotFoundError: + return name, False + + +@typing.overload +def remove_package( + package_names: str, +) -> DebianPackage: ... +@typing.overload +def remove_package( + package_names: list[str], +) -> DebianPackage | list[DebianPackage]: ... +def remove_package( + package_names: str | list[str], +) -> DebianPackage | list[DebianPackage]: + """Remove package(s) from the system. + + Args: + package_names: the name of a package + + Raises: + TypeError: if no packages are provided + """ + packages: list[DebianPackage] = [] + + package_names = [package_names] if isinstance(package_names, str) else package_names + if not package_names: + raise TypeError("Expected at least one package name to add, received zero!") + + for p in package_names: + try: + pkg = DebianPackage.from_installed_package(p) + pkg.ensure(state=PackageState.Absent) + packages.append(pkg) + except PackageNotFoundError: # noqa: PERF203 + logger.info("package '%s' was requested for removal, but it was not installed.", p) + + # the list of packages will be empty when no package is removed + logger.debug("packages: '%s'", packages) + return packages[0] if len(packages) == 1 else packages + + +def update() -> None: + """Update the apt cache via `apt-get update`.""" + cmd = ["apt-get", "update", "--error-on=any"] + try: + subprocess.run(cmd, capture_output=True, check=True) + except CalledProcessError as e: + logger.error( + "%s:\nstdout:\n%s\nstderr:\n%s", + " ".join(cmd), + e.stdout.decode(), + e.stderr.decode(), + ) + raise + + +def import_key(key: str) -> str: + """Import an ASCII Armor key. + + A Radix64 format keyid is also supported for backwards + compatibility. In this case Ubuntu keyserver will be + queried for a key via HTTPS by its keyid. This method + is less preferable because https proxy servers may + require traffic decryption which is equivalent to a + man-in-the-middle attack (a proxy server impersonates + keyserver TLS certificates and has to be explicitly + trusted by the system). + + Args: + key: A GPG key in ASCII armor format, including BEGIN + and END markers or a keyid. + + Returns: + The GPG key filename written. + + Raises: + GPGKeyError if the key could not be imported + """ + key = key.strip() + if "-" in key or "\n" in key: + # Send everything not obviously a keyid to GPG to import, as + # we trust its validation better than our own. eg. handling + # comments before the key. + logger.debug("PGP key found (looks like ASCII Armor format)") + if ( + "-----BEGIN PGP PUBLIC KEY BLOCK-----" in key + and "-----END PGP PUBLIC KEY BLOCK-----" in key + ): + logger.debug("Writing provided PGP key in the binary format") + key_bytes = key.encode("utf-8") + key_name = DebianRepository._get_keyid_by_gpg_key(key_bytes) + key_gpg = DebianRepository._dearmor_gpg_key(key_bytes) + gpg_key_filename = os.path.join(_GPG_KEY_DIR, f"{key_name}.gpg") + DebianRepository._write_apt_gpg_keyfile( + key_name=gpg_key_filename, key_material=key_gpg + ) + return gpg_key_filename + else: + raise GPGKeyError("ASCII armor markers missing from GPG key") + else: + logger.warning( + "PGP key found (looks like Radix64 format). " + "SECURELY importing PGP key from keyserver; " + "full key not provided." + ) + # as of bionic add-apt-repository uses curl with an HTTPS keyserver URL + # to retrieve GPG keys. `apt-key adv` command is deprecated as is + # apt-key in general as noted in its manpage. See lp:1433761 for more + # history. Instead, /etc/apt/trusted.gpg.d is used directly to drop + # gpg + key_asc = DebianRepository._get_key_by_keyid(key) + # write the key in GPG format so that apt-key list shows it + key_gpg = DebianRepository._dearmor_gpg_key(key_asc.encode("utf-8")) + gpg_key_filename = os.path.join(_GPG_KEY_DIR, f"{key}.gpg") + DebianRepository._write_apt_gpg_keyfile(key_name=gpg_key_filename, key_material=key_gpg) + return gpg_key_filename + + +class InvalidSourceError(Error): + """Exceptions for invalid source entries.""" + + +class GPGKeyError(Error): + """Exceptions for GPG keys.""" + + +class DebianRepository: + """An abstraction to represent a repository.""" + + _deb822_stanza: _Deb822Stanza | None = None + """set by Deb822Stanza after creating a DebianRepository""" + + def __init__( + self, + enabled: bool, + repotype: str, + uri: str, + release: str, + groups: list[str], + filename: str = "", + gpg_key_filename: str = "", + options: dict[str, str] | None = None, + ): + self._enabled = enabled + self._repotype = repotype + self._uri = uri + self._release = release + self._groups = groups + self._filename = filename + self._gpg_key_filename = gpg_key_filename + self._options = options + + @property + def enabled(self): + """Return whether or not the repository is enabled.""" + return self._enabled + + @property + def repotype(self): + """Return whether it is binary or source.""" + return self._repotype + + @property + def uri(self): + """Return the URI.""" + return self._uri + + @property + def release(self): + """Return which Debian/Ubuntu releases it is valid for.""" + return self._release + + @property + def groups(self): + """Return the enabled package groups.""" + return self._groups + + @property + def filename(self): + """Returns the filename for a repository.""" + return self._filename + + @filename.setter + def filename(self, fname: str) -> None: + """Set the filename used when a repo is written back to disk. + + Args: + fname: a filename to write the repository information to. + """ + if not fname.endswith((".list", ".sources")): + raise InvalidSourceError("apt source filenames should end in .list or .sources!") + self._filename = fname + + @property + def gpg_key(self): + """Returns the path to the GPG key for this repository.""" + if not self._gpg_key_filename and self._deb822_stanza is not None: + self._gpg_key_filename = self._deb822_stanza.get_gpg_key_filename() + return self._gpg_key_filename + + @property + def options(self): + """Returns any additional repo options which are set.""" + return self._options + + def make_options_string(self, include_signed_by: bool = True) -> str: + """Generate the complete one-line-style options string for a repository. + + Combining `gpg_key`, if set (and include_signed_by is True), with any other + provided options to form the options section of a one-line-style definition. + """ + options = self._options if self._options else {} + if include_signed_by and self.gpg_key: + options["signed-by"] = self.gpg_key + if not options: + return "" + pairs = (f"{k}={v}" for k, v in sorted(options.items())) + return "[{}] ".format(" ".join(pairs)) + + @staticmethod + def prefix_from_uri(uri: str) -> str: + """Get a repo list prefix from the uri, depending on whether a path is set.""" + uridetails = urlparse(uri) + path = ( + uridetails.path.lstrip("/").replace("/", "-") if uridetails.path else uridetails.netloc + ) + return f"/etc/apt/sources.list.d/{path}" + + @staticmethod + def from_repo_line(repo_line: str, write_file: bool | None = True) -> DebianRepository: + """Instantiate a new `DebianRepository` from a `sources.list` entry line. + + Args: + repo_line: a string representing a repository entry + write_file: boolean to enable writing the new repo to disk. True by default. + Expect it to result in an add-apt-repository call under the hood, like: + add-apt-repository --no-update --sourceslist="$repo_line" + """ + repo = RepositoryMapping._parse( + repo_line, + filename="UserInput", # temp filename + ) + repo.filename = repo._make_filename() + if write_file: + _add_repository(repo) + return repo + + def _make_filename(self) -> str: + """Construct a filename from uri and release. + + For internal use when a filename isn't set. + Should match the filename written to by add-apt-repository. + """ + return "{}-{}.list".format( + DebianRepository.prefix_from_uri(self.uri), + self.release.replace("/", "-"), + ) + + def disable(self) -> None: + """Remove this repository by disabling it in the source file. + + WARNING: This method does NOT alter the `self.enabled` flag. + + WARNING: disable is currently not implemented for repositories defined + by a deb822 stanza. Raises a NotImplementedError in this case. + """ + if self._deb822_stanza is not None: + raise NotImplementedError( + "Disabling a repository defined by a deb822 format source is not implemented." + " Please raise an issue if you require this feature." + ) + searcher = f"{self.repotype} {self.make_options_string()}{self.uri} {self.release}" + with fileinput.input(self._filename, inplace=True) as lines: + for line in lines: + if re.match(rf"^{re.escape(searcher)}\s", line): + print(f"# {line}", end="") + else: + print(line, end="") + + def import_key(self, key: str) -> None: + """Import an ASCII Armor key. + + A Radix64 format keyid is also supported for backwards + compatibility. In this case Ubuntu keyserver will be + queried for a key via HTTPS by its keyid. This method + is less preferable because https proxy servers may + require traffic decryption which is equivalent to a + man-in-the-middle attack (a proxy server impersonates + keyserver TLS certificates and has to be explicitly + trusted by the system). + + Args: + key: A GPG key in ASCII armor format, + including BEGIN and END markers or a keyid. + + Raises: + GPGKeyError if the key could not be imported + """ + self._gpg_key_filename = import_key(key) + + @staticmethod + def _get_keyid_by_gpg_key(key_material: bytes) -> str: + """Get a GPG key fingerprint by GPG key material. + + Gets a GPG key fingerprint (40-digit, 160-bit) by the ASCII armor-encoded + or binary GPG key material. Can be used, for example, to generate file + names for keys passed via charm options. + """ + # Use the same gpg command for both Xenial and Bionic + cmd = ["gpg", "--with-colons", "--with-fingerprint"] + ps = subprocess.run(cmd, capture_output=True, input=key_material) + out, err = ps.stdout.decode(), ps.stderr.decode() + if "gpg: no valid OpenPGP data found." in err: + raise GPGKeyError("Invalid GPG key material provided") + # from gnupg2 docs: fpr :: Fingerprint (fingerprint is in field 10) + result = re.search(r"^fpr:{9}([0-9A-F]{40}):$", out, re.MULTILINE) + assert result is not None + return result.group(1) + + @staticmethod + def _get_key_by_keyid(keyid: str) -> str: + """Get a key via HTTPS from the Ubuntu keyserver. + + Different key ID formats are supported by SKS keyservers (the longer ones + are more secure, see "dead beef attack" and https://evil32.com/). Since + HTTPS is used, if SSLBump-like HTTPS proxies are in place, they will + impersonate keyserver.ubuntu.com and generate a certificate with + keyserver.ubuntu.com in the CN field or in SubjAltName fields of a + certificate. If such proxy behavior is expected it is necessary to add the + CA certificate chain containing the intermediate CA of the SSLBump proxy to + every machine that this code runs on via ca-certs cloud-init directive (via + cloudinit-userdata model-config) or via other means (such as through a + custom charm option). Also note that DNS resolution for the hostname in a + URL is done at a proxy server - not at the client side. + 8-digit (32 bit) key ID + https://keyserver.ubuntu.com/pks/lookup?search=0x4652B4E6 + 16-digit (64 bit) key ID + https://keyserver.ubuntu.com/pks/lookup?search=0x6E85A86E4652B4E6 + 40-digit key ID: + https://keyserver.ubuntu.com/pks/lookup?search=0x35F77D63B5CEC106C577ED856E85A86E4652B4E6 + + Args: + keyid: An 8, 16 or 40 hex digit keyid to find a key for + + Returns: + A string containing key material for the specified GPG key id + + + Raises: + subprocess.CalledProcessError + """ + # options=mr - machine-readable output (disables html wrappers) + keyserver_url = ( + "https://keyserver.ubuntu.com" "/pks/lookup?op=get&options=mr&exact=on&search=0x{}" + ) + curl_cmd = ["curl", keyserver_url.format(keyid)] + # use proxy server settings in order to retrieve the key + return check_output(curl_cmd).decode() + + @staticmethod + def _dearmor_gpg_key(key_asc: bytes) -> bytes: + """Convert a GPG key in the ASCII armor format to the binary format. + + Args: + key_asc: A GPG key in ASCII armor format. + + Returns: + A GPG key in binary format as a string + + Raises: + GPGKeyError + """ + ps = subprocess.run(["gpg", "--dearmor"], capture_output=True, input=key_asc) + out, err = ps.stdout, ps.stderr.decode() + if "gpg: no valid OpenPGP data found." in err: + raise GPGKeyError( + "Invalid GPG key material. Check your network setup" + " (MTU, routing, DNS) and/or proxy server settings" + " as well as destination keyserver status." + ) + else: + return out + + @staticmethod + def _write_apt_gpg_keyfile(key_name: str, key_material: bytes) -> None: + """Write GPG key material into a file at a provided path. + + Args: + key_name: A key name to use for a key file (could be a fingerprint) + key_material: A GPG key material (binary) + """ + with open(key_name, "wb") as keyf: + keyf.write(key_material) + + +def _repo_to_identifier(repo: DebianRepository) -> str: + """Return str identifier derived from repotype, uri, and release. + + Private method used to produce the identifiers used by RepositoryMapping. + """ + return f"{repo.repotype}-{repo.uri}-{repo.release}" + + +def _repo_to_line(repo: DebianRepository, include_signed_by: bool = True) -> str: + """Return the one-per-line format repository definition.""" + return "{prefix}{repotype} {options}{uri} {release} {groups}".format( + prefix="" if repo.enabled else "#", + repotype=repo.repotype, + options=repo.make_options_string(include_signed_by=include_signed_by), + uri=repo.uri, + release=repo.release, + groups=" ".join(repo.groups), + ) + + +class RepositoryMapping(Mapping[str, DebianRepository]): + """An representation of known repositories. + + Instantiation of `RepositoryMapping` will iterate through the + filesystem, parse out repository files in `/etc/apt/...`, and create + `DebianRepository` objects in this list. + + Typical usage: + + repositories = apt.RepositoryMapping() + repositories.add(DebianRepository( + enabled=True, repotype="deb", uri="https://example.com", release="focal", + groups=["universe"] + )) + """ + + _apt_dir = "/etc/apt" + _sources_subdir = "sources.list.d" + _default_list_name = "sources.list" + _default_sources_name = "ubuntu.sources" + _last_errors: tuple[Error, ...] = () + + def __init__(self): + self._repository_map: dict[str, DebianRepository] = {} + self.default_file = os.path.join(self._apt_dir, self._default_list_name) + # ^ public attribute for backwards compatibility only + sources_dir = os.path.join(self._apt_dir, self._sources_subdir) + default_sources = os.path.join(sources_dir, self._default_sources_name) + + # read sources.list if it exists + # ignore InvalidSourceError if ubuntu.sources also exists + # -- in this case, sources.list just contains a comment + if os.path.isfile(self.default_file): + try: + self.load(self.default_file) + except InvalidSourceError: + if not os.path.isfile(default_sources): + raise + + # read sources.list.d + for file in glob.iglob(os.path.join(sources_dir, "*.list")): + self.load(file) + for file in glob.iglob(os.path.join(sources_dir, "*.sources")): + self.load_deb822(file) + + def __contains__(self, key: Any) -> bool: + """Magic method for checking presence of repo in mapping. + + Checks against the string names used to identify repositories. + """ + return key in self._repository_map + + def __len__(self) -> int: + """Return number of repositories in map.""" + return len(self._repository_map) + + def __iter__(self) -> Iterator[DebianRepository]: # pyright: ignore[reportIncompatibleMethodOverride] + """Return iterator for RepositoryMapping. + + Iterates over the DebianRepository values rather than the string names. + FIXME: this breaks the expectations of the Mapping abstract base class + for example when it provides methods like keys and items + """ + return iter(self._repository_map.values()) + + def __getitem__(self, repository_uri: str) -> DebianRepository: + """Return a given `DebianRepository`.""" + return self._repository_map[repository_uri] + + def __setitem__(self, repository_uri: str, repository: DebianRepository) -> None: + """Add a `DebianRepository` to the cache.""" + self._repository_map[repository_uri] = repository + + def load_deb822(self, filename: str) -> None: + """Load a deb822 format repository source file into the cache. + + In contrast to one-line-style, the deb822 format specifies a repository + using a multi-line stanza. Stanzas are separated by whitespace, + and each definition consists of lines that are either key: value pairs, + or continuations of the previous value. + + Read more about the deb822 format here: + https://manpages.ubuntu.com/manpages/noble/en/man5/sources.list.5.html + For instance, ubuntu 24.04 (noble) lists its sources using deb822 style in: + /etc/apt/sources.list.d/ubuntu.sources + """ + with open(filename) as f: + repos, errors = self._parse_deb822_lines(f, filename=filename) + for repo in repos: + self._repository_map[_repo_to_identifier(repo)] = repo + if errors: + self._last_errors = tuple(errors) + logger.debug( + "the following %d error(s) were encountered when reading deb822 sources:\n%s", + len(errors), + "\n".join(str(e) for e in errors), + ) + if repos: + logger.info("parsed %d apt package repositories from %s", len(repos), filename) + else: + raise InvalidSourceError(f"all repository lines in '{filename}' were invalid!") + + @classmethod + def _parse_deb822_lines( + cls, + lines: Iterable[str], + filename: str = "", + ) -> tuple[list[DebianRepository], list[InvalidSourceError]]: + """Parse lines from a deb822 file into a list of repos and a list of errors. + + The semantics of `_parse_deb822_lines` slightly different to `_parse`: + `_parse` reads a commented out line as an entry that is not enabled + `_parse_deb822_lines` strips out comments entirely when parsing a file into stanzas, + instead only reading the 'Enabled' key to determine if an entry is enabled + """ + repos: list[DebianRepository] = [] + errors: list[InvalidSourceError] = [] + for numbered_lines in _iter_deb822_stanzas(lines): + try: + stanza = _Deb822Stanza(numbered_lines=numbered_lines, filename=filename) + except InvalidSourceError as e: # noqa: PERF203 + errors.append(e) + else: + repos.extend(stanza.repos) + return repos, errors + + def load(self, filename: str): + """Load a one-line-style format repository source file into the cache. + + Args: + filename: the path to the repository file + """ + parsed: list[int] = [] + skipped: list[int] = [] + with open(filename) as f: + for n, line in enumerate(f, start=1): # 1 indexed line numbers + try: + repo = self._parse(line, filename) + except InvalidSourceError: # noqa: PERF203 + skipped.append(n) + else: + repo_identifier = _repo_to_identifier(repo) + self._repository_map[repo_identifier] = repo + parsed.append(n) + logger.debug("parsed repo: '%s'", repo_identifier) + + if skipped: + skip_list = ", ".join(str(s) for s in skipped) + logger.debug("skipped the following lines in file '%s': %s", filename, skip_list) + + if parsed: + logger.info("parsed %d apt package repositories from %s", len(parsed), filename) + else: + raise InvalidSourceError(f"all repository lines in '{filename}' were invalid!") + + @staticmethod + def _parse(line: str, filename: str) -> DebianRepository: + """Parse a line in a sources.list file. + + Args: + line: a single line from `load` to parse + filename: the filename being read + + Raises: + InvalidSourceError if the source type is unknown + """ + enabled = True + repotype = uri = release = gpg_key = "" + options = {} + groups = [] + + line = line.strip() + if line.startswith("#"): + enabled = False + line = line[1:] + + # Check for "#" in the line and treat a part after it as a comment then strip it off. + i = line.find("#") + if i > 0: + line = line[:i] + + # Split a source into substrings to initialize a new repo. + source = line.strip() + if source: + # Match any repo options, and get a dict representation. + for v in re.findall(OPTIONS_MATCHER, source): + opts = dict(o.split("=") for o in v.strip("[]").split()) + # Extract the 'signed-by' option for the gpg_key + gpg_key = opts.pop("signed-by", "") + options = opts + + # Remove any options from the source string and split the string into chunks + source = re.sub(OPTIONS_MATCHER, "", source) + chunks = source.split() + + # Check we've got a valid list of chunks + if len(chunks) < 3 or chunks[0] not in VALID_SOURCE_TYPES: + raise InvalidSourceError("An invalid sources line was found in %s!", filename) + + repotype = chunks[0] + uri = chunks[1] + release = chunks[2] + groups = chunks[3:] + + return DebianRepository( + enabled, repotype, uri, release, groups, filename, gpg_key, options + ) + else: + raise InvalidSourceError("An invalid sources line was found in %s!", filename) + + def add( # noqa: D417 # undocumented-param: default_filename intentionally undocumented + self, repo: DebianRepository, default_filename: bool | None = False + ) -> None: + """Add a new repository to the system using add-apt-repository. + + Args: + repo: a DebianRepository object + if repo.enabled is falsey, will return without adding the repository + Raises: + CalledProcessError: if there's an error running apt-add-repository + + WARNING: Does not associate the repository with a signing key. + Use `import_key` to add a signing key globally. + + WARNING: if repo.enabled is falsey, will return without adding the repository + + WARNING: Don't forget to call `apt.update` before installing any packages! + Or call `apt.add_package` with `update_cache=True`. + + WARNING: the default_filename keyword argument is provided for backwards compatibility + only. It is not used, and was not used in the previous revision of this library. + """ + if not repo.enabled: + logger.warning( + ( + "Returning from RepositoryMapping.add(repo=%s) without adding the repo" + " because repo.enabled is %s" + ), + repo, + repo.enabled, + ) + return + _add_repository(repo) + self._repository_map[_repo_to_identifier(repo)] = repo + + def disable(self, repo: DebianRepository) -> None: + """Remove a repository by disabling it in the source file. + + WARNING: disable is currently not implemented for repositories defined + by a deb822 stanza, and will raise a NotImplementedError if called on one. + + WARNING: This method does NOT alter the `.enabled` flag on the DebianRepository. + """ + repo.disable() + self._repository_map[_repo_to_identifier(repo)] = repo + # ^ adding to map on disable seems like a bug, but this is the previous behaviour + + +def _add_repository( + repo: DebianRepository, + remove: bool = False, + update_cache: bool = False, +) -> None: + line = _repo_to_line(repo, include_signed_by=False) + key_file = repo.gpg_key + if key_file and not remove and not os.path.exists(key_file): + msg = ( + "Adding repository '%s' with add-apt-repository." + " Key file '%s' does not exist." + " Ensure it is imported correctly to use this repository." + ) + logger.warning(msg, line, key_file) + cmd = [ + "add-apt-repository", + "--yes", + "--sourceslist=" + line, + ] + if remove: + cmd.append("--remove") + if not update_cache: + cmd.append("--no-update") + logger.info("%s", cmd) + try: + subprocess.run(cmd, check=True, capture_output=True) + except CalledProcessError as e: + logger.error( + "subprocess.run(%s):\nstdout:\n%s\nstderr:\n%s", + cmd, + e.stdout.decode(), + e.stderr.decode(), + ) + raise + + +class _Deb822Stanza: + """Representation of a stanza from a deb822 source file. + + May define multiple DebianRepository objects. + """ + + def __init__(self, numbered_lines: list[tuple[int, str]], filename: str = ""): + self._filename = filename + self._numbered_lines = numbered_lines + if not numbered_lines: + self._repos = () + self._gpg_key_filename = "" + self._gpg_key_from_stanza = None + return + options, line_numbers = _deb822_stanza_to_options(numbered_lines) + repos, gpg_key_info = _deb822_options_to_repos( + options, line_numbers=line_numbers, filename=filename + ) + for repo in repos: + repo._deb822_stanza = self + self._repos = repos + self._gpg_key_filename, self._gpg_key_from_stanza = gpg_key_info + + @property + def repos(self) -> tuple[DebianRepository, ...]: + """The repositories defined by this deb822 stanza.""" + return self._repos + + def get_gpg_key_filename(self) -> str: + """Return the path to the GPG key for this stanza. + + Import the key first, if the key itself was provided in the stanza. + Return an empty string if no filename or key was provided. + """ + if self._gpg_key_filename: + return self._gpg_key_filename + if self._gpg_key_from_stanza is None: + return "" + # a gpg key was provided in the stanza + # and we haven't already imported it + self._gpg_key_filename = import_key(self._gpg_key_from_stanza) + return self._gpg_key_filename + + +class MissingRequiredKeyError(InvalidSourceError): + """Missing a required value in a source file.""" + + def __init__(self, message: str = "", *, file: str, line: int | None, key: str) -> None: + super().__init__(message, file, line, key) + self.file = file + self.line = line + self.key = key + + +class BadValueError(InvalidSourceError): + """Bad value for an entry in a source file.""" + + def __init__( + self, + message: str = "", + *, + file: str, + line: int | None, + key: str, + value: str, + ) -> None: + super().__init__(message, file, line, key, value) + self.file = file + self.line = line + self.key = key + self.value = value + + +def _iter_deb822_stanzas(lines: Iterable[str]) -> Iterator[list[tuple[int, str]]]: + """Given lines from a deb822 format file, yield a stanza of lines. + + Args: + lines: an iterable of lines from a deb822 sources file + + Yields: + lists of numbered lines (a tuple of line number and line) that make up + a deb822 stanza, with comments stripped out (but accounted for in line numbering) + """ + current_stanza: list[tuple[int, str]] = [] + for n, line in enumerate(lines, start=1): # 1 indexed line numbers + if not line.strip(): # blank lines separate stanzas + if current_stanza: + yield current_stanza + current_stanza = [] + continue + content, _delim, _comment = line.partition("#") + if content.strip(): # skip (potentially indented) comment line + current_stanza.append((n, content.rstrip())) # preserve indent + if current_stanza: + yield current_stanza + + +def _deb822_stanza_to_options( + lines: Iterable[tuple[int, str]], +) -> tuple[dict[str, str], dict[str, int]]: + """Turn numbered lines into a dict of options and a dict of line numbers. + + Args: + lines: an iterable of numbered lines (a tuple of line number and line) + + Returns: + a dictionary of option names to (potentially multiline) values, and + a dictionary of option names to starting line number + """ + parts: dict[str, list[str]] = {} + line_numbers: dict[str, int] = {} + current = None + for n, line in lines: + assert "#" not in line # comments should be stripped out + if line.startswith(" "): # continuation of previous key's value + assert current is not None + parts[current].append(line.rstrip()) # preserve indent + continue + raw_key, _, raw_value = line.partition(":") + current = raw_key.strip() + parts[current] = [raw_value.strip()] + line_numbers[current] = n + options = {k: "\n".join(v) for k, v in parts.items()} + return options, line_numbers + + +def _deb822_options_to_repos( + options: dict[str, str], line_numbers: Mapping[str, int] = {}, filename: str = "" +) -> tuple[tuple[DebianRepository, ...], tuple[str, str | None]]: + """Return a collections of DebianRepository objects defined by this deb822 stanza. + + Args: + options: a dictionary of deb822 field names to string options + line_numbers: a dictionary of field names to line numbers (for error messages) + filename: the file the options were read from (for repository object and errors) + + Returns: + a tuple of `DebianRepository`s, and + a tuple of the gpg key filename and optional in-stanza provided key itself + + Raises: + InvalidSourceError if any options are malformed or required options are missing + """ + # Enabled + enabled_field = options.pop("Enabled", "yes") + if enabled_field == "yes": + enabled = True + elif enabled_field == "no": + enabled = False + else: + raise BadValueError( + "Must be one of yes or no (default: yes).", + file=filename, + line=line_numbers.get("Enabled"), + key="Enabled", + value=enabled_field, + ) + # Signed-By + gpg_key_file = options.pop("Signed-By", "") + gpg_key_from_stanza: str | None = None + if "\n" in gpg_key_file: + # actually a literal multi-line gpg-key rather than a filename + gpg_key_from_stanza = gpg_key_file + gpg_key_file = "" + # Types + try: + repotypes = options.pop("Types").split() + uris = options.pop("URIs").split() + suites = options.pop("Suites").split() + except KeyError as e: + [key] = e.args + raise MissingRequiredKeyError( + key=key, + line=min(line_numbers.values()) if line_numbers else None, + file=filename, + ) from e + # Components + # suite can specify an exact path, in which case the components must be omitted + # and suite must end with a slash (/). + # If suite does not specify an exact path, at least one component must be present. + # https://manpages.ubuntu.com/manpages/noble/man5/sources.list.5.html + components: list[str] + if len(suites) == 1 and suites[0].endswith("/"): + if "Components" in options: + msg = ( + "Since 'Suites' (line {suites_line}) specifies" + " a path relative to 'URIs' (line {uris_line})," + " 'Components' must be omitted." + ).format( + suites_line=line_numbers.get("Suites"), + uris_line=line_numbers.get("URIs"), + ) + raise BadValueError( + msg, + file=filename, + line=line_numbers.get("Components"), + key="Components", + value=options["Components"], + ) + components = [] + else: + if "Components" not in options: + msg = ( + "Since 'Suites' (line {suites_line}) does not specify" + " a path relative to 'URIs' (line {uris_line})," + " 'Components' must be present in this stanza." + ).format( + suites_line=line_numbers.get("Suites"), + uris_line=line_numbers.get("URIs"), + ) + raise MissingRequiredKeyError( + msg, + file=filename, + line=min(line_numbers.values()) if line_numbers else None, + key="Components", + ) + components = options.pop("Components").split() + repos = tuple( + DebianRepository( + enabled=enabled, + repotype=repotype, + uri=uri, + release=suite, + groups=components, + filename=filename, + gpg_key_filename=gpg_key_file, + options=options, + ) + for repotype in repotypes + for uri in uris + for suite in suites + ) + return repos, (gpg_key_file, gpg_key_from_stanza) diff --git a/lib/charms/operator_libs_linux/v1/systemd.py b/lib/charms/operator_libs_linux/v1/systemd.py new file mode 100644 index 0000000..cdcbad6 --- /dev/null +++ b/lib/charms/operator_libs_linux/v1/systemd.py @@ -0,0 +1,288 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Abstractions for stopping, starting and managing system services via systemd. + +This library assumes that your charm is running on a platform that uses systemd. E.g., +Centos 7 or later, Ubuntu Xenial (16.04) or later. + +For the most part, we transparently provide an interface to a commonly used selection of +systemd commands, with a few shortcuts baked in. For example, service_pause and +service_resume with run the mask/unmask and enable/disable invocations. + +Example usage: + +```python +from charms.operator_libs_linux.v0.systemd import service_running, service_reload + +# Start a service +if not service_running("mysql"): + success = service_start("mysql") + +# Attempt to reload a service, restarting if necessary +success = service_reload("nginx", restart_on_failure=True) +``` +""" + +__all__ = [ # Don't export `_systemctl`. (It's not the intended way of using this lib.) + "SystemdError", + "daemon_reload", + "service_disable", + "service_enable", + "service_failed", + "service_pause", + "service_reload", + "service_restart", + "service_resume", + "service_running", + "service_start", + "service_stop", +] + +import logging +import subprocess + +logger = logging.getLogger(__name__) + +# The unique Charmhub library identifier, never change it +LIBID = "045b0d179f6b4514a8bb9b48aee9ebaf" + +# Increment this major API version when introducing breaking changes +LIBAPI = 1 + +# Increment this PATCH version before using `charmcraft publish-lib` or reset +# to 0 if you are raising the major API version +LIBPATCH = 4 + + +class SystemdError(Exception): + """Custom exception for SystemD related errors.""" + + +def _systemctl(*args: str, check: bool = False) -> int: + """Control a system service using systemctl. + + Args: + *args: Arguments to pass to systemctl. + check: Check the output of the systemctl command. Default: False. + + Returns: + Returncode of systemctl command execution. + + Raises: + SystemdError: Raised if calling systemctl returns a non-zero returncode and check is True. + """ + cmd = ["systemctl", *args] + logger.debug(f"Executing command: {cmd}") + try: + proc = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + encoding="utf-8", + check=check, + ) + logger.debug( + f"Command {cmd} exit code: {proc.returncode}. systemctl output:\n{proc.stdout}" + ) + return proc.returncode + except subprocess.CalledProcessError as e: + raise SystemdError( + f"Command {cmd} failed with returncode {e.returncode}. systemctl output:\n{e.stdout}" + ) + + +def service_running(service_name: str) -> bool: + """Report whether a system service is running. + + Args: + service_name: The name of the service to check. + + Return: + True if service is running/active; False if not. + """ + # If returncode is 0, this means that is service is active. + return _systemctl("--quiet", "is-active", service_name) == 0 + + +def service_failed(service_name: str) -> bool: + """Report whether a system service has failed. + + Args: + service_name: The name of the service to check. + + Returns: + True if service is marked as failed; False if not. + """ + # If returncode is 0, this means that the service has failed. + return _systemctl("--quiet", "is-failed", service_name) == 0 + + +def service_start(*args: str) -> bool: + """Start a system service. + + Args: + *args: Arguments to pass to `systemctl start` (normally the service name). + + Returns: + On success, this function returns True for historical reasons. + + Raises: + SystemdError: Raised if `systemctl start ...` returns a non-zero returncode. + """ + return _systemctl("start", *args, check=True) == 0 + + +def service_stop(*args: str) -> bool: + """Stop a system service. + + Args: + *args: Arguments to pass to `systemctl stop` (normally the service name). + + Returns: + On success, this function returns True for historical reasons. + + Raises: + SystemdError: Raised if `systemctl stop ...` returns a non-zero returncode. + """ + return _systemctl("stop", *args, check=True) == 0 + + +def service_restart(*args: str) -> bool: + """Restart a system service. + + Args: + *args: Arguments to pass to `systemctl restart` (normally the service name). + + Returns: + On success, this function returns True for historical reasons. + + Raises: + SystemdError: Raised if `systemctl restart ...` returns a non-zero returncode. + """ + return _systemctl("restart", *args, check=True) == 0 + + +def service_enable(*args: str) -> bool: + """Enable a system service. + + Args: + *args: Arguments to pass to `systemctl enable` (normally the service name). + + Returns: + On success, this function returns True for historical reasons. + + Raises: + SystemdError: Raised if `systemctl enable ...` returns a non-zero returncode. + """ + return _systemctl("enable", *args, check=True) == 0 + + +def service_disable(*args: str) -> bool: + """Disable a system service. + + Args: + *args: Arguments to pass to `systemctl disable` (normally the service name). + + Returns: + On success, this function returns True for historical reasons. + + Raises: + SystemdError: Raised if `systemctl disable ...` returns a non-zero returncode. + """ + return _systemctl("disable", *args, check=True) == 0 + + +def service_reload(service_name: str, restart_on_failure: bool = False) -> bool: + """Reload a system service, optionally falling back to restart if reload fails. + + Args: + service_name: The name of the service to reload. + restart_on_failure: + Boolean indicating whether to fall back to a restart if the reload fails. + + Returns: + On success, this function returns True for historical reasons. + + Raises: + SystemdError: Raised if `systemctl reload|restart ...` returns a non-zero returncode. + """ + try: + return _systemctl("reload", service_name, check=True) == 0 + except SystemdError: + if restart_on_failure: + return service_restart(service_name) + else: + raise + + +def service_pause(service_name: str) -> bool: + """Pause a system service. + + Stops the service and prevents the service from starting again at boot. + + Args: + service_name: The name of the service to pause. + + Returns: + On success, this function returns True for historical reasons. + + Raises: + SystemdError: Raised if service is still running after being paused by systemctl. + """ + _systemctl("disable", "--now", service_name) + _systemctl("mask", service_name) + + if service_running(service_name): + raise SystemdError(f"Attempted to pause {service_name!r}, but it is still running.") + + return True + + +def service_resume(service_name: str) -> bool: + """Resume a system service. + + Re-enable starting the service again at boot. Start the service. + + Args: + service_name: The name of the service to resume. + + Returns: + On success, this function returns True for historical reasons. + + Raises: + SystemdError: Raised if service is not running after being resumed by systemctl. + """ + _systemctl("unmask", service_name) + _systemctl("enable", "--now", service_name) + + if not service_running(service_name): + raise SystemdError(f"Attempted to resume {service_name!r}, but it is not running.") + + return True + + +def daemon_reload() -> bool: + """Reload systemd manager configuration. + + Returns: + On success, this function returns True for historical reasons. + + Raises: + SystemdError: Raised if `systemctl daemon-reload` returns a non-zero returncode. + """ + return _systemctl("daemon-reload", check=True) == 0 diff --git a/pyproject.toml b/pyproject.toml index 0fce3f3..ae93610 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,8 +25,8 @@ select = ["E", "W", "F", "C", "N", "R", "D", "H"] # Ignore W503, E501 because using black creates errors with this # Ignore D107 Missing docstring in __init__ ignore = ["W503", "E501", "D107"] -# D100, D101, D102, D103: Ignore missing docstrings in tests -per-file-ignores = ["tests/*:D100,D101,D102,D103,D104,D205,D212,D415"] +# Ignore some rules that conflict with the test docstring format +per-file-ignores = ["tests/*:D205,D212,D415,DCO010,DCO020,DCO030"] docstring-convention = "google" [tool.isort] diff --git a/requirements.txt b/requirements.txt index aaa16b1..e271b2b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,4 @@ -ops >= 2.2.0 +jsonschema==4.24.0 +ops==2.22.0 +pydantic==2.11.7 +cosl==1.0.0 diff --git a/src/charm.py b/src/charm.py index 15e3064..8544d7b 100755 --- a/src/charm.py +++ b/src/charm.py @@ -3,29 +3,33 @@ # Copyright 2025 Canonical Ltd. # See LICENSE file for licensing details. -# Learn more at: https://documentation.ubuntu.com/juju/3.6/howto/manage-charms/#build-a-charm +# Learn more at: https://juju.is/docs/sdk -"""Charm the service. - -Refer to the following post for a quick-start guide that will help you -develop a new k8s charm using the Operator Framework: - -https://discourse.charmhub.io/t/4208 -""" +"""Chrony charm.""" import logging +import pathlib +import shutil +import textwrap import typing import ops -from ops import pebble +from charms.grafana_agent.v0.cos_agent import COSAgentProvider + +from chrony import Chrony, TimeSource -# Log messages can be retrieved using juju debug-log logger = logging.getLogger(__name__) -VALID_LOG_LEVELS = ["info", "debug", "warning", "error", "critical"] +CHRONY_CHARM_LOCK_FILE = pathlib.Path("/var/lib/chrony-charm/lock") +CHRONY_CHARM_CONFIG_HEADER = textwrap.dedent( + """\ + # This is managed by chrony-client charm (https://charmhub.io/chrony-client). + # Do not edit.\ + """ +) -class IsCharmsTemplateCharm(ops.CharmBase): +class ChronyClientCharm(ops.CharmBase): """Charm the service.""" def __init__(self, *args: typing.Any): @@ -35,85 +39,131 @@ def __init__(self, *args: typing.Any): args: Arguments passed to the CharmBase parent constructor. """ super().__init__(*args) - self.framework.observe(self.on.httpbin_pebble_ready, self._on_httpbin_pebble_ready) - self.framework.observe(self.on.config_changed, self._on_config_changed) + self.chrony = Chrony() + self._grafana_agent = COSAgentProvider( + self, + metrics_endpoints=[ + {"path": "/metrics", "port": 9123}, + ], + dashboard_dirs=["./src/grafana_dashboards"], + ) + self.framework.observe(self.on.install, self._do_install_and_config) + self.framework.observe(self.on.remove, self._on_remove) + self.framework.observe(self.on.upgrade_charm, self._do_install_and_config) + self.framework.observe(self.on.config_changed, self._do_install_and_config) + + def _do_install_and_config(self, _: ops.EventBase) -> None: + """Install required packages and open NTP port.""" + if self._try_acquire_chrony_lock(): + if not self.chrony.is_installed(): + self.unit.status = ops.MaintenanceStatus("installing chrony") + self.chrony.install() + self._configure_chrony() + else: + self._set_lock_failure_status() + + def _on_remove(self, _: ops.EventBase) -> None: + """Handle remove event.""" + if self._try_acquire_chrony_lock(): + self.chrony.uninstall() + self.chrony.restore_config() + self.chrony.restart() + self._release_chrony_lock() + + def _configure_chrony(self) -> None: + """Configure chrony.""" + try: + sources = self._get_time_sources() + except ValueError: + self.unit.status = ops.BlockedStatus("invalid sources configuration") + return + if not sources: + self.unit.status = ops.BlockedStatus("no time source configured") + return + if CHRONY_CHARM_CONFIG_HEADER not in self.chrony.read_config(): + self.chrony.backup_config() + new_config = self.chrony.new_config(sources=sources, header=CHRONY_CHARM_CONFIG_HEADER) + current_config = self.chrony.read_config() + if new_config != current_config: + logger.info("Chrony config changed, apply and restart chrony") + self.chrony.write_config(new_config) + self.chrony.restart() - def _on_httpbin_pebble_ready(self, event: ops.PebbleReadyEvent) -> None: - """Define and start a workload using the Pebble API. + self.unit.status = ops.ActiveStatus() + + def _get_time_sources(self) -> list[TimeSource]: + """Get time sources from charm configuration. - Change this example to suit your needs. You'll need to specify the right entrypoint and - environment configuration for your specific workload. + Returns: + Time source objects. + """ + urls = typing.cast(str, self.config.get("sources")) + return [ + self.chrony.parse_source_url(url.strip()) for url in urls.split(",") if url.strip() + ] - Learn more about interacting with Pebble at at - https://documentation.ubuntu.com/juju/3.6/reference/pebble/. + @staticmethod + def _write_chrony_lock_file(content: str) -> None: + """Write chrony charm lock file. Args: - event: event triggering the handler. + content: lock file content. """ - # Get a reference the container attribute on the PebbleReadyEvent - container = event.workload - # Add initial Pebble config layer using the Pebble API - container.add_layer("httpbin", self._pebble_layer, combine=True) - # Make Pebble reevaluate its plan, ensuring any services are started if enabled. - container.replan() - # Learn more about statuses in the SDK docs: - # https://documentation.ubuntu.com/juju/latest/reference/status/index.html - self.unit.status = ops.ActiveStatus() + CHRONY_CHARM_LOCK_FILE.parent.mkdir(parents=True, exist_ok=True) + CHRONY_CHARM_LOCK_FILE.write_text(content, encoding="utf-8") + + @staticmethod + def _read_chrony_lock_file() -> typing.Optional[str]: + """Read chrony charm lock file. - def _on_config_changed(self, event: ops.ConfigChangedEvent) -> None: - """Handle changed configuration. + Returns: + None if lock file doesn't exist, otherwise lock file content. + """ + if CHRONY_CHARM_LOCK_FILE.exists(): + return CHRONY_CHARM_LOCK_FILE.read_text(encoding="utf-8") + return None - Change this example to suit your needs. If you don't need to handle config, you can remove - this method. + @staticmethod + def _delete_chrony_lock_file() -> None: + """Delete chrony charm lock file.""" + shutil.rmtree(CHRONY_CHARM_LOCK_FILE.parent) - Learn more about config at - https://canonical-charmcraft.readthedocs-hosted.com/stable/reference/files/config-yaml-file/ + def _try_acquire_chrony_lock(self) -> bool: + """Try to acquire chrony lock. - Args: - event: event triggering the handler. + The chrony lock ensures that when multiple instances of the + chrony charm are installed on the same machine, only one + chrony charm application will execute. + + Returns: + True if lock acquired, False otherwise. + """ + lock_content = self.app.name + lock_file = self._read_chrony_lock_file() + if lock_file is None: + self._write_chrony_lock_file(lock_content) + return True + if lock_file.strip() == lock_content: + return True + return False + + def _release_chrony_lock(self) -> None: + """Release chrony lock. + + Remove the chrony charm lock file. """ - # Fetch the new config value - log_level = str(self.model.config["log-level"]).lower() - - # Do some validation of the configuration option - if log_level in VALID_LOG_LEVELS: - # The config is good, so update the configuration of the workload - container = self.unit.get_container("httpbin") - # Verify that we can connect to the Pebble API in the workload container - if container.can_connect(): - # Push an updated layer with the new config - container.add_layer("httpbin", self._pebble_layer, combine=True) - container.replan() - - logger.debug("Log level for gunicorn changed to '%s'", log_level) - self.unit.status = ops.ActiveStatus() - else: - # We were unable to connect to the Pebble API, so we defer this event - event.defer() - self.unit.status = ops.WaitingStatus("waiting for Pebble API") + if self._try_acquire_chrony_lock(): + self._delete_chrony_lock_file() else: - # In this case, the config option is bad, so block the charm and notify the operator. - self.unit.status = ops.BlockedStatus("invalid log level: '{log_level}'") - - @property - def _pebble_layer(self) -> pebble.LayerDict: - """Return a dictionary representing a Pebble layer.""" - return { - "summary": "httpbin layer", - "description": "pebble config layer for httpbin", - "services": { - "httpbin": { - "override": "replace", - "summary": "httpbin", - "command": "gunicorn -b 0.0.0.0:80 httpbin:app -k gevent", - "startup": "enabled", - "environment": { - "GUNICORN_CMD_ARGS": f"--log-level {self.model.config['log-level']}" - }, - } - }, - } + raise RuntimeError("failed to delete the lock file: owned by another charm") + + def _set_lock_failure_status(self) -> None: + """Set unit status to inform user to remove this charm application.""" + self.unit.status = ops.BlockedStatus( + "conflict: multiple chrony charms detected, " + f"remove this charm using `juju remove-application {self.app.name}`" + ) if __name__ == "__main__": # pragma: nocover - ops.main.main(IsCharmsTemplateCharm) + ops.main.main(ChronyClientCharm) diff --git a/src/chrony.py b/src/chrony.py new file mode 100644 index 0000000..df425b1 --- /dev/null +++ b/src/chrony.py @@ -0,0 +1,401 @@ +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Chrony controller.""" + +# check chrony.conf document for _PoolOptions attributes. +# flake8: noqa: DCO060 + +import collections +import itertools +import logging +import pathlib +import shutil +import subprocess # nosec +import textwrap +import typing +import urllib.parse + +import pydantic +from charms.operator_libs_linux.v0 import apt +from charms.operator_libs_linux.v1 import systemd + +logger = logging.getLogger(__name__) + + +class _PoolOptions(pydantic.BaseModel): + """Chrony pool directive options. + + For more detail: https://chrony-project.org/doc/4.5/chrony.conf.html + """ + + model_config = pydantic.ConfigDict(extra="forbid") + + minpoll: int | None = None + maxpoll: int | None = None + iburst: bool = False + burst: bool = False + key: str | None = None + nts: bool = False + certset: str | None = None + maxdelay: float | None = None + maxdelayratio: float | None = None + maxdelaydevratio: float | None = None + maxdelayquant: float | None = None + mindelay: float | None = None + asymmetry: float | None = None + offset: float | None = None + minsamples: int | None = None + maxsamples: int | None = None + filter: int | None = None + offline: bool = False + auto_offline: bool = False + prefer: bool = False + noselect: bool = False + trust: bool = False + require: bool = False + xleave: bool = False + polltarget: int | None = None + presend: int | None = None + minstratum: int | None = None + version: int | None = None + extfield: str | None = None + maxsources: int | None = None + + def render_options(self) -> str: + """Render pool options as chrony option string. + + Returns: + Chrony pool directive option string. + """ + options = [] + # mypy and pylint have problems handling the model_fields class attribute. + # pylint: disable=not-an-iterable + for field in sorted(f for f in _PoolOptions.model_fields if f != "copy"): # type: ignore + value = getattr(self, field) + # first, check if the value is of boolean type and True + # then, check if the value is of boolean type and False or None (unset) + # finally, check if the value is of a non-boolean type and set + if value is True: + options.append(field) + elif value is None or value is False: + continue + else: + options.extend([field, str(value)]) + return " ".join(options) + + +class _NtpSource(_PoolOptions): + """A NTP time source.""" + + host: typing.Annotated[str, pydantic.StringConstraints(min_length=1)] + port: int | None = None + + @classmethod + def from_source_url(cls, url: str) -> "_NtpSource": + """Parse a NTP time source from a URL. + + Args: + url: URL to parse. + + Returns: + Parsed NTP time source. + + Raises: + ValueError: If the URL is invalid. + """ + parsed = urllib.parse.urlparse(url) + if parsed.scheme != "ntp": + raise ValueError(f"Invalid NTP source URL: {url}") + query = dict(urllib.parse.parse_qsl(parsed.query)) + return cls(host=parsed.hostname, port=parsed.port, **query) # type: ignore + + def render(self) -> str: + """Render NTP time source as a chrony pool directive string. + + Returns: + Chrony pool directive string. + """ + directive = f"pool {self.host}" + if self.port is not None and self.port != 123: + directive += f" port {self.port}" + options = self.render_options() + if options: + directive += f" {options}" + return directive + + +class _NtsSource(_PoolOptions): + """A NTP time source with NTS enabled.""" + + host: typing.Annotated[str, pydantic.StringConstraints(min_length=1)] + ntsport: int | None = None + + @classmethod + def from_source_url(cls, url: str) -> "_NtsSource": + """Parse a NTP time source with NTS enabled from a URL. + + Args: + url: URL to parse. + + Returns: + Parsed NTP time source. + + Raises: + ValueError: If the URL is invalid. + """ + parsed = urllib.parse.urlparse(url) + if parsed.scheme != "nts": + raise ValueError(f"Invalid NTS source URL: {url}") + query = dict(urllib.parse.parse_qsl(parsed.query)) + return cls(host=parsed.hostname, ntsport=parsed.port, **query) # type: ignore + + def render(self) -> str: + """Render NTP time source as a chrony pool directive string with NTS enabled. + + Returns: + Chrony pool directive string. + """ + directive = f"pool {self.host} nts" + if self.ntsport is not None and self.ntsport != 4460: + directive += f" ntsport {self.ntsport}" + options = self.render_options() + if options: + directive += f" {options}" + return directive + + +TimeSource = _NtpSource | _NtsSource +TlsKeyPair = collections.namedtuple("TlsKeyPair", ["certificate", "key"]) + + +class Chrony: + """Chrony service manager.""" + + CONFIG_FILE = pathlib.Path("/etc/chrony/chrony.conf") + CONFIG_FILE_BACKUP = pathlib.Path("/var/lib/chrony/chrony.conf.bak") + CERTS_DIR = pathlib.Path("/etc/chrony/certs") + + @staticmethod + def is_installed() -> bool: + """Check if chrony related packages is installed. + + Returns: + True if installed, False otherwise. + """ + return bool(shutil.which("chrony_exporter") and shutil.which("chronyc")) + + @staticmethod + def install() -> None: # pragma: nocover + """Install the Chrony on the system.""" + subprocess.check_call( + ["add-apt-repository", "-y", "ppa:canonical-is-devops/chrony-charm"] + ) # nosec + apt.add_package( + ["chrony", "ca-certificates", "prometheus-chrony-exporter"], update_cache=True + ) + + @staticmethod + def uninstall() -> None: + """Uninstall installed packages from the system. + + Not all packages will be uninstalled, as some are system defaults. + For example, ca-certificates and chrony (as in Ubuntu 26.04). + """ + apt.remove_package(["prometheus-chrony-exporter"]) + + def read_config(self) -> str: + """Read the current chrony configuration file. + + Returns: + The current chrony configuration file content. + """ + return self.CONFIG_FILE.read_text(encoding="utf-8") # pragma: nocover + + def write_config(self, config: str) -> None: + """Write the chrony configuration file. + + Args: + config: The new chrony configuration file content. + """ + self.CONFIG_FILE.write_text(config, encoding="utf-8") # pragma: nocover + + def backup_config(self) -> None: + """Backup the current chrony configuration file.""" + if self.CONFIG_FILE_BACKUP.exists(): + logger.warning("failed to backup configuration file: backup already exists") + return + self.CONFIG_FILE_BACKUP.write_text(self.read_config(), encoding="utf-8") + + def restore_config(self) -> None: + """Restore the chrony configuration file from backup.""" + if not self.CONFIG_FILE_BACKUP.exists(): + logger.warning("failed to restore chrony configuration file from backup: no backup") + return + self.write_config(self.CONFIG_FILE_BACKUP.read_text(encoding="utf-8")) + self.CONFIG_FILE_BACKUP.unlink() + + def _make_certs_dir(self) -> None: # pragma: nocover + """Create the chrony TLS certificates directory.""" + self.CERTS_DIR.mkdir(exist_ok=True, mode=0o700) + shutil.chown(self.CERTS_DIR, "_chrony", "_chrony") + + def _iter_certs_dir(self) -> list[pathlib.Path]: # pragma: nocover + """Iterate over all certificate files in the certificate directory. + + Returns: + An iterator over the paths of the certificate files. + """ + return [f for f in self.CERTS_DIR.iterdir() if f.suffix in {".crt", ".key"}] + + @staticmethod + def _write_certs_file(path: pathlib.Path, content: str) -> None: # pragma: nocover + """Write content of a certificate file and set appropriate permissions and ownership. + + Args: + path: The path to the certificate file. + content: The content to write to the file. + """ + path.touch(mode=0o600, exist_ok=True) + path.write_text(content, encoding="utf-8") + shutil.chown(path, "_chrony", "_chrony") + + @staticmethod + def _read_certs_file(path: pathlib.Path) -> str: + """Read and return the content of a certificate file. + + Args: + path: The path to the certificate file. + + Returns: + The content of the certificate file as a string. + """ + return path.read_text(encoding="utf-8") # pragma: nocover + + @staticmethod + def _unlink_certs_file(path: pathlib.Path) -> None: + """Unlink (delete) a certificate file. + + Args: + path: The path to the certificate file to delete. + """ + path.unlink(missing_ok=True) # pragma: nocover + + def read_tls_key_pairs(self) -> list[TlsKeyPair]: + """Read TLS key pairs from the certificates directory. + + Returns: + A list of TlsKeyPair objects. + """ + self._make_certs_dir() + files = sorted(self._iter_certs_dir()) + key_pairs = [] + for crt, key in self._batched(files, 2): + key_pairs.append( + TlsKeyPair( + certificate=self._read_certs_file(crt), + key=self._read_certs_file(key), + ) + ) + return key_pairs + + def _batched(self, iterable: typing.Iterable, n: int) -> typing.Iterable: + """Batch data from the iterable into tuples of length n. The last may be shorter than n. + + Args: + iterable: The iterable to batch. + n: The number of elements to batch. + + Returns: + An iterator over the tuples of length n. + """ + if n < 1: + raise ValueError("n must be at least one") + iterator = iter(iterable) + while batch := tuple(itertools.islice(iterator, n)): + yield batch + + def write_tls_key_pairs(self, key_pairs: list[TlsKeyPair]) -> None: + """Write TLS key pairs to the certificates directory. + + Existing pairs are overwritten, and if more files exist than new key pairs provided, + the excess files are removed. + + Args: + key_pairs: A list of TlsKeyPair objects to write. + """ + self._make_certs_dir() + files = sorted(self._iter_certs_dir()) + for idx, (key_pair_files, key_pair) in enumerate( + itertools.zip_longest(self._batched(files, 2), key_pairs) + ): + if key_pair_files is None: + self._write_certs_file(self.CERTS_DIR / f"{idx:04}.crt", key_pair.certificate) + self._write_certs_file(self.CERTS_DIR / f"{idx:04}.key", key_pair.key) + continue + if key_pair is None: + for file in key_pair_files: + self._unlink_certs_file(file) + continue + crt_file, key_file = key_pair_files + if self._read_certs_file(crt_file) != key_pair.certificate: + self._write_certs_file(crt_file, key_pair.certificate) + if self._read_certs_file(key_file) != key_pair.key: + self._write_certs_file(key_file, key_pair.key) + + @staticmethod + def restart() -> None: + """Restart the chrony service.""" + systemd.service_restart("chrony") # pragma: nocover + + @staticmethod + def parse_source_url(url: str) -> TimeSource: + """Parse a time source from a URL. + + Args: + url: URL to parse. + + Returns: + Parsed TimeSource instance. + + Raises: + ValueError: If the URL is invalid. + """ + if url.startswith("ntp://"): + return _NtpSource.from_source_url(url) + if url.startswith("nts://"): + return _NtsSource.from_source_url(url) + raise ValueError(f"Invalid time source URL: {url}") + + @staticmethod + def new_config(sources: list[TimeSource], header: str = "") -> str: + """Generate the chrony configuration file content. + + Args: + header: Optional header in the configuration file. + sources: List of chrony time sources. + + Returns: + Generated chrony configuration file content. + + Raises: + ValueError: If no sources are provided. + """ + if not sources: + raise ValueError("No time sources provided") + sources_config = "\n".join(s.render() for s in sources) + static = textwrap.dedent( + """\ + sourcedir /run/chrony-dhcp + sourcedir /etc/chrony/sources.d + keyfile /etc/chrony/chrony.keys + driftfile /var/lib/chrony/chrony.drift + ntsdumpdir /var/lib/chrony + logdir /var/log/chrony + maxupdateskew 100.0 + rtcsync + makestep 1 3 + leapsectz right/UTC + """ + ) + return "\n\n".join(part for part in [header, sources_config, static] if part).lstrip() diff --git a/src/grafana_dashboards/chrony.json b/src/grafana_dashboards/chrony.json new file mode 100644 index 0000000..3fde952 --- /dev/null +++ b/src/grafana_dashboards/chrony.json @@ -0,0 +1,1212 @@ +{ + "__inputs": [], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.5.3" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Dashboard for Chrony Client Operator", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 11, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "refId": "A" + } + ], + "title": "Tracking", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "name" + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "chrony_tracking_info{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "{{ tracking_address }}", + "refId": "A" + } + ], + "title": "Reference Source Address", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 4 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value" + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "chrony_tracking_stratum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Stratum", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "chrony_tracking_last_offset_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", + "interval": "", + "legendFormat": "Last Offset", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "chrony_tracking_rms_offset_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", + "hide": false, + "interval": "", + "legendFormat": "RMS Offset (long term average)", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "chrony_tracking_system_time_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", + "hide": false, + "interval": "", + "legendFormat": "System Time", + "range": true, + "refId": "C" + } + ], + "title": "Offset", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 4 + }, + "id": 2, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "name" + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "chrony_tracking_info{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", + "instant": true, + "interval": "", + "legendFormat": "{{ tracking_refid }}", + "refId": "A" + } + ], + "title": "Reference Source ID", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Absolute bound on the computer’s clock accuracy (assuming the stratum-1 computer is correct)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "abs(chrony_tracking_last_offset_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}) + chrony_tracking_root_dispersion_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"} + (0.5 * chrony_tracking_root_delay_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Clock Error", + "range": true, + "refId": "A" + } + ], + "title": "Maximum Clock Error", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "chrony_tracking_root_delay_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", + "interval": "", + "legendFormat": "Root delay", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "chrony_tracking_root_dispersion_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", + "hide": false, + "interval": "", + "legendFormat": "Root dispersion", + "range": true, + "refId": "B" + } + ], + "title": "Source delay", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "mode": "basic", + "type": "color-background" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "State" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "candidate": { + "color": "light-green", + "index": 4, + "text": "combined" + }, + "falseticker": { + "color": "orange", + "index": 2, + "text": "falseticker" + }, + "jittery": { + "color": "yellow", + "index": 3, + "text": "jittery" + }, + "outlier": { + "color": "transparent", + "index": 5, + "text": "not combined" + }, + "sync": { + "color": "green", + "index": 0, + "text": "synchronized" + }, + "unreach": { + "color": "red", + "index": 1, + "text": "unreachable" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Polling Interval" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 19, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 0, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Juju Unit" + } + ] + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_join(chrony_sources_state_info{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}, \"source_uid\", \"-\", \"juju_model_uuid\", \"juju_unit\", \"source_address\")", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "__auto", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_join(chrony_sources_stratum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}, \"source_uid\", \"-\", \"juju_model_uuid\", \"juju_unit\", \"source_address\")", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_join(chrony_sources_polling_interval_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}, \"source_uid\", \"-\", \"juju_model_uuid\", \"juju_unit\", \"source_address\")", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "__auto", + "range": false, + "refId": "C" + } + ], + "title": "Current Status", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "source_uid", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 2": true, + "Value #A": true, + "Value #B": false, + "__name__": true, + "__name__ 2": true, + "instance": true, + "instance 2": true, + "job": true, + "job 2": true, + "juju_application 1": true, + "juju_application 2": true, + "juju_application 3": true, + "juju_model 2": true, + "juju_model 3": true, + "juju_model_uuid 2": true, + "juju_model_uuid 3": true, + "juju_unit 2": true, + "juju_unit 3": true, + "source_address 1": true, + "source_address 2": true, + "source_address 3": true, + "source_mode": true, + "source_name 2": true, + "source_name 3": true, + "source_uid": true + }, + "indexByName": { + "Time 1": 0, + "Time 2": 8, + "Time 3": 14, + "Value #A": 7, + "Value #B": 13, + "Value #C": 19, + "__name__ 1": 1, + "__name__ 2": 9, + "__name__ 3": 15, + "instance 1": 2, + "instance 2": 10, + "instance 3": 16, + "job 1": 3, + "job 2": 11, + "job 3": 17, + "juju_application 1": 21, + "juju_application 2": 26, + "juju_application 3": 31, + "juju_model 1": 22, + "juju_model 2": 27, + "juju_model 3": 32, + "juju_model_uuid 1": 24, + "juju_model_uuid 2": 28, + "juju_model_uuid 3": 33, + "juju_unit 1": 23, + "juju_unit 2": 29, + "juju_unit 3": 34, + "source_address 1": 25, + "source_address 2": 30, + "source_address 3": 35, + "source_mode": 4, + "source_name 1": 5, + "source_name 2": 12, + "source_name 3": 18, + "source_state": 6, + "source_uid": 20 + }, + "renameByName": { + "Value #B": "Stratum", + "Value #C": "Polling Interval", + "juju_application 1": "Juju Application", + "juju_model 1": "Juju Model", + "juju_model_uuid 1": "Juju Model UUID", + "juju_unit 1": "Juju Unit", + "source_address": "Address", + "source_name": "Name", + "source_name 1": "Name", + "source_state": "State" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 13, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "refId": "A" + } + ], + "title": "Sources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 24 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "chrony_sources_last_sample_age_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"} < 4.294967295e+09", + "interval": "", + "legendFormat": "{{ source_address }} ({{ source_name }})", + "range": true, + "refId": "A" + } + ], + "title": "Last Sample: Age", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 24 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "chrony_sources_last_sample_offset_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", + "interval": "", + "legendFormat": "{{ source_address }} ({{ source_name }})", + "range": true, + "refId": "A" + } + ], + "title": "Last Sample: Offset", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 24 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "chrony_sources_last_sample_error_margin_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", + "interval": "", + "legendFormat": "{{ source_address }} ({{ source_name }})", + "range": true, + "refId": "A" + } + ], + "title": "Last Sample: Error Margin", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "UTC", + "title": "Chrony Client Operator", + "version": 1, + "weekStart": "" +} diff --git a/src/prometheus_alert_rules/chrony.rule b/src/prometheus_alert_rules/chrony.rule new file mode 100644 index 0000000..0af4546 --- /dev/null +++ b/src/prometheus_alert_rules/chrony.rule @@ -0,0 +1,45 @@ +groups: + - name: chrony + rules: + - alert: ChronyTargetMissing + expr: up == 0 + for: 1m + labels: + severity: critical + annotations: + summary: Prometheus target missing (instance {{ $labels.instance }}) + description: | + Chrony target has disappeared. An exporter on {{ $labels.instance }} + might be crashed. + - alert: ChronyTrackingHighOffset + expr: chrony_tracking_last_offset_seconds > 1 + for: 1h + labels: + severity: critical + annotations: + summary: "Chrony tracking offset is high ({{ $value }}s)" + description: | + The last clock offset reported by Chrony on {{ $labels.instance }} + has been {{ $value }} seconds, exceeding the 1 s threshold for over 1 hour. + - alert: ChronyTrackingStaleMeasurement + expr: chrony_tracking_update_interval_seconds > 1800 + for: 1h + labels: + severity: critical + annotations: + summary: "Chrony update interval is too long ({{ $value }}s)" + description: | + Chrony on {{ $labels.instance }} has not processed a new measurement + for over 30 minutes. The current update interval is {{ $value }} seconds, + exceeding the 1800 s threshold for more than 1 hour. + - alert: ChronyHighStratum + expr: chrony_tracking_stratum > 3 + for: 1h + labels: + severity: warning + annotations: + summary: "Chrony tracking stratum is too high ({{ $value }})" + description: | + Chrony on instance {{ $labels.instance }} is tracking a source with + stratum {{ $value }}, which is above the acceptable threshold of 3 for + over 1 hour. diff --git a/tests/conftest.py b/tests/conftest.py index 09a84dd..b6a62ec 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,4 +10,22 @@ def pytest_addoption(parser): Args: parser: Pytest parser. """ - parser.addoption("--charm-file", action="store") + parser.addoption("--charm-file", action="append", default=[]) + parser.addoption( + "--use-existing", + action="store_true", + default=False, + help="This will skip deployment of the charms. Useful for local testing.", + ) + parser.addoption( + "--keep-models", + action="store_true", + default=False, + help="keep temporarily-created models", + ) + parser.addoption( + "--model", + action="store", + help="Juju model to use; if not provided, a new model " + "will be created for each test which requires one", + ) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index dddb292..0e44a50 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -1,2 +1,4 @@ # Copyright 2025 Canonical Ltd. # See LICENSE file for licensing details. + +"""Charm integration tests.""" diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 0000000..6a5d060 --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,198 @@ +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Fixtures for charm integration tests.""" + +import pathlib +import subprocess # nosec B404 +import typing + +import jubilant +import pytest + + +@pytest.fixture(name="chrony_client_charm_file", scope="session") +def chrony_client_charm_file_fixture(pytestconfig: pytest.Config): + """Build or get the chrony-client charm file.""" + charms = pytestconfig.getoption("--charm-file") + # if there's only one charm file supplied, use that one + if len(charms) == 1: + return charms[0] + + # else select the 24.04 based + charms = [c for c in charms if "24.04" in c] + if charms: + return charms[0] + + # else build the charm from source + try: + subprocess.run( + ["charmcraft", "pack", "--bases-index=0"], check=True, capture_output=True, text=True + ) # nosec B603, B607 + except subprocess.CalledProcessError as exc: + raise OSError(f"Error packing charm: {exc}; Stderr:\n{exc.stderr}") from None + + app_name = "chrony-client" + charm_path = pathlib.Path(__file__).parent.parent.parent + charms = [p.absolute() for p in charm_path.glob(f"{app_name}_*24.04*.charm")] + assert charms, f"{app_name} .charm file not found" + assert len(charms) == 1, f"{app_name} has more than one .charm file, unsure which to use" + return str(charms[0]) + + +@pytest.fixture(name="juju", scope="module") +def juju_fixture(request: pytest.FixtureRequest) -> typing.Generator[jubilant.Juju, None, None]: + """Pytest fixture that wraps :meth:`jubilant.with_model`.""" + + def show_debug_log(juju: jubilant.Juju) -> None: + if request.session.testsfailed: + log = juju.debug_log(limit=1000) + print(log, end="") + + use_existing = request.config.getoption("--use-existing", default=False) + if use_existing: + juju = jubilant.Juju() + yield juju + show_debug_log(juju) + return + + model = request.config.getoption("--model") + if model: + juju = jubilant.Juju(model=model) + yield juju + show_debug_log(juju) + return + + keep_models = typing.cast(bool, request.config.getoption("--keep-models")) + with jubilant.temp_model(keep=keep_models) as juju: + juju.wait_timeout = 10 * 60 + yield juju + show_debug_log(juju) + return + + +@pytest.fixture(name="deploy_charms", scope="module") +def deploy_charms_fixture(juju: jubilant.Juju, chrony_client_charm_file: str): + """Deploy charms fixture deploy all charms necessary for the integration test.""" + juju.deploy(charm="ubuntu", base="ubuntu@24.04") + juju.deploy(charm=chrony_client_charm_file) + juju.deploy( + charm="chrony", + config={"sources": "ntp://ntp.ubuntu.com?iburst=true&maxsources=4"}, + channel="latest/edge", + ) + juju.integrate("ubuntu", "chrony-client") + juju.wait(jubilant.all_active, timeout=20 * 60) + + +class App: + """A helper class for charm applications.""" + + def __init__(self, juju: jubilant.Juju, name: str) -> None: + """Initialize the charm application class. + + Args: + juju: Juju instance + name: Application name + """ + self._juju = juju + self.name = name + + def get_leader_unit(self) -> str: + """Get the leader unit name for this application. + + Returns: + Leader unit name. + + Raises: + RuntimeError: If no leader unit exists for this application. + """ + status = self._juju.status() + leader = [name for name, unit in status.get_units(self.name).items() if unit.leader] + if not leader: + raise RuntimeError(f"no leader unit found for {self.name}?") + return leader[0] + + def get_unit_ip(self, unit_num: int | None = None) -> str: + """Get the IP address of the unit. + + Args: + unit_num: unit number, if not provided, the leader unit number is used. + + Returns: + IP address of the unit. + """ + status = self._juju.status() + units = status.get_units(self.name) + if unit_num is None: + unit_name = self.get_leader_unit() + else: + unit_name = f"{self.name}/{unit_num}" + unit_ip = units[unit_name].public_address + return unit_ip + + def ssh(self, cmd: str, *, unit_num: int | None = None) -> str: + """Run a command on a charm unit. + + Args: + cmd: command to run + unit_num: unit number, if not provided, the leader unit number is used. + + Returns: + Output of the command. + """ + if unit_num is None: + unit_name = self.get_leader_unit() + else: + unit_name = f"{self.name}/{unit_num}" + return self._juju.ssh(target=unit_name, command=cmd) + + +@pytest.fixture(scope="module", name="principle_app") +def principle_app_fixture( + juju: jubilant.Juju, + # pylint: disable=unused-argument + deploy_charms, +): + """Deploy the principle charm app.""" + return App(juju=juju, name="ubuntu") + + +@pytest.fixture(scope="module", name="chrony_client_app") +def chrony_client_app_fixture( + juju: jubilant.Juju, + # pylint: disable=unused-argument + deploy_charms, +) -> App: + """Deployed chrony-client charm app.""" + return App(juju=juju, name="chrony-client") + + +@pytest.fixture(scope="module") +def chrony_app( + juju: jubilant.Juju, + # pylint: disable=unused-argument + deploy_charms, +) -> App: + """Deployed chrony charm app.""" + return App(juju=juju, name="chrony") + + +@pytest.fixture(scope="function") +def another_chrony_client_app( + juju: jubilant.Juju, + chrony_client_app, + chrony_client_charm_file, + # pylint: disable=unused-argument + principle_app, +): + """Deploy another chrony-client charm app.""" + name = "another-chrony-client" + + juju.deploy(charm=chrony_client_charm_file, app=name) + juju.integrate("ubuntu", name) + juju.wait(jubilant.all_agents_idle, timeout=20 * 60) + + yield App(juju=juju, name=name) + + juju.remove_application(name) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt new file mode 100644 index 0000000..cebd81e --- /dev/null +++ b/tests/integration/requirements.txt @@ -0,0 +1,2 @@ +pytest==8.4.1 +jubilant==1.2.0 \ No newline at end of file diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 3bc3268..7fe442f 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -5,35 +5,59 @@ """Integration tests.""" -import asyncio -import logging -from pathlib import Path - +import jubilant import pytest -import yaml -from pytest_operator.plugin import OpsTest -logger = logging.getLogger(__name__) -CHARMCRAFT_DATA = yaml.safe_load(Path("./charmcraft.yaml").read_text(encoding="utf-8")) -APP_NAME = CHARMCRAFT_DATA["name"] +def test_time_sources(juju, chrony_client_app, chrony_app): + """ + arrange: deploy the chrony-client and chrony charm. + act: use the chrony charm as the time source for the chrony-client charm. + assert: check if the chrony-client charm is using the time source. + """ + server_ip = chrony_app.get_unit_ip() + sources = f"ntp://{server_ip}?iburst=true" + juju.config(chrony_client_app.name, {"sources": sources}) + juju.wait( + lambda *args, **kwargs: jubilant.all_active(*args, **kwargs) + and jubilant.all_agents_idle(*args, **kwargs) + ) + assert server_ip in chrony_client_app.ssh("chronyc -N -n -c sources") -@pytest.mark.abort_on_fail -async def test_build_and_deploy(ops_test: OpsTest, pytestconfig: pytest.Config): - """Deploy the charm together with related charms. - Assert on the unit status before any relations/configurations take place. +def test_chrony_exporter(chrony_client_app): """ - # Deploy the charm and wait for active/idle status - charm = pytestconfig.getoption("--charm-file") - resources = {"httpbin-image": CHARMCRAFT_DATA["resources"]["httpbin-image"]["upstream-source"]} - assert ops_test.model - await asyncio.gather( - ops_test.model.deploy( - f"./{charm}", resources=resources, application_name=APP_NAME, series="jammy" - ), - ops_test.model.wait_for_idle( - apps=[APP_NAME], status="active", raise_on_blocked=True, timeout=1000 - ), - ) + arrange: deploy the chrony-client charm. + act: request chrony_exporter metrics endpoint. + assert: confirm that metrics are scraped. + """ + stdout = chrony_client_app.ssh("curl -m 10 localhost:9123/metrics") + assert "chrony_sources_reachability_success" in stdout + + +def test_charm_conflict(juju, another_chrony_client_app): + """ + arrange: deploy the chrony-client charm. + act: deploy another chrony-client charm on the principle charm. + assert: confirm that the second charm is in block state. + """ + units = juju.status().get_units(another_chrony_client_app.name) + status = units[another_chrony_client_app.get_leader_unit()].workload_status + assert status.current == "blocked" + assert "conflict" in status.message + + +def test_charm_uninstall_cleanup(juju, chrony_client_app, principle_app): + """ + arrange: deploy the chrony-client charm. + act: remove the chrony-client charm. + assert: confirm that the chrony-charm related configuration and packages are removed + """ + juju.remove_application(chrony_client_app.name) + juju.wait(jubilant.all_active, timeout=20 * 60) + + with pytest.raises(jubilant.CLIError): + principle_app.ssh("which chrony_exporter") + + assert "charm" not in principle_app.ssh("cat /etc/chrony/chrony.conf") diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index dddb292..6f65eaa 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -1,2 +1,4 @@ # Copyright 2025 Canonical Ltd. # See LICENSE file for licensing details. + +"""Charm unit tests.""" diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py new file mode 100644 index 0000000..5ef2682 --- /dev/null +++ b/tests/unit/conftest.py @@ -0,0 +1,112 @@ +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Fixtures for charm tests.""" + +import pathlib +from unittest.mock import patch + +import pytest + +import chrony + + +@pytest.fixture(name="patch_charm", autouse=True) +def patch_charm_fixture(): + """Patch necessary functions in the charm.""" + chrony_lock_file = None + + def _write_chrony_lock_file(content: str) -> None: + nonlocal chrony_lock_file + chrony_lock_file = content + + def _read_chrony_lock_file() -> None | str: + return chrony_lock_file + + def _delete_chrony_lock_file(): + nonlocal chrony_lock_file + chrony_lock_file = None + + with ( + patch("charm.ChronyClientCharm._write_chrony_lock_file") as mock_write_chrony_lock_file, + patch("charm.ChronyClientCharm._read_chrony_lock_file") as mock_read_chrony_lock_file, + patch("charm.ChronyClientCharm._delete_chrony_lock_file") as mock_delete_chrony_lock_file, + ): + mock_write_chrony_lock_file.side_effect = _write_chrony_lock_file + mock_read_chrony_lock_file.side_effect = _read_chrony_lock_file + mock_delete_chrony_lock_file.side_effect = _delete_chrony_lock_file + yield + + +@pytest.fixture(name="mock_chrony", autouse=True) +def mock_chrony_fixture(): # noqa: C901 pylint: disable=too-many-locals + """Create a Chrony object with necessary methods patched.""" + installed = False + + def install(): + nonlocal installed + installed = True + + def uninstall(): + nonlocal installed + installed = False + + mock_config = "" + + def read_config(): + return mock_config + + def write_config(config: str): + nonlocal mock_config + mock_config = config + + certs: dict[str, str] = {} + + def _iter_certs_dir(): + for file in certs: + yield pathlib.Path("/etc/chrony/certs") / file + + def _write_certs_file(path: pathlib.Path, content: str): + certs[path.name] = content + + def _read_certs_file(path: pathlib.Path): + return certs[path.name] + + def _unlink_certs_file(path: pathlib.Path) -> None: + del certs[path.name] + + backup_config_content = None + + def backup_config(): + nonlocal backup_config_content + backup_config_content = read_config() + + def restore_config(): + if backup_config_content is not None: + write_config(backup_config_content) + + with ( + patch("chrony.Chrony.install") as mock_install, + patch("chrony.Chrony.uninstall") as mock_uninstall, + patch("chrony.Chrony.restart"), + patch("chrony.Chrony.write_config") as mock_write_config, + patch("chrony.Chrony.read_config") as mock_read_config, + patch("chrony.Chrony.backup_config") as mock_backup_config, + patch("chrony.Chrony.restore_config") as mock_restore_config, + patch("chrony.Chrony._make_certs_dir"), + patch("chrony.Chrony._iter_certs_dir") as mock_iter_certs_dir, + patch("chrony.Chrony._write_certs_file") as mock_write_certs_file, + patch("chrony.Chrony._read_certs_file") as mock_read_certs_file, + patch("chrony.Chrony._unlink_certs_file") as mock_unlink_certs_file, + ): + mock_install.side_effect = install + mock_uninstall.side_effect = uninstall + mock_read_config.side_effect = read_config + mock_write_config.side_effect = write_config + mock_backup_config.side_effect = backup_config + mock_restore_config.side_effect = restore_config + mock_iter_certs_dir.side_effect = _iter_certs_dir + mock_write_certs_file.side_effect = _write_certs_file + mock_read_certs_file.side_effect = _read_certs_file + mock_unlink_certs_file.side_effect = _unlink_certs_file + yield chrony.Chrony() diff --git a/tests/unit/requirements.txt b/tests/unit/requirements.txt new file mode 100644 index 0000000..a69ffb7 --- /dev/null +++ b/tests/unit/requirements.txt @@ -0,0 +1,3 @@ +coverage[toml]==7.9.1 +pytest==8.4.1 +ops[testing]==2.22.0 diff --git a/tests/unit/test_base.py b/tests/unit/test_base.py deleted file mode 100644 index 8a55eb2..0000000 --- a/tests/unit/test_base.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2025 Canonical Ltd. -# See LICENSE file for licensing details. - -# Learn more about testing at: https://ops.readthedocs.io/en/latest/explanation/testing.html - -# pylint: disable=duplicate-code,missing-function-docstring -"""Unit tests.""" - -import unittest - -import ops -import ops.testing - -from charm import IsCharmsTemplateCharm - - -class TestCharm(unittest.TestCase): - """Test class.""" - - def setUp(self): - """Set up the testing environment.""" - self.harness = ops.testing.Harness(IsCharmsTemplateCharm) - self.addCleanup(self.harness.cleanup) - self.harness.begin() - - def test_httpbin_pebble_ready(self): - # Expected plan after Pebble ready with default config - expected_plan = { - "services": { - "httpbin": { - "override": "replace", - "summary": "httpbin", - "command": "gunicorn -b 0.0.0.0:80 httpbin:app -k gevent", - "startup": "enabled", - "environment": {"GUNICORN_CMD_ARGS": "--log-level info"}, - } - }, - } - # Simulate the container coming up and emission of pebble-ready event - self.harness.container_pebble_ready("httpbin") - # Get the plan now we've run PebbleReady - updated_plan = self.harness.get_container_pebble_plan("httpbin").to_dict() - # Check we've got the plan we expected - self.assertEqual(expected_plan, updated_plan) - # Check the service was started - service = self.harness.model.unit.get_container("httpbin").get_service("httpbin") - self.assertTrue(service.is_running()) - # Ensure we set an ActiveStatus with no message - self.assertEqual(self.harness.model.unit.status, ops.ActiveStatus()) - - def test_config_changed_valid_can_connect(self): - # Ensure the simulated Pebble API is reachable - self.harness.set_can_connect("httpbin", True) - # Trigger a config-changed event with an updated value - self.harness.update_config({"log-level": "debug"}) - # Get the plan now we've run PebbleReady - updated_plan = self.harness.get_container_pebble_plan("httpbin").to_dict() - updated_env = updated_plan["services"]["httpbin"]["environment"] - # Check the config change was effective - self.assertEqual(updated_env, {"GUNICORN_CMD_ARGS": "--log-level debug"}) - self.assertEqual(self.harness.model.unit.status, ops.ActiveStatus()) - - def test_config_changed_valid_cannot_connect(self): - # Trigger a config-changed event with an updated value - self.harness.update_config({"log-level": "debug"}) - # Check the charm is in WaitingStatus - self.assertIsInstance(self.harness.model.unit.status, ops.WaitingStatus) - - def test_config_changed_invalid(self): - # Ensure the simulated Pebble API is reachable - self.harness.set_can_connect("httpbin", True) - # Trigger a config-changed event with an updated value - self.harness.update_config({"log-level": "foobar"}) - # Check the charm is in BlockedStatus - self.assertIsInstance(self.harness.model.unit.status, ops.BlockedStatus) diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py new file mode 100644 index 0000000..d0249ae --- /dev/null +++ b/tests/unit/test_charm.py @@ -0,0 +1,183 @@ +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +# Learn more about testing at: https://ops.readthedocs.io/en/latest/explanation/testing.html + +# pylint: disable=duplicate-code,missing-function-docstring,protected-access + +"""Unit tests.""" + +import textwrap + +import pytest +from ops import testing + +import charm +import chrony + + +@pytest.mark.parametrize( + "sources, valid, source_config", + [ + pytest.param( + "ntp://example.com", + True, + "pool example.com", + id="ntp server", + ), + pytest.param( + "ntp://example.com:1234", + True, + "pool example.com port 1234", + id="ntp server with port", + ), + pytest.param( + "ntp://example.com?iburst=true", + True, + "pool example.com iburst", + id="ntp server with iburst option", + ), + pytest.param( + "ntp://example.com:1234?iburst=true&minpoll=10&polltarget=50", + True, + "pool example.com port 1234 iburst minpoll 10 polltarget 50", + id="ntp server with multiple option", + ), + pytest.param( + "nts://example.com?require=true&offset=-0.1", + True, + "pool example.com nts offset -0.1 require", + id="nts server", + ), + pytest.param( + textwrap.dedent( + """\ + ntp://ntp.ubuntu.com?iburst=true&maxsources=4, + ntp://0.ubuntu.pool.ntp.org?iburst=true&maxsources=1, + ntp://1.ubuntu.pool.ntp.org?iburst=true&maxsources=1, + ntp://2.ubuntu.pool.ntp.org?iburst=true&maxsources=2 + """ + ), + True, + textwrap.dedent( + """\ + pool ntp.ubuntu.com iburst maxsources 4 + pool 0.ubuntu.pool.ntp.org iburst maxsources 1 + pool 1.ubuntu.pool.ntp.org iburst maxsources 1 + pool 2.ubuntu.pool.ntp.org iburst maxsources 2 + """ + ), + id="multiple ntp server", + ), + pytest.param( + "example.com", + False, + "", + id="invalid ntp server", + ), + pytest.param( + "", + False, + "", + id="no sources", + ), + pytest.param( + "example.com", + False, + "", + id="invalid sources: no protocol", + ), + pytest.param( + "example.com:99999", + False, + "", + id="invalid sources: invalid port", + ), + pytest.param( + "https://example.com", + False, + "", + id="invalid sources: unknown protocol", + ), + pytest.param( + "ntp://example.com?foobar=true", + False, + "", + id="invalid sources: unknown param", + ), + ], +) +def test_chrony_config(sources: str, valid: bool, source_config: str, mock_chrony: chrony.Chrony): + """ + arrange: none. + act: trigger the 'config-changed' event with different sources charm configuration. + assert: check if configuration file content matches the charm configuration. + """ + mock_chrony.write_config("default") + + ctx = testing.Context(charm.ChronyClientCharm) + state_in = testing.State( + config={"sources": sources}, + relations=[testing.SubordinateRelation(endpoint="juju-info", id=1)], + ) + + state_out = ctx.run(ctx.on.config_changed(), state_in) + + assert charm.ChronyClientCharm._read_chrony_lock_file() == "chrony-client" + + if not valid: + assert state_out.unit_status.name == testing.BlockedStatus.name + assert mock_chrony.read_config() == "default" + assert not mock_chrony.restart.called + return + + assert state_out.unit_status == testing.ActiveStatus() + expected_config = ( + charm.CHRONY_CHARM_CONFIG_HEADER + + "\n\n" + + source_config.strip() + + "\n" + + textwrap.dedent( + """ + sourcedir /run/chrony-dhcp + sourcedir /etc/chrony/sources.d + keyfile /etc/chrony/chrony.keys + driftfile /var/lib/chrony/chrony.drift + ntsdumpdir /var/lib/chrony + logdir /var/log/chrony + maxupdateskew 100.0 + rtcsync + makestep 1 3 + leapsectz right/UTC + """ + ) + ) + assert mock_chrony.read_config() == expected_config + mock_chrony.restart.assert_called_once() + + +def test_chrony_uninstall(mock_chrony: chrony.Chrony): + """ + arrange: run the `config-changed` event + act: trigger the 'remove' event. + assert: check if configuration file content and packages are restored. + """ + mock_chrony.write_config("default") + + ctx = testing.Context(charm.ChronyClientCharm) + state_in = testing.State( + config={"sources": "ntp://example.com"}, + relations=[testing.SubordinateRelation(endpoint="juju-info", id=1)], + ) + ctx.run(ctx.on.config_changed(), state_in) + + ctx = testing.Context(charm.ChronyClientCharm) + state_in = testing.State( + config={"sources": "ntp://example.com"}, + relations=[testing.SubordinateRelation(endpoint="juju-info", id=1)], + ) + ctx.run(ctx.on.remove(), state_in) + + assert charm.ChronyClientCharm._read_chrony_lock_file() is None + assert mock_chrony.read_config() == "default" + mock_chrony.uninstall.assert_called_once() diff --git a/tox.ini b/tox.ini index 3c6615a..fa128cf 100644 --- a/tox.ini +++ b/tox.ini @@ -36,9 +36,9 @@ description = Check code against coding style standards deps = black codespell - flake8<6.0.0 + flake8 flake8-builtins - flake8-copyright<6.0.0 + flake8-copyright flake8-docstrings>=1.6.0 flake8-docstrings-complete>=1.0.3 flake8-test-docs>=1.0 @@ -47,7 +47,7 @@ deps = pep8-naming pydocstyle>=2.10 pylint - pyproject-flake8<6.0.0 + pyproject-flake8 pytest pytest-asyncio pytest-operator @@ -55,6 +55,8 @@ deps = types-PyYAML types-requests -r{toxinidir}/requirements.txt + -r{toxinidir}/tests/unit/requirements.txt + -r{toxinidir}/tests/integration/requirements.txt commands = pydocstyle {[vars]src_path} # uncomment the following line if this charm owns a lib @@ -72,9 +74,8 @@ commands = [testenv:unit] description = Run unit tests deps = - coverage[toml] - pytest -r{toxinidir}/requirements.txt + -r{toxinidir}/tests/unit/requirements.txt commands = coverage run --source={[vars]src_path} \ -m pytest --ignore={[vars]tst_path}integration -v --tb native -s {posargs} @@ -100,10 +101,7 @@ commands = [testenv:integration] description = Run integration tests deps = - juju==3.6.* - pytest - pytest-asyncio - pytest-operator -r{toxinidir}/requirements.txt + -r{toxinidir}/tests/integration/requirements.txt commands = pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} From 4134e950532b0cf0e98dad30501052c8039a5cc6 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Thu, 10 Jul 2025 14:15:11 +0800 Subject: [PATCH 02/15] Apply suggestions from code review Co-authored-by: Erin Conley --- docs/explanation/charm-architecture.md | 12 ++++++++---- docs/explanation/security.md | 2 +- docs/how-to/integrate-with-cos.md | 18 +++++++++--------- docs/how-to/upgrade.md | 2 +- docs/tutorial.md | 2 +- 5 files changed, 20 insertions(+), 16 deletions(-) diff --git a/docs/explanation/charm-architecture.md b/docs/explanation/charm-architecture.md index 8a2e9f9..cda3de9 100644 --- a/docs/explanation/charm-architecture.md +++ b/docs/explanation/charm-architecture.md @@ -3,7 +3,7 @@ At its core, the Chrony client charm is a simple Python program that installs and configures `chrony` and `chrony_exporter`. -The Chrony client charm is a subordinate charm which is a charm designed +The Chrony client charm is a subordinate charm, meaning it is designed to be deployed adjacent to another charm and to augment the functionality of that charm. In this case, it helps to set up Chrony as a NTP client. @@ -19,9 +19,10 @@ non-subordinate machine charm. ```mermaid C4Context title Component diagram for Chrony client charm - System_Boundary(vm, "VM machine") { - Container(principal, "Principal charm") + Container_Boundary(principal-charm, "Principal charm"){ + Component(principal, "Principal charm") + } Container_Boundary(chrony-client, "Chrony client charm") { Component(chrony, "Chrony") Component(chrony-exporter, "Chrony exporter") @@ -29,9 +30,12 @@ C4Context Container_Boundary(grafana-agent-charm, "Grafana agent charm") { Component(grafana-agent, "Grafana agent") } + Rel(principal, chrony, "Juju info") + UpdateRelStyle(principal, chrony, $offsetX="-22", $offsetY="10") Rel(chrony-exporter, grafana-agent, "Prometheus metrics") - UpdateRelStyle(chrony-exporter, grafana-agent, $offsetX="-50", $offsetY="10") + UpdateRelStyle(chrony-exporter, grafana-agent, $offsetX="-50", $offsetY="20") } + UpdateLayoutConfig($c4ShapeInRow="1", $c4BoundaryInRow="3") ``` ## Metrics diff --git a/docs/explanation/security.md b/docs/explanation/security.md index 01a698f..c13abc5 100644 --- a/docs/explanation/security.md +++ b/docs/explanation/security.md @@ -13,7 +13,7 @@ on localhost. `chrony` is installed from the Ubuntu archive, and security patches are delivered through Ubuntu archive updates. Use Ubuntu Pro for faster -security responses. Learn more about [Ubuntu Pro in Juju charms](https://charmhub.io/ubuntu-advantage). +security responses. See [the Ubuntu Pro charm](https://charmhub.io/ubuntu-advantage). `chrony_exporter` is installed from the Platform Engineering team’s PPA (`ppa:canonical-is-devops/chrony-charm`) and maintained by the diff --git a/docs/how-to/integrate-with-cos.md b/docs/how-to/integrate-with-cos.md index 2768a9a..81e5aef 100644 --- a/docs/how-to/integrate-with-cos.md +++ b/docs/how-to/integrate-with-cos.md @@ -8,32 +8,32 @@ The COS integration for the Chrony client charm is provided by the [Grafana Agent charm](https://charmhub.io/grafana-agent). Before integrating the COS charms, you must first integrate the Chrony client charm with the Grafana Agent charm. Because the Grafana Agent charm is -also a subordinate charm, you cannot directly relate it to the Chrony +also a subordinate charm, you cannot directly integrate it to the Chrony client charm. Instead, integrate the Grafana Agent charm with a -principal charm first, then relate it to the Chrony client charm. +principal charm first, then integrate it to the Chrony client charm. Assuming you have already integrated the Chrony client charm with the Ubuntu charm as the principal charm: ```bash juju deploy chrony-client -juju relate chrony-client:juju-info ubuntu +juju integrate chrony-client:juju-info ubuntu ``` -Use the Grafana Agent charm’s `juju-info` interface to relate it to the +Use the Grafana Agent charm’s `juju-info` interface to integrate it to the principal charm: ```bash juju relate grafana-agent:juju-info ubuntu ``` -Then relate the Chrony client charm to the Grafana Agent charm. +Then integrate the Chrony client charm to the Grafana Agent charm. ## Integrate with the Prometheus K8s operator -Deploy and relate +Deploy and integrate the [`prometheus-k8s`](https://charmhub.io/prometheus-k8s) charm with the Grafana Agent charm through the `send-remote-write` relation using the `prometheus_remote_write` interface. The Grafana Agent will push the @@ -46,14 +46,14 @@ how to add one, see [the cross-model relation documentation](https://documentati ```bash juju consume cos-juju-controller:cos-juju-user/cos-model.receive-remote-write -juju relate grafana-agent:send-remote-write receive-remote-write +juju integrate grafana-agent:send-remote-write receive-remote-write ``` ## Integrate with the Grafana K8s operator -Deploy and relate the [`grafana-k8s`](https://charmhub.io/grafana-k8s) +Deploy and integrate the [`grafana-k8s`](https://charmhub.io/grafana-k8s) charm with the Grafana Agent charm through the `grafana-dashboards-provider` relation using the `grafana_dashboard` interface. The Grafana Agent will relay the dashboards provided by the @@ -65,5 +65,5 @@ see [the cross-model relation documentation](https://documentation.ubuntu.com/ju ```bash juju consume cos-juju-controller:cos-juju-user/cos-model.grafana-dashboard -juju relate grafana-agent:grafana-dashboards-provider grafana-dashboard +juju integrate grafana-agent:grafana-dashboards-provider grafana-dashboard ``` diff --git a/docs/how-to/upgrade.md b/docs/how-to/upgrade.md index b9419b1..41a965c 100644 --- a/docs/how-to/upgrade.md +++ b/docs/how-to/upgrade.md @@ -1,4 +1,4 @@ # How to upgrade -You can use the [`juju refresh` command](https://documentation.ubuntu.com/juju/latest/reference/juju-cli/list-of-juju-cli-commands/refresh/) +Use the [`juju refresh` command](https://documentation.ubuntu.com/juju/latest/reference/juju-cli/list-of-juju-cli-commands/refresh/) to upgrade the Chrony client charm. No additional operations are needed. diff --git a/docs/tutorial.md b/docs/tutorial.md index 5ac947d..4e5028a 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -58,7 +58,7 @@ juju deploy ubuntu --base ubuntu@24.04 ``` -## Deploy the Chrony client charm on the Ubuntu charm +## Deploy the Chrony client charm The following commands deploy the Chrony client charm and integrate it From 20a9a6d64e882a414ea4518ce83ccb73f97c983e Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Wed, 23 Jul 2025 14:37:36 +0800 Subject: [PATCH 03/15] Update documents --- docs/explanation/charm-architecture.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/explanation/charm-architecture.md b/docs/explanation/charm-architecture.md index cda3de9..cde068c 100644 --- a/docs/explanation/charm-architecture.md +++ b/docs/explanation/charm-architecture.md @@ -1,7 +1,7 @@ # Charm architecture -At its core, the Chrony client charm is a simple Python program that -installs and configures `chrony` and `chrony_exporter`. +At its core, the Chrony client charm installs and configures the +`chrony` and `chrony_exporter` services. The Chrony client charm is a subordinate charm, meaning it is designed to be deployed adjacent to another charm and to augment the From 0cc02fb7e9b59035ea1f280cbaa0cbb09adbe180 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Wed, 23 Jul 2025 14:38:09 +0800 Subject: [PATCH 04/15] Apply suggestions from code review Co-authored-by: Erin Conley --- docs/explanation/charm-architecture.md | 4 ++-- docs/explanation/security.md | 2 +- docs/how-to/integrate-with-cos.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/explanation/charm-architecture.md b/docs/explanation/charm-architecture.md index cde068c..0eb65e5 100644 --- a/docs/explanation/charm-architecture.md +++ b/docs/explanation/charm-architecture.md @@ -68,14 +68,14 @@ event, the Chrony client charm will upgrade the installed `chrony` or ### `config-changed` The `config-changed` hook always runs once immediately after the initial -install, after leader-elected hooks, and after the `upgrade-charm` hook. +install, after `leader-elected` hooks, and after the `upgrade-charm` hook. It also runs whenever application configuration changes. During this event, the Chrony client charm will update the configuration of `chrony` and may restart the `chrony` service if the configuration has changed. See the documentation on the [`config-changed` event](https://documentation.ubuntu.com/juju/latest/reference/hook/index.html#config-changed). ### `remove` -The remove event is emitted only once per unit: when the Juju controller +The `remove` event is emitted only once per unit: when the Juju controller is ready to remove the unit completely. All necessary steps for handling removal should be handled there. During this event, the Chrony client charm remove some installed packages and reset the chrony configuration diff --git a/docs/explanation/security.md b/docs/explanation/security.md index c13abc5..5a91068 100644 --- a/docs/explanation/security.md +++ b/docs/explanation/security.md @@ -7,7 +7,7 @@ The Chrony client charm is a simple charm with a minimal attack surface. The Chrony service is configured as a pure NTP client, and the Chrony client charm does not expose any ports. The Chrony exporter only listens -on localhost. +on `localhost`. ## Security patches diff --git a/docs/how-to/integrate-with-cos.md b/docs/how-to/integrate-with-cos.md index 81e5aef..d9370f0 100644 --- a/docs/how-to/integrate-with-cos.md +++ b/docs/how-to/integrate-with-cos.md @@ -1,5 +1,5 @@ -# Integrate with COS +# How to integrate with COS ## Prerequisites From 439a779a0e1a08beb138b671b07958188d8189c9 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Wed, 23 Jul 2025 14:46:49 +0800 Subject: [PATCH 05/15] Update metrics.md --- docs/reference/metrics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/metrics.md b/docs/reference/metrics.md index 6975cf7..b111754 100644 --- a/docs/reference/metrics.md +++ b/docs/reference/metrics.md @@ -18,7 +18,7 @@ - **`chrony_sources_reachability_ratio`**: Chrony sources ratio of packet reachability - **`chrony_sources_reachability_success`**: Chrony sources last poll reachability success - **`chrony_sources_state_info`**: Chrony sources state info -- **`chrony_sources_stratum`**: Chrony sources stratum +- **`chrony_sources_stratum`**: Chrony sources stratum number - **`chrony_tracking_frequency_ppms`**: Rate by which the system's clock would be wrong if chronyd was not correcting it, in PPMs - **`chrony_tracking_info`**: Chrony tracking info - **`chrony_tracking_last_offset_seconds`**: Chrony tracking last offset in seconds From 3c721232fb4fa3aa15a56267e5c0f28da955d6d6 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Wed, 23 Jul 2025 14:49:34 +0800 Subject: [PATCH 06/15] Update metrics.md --- docs/reference/metrics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/metrics.md b/docs/reference/metrics.md index b111754..9095517 100644 --- a/docs/reference/metrics.md +++ b/docs/reference/metrics.md @@ -18,7 +18,7 @@ - **`chrony_sources_reachability_ratio`**: Chrony sources ratio of packet reachability - **`chrony_sources_reachability_success`**: Chrony sources last poll reachability success - **`chrony_sources_state_info`**: Chrony sources state info -- **`chrony_sources_stratum`**: Chrony sources stratum number +- **`chrony_sources_stratum`**: Chrony sources stratum numbers - **`chrony_tracking_frequency_ppms`**: Rate by which the system's clock would be wrong if chronyd was not correcting it, in PPMs - **`chrony_tracking_info`**: Chrony tracking info - **`chrony_tracking_last_offset_seconds`**: Chrony tracking last offset in seconds From 5e07bd67436ad7cb9341328d9492e18c608907ac Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Fri, 25 Jul 2025 14:02:05 +0800 Subject: [PATCH 07/15] Add changelog.md and .woke.yaml --- .woke.yaml | 9 +++++---- docs/changelog.md | 8 +++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.woke.yaml b/.woke.yaml index 1a76403..0ef21cb 100644 --- a/.woke.yaml +++ b/.woke.yaml @@ -1,7 +1,8 @@ # Copyright 2025 Canonical Ltd. # See LICENSE file for licensing details. -ignore_files: - # Ignore apt charm library as it uses non compliant terminology: - # man-in-the-middle. - - lib/charms/operator_libs_linux/v0/apt.py +rules: + - name: man-in-the-middle + terms: + - man-in-the-middle + severity: warning diff --git a/docs/changelog.md b/docs/changelog.md index abe600d..bd6a102 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,7 +1,9 @@ # Changelog -All notable changes to this project will be documented in this file. +## 2014-05-31 -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +### Added -Each revision is versioned by the date of the revision. \ No newline at end of file +* Created the initial version of the Chrony client charm, a subordinate + charm that configures Chrony as an NTP client on the target machine. +* Added the initial set of charm documentation. From 36f3195f8a76a9cf8a90535ebfd51aa4eb7fcc54 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Fri, 25 Jul 2025 14:14:32 +0800 Subject: [PATCH 08/15] Revert changes in .woke.yaml --- .woke.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.woke.yaml b/.woke.yaml index 0ef21cb..1a76403 100644 --- a/.woke.yaml +++ b/.woke.yaml @@ -1,8 +1,7 @@ # Copyright 2025 Canonical Ltd. # See LICENSE file for licensing details. -rules: - - name: man-in-the-middle - terms: - - man-in-the-middle - severity: warning +ignore_files: + # Ignore apt charm library as it uses non compliant terminology: + # man-in-the-middle. + - lib/charms/operator_libs_linux/v0/apt.py From f62f15bd2e78651b52213f6c1110ccf680291456 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Fri, 25 Jul 2025 17:45:55 +0800 Subject: [PATCH 09/15] Update charmcraft.yaml to include changelog.md --- charmcraft.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/charmcraft.yaml b/charmcraft.yaml index 8ad9abf..dd2b1e4 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -65,3 +65,12 @@ parts: - libffi-dev - libssl-dev - pkg-config + changelog: + source: . + plugin: dump + stage: + - docs/changelog.md + organize: + docs/changelog.md: changelog.md + prime: + - changelog.md From eb866e1137e2a50ce5c8f948a4cdfc2cfe0fa62e Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Fri, 5 Sep 2025 00:48:32 +0800 Subject: [PATCH 10/15] Update charmcraft.yaml --- charmcraft.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/charmcraft.yaml b/charmcraft.yaml index dd2b1e4..8a2aa71 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -66,11 +66,7 @@ parts: - libssl-dev - pkg-config changelog: - source: . + source: ./docs plugin: dump stage: - - docs/changelog.md - organize: - docs/changelog.md: changelog.md - prime: - changelog.md From 0982c10a7c30b096be269e898e611045dac5171e Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Fri, 5 Sep 2025 01:23:48 +0800 Subject: [PATCH 11/15] Relocate vale wordlist --- .../styles/config/vocabularies/local/accept.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .custom_wordlist.txt => .vale/styles/config/vocabularies/local/accept.txt (100%) diff --git a/.custom_wordlist.txt b/.vale/styles/config/vocabularies/local/accept.txt similarity index 100% rename from .custom_wordlist.txt rename to .vale/styles/config/vocabularies/local/accept.txt From 940161c4a7fda9a85dd255f7e687b2a05e47c60f Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Fri, 5 Sep 2025 01:34:17 +0800 Subject: [PATCH 12/15] Update test workflows --- .github/workflows/integration_test.yaml | 14 -------------- .github/workflows/publish_charm.yaml | 1 + .github/workflows/test.yaml | 12 +++++++++++- 3 files changed, 12 insertions(+), 15 deletions(-) delete mode 100644 .github/workflows/integration_test.yaml diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml deleted file mode 100644 index fc0ee36..0000000 --- a/.github/workflows/integration_test.yaml +++ /dev/null @@ -1,14 +0,0 @@ -name: Integration tests - -on: - pull_request: - -jobs: - integration-tests: - uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main - secrets: inherit - with: - self-hosted-runner: true - self-hosted-runner-label: "edge" - juju-channel: '3/stable' - provider: 'lxd' diff --git a/.github/workflows/publish_charm.yaml b/.github/workflows/publish_charm.yaml index e14e332..d410dc5 100644 --- a/.github/workflows/publish_charm.yaml +++ b/.github/workflows/publish_charm.yaml @@ -12,3 +12,4 @@ jobs: secrets: inherit with: channel: latest/edge + integration-test-workflow-file: test.yaml diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 21f4024..dd086c3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -10,4 +10,14 @@ jobs: with: self-hosted-runner: true self-hosted-runner-label: "edge" - vale-style-check: true + docs-checks: + uses: canonical/operator-workflows/.github/workflows/docs.yaml@main + secrets: inherit + integration-tests: + uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main + secrets: inherit + with: + self-hosted-runner: true + self-hosted-runner-label: "edge" + juju-channel: '3/stable' + provider: 'lxd' From 58d0eb51382c2ebb21200a527eac48f6c0f3578b Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Fri, 5 Sep 2025 01:38:20 +0800 Subject: [PATCH 13/15] Add .vale.ini file --- .vale.ini | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .vale.ini diff --git a/.vale.ini b/.vale.ini new file mode 100644 index 0000000..9e8948d --- /dev/null +++ b/.vale.ini @@ -0,0 +1,11 @@ +; Copyright 2025 Canonical Ltd. +; See LICENSE file for licensing details. + +StylesPath = .vale/styles + +Packages = https://github.com/canonical/platform-engineering-vale-package/releases/download/latest/pfe-vale.zip + +Vocab = PFE, local + +[*] +BasedOnStyles = PFE From d2f83dfab5e7836215cc4db99f01861ebb291831 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Fri, 5 Sep 2025 01:43:37 +0800 Subject: [PATCH 14/15] Add .vale.ini file --- .vale/styles/config/vocabularies/local/accept.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.vale/styles/config/vocabularies/local/accept.txt b/.vale/styles/config/vocabularies/local/accept.txt index 68a0460..0576b06 100644 --- a/.vale/styles/config/vocabularies/local/accept.txt +++ b/.vale/styles/config/vocabularies/local/accept.txt @@ -1,4 +1,6 @@ chrony Chrony +chronyc +chronyd PPMs reachability \ No newline at end of file From 74330ee3ede7843ae3697429091b5774c1b6f447 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Fri, 5 Sep 2025 01:45:22 +0800 Subject: [PATCH 15/15] Update linkcheck --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c5d16e1..11b67e2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,7 +32,7 @@ When contributing, you must abide by the This project uses [semantic versioning](https://semver.org/). Please ensure that any new feature, fix, or significant change is documented by -adding an entry to the [CHANGELOG.md](link-to-changelog) file. +adding an entry to the [CHANGELOG.md](./docs/changelog.md) file. To learn more about changelog best practices, visit [Keep a Changelog](https://keepachangelog.com/). @@ -43,7 +43,7 @@ notify in advance the people involved to avoid confusion; also, reference the issue or bug number when you submit the changes. - [Fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-forks) - our [GitHub repository](link-to-github-repo) + our [GitHub repository](https://github.com/canonical/chrony-client-operator) and add the changes to your fork, properly structuring your commits, providing detailed commit messages and signing your commits. - Make sure the updated project builds and runs without warnings or errors;