diff --git a/.github/pull_request_template.yaml b/.github/pull_request_template.yaml index 3633e3d..bea9d6a 100644 --- a/.github/pull_request_template.yaml +++ b/.github/pull_request_template.yaml @@ -8,15 +8,15 @@ Applicable spec: -### Juju Events Changes +### Juju events changes -### Module Changes +### Module changes -### Library Changes +### Library changes diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml deleted file mode 100644 index 19a5606..0000000 --- a/.github/workflows/integration_test.yaml +++ /dev/null @@ -1,18 +0,0 @@ -name: Integration tests - -on: - pull_request: - -jobs: - integration-tests: - uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main - secrets: inherit - with: - load-test-enabled: false - load-test-run-args: "-e LOAD_TEST_HOST=localhost" - trivy-fs-enabled: true - trivy-image-config: "trivy.yaml" - self-hosted-runner: true - self-hosted-runner-label: "edge" - juju-channel: '3/stable' - channel: '1.32-strict/stable' diff --git a/.github/workflows/publish_charm.yaml b/.github/workflows/publish_charm.yaml index e14e332..d410dc5 100644 --- a/.github/workflows/publish_charm.yaml +++ b/.github/workflows/publish_charm.yaml @@ -12,3 +12,4 @@ jobs: secrets: inherit with: channel: latest/edge + integration-test-workflow-file: test.yaml diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bd1426c..dd086c3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -10,3 +10,14 @@ jobs: with: self-hosted-runner: true self-hosted-runner-label: "edge" + docs-checks: + uses: canonical/operator-workflows/.github/workflows/docs.yaml@main + secrets: inherit + integration-tests: + uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main + secrets: inherit + with: + self-hosted-runner: true + self-hosted-runner-label: "edge" + juju-channel: '3/stable' + provider: 'lxd' diff --git a/.licenserc.yaml b/.licenserc.yaml index 2994aef..3315013 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -6,8 +6,8 @@ header: Copyright [year] [owner] See LICENSE file for licensing details. pattern: | - Copyright \d{4} Canonical Ltd. - See LICENSE file for licensing details. + Copyright \d{4} Canonical Ltd. + See LICENSE file for licensing details. paths: - '**' paths-ignore: @@ -23,4 +23,7 @@ header: - 'trivy.yaml' - 'pyproject.toml' - 'zap_rules.tsv' + - 'lib/**' + - 'src/grafana_dashboards/**' + - 'src/prometheus_alert_rules/**' comment: on-failure diff --git a/.vale.ini b/.vale.ini new file mode 100644 index 0000000..9e8948d --- /dev/null +++ b/.vale.ini @@ -0,0 +1,11 @@ +; Copyright 2025 Canonical Ltd. +; See LICENSE file for licensing details. + +StylesPath = .vale/styles + +Packages = https://github.com/canonical/platform-engineering-vale-package/releases/download/latest/pfe-vale.zip + +Vocab = PFE, local + +[*] +BasedOnStyles = PFE diff --git a/.vale/styles/config/vocabularies/local/accept.txt b/.vale/styles/config/vocabularies/local/accept.txt new file mode 100644 index 0000000..0576b06 --- /dev/null +++ b/.vale/styles/config/vocabularies/local/accept.txt @@ -0,0 +1,6 @@ +chrony +Chrony +chronyc +chronyd +PPMs +reachability \ No newline at end of file diff --git a/.woke.yaml b/.woke.yaml new file mode 100644 index 0000000..1a76403 --- /dev/null +++ b/.woke.yaml @@ -0,0 +1,7 @@ +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +ignore_files: + # Ignore apt charm library as it uses non compliant terminology: + # man-in-the-middle. + - lib/charms/operator_libs_linux/v0/apt.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c5d16e1..11b67e2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,7 +32,7 @@ When contributing, you must abide by the This project uses [semantic versioning](https://semver.org/). Please ensure that any new feature, fix, or significant change is documented by -adding an entry to the [CHANGELOG.md](link-to-changelog) file. +adding an entry to the [CHANGELOG.md](./docs/changelog.md) file. To learn more about changelog best practices, visit [Keep a Changelog](https://keepachangelog.com/). @@ -43,7 +43,7 @@ notify in advance the people involved to avoid confusion; also, reference the issue or bug number when you submit the changes. - [Fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-forks) - our [GitHub repository](link-to-github-repo) + our [GitHub repository](https://github.com/canonical/chrony-client-operator) and add the changes to your fork, properly structuring your commits, providing detailed commit messages and signing your commits. - Make sure the updated project builds and runs without warnings or errors; diff --git a/README.md b/README.md index 206e585..7c7f28c 100644 --- a/README.md +++ b/README.md @@ -1,41 +1,28 @@ - - -# platform-engineering-charm-template - +A [Juju](https://juju.is/) [subordinate charm](https://documentation.ubuntu.com/juju/latest/reference/charm/#subordinate) +that deploys and configures Chrony as an NTP client. -Describe your charm in 1-2 sentences. Include the software that the charm deploys (if applicable), and the substrate (VM/K8s). +Like any Juju charm, it supports one-line deployment, configuration, +integration, scaling, and more. Specifically, the Chrony client charm +can: -Like any Juju charm, this charm supports one-line deployment, configuration, integration, scaling, and more. For Charmed {Name}, this includes: -* list or summary of app-specific features +* Install Chrony as an NTP client replacing the system default NTP client +* Configure time sources +* Integrate with COS for time tracking status observability -For information about how to deploy, integrate, and manage this charm, see the Official [platform-engineering-charm-template Documentation](external link). +For information on deploying, integrating, and managing this charm, see +the official [Chrony Client documentation](https://charmhub.io/chrony-client). ## Get started - +To begin, refer to the [Getting Started](./docs/tutorial.md) tutorial +for step-by-step instructions. ### Basic operations -## (Optional) Integrations - - ## Learn more - -* [Read more]() -* [Developer documentation]() -* [Official webpage]() -* [Troubleshooting]() +* [Read more](https://charmhub.io/chrony-client) +* [Developer documentation](https://chrony-project.org/documentation.html) +* [Official webpage](https://chrony-project.org/) +* [Troubleshooting](https://matrix.to/#/#charmhub-charmdev:ubuntu.com) ## Project and community -* [Issues]() -* [Contributing]() -* [Matrix]() -* [Launchpad]() - -## (Optional) Licensing and trademark +* [Issues](https://github.com/canonical/chrony-client-operator/issues) +* [Contributing](./CONTRIBUTING.md) +* [Matrix](https://matrix.to/#/#charmhub-charmdev:ubuntu.com) diff --git a/charmcraft.yaml b/charmcraft.yaml index 9ad69f6..8a2aa71 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -1,65 +1,72 @@ # Copyright 2025 Canonical Ltd. # See LICENSE file for licensing details. -# This file configures Charmcraft. -# See https://canonical-charmcraft.readthedocs-hosted.com/stable/howto/manage-charmcraft/ -# for guidance. -type: charm -name: is-charms-template -title: Charm Template -summary: A very short one-line summary of the charm. +name: chrony-client +title: Chrony Client Charm +summary: Chrony NTP client charm. links: - documentation: https://discourse.charmhub.io - issues: https://github.com/canonical/is-charms-template-repo/issues - source: https://github.com/canonical/is-charms-template-repo + issues: https://github.com/canonical/chrony-client-operator/issues + source: https://github.com/canonical/chrony-client-operator contact: https://launchpad.net/~canonical-is-devops description: | - A single sentence that says what the charm is, concisely and memorably. - - A paragraph of one to three short sentences, that describe what the charm does. - - A third paragraph that explains what need the charm meets. - - Finally, a paragraph that describes whom the charm is useful for. - -bases: - - build-on: - - name: ubuntu - channel: "22.04" - run-on: - - name: ubuntu - channel: "22.04" - -# The containers and resources metadata apply to Kubernetes charms only. -# Remove them if not required. + A [Juju](https://juju.is/) [charm](https://juju.is/docs/olm/charmed-operators) + for deploying and managing the [Chrony](https://chrony-project.org) NTP server + in your systems. + + This charm simplifies the configuration and maintenance of `chrony` as a + NTP client. -# Your workload’s containers. -containers: - httpbin: - resource: httpbin-image - -# This field populates the Resources tab on Charmhub. -resources: - # An OCI image resource for each container listed above. - # You may remove this if your charm will run without a workload sidecar container. - httpbin-image: - type: oci-image - description: OCI image for httpbin - # The upstream-source field is ignored by Juju. It is included here as a reference - # so the integration testing suite knows which image to deploy during testing. This field - # is also used by the 'canonical/charming-actions' Github action for automated releasing. - upstream-source: kennethreitz/httpbin +type: charm +subordinate: true -# See https://canonical-charmcraft.readthedocs-hosted.com/stable/reference/files/config-yaml-file/ -# for guidance. config: options: - # An example config option to customise the log level of the workload - log-level: - description: | - Configures the log level of gunicorn. - - Acceptable values are: "info", "debug", "warning", "error" and "critical" - default: "info" + sources: + description: >- + Time sources for the Chrony server are specified as a comma-separated list of URLs. + Currently, the Chrony charm supports using only another NTP server as the time source. + The format for NTP source URLs is as follows: `ntp://host[:port][?options]`, where the options are the same as + Chrony pool configuration settings (https://chrony-project.org/doc/4.5/chrony.conf.html). + For NTP sources that support NTS, the URL format is: `nts://host[:nts-ke-port][?options]`. + The options for NTS URLs are the same as those for NTP sources. + Here are some examples of supported sources: + `ntp://ntp.example.com` + `ntp://ntp.example.com:1234` + `ntp://ntp.example.com?iburst=true&maxsources=2` + `nts://ntp.example.com` type: string + default: |- + ntp://ntp.ubuntu.com?iburst=true&maxsources=4, + ntp://0.ubuntu.pool.ntp.org?iburst=true&maxsources=1, + ntp://1.ubuntu.pool.ntp.org?iburst=true&maxsources=1, + ntp://2.ubuntu.pool.ntp.org?iburst=true&maxsources=2 + +requires: + juju-info: + interface: juju-info + scope: container + +provides: + cos-agent: + interface: cos_agent + +platforms: + ubuntu@24.04:amd64: + ubuntu@22.04:amd64: +parts: + charm: + build-snaps: + - rustup + override-build: | + rustup default stable + craftctl default + build-packages: + - libffi-dev + - libssl-dev + - pkg-config + changelog: + source: ./docs + plugin: dump + stage: + - changelog.md diff --git a/docs-template/changelog.md b/docs-template/changelog.md deleted file mode 100644 index abe600d..0000000 --- a/docs-template/changelog.md +++ /dev/null @@ -1,7 +0,0 @@ -# Changelog - -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). - -Each revision is versioned by the date of the revision. \ No newline at end of file diff --git a/docs-template/explanation/charm-architecture.md b/docs-template/explanation/charm-architecture.md deleted file mode 100644 index 7aeec6b..0000000 --- a/docs-template/explanation/charm-architecture.md +++ /dev/null @@ -1,122 +0,0 @@ -# Charm architecture - -Add overview material here: - -1. What kind of application is it? What kind of software does it use? -2. Describe Pebble services. - - - -## High-level overview of deployment - -The following diagram shows a typical deployment of the charm. - - - - -## Charm architecture - -The following diagram shows the architecture of the charm: - - - -### Containers - -Configuration files for the containers can be found in the respective directories that define the rock. - - - -## OCI images - -We use [Rockcraft](https://canonical-rockcraft.readthedocs-hosted.com/en/latest/) to build OCI Images for . -The images are defined in [ rock](link to rock). -They are published to [Charmhub](https://charmhub.io/), the official repository of charms. - -> See more: [How to publish your charm on Charmhub](https://canonical-charmcraft.readthedocs-hosted.com/en/stable/howto/manage-charms/#publish-a-charm-on-charmhub) - -## Metrics - - - -## Juju events - -For this charm, the following Juju events are observed: - - - -> See more in the Juju docs: [Hook](https://documentation.ubuntu.com/juju/latest/user/reference/hook/) - -## Charm code overview - -The `src/charm.py` is the default entry point for a charm and has the Python class which inherits -from CharmBase. CharmBase is the base class from which all charms are formed, defined -by [Ops](https://ops.readthedocs.io/en/latest/index.html) (Python framework for developing charms). - -> See more in the Juju docs: [Charm](https://documentation.ubuntu.com/juju/latest/user/reference/charm/) - -The `__init__` method guarantees that the charm observes all events relevant to its operation and handles them. - -Take, for example, when a configuration is changed by using the CLI. - -1. User runs the configuration command: - -```bash -juju config -``` - -2. A `config-changed` event is emitted. -3. In the `__init__` method is defined how to handle this event like this: - -```python -self.framework.observe(self.on.config_changed, self._on_config_changed) -``` - -4. The method `_on_config_changed`, for its turn, will take the necessary actions such as waiting for all the relations to be ready and then configuring the containers. diff --git a/docs-template/explanation/security.md b/docs-template/explanation/security.md deleted file mode 100644 index af25027..0000000 --- a/docs-template/explanation/security.md +++ /dev/null @@ -1,26 +0,0 @@ -# Security overview - - \ No newline at end of file diff --git a/docs-template/how-to/back-up-restore.md b/docs-template/how-to/back-up-restore.md deleted file mode 100644 index 81fb56f..0000000 --- a/docs-template/how-to/back-up-restore.md +++ /dev/null @@ -1,9 +0,0 @@ -# How to back up and restore - - \ No newline at end of file diff --git a/docs-template/how-to/contribute.md b/docs-template/how-to/contribute.md deleted file mode 100644 index c842f82..0000000 --- a/docs-template/how-to/contribute.md +++ /dev/null @@ -1,15 +0,0 @@ -# How to contribute - - - -Our documentation is hosted on the [Charmhub forum](link-to-charmhub-overview-page) to enable collaboration. -Please use the "Help us improve this documentation" links on each documentation page to either -directly change something you see that's wrong, ask a question, or make a suggestion about a potential -change via the comments section. - -Our documentation is also available alongside the [source code on GitHub](link-to-github-repo). -You may open a pull request with your documentation changes, or you can -[file a bug](link-to-issues) to provide constructive feedback or suggestions. - -See [CONTRIBUTING.md](link-to-contributing-md) -for information on contributing to the source code. \ No newline at end of file diff --git a/docs-template/how-to/integrate-with-cos.md b/docs-template/how-to/integrate-with-cos.md deleted file mode 100644 index 7f4f9de..0000000 --- a/docs-template/how-to/integrate-with-cos.md +++ /dev/null @@ -1,7 +0,0 @@ -# Integrate with COS - - \ No newline at end of file diff --git a/docs-template/how-to/upgrade.md b/docs-template/how-to/upgrade.md deleted file mode 100644 index 7e15ae1..0000000 --- a/docs-template/how-to/upgrade.md +++ /dev/null @@ -1,10 +0,0 @@ -# How to upgrade - - \ No newline at end of file diff --git a/docs-template/index.md b/docs-template/index.md deleted file mode 100644 index b9db875..0000000 --- a/docs-template/index.md +++ /dev/null @@ -1,57 +0,0 @@ -# Operator - - - -A [Juju](https://juju.is/) [charm](https://documentation.ubuntu.com/juju/3.6/reference/charm/) deploying and managing on -Kubernetes. - - - -Like any Juju charm, this charm supports one-line deployment, configuration, integration, scaling, and more. -For , this includes: -* list or summary of app-specific features - -The charm allows for deployment on many different Kubernetes platforms, from [MicroK8s](https://microk8s.io/) to -[Charmed Kubernetes](https://ubuntu.com/kubernetes) to public cloud Kubernetes offerings. - - - -This charm will make operating simple and straightforward for DevOps or SRE teams through Juju's clean interface. - -## In this documentation - -| | | -|--|--| -| [Tutorials](link to tutorial)
Get started - a hands-on introduction to using the charm for new users
| [How-to guides](link to how-to guide)
Step-by-step guides covering key operations and common tasks | -| [Reference](link to reference)
Technical information - specifications, APIs, architecture | [Explanation](link to explanation)
Concepts - discussion and clarification of key topics | - -## Contributing to this documentation - -Documentation is an important part of this project, and we take the same open-source approach -to the documentation as the code. As such, we welcome community contributions, suggestions, and -constructive feedback on our documentation. -See [How to contribute](link to contribute page) for more information. - - -If there's a particular area of documentation that you'd like to see that's missing, please -[file a bug](link to issues page). - -## Project and community - -The Operator is a member of the Ubuntu family. It's an open-source project that warmly welcomes community -projects, contributions, suggestions, fixes, and constructive feedback. - -- [Code of conduct](https://ubuntu.com/community/code-of-conduct) -- [Get support](https://discourse.charmhub.io/) -- [Join our online chat](https://matrix.to/#/#charmhub-charmdev:ubuntu.com) -- [Contribute](link to Contribute page) - -Thinking about using the Operator for your next project? -[Get in touch](https://matrix.to/#/#charmhub-charmdev:ubuntu.com)! - -# Contents - -1. [Tutorial](link to tutorial) -1. [How-to](link to how-to) -1. [Reference](link to reference) -1. [Explanation](link to explanation) diff --git a/docs-template/reference/actions.md b/docs-template/reference/actions.md deleted file mode 100644 index 5fac2eb..0000000 --- a/docs-template/reference/actions.md +++ /dev/null @@ -1,5 +0,0 @@ -# Actions - -See [Actions](link to actions page). - -> Read more about actions in the Juju docs: [Action](https://documentation.ubuntu.com/juju/latest/user/reference/action/) diff --git a/docs-template/reference/integrations.md b/docs-template/reference/integrations.md deleted file mode 100644 index 0d4d824..0000000 --- a/docs-template/reference/integrations.md +++ /dev/null @@ -1,16 +0,0 @@ -# Integrations - - - -### Integration example - -_Interface_: -_Supported charms_: - -Description here. - -Example integrate command: - -``` -juju integrate : -``` diff --git a/docs-template/reference/metrics.md b/docs-template/reference/metrics.md deleted file mode 100644 index 400de84..0000000 --- a/docs-template/reference/metrics.md +++ /dev/null @@ -1,24 +0,0 @@ -## Metrics - - \ No newline at end of file diff --git a/docs/changelog.md b/docs/changelog.md new file mode 100644 index 0000000..bd6a102 --- /dev/null +++ b/docs/changelog.md @@ -0,0 +1,9 @@ +# Changelog + +## 2014-05-31 + +### Added + +* Created the initial version of the Chrony client charm, a subordinate + charm that configures Chrony as an NTP client on the target machine. +* Added the initial set of charm documentation. diff --git a/docs/explanation/charm-architecture.md b/docs/explanation/charm-architecture.md new file mode 100644 index 0000000..0eb65e5 --- /dev/null +++ b/docs/explanation/charm-architecture.md @@ -0,0 +1,114 @@ +# Charm architecture + +At its core, the Chrony client charm installs and configures the +`chrony` and `chrony_exporter` services. + +The Chrony client charm is a subordinate charm, meaning it is designed +to be deployed adjacent to another charm and to augment the +functionality of that charm. In this case, it helps to set up Chrony as +a NTP client. + + +## High-level overview of Chrony client charm deployment + + +The following diagram shows a typical deployment of the Chrony client +charm in a VM environment. The principal charm here can be any +non-subordinate machine charm. + +```mermaid +C4Context + title Component diagram for Chrony client charm + System_Boundary(vm, "VM machine") { + Container_Boundary(principal-charm, "Principal charm"){ + Component(principal, "Principal charm") + } + Container_Boundary(chrony-client, "Chrony client charm") { + Component(chrony, "Chrony") + Component(chrony-exporter, "Chrony exporter") + } + Container_Boundary(grafana-agent-charm, "Grafana agent charm") { + Component(grafana-agent, "Grafana agent") + } + Rel(principal, chrony, "Juju info") + UpdateRelStyle(principal, chrony, $offsetX="-22", $offsetY="10") + Rel(chrony-exporter, grafana-agent, "Prometheus metrics") + UpdateRelStyle(chrony-exporter, grafana-agent, $offsetX="-50", $offsetY="20") + } + UpdateLayoutConfig($c4ShapeInRow="1", $c4BoundaryInRow="3") +``` + +## Metrics + +See [metrics](../reference/metrics.md) for more information. + +## Juju events + +Juju events allow progression of the charm through its lifecycle and +encapsulate part of the execution context of a charm. Below is a list of +observed events for the Chrony client charm and how the charm reacts +to each event. For more information about the charm’s lifecycle in +general, refer to the charm’s lifecycle [documentation](https://canonical-juju.readthedocs-hosted.com/en/latest/user/reference/hook/). + +### `install` + +The `install` event is emitted once per unit at the beginning of a +charm’s lifecycle. The charm will install `chrony` and `chrony_exporter` +during this event. See the documentation on the [`install` event](https://documentation.ubuntu.com/juju/latest/reference/hook/index.html#install). + +### `upgrade-charm` + +The `upgrade-charm` hook always runs once immediately after the charm +directory contents have been changed by an unforced charm upgrade +operation and may run after a forced upgrade, but it will not run +following a forced upgrade from an existing error state. During this +event, the Chrony client charm will upgrade the installed `chrony` or +`chrony_exporter`. See the documentation on the [`upgrade-charm` event](https://documentation.ubuntu.com/juju/latest/reference/hook/index.html#hook-upgrade-charm). + +### `config-changed` + +The `config-changed` hook always runs once immediately after the initial +install, after `leader-elected` hooks, and after the `upgrade-charm` hook. +It also runs whenever application configuration changes. During this +event, the Chrony client charm will update the configuration of `chrony` +and may restart the `chrony` service if the configuration has changed. +See the documentation on the [`config-changed` event](https://documentation.ubuntu.com/juju/latest/reference/hook/index.html#config-changed). + +### `remove` +The `remove` event is emitted only once per unit: when the Juju controller +is ready to remove the unit completely. All necessary steps for handling +removal should be handled there. During this event, the Chrony client +charm remove some installed packages and reset the chrony configuration +back to default. See the documentation on the [`remove` event](https://documentation.ubuntu.com/juju/latest/reference/hook/index.html#remove). + +## Charm code overview + +The `src/charm.py` is the default entry point for a charm and has the +`ChronyClientCharm` Python class which inherits from `CharmBase`. +`CharmBase` is the base class from which all charms are formed, defined +by [Ops](https://ops.readthedocs.io/en/latest/index.html) (Python +framework for developing charms). + +> See more in the Juju docs: [Charm](https://documentation.ubuntu.com/juju/latest/user/reference/charm/) + +The `__init__` method guarantees that the charm observes all events +relevant to its operation and handles them. + +Take, for example, when a configuration is changed by using the CLI. + +1. User runs the configuration command: + +```bash +juju config chrony-client sources=ntp://0.pool.ntp.org +``` + +2. A `config-changed` event is emitted. +3. In the `__init__` method is defined how to handle this event like this: + +```python +self.framework.observe(self.on.config_changed, self._on_config_changed) +``` + +4. The method `_on_config_changed`, for its turn, will take the + necessary actions such as waiting for all the relations to be ready + and then configuring the containers. diff --git a/docs/explanation/security.md b/docs/explanation/security.md new file mode 100644 index 0000000..5a91068 --- /dev/null +++ b/docs/explanation/security.md @@ -0,0 +1,20 @@ +# Security overview + + + +## Risks + +The Chrony client charm is a simple charm with a minimal attack surface. +The Chrony service is configured as a pure NTP client, and the Chrony +client charm does not expose any ports. The Chrony exporter only listens +on `localhost`. + +## Security patches + +`chrony` is installed from the Ubuntu archive, and security patches are +delivered through Ubuntu archive updates. Use Ubuntu Pro for faster +security responses. See [the Ubuntu Pro charm](https://charmhub.io/ubuntu-advantage). + +`chrony_exporter` is installed from the Platform Engineering team’s +PPA (`ppa:canonical-is-devops/chrony-charm`) and maintained by the +Platform Engineering team. diff --git a/docs/how-to/contribute.md b/docs/how-to/contribute.md new file mode 100644 index 0000000..6daf95b --- /dev/null +++ b/docs/how-to/contribute.md @@ -0,0 +1,15 @@ +# How to contribute + +Our documentation is hosted on the [Charmhub forum](https://charmhub.io/chrony-client) +to enable collaboration. Please use the "Help us improve this documentation" +links on each documentation page to either directly change something you +see that's wrong, ask a question, or make a suggestion about a potential +change via the comments section. + +Our documentation is also available alongside the [source code on GitHub](https://github.com/canonical/chrony-client-operator). +You may open a pull request with your documentation changes, or you can +[file a bug](https://github.com/canonical/chrony-client-operator/issues) +to provide constructive feedback or suggestions. + +See [CONTRIBUTING.md](https://github.com/canonical/chrony-client-operator/blob/main/CONTRIBUTING.md) +for information on contributing to the source code. \ No newline at end of file diff --git a/docs/how-to/integrate-with-cos.md b/docs/how-to/integrate-with-cos.md new file mode 100644 index 0000000..d9370f0 --- /dev/null +++ b/docs/how-to/integrate-with-cos.md @@ -0,0 +1,69 @@ + +# How to integrate with COS + + +## Prerequisites + +The COS integration for the Chrony client charm is provided by +the [Grafana Agent charm](https://charmhub.io/grafana-agent). Before +integrating the COS charms, you must first integrate the Chrony client +charm with the Grafana Agent charm. Because the Grafana Agent charm is +also a subordinate charm, you cannot directly integrate it to the Chrony +client charm. Instead, integrate the Grafana Agent charm with a +principal charm first, then integrate it to the Chrony client charm. + +Assuming you have already integrated the Chrony client charm with the +Ubuntu charm as the principal charm: + +```bash +juju deploy chrony-client +juju integrate chrony-client:juju-info ubuntu +``` + +Use the Grafana Agent charm’s `juju-info` interface to integrate it to the +principal charm: + +```bash +juju relate grafana-agent:juju-info ubuntu +``` + +Then integrate the Chrony client charm to the Grafana Agent charm. + + +## Integrate with the Prometheus K8s operator + + +Deploy and integrate +the [`prometheus-k8s`](https://charmhub.io/prometheus-k8s) charm with the +Grafana Agent charm through the `send-remote-write` relation using the +`prometheus_remote_write` interface. The Grafana Agent will push the +Prometheus metrics collected from the Chrony exporter to the Prometheus +charm. + +Because the Prometheus charm is a Kubernetes charm, you must establish a +cross-model relation. For more information on cross-model relations and +how to add one, see [the cross-model relation documentation](https://documentation.ubuntu.com/juju/latest/reference/relation/#cross-model). + +```bash +juju consume cos-juju-controller:cos-juju-user/cos-model.receive-remote-write +juju integrate grafana-agent:send-remote-write receive-remote-write +``` + + +## Integrate with the Grafana K8s operator + + +Deploy and integrate the [`grafana-k8s`](https://charmhub.io/grafana-k8s) +charm with the Grafana Agent charm through the +`grafana-dashboards-provider` relation using the `grafana_dashboard` +interface. The Grafana Agent will relay the dashboards provided by the +Chrony client charm to the Grafana charm. + +As with the Prometheus charm, the Grafana charm is a Kubernetes charm, +so you must establish a cross-model relation. For details, +see [the cross-model relation documentation](https://documentation.ubuntu.com/juju/latest/reference/relation/#cross-model). + +```bash +juju consume cos-juju-controller:cos-juju-user/cos-model.grafana-dashboard +juju integrate grafana-agent:grafana-dashboards-provider grafana-dashboard +``` diff --git a/docs/how-to/upgrade.md b/docs/how-to/upgrade.md new file mode 100644 index 0000000..41a965c --- /dev/null +++ b/docs/how-to/upgrade.md @@ -0,0 +1,4 @@ +# How to upgrade + +Use the [`juju refresh` command](https://documentation.ubuntu.com/juju/latest/reference/juju-cli/list-of-juju-cli-commands/refresh/) +to upgrade the Chrony client charm. No additional operations are needed. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..56bc205 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,50 @@ +# Chrony client operator + +A [Juju](https://juju.is/) [subordinate charm](https://documentation.ubuntu.com/juju/latest/reference/charm/#subordinate) +install and managing Chrony as a NTP client. + +Like any Juju charm, it supports one-line deployment, configuration, +integration, scaling, and more. Specifically, the Chrony client charm +can: + +* Install Chrony as an NTP client +* Configure time sources +* Integrate with COS for time tracking status observability + +The Chrony client charm allows for deployment on many different machine +platforms, from [MAAS](https://maas.io/) to [Charmed OpenStack](https://ubuntu.com/openstack) +to public cloud offerings. + +This charm will make managing Chrony as NTP client simple and +straightforward for DevOps or SRE teams through Juju's clean interface. + +## In this documentation + +| | | +|---------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------| +| [Tutorials](./tutorial.md)
Get started - a hands-on introduction to using the charm for new users
| [How-to guides](./how-to)
Step-by-step guides covering key operations and common tasks | +| [Reference](./reference)
Technical information - specifications, APIs, architecture | [Explanation](./explanation)
Concepts - discussion and clarification of key topics | + +## Contributing to this documentation + +Documentation is an important part of this project, and we take the same open-source approach +to the documentation as the code. As such, we welcome community contributions, suggestions, and +constructive feedback on our documentation. +See [How to contribute](./how-to/contribute.md) for more information. + + +If there's a particular area of documentation that you'd like to see that's missing, please +[file a bug](https://github.com/canonical/chrony-client-operator/issues). + +## Project and community + +The Chrony client charm is a member of the Ubuntu family. It's an open-source project that warmly welcomes community +projects, contributions, suggestions, fixes, and constructive feedback. + +- [Code of conduct](https://ubuntu.com/community/code-of-conduct) +- [Get support](https://discourse.charmhub.io/) +- [Join our online chat](https://matrix.to/#/#charmhub-charmdev:ubuntu.com) +- [Contribute](https://github.com/canonical/chrony-client-operator/blob/main/CONTRIBUTING.md) + +Thinking about using the Chrony client charm for your next project? +[Get in touch](https://matrix.to/#/#charmhub-charmdev:ubuntu.com)! diff --git a/docs-template/reference/configurations.md b/docs/reference/configurations.md similarity index 69% rename from docs-template/reference/configurations.md rename to docs/reference/configurations.md index 2577516..047baf4 100644 --- a/docs-template/reference/configurations.md +++ b/docs/reference/configurations.md @@ -1,5 +1,5 @@ # Configurations -See [Configurations](link to configurations page). +See [Configurations](https://charmhub.io/chrony-client/configurations). > Read more about configurations in the Juju docs: [Configuration](https://documentation.ubuntu.com/juju/latest/user/reference/configuration/) diff --git a/docs/reference/integrations.md b/docs/reference/integrations.md new file mode 100644 index 0000000..a1ff86f --- /dev/null +++ b/docs/reference/integrations.md @@ -0,0 +1,34 @@ +# Integrations + + + +### `cos-agent` + +_Interface_: `cos_agent` +_Supported charms_: [Grafana agent](https://charmhub.io/grafana-agent) + +Chrony client charm uses the `cos-agent` relation to provide COS-related +information, such as Prometheus metrics, Grafana dashboards, and +Prometheus alerts, to the Grafana agent. + +Example `cos-agent` integrate command: + +``` +juju integrate chrony-client:cos-agent grafana-agent +``` + +### `juju-info` + +_Interface_: `juju-info` +_Supported charms_: every charm + +The `juju-info` interface is a special and implicit relationship that +works with any charm. It is mainly useful for subordinate charms +, in this case Chrony client charm, that can add functionality to any +existing machine without the host charm being aware of it. + +Example `juju-info` integrate command: + +``` +juju integrate chrony-client:juju-info ubuntu +``` diff --git a/docs/reference/metrics.md b/docs/reference/metrics.md new file mode 100644 index 0000000..9095517 --- /dev/null +++ b/docs/reference/metrics.md @@ -0,0 +1,35 @@ +## Metrics + +- **`chrony_serverstats_authenticated_ntp_packets_total`**: The number of received NTP requests that were authenticated (with a symmetric key or NTS). +- **`chrony_serverstats_client_log_records_dropped_total`**: The number of client log records dropped by the server to limit the memory use. +- **`chrony_serverstats_command_packets_dropped_total`**: The number of command requests dropped by the server due to rate limiting. +- **`chrony_serverstats_command_packets_received_total`**: The number of command requests received by the server. +- **`chrony_serverstats_interleaved_ntp_packets_total`**: The number of received NTP requests that were detected to be in the interleaved mode. +- **`chrony_serverstats_ntp_packets_dropped_total`**: The number of NTP requests dropped by the server due to rate limiting. +- **`chrony_serverstats_ntp_packets_received_total`**: The number of valid NTP requests received by the server. +- **`chrony_serverstats_ntp_timestamp_span_seconds`**: The interval (in seconds) covered by the currently held NTP timestamps. +- **`chrony_serverstats_ntp_timestamps_held`**: The number of pairs of receive and transmit timestamps that the server is currently holding in memory for clients using the interleaved mode. +- **`chrony_serverstats_nts_ke_connections_accepted_total`**: The number of NTS-KE connections accepted by the server. +- **`chrony_serverstats_nts_ke_connections_dropped_total`**: The number of NTS-KE connections dropped by the server due to rate limiting. +- **`chrony_sources_last_sample_age_seconds`**: Chrony sources last good sample age in seconds +- **`chrony_sources_last_sample_error_margin_seconds`**: Chrony sources last sample margin of error in seconds +- **`chrony_sources_last_sample_offset_seconds`**: Chrony sources last sample offset in seconds +- **`chrony_sources_polling_interval_seconds`**: Chrony sources polling interval in seconds +- **`chrony_sources_reachability_ratio`**: Chrony sources ratio of packet reachability +- **`chrony_sources_reachability_success`**: Chrony sources last poll reachability success +- **`chrony_sources_state_info`**: Chrony sources state info +- **`chrony_sources_stratum`**: Chrony sources stratum numbers +- **`chrony_tracking_frequency_ppms`**: Rate by which the system's clock would be wrong if chronyd was not correcting it, in PPMs +- **`chrony_tracking_info`**: Chrony tracking info +- **`chrony_tracking_last_offset_seconds`**: Chrony tracking last offset in seconds +- **`chrony_tracking_reference_timestamp_seconds`**: Chrony tracking Reference timestamp +- **`chrony_tracking_remote_reference`**: Chrony tracking is connected to a remote source +- **`chrony_tracking_residual_frequency_ppms`**: For the currently selected reference source, the difference between the frequency it suggests and the one currently in use, in PPMs +- **`chrony_tracking_rms_offset_seconds`**: Chrony tracking long-term average of the offset +- **`chrony_tracking_root_delay_seconds`**: This is the total of the network path delays to the stratum-1 computer from which the computer is ultimately synchronised +- **`chrony_tracking_root_dispersion_seconds`**: Chrony tracking total of all measurement errors to the NTP root +- **`chrony_tracking_skew_ppms`**: The estimated error bound on the frequency, in PPMs +- **`chrony_tracking_stratum`**: Chrony tracking client stratum +- **`chrony_tracking_system_time_seconds`**: Chrony tracking System time +- **`chrony_tracking_update_interval_seconds`**: The time elapsed since the last measurement from the reference source was processed, in seconds +- **`chrony_up`**: Whether the chrony server is up. diff --git a/docs/tutorial.md b/docs/tutorial.md new file mode 100644 index 0000000..4e5028a --- /dev/null +++ b/docs/tutorial.md @@ -0,0 +1,117 @@ + +# Deploy the Chrony client charm + + +The Chrony client charm installs and configures Chrony as the NTP client +on target systems. It also provides observability into the target +system’s time-tracking status. This tutorial will walk you through each +step of deploying the Chrony client charm. + +## What you'll need + + +- A working station, e.g., a laptop, with AMD64 architecture. +- Juju 3 installed and bootstrapped to a LXD controller. You can + accomplish this process by using a Multipass VM as outlined in this + guide: [Set up your test environment](https://canonical-juju.readthedocs-hosted.com/en/latest/user/howto/manage-your-deployment/manage-your-deployment-environment/#set-things-up) + + +## What you'll do + +- Deploy the [Ubuntu charm](https://charmhub.io/ubuntu) +- [Deploy and Chrony client charm on the Ubuntu charm](#deploy-the-chrony-client-charm-on-the-ubuntu-charm) +- [Configure time sources] + +## Set up the environment + +To be able to work inside the Multipass VM first you need to log in with +the following command: + +```bash +multipass shell my-juju-vm +``` + +[note] +If you're working locally, you don't need to do this step. +[/note] + +To manage resources effectively and to separate this tutorial's workload +from your usual work, create a new model in the MicroK8s controller +using the following command: + +```bash +juju add-model chrony-client-tutorial +``` + + +## Deploy Ubuntu charm + + +As the Chrony client charm is +a [subordinate charm](https://documentation.ubuntu.com/juju/latest/reference/charm/#subordinate), +it requires a principal charm to be deployed on. The Chrony client charm +can be deployed with any charm. In this tutorial, we will choose +the [Ubuntu charm](https://charmhub.io/ubuntu). + +```bash +juju deploy ubuntu --base ubuntu@24.04 +``` + + +## Deploy the Chrony client charm + + +The following commands deploy the Chrony client charm and integrate it +with the Ubuntu charm to create a principal-subordinate relation. + +```bash +juju deploy chrony-client --channel latest/edge --base ubuntu@24.04 +juju integrate chrony-client:juju-info ubuntu +``` + +Run `juju status` to see the current status of the deployment. The +output should be similar to the following: + +``` +Model Controller Cloud/Region Version SLA Timestamp +chrony-client-tutorial lxd localhost/localhost 3.6.2 unsupported 13:45:19+08:00 + +App Version Status Scale Charm Channel Rev Exposed Message +chrony-client active 1 chrony-client 1 no +ubuntu 24.04 active 1 ubuntu latest/stable 26 no + +Unit Workload Agent Machine Public address Ports Message +ubuntu/0* active idle 0 10.212.71.96 + chrony-client/0* active idle 10.212.71.96 + +Machine State Address Inst id Base AZ Message +0 started 10.212.71.96 juju-2cbd10-0 ubuntu@24.04 Running +``` + +The deployment finishes when the status shows "active" for both the +Ubuntu and Chrony client charms. + +## Configure time sources + +By default, the Chrony client charm uses Ubuntu’s NTP servers +(ntp.ubuntu.com) as its time source. You can configure the charm to use +different time sources, for example, switching to the NTP Pool servers, +by running the following command: + +```bash +juju config chrony-client sources='ntp://0.pool.ntp.org?iburst=true&maxsources=4, +ntp://1.pool.ntp.org?iburst=true&maxsources=4, +ntp://2.pool.ntp.org?iburst=true&maxsources=4, +ntp://3.pool.ntp.org?iburst=true&maxsources=4' +``` + +The charm should reach the active state when the configuration is +successful. It will end up in the blocked state if the configuration +value is invalid. + +## Clean up the environment + +Congratulations! You have successfully deployed the Chrony client charm. + +You can clean up your environment by following this guide: +[Tear down your test environment](https://canonical-juju.readthedocs-hosted.com/en/3.6/user/howto/manage-your-deployment/manage-your-deployment-environment/#tear-things-down) diff --git a/lib/charms/grafana_agent/v0/cos_agent.py b/lib/charms/grafana_agent/v0/cos_agent.py new file mode 100644 index 0000000..64c2d77 --- /dev/null +++ b/lib/charms/grafana_agent/v0/cos_agent.py @@ -0,0 +1,1427 @@ +# Copyright 2023 Canonical Ltd. +# See LICENSE file for licensing details. + +r"""## Overview. + +This library can be used to manage the cos_agent relation interface: + +- `COSAgentProvider`: Use in machine charms that need to have a workload's metrics + or logs scraped, or forward rule files or dashboards to Prometheus, Loki or Grafana through + the Grafana Agent machine charm. + NOTE: Be sure to add `limit: 1` in your charm for the cos-agent relation. That is the only + way we currently have to prevent two different grafana agent apps deployed on the same VM. + +- `COSAgentConsumer`: Used in the Grafana Agent machine charm to manage the requirer side of + the `cos_agent` interface. + + +## COSAgentProvider Library Usage + +Grafana Agent machine Charmed Operator interacts with its clients using the cos_agent library. +Charms seeking to send telemetry, must do so using the `COSAgentProvider` object from +this charm library. + +Using the `COSAgentProvider` object only requires instantiating it, +typically in the `__init__` method of your charm (the one which sends telemetry). + + +```python + def __init__( + self, + charm: CharmType, + relation_name: str = DEFAULT_RELATION_NAME, + metrics_endpoints: Optional[List[_MetricsEndpointDict]] = None, + metrics_rules_dir: str = "./src/prometheus_alert_rules", + logs_rules_dir: str = "./src/loki_alert_rules", + recurse_rules_dirs: bool = False, + log_slots: Optional[List[str]] = None, + dashboard_dirs: Optional[List[str]] = None, + refresh_events: Optional[List] = None, + tracing_protocols: Optional[List[str]] = None, + scrape_configs: Optional[Union[List[Dict], Callable]] = None, + ): +``` + +### Parameters + +- `charm`: The instance of the charm that instantiates `COSAgentProvider`, typically `self`. + +- `relation_name`: If your charmed operator uses a relation name other than `cos-agent` to use + the `cos_agent` interface, this is where you have to specify that. + +- `metrics_endpoints`: In this parameter you can specify the metrics endpoints that Grafana Agent + machine Charmed Operator will scrape. The configs of this list will be merged with the configs + from `scrape_configs`. + +- `metrics_rules_dir`: The directory in which the Charmed Operator stores its metrics alert rules + files. + +- `logs_rules_dir`: The directory in which the Charmed Operator stores its logs alert rules files. + +- `recurse_rules_dirs`: This parameters set whether Grafana Agent machine Charmed Operator has to + search alert rules files recursively in the previous two directories or not. + +- `log_slots`: Snap slots to connect to for scraping logs in the form ["snap-name:slot", ...]. + +- `dashboard_dirs`: List of directories where the dashboards are stored in the Charmed Operator. + +- `refresh_events`: List of events on which to refresh relation data. + +- `tracing_protocols`: List of requested tracing protocols that the charm requires to send traces. + +- `scrape_configs`: List of standard scrape_configs dicts or a callable that returns the list in + case the configs need to be generated dynamically. The contents of this list will be merged + with the configs from `metrics_endpoints`. + + +### Example 1 - Minimal instrumentation: + +In order to use this object the following should be in the `charm.py` file. + +```python +from charms.grafana_agent.v0.cos_agent import COSAgentProvider +... +class TelemetryProviderCharm(CharmBase): + def __init__(self, *args): + ... + self._grafana_agent = COSAgentProvider(self) +``` + +### Example 2 - Full instrumentation: + +In order to use this object the following should be in the `charm.py` file. + +```python +from charms.grafana_agent.v0.cos_agent import COSAgentProvider +... +class TelemetryProviderCharm(CharmBase): + def __init__(self, *args): + ... + self._grafana_agent = COSAgentProvider( + self, + relation_name="custom-cos-agent", + metrics_endpoints=[ + # specify "path" and "port" to scrape from localhost + {"path": "/metrics", "port": 9000}, + {"path": "/metrics", "port": 9001}, + {"path": "/metrics", "port": 9002}, + ], + metrics_rules_dir="./src/alert_rules/prometheus", + logs_rules_dir="./src/alert_rules/loki", + recursive_rules_dir=True, + log_slots=["my-app:slot"], + dashboard_dirs=["./src/dashboards_1", "./src/dashboards_2"], + refresh_events=["update-status", "upgrade-charm"], + tracing_protocols=["otlp_http", "otlp_grpc"], + scrape_configs=[ + { + "job_name": "custom_job", + "metrics_path": "/metrics", + "authorization": {"credentials": "bearer-token"}, + "static_configs": [ + { + "targets": ["localhost:9003"]}, + "labels": {"key": "value"}, + }, + ], + }, + ] + ) +``` + +### Example 3 - Dynamic scrape configs generation: + +Pass a function to the `scrape_configs` to decouple the generation of the configs +from the instantiation of the COSAgentProvider object. + +```python +from charms.grafana_agent.v0.cos_agent import COSAgentProvider +... + +class TelemetryProviderCharm(CharmBase): + def generate_scrape_configs(self): + return [ + { + "job_name": "custom", + "metrics_path": "/metrics", + "static_configs": [{"targets": ["localhost:9000"]}], + }, + ] + + def __init__(self, *args): + ... + self._grafana_agent = COSAgentProvider( + self, + scrape_configs=self.generate_scrape_configs, + ) +``` + +## COSAgentConsumer Library Usage + +This object may be used by any Charmed Operator which gathers telemetry data by +implementing the consumer side of the `cos_agent` interface. +For instance Grafana Agent machine Charmed Operator. + +For this purpose the charm needs to instantiate the `COSAgentConsumer` object with one mandatory +and two optional arguments. + +### Parameters + +- `charm`: A reference to the parent (Grafana Agent machine) charm. + +- `relation_name`: The name of the relation that the charm uses to interact + with its clients that provides telemetry data using the `COSAgentProvider` object. + + If provided, this relation name must match a provided relation in metadata.yaml with the + `cos_agent` interface. + The default value of this argument is "cos-agent". + +- `refresh_events`: List of events on which to refresh relation data. + + +### Example 1 - Minimal instrumentation: + +In order to use this object the following should be in the `charm.py` file. + +```python +from charms.grafana_agent.v0.cos_agent import COSAgentConsumer +... +class GrafanaAgentMachineCharm(GrafanaAgentCharm) + def __init__(self, *args): + ... + self._cos = COSAgentRequirer(self) +``` + + +### Example 2 - Full instrumentation: + +In order to use this object the following should be in the `charm.py` file. + +```python +from charms.grafana_agent.v0.cos_agent import COSAgentConsumer +... +class GrafanaAgentMachineCharm(GrafanaAgentCharm) + def __init__(self, *args): + ... + self._cos = COSAgentRequirer( + self, + relation_name="cos-agent-consumer", + refresh_events=["update-status", "upgrade-charm"], + ) +``` +""" + +import enum +import json +import logging +import socket +from collections import namedtuple +from itertools import chain +from pathlib import Path +from typing import ( + TYPE_CHECKING, + Any, + Callable, + ClassVar, + Dict, + List, + Literal, + MutableMapping, + Optional, + Set, + Tuple, + Union, +) + +import pydantic +from cosl import DashboardPath40UID, JujuTopology, LZMABase64 +from cosl.rules import AlertRules, generic_alert_groups +from ops.charm import RelationChangedEvent +from ops.framework import EventBase, EventSource, Object, ObjectEvents +from ops.model import ModelError, Relation +from ops.testing import CharmType + +if TYPE_CHECKING: + try: + from typing import TypedDict + + class _MetricsEndpointDict(TypedDict): + path: str + port: int + + except ModuleNotFoundError: + _MetricsEndpointDict = Dict # pyright: ignore + +LIBID = "dc15fa84cef84ce58155fb84f6c6213a" +LIBAPI = 0 +LIBPATCH = 20 + +PYDEPS = ["cosl >= 0.0.50", "pydantic"] + +DEFAULT_RELATION_NAME = "cos-agent" +DEFAULT_PEER_RELATION_NAME = "peers" +DEFAULT_SCRAPE_CONFIG = { + "static_configs": [{"targets": ["localhost:80"]}], + "metrics_path": "/metrics", +} + +logger = logging.getLogger(__name__) +SnapEndpoint = namedtuple("SnapEndpoint", "owner, name") + +# Note: MutableMapping is imported from the typing module and not collections.abc +# because subscripting collections.abc.MutableMapping was added in python 3.9, but +# most of our charms are based on 20.04, which has python 3.8. + +_RawDatabag = MutableMapping[str, str] + + +class TransportProtocolType(str, enum.Enum): + """Receiver Type.""" + + http = "http" + grpc = "grpc" + + +receiver_protocol_to_transport_protocol = { + "zipkin": TransportProtocolType.http, + "kafka": TransportProtocolType.http, + "tempo_http": TransportProtocolType.http, + "tempo_grpc": TransportProtocolType.grpc, + "otlp_grpc": TransportProtocolType.grpc, + "otlp_http": TransportProtocolType.http, + "jaeger_thrift_http": TransportProtocolType.http, +} + +_tracing_receivers_ports = { + # OTLP receiver: see + # https://github.com/open-telemetry/opentelemetry-collector/tree/v0.96.0/receiver/otlpreceiver + "otlp_http": 4318, + "otlp_grpc": 4317, + # Jaeger receiver: see + # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/v0.96.0/receiver/jaegerreceiver + "jaeger_grpc": 14250, + "jaeger_thrift_http": 14268, + # Zipkin receiver: see + # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/v0.96.0/receiver/zipkinreceiver + "zipkin": 9411, +} + +ReceiverProtocol = Literal["otlp_grpc", "otlp_http", "zipkin", "jaeger_thrift_http", "jaeger_grpc"] + + +class TracingError(Exception): + """Base class for custom errors raised by tracing.""" + + +class NotReadyError(TracingError): + """Raised by the provider wrapper if a requirer hasn't published the required data (yet).""" + + +class ProtocolNotFoundError(TracingError): + """Raised if the user doesn't receive an endpoint for a protocol it requested.""" + + +class ProtocolNotRequestedError(ProtocolNotFoundError): + """Raised if the user attempts to obtain an endpoint for a protocol it did not request.""" + + +class DataValidationError(TracingError): + """Raised when data validation fails on IPU relation data.""" + + +class AmbiguousRelationUsageError(TracingError): + """Raised when one wrongly assumes that there can only be one relation on an endpoint.""" + + +# TODO we want to eventually use `DatabagModel` from cosl but it likely needs a move to common package first +if int(pydantic.version.VERSION.split(".")[0]) < 2: # type: ignore + + class DatabagModel(pydantic.BaseModel): # type: ignore + """Base databag model.""" + + class Config: + """Pydantic config.""" + + # ignore any extra fields in the databag + extra = "ignore" + """Ignore any extra fields in the databag.""" + allow_population_by_field_name = True + """Allow instantiating this class by field name (instead of forcing alias).""" + + _NEST_UNDER = None + + @classmethod + def load(cls, databag: MutableMapping): + """Load this model from a Juju databag.""" + if cls._NEST_UNDER: + return cls.parse_obj(json.loads(databag[cls._NEST_UNDER])) + + try: + data = { + k: json.loads(v) + for k, v in databag.items() + # Don't attempt to parse model-external values + if k in {f.alias for f in cls.__fields__.values()} + } + except json.JSONDecodeError as e: + msg = f"invalid databag contents: expecting json. {databag}" + logger.error(msg) + raise DataValidationError(msg) from e + + try: + return cls.parse_raw(json.dumps(data)) # type: ignore + except pydantic.ValidationError as e: + msg = f"failed to validate databag: {databag}" + logger.debug(msg, exc_info=True) + raise DataValidationError(msg) from e + + def dump(self, databag: Optional[MutableMapping] = None, clear: bool = True): + """Write the contents of this model to Juju databag. + + :param databag: the databag to write the data to. + :param clear: ensure the databag is cleared before writing it. + """ + if clear and databag: + databag.clear() + + if databag is None: + databag = {} + + if self._NEST_UNDER: + databag[self._NEST_UNDER] = self.json(by_alias=True) + return databag + + dct = self.dict() + for key, field in self.__fields__.items(): # type: ignore + value = dct[key] + databag[field.alias or key] = json.dumps(value) + + return databag + +else: + from pydantic import ConfigDict + + class DatabagModel(pydantic.BaseModel): + """Base databag model.""" + + model_config = ConfigDict( + # ignore any extra fields in the databag + extra="ignore", + # Allow instantiating this class by field name (instead of forcing alias). + populate_by_name=True, + # Custom config key: whether to nest the whole datastructure (as json) + # under a field or spread it out at the toplevel. + _NEST_UNDER=None, # type: ignore + arbitrary_types_allowed=True, + ) + """Pydantic config.""" + + @classmethod + def load(cls, databag: MutableMapping): + """Load this model from a Juju databag.""" + nest_under = cls.model_config.get("_NEST_UNDER") # type: ignore + if nest_under: + return cls.model_validate(json.loads(databag[nest_under])) # type: ignore + + try: + data = { + k: json.loads(v) + for k, v in databag.items() + # Don't attempt to parse model-external values + if k in {(f.alias or n) for n, f in cls.__fields__.items()} + } + except json.JSONDecodeError as e: + msg = f"invalid databag contents: expecting json. {databag}" + logger.error(msg) + raise DataValidationError(msg) from e + + try: + return cls.model_validate_json(json.dumps(data)) # type: ignore + except pydantic.ValidationError as e: + msg = f"failed to validate databag: {databag}" + logger.debug(msg, exc_info=True) + raise DataValidationError(msg) from e + + def dump(self, databag: Optional[MutableMapping] = None, clear: bool = True): + """Write the contents of this model to Juju databag. + + :param databag: the databag to write the data to. + :param clear: ensure the databag is cleared before writing it. + """ + if clear and databag: + databag.clear() + + if databag is None: + databag = {} + nest_under = self.model_config.get("_NEST_UNDER") + if nest_under: + databag[nest_under] = self.model_dump_json( # type: ignore + by_alias=True, + # skip keys whose values are default + exclude_defaults=True, + ) + return databag + + dct = self.model_dump() # type: ignore + for key, field in self.model_fields.items(): # type: ignore + value = dct[key] + if value == field.default: + continue + databag[field.alias or key] = json.dumps(value) + + return databag + + +class CosAgentProviderUnitData(DatabagModel): + """Unit databag model for `cos-agent` relation.""" + + # The following entries are the same for all units of the same principal. + # Note that the same grafana agent subordinate may be related to several apps. + # this needs to make its way to the gagent leader + metrics_alert_rules: dict + log_alert_rules: dict + dashboards: List[str] + # subordinate is no longer used but we should keep it until we bump the library to ensure + # we don't break compatibility. + subordinate: Optional[bool] = None + + # The following entries may vary across units of the same principal app. + # this data does not need to be forwarded to the gagent leader + metrics_scrape_jobs: List[Dict] + log_slots: List[str] + + # Requested tracing protocols. + tracing_protocols: Optional[List[str]] = None + + # when this whole datastructure is dumped into a databag, it will be nested under this key. + # while not strictly necessary (we could have it 'flattened out' into the databag), + # this simplifies working with the model. + KEY: ClassVar[str] = "config" + + +class CosAgentPeersUnitData(DatabagModel): + """Unit databag model for `peers` cos-agent machine charm peer relation.""" + + # We need the principal unit name and relation metadata to be able to render identifiers + # (e.g. topology) on the leader side, after all the data moves into peer data (the grafana + # agent leader can only see its own principal, because it is a subordinate charm). + unit_name: str + relation_id: str + relation_name: str + + # The only data that is forwarded to the leader is data that needs to go into the app databags + # of the outgoing o11y relations. + metrics_alert_rules: Optional[dict] + log_alert_rules: Optional[dict] + dashboards: Optional[List[str]] + + # when this whole datastructure is dumped into a databag, it will be nested under this key. + # while not strictly necessary (we could have it 'flattened out' into the databag), + # this simplifies working with the model. + KEY: ClassVar[str] = "config" + + @property + def app_name(self) -> str: + """Parse out the app name from the unit name. + + TODO: Switch to using `model_post_init` when pydantic v2 is released? + https://github.com/pydantic/pydantic/issues/1729#issuecomment-1300576214 + """ + return self.unit_name.split("/")[0] + + +if int(pydantic.version.VERSION.split(".")[0]) < 2: # type: ignore + + class ProtocolType(pydantic.BaseModel): # type: ignore + """Protocol Type.""" + + class Config: + """Pydantic config.""" + + use_enum_values = True + """Allow serializing enum values.""" + + name: str = pydantic.Field( + ..., + description="Receiver protocol name. What protocols are supported (and what they are called) " + "may differ per provider.", + examples=["otlp_grpc", "otlp_http", "tempo_http"], + ) + + type: TransportProtocolType = pydantic.Field( + ..., + description="The transport protocol used by this receiver.", + examples=["http", "grpc"], + ) + +else: + + class ProtocolType(pydantic.BaseModel): + """Protocol Type.""" + + model_config = pydantic.ConfigDict( + # Allow serializing enum values. + use_enum_values=True + ) + """Pydantic config.""" + + name: str = pydantic.Field( + ..., + description="Receiver protocol name. What protocols are supported (and what they are called) " + "may differ per provider.", + examples=["otlp_grpc", "otlp_http", "tempo_http"], + ) + + type: TransportProtocolType = pydantic.Field( + ..., + description="The transport protocol used by this receiver.", + examples=["http", "grpc"], + ) + + +class Receiver(pydantic.BaseModel): + """Specification of an active receiver.""" + + protocol: ProtocolType = pydantic.Field(..., description="Receiver protocol name and type.") + url: Optional[str] = pydantic.Field( + ..., + description="""URL at which the receiver is reachable. If there's an ingress, it would be the external URL. + Otherwise, it would be the service's fqdn or internal IP. + If the protocol type is grpc, the url will not contain a scheme.""", + examples=[ + "http://traefik_address:2331", + "https://traefik_address:2331", + "http://tempo_public_ip:2331", + "https://tempo_public_ip:2331", + "tempo_public_ip:2331", + ], + ) + + +class CosAgentRequirerUnitData(DatabagModel): # noqa: D101 + """Application databag model for the COS-agent requirer.""" + + receivers: List[Receiver] = pydantic.Field( + ..., + description="List of all receivers enabled on the tracing provider.", + ) + + +class COSAgentProvider(Object): + """Integration endpoint wrapper for the provider side of the cos_agent interface.""" + + def __init__( + self, + charm: CharmType, + relation_name: str = DEFAULT_RELATION_NAME, + metrics_endpoints: Optional[List["_MetricsEndpointDict"]] = None, + metrics_rules_dir: str = "./src/prometheus_alert_rules", + logs_rules_dir: str = "./src/loki_alert_rules", + recurse_rules_dirs: bool = False, + log_slots: Optional[List[str]] = None, + dashboard_dirs: Optional[List[str]] = None, + refresh_events: Optional[List] = None, + tracing_protocols: Optional[List[str]] = None, + *, + scrape_configs: Optional[Union[List[dict], Callable]] = None, + ): + """Create a COSAgentProvider instance. + + Args: + charm: The `CharmBase` instance that is instantiating this object. + relation_name: The name of the relation to communicate over. + metrics_endpoints: List of endpoints in the form [{"path": path, "port": port}, ...]. + This argument is a simplified form of the `scrape_configs`. + The contents of this list will be merged with the contents of `scrape_configs`. + metrics_rules_dir: Directory where the metrics rules are stored. + logs_rules_dir: Directory where the logs rules are stored. + recurse_rules_dirs: Whether to recurse into rule paths. + log_slots: Snap slots to connect to for scraping logs + in the form ["snap-name:slot", ...]. + dashboard_dirs: Directory where the dashboards are stored. + refresh_events: List of events on which to refresh relation data. + tracing_protocols: List of protocols that the charm will be using for sending traces. + scrape_configs: List of standard scrape_configs dicts or a callable + that returns the list in case the configs need to be generated dynamically. + The contents of this list will be merged with the contents of `metrics_endpoints`. + """ + super().__init__(charm, relation_name) + dashboard_dirs = dashboard_dirs or ["./src/grafana_dashboards"] + + self._charm = charm + self._relation_name = relation_name + self._metrics_endpoints = metrics_endpoints or [] + self._scrape_configs = scrape_configs or [] + self._metrics_rules = metrics_rules_dir + self._logs_rules = logs_rules_dir + self._recursive = recurse_rules_dirs + self._log_slots = log_slots or [] + self._dashboard_dirs = dashboard_dirs + self._refresh_events = refresh_events or [self._charm.on.config_changed] + self._tracing_protocols = tracing_protocols + self._is_single_endpoint = charm.meta.relations[relation_name].limit == 1 + + events = self._charm.on[relation_name] + self.framework.observe(events.relation_joined, self._on_refresh) + self.framework.observe(events.relation_changed, self._on_refresh) + for event in self._refresh_events: + self.framework.observe(event, self._on_refresh) + + def _on_refresh(self, event): + """Trigger the class to update relation data.""" + relations = self._charm.model.relations[self._relation_name] + + for relation in relations: + # Before a principal is related to the grafana-agent subordinate, we'd get + # ModelError: ERROR cannot read relation settings: unit "zk/2": settings not found + # Add a guard to make sure it doesn't happen. + if relation.data and self._charm.unit in relation.data: + # Subordinate relations can communicate only over unit data. + try: + data = CosAgentProviderUnitData( + metrics_alert_rules=self._metrics_alert_rules, + log_alert_rules=self._log_alert_rules, + dashboards=self._dashboards, + metrics_scrape_jobs=self._scrape_jobs, + log_slots=self._log_slots, + tracing_protocols=self._tracing_protocols, + ) + relation.data[self._charm.unit][data.KEY] = data.json() + except ( + pydantic.ValidationError, + json.decoder.JSONDecodeError, + ) as e: + logger.error("Invalid relation data provided: %s", e) + + @property + def _scrape_jobs(self) -> List[Dict]: + """Return a prometheus_scrape-like data structure for jobs. + + https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config + """ + if callable(self._scrape_configs): + scrape_configs = self._scrape_configs() + else: + # Create a copy of the user scrape_configs, since we will mutate this object + scrape_configs = self._scrape_configs.copy() + + # Convert "metrics_endpoints" to standard scrape_configs, and add them in + for endpoint in self._metrics_endpoints: + scrape_configs.append( + { + "metrics_path": endpoint["path"], + "static_configs": [{"targets": [f"localhost:{endpoint['port']}"]}], + } + ) + + scrape_configs = scrape_configs or [DEFAULT_SCRAPE_CONFIG] + + # Augment job name to include the app name and a unique id (index) + for idx, scrape_config in enumerate(scrape_configs): + scrape_config["job_name"] = "_".join( + [self._charm.app.name, str(idx), scrape_config.get("job_name", "default")] + ) + + return scrape_configs + + @property + def _metrics_alert_rules(self) -> Dict: + """Use (for now) the prometheus_scrape AlertRules to initialize this.""" + alert_rules = AlertRules( + query_type="promql", topology=JujuTopology.from_charm(self._charm) + ) + alert_rules.add_path(self._metrics_rules, recursive=self._recursive) + alert_rules.add( + generic_alert_groups.application_rules, + group_name_prefix=JujuTopology.from_charm(self._charm).identifier, + ) + return alert_rules.as_dict() + + @property + def _log_alert_rules(self) -> Dict: + """Use (for now) the loki_push_api AlertRules to initialize this.""" + alert_rules = AlertRules(query_type="logql", topology=JujuTopology.from_charm(self._charm)) + alert_rules.add_path(self._logs_rules, recursive=self._recursive) + return alert_rules.as_dict() + + @property + def _dashboards(self) -> List[str]: + dashboards: List[str] = [] + for d in self._dashboard_dirs: + for path in Path(d).glob("*"): + with open(path, "rt") as fp: + dashboard = json.load(fp) + rel_path = str( + path.relative_to(self._charm.charm_dir) if path.is_absolute() else path + ) + # COSAgentProvider is somewhat analogous to GrafanaDashboardProvider. We need to overwrite the uid here + # because there is currently no other way to communicate the dashboard path separately. + # https://github.com/canonical/grafana-k8s-operator/pull/363 + dashboard["uid"] = DashboardPath40UID.generate(self._charm.meta.name, rel_path) + + # Add tags + tags: List[str] = dashboard.get("tags", []) + if not any(tag.startswith("charm: ") for tag in tags): + tags.append(f"charm: {self._charm.meta.name}") + dashboard["tags"] = tags + + dashboards.append(LZMABase64.compress(json.dumps(dashboard))) + return dashboards + + @property + def relations(self) -> List[Relation]: + """The tracing relations associated with this endpoint.""" + return self._charm.model.relations[self._relation_name] + + @property + def _relation(self) -> Optional[Relation]: + """If this wraps a single endpoint, the relation bound to it, if any.""" + if not self._is_single_endpoint: + objname = type(self).__name__ + raise AmbiguousRelationUsageError( + f"This {objname} wraps a {self._relation_name} endpoint that has " + "limit != 1. We can't determine what relation, of the possibly many, you are " + f"referring to. Please pass a relation instance while calling {objname}, " + "or set limit=1 in the charm metadata." + ) + relations = self.relations + return relations[0] if relations else None + + def is_ready(self, relation: Optional[Relation] = None): + """Is this endpoint ready?""" + relation = relation or self._relation + if not relation: + logger.debug(f"no relation on {self._relation_name!r}: tracing not ready") + return False + if relation.data is None: + logger.error(f"relation data is None for {relation}") + return False + if not relation.app: + logger.error(f"{relation} event received but there is no relation.app") + return False + try: + unit = next(iter(relation.units), None) + if not unit: + return False + databag = dict(relation.data[unit]) + CosAgentRequirerUnitData.load(databag) + + except (json.JSONDecodeError, pydantic.ValidationError, DataValidationError): + logger.info(f"failed validating relation data for {relation}") + return False + return True + + def get_all_endpoints( + self, relation: Optional[Relation] = None + ) -> Optional[CosAgentRequirerUnitData]: + """Unmarshalled relation data.""" + relation = relation or self._relation + if not relation or not self.is_ready(relation): + return None + unit = next(iter(relation.units), None) + if not unit: + return None + return CosAgentRequirerUnitData.load(relation.data[unit]) # type: ignore + + def _get_tracing_endpoint( + self, relation: Optional[Relation], protocol: ReceiverProtocol + ) -> str: + """Return a tracing endpoint URL if it is available or raise a ProtocolNotFoundError.""" + unit_data = self.get_all_endpoints(relation) + if not unit_data: + # we didn't find the protocol because the remote end didn't publish any data yet + # it might also mean that grafana-agent doesn't have a relation to the tracing backend + raise ProtocolNotFoundError(protocol) + receivers: List[Receiver] = [i for i in unit_data.receivers if i.protocol.name == protocol] + if not receivers: + # we didn't find the protocol because grafana-agent didn't return us the protocol that we requested + # the caller might want to verify that we did indeed request this protocol + raise ProtocolNotFoundError(protocol) + if len(receivers) > 1: + logger.warning( + f"too many receivers with protocol={protocol!r}; using first one. Found: {receivers}" + ) + + receiver = receivers[0] + if not receiver.url: + # grafana-agent isn't connected to the tracing backend yet + raise ProtocolNotFoundError(protocol) + return receiver.url + + def get_tracing_endpoint( + self, protocol: ReceiverProtocol, relation: Optional[Relation] = None + ) -> str: + """Receiver endpoint for the given protocol. + + It could happen that this function gets called before the provider publishes the endpoints. + In such a scenario, if a non-leader unit calls this function, a permission denied exception will be raised due to + restricted access. To prevent this, this function needs to be guarded by the `is_ready` check. + + Raises: + ProtocolNotRequestedError: + If the charm unit is the leader unit and attempts to obtain an endpoint for a protocol it did not request. + ProtocolNotFoundError: + If the charm attempts to obtain an endpoint when grafana-agent isn't related to a tracing backend. + """ + try: + return self._get_tracing_endpoint(relation or self._relation, protocol=protocol) + except ProtocolNotFoundError: + # let's see if we didn't find it because we didn't request the endpoint + requested_protocols = set() + relations = [relation] if relation else self.relations + for relation in relations: + try: + databag = CosAgentProviderUnitData.load(relation.data[self._charm.unit]) + except DataValidationError: + continue + + if databag.tracing_protocols: + requested_protocols.update(databag.tracing_protocols) + + if protocol not in requested_protocols: + raise ProtocolNotRequestedError(protocol, relation) + + raise + + +class COSAgentDataChanged(EventBase): + """Event emitted by `COSAgentRequirer` when relation data changes.""" + + +class COSAgentValidationError(EventBase): + """Event emitted by `COSAgentRequirer` when there is an error in the relation data.""" + + def __init__(self, handle, message: str = ""): + super().__init__(handle) + self.message = message + + def snapshot(self) -> Dict: + """Save COSAgentValidationError source information.""" + return {"message": self.message} + + def restore(self, snapshot): + """Restore COSAgentValidationError source information.""" + self.message = snapshot["message"] + + +class COSAgentRequirerEvents(ObjectEvents): + """`COSAgentRequirer` events.""" + + data_changed = EventSource(COSAgentDataChanged) + validation_error = EventSource(COSAgentValidationError) + + +class COSAgentRequirer(Object): + """Integration endpoint wrapper for the Requirer side of the cos_agent interface.""" + + on = COSAgentRequirerEvents() # pyright: ignore + + def __init__( + self, + charm: CharmType, + *, + relation_name: str = DEFAULT_RELATION_NAME, + peer_relation_name: str = DEFAULT_PEER_RELATION_NAME, + refresh_events: Optional[List[str]] = None, + ): + """Create a COSAgentRequirer instance. + + Args: + charm: The `CharmBase` instance that is instantiating this object. + relation_name: The name of the relation to communicate over. + peer_relation_name: The name of the peer relation to communicate over. + refresh_events: List of events on which to refresh relation data. + """ + super().__init__(charm, relation_name) + self._charm = charm + self._relation_name = relation_name + self._peer_relation_name = peer_relation_name + self._refresh_events = refresh_events or [self._charm.on.config_changed] + + events = self._charm.on[relation_name] + self.framework.observe( + events.relation_joined, self._on_relation_data_changed + ) # TODO: do we need this? + self.framework.observe(events.relation_changed, self._on_relation_data_changed) + self.framework.observe(events.relation_departed, self._on_relation_departed) + + for event in self._refresh_events: + self.framework.observe(event, self.trigger_refresh) # pyright: ignore + + # Peer relation events + # A peer relation is needed as it is the only mechanism for exchanging data across + # subordinate units. + # self.framework.observe( + # self.on[self._peer_relation_name].relation_joined, self._on_peer_relation_joined + # ) + peer_events = self._charm.on[peer_relation_name] + self.framework.observe(peer_events.relation_changed, self._on_peer_relation_changed) + + @property + def peer_relation(self) -> Optional["Relation"]: + """Helper function for obtaining the peer relation object. + + Returns: peer relation object + (NOTE: would return None if called too early, e.g. during install). + """ + return self.model.get_relation(self._peer_relation_name) + + def _on_peer_relation_changed(self, _): + # Peer data is used for forwarding data from principal units to the grafana agent + # subordinate leader, for updating the app data of the outgoing o11y relations. + if self._charm.unit.is_leader(): + self.on.data_changed.emit() # pyright: ignore + + def _on_relation_departed(self, event): + """Remove provider's (principal's) alert rules and dashboards from peer data when the cos-agent relation to the principal is removed.""" + if not self.peer_relation: + event.defer() + return + # empty the departing unit's alert rules and dashboards from peer data + data = CosAgentPeersUnitData( + unit_name=event.unit.name, + relation_id=str(event.relation.id), + relation_name=event.relation.name, + metrics_alert_rules={}, + log_alert_rules={}, + dashboards=[], + ) + self.peer_relation.data[self._charm.unit][ + f"{CosAgentPeersUnitData.KEY}-{event.unit.name}" + ] = data.json() + + self.on.data_changed.emit() # pyright: ignore + + def _on_relation_data_changed(self, event: RelationChangedEvent): + # Peer data is the only means of communication between subordinate units. + if not self.peer_relation: + event.defer() + return + + cos_agent_relation = event.relation + if not event.unit or not cos_agent_relation.data.get(event.unit): + return + principal_unit = event.unit + + # Coherence check + units = cos_agent_relation.units + if len(units) > 1: + # should never happen + raise ValueError( + f"unexpected error: subordinate relation {cos_agent_relation} " + f"should have exactly one unit" + ) + + if not (raw := cos_agent_relation.data[principal_unit].get(CosAgentProviderUnitData.KEY)): + return + + if not (provider_data := self._validated_provider_data(raw)): + return + + # write enabled receivers to cos-agent relation + try: + self.update_tracing_receivers() + except ModelError: + raise + + # Copy data from the cos_agent relation to the peer relation, so the leader could + # follow up. + # Save the originating unit name, so it could be used for topology later on by the leader. + data = CosAgentPeersUnitData( # peer relation databag model + unit_name=event.unit.name, + relation_id=str(event.relation.id), + relation_name=event.relation.name, + metrics_alert_rules=provider_data.metrics_alert_rules, + log_alert_rules=provider_data.log_alert_rules, + dashboards=provider_data.dashboards, + ) + self.peer_relation.data[self._charm.unit][ + f"{CosAgentPeersUnitData.KEY}-{event.unit.name}" + ] = data.json() + + # We can't easily tell if the data that was changed is limited to only the data + # that goes into peer relation (in which case, if this is not a leader unit, we wouldn't + # need to emit `on.data_changed`), so we're emitting `on.data_changed` either way. + self.on.data_changed.emit() # pyright: ignore + + def update_tracing_receivers(self): + """Updates the list of exposed tracing receivers in all relations.""" + try: + for relation in self._charm.model.relations[self._relation_name]: + CosAgentRequirerUnitData( + receivers=[ + Receiver( + # if tracing isn't ready, we don't want the wrong receiver URLs present in the databag. + # however, because of the backwards compatibility requirements, we need to still provide + # the protocols list so that the charm with older cos_agent version doesn't error its hooks. + # before this change was added, the charm with old cos_agent version threw exceptions with + # connections to grafana-agent timing out. After the change, the charm will fail validating + # databag contents (as it expects a string in URL) but that won't cause any errors as + # tracing endpoints are the only content in the grafana-agent's side of the databag. + url=f"{self._get_tracing_receiver_url(protocol)}" + if self._charm.tracing.is_ready() # type: ignore + else None, + protocol=ProtocolType( + name=protocol, + type=receiver_protocol_to_transport_protocol[protocol], + ), + ) + for protocol in self.requested_tracing_protocols() + ], + ).dump(relation.data[self._charm.unit]) + + except ModelError as e: + # args are bytes + msg = e.args[0] + if isinstance(msg, bytes): + if msg.startswith( + b"ERROR cannot read relation application settings: permission denied" + ): + logger.error( + f"encountered error {e} while attempting to update_relation_data." + f"The relation must be gone." + ) + return + raise + + def _validated_provider_data(self, raw) -> Optional[CosAgentProviderUnitData]: + try: + return CosAgentProviderUnitData(**json.loads(raw)) + except (pydantic.ValidationError, json.decoder.JSONDecodeError) as e: + self.on.validation_error.emit(message=str(e)) # pyright: ignore + return None + + def trigger_refresh(self, _): + """Trigger a refresh of relation data.""" + # FIXME: Figure out what we should do here + self.on.data_changed.emit() # pyright: ignore + + def _get_requested_protocols(self, relation: Relation): + # Coherence check + units = relation.units + if len(units) > 1: + # should never happen + raise ValueError( + f"unexpected error: subordinate relation {relation} should have exactly one unit" + ) + + unit = next(iter(units), None) + + if not unit: + return None + + if not (raw := relation.data[unit].get(CosAgentProviderUnitData.KEY)): + return None + + if not (provider_data := self._validated_provider_data(raw)): + return None + + return provider_data.tracing_protocols + + def requested_tracing_protocols(self): + """All receiver protocols that have been requested by our related apps.""" + requested_protocols = set() + for relation in self._charm.model.relations[self._relation_name]: + try: + protocols = self._get_requested_protocols(relation) + except NotReadyError: + continue + if protocols: + requested_protocols.update(protocols) + return requested_protocols + + def _get_tracing_receiver_url(self, protocol: str): + scheme = "http" + try: + if self._charm.cert.enabled: # type: ignore + scheme = "https" + # not only Grafana Agent can implement cos_agent. If the charm doesn't have the `cert` attribute + # using our cert_handler, it won't have the `enabled` parameter. In this case, we pass and assume http. + except AttributeError: + pass + # the assumption is that a subordinate charm will always be accessible to its principal charm under its fqdn + if receiver_protocol_to_transport_protocol[protocol] == TransportProtocolType.grpc: + return f"{socket.getfqdn()}:{_tracing_receivers_ports[protocol]}" + return f"{scheme}://{socket.getfqdn()}:{_tracing_receivers_ports[protocol]}" + + @property + def _remote_data(self) -> List[Tuple[CosAgentProviderUnitData, JujuTopology]]: + """Return a list of remote data from each of the related units. + + Assumes that the relation is of type subordinate. + Relies on the fact that, for subordinate relations, the only remote unit visible to + *this unit* is the principal unit that this unit is attached to. + """ + all_data = [] + + for relation in self._charm.model.relations[self._relation_name]: + if not relation.units: + continue + unit = next(iter(relation.units)) + if not (raw := relation.data[unit].get(CosAgentProviderUnitData.KEY)): + continue + if not (provider_data := self._validated_provider_data(raw)): + continue + + topology = JujuTopology( + model=self._charm.model.name, + model_uuid=self._charm.model.uuid, + application=unit.name.name, + unit=unit.name, + ) + + all_data.append((provider_data, topology)) + + return all_data + + def _gather_peer_data(self) -> List[CosAgentPeersUnitData]: + """Collect data from the peers. + + Returns a trimmed-down list of CosAgentPeersUnitData. + """ + relation = self.peer_relation + + # Ensure that whatever context we're running this in, we take the necessary precautions: + if not relation or not relation.data or not relation.app: + return [] + + # Iterate over all peer unit data and only collect every principal once. + peer_data: List[CosAgentPeersUnitData] = [] + app_names: Set[str] = set() + + for unit in chain((self._charm.unit,), relation.units): + if not relation.data.get(unit): + continue + + for unit_name in relation.data.get(unit): # pyright: ignore + if not unit_name.startswith(CosAgentPeersUnitData.KEY): + continue + raw = relation.data[unit].get(unit_name) + if raw is None: + continue + data = CosAgentPeersUnitData(**json.loads(raw)) + # Have we already seen this principal app? + if (app_name := data.app_name) in app_names: + continue + peer_data.append(data) + app_names.add(app_name) + + return peer_data + + @property + def metrics_alerts(self) -> Dict[str, Any]: + """Fetch metrics alerts.""" + alert_rules = {} + + seen_apps: List[str] = [] + for data in self._gather_peer_data(): + if rules := data.metrics_alert_rules: + app_name = data.app_name + if app_name in seen_apps: + continue # dedup! + seen_apps.append(app_name) + # This is only used for naming the file, so be as specific as we can be + identifier = JujuTopology( + model=self._charm.model.name, + model_uuid=self._charm.model.uuid, + application=app_name, + # For the topology unit, we could use `data.principal_unit_name`, but that unit + # name may not be very stable: `_gather_peer_data` de-duplicates by app name so + # the exact unit name that turns up first in the iterator may vary from time to + # time. So using the grafana-agent unit name instead. + unit=self._charm.unit.name, + ).identifier + + alert_rules[identifier] = rules + + return alert_rules + + @property + def metrics_jobs(self) -> List[Dict]: + """Parse the relation data contents and extract the metrics jobs.""" + scrape_jobs = [] + for data, topology in self._remote_data: + for job in data.metrics_scrape_jobs: + # In #220, relation schema changed from a simplified dict to the standard + # `scrape_configs`. + # This is to ensure backwards compatibility with Providers older than v0.5. + if "path" in job and "port" in job and "job_name" in job: + job = { + "job_name": job["job_name"], + "metrics_path": job["path"], + "static_configs": [{"targets": [f"localhost:{job['port']}"]}], + # We include insecure_skip_verify because we are always scraping localhost. + # Even if we have the certs for the scrape targets, we'd rather specify the scrape + # jobs with localhost rather than the SAN DNS the cert was issued for. + "tls_config": {"insecure_skip_verify": True}, + } + + # Apply labels to the scrape jobs + for static_config in job.get("static_configs", []): + topo_as_dict = topology.as_dict(excluded_keys=["charm_name"]) + static_config["labels"] = { + # Be sure to keep labels from static_config + **static_config.get("labels", {}), + # TODO: We should add a new method in juju_topology.py + # that like `as_dict` method, returns the keys with juju_ prefix + # https://github.com/canonical/cos-lib/issues/18 + **{ + "juju_{}".format(key): value + for key, value in topo_as_dict.items() + if value + }, + } + + scrape_jobs.append(job) + + return scrape_jobs + + @property + def snap_log_endpoints(self) -> List[SnapEndpoint]: + """Fetch logging endpoints exposed by related snaps.""" + endpoints = [] + endpoints_with_topology = self.snap_log_endpoints_with_topology + for endpoint, _ in endpoints_with_topology: + endpoints.append(endpoint) + + return endpoints + + @property + def snap_log_endpoints_with_topology(self) -> List[Tuple[SnapEndpoint, JujuTopology]]: + """Fetch logging endpoints and charm topology for each related snap.""" + plugs = [] + for data, topology in self._remote_data: + targets = data.log_slots + if targets: + for target in targets: + if target in plugs: + logger.warning( + f"plug {target} already listed. " + "The same snap is being passed from multiple " + "endpoints; this should not happen." + ) + else: + plugs.append((target, topology)) + + endpoints = [] + for plug, topology in plugs: + if ":" not in plug: + logger.error(f"invalid plug definition received: {plug}. Ignoring...") + else: + endpoint = SnapEndpoint(*plug.split(":")) + endpoints.append((endpoint, topology)) + + return endpoints + + @property + def logs_alerts(self) -> Dict[str, Any]: + """Fetch log alerts.""" + alert_rules = {} + seen_apps: List[str] = [] + + for data in self._gather_peer_data(): + if rules := data.log_alert_rules: + # This is only used for naming the file, so be as specific as we can be + app_name = data.app_name + if app_name in seen_apps: + continue # dedup! + seen_apps.append(app_name) + + identifier = JujuTopology( + model=self._charm.model.name, + model_uuid=self._charm.model.uuid, + application=app_name, + # For the topology unit, we could use `data.unit_name`, but that unit + # name may not be very stable: `_gather_peer_data` de-duplicates by app name so + # the exact unit name that turns up first in the iterator may vary from time to + # time. So using the grafana-agent unit name instead. + unit=self._charm.unit.name, + ).identifier + + alert_rules[identifier] = rules + + return alert_rules + + @property + def dashboards(self) -> List[Dict[str, str]]: + """Fetch dashboards as encoded content. + + Dashboards are assumed not to vary across units of the same primary. + """ + dashboards: List[Dict[str, Any]] = [] + + seen_apps: List[str] = [] + for data in self._gather_peer_data(): + app_name = data.app_name + if app_name in seen_apps: + continue # dedup! + seen_apps.append(app_name) + + for encoded_dashboard in data.dashboards or (): + content = json.loads(LZMABase64.decompress(encoded_dashboard)) + + title = content.get("title", "no_title") + + dashboards.append( + { + "relation_id": data.relation_id, + # We have the remote charm name - use it for the identifier + "charm": f"{data.relation_name}-{app_name}", + "content": content, + "title": title, + } + ) + + return dashboards + + +def charm_tracing_config( + endpoint_requirer: COSAgentProvider, cert_path: Optional[Union[Path, str]] +) -> Tuple[Optional[str], Optional[str]]: + """Utility function to determine the charm_tracing config you will likely want. + + If no endpoint is provided: + disable charm tracing. + If https endpoint is provided but cert_path is not found on disk: + disable charm tracing. + If https endpoint is provided and cert_path is None: + raise TracingError + Else: + proceed with charm tracing (with or without tls, as appropriate) + + Usage: + >>> from lib.charms.tempo_coordinator_k8s.v0.charm_tracing import trace_charm + >>> from lib.charms.tempo_coordinator_k8s.v0.tracing import charm_tracing_config + >>> @trace_charm(tracing_endpoint="my_endpoint", cert_path="cert_path") + >>> class MyCharm(...): + >>> _cert_path = "/path/to/cert/on/charm/container.crt" + >>> def __init__(self, ...): + >>> self.tracing = TracingEndpointRequirer(...) + >>> self.my_endpoint, self.cert_path = charm_tracing_config( + ... self.tracing, self._cert_path) + """ + if not endpoint_requirer.is_ready(): + return None, None + + try: + endpoint = endpoint_requirer.get_tracing_endpoint("otlp_http") + except ProtocolNotFoundError: + logger.warn( + "Endpoint for tracing wasn't provided as tracing backend isn't ready yet. If grafana-agent isn't connected to a tracing backend, integrate it. Otherwise this issue should resolve itself in a few events." + ) + return None, None + + if not endpoint: + return None, None + + is_https = endpoint.startswith("https://") + + if is_https: + if cert_path is None: + raise TracingError("Cannot send traces to an https endpoint without a certificate.") + if not Path(cert_path).exists(): + # if endpoint is https BUT we don't have a server_cert yet: + # disable charm tracing until we do to prevent tls errors + return None, None + return endpoint, str(cert_path) + return endpoint, None diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py new file mode 100644 index 0000000..27ab939 --- /dev/null +++ b/lib/charms/operator_libs_linux/v0/apt.py @@ -0,0 +1,1779 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Abstractions for the system's Debian/Ubuntu package information and repositories. + +This module contains abstractions and wrappers around Debian/Ubuntu-style repositories and +packages, in order to easily provide an idiomatic and Pythonic mechanism for adding packages and/or +repositories to systems for use in machine charms. + +A sane default configuration is attainable through nothing more than instantiation of the +appropriate classes. `DebianPackage` objects provide information about the architecture, version, +name, and status of a package. + +`DebianPackage` will try to look up a package either from `dpkg -L` or from `apt-cache` when +provided with a string indicating the package name. If it cannot be located, `PackageNotFoundError` +will be returned, as `apt` and `dpkg` otherwise return `100` for all errors, and a meaningful error +message if the package is not known is desirable. + +To install packages with convenience methods: + +```python +try: + # Run `apt-get update` + apt.update() + apt.add_package("zsh") + apt.add_package(["vim", "htop", "wget"]) +except PackageNotFoundError: + logger.error("a specified package not found in package cache or on system") +except PackageError as e: + logger.error("could not install package. Reason: %s", e.message) +```` + +To find details of a specific package: + +```python +try: + vim = apt.DebianPackage.from_system("vim") + + # To find from the apt cache only + # apt.DebianPackage.from_apt_cache("vim") + + # To find from installed packages only + # apt.DebianPackage.from_installed_package("vim") + + vim.ensure(PackageState.Latest) + logger.info("updated vim to version: %s", vim.fullversion) +except PackageNotFoundError: + logger.error("a specified package not found in package cache or on system") +except PackageError as e: + logger.error("could not install package. Reason: %s", e.message) +``` + + +`RepositoryMapping` will return a dict-like object containing enabled system repositories +and their properties (available groups, baseuri. gpg key). This class can add, disable, or +manipulate repositories. Items can be retrieved as `DebianRepository` objects. + +In order add a new repository with explicit details for fields, a new `DebianRepository` can +be added to `RepositoryMapping` + +`RepositoryMapping` provides an abstraction around the existing repositories on the system, +and can be accessed and iterated over like any `Mapping` object, to retrieve values by key, +iterate, or perform other operations. + +Keys are constructed as `{repo_type}-{}-{release}` in order to uniquely identify a repository. + +Repositories can be added with explicit values through a Python constructor. + +Example: +```python +repositories = apt.RepositoryMapping() + +if "deb-example.com-focal" not in repositories: + repositories.add(DebianRepository(enabled=True, repotype="deb", + uri="https://example.com", release="focal", groups=["universe"])) +``` + +Alternatively, any valid `sources.list` line may be used to construct a new +`DebianRepository`. + +Example: +```python +repositories = apt.RepositoryMapping() + +if "deb-us.archive.ubuntu.com-xenial" not in repositories: + line = "deb http://us.archive.ubuntu.com/ubuntu xenial main restricted" + repo = DebianRepository.from_repo_line(line) + repositories.add(repo) +``` +""" + +from __future__ import annotations + +import fileinput +import glob +import logging +import os +import re +import subprocess +import typing +from enum import Enum +from subprocess import PIPE, CalledProcessError, check_output +from typing import Any, Iterable, Iterator, Literal, Mapping +from urllib.parse import urlparse + +logger = logging.getLogger(__name__) + +# The unique Charmhub library identifier, never change it +LIBID = "7c3dbc9c2ad44a47bd6fcb25caa270e5" + +# Increment this major API version when introducing breaking changes +LIBAPI = 0 + +# Increment this PATCH version before using `charmcraft publish-lib` or reset +# to 0 if you are raising the major API version +LIBPATCH = 17 + + +VALID_SOURCE_TYPES = ("deb", "deb-src") +OPTIONS_MATCHER = re.compile(r"\[.*?\]") +_GPG_KEY_DIR = "/etc/apt/trusted.gpg.d/" + + +class Error(Exception): + """Base class of most errors raised by this library.""" + + def __repr__(self): + """Represent the Error.""" + return f"<{type(self).__module__}.{type(self).__name__} {self.args}>" + + @property + def name(self): + """Return a string representation of the model plus class.""" + return f"<{type(self).__module__}.{type(self).__name__}>" + + @property + def message(self): + """Return the message passed as an argument.""" + return self.args[0] + + +class PackageError(Error): + """Raised when there's an error installing or removing a package.""" + + +class PackageNotFoundError(Error): + """Raised when a requested package is not known to the system.""" + + +class PackageState(Enum): + """A class to represent possible package states.""" + + Present = "present" + Absent = "absent" + Latest = "latest" + Available = "available" + + +class DebianPackage: + """Represents a traditional Debian package and its utility functions. + + `DebianPackage` wraps information and functionality around a known package, whether installed + or available. The version, epoch, name, and architecture can be easily queried and compared + against other `DebianPackage` objects to determine the latest version or to install a specific + version. + + The representation of this object as a string mimics the output from `dpkg` for familiarity. + + Installation and removal of packages is handled through the `state` property or `ensure` + method, with the following options: + + apt.PackageState.Absent + apt.PackageState.Available + apt.PackageState.Present + apt.PackageState.Latest + + When `DebianPackage` is initialized, the state of a given `DebianPackage` object will be set to + `Available`, `Present`, or `Latest`, with `Absent` implemented as a convenience for removal + (though it operates essentially the same as `Available`). + """ + + def __init__( + self, name: str, version: str, epoch: str, arch: str, state: PackageState + ) -> None: + self._name = name + self._arch = arch + self._state = state + self._version = Version(version, epoch) + + def __eq__(self, other: object) -> bool: + """Equality for comparison. + + Args: + other: a `DebianPackage` object for comparison + + Returns: + A boolean reflecting equality + """ + return isinstance(other, self.__class__) and ( + self._name, + self._version.number, + ) == (other._name, other._version.number) + + def __hash__(self): + """Return a hash of this package.""" + return hash((self._name, self._version.number)) + + def __repr__(self): + """Represent the package.""" + return f"<{self.__module__}.{type(self).__name__}: {self.__dict__}>" + + def __str__(self): + """Return a human-readable representation of the package.""" + return ( + f"<{type(self).__name__}: {self._name}-{self._version}.{self._arch} -- {self._state}>" + ) + + @staticmethod + def _apt( + command: str, + package_names: str | list[str], + optargs: list[str] | None = None, + ) -> None: + """Wrap package management commands for Debian/Ubuntu systems. + + Args: + command: the command given to `apt-get` + package_names: a package name or list of package names to operate on + optargs: an (Optional) list of additional arguments + + Raises: + PackageError if an error is encountered + """ + optargs = optargs if optargs is not None else [] + if isinstance(package_names, str): + package_names = [package_names] + _cmd = ["apt-get", "-y", *optargs, command, *package_names] + try: + env = os.environ.copy() + env["DEBIAN_FRONTEND"] = "noninteractive" + subprocess.run(_cmd, capture_output=True, check=True, text=True, env=env) + except CalledProcessError as e: + raise PackageError( + f"Could not {command} package(s) {package_names}: {e.stderr}" + ) from None + + def _add(self) -> None: + """Add a package to the system.""" + self._apt( + "install", + f"{self.name}={self.version}", + optargs=["--option=Dpkg::Options::=--force-confold"], + ) + + def _remove(self) -> None: + """Remove a package from the system. Implementation-specific.""" + return self._apt("remove", f"{self.name}={self.version}") + + @property + def name(self) -> str: + """Returns the name of the package.""" + return self._name + + def ensure(self, state: PackageState): + """Ensure that a package is in a given state. + + Args: + state: a `PackageState` to reconcile the package to + + Raises: + PackageError from the underlying call to apt + """ + if self._state is not state: + if state not in (PackageState.Present, PackageState.Latest): + self._remove() + else: + self._add() + self._state = state + + @property + def present(self) -> bool: + """Returns whether or not a package is present.""" + return self._state in (PackageState.Present, PackageState.Latest) + + @property + def latest(self) -> bool: + """Returns whether the package is the most recent version.""" + return self._state is PackageState.Latest + + @property + def state(self) -> PackageState: + """Returns the current package state.""" + return self._state + + @state.setter + def state(self, state: PackageState) -> None: + """Set the package state to a given value. + + Args: + state: a `PackageState` to reconcile the package to + + Raises: + PackageError from the underlying call to apt + """ + if state in (PackageState.Latest, PackageState.Present): + self._add() + else: + self._remove() + self._state = state + + @property + def version(self) -> Version: + """Returns the version for a package.""" + return self._version + + @property + def epoch(self) -> str: + """Returns the epoch for a package. May be unset.""" + return self._version.epoch + + @property + def arch(self) -> str: + """Returns the architecture for a package.""" + return self._arch + + @property + def fullversion(self) -> str: + """Returns the name+epoch for a package.""" + return f"{self._version}.{self._arch}" + + @staticmethod + def _get_epoch_from_version(version: str) -> tuple[str, str]: + """Pull the epoch, if any, out of a version string.""" + epoch_matcher = re.compile(r"^((?P\d+):)?(?P.*)") + result = epoch_matcher.search(version) + assert result is not None + matches = result.groupdict() + return matches.get("epoch", ""), matches["version"] + + @classmethod + def from_system( + cls, package: str, version: str | None = "", arch: str | None = "" + ) -> DebianPackage: + """Locates a package, either on the system or known to apt, and serializes the information. + + Args: + package: a string representing the package + version: an optional string if a specific version is requested + arch: an optional architecture, defaulting to `dpkg --print-architecture`. If an + architecture is not specified, this will be used for selection. + + """ + try: + return DebianPackage.from_installed_package(package, version, arch) + except PackageNotFoundError: + logger.debug( + "package '%s' is not currently installed or has the wrong architecture.", package + ) + + # Ok, try `apt-cache ...` + try: + return DebianPackage.from_apt_cache(package, version, arch) + except (PackageNotFoundError, PackageError): + # If we get here, it's not known to the systems. + # This seems unnecessary, but virtually all `apt` commands have a return code of `100`, + # and providing meaningful error messages without this is ugly. + arch_str = f".{arch}" if arch else "" + raise PackageNotFoundError( + f"Package '{package}{arch_str}' " + "could not be found on the system or in the apt cache!" + ) from None + + @classmethod + def from_installed_package( + cls, package: str, version: str | None = "", arch: str | None = "" + ) -> DebianPackage: + """Check whether the package is already installed and return an instance. + + Args: + package: a string representing the package + version: an optional string if a specific version is requested + arch: an optional architecture, defaulting to `dpkg --print-architecture`. + If an architecture is not specified, this will be used for selection. + """ + system_arch = check_output( + ["dpkg", "--print-architecture"], universal_newlines=True + ).strip() + arch = arch if arch else system_arch + + # Regexps are a really terrible way to do this. Thanks dpkg + output = "" + try: + output = check_output(["dpkg", "-l", package], stderr=PIPE, universal_newlines=True) + except CalledProcessError: + raise PackageNotFoundError(f"Package is not installed: {package}") from None + + # Pop off the output from `dpkg -l' because there's no flag to + # omit it` + lines = str(output).splitlines()[5:] + + dpkg_matcher = re.compile( + r""" + ^(?P\w+?)\s+ + (?P.*?)(?P:\w+?)?\s+ + (?P.*?)\s+ + (?P\w+?)\s+ + (?P.*) + """, + re.VERBOSE, + ) + + for line in lines: + result = dpkg_matcher.search(line) + if result is None: + logger.warning("dpkg matcher could not parse line: %s", line) + continue + matches = result.groupdict() + package_status = matches["package_status"] + + if not package_status.endswith("i"): + logger.debug( + "package '%s' in dpkg output but not installed, status: '%s'", + package, + package_status, + ) + break + + epoch, split_version = DebianPackage._get_epoch_from_version(matches["version"]) + pkg = DebianPackage( + name=matches["package_name"], + version=split_version, + epoch=epoch, + arch=matches["arch"], + state=PackageState.Present, + ) + if (pkg.arch == "all" or pkg.arch == arch) and ( + version == "" or str(pkg.version) == version + ): + return pkg + + # If we didn't find it, fail through + raise PackageNotFoundError(f"Package {package}.{arch} is not installed!") + + @classmethod + def from_apt_cache( + cls, package: str, version: str | None = "", arch: str | None = "" + ) -> DebianPackage: + """Check whether the package is already installed and return an instance. + + Args: + package: a string representing the package + version: an optional string if a specific version is requested + arch: an optional architecture, defaulting to `dpkg --print-architecture`. + If an architecture is not specified, this will be used for selection. + """ + system_arch = check_output( + ["dpkg", "--print-architecture"], universal_newlines=True + ).strip() + arch = arch if arch else system_arch + + # Regexps are a really terrible way to do this. Thanks dpkg + keys = ("Package", "Architecture", "Version") + + try: + output = check_output( + ["apt-cache", "show", package], stderr=PIPE, universal_newlines=True + ) + except CalledProcessError as e: + raise PackageError(f"Could not list packages in apt-cache: {e.stderr}") from None + + pkg_groups = output.strip().split("\n\n") + keys = ("Package", "Architecture", "Version") + + for pkg_raw in pkg_groups: + lines = str(pkg_raw).splitlines() + vals: dict[str, str] = {} + for line in lines: + if line.startswith(keys): + items = line.split(":", 1) + vals[items[0]] = items[1].strip() + else: + continue + + epoch, split_version = DebianPackage._get_epoch_from_version(vals["Version"]) + pkg = DebianPackage( + name=vals["Package"], + version=split_version, + epoch=epoch, + arch=vals["Architecture"], + state=PackageState.Available, + ) + + if (pkg.arch == "all" or pkg.arch == arch) and ( + version == "" or str(pkg.version) == version + ): + return pkg + + # If we didn't find it, fail through + raise PackageNotFoundError(f"Package {package}.{arch} is not in the apt cache!") + + +class Version: + """An abstraction around package versions. + + This seems like it should be strictly unnecessary, except that `apt_pkg` is not usable inside a + venv, and wedging version comparisons into `DebianPackage` would overcomplicate it. + + This class implements the algorithm found here: + https://www.debian.org/doc/debian-policy/ch-controlfields.html#version + """ + + def __init__(self, version: str, epoch: str): + self._version = version + self._epoch = epoch or "" + + def __repr__(self): + """Represent the package.""" + return f"<{self.__module__}.{type(self).__name__}: {self.__dict__}>" + + def __str__(self): + """Return human-readable representation of the package.""" + epoch = f"{self._epoch}:" if self._epoch else "" + return f"{epoch}{self._version}" + + @property + def epoch(self): + """Returns the epoch for a package. May be empty.""" + return self._epoch + + @property + def number(self) -> str: + """Returns the version number for a package.""" + return self._version + + def _get_parts(self, version: str) -> tuple[str, str]: + """Separate the version into component upstream and Debian pieces.""" + try: + version.rindex("-") + except ValueError: + # No hyphens means no Debian version + return version, "0" + + upstream, debian = version.rsplit("-", 1) + return upstream, debian + + def _listify(self, revision: str) -> list[str | int]: + """Split a revision string into a list. + + This list is comprised of alternating between strings and numbers, + padded on either end to always be "str, int, str, int..." and + always be of even length. This allows us to trivially implement the + comparison algorithm described. + """ + result: list[str | int] = [] + while revision: + rev_1, remains = self._get_alphas(revision) + rev_2, remains = self._get_digits(remains) + result.extend([rev_1, rev_2]) + revision = remains + return result + + def _get_alphas(self, revision: str) -> tuple[str, str]: + """Return a tuple of the first non-digit characters of a revision.""" + # get the index of the first digit + for i, char in enumerate(revision): + if char.isdigit(): + if i == 0: + return "", revision + return revision[0:i], revision[i:] + # string is entirely alphas + return revision, "" + + def _get_digits(self, revision: str) -> tuple[int, str]: + """Return a tuple of the first integer characters of a revision.""" + # If the string is empty, return (0,'') + if not revision: + return 0, "" + # get the index of the first non-digit + for i, char in enumerate(revision): + if not char.isdigit(): + if i == 0: + return 0, revision + return int(revision[0:i]), revision[i:] + # string is entirely digits + return int(revision), "" + + def _dstringcmp(self, a: str, b: str) -> Literal[-1, 0, 1]: + """Debian package version string section lexical sort algorithm. + + The lexical comparison is a comparison of ASCII values modified so + that all the letters sort earlier than all the non-letters and so that + a tilde sorts before anything, even the end of a part. + """ + if a == b: + return 0 + try: + for i, char in enumerate(a): + if char == b[i]: + continue + # "a tilde sorts before anything, even the end of a part" + # (emptyness) + if char == "~": + return -1 + if b[i] == "~": + return 1 + # "all the letters sort earlier than all the non-letters" + if char.isalpha() and not b[i].isalpha(): + return -1 + if not char.isalpha() and b[i].isalpha(): + return 1 + # otherwise lexical sort + if ord(char) > ord(b[i]): + return 1 + if ord(char) < ord(b[i]): + return -1 + except IndexError: + # a is longer than b but otherwise equal, greater unless there are tildes + # FIXME: type checker thinks "char" is possibly unbound as it's a loop variable + # but it won't be since the IndexError can only occur inside the loop + # -- I'd like to refactor away this `try ... except` anyway + if char == "~": # pyright: ignore[reportPossiblyUnboundVariable] + return -1 + return 1 + # if we get here, a is shorter than b but otherwise equal, so check for tildes... + if b[len(a)] == "~": + return 1 + return -1 + + def _compare_revision_strings(self, first: str, second: str) -> Literal[-1, 0, 1]: + """Compare two debian revision strings.""" + if first == second: + return 0 + + # listify pads results so that we will always be comparing ints to ints + # and strings to strings (at least until we fall off the end of a list) + first_list = self._listify(first) + second_list = self._listify(second) + if first_list == second_list: + return 0 + try: + for i, item in enumerate(first_list): + # explicitly raise IndexError if we've fallen off the edge of list2 + if i >= len(second_list): + raise IndexError + other = second_list[i] + # if the items are equal, next + if item == other: + continue + # numeric comparison + if isinstance(item, int): + assert isinstance(other, int) + if item > other: + return 1 + if item < other: + return -1 + else: + # string comparison + assert isinstance(other, str) + return self._dstringcmp(item, other) + except IndexError: + # rev1 is longer than rev2 but otherwise equal, hence greater + # ...except for goddamn tildes + # FIXME: bug?? we return 1 in both cases + # FIXME: first_list[len(second_list)] should be a string + # why are we indexing to 0 twice? + if first_list[len(second_list)][0][0] == "~": # type: ignore + return 1 + return 1 + # rev1 is shorter than rev2 but otherwise equal, hence lesser + # ...except for goddamn tildes + # FIXME: bug?? we return -1 in both cases + # FIXME: first_list[len(second_list)] should be a string, why are we indexing to 0 twice? + if second_list[len(first_list)][0][0] == "~": # type: ignore + return -1 + return -1 + + def _compare_version(self, other: Version) -> Literal[-1, 0, 1]: + if (self.number, self.epoch) == (other.number, other.epoch): + return 0 + + if self.epoch < other.epoch: + return -1 + if self.epoch > other.epoch: + return 1 + + # If none of these are true, follow the algorithm + upstream_version, debian_version = self._get_parts(self.number) + other_upstream_version, other_debian_version = self._get_parts(other.number) + + upstream_cmp = self._compare_revision_strings(upstream_version, other_upstream_version) + if upstream_cmp != 0: + return upstream_cmp + + debian_cmp = self._compare_revision_strings(debian_version, other_debian_version) + if debian_cmp != 0: + return debian_cmp + + return 0 + + def __lt__(self, other: Version) -> bool: + """Less than magic method impl.""" + return self._compare_version(other) < 0 + + def __eq__(self, other: object) -> bool: + """Equality magic method impl.""" + if not isinstance(other, Version): + return False + return self._compare_version(other) == 0 + + def __gt__(self, other: Version) -> bool: + """Greater than magic method impl.""" + return self._compare_version(other) > 0 + + def __le__(self, other: Version) -> bool: + """Less than or equal to magic method impl.""" + return self.__eq__(other) or self.__lt__(other) + + def __ge__(self, other: Version) -> bool: + """Greater than or equal to magic method impl.""" + return self.__gt__(other) or self.__eq__(other) + + def __ne__(self, other: object) -> bool: + """Not equal to magic method impl.""" + return not self.__eq__(other) + + +@typing.overload +def add_package( + package_names: str, + version: str | None = "", + arch: str | None = "", + update_cache: bool = False, +) -> DebianPackage: ... +@typing.overload +def add_package( + package_names: list[str], + version: str | None = "", + arch: str | None = "", + update_cache: bool = False, +) -> DebianPackage | list[DebianPackage]: ... +def add_package( + package_names: str | list[str], + version: str | None = "", + arch: str | None = "", + update_cache: bool = False, +) -> DebianPackage | list[DebianPackage]: + """Add a package or list of packages to the system. + + Args: + package_names: single package name, or list of package names + name: the name(s) of the package(s) + version: an (Optional) version as a string. Defaults to the latest known + arch: an optional architecture for the package + update_cache: whether or not to run `apt-get update` prior to operating + + Raises: + TypeError if no package name is given, or explicit version is set for multiple packages + PackageNotFoundError if the package is not in the cache. + PackageError if packages fail to install + """ + cache_refreshed = False + if update_cache: + update() + cache_refreshed = True + + package_names = [package_names] if isinstance(package_names, str) else package_names + if not package_names: + raise TypeError("Expected at least one package name to add, received zero!") + + if len(package_names) != 1 and version: + raise TypeError( + "Explicit version should not be set if more than one package is being added!" + ) + + succeeded: list[DebianPackage] = [] + retry: list[str] = [] + failed: list[str] = [] + + for p in package_names: + pkg, _ = _add(p, version, arch) + if isinstance(pkg, DebianPackage): + succeeded.append(pkg) + elif cache_refreshed: + logger.warning("failed to locate and install/update '%s'", pkg) + failed.append(p) + else: + logger.warning("failed to locate and install/update '%s', will retry later", pkg) + retry.append(p) + + if retry: + logger.info("updating the apt-cache and retrying installation of failed packages.") + update() + + for p in retry: + pkg, _ = _add(p, version, arch) + if isinstance(pkg, DebianPackage): + succeeded.append(pkg) + else: + failed.append(p) + + if failed: + raise PackageError(f"Failed to install packages: {', '.join(failed)}") + + return succeeded[0] if len(succeeded) == 1 else succeeded + + +def _add( + name: str, + version: str | None = "", + arch: str | None = "", +) -> tuple[DebianPackage, Literal[True]] | tuple[str, Literal[False]]: + """Add a package to the system. + + Args: + name: the name(s) of the package(s) + version: an (Optional) version as a string. Defaults to the latest known + arch: an optional architecture for the package + + Returns: a tuple of `DebianPackage` if found, or a :str: if it is not, and + a boolean indicating success + """ + try: + pkg = DebianPackage.from_system(name, version, arch) + pkg.ensure(state=PackageState.Present) + return pkg, True + except PackageNotFoundError: + return name, False + + +@typing.overload +def remove_package( + package_names: str, +) -> DebianPackage: ... +@typing.overload +def remove_package( + package_names: list[str], +) -> DebianPackage | list[DebianPackage]: ... +def remove_package( + package_names: str | list[str], +) -> DebianPackage | list[DebianPackage]: + """Remove package(s) from the system. + + Args: + package_names: the name of a package + + Raises: + TypeError: if no packages are provided + """ + packages: list[DebianPackage] = [] + + package_names = [package_names] if isinstance(package_names, str) else package_names + if not package_names: + raise TypeError("Expected at least one package name to add, received zero!") + + for p in package_names: + try: + pkg = DebianPackage.from_installed_package(p) + pkg.ensure(state=PackageState.Absent) + packages.append(pkg) + except PackageNotFoundError: # noqa: PERF203 + logger.info("package '%s' was requested for removal, but it was not installed.", p) + + # the list of packages will be empty when no package is removed + logger.debug("packages: '%s'", packages) + return packages[0] if len(packages) == 1 else packages + + +def update() -> None: + """Update the apt cache via `apt-get update`.""" + cmd = ["apt-get", "update", "--error-on=any"] + try: + subprocess.run(cmd, capture_output=True, check=True) + except CalledProcessError as e: + logger.error( + "%s:\nstdout:\n%s\nstderr:\n%s", + " ".join(cmd), + e.stdout.decode(), + e.stderr.decode(), + ) + raise + + +def import_key(key: str) -> str: + """Import an ASCII Armor key. + + A Radix64 format keyid is also supported for backwards + compatibility. In this case Ubuntu keyserver will be + queried for a key via HTTPS by its keyid. This method + is less preferable because https proxy servers may + require traffic decryption which is equivalent to a + man-in-the-middle attack (a proxy server impersonates + keyserver TLS certificates and has to be explicitly + trusted by the system). + + Args: + key: A GPG key in ASCII armor format, including BEGIN + and END markers or a keyid. + + Returns: + The GPG key filename written. + + Raises: + GPGKeyError if the key could not be imported + """ + key = key.strip() + if "-" in key or "\n" in key: + # Send everything not obviously a keyid to GPG to import, as + # we trust its validation better than our own. eg. handling + # comments before the key. + logger.debug("PGP key found (looks like ASCII Armor format)") + if ( + "-----BEGIN PGP PUBLIC KEY BLOCK-----" in key + and "-----END PGP PUBLIC KEY BLOCK-----" in key + ): + logger.debug("Writing provided PGP key in the binary format") + key_bytes = key.encode("utf-8") + key_name = DebianRepository._get_keyid_by_gpg_key(key_bytes) + key_gpg = DebianRepository._dearmor_gpg_key(key_bytes) + gpg_key_filename = os.path.join(_GPG_KEY_DIR, f"{key_name}.gpg") + DebianRepository._write_apt_gpg_keyfile( + key_name=gpg_key_filename, key_material=key_gpg + ) + return gpg_key_filename + else: + raise GPGKeyError("ASCII armor markers missing from GPG key") + else: + logger.warning( + "PGP key found (looks like Radix64 format). " + "SECURELY importing PGP key from keyserver; " + "full key not provided." + ) + # as of bionic add-apt-repository uses curl with an HTTPS keyserver URL + # to retrieve GPG keys. `apt-key adv` command is deprecated as is + # apt-key in general as noted in its manpage. See lp:1433761 for more + # history. Instead, /etc/apt/trusted.gpg.d is used directly to drop + # gpg + key_asc = DebianRepository._get_key_by_keyid(key) + # write the key in GPG format so that apt-key list shows it + key_gpg = DebianRepository._dearmor_gpg_key(key_asc.encode("utf-8")) + gpg_key_filename = os.path.join(_GPG_KEY_DIR, f"{key}.gpg") + DebianRepository._write_apt_gpg_keyfile(key_name=gpg_key_filename, key_material=key_gpg) + return gpg_key_filename + + +class InvalidSourceError(Error): + """Exceptions for invalid source entries.""" + + +class GPGKeyError(Error): + """Exceptions for GPG keys.""" + + +class DebianRepository: + """An abstraction to represent a repository.""" + + _deb822_stanza: _Deb822Stanza | None = None + """set by Deb822Stanza after creating a DebianRepository""" + + def __init__( + self, + enabled: bool, + repotype: str, + uri: str, + release: str, + groups: list[str], + filename: str = "", + gpg_key_filename: str = "", + options: dict[str, str] | None = None, + ): + self._enabled = enabled + self._repotype = repotype + self._uri = uri + self._release = release + self._groups = groups + self._filename = filename + self._gpg_key_filename = gpg_key_filename + self._options = options + + @property + def enabled(self): + """Return whether or not the repository is enabled.""" + return self._enabled + + @property + def repotype(self): + """Return whether it is binary or source.""" + return self._repotype + + @property + def uri(self): + """Return the URI.""" + return self._uri + + @property + def release(self): + """Return which Debian/Ubuntu releases it is valid for.""" + return self._release + + @property + def groups(self): + """Return the enabled package groups.""" + return self._groups + + @property + def filename(self): + """Returns the filename for a repository.""" + return self._filename + + @filename.setter + def filename(self, fname: str) -> None: + """Set the filename used when a repo is written back to disk. + + Args: + fname: a filename to write the repository information to. + """ + if not fname.endswith((".list", ".sources")): + raise InvalidSourceError("apt source filenames should end in .list or .sources!") + self._filename = fname + + @property + def gpg_key(self): + """Returns the path to the GPG key for this repository.""" + if not self._gpg_key_filename and self._deb822_stanza is not None: + self._gpg_key_filename = self._deb822_stanza.get_gpg_key_filename() + return self._gpg_key_filename + + @property + def options(self): + """Returns any additional repo options which are set.""" + return self._options + + def make_options_string(self, include_signed_by: bool = True) -> str: + """Generate the complete one-line-style options string for a repository. + + Combining `gpg_key`, if set (and include_signed_by is True), with any other + provided options to form the options section of a one-line-style definition. + """ + options = self._options if self._options else {} + if include_signed_by and self.gpg_key: + options["signed-by"] = self.gpg_key + if not options: + return "" + pairs = (f"{k}={v}" for k, v in sorted(options.items())) + return "[{}] ".format(" ".join(pairs)) + + @staticmethod + def prefix_from_uri(uri: str) -> str: + """Get a repo list prefix from the uri, depending on whether a path is set.""" + uridetails = urlparse(uri) + path = ( + uridetails.path.lstrip("/").replace("/", "-") if uridetails.path else uridetails.netloc + ) + return f"/etc/apt/sources.list.d/{path}" + + @staticmethod + def from_repo_line(repo_line: str, write_file: bool | None = True) -> DebianRepository: + """Instantiate a new `DebianRepository` from a `sources.list` entry line. + + Args: + repo_line: a string representing a repository entry + write_file: boolean to enable writing the new repo to disk. True by default. + Expect it to result in an add-apt-repository call under the hood, like: + add-apt-repository --no-update --sourceslist="$repo_line" + """ + repo = RepositoryMapping._parse( + repo_line, + filename="UserInput", # temp filename + ) + repo.filename = repo._make_filename() + if write_file: + _add_repository(repo) + return repo + + def _make_filename(self) -> str: + """Construct a filename from uri and release. + + For internal use when a filename isn't set. + Should match the filename written to by add-apt-repository. + """ + return "{}-{}.list".format( + DebianRepository.prefix_from_uri(self.uri), + self.release.replace("/", "-"), + ) + + def disable(self) -> None: + """Remove this repository by disabling it in the source file. + + WARNING: This method does NOT alter the `self.enabled` flag. + + WARNING: disable is currently not implemented for repositories defined + by a deb822 stanza. Raises a NotImplementedError in this case. + """ + if self._deb822_stanza is not None: + raise NotImplementedError( + "Disabling a repository defined by a deb822 format source is not implemented." + " Please raise an issue if you require this feature." + ) + searcher = f"{self.repotype} {self.make_options_string()}{self.uri} {self.release}" + with fileinput.input(self._filename, inplace=True) as lines: + for line in lines: + if re.match(rf"^{re.escape(searcher)}\s", line): + print(f"# {line}", end="") + else: + print(line, end="") + + def import_key(self, key: str) -> None: + """Import an ASCII Armor key. + + A Radix64 format keyid is also supported for backwards + compatibility. In this case Ubuntu keyserver will be + queried for a key via HTTPS by its keyid. This method + is less preferable because https proxy servers may + require traffic decryption which is equivalent to a + man-in-the-middle attack (a proxy server impersonates + keyserver TLS certificates and has to be explicitly + trusted by the system). + + Args: + key: A GPG key in ASCII armor format, + including BEGIN and END markers or a keyid. + + Raises: + GPGKeyError if the key could not be imported + """ + self._gpg_key_filename = import_key(key) + + @staticmethod + def _get_keyid_by_gpg_key(key_material: bytes) -> str: + """Get a GPG key fingerprint by GPG key material. + + Gets a GPG key fingerprint (40-digit, 160-bit) by the ASCII armor-encoded + or binary GPG key material. Can be used, for example, to generate file + names for keys passed via charm options. + """ + # Use the same gpg command for both Xenial and Bionic + cmd = ["gpg", "--with-colons", "--with-fingerprint"] + ps = subprocess.run(cmd, capture_output=True, input=key_material) + out, err = ps.stdout.decode(), ps.stderr.decode() + if "gpg: no valid OpenPGP data found." in err: + raise GPGKeyError("Invalid GPG key material provided") + # from gnupg2 docs: fpr :: Fingerprint (fingerprint is in field 10) + result = re.search(r"^fpr:{9}([0-9A-F]{40}):$", out, re.MULTILINE) + assert result is not None + return result.group(1) + + @staticmethod + def _get_key_by_keyid(keyid: str) -> str: + """Get a key via HTTPS from the Ubuntu keyserver. + + Different key ID formats are supported by SKS keyservers (the longer ones + are more secure, see "dead beef attack" and https://evil32.com/). Since + HTTPS is used, if SSLBump-like HTTPS proxies are in place, they will + impersonate keyserver.ubuntu.com and generate a certificate with + keyserver.ubuntu.com in the CN field or in SubjAltName fields of a + certificate. If such proxy behavior is expected it is necessary to add the + CA certificate chain containing the intermediate CA of the SSLBump proxy to + every machine that this code runs on via ca-certs cloud-init directive (via + cloudinit-userdata model-config) or via other means (such as through a + custom charm option). Also note that DNS resolution for the hostname in a + URL is done at a proxy server - not at the client side. + 8-digit (32 bit) key ID + https://keyserver.ubuntu.com/pks/lookup?search=0x4652B4E6 + 16-digit (64 bit) key ID + https://keyserver.ubuntu.com/pks/lookup?search=0x6E85A86E4652B4E6 + 40-digit key ID: + https://keyserver.ubuntu.com/pks/lookup?search=0x35F77D63B5CEC106C577ED856E85A86E4652B4E6 + + Args: + keyid: An 8, 16 or 40 hex digit keyid to find a key for + + Returns: + A string containing key material for the specified GPG key id + + + Raises: + subprocess.CalledProcessError + """ + # options=mr - machine-readable output (disables html wrappers) + keyserver_url = ( + "https://keyserver.ubuntu.com" "/pks/lookup?op=get&options=mr&exact=on&search=0x{}" + ) + curl_cmd = ["curl", keyserver_url.format(keyid)] + # use proxy server settings in order to retrieve the key + return check_output(curl_cmd).decode() + + @staticmethod + def _dearmor_gpg_key(key_asc: bytes) -> bytes: + """Convert a GPG key in the ASCII armor format to the binary format. + + Args: + key_asc: A GPG key in ASCII armor format. + + Returns: + A GPG key in binary format as a string + + Raises: + GPGKeyError + """ + ps = subprocess.run(["gpg", "--dearmor"], capture_output=True, input=key_asc) + out, err = ps.stdout, ps.stderr.decode() + if "gpg: no valid OpenPGP data found." in err: + raise GPGKeyError( + "Invalid GPG key material. Check your network setup" + " (MTU, routing, DNS) and/or proxy server settings" + " as well as destination keyserver status." + ) + else: + return out + + @staticmethod + def _write_apt_gpg_keyfile(key_name: str, key_material: bytes) -> None: + """Write GPG key material into a file at a provided path. + + Args: + key_name: A key name to use for a key file (could be a fingerprint) + key_material: A GPG key material (binary) + """ + with open(key_name, "wb") as keyf: + keyf.write(key_material) + + +def _repo_to_identifier(repo: DebianRepository) -> str: + """Return str identifier derived from repotype, uri, and release. + + Private method used to produce the identifiers used by RepositoryMapping. + """ + return f"{repo.repotype}-{repo.uri}-{repo.release}" + + +def _repo_to_line(repo: DebianRepository, include_signed_by: bool = True) -> str: + """Return the one-per-line format repository definition.""" + return "{prefix}{repotype} {options}{uri} {release} {groups}".format( + prefix="" if repo.enabled else "#", + repotype=repo.repotype, + options=repo.make_options_string(include_signed_by=include_signed_by), + uri=repo.uri, + release=repo.release, + groups=" ".join(repo.groups), + ) + + +class RepositoryMapping(Mapping[str, DebianRepository]): + """An representation of known repositories. + + Instantiation of `RepositoryMapping` will iterate through the + filesystem, parse out repository files in `/etc/apt/...`, and create + `DebianRepository` objects in this list. + + Typical usage: + + repositories = apt.RepositoryMapping() + repositories.add(DebianRepository( + enabled=True, repotype="deb", uri="https://example.com", release="focal", + groups=["universe"] + )) + """ + + _apt_dir = "/etc/apt" + _sources_subdir = "sources.list.d" + _default_list_name = "sources.list" + _default_sources_name = "ubuntu.sources" + _last_errors: tuple[Error, ...] = () + + def __init__(self): + self._repository_map: dict[str, DebianRepository] = {} + self.default_file = os.path.join(self._apt_dir, self._default_list_name) + # ^ public attribute for backwards compatibility only + sources_dir = os.path.join(self._apt_dir, self._sources_subdir) + default_sources = os.path.join(sources_dir, self._default_sources_name) + + # read sources.list if it exists + # ignore InvalidSourceError if ubuntu.sources also exists + # -- in this case, sources.list just contains a comment + if os.path.isfile(self.default_file): + try: + self.load(self.default_file) + except InvalidSourceError: + if not os.path.isfile(default_sources): + raise + + # read sources.list.d + for file in glob.iglob(os.path.join(sources_dir, "*.list")): + self.load(file) + for file in glob.iglob(os.path.join(sources_dir, "*.sources")): + self.load_deb822(file) + + def __contains__(self, key: Any) -> bool: + """Magic method for checking presence of repo in mapping. + + Checks against the string names used to identify repositories. + """ + return key in self._repository_map + + def __len__(self) -> int: + """Return number of repositories in map.""" + return len(self._repository_map) + + def __iter__(self) -> Iterator[DebianRepository]: # pyright: ignore[reportIncompatibleMethodOverride] + """Return iterator for RepositoryMapping. + + Iterates over the DebianRepository values rather than the string names. + FIXME: this breaks the expectations of the Mapping abstract base class + for example when it provides methods like keys and items + """ + return iter(self._repository_map.values()) + + def __getitem__(self, repository_uri: str) -> DebianRepository: + """Return a given `DebianRepository`.""" + return self._repository_map[repository_uri] + + def __setitem__(self, repository_uri: str, repository: DebianRepository) -> None: + """Add a `DebianRepository` to the cache.""" + self._repository_map[repository_uri] = repository + + def load_deb822(self, filename: str) -> None: + """Load a deb822 format repository source file into the cache. + + In contrast to one-line-style, the deb822 format specifies a repository + using a multi-line stanza. Stanzas are separated by whitespace, + and each definition consists of lines that are either key: value pairs, + or continuations of the previous value. + + Read more about the deb822 format here: + https://manpages.ubuntu.com/manpages/noble/en/man5/sources.list.5.html + For instance, ubuntu 24.04 (noble) lists its sources using deb822 style in: + /etc/apt/sources.list.d/ubuntu.sources + """ + with open(filename) as f: + repos, errors = self._parse_deb822_lines(f, filename=filename) + for repo in repos: + self._repository_map[_repo_to_identifier(repo)] = repo + if errors: + self._last_errors = tuple(errors) + logger.debug( + "the following %d error(s) were encountered when reading deb822 sources:\n%s", + len(errors), + "\n".join(str(e) for e in errors), + ) + if repos: + logger.info("parsed %d apt package repositories from %s", len(repos), filename) + else: + raise InvalidSourceError(f"all repository lines in '{filename}' were invalid!") + + @classmethod + def _parse_deb822_lines( + cls, + lines: Iterable[str], + filename: str = "", + ) -> tuple[list[DebianRepository], list[InvalidSourceError]]: + """Parse lines from a deb822 file into a list of repos and a list of errors. + + The semantics of `_parse_deb822_lines` slightly different to `_parse`: + `_parse` reads a commented out line as an entry that is not enabled + `_parse_deb822_lines` strips out comments entirely when parsing a file into stanzas, + instead only reading the 'Enabled' key to determine if an entry is enabled + """ + repos: list[DebianRepository] = [] + errors: list[InvalidSourceError] = [] + for numbered_lines in _iter_deb822_stanzas(lines): + try: + stanza = _Deb822Stanza(numbered_lines=numbered_lines, filename=filename) + except InvalidSourceError as e: # noqa: PERF203 + errors.append(e) + else: + repos.extend(stanza.repos) + return repos, errors + + def load(self, filename: str): + """Load a one-line-style format repository source file into the cache. + + Args: + filename: the path to the repository file + """ + parsed: list[int] = [] + skipped: list[int] = [] + with open(filename) as f: + for n, line in enumerate(f, start=1): # 1 indexed line numbers + try: + repo = self._parse(line, filename) + except InvalidSourceError: # noqa: PERF203 + skipped.append(n) + else: + repo_identifier = _repo_to_identifier(repo) + self._repository_map[repo_identifier] = repo + parsed.append(n) + logger.debug("parsed repo: '%s'", repo_identifier) + + if skipped: + skip_list = ", ".join(str(s) for s in skipped) + logger.debug("skipped the following lines in file '%s': %s", filename, skip_list) + + if parsed: + logger.info("parsed %d apt package repositories from %s", len(parsed), filename) + else: + raise InvalidSourceError(f"all repository lines in '{filename}' were invalid!") + + @staticmethod + def _parse(line: str, filename: str) -> DebianRepository: + """Parse a line in a sources.list file. + + Args: + line: a single line from `load` to parse + filename: the filename being read + + Raises: + InvalidSourceError if the source type is unknown + """ + enabled = True + repotype = uri = release = gpg_key = "" + options = {} + groups = [] + + line = line.strip() + if line.startswith("#"): + enabled = False + line = line[1:] + + # Check for "#" in the line and treat a part after it as a comment then strip it off. + i = line.find("#") + if i > 0: + line = line[:i] + + # Split a source into substrings to initialize a new repo. + source = line.strip() + if source: + # Match any repo options, and get a dict representation. + for v in re.findall(OPTIONS_MATCHER, source): + opts = dict(o.split("=") for o in v.strip("[]").split()) + # Extract the 'signed-by' option for the gpg_key + gpg_key = opts.pop("signed-by", "") + options = opts + + # Remove any options from the source string and split the string into chunks + source = re.sub(OPTIONS_MATCHER, "", source) + chunks = source.split() + + # Check we've got a valid list of chunks + if len(chunks) < 3 or chunks[0] not in VALID_SOURCE_TYPES: + raise InvalidSourceError("An invalid sources line was found in %s!", filename) + + repotype = chunks[0] + uri = chunks[1] + release = chunks[2] + groups = chunks[3:] + + return DebianRepository( + enabled, repotype, uri, release, groups, filename, gpg_key, options + ) + else: + raise InvalidSourceError("An invalid sources line was found in %s!", filename) + + def add( # noqa: D417 # undocumented-param: default_filename intentionally undocumented + self, repo: DebianRepository, default_filename: bool | None = False + ) -> None: + """Add a new repository to the system using add-apt-repository. + + Args: + repo: a DebianRepository object + if repo.enabled is falsey, will return without adding the repository + Raises: + CalledProcessError: if there's an error running apt-add-repository + + WARNING: Does not associate the repository with a signing key. + Use `import_key` to add a signing key globally. + + WARNING: if repo.enabled is falsey, will return without adding the repository + + WARNING: Don't forget to call `apt.update` before installing any packages! + Or call `apt.add_package` with `update_cache=True`. + + WARNING: the default_filename keyword argument is provided for backwards compatibility + only. It is not used, and was not used in the previous revision of this library. + """ + if not repo.enabled: + logger.warning( + ( + "Returning from RepositoryMapping.add(repo=%s) without adding the repo" + " because repo.enabled is %s" + ), + repo, + repo.enabled, + ) + return + _add_repository(repo) + self._repository_map[_repo_to_identifier(repo)] = repo + + def disable(self, repo: DebianRepository) -> None: + """Remove a repository by disabling it in the source file. + + WARNING: disable is currently not implemented for repositories defined + by a deb822 stanza, and will raise a NotImplementedError if called on one. + + WARNING: This method does NOT alter the `.enabled` flag on the DebianRepository. + """ + repo.disable() + self._repository_map[_repo_to_identifier(repo)] = repo + # ^ adding to map on disable seems like a bug, but this is the previous behaviour + + +def _add_repository( + repo: DebianRepository, + remove: bool = False, + update_cache: bool = False, +) -> None: + line = _repo_to_line(repo, include_signed_by=False) + key_file = repo.gpg_key + if key_file and not remove and not os.path.exists(key_file): + msg = ( + "Adding repository '%s' with add-apt-repository." + " Key file '%s' does not exist." + " Ensure it is imported correctly to use this repository." + ) + logger.warning(msg, line, key_file) + cmd = [ + "add-apt-repository", + "--yes", + "--sourceslist=" + line, + ] + if remove: + cmd.append("--remove") + if not update_cache: + cmd.append("--no-update") + logger.info("%s", cmd) + try: + subprocess.run(cmd, check=True, capture_output=True) + except CalledProcessError as e: + logger.error( + "subprocess.run(%s):\nstdout:\n%s\nstderr:\n%s", + cmd, + e.stdout.decode(), + e.stderr.decode(), + ) + raise + + +class _Deb822Stanza: + """Representation of a stanza from a deb822 source file. + + May define multiple DebianRepository objects. + """ + + def __init__(self, numbered_lines: list[tuple[int, str]], filename: str = ""): + self._filename = filename + self._numbered_lines = numbered_lines + if not numbered_lines: + self._repos = () + self._gpg_key_filename = "" + self._gpg_key_from_stanza = None + return + options, line_numbers = _deb822_stanza_to_options(numbered_lines) + repos, gpg_key_info = _deb822_options_to_repos( + options, line_numbers=line_numbers, filename=filename + ) + for repo in repos: + repo._deb822_stanza = self + self._repos = repos + self._gpg_key_filename, self._gpg_key_from_stanza = gpg_key_info + + @property + def repos(self) -> tuple[DebianRepository, ...]: + """The repositories defined by this deb822 stanza.""" + return self._repos + + def get_gpg_key_filename(self) -> str: + """Return the path to the GPG key for this stanza. + + Import the key first, if the key itself was provided in the stanza. + Return an empty string if no filename or key was provided. + """ + if self._gpg_key_filename: + return self._gpg_key_filename + if self._gpg_key_from_stanza is None: + return "" + # a gpg key was provided in the stanza + # and we haven't already imported it + self._gpg_key_filename = import_key(self._gpg_key_from_stanza) + return self._gpg_key_filename + + +class MissingRequiredKeyError(InvalidSourceError): + """Missing a required value in a source file.""" + + def __init__(self, message: str = "", *, file: str, line: int | None, key: str) -> None: + super().__init__(message, file, line, key) + self.file = file + self.line = line + self.key = key + + +class BadValueError(InvalidSourceError): + """Bad value for an entry in a source file.""" + + def __init__( + self, + message: str = "", + *, + file: str, + line: int | None, + key: str, + value: str, + ) -> None: + super().__init__(message, file, line, key, value) + self.file = file + self.line = line + self.key = key + self.value = value + + +def _iter_deb822_stanzas(lines: Iterable[str]) -> Iterator[list[tuple[int, str]]]: + """Given lines from a deb822 format file, yield a stanza of lines. + + Args: + lines: an iterable of lines from a deb822 sources file + + Yields: + lists of numbered lines (a tuple of line number and line) that make up + a deb822 stanza, with comments stripped out (but accounted for in line numbering) + """ + current_stanza: list[tuple[int, str]] = [] + for n, line in enumerate(lines, start=1): # 1 indexed line numbers + if not line.strip(): # blank lines separate stanzas + if current_stanza: + yield current_stanza + current_stanza = [] + continue + content, _delim, _comment = line.partition("#") + if content.strip(): # skip (potentially indented) comment line + current_stanza.append((n, content.rstrip())) # preserve indent + if current_stanza: + yield current_stanza + + +def _deb822_stanza_to_options( + lines: Iterable[tuple[int, str]], +) -> tuple[dict[str, str], dict[str, int]]: + """Turn numbered lines into a dict of options and a dict of line numbers. + + Args: + lines: an iterable of numbered lines (a tuple of line number and line) + + Returns: + a dictionary of option names to (potentially multiline) values, and + a dictionary of option names to starting line number + """ + parts: dict[str, list[str]] = {} + line_numbers: dict[str, int] = {} + current = None + for n, line in lines: + assert "#" not in line # comments should be stripped out + if line.startswith(" "): # continuation of previous key's value + assert current is not None + parts[current].append(line.rstrip()) # preserve indent + continue + raw_key, _, raw_value = line.partition(":") + current = raw_key.strip() + parts[current] = [raw_value.strip()] + line_numbers[current] = n + options = {k: "\n".join(v) for k, v in parts.items()} + return options, line_numbers + + +def _deb822_options_to_repos( + options: dict[str, str], line_numbers: Mapping[str, int] = {}, filename: str = "" +) -> tuple[tuple[DebianRepository, ...], tuple[str, str | None]]: + """Return a collections of DebianRepository objects defined by this deb822 stanza. + + Args: + options: a dictionary of deb822 field names to string options + line_numbers: a dictionary of field names to line numbers (for error messages) + filename: the file the options were read from (for repository object and errors) + + Returns: + a tuple of `DebianRepository`s, and + a tuple of the gpg key filename and optional in-stanza provided key itself + + Raises: + InvalidSourceError if any options are malformed or required options are missing + """ + # Enabled + enabled_field = options.pop("Enabled", "yes") + if enabled_field == "yes": + enabled = True + elif enabled_field == "no": + enabled = False + else: + raise BadValueError( + "Must be one of yes or no (default: yes).", + file=filename, + line=line_numbers.get("Enabled"), + key="Enabled", + value=enabled_field, + ) + # Signed-By + gpg_key_file = options.pop("Signed-By", "") + gpg_key_from_stanza: str | None = None + if "\n" in gpg_key_file: + # actually a literal multi-line gpg-key rather than a filename + gpg_key_from_stanza = gpg_key_file + gpg_key_file = "" + # Types + try: + repotypes = options.pop("Types").split() + uris = options.pop("URIs").split() + suites = options.pop("Suites").split() + except KeyError as e: + [key] = e.args + raise MissingRequiredKeyError( + key=key, + line=min(line_numbers.values()) if line_numbers else None, + file=filename, + ) from e + # Components + # suite can specify an exact path, in which case the components must be omitted + # and suite must end with a slash (/). + # If suite does not specify an exact path, at least one component must be present. + # https://manpages.ubuntu.com/manpages/noble/man5/sources.list.5.html + components: list[str] + if len(suites) == 1 and suites[0].endswith("/"): + if "Components" in options: + msg = ( + "Since 'Suites' (line {suites_line}) specifies" + " a path relative to 'URIs' (line {uris_line})," + " 'Components' must be omitted." + ).format( + suites_line=line_numbers.get("Suites"), + uris_line=line_numbers.get("URIs"), + ) + raise BadValueError( + msg, + file=filename, + line=line_numbers.get("Components"), + key="Components", + value=options["Components"], + ) + components = [] + else: + if "Components" not in options: + msg = ( + "Since 'Suites' (line {suites_line}) does not specify" + " a path relative to 'URIs' (line {uris_line})," + " 'Components' must be present in this stanza." + ).format( + suites_line=line_numbers.get("Suites"), + uris_line=line_numbers.get("URIs"), + ) + raise MissingRequiredKeyError( + msg, + file=filename, + line=min(line_numbers.values()) if line_numbers else None, + key="Components", + ) + components = options.pop("Components").split() + repos = tuple( + DebianRepository( + enabled=enabled, + repotype=repotype, + uri=uri, + release=suite, + groups=components, + filename=filename, + gpg_key_filename=gpg_key_file, + options=options, + ) + for repotype in repotypes + for uri in uris + for suite in suites + ) + return repos, (gpg_key_file, gpg_key_from_stanza) diff --git a/lib/charms/operator_libs_linux/v1/systemd.py b/lib/charms/operator_libs_linux/v1/systemd.py new file mode 100644 index 0000000..cdcbad6 --- /dev/null +++ b/lib/charms/operator_libs_linux/v1/systemd.py @@ -0,0 +1,288 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +"""Abstractions for stopping, starting and managing system services via systemd. + +This library assumes that your charm is running on a platform that uses systemd. E.g., +Centos 7 or later, Ubuntu Xenial (16.04) or later. + +For the most part, we transparently provide an interface to a commonly used selection of +systemd commands, with a few shortcuts baked in. For example, service_pause and +service_resume with run the mask/unmask and enable/disable invocations. + +Example usage: + +```python +from charms.operator_libs_linux.v0.systemd import service_running, service_reload + +# Start a service +if not service_running("mysql"): + success = service_start("mysql") + +# Attempt to reload a service, restarting if necessary +success = service_reload("nginx", restart_on_failure=True) +``` +""" + +__all__ = [ # Don't export `_systemctl`. (It's not the intended way of using this lib.) + "SystemdError", + "daemon_reload", + "service_disable", + "service_enable", + "service_failed", + "service_pause", + "service_reload", + "service_restart", + "service_resume", + "service_running", + "service_start", + "service_stop", +] + +import logging +import subprocess + +logger = logging.getLogger(__name__) + +# The unique Charmhub library identifier, never change it +LIBID = "045b0d179f6b4514a8bb9b48aee9ebaf" + +# Increment this major API version when introducing breaking changes +LIBAPI = 1 + +# Increment this PATCH version before using `charmcraft publish-lib` or reset +# to 0 if you are raising the major API version +LIBPATCH = 4 + + +class SystemdError(Exception): + """Custom exception for SystemD related errors.""" + + +def _systemctl(*args: str, check: bool = False) -> int: + """Control a system service using systemctl. + + Args: + *args: Arguments to pass to systemctl. + check: Check the output of the systemctl command. Default: False. + + Returns: + Returncode of systemctl command execution. + + Raises: + SystemdError: Raised if calling systemctl returns a non-zero returncode and check is True. + """ + cmd = ["systemctl", *args] + logger.debug(f"Executing command: {cmd}") + try: + proc = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + encoding="utf-8", + check=check, + ) + logger.debug( + f"Command {cmd} exit code: {proc.returncode}. systemctl output:\n{proc.stdout}" + ) + return proc.returncode + except subprocess.CalledProcessError as e: + raise SystemdError( + f"Command {cmd} failed with returncode {e.returncode}. systemctl output:\n{e.stdout}" + ) + + +def service_running(service_name: str) -> bool: + """Report whether a system service is running. + + Args: + service_name: The name of the service to check. + + Return: + True if service is running/active; False if not. + """ + # If returncode is 0, this means that is service is active. + return _systemctl("--quiet", "is-active", service_name) == 0 + + +def service_failed(service_name: str) -> bool: + """Report whether a system service has failed. + + Args: + service_name: The name of the service to check. + + Returns: + True if service is marked as failed; False if not. + """ + # If returncode is 0, this means that the service has failed. + return _systemctl("--quiet", "is-failed", service_name) == 0 + + +def service_start(*args: str) -> bool: + """Start a system service. + + Args: + *args: Arguments to pass to `systemctl start` (normally the service name). + + Returns: + On success, this function returns True for historical reasons. + + Raises: + SystemdError: Raised if `systemctl start ...` returns a non-zero returncode. + """ + return _systemctl("start", *args, check=True) == 0 + + +def service_stop(*args: str) -> bool: + """Stop a system service. + + Args: + *args: Arguments to pass to `systemctl stop` (normally the service name). + + Returns: + On success, this function returns True for historical reasons. + + Raises: + SystemdError: Raised if `systemctl stop ...` returns a non-zero returncode. + """ + return _systemctl("stop", *args, check=True) == 0 + + +def service_restart(*args: str) -> bool: + """Restart a system service. + + Args: + *args: Arguments to pass to `systemctl restart` (normally the service name). + + Returns: + On success, this function returns True for historical reasons. + + Raises: + SystemdError: Raised if `systemctl restart ...` returns a non-zero returncode. + """ + return _systemctl("restart", *args, check=True) == 0 + + +def service_enable(*args: str) -> bool: + """Enable a system service. + + Args: + *args: Arguments to pass to `systemctl enable` (normally the service name). + + Returns: + On success, this function returns True for historical reasons. + + Raises: + SystemdError: Raised if `systemctl enable ...` returns a non-zero returncode. + """ + return _systemctl("enable", *args, check=True) == 0 + + +def service_disable(*args: str) -> bool: + """Disable a system service. + + Args: + *args: Arguments to pass to `systemctl disable` (normally the service name). + + Returns: + On success, this function returns True for historical reasons. + + Raises: + SystemdError: Raised if `systemctl disable ...` returns a non-zero returncode. + """ + return _systemctl("disable", *args, check=True) == 0 + + +def service_reload(service_name: str, restart_on_failure: bool = False) -> bool: + """Reload a system service, optionally falling back to restart if reload fails. + + Args: + service_name: The name of the service to reload. + restart_on_failure: + Boolean indicating whether to fall back to a restart if the reload fails. + + Returns: + On success, this function returns True for historical reasons. + + Raises: + SystemdError: Raised if `systemctl reload|restart ...` returns a non-zero returncode. + """ + try: + return _systemctl("reload", service_name, check=True) == 0 + except SystemdError: + if restart_on_failure: + return service_restart(service_name) + else: + raise + + +def service_pause(service_name: str) -> bool: + """Pause a system service. + + Stops the service and prevents the service from starting again at boot. + + Args: + service_name: The name of the service to pause. + + Returns: + On success, this function returns True for historical reasons. + + Raises: + SystemdError: Raised if service is still running after being paused by systemctl. + """ + _systemctl("disable", "--now", service_name) + _systemctl("mask", service_name) + + if service_running(service_name): + raise SystemdError(f"Attempted to pause {service_name!r}, but it is still running.") + + return True + + +def service_resume(service_name: str) -> bool: + """Resume a system service. + + Re-enable starting the service again at boot. Start the service. + + Args: + service_name: The name of the service to resume. + + Returns: + On success, this function returns True for historical reasons. + + Raises: + SystemdError: Raised if service is not running after being resumed by systemctl. + """ + _systemctl("unmask", service_name) + _systemctl("enable", "--now", service_name) + + if not service_running(service_name): + raise SystemdError(f"Attempted to resume {service_name!r}, but it is not running.") + + return True + + +def daemon_reload() -> bool: + """Reload systemd manager configuration. + + Returns: + On success, this function returns True for historical reasons. + + Raises: + SystemdError: Raised if `systemctl daemon-reload` returns a non-zero returncode. + """ + return _systemctl("daemon-reload", check=True) == 0 diff --git a/pyproject.toml b/pyproject.toml index 0fce3f3..ae93610 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,8 +25,8 @@ select = ["E", "W", "F", "C", "N", "R", "D", "H"] # Ignore W503, E501 because using black creates errors with this # Ignore D107 Missing docstring in __init__ ignore = ["W503", "E501", "D107"] -# D100, D101, D102, D103: Ignore missing docstrings in tests -per-file-ignores = ["tests/*:D100,D101,D102,D103,D104,D205,D212,D415"] +# Ignore some rules that conflict with the test docstring format +per-file-ignores = ["tests/*:D205,D212,D415,DCO010,DCO020,DCO030"] docstring-convention = "google" [tool.isort] diff --git a/requirements.txt b/requirements.txt index aaa16b1..e271b2b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,4 @@ -ops >= 2.2.0 +jsonschema==4.24.0 +ops==2.22.0 +pydantic==2.11.7 +cosl==1.0.0 diff --git a/src/charm.py b/src/charm.py index 15e3064..8544d7b 100755 --- a/src/charm.py +++ b/src/charm.py @@ -3,29 +3,33 @@ # Copyright 2025 Canonical Ltd. # See LICENSE file for licensing details. -# Learn more at: https://documentation.ubuntu.com/juju/3.6/howto/manage-charms/#build-a-charm +# Learn more at: https://juju.is/docs/sdk -"""Charm the service. - -Refer to the following post for a quick-start guide that will help you -develop a new k8s charm using the Operator Framework: - -https://discourse.charmhub.io/t/4208 -""" +"""Chrony charm.""" import logging +import pathlib +import shutil +import textwrap import typing import ops -from ops import pebble +from charms.grafana_agent.v0.cos_agent import COSAgentProvider + +from chrony import Chrony, TimeSource -# Log messages can be retrieved using juju debug-log logger = logging.getLogger(__name__) -VALID_LOG_LEVELS = ["info", "debug", "warning", "error", "critical"] +CHRONY_CHARM_LOCK_FILE = pathlib.Path("/var/lib/chrony-charm/lock") +CHRONY_CHARM_CONFIG_HEADER = textwrap.dedent( + """\ + # This is managed by chrony-client charm (https://charmhub.io/chrony-client). + # Do not edit.\ + """ +) -class IsCharmsTemplateCharm(ops.CharmBase): +class ChronyClientCharm(ops.CharmBase): """Charm the service.""" def __init__(self, *args: typing.Any): @@ -35,85 +39,131 @@ def __init__(self, *args: typing.Any): args: Arguments passed to the CharmBase parent constructor. """ super().__init__(*args) - self.framework.observe(self.on.httpbin_pebble_ready, self._on_httpbin_pebble_ready) - self.framework.observe(self.on.config_changed, self._on_config_changed) + self.chrony = Chrony() + self._grafana_agent = COSAgentProvider( + self, + metrics_endpoints=[ + {"path": "/metrics", "port": 9123}, + ], + dashboard_dirs=["./src/grafana_dashboards"], + ) + self.framework.observe(self.on.install, self._do_install_and_config) + self.framework.observe(self.on.remove, self._on_remove) + self.framework.observe(self.on.upgrade_charm, self._do_install_and_config) + self.framework.observe(self.on.config_changed, self._do_install_and_config) + + def _do_install_and_config(self, _: ops.EventBase) -> None: + """Install required packages and open NTP port.""" + if self._try_acquire_chrony_lock(): + if not self.chrony.is_installed(): + self.unit.status = ops.MaintenanceStatus("installing chrony") + self.chrony.install() + self._configure_chrony() + else: + self._set_lock_failure_status() + + def _on_remove(self, _: ops.EventBase) -> None: + """Handle remove event.""" + if self._try_acquire_chrony_lock(): + self.chrony.uninstall() + self.chrony.restore_config() + self.chrony.restart() + self._release_chrony_lock() + + def _configure_chrony(self) -> None: + """Configure chrony.""" + try: + sources = self._get_time_sources() + except ValueError: + self.unit.status = ops.BlockedStatus("invalid sources configuration") + return + if not sources: + self.unit.status = ops.BlockedStatus("no time source configured") + return + if CHRONY_CHARM_CONFIG_HEADER not in self.chrony.read_config(): + self.chrony.backup_config() + new_config = self.chrony.new_config(sources=sources, header=CHRONY_CHARM_CONFIG_HEADER) + current_config = self.chrony.read_config() + if new_config != current_config: + logger.info("Chrony config changed, apply and restart chrony") + self.chrony.write_config(new_config) + self.chrony.restart() - def _on_httpbin_pebble_ready(self, event: ops.PebbleReadyEvent) -> None: - """Define and start a workload using the Pebble API. + self.unit.status = ops.ActiveStatus() + + def _get_time_sources(self) -> list[TimeSource]: + """Get time sources from charm configuration. - Change this example to suit your needs. You'll need to specify the right entrypoint and - environment configuration for your specific workload. + Returns: + Time source objects. + """ + urls = typing.cast(str, self.config.get("sources")) + return [ + self.chrony.parse_source_url(url.strip()) for url in urls.split(",") if url.strip() + ] - Learn more about interacting with Pebble at at - https://documentation.ubuntu.com/juju/3.6/reference/pebble/. + @staticmethod + def _write_chrony_lock_file(content: str) -> None: + """Write chrony charm lock file. Args: - event: event triggering the handler. + content: lock file content. """ - # Get a reference the container attribute on the PebbleReadyEvent - container = event.workload - # Add initial Pebble config layer using the Pebble API - container.add_layer("httpbin", self._pebble_layer, combine=True) - # Make Pebble reevaluate its plan, ensuring any services are started if enabled. - container.replan() - # Learn more about statuses in the SDK docs: - # https://documentation.ubuntu.com/juju/latest/reference/status/index.html - self.unit.status = ops.ActiveStatus() + CHRONY_CHARM_LOCK_FILE.parent.mkdir(parents=True, exist_ok=True) + CHRONY_CHARM_LOCK_FILE.write_text(content, encoding="utf-8") + + @staticmethod + def _read_chrony_lock_file() -> typing.Optional[str]: + """Read chrony charm lock file. - def _on_config_changed(self, event: ops.ConfigChangedEvent) -> None: - """Handle changed configuration. + Returns: + None if lock file doesn't exist, otherwise lock file content. + """ + if CHRONY_CHARM_LOCK_FILE.exists(): + return CHRONY_CHARM_LOCK_FILE.read_text(encoding="utf-8") + return None - Change this example to suit your needs. If you don't need to handle config, you can remove - this method. + @staticmethod + def _delete_chrony_lock_file() -> None: + """Delete chrony charm lock file.""" + shutil.rmtree(CHRONY_CHARM_LOCK_FILE.parent) - Learn more about config at - https://canonical-charmcraft.readthedocs-hosted.com/stable/reference/files/config-yaml-file/ + def _try_acquire_chrony_lock(self) -> bool: + """Try to acquire chrony lock. - Args: - event: event triggering the handler. + The chrony lock ensures that when multiple instances of the + chrony charm are installed on the same machine, only one + chrony charm application will execute. + + Returns: + True if lock acquired, False otherwise. + """ + lock_content = self.app.name + lock_file = self._read_chrony_lock_file() + if lock_file is None: + self._write_chrony_lock_file(lock_content) + return True + if lock_file.strip() == lock_content: + return True + return False + + def _release_chrony_lock(self) -> None: + """Release chrony lock. + + Remove the chrony charm lock file. """ - # Fetch the new config value - log_level = str(self.model.config["log-level"]).lower() - - # Do some validation of the configuration option - if log_level in VALID_LOG_LEVELS: - # The config is good, so update the configuration of the workload - container = self.unit.get_container("httpbin") - # Verify that we can connect to the Pebble API in the workload container - if container.can_connect(): - # Push an updated layer with the new config - container.add_layer("httpbin", self._pebble_layer, combine=True) - container.replan() - - logger.debug("Log level for gunicorn changed to '%s'", log_level) - self.unit.status = ops.ActiveStatus() - else: - # We were unable to connect to the Pebble API, so we defer this event - event.defer() - self.unit.status = ops.WaitingStatus("waiting for Pebble API") + if self._try_acquire_chrony_lock(): + self._delete_chrony_lock_file() else: - # In this case, the config option is bad, so block the charm and notify the operator. - self.unit.status = ops.BlockedStatus("invalid log level: '{log_level}'") - - @property - def _pebble_layer(self) -> pebble.LayerDict: - """Return a dictionary representing a Pebble layer.""" - return { - "summary": "httpbin layer", - "description": "pebble config layer for httpbin", - "services": { - "httpbin": { - "override": "replace", - "summary": "httpbin", - "command": "gunicorn -b 0.0.0.0:80 httpbin:app -k gevent", - "startup": "enabled", - "environment": { - "GUNICORN_CMD_ARGS": f"--log-level {self.model.config['log-level']}" - }, - } - }, - } + raise RuntimeError("failed to delete the lock file: owned by another charm") + + def _set_lock_failure_status(self) -> None: + """Set unit status to inform user to remove this charm application.""" + self.unit.status = ops.BlockedStatus( + "conflict: multiple chrony charms detected, " + f"remove this charm using `juju remove-application {self.app.name}`" + ) if __name__ == "__main__": # pragma: nocover - ops.main.main(IsCharmsTemplateCharm) + ops.main.main(ChronyClientCharm) diff --git a/src/chrony.py b/src/chrony.py new file mode 100644 index 0000000..df425b1 --- /dev/null +++ b/src/chrony.py @@ -0,0 +1,401 @@ +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Chrony controller.""" + +# check chrony.conf document for _PoolOptions attributes. +# flake8: noqa: DCO060 + +import collections +import itertools +import logging +import pathlib +import shutil +import subprocess # nosec +import textwrap +import typing +import urllib.parse + +import pydantic +from charms.operator_libs_linux.v0 import apt +from charms.operator_libs_linux.v1 import systemd + +logger = logging.getLogger(__name__) + + +class _PoolOptions(pydantic.BaseModel): + """Chrony pool directive options. + + For more detail: https://chrony-project.org/doc/4.5/chrony.conf.html + """ + + model_config = pydantic.ConfigDict(extra="forbid") + + minpoll: int | None = None + maxpoll: int | None = None + iburst: bool = False + burst: bool = False + key: str | None = None + nts: bool = False + certset: str | None = None + maxdelay: float | None = None + maxdelayratio: float | None = None + maxdelaydevratio: float | None = None + maxdelayquant: float | None = None + mindelay: float | None = None + asymmetry: float | None = None + offset: float | None = None + minsamples: int | None = None + maxsamples: int | None = None + filter: int | None = None + offline: bool = False + auto_offline: bool = False + prefer: bool = False + noselect: bool = False + trust: bool = False + require: bool = False + xleave: bool = False + polltarget: int | None = None + presend: int | None = None + minstratum: int | None = None + version: int | None = None + extfield: str | None = None + maxsources: int | None = None + + def render_options(self) -> str: + """Render pool options as chrony option string. + + Returns: + Chrony pool directive option string. + """ + options = [] + # mypy and pylint have problems handling the model_fields class attribute. + # pylint: disable=not-an-iterable + for field in sorted(f for f in _PoolOptions.model_fields if f != "copy"): # type: ignore + value = getattr(self, field) + # first, check if the value is of boolean type and True + # then, check if the value is of boolean type and False or None (unset) + # finally, check if the value is of a non-boolean type and set + if value is True: + options.append(field) + elif value is None or value is False: + continue + else: + options.extend([field, str(value)]) + return " ".join(options) + + +class _NtpSource(_PoolOptions): + """A NTP time source.""" + + host: typing.Annotated[str, pydantic.StringConstraints(min_length=1)] + port: int | None = None + + @classmethod + def from_source_url(cls, url: str) -> "_NtpSource": + """Parse a NTP time source from a URL. + + Args: + url: URL to parse. + + Returns: + Parsed NTP time source. + + Raises: + ValueError: If the URL is invalid. + """ + parsed = urllib.parse.urlparse(url) + if parsed.scheme != "ntp": + raise ValueError(f"Invalid NTP source URL: {url}") + query = dict(urllib.parse.parse_qsl(parsed.query)) + return cls(host=parsed.hostname, port=parsed.port, **query) # type: ignore + + def render(self) -> str: + """Render NTP time source as a chrony pool directive string. + + Returns: + Chrony pool directive string. + """ + directive = f"pool {self.host}" + if self.port is not None and self.port != 123: + directive += f" port {self.port}" + options = self.render_options() + if options: + directive += f" {options}" + return directive + + +class _NtsSource(_PoolOptions): + """A NTP time source with NTS enabled.""" + + host: typing.Annotated[str, pydantic.StringConstraints(min_length=1)] + ntsport: int | None = None + + @classmethod + def from_source_url(cls, url: str) -> "_NtsSource": + """Parse a NTP time source with NTS enabled from a URL. + + Args: + url: URL to parse. + + Returns: + Parsed NTP time source. + + Raises: + ValueError: If the URL is invalid. + """ + parsed = urllib.parse.urlparse(url) + if parsed.scheme != "nts": + raise ValueError(f"Invalid NTS source URL: {url}") + query = dict(urllib.parse.parse_qsl(parsed.query)) + return cls(host=parsed.hostname, ntsport=parsed.port, **query) # type: ignore + + def render(self) -> str: + """Render NTP time source as a chrony pool directive string with NTS enabled. + + Returns: + Chrony pool directive string. + """ + directive = f"pool {self.host} nts" + if self.ntsport is not None and self.ntsport != 4460: + directive += f" ntsport {self.ntsport}" + options = self.render_options() + if options: + directive += f" {options}" + return directive + + +TimeSource = _NtpSource | _NtsSource +TlsKeyPair = collections.namedtuple("TlsKeyPair", ["certificate", "key"]) + + +class Chrony: + """Chrony service manager.""" + + CONFIG_FILE = pathlib.Path("/etc/chrony/chrony.conf") + CONFIG_FILE_BACKUP = pathlib.Path("/var/lib/chrony/chrony.conf.bak") + CERTS_DIR = pathlib.Path("/etc/chrony/certs") + + @staticmethod + def is_installed() -> bool: + """Check if chrony related packages is installed. + + Returns: + True if installed, False otherwise. + """ + return bool(shutil.which("chrony_exporter") and shutil.which("chronyc")) + + @staticmethod + def install() -> None: # pragma: nocover + """Install the Chrony on the system.""" + subprocess.check_call( + ["add-apt-repository", "-y", "ppa:canonical-is-devops/chrony-charm"] + ) # nosec + apt.add_package( + ["chrony", "ca-certificates", "prometheus-chrony-exporter"], update_cache=True + ) + + @staticmethod + def uninstall() -> None: + """Uninstall installed packages from the system. + + Not all packages will be uninstalled, as some are system defaults. + For example, ca-certificates and chrony (as in Ubuntu 26.04). + """ + apt.remove_package(["prometheus-chrony-exporter"]) + + def read_config(self) -> str: + """Read the current chrony configuration file. + + Returns: + The current chrony configuration file content. + """ + return self.CONFIG_FILE.read_text(encoding="utf-8") # pragma: nocover + + def write_config(self, config: str) -> None: + """Write the chrony configuration file. + + Args: + config: The new chrony configuration file content. + """ + self.CONFIG_FILE.write_text(config, encoding="utf-8") # pragma: nocover + + def backup_config(self) -> None: + """Backup the current chrony configuration file.""" + if self.CONFIG_FILE_BACKUP.exists(): + logger.warning("failed to backup configuration file: backup already exists") + return + self.CONFIG_FILE_BACKUP.write_text(self.read_config(), encoding="utf-8") + + def restore_config(self) -> None: + """Restore the chrony configuration file from backup.""" + if not self.CONFIG_FILE_BACKUP.exists(): + logger.warning("failed to restore chrony configuration file from backup: no backup") + return + self.write_config(self.CONFIG_FILE_BACKUP.read_text(encoding="utf-8")) + self.CONFIG_FILE_BACKUP.unlink() + + def _make_certs_dir(self) -> None: # pragma: nocover + """Create the chrony TLS certificates directory.""" + self.CERTS_DIR.mkdir(exist_ok=True, mode=0o700) + shutil.chown(self.CERTS_DIR, "_chrony", "_chrony") + + def _iter_certs_dir(self) -> list[pathlib.Path]: # pragma: nocover + """Iterate over all certificate files in the certificate directory. + + Returns: + An iterator over the paths of the certificate files. + """ + return [f for f in self.CERTS_DIR.iterdir() if f.suffix in {".crt", ".key"}] + + @staticmethod + def _write_certs_file(path: pathlib.Path, content: str) -> None: # pragma: nocover + """Write content of a certificate file and set appropriate permissions and ownership. + + Args: + path: The path to the certificate file. + content: The content to write to the file. + """ + path.touch(mode=0o600, exist_ok=True) + path.write_text(content, encoding="utf-8") + shutil.chown(path, "_chrony", "_chrony") + + @staticmethod + def _read_certs_file(path: pathlib.Path) -> str: + """Read and return the content of a certificate file. + + Args: + path: The path to the certificate file. + + Returns: + The content of the certificate file as a string. + """ + return path.read_text(encoding="utf-8") # pragma: nocover + + @staticmethod + def _unlink_certs_file(path: pathlib.Path) -> None: + """Unlink (delete) a certificate file. + + Args: + path: The path to the certificate file to delete. + """ + path.unlink(missing_ok=True) # pragma: nocover + + def read_tls_key_pairs(self) -> list[TlsKeyPair]: + """Read TLS key pairs from the certificates directory. + + Returns: + A list of TlsKeyPair objects. + """ + self._make_certs_dir() + files = sorted(self._iter_certs_dir()) + key_pairs = [] + for crt, key in self._batched(files, 2): + key_pairs.append( + TlsKeyPair( + certificate=self._read_certs_file(crt), + key=self._read_certs_file(key), + ) + ) + return key_pairs + + def _batched(self, iterable: typing.Iterable, n: int) -> typing.Iterable: + """Batch data from the iterable into tuples of length n. The last may be shorter than n. + + Args: + iterable: The iterable to batch. + n: The number of elements to batch. + + Returns: + An iterator over the tuples of length n. + """ + if n < 1: + raise ValueError("n must be at least one") + iterator = iter(iterable) + while batch := tuple(itertools.islice(iterator, n)): + yield batch + + def write_tls_key_pairs(self, key_pairs: list[TlsKeyPair]) -> None: + """Write TLS key pairs to the certificates directory. + + Existing pairs are overwritten, and if more files exist than new key pairs provided, + the excess files are removed. + + Args: + key_pairs: A list of TlsKeyPair objects to write. + """ + self._make_certs_dir() + files = sorted(self._iter_certs_dir()) + for idx, (key_pair_files, key_pair) in enumerate( + itertools.zip_longest(self._batched(files, 2), key_pairs) + ): + if key_pair_files is None: + self._write_certs_file(self.CERTS_DIR / f"{idx:04}.crt", key_pair.certificate) + self._write_certs_file(self.CERTS_DIR / f"{idx:04}.key", key_pair.key) + continue + if key_pair is None: + for file in key_pair_files: + self._unlink_certs_file(file) + continue + crt_file, key_file = key_pair_files + if self._read_certs_file(crt_file) != key_pair.certificate: + self._write_certs_file(crt_file, key_pair.certificate) + if self._read_certs_file(key_file) != key_pair.key: + self._write_certs_file(key_file, key_pair.key) + + @staticmethod + def restart() -> None: + """Restart the chrony service.""" + systemd.service_restart("chrony") # pragma: nocover + + @staticmethod + def parse_source_url(url: str) -> TimeSource: + """Parse a time source from a URL. + + Args: + url: URL to parse. + + Returns: + Parsed TimeSource instance. + + Raises: + ValueError: If the URL is invalid. + """ + if url.startswith("ntp://"): + return _NtpSource.from_source_url(url) + if url.startswith("nts://"): + return _NtsSource.from_source_url(url) + raise ValueError(f"Invalid time source URL: {url}") + + @staticmethod + def new_config(sources: list[TimeSource], header: str = "") -> str: + """Generate the chrony configuration file content. + + Args: + header: Optional header in the configuration file. + sources: List of chrony time sources. + + Returns: + Generated chrony configuration file content. + + Raises: + ValueError: If no sources are provided. + """ + if not sources: + raise ValueError("No time sources provided") + sources_config = "\n".join(s.render() for s in sources) + static = textwrap.dedent( + """\ + sourcedir /run/chrony-dhcp + sourcedir /etc/chrony/sources.d + keyfile /etc/chrony/chrony.keys + driftfile /var/lib/chrony/chrony.drift + ntsdumpdir /var/lib/chrony + logdir /var/log/chrony + maxupdateskew 100.0 + rtcsync + makestep 1 3 + leapsectz right/UTC + """ + ) + return "\n\n".join(part for part in [header, sources_config, static] if part).lstrip() diff --git a/src/grafana_dashboards/chrony.json b/src/grafana_dashboards/chrony.json new file mode 100644 index 0000000..3fde952 --- /dev/null +++ b/src/grafana_dashboards/chrony.json @@ -0,0 +1,1212 @@ +{ + "__inputs": [], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.5.3" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Dashboard for Chrony Client Operator", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 11, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "refId": "A" + } + ], + "title": "Tracking", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "name" + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "chrony_tracking_info{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "{{ tracking_address }}", + "refId": "A" + } + ], + "title": "Reference Source Address", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 4 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value" + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "chrony_tracking_stratum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Stratum", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "chrony_tracking_last_offset_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", + "interval": "", + "legendFormat": "Last Offset", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "chrony_tracking_rms_offset_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", + "hide": false, + "interval": "", + "legendFormat": "RMS Offset (long term average)", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "chrony_tracking_system_time_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", + "hide": false, + "interval": "", + "legendFormat": "System Time", + "range": true, + "refId": "C" + } + ], + "title": "Offset", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 4 + }, + "id": 2, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "name" + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "chrony_tracking_info{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", + "instant": true, + "interval": "", + "legendFormat": "{{ tracking_refid }}", + "refId": "A" + } + ], + "title": "Reference Source ID", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "Absolute bound on the computer’s clock accuracy (assuming the stratum-1 computer is correct)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "abs(chrony_tracking_last_offset_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}) + chrony_tracking_root_dispersion_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"} + (0.5 * chrony_tracking_root_delay_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"})", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Clock Error", + "range": true, + "refId": "A" + } + ], + "title": "Maximum Clock Error", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "chrony_tracking_root_delay_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", + "interval": "", + "legendFormat": "Root delay", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "chrony_tracking_root_dispersion_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", + "hide": false, + "interval": "", + "legendFormat": "Root dispersion", + "range": true, + "refId": "B" + } + ], + "title": "Source delay", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "mode": "basic", + "type": "color-background" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "State" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "candidate": { + "color": "light-green", + "index": 4, + "text": "combined" + }, + "falseticker": { + "color": "orange", + "index": 2, + "text": "falseticker" + }, + "jittery": { + "color": "yellow", + "index": 3, + "text": "jittery" + }, + "outlier": { + "color": "transparent", + "index": 5, + "text": "not combined" + }, + "sync": { + "color": "green", + "index": 0, + "text": "synchronized" + }, + "unreach": { + "color": "red", + "index": 1, + "text": "unreachable" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Polling Interval" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 19, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 0, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Juju Unit" + } + ] + }, + "pluginVersion": "9.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_join(chrony_sources_state_info{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}, \"source_uid\", \"-\", \"juju_model_uuid\", \"juju_unit\", \"source_address\")", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "__auto", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_join(chrony_sources_stratum{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}, \"source_uid\", \"-\", \"juju_model_uuid\", \"juju_unit\", \"source_address\")", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_join(chrony_sources_polling_interval_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}, \"source_uid\", \"-\", \"juju_model_uuid\", \"juju_unit\", \"source_address\")", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "__auto", + "range": false, + "refId": "C" + } + ], + "title": "Current Status", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "source_uid", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 2": true, + "Value #A": true, + "Value #B": false, + "__name__": true, + "__name__ 2": true, + "instance": true, + "instance 2": true, + "job": true, + "job 2": true, + "juju_application 1": true, + "juju_application 2": true, + "juju_application 3": true, + "juju_model 2": true, + "juju_model 3": true, + "juju_model_uuid 2": true, + "juju_model_uuid 3": true, + "juju_unit 2": true, + "juju_unit 3": true, + "source_address 1": true, + "source_address 2": true, + "source_address 3": true, + "source_mode": true, + "source_name 2": true, + "source_name 3": true, + "source_uid": true + }, + "indexByName": { + "Time 1": 0, + "Time 2": 8, + "Time 3": 14, + "Value #A": 7, + "Value #B": 13, + "Value #C": 19, + "__name__ 1": 1, + "__name__ 2": 9, + "__name__ 3": 15, + "instance 1": 2, + "instance 2": 10, + "instance 3": 16, + "job 1": 3, + "job 2": 11, + "job 3": 17, + "juju_application 1": 21, + "juju_application 2": 26, + "juju_application 3": 31, + "juju_model 1": 22, + "juju_model 2": 27, + "juju_model 3": 32, + "juju_model_uuid 1": 24, + "juju_model_uuid 2": 28, + "juju_model_uuid 3": 33, + "juju_unit 1": 23, + "juju_unit 2": 29, + "juju_unit 3": 34, + "source_address 1": 25, + "source_address 2": 30, + "source_address 3": 35, + "source_mode": 4, + "source_name 1": 5, + "source_name 2": 12, + "source_name 3": 18, + "source_state": 6, + "source_uid": 20 + }, + "renameByName": { + "Value #B": "Stratum", + "Value #C": "Polling Interval", + "juju_application 1": "Juju Application", + "juju_model 1": "Juju Model", + "juju_model_uuid 1": "Juju Model UUID", + "juju_unit 1": "Juju Unit", + "source_address": "Address", + "source_name": "Name", + "source_name 1": "Name", + "source_state": "State" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 13, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "refId": "A" + } + ], + "title": "Sources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 24 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "chrony_sources_last_sample_age_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"} < 4.294967295e+09", + "interval": "", + "legendFormat": "{{ source_address }} ({{ source_name }})", + "range": true, + "refId": "A" + } + ], + "title": "Last Sample: Age", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 24 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "chrony_sources_last_sample_offset_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", + "interval": "", + "legendFormat": "{{ source_address }} ({{ source_name }})", + "range": true, + "refId": "A" + } + ], + "title": "Last Sample: Offset", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 24 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheusds}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "chrony_sources_last_sample_error_margin_seconds{juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\"}", + "interval": "", + "legendFormat": "{{ source_address }} ({{ source_name }})", + "range": true, + "refId": "A" + } + ], + "title": "Last Sample: Error Margin", + "type": "timeseries" + } + ], + "refresh": "30s", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "UTC", + "title": "Chrony Client Operator", + "version": 1, + "weekStart": "" +} diff --git a/src/prometheus_alert_rules/chrony.rule b/src/prometheus_alert_rules/chrony.rule new file mode 100644 index 0000000..0af4546 --- /dev/null +++ b/src/prometheus_alert_rules/chrony.rule @@ -0,0 +1,45 @@ +groups: + - name: chrony + rules: + - alert: ChronyTargetMissing + expr: up == 0 + for: 1m + labels: + severity: critical + annotations: + summary: Prometheus target missing (instance {{ $labels.instance }}) + description: | + Chrony target has disappeared. An exporter on {{ $labels.instance }} + might be crashed. + - alert: ChronyTrackingHighOffset + expr: chrony_tracking_last_offset_seconds > 1 + for: 1h + labels: + severity: critical + annotations: + summary: "Chrony tracking offset is high ({{ $value }}s)" + description: | + The last clock offset reported by Chrony on {{ $labels.instance }} + has been {{ $value }} seconds, exceeding the 1 s threshold for over 1 hour. + - alert: ChronyTrackingStaleMeasurement + expr: chrony_tracking_update_interval_seconds > 1800 + for: 1h + labels: + severity: critical + annotations: + summary: "Chrony update interval is too long ({{ $value }}s)" + description: | + Chrony on {{ $labels.instance }} has not processed a new measurement + for over 30 minutes. The current update interval is {{ $value }} seconds, + exceeding the 1800 s threshold for more than 1 hour. + - alert: ChronyHighStratum + expr: chrony_tracking_stratum > 3 + for: 1h + labels: + severity: warning + annotations: + summary: "Chrony tracking stratum is too high ({{ $value }})" + description: | + Chrony on instance {{ $labels.instance }} is tracking a source with + stratum {{ $value }}, which is above the acceptable threshold of 3 for + over 1 hour. diff --git a/tests/conftest.py b/tests/conftest.py index 09a84dd..b6a62ec 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,4 +10,22 @@ def pytest_addoption(parser): Args: parser: Pytest parser. """ - parser.addoption("--charm-file", action="store") + parser.addoption("--charm-file", action="append", default=[]) + parser.addoption( + "--use-existing", + action="store_true", + default=False, + help="This will skip deployment of the charms. Useful for local testing.", + ) + parser.addoption( + "--keep-models", + action="store_true", + default=False, + help="keep temporarily-created models", + ) + parser.addoption( + "--model", + action="store", + help="Juju model to use; if not provided, a new model " + "will be created for each test which requires one", + ) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index dddb292..0e44a50 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -1,2 +1,4 @@ # Copyright 2025 Canonical Ltd. # See LICENSE file for licensing details. + +"""Charm integration tests.""" diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 0000000..6a5d060 --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,198 @@ +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Fixtures for charm integration tests.""" + +import pathlib +import subprocess # nosec B404 +import typing + +import jubilant +import pytest + + +@pytest.fixture(name="chrony_client_charm_file", scope="session") +def chrony_client_charm_file_fixture(pytestconfig: pytest.Config): + """Build or get the chrony-client charm file.""" + charms = pytestconfig.getoption("--charm-file") + # if there's only one charm file supplied, use that one + if len(charms) == 1: + return charms[0] + + # else select the 24.04 based + charms = [c for c in charms if "24.04" in c] + if charms: + return charms[0] + + # else build the charm from source + try: + subprocess.run( + ["charmcraft", "pack", "--bases-index=0"], check=True, capture_output=True, text=True + ) # nosec B603, B607 + except subprocess.CalledProcessError as exc: + raise OSError(f"Error packing charm: {exc}; Stderr:\n{exc.stderr}") from None + + app_name = "chrony-client" + charm_path = pathlib.Path(__file__).parent.parent.parent + charms = [p.absolute() for p in charm_path.glob(f"{app_name}_*24.04*.charm")] + assert charms, f"{app_name} .charm file not found" + assert len(charms) == 1, f"{app_name} has more than one .charm file, unsure which to use" + return str(charms[0]) + + +@pytest.fixture(name="juju", scope="module") +def juju_fixture(request: pytest.FixtureRequest) -> typing.Generator[jubilant.Juju, None, None]: + """Pytest fixture that wraps :meth:`jubilant.with_model`.""" + + def show_debug_log(juju: jubilant.Juju) -> None: + if request.session.testsfailed: + log = juju.debug_log(limit=1000) + print(log, end="") + + use_existing = request.config.getoption("--use-existing", default=False) + if use_existing: + juju = jubilant.Juju() + yield juju + show_debug_log(juju) + return + + model = request.config.getoption("--model") + if model: + juju = jubilant.Juju(model=model) + yield juju + show_debug_log(juju) + return + + keep_models = typing.cast(bool, request.config.getoption("--keep-models")) + with jubilant.temp_model(keep=keep_models) as juju: + juju.wait_timeout = 10 * 60 + yield juju + show_debug_log(juju) + return + + +@pytest.fixture(name="deploy_charms", scope="module") +def deploy_charms_fixture(juju: jubilant.Juju, chrony_client_charm_file: str): + """Deploy charms fixture deploy all charms necessary for the integration test.""" + juju.deploy(charm="ubuntu", base="ubuntu@24.04") + juju.deploy(charm=chrony_client_charm_file) + juju.deploy( + charm="chrony", + config={"sources": "ntp://ntp.ubuntu.com?iburst=true&maxsources=4"}, + channel="latest/edge", + ) + juju.integrate("ubuntu", "chrony-client") + juju.wait(jubilant.all_active, timeout=20 * 60) + + +class App: + """A helper class for charm applications.""" + + def __init__(self, juju: jubilant.Juju, name: str) -> None: + """Initialize the charm application class. + + Args: + juju: Juju instance + name: Application name + """ + self._juju = juju + self.name = name + + def get_leader_unit(self) -> str: + """Get the leader unit name for this application. + + Returns: + Leader unit name. + + Raises: + RuntimeError: If no leader unit exists for this application. + """ + status = self._juju.status() + leader = [name for name, unit in status.get_units(self.name).items() if unit.leader] + if not leader: + raise RuntimeError(f"no leader unit found for {self.name}?") + return leader[0] + + def get_unit_ip(self, unit_num: int | None = None) -> str: + """Get the IP address of the unit. + + Args: + unit_num: unit number, if not provided, the leader unit number is used. + + Returns: + IP address of the unit. + """ + status = self._juju.status() + units = status.get_units(self.name) + if unit_num is None: + unit_name = self.get_leader_unit() + else: + unit_name = f"{self.name}/{unit_num}" + unit_ip = units[unit_name].public_address + return unit_ip + + def ssh(self, cmd: str, *, unit_num: int | None = None) -> str: + """Run a command on a charm unit. + + Args: + cmd: command to run + unit_num: unit number, if not provided, the leader unit number is used. + + Returns: + Output of the command. + """ + if unit_num is None: + unit_name = self.get_leader_unit() + else: + unit_name = f"{self.name}/{unit_num}" + return self._juju.ssh(target=unit_name, command=cmd) + + +@pytest.fixture(scope="module", name="principle_app") +def principle_app_fixture( + juju: jubilant.Juju, + # pylint: disable=unused-argument + deploy_charms, +): + """Deploy the principle charm app.""" + return App(juju=juju, name="ubuntu") + + +@pytest.fixture(scope="module", name="chrony_client_app") +def chrony_client_app_fixture( + juju: jubilant.Juju, + # pylint: disable=unused-argument + deploy_charms, +) -> App: + """Deployed chrony-client charm app.""" + return App(juju=juju, name="chrony-client") + + +@pytest.fixture(scope="module") +def chrony_app( + juju: jubilant.Juju, + # pylint: disable=unused-argument + deploy_charms, +) -> App: + """Deployed chrony charm app.""" + return App(juju=juju, name="chrony") + + +@pytest.fixture(scope="function") +def another_chrony_client_app( + juju: jubilant.Juju, + chrony_client_app, + chrony_client_charm_file, + # pylint: disable=unused-argument + principle_app, +): + """Deploy another chrony-client charm app.""" + name = "another-chrony-client" + + juju.deploy(charm=chrony_client_charm_file, app=name) + juju.integrate("ubuntu", name) + juju.wait(jubilant.all_agents_idle, timeout=20 * 60) + + yield App(juju=juju, name=name) + + juju.remove_application(name) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt new file mode 100644 index 0000000..cebd81e --- /dev/null +++ b/tests/integration/requirements.txt @@ -0,0 +1,2 @@ +pytest==8.4.1 +jubilant==1.2.0 \ No newline at end of file diff --git a/tests/integration/test_charm.py b/tests/integration/test_charm.py index 3bc3268..7fe442f 100644 --- a/tests/integration/test_charm.py +++ b/tests/integration/test_charm.py @@ -5,35 +5,59 @@ """Integration tests.""" -import asyncio -import logging -from pathlib import Path - +import jubilant import pytest -import yaml -from pytest_operator.plugin import OpsTest -logger = logging.getLogger(__name__) -CHARMCRAFT_DATA = yaml.safe_load(Path("./charmcraft.yaml").read_text(encoding="utf-8")) -APP_NAME = CHARMCRAFT_DATA["name"] +def test_time_sources(juju, chrony_client_app, chrony_app): + """ + arrange: deploy the chrony-client and chrony charm. + act: use the chrony charm as the time source for the chrony-client charm. + assert: check if the chrony-client charm is using the time source. + """ + server_ip = chrony_app.get_unit_ip() + sources = f"ntp://{server_ip}?iburst=true" + juju.config(chrony_client_app.name, {"sources": sources}) + juju.wait( + lambda *args, **kwargs: jubilant.all_active(*args, **kwargs) + and jubilant.all_agents_idle(*args, **kwargs) + ) + assert server_ip in chrony_client_app.ssh("chronyc -N -n -c sources") -@pytest.mark.abort_on_fail -async def test_build_and_deploy(ops_test: OpsTest, pytestconfig: pytest.Config): - """Deploy the charm together with related charms. - Assert on the unit status before any relations/configurations take place. +def test_chrony_exporter(chrony_client_app): """ - # Deploy the charm and wait for active/idle status - charm = pytestconfig.getoption("--charm-file") - resources = {"httpbin-image": CHARMCRAFT_DATA["resources"]["httpbin-image"]["upstream-source"]} - assert ops_test.model - await asyncio.gather( - ops_test.model.deploy( - f"./{charm}", resources=resources, application_name=APP_NAME, series="jammy" - ), - ops_test.model.wait_for_idle( - apps=[APP_NAME], status="active", raise_on_blocked=True, timeout=1000 - ), - ) + arrange: deploy the chrony-client charm. + act: request chrony_exporter metrics endpoint. + assert: confirm that metrics are scraped. + """ + stdout = chrony_client_app.ssh("curl -m 10 localhost:9123/metrics") + assert "chrony_sources_reachability_success" in stdout + + +def test_charm_conflict(juju, another_chrony_client_app): + """ + arrange: deploy the chrony-client charm. + act: deploy another chrony-client charm on the principle charm. + assert: confirm that the second charm is in block state. + """ + units = juju.status().get_units(another_chrony_client_app.name) + status = units[another_chrony_client_app.get_leader_unit()].workload_status + assert status.current == "blocked" + assert "conflict" in status.message + + +def test_charm_uninstall_cleanup(juju, chrony_client_app, principle_app): + """ + arrange: deploy the chrony-client charm. + act: remove the chrony-client charm. + assert: confirm that the chrony-charm related configuration and packages are removed + """ + juju.remove_application(chrony_client_app.name) + juju.wait(jubilant.all_active, timeout=20 * 60) + + with pytest.raises(jubilant.CLIError): + principle_app.ssh("which chrony_exporter") + + assert "charm" not in principle_app.ssh("cat /etc/chrony/chrony.conf") diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index dddb292..6f65eaa 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -1,2 +1,4 @@ # Copyright 2025 Canonical Ltd. # See LICENSE file for licensing details. + +"""Charm unit tests.""" diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py new file mode 100644 index 0000000..5ef2682 --- /dev/null +++ b/tests/unit/conftest.py @@ -0,0 +1,112 @@ +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Fixtures for charm tests.""" + +import pathlib +from unittest.mock import patch + +import pytest + +import chrony + + +@pytest.fixture(name="patch_charm", autouse=True) +def patch_charm_fixture(): + """Patch necessary functions in the charm.""" + chrony_lock_file = None + + def _write_chrony_lock_file(content: str) -> None: + nonlocal chrony_lock_file + chrony_lock_file = content + + def _read_chrony_lock_file() -> None | str: + return chrony_lock_file + + def _delete_chrony_lock_file(): + nonlocal chrony_lock_file + chrony_lock_file = None + + with ( + patch("charm.ChronyClientCharm._write_chrony_lock_file") as mock_write_chrony_lock_file, + patch("charm.ChronyClientCharm._read_chrony_lock_file") as mock_read_chrony_lock_file, + patch("charm.ChronyClientCharm._delete_chrony_lock_file") as mock_delete_chrony_lock_file, + ): + mock_write_chrony_lock_file.side_effect = _write_chrony_lock_file + mock_read_chrony_lock_file.side_effect = _read_chrony_lock_file + mock_delete_chrony_lock_file.side_effect = _delete_chrony_lock_file + yield + + +@pytest.fixture(name="mock_chrony", autouse=True) +def mock_chrony_fixture(): # noqa: C901 pylint: disable=too-many-locals + """Create a Chrony object with necessary methods patched.""" + installed = False + + def install(): + nonlocal installed + installed = True + + def uninstall(): + nonlocal installed + installed = False + + mock_config = "" + + def read_config(): + return mock_config + + def write_config(config: str): + nonlocal mock_config + mock_config = config + + certs: dict[str, str] = {} + + def _iter_certs_dir(): + for file in certs: + yield pathlib.Path("/etc/chrony/certs") / file + + def _write_certs_file(path: pathlib.Path, content: str): + certs[path.name] = content + + def _read_certs_file(path: pathlib.Path): + return certs[path.name] + + def _unlink_certs_file(path: pathlib.Path) -> None: + del certs[path.name] + + backup_config_content = None + + def backup_config(): + nonlocal backup_config_content + backup_config_content = read_config() + + def restore_config(): + if backup_config_content is not None: + write_config(backup_config_content) + + with ( + patch("chrony.Chrony.install") as mock_install, + patch("chrony.Chrony.uninstall") as mock_uninstall, + patch("chrony.Chrony.restart"), + patch("chrony.Chrony.write_config") as mock_write_config, + patch("chrony.Chrony.read_config") as mock_read_config, + patch("chrony.Chrony.backup_config") as mock_backup_config, + patch("chrony.Chrony.restore_config") as mock_restore_config, + patch("chrony.Chrony._make_certs_dir"), + patch("chrony.Chrony._iter_certs_dir") as mock_iter_certs_dir, + patch("chrony.Chrony._write_certs_file") as mock_write_certs_file, + patch("chrony.Chrony._read_certs_file") as mock_read_certs_file, + patch("chrony.Chrony._unlink_certs_file") as mock_unlink_certs_file, + ): + mock_install.side_effect = install + mock_uninstall.side_effect = uninstall + mock_read_config.side_effect = read_config + mock_write_config.side_effect = write_config + mock_backup_config.side_effect = backup_config + mock_restore_config.side_effect = restore_config + mock_iter_certs_dir.side_effect = _iter_certs_dir + mock_write_certs_file.side_effect = _write_certs_file + mock_read_certs_file.side_effect = _read_certs_file + mock_unlink_certs_file.side_effect = _unlink_certs_file + yield chrony.Chrony() diff --git a/tests/unit/requirements.txt b/tests/unit/requirements.txt new file mode 100644 index 0000000..a69ffb7 --- /dev/null +++ b/tests/unit/requirements.txt @@ -0,0 +1,3 @@ +coverage[toml]==7.9.1 +pytest==8.4.1 +ops[testing]==2.22.0 diff --git a/tests/unit/test_base.py b/tests/unit/test_base.py deleted file mode 100644 index 8a55eb2..0000000 --- a/tests/unit/test_base.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2025 Canonical Ltd. -# See LICENSE file for licensing details. - -# Learn more about testing at: https://ops.readthedocs.io/en/latest/explanation/testing.html - -# pylint: disable=duplicate-code,missing-function-docstring -"""Unit tests.""" - -import unittest - -import ops -import ops.testing - -from charm import IsCharmsTemplateCharm - - -class TestCharm(unittest.TestCase): - """Test class.""" - - def setUp(self): - """Set up the testing environment.""" - self.harness = ops.testing.Harness(IsCharmsTemplateCharm) - self.addCleanup(self.harness.cleanup) - self.harness.begin() - - def test_httpbin_pebble_ready(self): - # Expected plan after Pebble ready with default config - expected_plan = { - "services": { - "httpbin": { - "override": "replace", - "summary": "httpbin", - "command": "gunicorn -b 0.0.0.0:80 httpbin:app -k gevent", - "startup": "enabled", - "environment": {"GUNICORN_CMD_ARGS": "--log-level info"}, - } - }, - } - # Simulate the container coming up and emission of pebble-ready event - self.harness.container_pebble_ready("httpbin") - # Get the plan now we've run PebbleReady - updated_plan = self.harness.get_container_pebble_plan("httpbin").to_dict() - # Check we've got the plan we expected - self.assertEqual(expected_plan, updated_plan) - # Check the service was started - service = self.harness.model.unit.get_container("httpbin").get_service("httpbin") - self.assertTrue(service.is_running()) - # Ensure we set an ActiveStatus with no message - self.assertEqual(self.harness.model.unit.status, ops.ActiveStatus()) - - def test_config_changed_valid_can_connect(self): - # Ensure the simulated Pebble API is reachable - self.harness.set_can_connect("httpbin", True) - # Trigger a config-changed event with an updated value - self.harness.update_config({"log-level": "debug"}) - # Get the plan now we've run PebbleReady - updated_plan = self.harness.get_container_pebble_plan("httpbin").to_dict() - updated_env = updated_plan["services"]["httpbin"]["environment"] - # Check the config change was effective - self.assertEqual(updated_env, {"GUNICORN_CMD_ARGS": "--log-level debug"}) - self.assertEqual(self.harness.model.unit.status, ops.ActiveStatus()) - - def test_config_changed_valid_cannot_connect(self): - # Trigger a config-changed event with an updated value - self.harness.update_config({"log-level": "debug"}) - # Check the charm is in WaitingStatus - self.assertIsInstance(self.harness.model.unit.status, ops.WaitingStatus) - - def test_config_changed_invalid(self): - # Ensure the simulated Pebble API is reachable - self.harness.set_can_connect("httpbin", True) - # Trigger a config-changed event with an updated value - self.harness.update_config({"log-level": "foobar"}) - # Check the charm is in BlockedStatus - self.assertIsInstance(self.harness.model.unit.status, ops.BlockedStatus) diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py new file mode 100644 index 0000000..d0249ae --- /dev/null +++ b/tests/unit/test_charm.py @@ -0,0 +1,183 @@ +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +# Learn more about testing at: https://ops.readthedocs.io/en/latest/explanation/testing.html + +# pylint: disable=duplicate-code,missing-function-docstring,protected-access + +"""Unit tests.""" + +import textwrap + +import pytest +from ops import testing + +import charm +import chrony + + +@pytest.mark.parametrize( + "sources, valid, source_config", + [ + pytest.param( + "ntp://example.com", + True, + "pool example.com", + id="ntp server", + ), + pytest.param( + "ntp://example.com:1234", + True, + "pool example.com port 1234", + id="ntp server with port", + ), + pytest.param( + "ntp://example.com?iburst=true", + True, + "pool example.com iburst", + id="ntp server with iburst option", + ), + pytest.param( + "ntp://example.com:1234?iburst=true&minpoll=10&polltarget=50", + True, + "pool example.com port 1234 iburst minpoll 10 polltarget 50", + id="ntp server with multiple option", + ), + pytest.param( + "nts://example.com?require=true&offset=-0.1", + True, + "pool example.com nts offset -0.1 require", + id="nts server", + ), + pytest.param( + textwrap.dedent( + """\ + ntp://ntp.ubuntu.com?iburst=true&maxsources=4, + ntp://0.ubuntu.pool.ntp.org?iburst=true&maxsources=1, + ntp://1.ubuntu.pool.ntp.org?iburst=true&maxsources=1, + ntp://2.ubuntu.pool.ntp.org?iburst=true&maxsources=2 + """ + ), + True, + textwrap.dedent( + """\ + pool ntp.ubuntu.com iburst maxsources 4 + pool 0.ubuntu.pool.ntp.org iburst maxsources 1 + pool 1.ubuntu.pool.ntp.org iburst maxsources 1 + pool 2.ubuntu.pool.ntp.org iburst maxsources 2 + """ + ), + id="multiple ntp server", + ), + pytest.param( + "example.com", + False, + "", + id="invalid ntp server", + ), + pytest.param( + "", + False, + "", + id="no sources", + ), + pytest.param( + "example.com", + False, + "", + id="invalid sources: no protocol", + ), + pytest.param( + "example.com:99999", + False, + "", + id="invalid sources: invalid port", + ), + pytest.param( + "https://example.com", + False, + "", + id="invalid sources: unknown protocol", + ), + pytest.param( + "ntp://example.com?foobar=true", + False, + "", + id="invalid sources: unknown param", + ), + ], +) +def test_chrony_config(sources: str, valid: bool, source_config: str, mock_chrony: chrony.Chrony): + """ + arrange: none. + act: trigger the 'config-changed' event with different sources charm configuration. + assert: check if configuration file content matches the charm configuration. + """ + mock_chrony.write_config("default") + + ctx = testing.Context(charm.ChronyClientCharm) + state_in = testing.State( + config={"sources": sources}, + relations=[testing.SubordinateRelation(endpoint="juju-info", id=1)], + ) + + state_out = ctx.run(ctx.on.config_changed(), state_in) + + assert charm.ChronyClientCharm._read_chrony_lock_file() == "chrony-client" + + if not valid: + assert state_out.unit_status.name == testing.BlockedStatus.name + assert mock_chrony.read_config() == "default" + assert not mock_chrony.restart.called + return + + assert state_out.unit_status == testing.ActiveStatus() + expected_config = ( + charm.CHRONY_CHARM_CONFIG_HEADER + + "\n\n" + + source_config.strip() + + "\n" + + textwrap.dedent( + """ + sourcedir /run/chrony-dhcp + sourcedir /etc/chrony/sources.d + keyfile /etc/chrony/chrony.keys + driftfile /var/lib/chrony/chrony.drift + ntsdumpdir /var/lib/chrony + logdir /var/log/chrony + maxupdateskew 100.0 + rtcsync + makestep 1 3 + leapsectz right/UTC + """ + ) + ) + assert mock_chrony.read_config() == expected_config + mock_chrony.restart.assert_called_once() + + +def test_chrony_uninstall(mock_chrony: chrony.Chrony): + """ + arrange: run the `config-changed` event + act: trigger the 'remove' event. + assert: check if configuration file content and packages are restored. + """ + mock_chrony.write_config("default") + + ctx = testing.Context(charm.ChronyClientCharm) + state_in = testing.State( + config={"sources": "ntp://example.com"}, + relations=[testing.SubordinateRelation(endpoint="juju-info", id=1)], + ) + ctx.run(ctx.on.config_changed(), state_in) + + ctx = testing.Context(charm.ChronyClientCharm) + state_in = testing.State( + config={"sources": "ntp://example.com"}, + relations=[testing.SubordinateRelation(endpoint="juju-info", id=1)], + ) + ctx.run(ctx.on.remove(), state_in) + + assert charm.ChronyClientCharm._read_chrony_lock_file() is None + assert mock_chrony.read_config() == "default" + mock_chrony.uninstall.assert_called_once() diff --git a/tox.ini b/tox.ini index 3c6615a..fa128cf 100644 --- a/tox.ini +++ b/tox.ini @@ -36,9 +36,9 @@ description = Check code against coding style standards deps = black codespell - flake8<6.0.0 + flake8 flake8-builtins - flake8-copyright<6.0.0 + flake8-copyright flake8-docstrings>=1.6.0 flake8-docstrings-complete>=1.0.3 flake8-test-docs>=1.0 @@ -47,7 +47,7 @@ deps = pep8-naming pydocstyle>=2.10 pylint - pyproject-flake8<6.0.0 + pyproject-flake8 pytest pytest-asyncio pytest-operator @@ -55,6 +55,8 @@ deps = types-PyYAML types-requests -r{toxinidir}/requirements.txt + -r{toxinidir}/tests/unit/requirements.txt + -r{toxinidir}/tests/integration/requirements.txt commands = pydocstyle {[vars]src_path} # uncomment the following line if this charm owns a lib @@ -72,9 +74,8 @@ commands = [testenv:unit] description = Run unit tests deps = - coverage[toml] - pytest -r{toxinidir}/requirements.txt + -r{toxinidir}/tests/unit/requirements.txt commands = coverage run --source={[vars]src_path} \ -m pytest --ignore={[vars]tst_path}integration -v --tb native -s {posargs} @@ -100,10 +101,7 @@ commands = [testenv:integration] description = Run integration tests deps = - juju==3.6.* - pytest - pytest-asyncio - pytest-operator -r{toxinidir}/requirements.txt + -r{toxinidir}/tests/integration/requirements.txt commands = pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs}