From 3c84f04e6fbc42e461685f1c2aee9df5f9c61636 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Mon, 23 Jun 2025 04:27:13 +0000 Subject: [PATCH 01/75] add prometheus client to dependency & order deps alphabetically --- github-runner-manager/requirements.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/github-runner-manager/requirements.txt b/github-runner-manager/requirements.txt index 38bb457784..a11a02eac5 100644 --- a/github-runner-manager/requirements.txt +++ b/github-runner-manager/requirements.txt @@ -1,9 +1,10 @@ +click==8.2.1 fabric >=3,<4 +flask==3.1.1 ghapi jinja2 kombu==5.5.3 openstacksdk==4.5.0 +prometheus-client==0.22.1 pydantic < 2 pymongo==4.13.0 -click==8.2.1 -flask==3.1.1 From 5188ef2b21bffb422003ab6a81314c71a0a54810 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Mon, 23 Jun 2025 04:29:04 +0000 Subject: [PATCH 02/75] add metrics route --- .../src/github_runner_manager/http_server.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/github-runner-manager/src/github_runner_manager/http_server.py b/github-runner-manager/src/github_runner_manager/http_server.py index be806a88fe..c66ed70cbe 100644 --- a/github-runner-manager/src/github_runner_manager/http_server.py +++ b/github-runner-manager/src/github_runner_manager/http_server.py @@ -12,6 +12,7 @@ from threading import Lock from flask import Flask, request +from prometheus_client import generate_latest from github_runner_manager.configuration import ApplicationConfiguration from github_runner_manager.errors import CloudError, LockError @@ -103,6 +104,12 @@ def get_lock() -> Lock: raise LockError("Lock not configured") +@app.route("/metrics", methods=["GET"]) +def metrics(): + """Return prometheus metrics from default registry.""" + return generate_latest() + + @dataclass class FlaskArgs: """Arguments for Flask HTTP server. From 98cd349b5747d78bf916479df8b3e514dabd7a2f Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Mon, 23 Jun 2025 04:29:14 +0000 Subject: [PATCH 03/75] refactor internal function --- .../src/github_runner_manager/http_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/github-runner-manager/src/github_runner_manager/http_server.py b/github-runner-manager/src/github_runner_manager/http_server.py index c66ed70cbe..6b852d651a 100644 --- a/github-runner-manager/src/github_runner_manager/http_server.py +++ b/github-runner-manager/src/github_runner_manager/http_server.py @@ -73,7 +73,7 @@ def flush_runner() -> tuple[str, int]: if flush_busy_str in ("True", "true"): flush_busy = True - lock = get_lock() + lock = _get_lock() with lock: app.logger.info("Flushing runners...") runner_scaler = get_runner_scaler(app_config) @@ -88,7 +88,7 @@ def flush_runner() -> tuple[str, int]: return ("", 204) -def get_lock() -> Lock: +def _get_lock() -> Lock: """Get the lock representing modification access to the set of runners. Raises: From 17665195829afb73c79b3991151e604d1e319a7d Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Mon, 23 Jun 2025 04:29:36 +0000 Subject: [PATCH 04/75] add reconcile metrics --- .../metrics/reconcile.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 github-runner-manager/src/github_runner_manager/metrics/reconcile.py diff --git a/github-runner-manager/src/github_runner_manager/metrics/reconcile.py b/github-runner-manager/src/github_runner_manager/metrics/reconcile.py new file mode 100644 index 0000000000..dbd8df267e --- /dev/null +++ b/github-runner-manager/src/github_runner_manager/metrics/reconcile.py @@ -0,0 +1,34 @@ +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Module for collecting metrics related to the reconciliation process.""" + +from prometheus_client import Gauge, Histogram + +LABEL_FLAVOR = "flavor" + +RECONCILE_DURATION_SECONDS = Histogram( + name="reconcile_duration_seconds", + documentation="Duration of reconciliation (seconds)", + labelnames=[LABEL_FLAVOR], +) +EXPECTED_RUNNERS_COUNT = Gauge( + name="expected_runners_count", + documentation="Expected number of runners", + labelnames=[LABEL_FLAVOR], +) +BUSY_RUNNERS_COUNT = Gauge( + name="busy_runners_count", + documentation="Number of busy runners", + labelnames=[LABEL_FLAVOR], +) +IDLE_RUNNERS_COUNT = Gauge( + name="idle_runners_count", + documentation="Number of idle runners", + labelnames=[LABEL_FLAVOR], +) +CLEANED_RUNNERS_TOTAL = Gauge( + name="cleaned_runners_total", + documentation="Total number of runners cleaned up", + labelnames=[LABEL_FLAVOR], +) From c2b7fc39e628f110eb84960169a7b3ca8dc62325 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Mon, 23 Jun 2025 04:30:00 +0000 Subject: [PATCH 05/75] implement reconcile metrics --- .../manager/runner_manager.py | 2 ++ .../manager/runner_scaler.py | 19 +++++++++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/github-runner-manager/src/github_runner_manager/manager/runner_manager.py b/github-runner-manager/src/github_runner_manager/manager/runner_manager.py index 8624c6d9d7..a11c40cfc9 100644 --- a/github-runner-manager/src/github_runner_manager/manager/runner_manager.py +++ b/github-runner-manager/src/github_runner_manager/manager/runner_manager.py @@ -24,6 +24,7 @@ from github_runner_manager.metrics import events as metric_events from github_runner_manager.metrics import github as github_metrics from github_runner_manager.metrics import runner as runner_metrics +from github_runner_manager.metrics.reconcile import CLEANED_RUNNERS_TOTAL from github_runner_manager.metrics.runner import RunnerMetrics from github_runner_manager.openstack_cloud.constants import CREATE_SERVER_TIMEOUT from github_runner_manager.platform.platform_provider import ( @@ -301,6 +302,7 @@ def _cleanup_resources( ) cloud_runners_to_delete = cloud_runners_to_delete[:maximum_runners_to_delete] + CLEANED_RUNNERS_TOTAL.labels("flavor", len(cloud_runners_to_delete)) return self._delete_cloud_runners( cloud_runners_to_delete, runners_health_response.requested_runners, diff --git a/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py b/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py index 34efa53e79..33b5fc7a6a 100644 --- a/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py +++ b/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py @@ -8,10 +8,7 @@ from dataclasses import dataclass import github_runner_manager.reactive.runner_manager as reactive_runner_manager -from github_runner_manager.configuration import ( - ApplicationConfiguration, - UserInfo, -) +from github_runner_manager.configuration import ApplicationConfiguration, UserInfo from github_runner_manager.constants import GITHUB_SELF_HOSTED_ARCH_LABELS from github_runner_manager.errors import ( CloudError, @@ -28,6 +25,13 @@ RunnerMetadata, ) from github_runner_manager.metrics import events as metric_events +from github_runner_manager.metrics.reconcile import ( + BUSY_RUNNERS_COUNT, + CLEANED_RUNNERS_TOTAL, + EXPECTED_RUNNERS_COUNT, + IDLE_RUNNERS_COUNT, + RECONCILE_DURATION_SECONDS, +) from github_runner_manager.openstack_cloud.models import OpenStackServerConfig from github_runner_manager.openstack_cloud.openstack_runner_manager import ( OpenStackRunnerManager, @@ -216,6 +220,8 @@ def __init__( # pylint: disable=too-many-arguments, too-many-positional-argumen self._platform_name = platform_name self._python_path = python_path + EXPECTED_RUNNERS_COUNT.labels(self._manager.manager_name).set(self._base_quantity) + def get_runner_info(self) -> RunnerInfo: """Get information on the runners. @@ -243,6 +249,8 @@ def get_runner_info(self) -> RunnerInfo: offline += 1 case _: unknown += 1 + BUSY_RUNNERS_COUNT.labels("flavor", self._manager.manager_name).set(busy) + IDLE_RUNNERS_COUNT.labels("flavor", self._manager.manager_name).set(online) return RunnerInfo( online=online, busy=busy, @@ -321,6 +329,9 @@ def reconcile(self) -> int: flavor=self._manager.manager_name, expected_runner_quantity=expected_runner_quantity, ) + RECONCILE_DURATION_SECONDS.labels("flavor", self._manager.manager_name).observe( + end_timestamp - start_timestamp + ) _issue_reconciliation_metric(reconcile_metric_data) logger.info("Finished reconciliation.") From d55b89041a8c861b82feb04dcbc64efa43d466a5 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Mon, 23 Jun 2025 10:06:57 +0000 Subject: [PATCH 06/75] initial implementation --- .github/workflows/auto_update_libs.yaml | 10 --- .github/workflows/bot_pr_approval.yaml | 9 -- .github/workflows/comment.yaml | 12 --- .github/workflows/comment_contributing.yaml | 13 --- .github/workflows/e2e_test.yaml | 36 -------- .github/workflows/e2e_test_run.yaml | 89 ------------------- .github/workflows/integration_test.yaml | 43 ++++----- .github/workflows/promote_charm.yaml | 32 ------- .github/workflows/publish_charm.yaml | 12 --- .github/workflows/push_test.yaml | 14 --- .github/workflows/schedule_issues_test.yaml | 26 ------ .github/workflows/test.yaml | 13 --- .../workflows/test_github_runner_manager.yaml | 13 --- .github/workflows/tics.yaml | 36 -------- .../workflow_dispatch_crash_test.yaml | 17 ---- .../workflow_dispatch_failure_test.yaml | 23 ----- .../workflow_dispatch_ssh_debug.yaml | 17 ---- .github/workflows/workflow_dispatch_test.yaml | 20 ----- .../workflow_dispatch_wait_test.yaml | 28 ------ scripts/setup-integration-tests.sh | 6 -- src/charm.py | 7 +- tests/integration/setup-integration-tests.sh | 25 ++++++ tests/integration/test_prometheus_metrics.py | 78 ++++++++++++++++ 23 files changed, 131 insertions(+), 448 deletions(-) delete mode 100644 .github/workflows/auto_update_libs.yaml delete mode 100644 .github/workflows/bot_pr_approval.yaml delete mode 100644 .github/workflows/comment.yaml delete mode 100644 .github/workflows/comment_contributing.yaml delete mode 100644 .github/workflows/e2e_test.yaml delete mode 100644 .github/workflows/e2e_test_run.yaml delete mode 100644 .github/workflows/promote_charm.yaml delete mode 100644 .github/workflows/publish_charm.yaml delete mode 100644 .github/workflows/push_test.yaml delete mode 100644 .github/workflows/schedule_issues_test.yaml delete mode 100644 .github/workflows/test.yaml delete mode 100644 .github/workflows/test_github_runner_manager.yaml delete mode 100644 .github/workflows/tics.yaml delete mode 100644 .github/workflows/workflow_dispatch_crash_test.yaml delete mode 100644 .github/workflows/workflow_dispatch_failure_test.yaml delete mode 100644 .github/workflows/workflow_dispatch_ssh_debug.yaml delete mode 100644 .github/workflows/workflow_dispatch_test.yaml delete mode 100644 .github/workflows/workflow_dispatch_wait_test.yaml delete mode 100644 scripts/setup-integration-tests.sh create mode 100644 tests/integration/setup-integration-tests.sh create mode 100644 tests/integration/test_prometheus_metrics.py diff --git a/.github/workflows/auto_update_libs.yaml b/.github/workflows/auto_update_libs.yaml deleted file mode 100644 index 02b7b204e7..0000000000 --- a/.github/workflows/auto_update_libs.yaml +++ /dev/null @@ -1,10 +0,0 @@ -name: Auto-update charm libraries - -on: - schedule: - - cron: "0 1 * * *" - -jobs: - auto-update-libs: - uses: canonical/operator-workflows/.github/workflows/auto_update_charm_libs.yaml@main - secrets: inherit diff --git a/.github/workflows/bot_pr_approval.yaml b/.github/workflows/bot_pr_approval.yaml deleted file mode 100644 index e38c5841dd..0000000000 --- a/.github/workflows/bot_pr_approval.yaml +++ /dev/null @@ -1,9 +0,0 @@ -name: Provide approval for bot PRs - -on: - pull_request: - -jobs: - bot_pr_approval: - uses: canonical/operator-workflows/.github/workflows/bot_pr_approval.yaml@main - secrets: inherit diff --git a/.github/workflows/comment.yaml b/.github/workflows/comment.yaml deleted file mode 100644 index 26ac226df8..0000000000 --- a/.github/workflows/comment.yaml +++ /dev/null @@ -1,12 +0,0 @@ -name: Comment on the pull request - -on: - workflow_run: - workflows: ["Tests"] - types: - - completed - -jobs: - comment-on-pr: - uses: canonical/operator-workflows/.github/workflows/comment.yaml@main - secrets: inherit diff --git a/.github/workflows/comment_contributing.yaml b/.github/workflows/comment_contributing.yaml deleted file mode 100644 index c51f873143..0000000000 --- a/.github/workflows/comment_contributing.yaml +++ /dev/null @@ -1,13 +0,0 @@ -name: Comment on the pull request - -on: - pull_request: - types: - - opened - branches: - - 'track/**' - -jobs: - comment-on-pr: - uses: canonical/operator-workflows/.github/workflows/comment_contributing.yaml@main - secrets: inherit diff --git a/.github/workflows/e2e_test.yaml b/.github/workflows/e2e_test.yaml deleted file mode 100644 index 60fdbacbdb..0000000000 --- a/.github/workflows/e2e_test.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: End-to-End tests - -on: - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - # test option values defined at test/conftest.py are passed on via repository secret - # INTEGRATION_TEST_ARGS to operator-workflows automatically. - openstack-integration-end-to-end-test: - name: end-to-end test using private-endpoint - uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main - secrets: inherit - with: - juju-channel: 3.6/stable - pre-run-script: scripts/setup-integration-tests.sh - provider: lxd - test-tox-env: integration-juju3.6 - modules: '["test_e2e"]' - extra-arguments: "-m openstack" - self-hosted-runner: true - self-hosted-runner-label: stg-private-endpoint - - required_status_checks: - name: Required E2E Test Status Checks - runs-on: ubuntu-latest - needs: - - openstack-integration-end-to-end-test - if: always() && !cancelled() - timeout-minutes: 5 - steps: - - run: | - [ '${{ needs.openstack-integration-end-to-end-test.result }}' = 'success' ] || (echo openstack-integration-end-to-end-test failed && false) diff --git a/.github/workflows/e2e_test_run.yaml b/.github/workflows/e2e_test_run.yaml deleted file mode 100644 index bebbc1a2fc..0000000000 --- a/.github/workflows/e2e_test_run.yaml +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2025 Canonical Ltd. -# See LICENSE file for licensing details. - -name: Run End-to-End test - -on: - # The inputs are the same but they cannot be shared between triggers. - # See https://github.com/orgs/community/discussions/39357 - workflow_call: - inputs: - runner-tag: - description: The e2e test runner tag to run the workflow on. - type: string - required: true - workflow_dispatch: - inputs: - runner-tag: - description: The e2e test runner tag to run the workflow on. - type: string - required: true - -jobs: - e2e-test: - name: End-to-End Test Run - runs-on: [self-hosted, linux, "${{ inputs.runner-tag }}"] - steps: - - name: Hostname is set to "github-runner" - run: sudo hostnamectl hostname | grep github-runner - # Below is a series of simple tests to assess the functionality of the newly spawned runner. - - name: Echo hello world - run: echo "hello world" - - name: File permission for /usr/local/bin - run: ls -ld /usr/local/bin | grep drwxrwxrwx - - name: Test file permission for /usr/local/bin - run: touch /usr/local/bin/test_file - # "Install microk8s" step will test if the proxies settings are correct. - - name: Proxy set in /etc/environment - run: cat /etc/environment - # "Update apt in python docker container" step will test docker default proxy settings due to - # pulling the python image. - - name: Proxy set in docker daemon - run: | - [[ -z "${http_proxy}" && -z "${HTTP_PROXY}" ]] \ - || sudo cat /etc/systemd/system/docker.service.d/http-proxy.conf | grep HTTP_PROXY - # "Update apt in python docker container" step will test docker client default proxy settings. - - name: Proxy set in docker client - run: | - [[ -z "${http_proxy}" && -z "${HTTP_PROXY}" ]] \ - || cat /home/ubuntu/.docker/config.json | grep httpProxy - - name: Install microk8s - run: sudo snap install microk8s --classic - - name: Wait for microk8s - timeout-minutes: 10 - run: microk8s status --wait-ready - - name: Deploy nginx for testing - run: microk8s kubectl create deployment nginx --image=nginx - - name: Wait for nginx to be ready - run: microk8s kubectl rollout status deployment/nginx --timeout=30m - - name: Update apt in python docker container - run: docker run python:3.10-slim apt-get update - - name: Docker version - run: docker version - - name: Check python alias for python3 - run: python --version - - name: pip version - run: python3 -m pip --version - - name: npm version - run: npm --version - - name: shellcheck version - run: shellcheck --version - - name: jq version - run: jq --version - - name: yq version - run: yq --version - - name: apt update - run: sudo apt-get update -y - # Use pipx for 24.04 noble, check-jsonschema breaks OS system packages. - - name: install pipx - run: sudo apt-get install -y pipx - - name: install check-jsonschema - run: python3 -m pip install check-jsonschema || pipx install check-jsonschema - - name: unzip version - run: unzip -v - - name: gh version - run: gh --version - # `check-jsonschema` is installed using pip. The directory `~/.local/bin` needs to be added to PATH. - # ~/.local/bin is added to path runner env through in scripts/env.j2 - - name: test check-jsonschema - run: check-jsonschema --version diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index b274e4d925..47b0e1ae48 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -12,35 +12,36 @@ concurrency: cancel-in-progress: true jobs: - openstack-interface-tests-private-endpoint: - name: openstack interface test using private-endpoint - uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main - secrets: inherit - with: - juju-channel: 3.6/stable - pre-run-script: scripts/setup-integration-tests.sh - provider: lxd - test-tox-env: integration-juju3.6 - modules: '["test_runner_manager_openstack"]' - extra-arguments: '--log-format="%(asctime)s %(levelname)s %(message)s"' - self-hosted-runner: true - self-hosted-runner-label: stg-private-endpoint + # openstack-interface-tests-private-endpoint: + # name: openstack interface test using private-endpoint + # uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main + # secrets: inherit + # with: + # juju-channel: 3.6/stable + # provider: lxd + # test-tox-env: integration-juju3.6 + # modules: '["test_runner_manager_openstack"]' + # extra-arguments: '--log-format="%(asctime)s %(levelname)s %(message)s"' + # self-hosted-runner: true + # self-hosted-runner-label: stg-private-endpoint openstack-integration-tests-private-endpoint: name: Integration test using private-endpoint uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main secrets: inherit with: juju-channel: 3.6/stable - pre-run-script: scripts/setup-integration-tests.sh + pre-run-script: tests/integration/setup-integration-tests.sh provider: lxd test-tox-env: integration-juju3.6 - modules: '["test_charm_metrics_failure", "test_charm_metrics_success", "test_charm_fork_repo", "test_charm_fork_path_change", "test_charm_no_runner", "test_charm_runner", "test_debug_ssh", "test_charm_upgrade", "test_reactive", "test_jobmanager_prespawned", "test_jobmanager_reactive"]' + modules: '["test_prometheus_metrics"]' extra-arguments: '-m openstack --log-format="%(asctime)s %(levelname)s %(message)s"' self-hosted-runner: true self-hosted-runner-label: stg-private-endpoint - allure-report: - if: ${{ (success() || failure()) && github.event_name == 'schedule' }} - needs: - - openstack-interface-tests-private-endpoint - - openstack-integration-tests-private-endpoint - uses: canonical/operator-workflows/.github/workflows/allure_report.yaml@main + tmate-debug: true + tmate-timeout: 90 + # allure-report: + # if: ${{ (success() || failure()) && github.event_name == 'schedule' }} + # needs: + # - openstack-interface-tests-private-endpoint + # - openstack-integration-tests-private-endpoint + # uses: canonical/operator-workflows/.github/workflows/allure_report.yaml@main diff --git a/.github/workflows/promote_charm.yaml b/.github/workflows/promote_charm.yaml deleted file mode 100644 index d1a0ac0ffb..0000000000 --- a/.github/workflows/promote_charm.yaml +++ /dev/null @@ -1,32 +0,0 @@ -name: Promote charm - -on: - workflow_dispatch: - inputs: - origin-channel: - type: choice - description: 'Origin Channel' - options: - - latest/beta - - latest/edge - - 1/edge - destination-channel: - type: choice - description: 'Destination Channel' - options: - - latest/stable - - latest/beta - - 1/stable - secrets: - CHARMHUB_TOKEN: - required: true - -jobs: - promote-charm: - uses: canonical/operator-workflows/.github/workflows/promote_charm.yaml@main - with: - origin-channel: ${{ github.event.inputs.origin-channel }} - destination-channel: ${{ github.event.inputs.destination-channel }} - doc-automation-disabled: false - base-channel: '22.04' - secrets: inherit diff --git a/.github/workflows/publish_charm.yaml b/.github/workflows/publish_charm.yaml deleted file mode 100644 index a84cba1f56..0000000000 --- a/.github/workflows/publish_charm.yaml +++ /dev/null @@ -1,12 +0,0 @@ -name: Publish to edge - -on: - push: - branches: - - main - - track/* - -jobs: - publish-to-edge: - uses: canonical/operator-workflows/.github/workflows/publish_charm.yaml@main - secrets: inherit diff --git a/.github/workflows/push_test.yaml b/.github/workflows/push_test.yaml deleted file mode 100644 index e19f300c28..0000000000 --- a/.github/workflows/push_test.yaml +++ /dev/null @@ -1,14 +0,0 @@ -name: Push Event Tests - -on: - push: - branches: - - push-e2e-* - -jobs: - push-event-tests: - runs-on: [self-hosted, linux, "${{ github.ref_name }}"] - steps: - - name: Echo runner name - run: | - echo "Hello, runner: ${{ github.ref_name }}" diff --git a/.github/workflows/schedule_issues_test.yaml b/.github/workflows/schedule_issues_test.yaml deleted file mode 100644 index 5140353f1e..0000000000 --- a/.github/workflows/schedule_issues_test.yaml +++ /dev/null @@ -1,26 +0,0 @@ -name: Issues and Schedule Event Tests - -on: - workflow_dispatch: # Replaced by end-to-end test to match 'issues' or 'schedule' - -jobs: - - extract-runner-name: - runs-on: ubuntu-latest - outputs: - name: ${{ steps.extract-runner-name.outputs.name }} - steps: - - name: Extract runner name - id: extract-runner-name - run: | - REPO=${{ github.repository }} - RUNNER_NAME=${REPO#${{ github.repository_owner }}/} - echo name=$RUNNER_NAME >> $GITHUB_OUTPUT - - issues-schedule-event-tests: - needs: [ extract-runner-name ] - runs-on: [ self-hosted, linux, "${{ needs.extract-runner-name.outputs.name }}" ] - steps: - - name: Echo runner name - run: | - echo "Hello, runner: ${{ needs.extract-runner-name.outputs.name }}" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml deleted file mode 100644 index 165fae6d27..0000000000 --- a/.github/workflows/test.yaml +++ /dev/null @@ -1,13 +0,0 @@ -name: Tests - -on: - pull_request: - -jobs: - unit-tests: - uses: canonical/operator-workflows/.github/workflows/test.yaml@main - secrets: inherit - with: - self-hosted-runner: true - self-hosted-runner-label: edge - vale-style-check: true diff --git a/.github/workflows/test_github_runner_manager.yaml b/.github/workflows/test_github_runner_manager.yaml deleted file mode 100644 index 3f7eff3758..0000000000 --- a/.github/workflows/test_github_runner_manager.yaml +++ /dev/null @@ -1,13 +0,0 @@ -name: Tests for github-runner-manager - -on: - pull_request: - -jobs: - unit-tests-github-runner-manager: - uses: canonical/operator-workflows/.github/workflows/test.yaml@main - secrets: inherit - with: - self-hosted-runner: true - self-hosted-runner-label: edge - working-directory: ./github-runner-manager/ diff --git a/.github/workflows/tics.yaml b/.github/workflows/tics.yaml deleted file mode 100644 index 52067ad7a3..0000000000 --- a/.github/workflows/tics.yaml +++ /dev/null @@ -1,36 +0,0 @@ -on: [pull_request] - -jobs: - TICS: - name: TICS - runs-on: [self-hosted, linux, amd64, tiobe, jammy] - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Run Unit Tests (manager app) - run: | - cd github-runner-manager - pip install tox pylint flake8 -r ./requirements.txt - tox -e unit,coverage-report - rm -rf .tox - - name: Run Unit Tests - run: | - pip install tox pylint flake8 -r ./requirements.txt -r ./github-runner-manager/requirements.txt - tox -e unit,coverage-report - rm -rf .tox - - name: TICS GitHub Action - uses: tiobe/tics-github-action@v3 - env: - PYTHONPATH: "/home/ubuntu/actions-runner/_work/github-runner-operator/github-runner-operator/src:/home/ubuntu/actions-runner/_work/github-runner-operator/github-runner-operator/lib:/home/ubuntu/actions-runner/_work/github-runner-operator/github-runner-operator/github-runner-manager/src/github_runner_manager:/home/ubuntu/actions-runner/_work/github-runner-operator/github-runner-operator/github-runner-manager/src" - with: - mode: qserver - project: github-runner-operator - branchdir: ${{ github.workspace }} - viewerUrl: https://canonical.tiobe.com/tiobeweb/TICS/api/cfg?name=default - ticsAuthToken: ${{ secrets.TICSAUTHTOKEN }} - installTics: true - -concurrency: - group: tics - cancel-in-progress: false diff --git a/.github/workflows/workflow_dispatch_crash_test.yaml b/.github/workflows/workflow_dispatch_crash_test.yaml deleted file mode 100644 index 253513b641..0000000000 --- a/.github/workflows/workflow_dispatch_crash_test.yaml +++ /dev/null @@ -1,17 +0,0 @@ -name: Workflow Dispatch Crash Tests - -on: - # Manually dispatched workflow action - workflow_dispatch: - inputs: - runner: - description: 'Self hosted gh runner' - required: true - -jobs: - workflow-dispatch-tests: - runs-on: [self-hosted, linux, "${{ inputs.runner }}"] - steps: - - name: Sleep for an hour to give the integration test some time to crash the runner. - run: | - sleep 3600 diff --git a/.github/workflows/workflow_dispatch_failure_test.yaml b/.github/workflows/workflow_dispatch_failure_test.yaml deleted file mode 100644 index ef5c6ac8a7..0000000000 --- a/.github/workflows/workflow_dispatch_failure_test.yaml +++ /dev/null @@ -1,23 +0,0 @@ -name: Workflow Dispatch Failure Tests 2a34f8b1-41e4-4bcb-9bbf-7a74e6c482f7 - -# This workflow is for testing only. -# Self-hosted runner managed by the charm is setup to fail this workflow at pre-job step. - -on: - # Manually dispatched workflow action - workflow_dispatch: - inputs: - runner: - description: 'Self hosted gh runner' - required: true - -jobs: - workflow-dispatch-tests: - runs-on: [self-hosted, linux, "${{ inputs.runner }}"] - steps: - - name: Echo input variable and message - run: | - echo "Hello, runner: ${{ inputs.runner }}" - - name: Always echo a message - if: always() - run: echo "Should not echo if pre-job script failed" diff --git a/.github/workflows/workflow_dispatch_ssh_debug.yaml b/.github/workflows/workflow_dispatch_ssh_debug.yaml deleted file mode 100644 index edbfb39206..0000000000 --- a/.github/workflows/workflow_dispatch_ssh_debug.yaml +++ /dev/null @@ -1,17 +0,0 @@ -name: Workflow Dispatch Tests (ssh-debug/tmate) - -on: - # Manually dispatched workflow action - workflow_dispatch: - inputs: - runner: - description: 'Self hosted gh runner' - required: true - -jobs: - workflow-dispatch-tests: - runs-on: [self-hosted, linux, "${{ inputs.runner }}"] - steps: - - name: Setup tmate session - uses: canonical/action-tmate@main - timeout-minutes: 5 diff --git a/.github/workflows/workflow_dispatch_test.yaml b/.github/workflows/workflow_dispatch_test.yaml deleted file mode 100644 index 513dc4232e..0000000000 --- a/.github/workflows/workflow_dispatch_test.yaml +++ /dev/null @@ -1,20 +0,0 @@ -name: Workflow Dispatch Tests - -on: - # Manually dispatched workflow action - workflow_dispatch: - inputs: - runner: - description: 'Self hosted gh runner' - required: true - -jobs: - workflow-dispatch-tests: - runs-on: [self-hosted, linux, "${{ inputs.runner }}"] - steps: - - name: Echo input variable and message - run: | - echo "Hello, runner: ${{ inputs.runner }}" - - name: Always echo a message - if: always() - run: echo "Should not echo if pre-job script failed" diff --git a/.github/workflows/workflow_dispatch_wait_test.yaml b/.github/workflows/workflow_dispatch_wait_test.yaml deleted file mode 100644 index f2470bec4b..0000000000 --- a/.github/workflows/workflow_dispatch_wait_test.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: Workflow Dispatch Wait Tests - -on: - # Manually dispatched workflow action - workflow_dispatch: - inputs: - runner: - description: 'Self hosted gh runner' - required: true - minutes: - description: 'Number of minutes to wait' - # Number type not supported in workflow dispatch: https://github.com/orgs/community/discussions/67182 - # Seems to be by design: https://github.blog/changelog/2021-11-10-github-actions-input-types-for-manual-workflows/ - default: '2' - -jobs: - workflow-dispatch-tests: - runs-on: [self-hosted, linux, "${{ inputs.runner }}"] - steps: - - name: Echo input variable and message - run: | - echo "Hello, runner: ${{ inputs.runner }}" - - name: Wait - run: | - sleep ${{ inputs.minutes }}m - - name: Always echo a message - if: always() - run: echo "Should not echo if pre-job script failed" diff --git a/scripts/setup-integration-tests.sh b/scripts/setup-integration-tests.sh deleted file mode 100644 index c6a86227db..0000000000 --- a/scripts/setup-integration-tests.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2025 Canonical Ltd. -# See LICENSE file for licensing details. - -# Script to setup localhost for integration tests diff --git a/src/charm.py b/src/charm.py index 66a6e29283..419b40938d 100755 --- a/src/charm.py +++ b/src/charm.py @@ -186,7 +186,12 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) self._log_charm_status() - self._grafana_agent = COSAgentProvider(self) + self._grafana_agent = COSAgentProvider( + self, + metrics_endpoints=[ + {"path": "/metrics", "port": int(manager_service.GITHUB_RUNNER_MANAGER_PORT)} + ], + ) self._stored.set_default( path=self.config[PATH_CONFIG_NAME], # for detecting changes diff --git a/tests/integration/setup-integration-tests.sh b/tests/integration/setup-integration-tests.sh new file mode 100644 index 0000000000..ffe4deb61c --- /dev/null +++ b/tests/integration/setup-integration-tests.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +# Pre-run script for integration test operator-workflows action. +# https://github.com/canonical/operator-workflows/blob/main/.github/workflows/integration_test.yaml + +# The COS observability stack are deployed on K8s models. + +TESTING_MODEL="$(juju switch)" + +echo "bootstrapping microk8s juju controller" +sudo snap install microk8s --channel=1.32-strict/stable +GROUP=snap_microk8s +sudo usermod -a -G $GROUP $USER +if [ $(id -gn) != $GROUP ]; then + exec sg $GROUP "$0 $*" +fi +sudo microk8s enable hostpath-storage +microk8s status --wait-ready +juju bootstrap microk8s microk8s + +echo "Switching to testing model" +juju switch $TESTING_MODEL diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py new file mode 100644 index 0000000000..a89d982d3e --- /dev/null +++ b/tests/integration/test_prometheus_metrics.py @@ -0,0 +1,78 @@ +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Module for collecting metrics related to the reconciliation process.""" + +import secrets +from typing import Any, AsyncGenerator + +import pytest +import pytest_asyncio +from github.Branch import Branch +from github.Repository import Repository +from juju.action import Action +from juju.application import Application +from juju.controller import Controller +from juju.model import Model +from openstack.connection import Connection + +from charm_state import BASE_VIRTUAL_MACHINES_CONFIG_NAME, CUSTOM_PRE_JOB_SCRIPT_CONFIG_NAME +from tests.integration.helpers.common import ( + DISPATCH_TEST_WORKFLOW_FILENAME, + DISPATCH_WAIT_TEST_WORKFLOW_FILENAME, + dispatch_workflow, + get_job_logs, + wait_for, + wait_for_reconcile, +) +from tests.integration.helpers.openstack import OpenStackInstanceHelper, setup_repo_policy + + +@pytest_asyncio.fixture(scope="module", name="k8s_controller") +async def k8s_controller_fixture() -> AsyncGenerator[Controller, None]: + """The k8s controller. + + See scripts/setup-integration-tests.sh. + """ + controller = Controller() + await controller.connect_controller("microk8s") + yield controller + await controller.disconnect() + + +@pytest_asyncio.fixture(scope="module", name="k8s_model") +async def k8s_model_fixture(k8s_controller: Controller) -> AsyncGenerator[Model, None]: + """The machine model for jenkins agent machine charm.""" + k8s_model_name = f"k8s-{secrets.token_hex(2)}" + model = await k8s_controller.add_model(k8s_model_name) + await model.connect(f"localhost:admin/{model.name}") + yield model + await k8s_controller.destroy_models( + model.name, destroy_storage=True, force=True, max_wait=10 * 60 + ) + await model.disconnect() + + +@pytest_asyncio.fixture(scope="module", name="prometheus_app") +async def prometheus_app_fixture(k8s_model: Model): + """Deploy prometheus charm.""" + prometheus_app: Application = await k8s_model.deploy("prometheus-k8s", channel="1/stable") + return prometheus_app + + +async def test_prometheus_metrics( + model: Model, k8s_model: Model, app_openstack_runner: Application, prometheus_app: Application +): + """ + arrange: given a prometheus charm application. + act: when GitHub runner is integrated. + assert: the datasource is registered and basic metrics are available. + """ + offer = await k8s_model.create_offer(f"{prometheus_app.name}:metrics-endpoint", "metrics") + await model.integrate( + app_openstack_runner.name, f"microk8s:admin/{k8s_model.name}.{offer.name}" + ) + await k8s_model.wait_for_idle(apps=[prometheus_app.name], timeout=300) + await model.wait_for_idle(apps=[app_openstack_runner.name], timeout=300) + + assert False From cb74f9770a78e315d39698b8e3c6c603f944f625 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Mon, 23 Jun 2025 19:11:32 +0000 Subject: [PATCH 07/75] test: add integration test --- tests/integration/helpers/common.py | 25 +++++++++++++ tests/integration/requirements.txt | 5 +-- tests/integration/setup-integration-tests.sh | 0 tests/integration/test_prometheus_metrics.py | 38 ++++++++++---------- 4 files changed, 48 insertions(+), 20 deletions(-) mode change 100644 => 100755 tests/integration/setup-integration-tests.sh diff --git a/tests/integration/helpers/common.py b/tests/integration/helpers/common.py index 0d9566c941..02e72ad705 100644 --- a/tests/integration/helpers/common.py +++ b/tests/integration/helpers/common.py @@ -22,6 +22,7 @@ from github.WorkflowRun import WorkflowRun from juju.action import Action from juju.application import Application +from juju.client._definitions import ApplicationStatus, FullStatus, UnitStatus from juju.model import Model from juju.unit import Unit @@ -471,3 +472,27 @@ async def get_github_runner_manager_service_log(unit: Unit) -> str: assert return_code == 0, f"Get log with cat {log_file_path} failed with: {stderr}" assert stdout is not None return stdout + + +async def get_model_unit_addresses(model: Model, app_name: str) -> list[str]: + """Extract the address of a given unit. + + Args: + model: Juju model + app_name: Juju application name + + Returns: + the IP address of the Jenkins unit. + """ + status: FullStatus = await model.get_status() + # mypy cannot infer the type ApplicationStatus but thinks its the base class type "Type". + application_status: ApplicationStatus | None = status.applications[app_name] # type: ignore + assert application_status, f"Application status {app_name} not found in {status}" + # mypy cannot infer the type UnitStatus but thinks its the base class type "Type". + unit_status_map: dict[typing.Any, UnitStatus | None] = application_status.units # type: ignore + units_statuses: list[UnitStatus | None] = list(unit_status_map.values()) + return [ + str(unit_status.address) + for unit_status in units_statuses + if unit_status and unit_status.address + ] diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 524736dde0..062f66c0ba 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,5 +1,6 @@ GitPython>3,<4 -pygithub +jubilant==1.1.* kombu==5.* +pygithub pymongo==4.* -jubilant==1.1.* +tenacity==9.1.2 diff --git a/tests/integration/setup-integration-tests.sh b/tests/integration/setup-integration-tests.sh old mode 100644 new mode 100755 diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index a89d982d3e..280e70d612 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -3,29 +3,20 @@ """Module for collecting metrics related to the reconciliation process.""" +import logging import secrets -from typing import Any, AsyncGenerator +from typing import AsyncGenerator -import pytest import pytest_asyncio -from github.Branch import Branch -from github.Repository import Repository -from juju.action import Action +import requests from juju.application import Application from juju.controller import Controller from juju.model import Model -from openstack.connection import Connection +from tenacity import retry, stop_after_attempt, wait_exponential -from charm_state import BASE_VIRTUAL_MACHINES_CONFIG_NAME, CUSTOM_PRE_JOB_SCRIPT_CONFIG_NAME -from tests.integration.helpers.common import ( - DISPATCH_TEST_WORKFLOW_FILENAME, - DISPATCH_WAIT_TEST_WORKFLOW_FILENAME, - dispatch_workflow, - get_job_logs, - wait_for, - wait_for_reconcile, -) -from tests.integration.helpers.openstack import OpenStackInstanceHelper, setup_repo_policy +from tests.integration.helpers.common import get_model_unit_addresses + +logger = logging.getLogger(__name__) @pytest_asyncio.fixture(scope="module", name="k8s_controller") @@ -42,10 +33,11 @@ async def k8s_controller_fixture() -> AsyncGenerator[Controller, None]: @pytest_asyncio.fixture(scope="module", name="k8s_model") async def k8s_model_fixture(k8s_controller: Controller) -> AsyncGenerator[Model, None]: - """The machine model for jenkins agent machine charm.""" + """The machine model for K8s charms.""" k8s_model_name = f"k8s-{secrets.token_hex(2)}" model = await k8s_controller.add_model(k8s_model_name) - await model.connect(f"localhost:admin/{model.name}") + logger.info("Added model: %s", model.name) + await model.connect(f"microk8s:admin/{model.name}") yield model await k8s_controller.destroy_models( model.name, destroy_storage=True, force=True, max_wait=10 * 60 @@ -75,4 +67,14 @@ async def test_prometheus_metrics( await k8s_model.wait_for_idle(apps=[prometheus_app.name], timeout=300) await model.wait_for_idle(apps=[app_openstack_runner.name], timeout=300) + addresses = await get_model_unit_addresses(model=k8s_model, app_name=prometheus_app.name) + assert addresses, f"Unit addresses not found for {prometheus_app.name}" + address = addresses[0] + _get_active_target_patiently(prometheus_ip=address, target_name="github-runner") assert False + + +@retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=1, min=4, max=10)) +def _get_active_target_patiently(prometheus_ip: str, target_name: str): + query_targets = requests.get(f"http://{prometheus_ip}:9090/api/v1/targets", timeout=10).json() + assert target_name in query_targets["data"]["activeTargets"] From a22a73609c6a0f85c623207ec8c2702e4f10a68a Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Mon, 23 Jun 2025 19:29:50 +0000 Subject: [PATCH 08/75] test: test flavor label --- tests/integration/test_prometheus_metrics.py | 24 ++++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index 280e70d612..457bc3e701 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -70,11 +70,25 @@ async def test_prometheus_metrics( addresses = await get_model_unit_addresses(model=k8s_model, app_name=prometheus_app.name) assert addresses, f"Unit addresses not found for {prometheus_app.name}" address = addresses[0] - _get_active_target_patiently(prometheus_ip=address, target_name="github-runner") - assert False + + _assert_app_in_prometheus_target_patiently( + prometheus_ip=address, target_name=app_openstack_runner.name + ) + _assert_metrics_in_prometheus_labels_patiently(prometheus_ip=address, labels=["flavor"]) + + +@retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=1, min=4, max=10)) +def _assert_app_in_prometheus_target_patiently(prometheus_ip: str, target_name: str): + response = requests.get(f"http://{prometheus_ip}:9090/api/v1/targets", timeout=10).json() + assert any( + [ + target["labels"]["juju_charm"] == target_name + for target in response["data"]["activeTargets"] + ] + ) @retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=1, min=4, max=10)) -def _get_active_target_patiently(prometheus_ip: str, target_name: str): - query_targets = requests.get(f"http://{prometheus_ip}:9090/api/v1/targets", timeout=10).json() - assert target_name in query_targets["data"]["activeTargets"] +def _assert_metrics_in_prometheus_labels_patiently(prometheus_ip: str, labels: list[str]): + response = requests.get(f"http://{prometheus_ip}:9090/api/v1/labels", timeout=10).json() + assert set(labels).issubset(set(response["data"])) From 93aeb5e203b0527a8137fbb2b038dd7467a18c5c Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 24 Jun 2025 02:37:44 +0000 Subject: [PATCH 09/75] fix: enable allure test to collect reports before running tests --- .github/workflows/integration_test.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 47b0e1ae48..5578841229 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -39,9 +39,9 @@ jobs: self-hosted-runner-label: stg-private-endpoint tmate-debug: true tmate-timeout: 90 - # allure-report: - # if: ${{ (success() || failure()) && github.event_name == 'schedule' }} - # needs: - # - openstack-interface-tests-private-endpoint - # - openstack-integration-tests-private-endpoint - # uses: canonical/operator-workflows/.github/workflows/allure_report.yaml@main + allure-report: + if: ${{ (success() || failure()) && github.event_name == 'schedule' }} + needs: + - openstack-interface-tests-private-endpoint + - openstack-integration-tests-private-endpoint + uses: canonical/operator-workflows/.github/workflows/allure_report.yaml@main From 949ec41ceea18b90eae7c53abfca2ec4561a50ed Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 24 Jun 2025 02:42:10 +0000 Subject: [PATCH 10/75] remove dep --- .github/workflows/integration_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 5578841229..e37d7609a5 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -42,6 +42,6 @@ jobs: allure-report: if: ${{ (success() || failure()) && github.event_name == 'schedule' }} needs: - - openstack-interface-tests-private-endpoint + # - openstack-interface-tests-private-endpoint - openstack-integration-tests-private-endpoint uses: canonical/operator-workflows/.github/workflows/allure_report.yaml@main From 9649c53dc825338094b3334ced1fe85d86859083 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 24 Jun 2025 03:29:58 +0000 Subject: [PATCH 11/75] debug: try two tests --- .github/workflows/integration_test.yaml | 33 ++++++++++++------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index e37d7609a5..3d07c863c5 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -12,36 +12,35 @@ concurrency: cancel-in-progress: true jobs: - # openstack-interface-tests-private-endpoint: - # name: openstack interface test using private-endpoint - # uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main - # secrets: inherit - # with: - # juju-channel: 3.6/stable - # provider: lxd - # test-tox-env: integration-juju3.6 - # modules: '["test_runner_manager_openstack"]' - # extra-arguments: '--log-format="%(asctime)s %(levelname)s %(message)s"' - # self-hosted-runner: true - # self-hosted-runner-label: stg-private-endpoint + openstack-interface-tests-private-endpoint: + name: openstack interface test using private-endpoint + uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main + secrets: inherit + with: + juju-channel: 3.6/stable + pre-run-script: scripts/setup-integration-tests.sh + provider: lxd + test-tox-env: integration-juju3.6 + modules: '["test_runner_manager_openstack"]' + extra-arguments: '--log-format="%(asctime)s %(levelname)s %(message)s"' + self-hosted-runner: true + self-hosted-runner-label: stg-private-endpoint openstack-integration-tests-private-endpoint: name: Integration test using private-endpoint uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main secrets: inherit with: juju-channel: 3.6/stable - pre-run-script: tests/integration/setup-integration-tests.sh + pre-run-script: scripts/setup-integration-tests.sh provider: lxd test-tox-env: integration-juju3.6 - modules: '["test_prometheus_metrics"]' + modules: '["test_jobmanager_reactive", "test_prometheus_metrics"]' extra-arguments: '-m openstack --log-format="%(asctime)s %(levelname)s %(message)s"' self-hosted-runner: true self-hosted-runner-label: stg-private-endpoint - tmate-debug: true - tmate-timeout: 90 allure-report: if: ${{ (success() || failure()) && github.event_name == 'schedule' }} needs: - # - openstack-interface-tests-private-endpoint + - openstack-interface-tests-private-endpoint - openstack-integration-tests-private-endpoint uses: canonical/operator-workflows/.github/workflows/allure_report.yaml@main From bc76e0603f030bf518455ac9560bc26708e339c6 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 24 Jun 2025 05:18:09 +0000 Subject: [PATCH 12/75] test: mark openstack test --- tests/integration/test_prometheus_metrics.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index 457bc3e701..3612eb8683 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -7,6 +7,7 @@ import secrets from typing import AsyncGenerator +import pytest import pytest_asyncio import requests from juju.application import Application @@ -52,6 +53,7 @@ async def prometheus_app_fixture(k8s_model: Model): return prometheus_app +@pytest.mark.openstack async def test_prometheus_metrics( model: Model, k8s_model: Model, app_openstack_runner: Application, prometheus_app: Application ): From cda9318b59904eb947b96c3afff2da8380193e0c Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 24 Jun 2025 05:18:25 +0000 Subject: [PATCH 13/75] test: move integration test pre run script --- .github/workflows/integration_test.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 3d07c863c5..f81ff31d79 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -18,7 +18,6 @@ jobs: secrets: inherit with: juju-channel: 3.6/stable - pre-run-script: scripts/setup-integration-tests.sh provider: lxd test-tox-env: integration-juju3.6 modules: '["test_runner_manager_openstack"]' @@ -31,7 +30,7 @@ jobs: secrets: inherit with: juju-channel: 3.6/stable - pre-run-script: scripts/setup-integration-tests.sh + pre-run-script: tests/integration/setup-integration-tests.sh provider: lxd test-tox-env: integration-juju3.6 modules: '["test_jobmanager_reactive", "test_prometheus_metrics"]' From ea5dc147da5ee971fe5f6f9953a8435613463ce9 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 24 Jun 2025 05:43:55 +0000 Subject: [PATCH 14/75] fix: juju microk8s setup script --- tests/integration/setup-integration-tests.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/integration/setup-integration-tests.sh b/tests/integration/setup-integration-tests.sh index ffe4deb61c..c9b3c258e5 100755 --- a/tests/integration/setup-integration-tests.sh +++ b/tests/integration/setup-integration-tests.sh @@ -8,7 +8,8 @@ # The COS observability stack are deployed on K8s models. -TESTING_MODEL="$(juju switch)" +# save original controller that is used for testing +ORIGINAL_CONTROLLER=$(juju controllers --format json | jq '.controllers | keys | .[0]') echo "bootstrapping microk8s juju controller" sudo snap install microk8s --channel=1.32-strict/stable @@ -21,5 +22,5 @@ sudo microk8s enable hostpath-storage microk8s status --wait-ready juju bootstrap microk8s microk8s -echo "Switching to testing model" -juju switch $TESTING_MODEL +echo "Switching to original controller $ORIGINAL_CONTROLLER" +juju switch $ORIGINAL_CONTROLLER From 744769c05c71eb2f58e8bd76fcff2b5cbf1ea30a Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 24 Jun 2025 08:45:20 +0000 Subject: [PATCH 15/75] test: dont switch back to original controller --- tests/integration/setup-integration-tests.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/integration/setup-integration-tests.sh b/tests/integration/setup-integration-tests.sh index c9b3c258e5..4d776b8e1a 100755 --- a/tests/integration/setup-integration-tests.sh +++ b/tests/integration/setup-integration-tests.sh @@ -21,6 +21,3 @@ fi sudo microk8s enable hostpath-storage microk8s status --wait-ready juju bootstrap microk8s microk8s - -echo "Switching to original controller $ORIGINAL_CONTROLLER" -juju switch $ORIGINAL_CONTROLLER From 83b5cccad9db49f37aa8163ea5de4c76305c5bff Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 24 Jun 2025 15:46:34 +0000 Subject: [PATCH 16/75] switch back to original controller git push --- tests/integration/setup-integration-tests.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/setup-integration-tests.sh b/tests/integration/setup-integration-tests.sh index 4d776b8e1a..44bdf3b630 100755 --- a/tests/integration/setup-integration-tests.sh +++ b/tests/integration/setup-integration-tests.sh @@ -10,7 +10,6 @@ # save original controller that is used for testing ORIGINAL_CONTROLLER=$(juju controllers --format json | jq '.controllers | keys | .[0]') - echo "bootstrapping microk8s juju controller" sudo snap install microk8s --channel=1.32-strict/stable GROUP=snap_microk8s @@ -21,3 +20,5 @@ fi sudo microk8s enable hostpath-storage microk8s status --wait-ready juju bootstrap microk8s microk8s +unset JUJU_CONTROLLER +juju switch $ORIGINAL_CONTROLLER From 9c3fd26e1137416cce76bc83fd9b40c9fe338b70 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 24 Jun 2025 16:53:20 +0000 Subject: [PATCH 17/75] unset juju model env var --- tests/integration/setup-integration-tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/setup-integration-tests.sh b/tests/integration/setup-integration-tests.sh index 44bdf3b630..7ae15eb059 100755 --- a/tests/integration/setup-integration-tests.sh +++ b/tests/integration/setup-integration-tests.sh @@ -21,4 +21,5 @@ sudo microk8s enable hostpath-storage microk8s status --wait-ready juju bootstrap microk8s microk8s unset JUJU_CONTROLLER +unset JUJU_MODEL juju switch $ORIGINAL_CONTROLLER From b99962a48f0f546b9d99bf97ae6367e0147e6ff7 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Tue, 24 Jun 2025 17:21:35 +0000 Subject: [PATCH 18/75] raw json output --- tests/integration/setup-integration-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/setup-integration-tests.sh b/tests/integration/setup-integration-tests.sh index 7ae15eb059..4d32abe362 100755 --- a/tests/integration/setup-integration-tests.sh +++ b/tests/integration/setup-integration-tests.sh @@ -9,7 +9,7 @@ # The COS observability stack are deployed on K8s models. # save original controller that is used for testing -ORIGINAL_CONTROLLER=$(juju controllers --format json | jq '.controllers | keys | .[0]') +ORIGINAL_CONTROLLER=$(juju controllers --format json | jq -r '.controllers | keys | .[0]') echo "bootstrapping microk8s juju controller" sudo snap install microk8s --channel=1.32-strict/stable GROUP=snap_microk8s From dec85d2e2e01ea356463f9fb5854b20f4a9c6323 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 25 Jun 2025 02:47:37 +0000 Subject: [PATCH 19/75] debug --- .github/workflows/integration_test.yaml | 40 +++++++++++++------------ 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index f81ff31d79..47b0e1ae48 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -12,18 +12,18 @@ concurrency: cancel-in-progress: true jobs: - openstack-interface-tests-private-endpoint: - name: openstack interface test using private-endpoint - uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main - secrets: inherit - with: - juju-channel: 3.6/stable - provider: lxd - test-tox-env: integration-juju3.6 - modules: '["test_runner_manager_openstack"]' - extra-arguments: '--log-format="%(asctime)s %(levelname)s %(message)s"' - self-hosted-runner: true - self-hosted-runner-label: stg-private-endpoint + # openstack-interface-tests-private-endpoint: + # name: openstack interface test using private-endpoint + # uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main + # secrets: inherit + # with: + # juju-channel: 3.6/stable + # provider: lxd + # test-tox-env: integration-juju3.6 + # modules: '["test_runner_manager_openstack"]' + # extra-arguments: '--log-format="%(asctime)s %(levelname)s %(message)s"' + # self-hosted-runner: true + # self-hosted-runner-label: stg-private-endpoint openstack-integration-tests-private-endpoint: name: Integration test using private-endpoint uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main @@ -33,13 +33,15 @@ jobs: pre-run-script: tests/integration/setup-integration-tests.sh provider: lxd test-tox-env: integration-juju3.6 - modules: '["test_jobmanager_reactive", "test_prometheus_metrics"]' + modules: '["test_prometheus_metrics"]' extra-arguments: '-m openstack --log-format="%(asctime)s %(levelname)s %(message)s"' self-hosted-runner: true self-hosted-runner-label: stg-private-endpoint - allure-report: - if: ${{ (success() || failure()) && github.event_name == 'schedule' }} - needs: - - openstack-interface-tests-private-endpoint - - openstack-integration-tests-private-endpoint - uses: canonical/operator-workflows/.github/workflows/allure_report.yaml@main + tmate-debug: true + tmate-timeout: 90 + # allure-report: + # if: ${{ (success() || failure()) && github.event_name == 'schedule' }} + # needs: + # - openstack-interface-tests-private-endpoint + # - openstack-integration-tests-private-endpoint + # uses: canonical/operator-workflows/.github/workflows/allure_report.yaml@main From d757a98d686b2b2f91841d189f93c3b34a2835b5 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 25 Jun 2025 04:57:14 +0000 Subject: [PATCH 20/75] fix: label name --- .../src/github_runner_manager/manager/runner_manager.py | 2 +- .../src/github_runner_manager/manager/runner_scaler.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/github-runner-manager/src/github_runner_manager/manager/runner_manager.py b/github-runner-manager/src/github_runner_manager/manager/runner_manager.py index b2ec2530c0..b6e143a9f9 100644 --- a/github-runner-manager/src/github_runner_manager/manager/runner_manager.py +++ b/github-runner-manager/src/github_runner_manager/manager/runner_manager.py @@ -301,7 +301,7 @@ def _cleanup_resources( ) cloud_runners_to_delete = cloud_runners_to_delete[:maximum_runners_to_delete] - CLEANED_RUNNERS_TOTAL.labels("flavor", len(cloud_runners_to_delete)) + CLEANED_RUNNERS_TOTAL.labels(self.manager_name).inc(len(cloud_runners_to_delete)) return self._delete_cloud_runners( cloud_runners_to_delete, runners_health_response.requested_runners, diff --git a/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py b/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py index 33b5fc7a6a..50eb77b85c 100644 --- a/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py +++ b/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py @@ -27,7 +27,6 @@ from github_runner_manager.metrics import events as metric_events from github_runner_manager.metrics.reconcile import ( BUSY_RUNNERS_COUNT, - CLEANED_RUNNERS_TOTAL, EXPECTED_RUNNERS_COUNT, IDLE_RUNNERS_COUNT, RECONCILE_DURATION_SECONDS, @@ -249,8 +248,8 @@ def get_runner_info(self) -> RunnerInfo: offline += 1 case _: unknown += 1 - BUSY_RUNNERS_COUNT.labels("flavor", self._manager.manager_name).set(busy) - IDLE_RUNNERS_COUNT.labels("flavor", self._manager.manager_name).set(online) + BUSY_RUNNERS_COUNT.labels(self._manager.manager_name).set(busy) + IDLE_RUNNERS_COUNT.labels(self._manager.manager_name).set(online) return RunnerInfo( online=online, busy=busy, @@ -329,7 +328,7 @@ def reconcile(self) -> int: flavor=self._manager.manager_name, expected_runner_quantity=expected_runner_quantity, ) - RECONCILE_DURATION_SECONDS.labels("flavor", self._manager.manager_name).observe( + RECONCILE_DURATION_SECONDS.labels(self._manager.manager_name).observe( end_timestamp - start_timestamp ) _issue_reconciliation_metric(reconcile_metric_data) From edde27eb8f30fe1606c9a74a8e654f80220ae67b Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 25 Jun 2025 05:37:37 +0000 Subject: [PATCH 21/75] test: fix offer name relation --- tests/integration/test_prometheus_metrics.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index 3612eb8683..83939f5567 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -50,6 +50,7 @@ async def k8s_model_fixture(k8s_controller: Controller) -> AsyncGenerator[Model, async def prometheus_app_fixture(k8s_model: Model): """Deploy prometheus charm.""" prometheus_app: Application = await k8s_model.deploy("prometheus-k8s", channel="1/stable") + await k8s_model.wait_for_idle(apps=[prometheus_app.name]) return prometheus_app @@ -62,9 +63,10 @@ async def test_prometheus_metrics( act: when GitHub runner is integrated. assert: the datasource is registered and basic metrics are available. """ - offer = await k8s_model.create_offer(f"{prometheus_app.name}:metrics-endpoint", "metrics") + offer_name = "metrics" + await k8s_model.create_offer(f"{prometheus_app.name}:metrics-endpoint", offer_name) await model.integrate( - app_openstack_runner.name, f"microk8s:admin/{k8s_model.name}.{offer.name}" + app_openstack_runner.name, f"microk8s:admin/{k8s_model.name}.{offer_name}" ) await k8s_model.wait_for_idle(apps=[prometheus_app.name], timeout=300) await model.wait_for_idle(apps=[app_openstack_runner.name], timeout=300) From ff43e8aee43d2f8e1a57d3a9dfb4416b2efef911 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 25 Jun 2025 06:20:13 +0000 Subject: [PATCH 22/75] test: do not use juju controller from env var --- tests/integration/setup-integration-tests.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/setup-integration-tests.sh b/tests/integration/setup-integration-tests.sh index 4d32abe362..5504571582 100755 --- a/tests/integration/setup-integration-tests.sh +++ b/tests/integration/setup-integration-tests.sh @@ -19,7 +19,8 @@ if [ $(id -gn) != $GROUP ]; then fi sudo microk8s enable hostpath-storage microk8s status --wait-ready -juju bootstrap microk8s microk8s + unset JUJU_CONTROLLER unset JUJU_MODEL +juju bootstrap microk8s microk8s juju switch $ORIGINAL_CONTROLLER From a4debe22b2ec3d124bcd0e86fc8413db00b601a9 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 25 Jun 2025 06:21:29 +0000 Subject: [PATCH 23/75] test: do not raise on error while waiting for idle --- tests/integration/test_prometheus_metrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index 83939f5567..c8de99e605 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -68,8 +68,8 @@ async def test_prometheus_metrics( await model.integrate( app_openstack_runner.name, f"microk8s:admin/{k8s_model.name}.{offer_name}" ) - await k8s_model.wait_for_idle(apps=[prometheus_app.name], timeout=300) - await model.wait_for_idle(apps=[app_openstack_runner.name], timeout=300) + await k8s_model.wait_for_idle(apps=[prometheus_app.name], raise_on_error=False, timeout=300) + await model.wait_for_idle(apps=[app_openstack_runner.name], raise_on_error=False, timeout=300) addresses = await get_model_unit_addresses(model=k8s_model, app_name=prometheus_app.name) assert addresses, f"Unit addresses not found for {prometheus_app.name}" From 3bf6c5acc3e1fd0a7609ba74513d607494d3d242 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 25 Jun 2025 08:15:06 +0000 Subject: [PATCH 24/75] test: model integrate & consume --- tests/integration/test_prometheus_metrics.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index c8de99e605..43eb44a564 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -65,9 +65,8 @@ async def test_prometheus_metrics( """ offer_name = "metrics" await k8s_model.create_offer(f"{prometheus_app.name}:metrics-endpoint", offer_name) - await model.integrate( - app_openstack_runner.name, f"microk8s:admin/{k8s_model.name}.{offer_name}" - ) + await model.consume(f"microk8s:admin/{k8s_model.name}.{offer_name}") + await model.integrate(app_openstack_runner.name, offer_name) await k8s_model.wait_for_idle(apps=[prometheus_app.name], raise_on_error=False, timeout=300) await model.wait_for_idle(apps=[app_openstack_runner.name], raise_on_error=False, timeout=300) From 4d65ffc6c2067b8028ab1b8725cde46bd09b51f2 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 25 Jun 2025 11:53:25 +0000 Subject: [PATCH 25/75] test: keep microk8s model --- tests/integration/test_prometheus_metrics.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index 43eb44a564..96b98593a5 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -13,6 +13,7 @@ from juju.application import Application from juju.controller import Controller from juju.model import Model +from pytest_operator.plugin import OpsTest from tenacity import retry, stop_after_attempt, wait_exponential from tests.integration.helpers.common import get_model_unit_addresses @@ -33,17 +34,20 @@ async def k8s_controller_fixture() -> AsyncGenerator[Controller, None]: @pytest_asyncio.fixture(scope="module", name="k8s_model") -async def k8s_model_fixture(k8s_controller: Controller) -> AsyncGenerator[Model, None]: +async def k8s_model_fixture( + ops_test: OpsTest, k8s_controller: Controller +) -> AsyncGenerator[Model, None]: """The machine model for K8s charms.""" k8s_model_name = f"k8s-{secrets.token_hex(2)}" model = await k8s_controller.add_model(k8s_model_name) logger.info("Added model: %s", model.name) await model.connect(f"microk8s:admin/{model.name}") yield model + if ops_test.keep_model: + return await k8s_controller.destroy_models( model.name, destroy_storage=True, force=True, max_wait=10 * 60 ) - await model.disconnect() @pytest_asyncio.fixture(scope="module", name="prometheus_app") From eb13f7a74b07a202623d7839a05f245774de5f77 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 25 Jun 2025 12:41:25 +0000 Subject: [PATCH 26/75] test: deploy cos agent --- tests/integration/test_prometheus_metrics.py | 25 ++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index 96b98593a5..c0c07be732 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -58,9 +58,24 @@ async def prometheus_app_fixture(k8s_model: Model): return prometheus_app +@pytest_asyncio.fixture(scope="module", name="openstack_app_cos_agent") +async def openstack_app_cos_agent_fixture(app_openstack_runner: Application): + """Deploy cos-agent subordinate charm on OpenStack runner application.""" + model = app_openstack_runner.model + grafana_agent = await model.deploy("grafana-agent") + await model.relate(grafana_agent.name, app_openstack_runner.name) + await model.wait_for_idle( + apps=[grafana_agent.name, app_openstack_runner.name], raise_on_error=False + ) + return app_openstack_runner + + @pytest.mark.openstack async def test_prometheus_metrics( - model: Model, k8s_model: Model, app_openstack_runner: Application, prometheus_app: Application + model: Model, + k8s_model: Model, + openstack_app_cos_agent_fixture: Application, + prometheus_app: Application, ): """ arrange: given a prometheus charm application. @@ -70,16 +85,18 @@ async def test_prometheus_metrics( offer_name = "metrics" await k8s_model.create_offer(f"{prometheus_app.name}:metrics-endpoint", offer_name) await model.consume(f"microk8s:admin/{k8s_model.name}.{offer_name}") - await model.integrate(app_openstack_runner.name, offer_name) + await model.integrate(openstack_app_cos_agent_fixture.name, offer_name) await k8s_model.wait_for_idle(apps=[prometheus_app.name], raise_on_error=False, timeout=300) - await model.wait_for_idle(apps=[app_openstack_runner.name], raise_on_error=False, timeout=300) + await model.wait_for_idle( + apps=[openstack_app_cos_agent_fixture.name], raise_on_error=False, timeout=300 + ) addresses = await get_model_unit_addresses(model=k8s_model, app_name=prometheus_app.name) assert addresses, f"Unit addresses not found for {prometheus_app.name}" address = addresses[0] _assert_app_in_prometheus_target_patiently( - prometheus_ip=address, target_name=app_openstack_runner.name + prometheus_ip=address, target_name=openstack_app_cos_agent_fixture.name ) _assert_metrics_in_prometheus_labels_patiently(prometheus_ip=address, labels=["flavor"]) From 61cad416d768ad718a926bbdebcc39a0300cfeee Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 25 Jun 2025 12:44:54 +0000 Subject: [PATCH 27/75] test: add grafana agent series --- tests/integration/test_prometheus_metrics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index c0c07be732..b75d45569e 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -62,7 +62,8 @@ async def prometheus_app_fixture(k8s_model: Model): async def openstack_app_cos_agent_fixture(app_openstack_runner: Application): """Deploy cos-agent subordinate charm on OpenStack runner application.""" model = app_openstack_runner.model - grafana_agent = await model.deploy("grafana-agent") + series = await app_openstack_runner.get_series() + grafana_agent = await model.deploy("grafana-agent", base=series) await model.relate(grafana_agent.name, app_openstack_runner.name) await model.wait_for_idle( apps=[grafana_agent.name, app_openstack_runner.name], raise_on_error=False From f28f81402697cd90160517cb6b50791ec3ea10cc Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 25 Jun 2025 13:50:07 +0000 Subject: [PATCH 28/75] test: fix typo --- tests/integration/test_prometheus_metrics.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index b75d45569e..c24863de79 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -75,7 +75,7 @@ async def openstack_app_cos_agent_fixture(app_openstack_runner: Application): async def test_prometheus_metrics( model: Model, k8s_model: Model, - openstack_app_cos_agent_fixture: Application, + openstack_app_cos_agent: Application, prometheus_app: Application, ): """ @@ -86,10 +86,10 @@ async def test_prometheus_metrics( offer_name = "metrics" await k8s_model.create_offer(f"{prometheus_app.name}:metrics-endpoint", offer_name) await model.consume(f"microk8s:admin/{k8s_model.name}.{offer_name}") - await model.integrate(openstack_app_cos_agent_fixture.name, offer_name) + await model.integrate(openstack_app_cos_agent.name, offer_name) await k8s_model.wait_for_idle(apps=[prometheus_app.name], raise_on_error=False, timeout=300) await model.wait_for_idle( - apps=[openstack_app_cos_agent_fixture.name], raise_on_error=False, timeout=300 + apps=[openstack_app_cos_agent.name], raise_on_error=False, timeout=300 ) addresses = await get_model_unit_addresses(model=k8s_model, app_name=prometheus_app.name) @@ -97,7 +97,7 @@ async def test_prometheus_metrics( address = addresses[0] _assert_app_in_prometheus_target_patiently( - prometheus_ip=address, target_name=openstack_app_cos_agent_fixture.name + prometheus_ip=address, target_name=openstack_app_cos_agent.name ) _assert_metrics_in_prometheus_labels_patiently(prometheus_ip=address, labels=["flavor"]) From 9a4b9563d0d010e3686cd8fa2c80318e9fe8b199 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 25 Jun 2025 14:37:09 +0000 Subject: [PATCH 29/75] test: assign grafana agent channel --- tests/integration/test_prometheus_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index c24863de79..ed58d5e09c 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -63,7 +63,7 @@ async def openstack_app_cos_agent_fixture(app_openstack_runner: Application): """Deploy cos-agent subordinate charm on OpenStack runner application.""" model = app_openstack_runner.model series = await app_openstack_runner.get_series() - grafana_agent = await model.deploy("grafana-agent", base=series) + grafana_agent = await model.deploy("grafana-agent", channel="1/stable", base=series) await model.relate(grafana_agent.name, app_openstack_runner.name) await model.wait_for_idle( apps=[grafana_agent.name, app_openstack_runner.name], raise_on_error=False From 0fb23f00688914d31909c516e42dffe9c4a82ed5 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 25 Jun 2025 15:20:51 +0000 Subject: [PATCH 30/75] test: juju ops lib is ... --- tests/integration/test_prometheus_metrics.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index ed58d5e09c..649a0645c1 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -62,8 +62,10 @@ async def prometheus_app_fixture(k8s_model: Model): async def openstack_app_cos_agent_fixture(app_openstack_runner: Application): """Deploy cos-agent subordinate charm on OpenStack runner application.""" model = app_openstack_runner.model - series = await app_openstack_runner.get_series() - grafana_agent = await model.deploy("grafana-agent", channel="1/stable", base=series) + # do not use series, it returns jammy and isn't compatible with base tag which only accepts + # ubuntu@. Hence hard code the base for now for testing only. + # series = await app_openstack_runner.get_series() + grafana_agent = await model.deploy("grafana-agent", channel="1/stable", base="ubuntu@22.04") await model.relate(grafana_agent.name, app_openstack_runner.name) await model.wait_for_idle( apps=[grafana_agent.name, app_openstack_runner.name], raise_on_error=False From 61ceb1fd6ac9b6cce7e51fcf45e60e7a49579217 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 25 Jun 2025 16:12:06 +0000 Subject: [PATCH 31/75] test: use series --- tests/integration/test_prometheus_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index 649a0645c1..f7053f45a9 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -65,7 +65,7 @@ async def openstack_app_cos_agent_fixture(app_openstack_runner: Application): # do not use series, it returns jammy and isn't compatible with base tag which only accepts # ubuntu@. Hence hard code the base for now for testing only. # series = await app_openstack_runner.get_series() - grafana_agent = await model.deploy("grafana-agent", channel="1/stable", base="ubuntu@22.04") + grafana_agent = await model.deploy("grafana-agent", channel="1/stable", series="ubuntu@22.04") await model.relate(grafana_agent.name, app_openstack_runner.name) await model.wait_for_idle( apps=[grafana_agent.name, app_openstack_runner.name], raise_on_error=False From e2a558b203c58abedecca52f710d9ad829562038 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 26 Jun 2025 03:46:07 +0000 Subject: [PATCH 32/75] test: swap out ops_test with jubilant --- tests/integration/helpers/common.py | 24 ---- tests/integration/test_prometheus_metrics.py | 118 +++++-------------- 2 files changed, 29 insertions(+), 113 deletions(-) diff --git a/tests/integration/helpers/common.py b/tests/integration/helpers/common.py index 5109c143a9..1f0c5f5d1e 100644 --- a/tests/integration/helpers/common.py +++ b/tests/integration/helpers/common.py @@ -494,27 +494,3 @@ async def get_github_runner_manager_service_log(unit: Unit) -> str: assert return_code == 0, f"Get log with cat {log_file_path} failed with: {stderr}" assert stdout is not None return stdout - - -async def get_model_unit_addresses(model: Model, app_name: str) -> list[str]: - """Extract the address of a given unit. - - Args: - model: Juju model - app_name: Juju application name - - Returns: - the IP address of the Jenkins unit. - """ - status: FullStatus = await model.get_status() - # mypy cannot infer the type ApplicationStatus but thinks its the base class type "Type". - application_status: ApplicationStatus | None = status.applications[app_name] # type: ignore - assert application_status, f"Application status {app_name} not found in {status}" - # mypy cannot infer the type UnitStatus but thinks its the base class type "Type". - unit_status_map: dict[typing.Any, UnitStatus | None] = application_status.units # type: ignore - units_statuses: list[UnitStatus | None] = list(unit_status_map.values()) - return [ - str(unit_status.address) - for unit_status in units_statuses - if unit_status and unit_status.address - ] diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index f7053f45a9..5832e9abb0 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -4,118 +4,58 @@ """Module for collecting metrics related to the reconciliation process.""" import logging -import secrets -from typing import AsyncGenerator +from typing import Generator, cast +import jubilant import pytest import pytest_asyncio -import requests from juju.application import Application -from juju.controller import Controller -from juju.model import Model -from pytest_operator.plugin import OpsTest -from tenacity import retry, stop_after_attempt, wait_exponential - -from tests.integration.helpers.common import get_model_unit_addresses logger = logging.getLogger(__name__) -@pytest_asyncio.fixture(scope="module", name="k8s_controller") -async def k8s_controller_fixture() -> AsyncGenerator[Controller, None]: - """The k8s controller. - - See scripts/setup-integration-tests.sh. - """ - controller = Controller() - await controller.connect_controller("microk8s") - yield controller - await controller.disconnect() - - -@pytest_asyncio.fixture(scope="module", name="k8s_model") -async def k8s_model_fixture( - ops_test: OpsTest, k8s_controller: Controller -) -> AsyncGenerator[Model, None]: +@pytest_asyncio.fixture(scope="module", name="k8s_juju") +def k8s_juju_fixture( + juju: jubilant.Juju, request: pytest.FixtureRequest +) -> Generator[jubilant.Juju, None]: """The machine model for K8s charms.""" - k8s_model_name = f"k8s-{secrets.token_hex(2)}" - model = await k8s_controller.add_model(k8s_model_name) - logger.info("Added model: %s", model.name) - await model.connect(f"microk8s:admin/{model.name}") - yield model - if ops_test.keep_model: - return - await k8s_controller.destroy_models( - model.name, destroy_storage=True, force=True, max_wait=10 * 60 - ) + keep_models = cast(bool, request.config.getoption("--keep-models")) + with jubilant.temp_model(keep=keep_models, controller="microk8s") as juju: + yield juju -@pytest_asyncio.fixture(scope="module", name="prometheus_app") -async def prometheus_app_fixture(k8s_model: Model): +@pytest.fixture(scope="module", name="prometheus_app") +def prometheus_app_fixture(k8s_juju: jubilant.Juju): """Deploy prometheus charm.""" - prometheus_app: Application = await k8s_model.deploy("prometheus-k8s", channel="1/stable") - await k8s_model.wait_for_idle(apps=[prometheus_app.name]) - return prometheus_app + k8s_juju.deploy("grafana-k8s", channel="1/stable") + k8s_juju.wait(lambda status: jubilant.all_active(status, "grafana-k8s")) + k8s_juju.offer("grafana-k8s", endpoint="grafana-dashboards-provider") + return k8s_juju.status().apps["prometheus-k8s"] -@pytest_asyncio.fixture(scope="module", name="openstack_app_cos_agent") -async def openstack_app_cos_agent_fixture(app_openstack_runner: Application): +@pytest.fixture(scope="module", name="openstack_app_cos_agent") +def openstack_app_cos_agent_fixture(juju: jubilant.Juju, app_openstack_runner: Application): """Deploy cos-agent subordinate charm on OpenStack runner application.""" - model = app_openstack_runner.model - # do not use series, it returns jammy and isn't compatible with base tag which only accepts - # ubuntu@. Hence hard code the base for now for testing only. - # series = await app_openstack_runner.get_series() - grafana_agent = await model.deploy("grafana-agent", channel="1/stable", series="ubuntu@22.04") - await model.relate(grafana_agent.name, app_openstack_runner.name) - await model.wait_for_idle( - apps=[grafana_agent.name, app_openstack_runner.name], raise_on_error=False + juju.deploy("grafana-agent", channel="1/stable", base="ubuntu@22.04") + juju.integrate(app_openstack_runner.name, "grafana-agent") + juju.wait( + lambda status: jubilant.all_agents_idle(status, app_openstack_runner.name, "grafana-agent") ) return app_openstack_runner @pytest.mark.openstack -async def test_prometheus_metrics( - model: Model, - k8s_model: Model, - openstack_app_cos_agent: Application, - prometheus_app: Application, +def test_prometheus_metrics( + juju: jubilant.Juju, k8s_juju: jubilant.Juju, openstack_app_cos_agent: Application ): """ arrange: given a prometheus charm application. act: when GitHub runner is integrated. assert: the datasource is registered and basic metrics are available. """ - offer_name = "metrics" - await k8s_model.create_offer(f"{prometheus_app.name}:metrics-endpoint", offer_name) - await model.consume(f"microk8s:admin/{k8s_model.name}.{offer_name}") - await model.integrate(openstack_app_cos_agent.name, offer_name) - await k8s_model.wait_for_idle(apps=[prometheus_app.name], raise_on_error=False, timeout=300) - await model.wait_for_idle( - apps=[openstack_app_cos_agent.name], raise_on_error=False, timeout=300 - ) - - addresses = await get_model_unit_addresses(model=k8s_model, app_name=prometheus_app.name) - assert addresses, f"Unit addresses not found for {prometheus_app.name}" - address = addresses[0] - - _assert_app_in_prometheus_target_patiently( - prometheus_ip=address, target_name=openstack_app_cos_agent.name - ) - _assert_metrics_in_prometheus_labels_patiently(prometheus_ip=address, labels=["flavor"]) - - -@retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=1, min=4, max=10)) -def _assert_app_in_prometheus_target_patiently(prometheus_ip: str, target_name: str): - response = requests.get(f"http://{prometheus_ip}:9090/api/v1/targets", timeout=10).json() - assert any( - [ - target["labels"]["juju_charm"] == target_name - for target in response["data"]["activeTargets"] - ] - ) - - -@retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=1, min=4, max=10)) -def _assert_metrics_in_prometheus_labels_patiently(prometheus_ip: str, labels: list[str]): - response = requests.get(f"http://{prometheus_ip}:9090/api/v1/labels", timeout=10).json() - assert set(labels).issubset(set(response["data"])) + offer_name = "grafana-k8s" + juju.cli(f"consume microk8s:{k8s_juju.model}.{offer_name}") + juju.integrate(openstack_app_cos_agent.name, offer_name) + juju.wait(lambda status: jubilant.all_agents_idle(status, openstack_app_cos_agent.name)) + grafana_ip = k8s_juju.status().apps["grafana-k8s"].units["grafana-k8s/0"].address + assert False, grafana_ip From 580aa6888fd442ddf9a3b5c4d7de53530c8ababc Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 26 Jun 2025 04:07:24 +0000 Subject: [PATCH 33/75] test: generator type hint fix --- tests/integration/test_prometheus_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index 5832e9abb0..1bfd25289d 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -17,7 +17,7 @@ @pytest_asyncio.fixture(scope="module", name="k8s_juju") def k8s_juju_fixture( juju: jubilant.Juju, request: pytest.FixtureRequest -) -> Generator[jubilant.Juju, None]: +) -> Generator[jubilant.Juju, None, None]: """The machine model for K8s charms.""" keep_models = cast(bool, request.config.getoption("--keep-models")) with jubilant.temp_model(keep=keep_models, controller="microk8s") as juju: From 09fc69a4e5399bb204f28b37f380a8eba9434d4d Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 26 Jun 2025 06:25:27 +0000 Subject: [PATCH 34/75] test: add prometheus datasource --- tests/integration/test_prometheus_metrics.py | 50 ++++++++++++++++---- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index 1bfd25289d..a34736f5df 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -9,6 +9,7 @@ import jubilant import pytest import pytest_asyncio +from jubilant.statustypes import AppStatus from juju.application import Application logger = logging.getLogger(__name__) @@ -27,12 +28,30 @@ def k8s_juju_fixture( @pytest.fixture(scope="module", name="prometheus_app") def prometheus_app_fixture(k8s_juju: jubilant.Juju): """Deploy prometheus charm.""" - k8s_juju.deploy("grafana-k8s", channel="1/stable") - k8s_juju.wait(lambda status: jubilant.all_active(status, "grafana-k8s")) - k8s_juju.offer("grafana-k8s", endpoint="grafana-dashboards-provider") + k8s_juju.deploy("prometheus-k8s", channel="1/stable") + k8s_juju.wait(lambda status: jubilant.all_active(status, "prometheus-k8s")) + k8s_juju.offer("prometheus-k8s", endpoint="receive-remote-write") return k8s_juju.status().apps["prometheus-k8s"] +@pytest.fixture(scope="module", name="grafana_app") +def grafana_app_fixture(k8s_juju: jubilant.Juju): + """Deploy prometheus charm.""" + k8s_juju.deploy("grafana-k8s", channel="1/stable") + k8s_juju.integrate("grafana-k8s:grafana-source", "prometheus-k8s:grafana-source") + k8s_juju.wait(lambda status: jubilant.all_active(status, "grafana-k8s", "prometheus-k8s")) + k8s_juju.offer("grafana-k8s", endpoint="grafana-dashboard") + return k8s_juju.status().apps["grafana-k8s"] + + +@pytest.fixture(scope="module", name="grafana_password") +def grafana_password_fixture(k8s_juju: jubilant.Juju, grafana_app: AppStatus): + """Get Grafana dashboard password.""" + unit = next(iter(grafana_app.units.keys())) + result = k8s_juju.run(unit, "get-admin-password") + return result.results["admin-password"] + + @pytest.fixture(scope="module", name="openstack_app_cos_agent") def openstack_app_cos_agent_fixture(juju: jubilant.Juju, app_openstack_runner: Application): """Deploy cos-agent subordinate charm on OpenStack runner application.""" @@ -46,16 +65,27 @@ def openstack_app_cos_agent_fixture(juju: jubilant.Juju, app_openstack_runner: A @pytest.mark.openstack def test_prometheus_metrics( - juju: jubilant.Juju, k8s_juju: jubilant.Juju, openstack_app_cos_agent: Application + juju: jubilant.Juju, + k8s_juju: jubilant.Juju, + openstack_app_cos_agent: Application, + grafana_app: AppStatus, + grafana_password: str, ): """ arrange: given a prometheus charm application. act: when GitHub runner is integrated. assert: the datasource is registered and basic metrics are available. """ - offer_name = "grafana-k8s" - juju.cli(f"consume microk8s:{k8s_juju.model}.{offer_name}") - juju.integrate(openstack_app_cos_agent.name, offer_name) - juju.wait(lambda status: jubilant.all_agents_idle(status, openstack_app_cos_agent.name)) - grafana_ip = k8s_juju.status().apps["grafana-k8s"].units["grafana-k8s/0"].address - assert False, grafana_ip + prometheus_offer_name = "prometheus-k8s" + grafana_offer_name = "grafana-k8s" + juju.cli(f"consume microk8s:{k8s_juju.model}.{prometheus_offer_name}") + juju.cli(f"consume microk8s:{k8s_juju.model}.{grafana_offer_name}") + juju.integrate("grafana-agent", prometheus_offer_name) + juju.integrate("grafana-agent", grafana_offer_name) + juju.wait( + lambda status: jubilant.all_agents_idle( + status, openstack_app_cos_agent.name, "grafana-agent" + ) + ) + grafana_ip = grafana_app.units["grafana-k8s/0"].address + assert False, f"admin:{grafana_password}@{grafana_ip}:3000/" From 717af98e0a27182ed19571da5144c2beb10651b2 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 26 Jun 2025 07:03:04 +0000 Subject: [PATCH 35/75] test: separate micrlk8s juju from lxd juju --- tests/integration/test_prometheus_metrics.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index a34736f5df..eb0fc570de 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -16,9 +16,7 @@ @pytest_asyncio.fixture(scope="module", name="k8s_juju") -def k8s_juju_fixture( - juju: jubilant.Juju, request: pytest.FixtureRequest -) -> Generator[jubilant.Juju, None, None]: +def k8s_juju_fixture(request: pytest.FixtureRequest) -> Generator[jubilant.Juju, None, None]: """The machine model for K8s charms.""" keep_models = cast(bool, request.config.getoption("--keep-models")) with jubilant.temp_model(keep=keep_models, controller="microk8s") as juju: From 5639311ea2f4ec914b793be69e55bc169368e97f Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 26 Jun 2025 07:13:30 +0000 Subject: [PATCH 36/75] test: add controller prefix to model --- tests/integration/test_prometheus_metrics.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index eb0fc570de..47c1b02c73 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -26,6 +26,7 @@ def k8s_juju_fixture(request: pytest.FixtureRequest) -> Generator[jubilant.Juju, @pytest.fixture(scope="module", name="prometheus_app") def prometheus_app_fixture(k8s_juju: jubilant.Juju): """Deploy prometheus charm.""" + k8s_juju.model = f"microk8s:{k8s_juju.model}" k8s_juju.deploy("prometheus-k8s", channel="1/stable") k8s_juju.wait(lambda status: jubilant.all_active(status, "prometheus-k8s")) k8s_juju.offer("prometheus-k8s", endpoint="receive-remote-write") @@ -35,6 +36,7 @@ def prometheus_app_fixture(k8s_juju: jubilant.Juju): @pytest.fixture(scope="module", name="grafana_app") def grafana_app_fixture(k8s_juju: jubilant.Juju): """Deploy prometheus charm.""" + k8s_juju.model = f"microk8s:{k8s_juju.model}" k8s_juju.deploy("grafana-k8s", channel="1/stable") k8s_juju.integrate("grafana-k8s:grafana-source", "prometheus-k8s:grafana-source") k8s_juju.wait(lambda status: jubilant.all_active(status, "grafana-k8s", "prometheus-k8s")) @@ -45,6 +47,7 @@ def grafana_app_fixture(k8s_juju: jubilant.Juju): @pytest.fixture(scope="module", name="grafana_password") def grafana_password_fixture(k8s_juju: jubilant.Juju, grafana_app: AppStatus): """Get Grafana dashboard password.""" + k8s_juju.model = f"microk8s:{k8s_juju.model}" unit = next(iter(grafana_app.units.keys())) result = k8s_juju.run(unit, "get-admin-password") return result.results["admin-password"] @@ -74,6 +77,7 @@ def test_prometheus_metrics( act: when GitHub runner is integrated. assert: the datasource is registered and basic metrics are available. """ + k8s_juju.model = f"microk8s:{k8s_juju.model}" prometheus_offer_name = "prometheus-k8s" grafana_offer_name = "grafana-k8s" juju.cli(f"consume microk8s:{k8s_juju.model}.{prometheus_offer_name}") From 62ee73dea64d51b4bc5de7a4f27796c1a56bc695 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 26 Jun 2025 07:16:20 +0000 Subject: [PATCH 37/75] test: store model & controller name --- tests/integration/conftest.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 0686c01da7..bcce3516c3 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -9,7 +9,7 @@ import string from pathlib import Path from time import sleep -from typing import Any, AsyncIterator, Generator, Iterator, Optional, cast +from typing import Any, AsyncGenerator, AsyncIterator, Generator, Iterator, Optional, cast import jubilant import nest_asyncio @@ -748,8 +748,10 @@ async def instance_helper_fixture(request: pytest.FixtureRequest) -> OpenStackIn return OpenStackInstanceHelper(openstack_connection=openstack_connection) -@pytest.fixture(scope="module") -def juju(request: pytest.FixtureRequest, model: Model) -> Generator[jubilant.Juju, None, None]: +@pytest_asyncio.fixture(scope="module") +async def juju( + request: pytest.FixtureRequest, model: Model +) -> AsyncGenerator[jubilant.Juju, None]: """Pytest fixture that wraps :meth:`jubilant.with_model`.""" def show_debug_log(juju: jubilant.Juju): @@ -762,14 +764,16 @@ def show_debug_log(juju: jubilant.Juju): log = juju.debug_log(limit=1000) print(log, end="") + controller = await model.get_controller() if model: - juju = jubilant.Juju(model=model.name) + juju = jubilant.Juju(model=f"{controller.controller_name}:{model.name}") yield juju show_debug_log(juju) return keep_models = cast(bool, request.config.getoption("--keep-models")) - with jubilant.temp_model(keep=keep_models) as juju: + with jubilant.temp_model(keep=keep_models, controller=controller.controller_name) as juju: + juju.model = f"{controller.controller_name}:{juju.model}" juju.wait_timeout = 10 * 60 yield juju show_debug_log(juju) From d187de6373df21c2587fb57a3a95835aee274a96 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 26 Jun 2025 07:32:33 +0000 Subject: [PATCH 38/75] test: remove duplicate model naming w/ controller name prefix --- tests/integration/conftest.py | 3 +++ tests/integration/test_prometheus_metrics.py | 13 +++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index bcce3516c3..f5ff30658f 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -766,6 +766,9 @@ def show_debug_log(juju: jubilant.Juju): controller = await model.get_controller() if model: + # Currently juju has no way of switching controller context, this is required to operate + # in the right controller's right model when using multiple controllers. + # See: https://github.com/canonical/jubilant/issues/158 juju = jubilant.Juju(model=f"{controller.controller_name}:{model.name}") yield juju show_debug_log(juju) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index 47c1b02c73..2ed38efed2 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -20,13 +20,16 @@ def k8s_juju_fixture(request: pytest.FixtureRequest) -> Generator[jubilant.Juju, """The machine model for K8s charms.""" keep_models = cast(bool, request.config.getoption("--keep-models")) with jubilant.temp_model(keep=keep_models, controller="microk8s") as juju: + # Currently juju has no way of switching controller context, this is required to operate + # in the right controller's right model when using multiple controllers. + # See: https://github.com/canonical/jubilant/issues/158 + juju.model = f"microk8s:{juju.model}" yield juju @pytest.fixture(scope="module", name="prometheus_app") def prometheus_app_fixture(k8s_juju: jubilant.Juju): """Deploy prometheus charm.""" - k8s_juju.model = f"microk8s:{k8s_juju.model}" k8s_juju.deploy("prometheus-k8s", channel="1/stable") k8s_juju.wait(lambda status: jubilant.all_active(status, "prometheus-k8s")) k8s_juju.offer("prometheus-k8s", endpoint="receive-remote-write") @@ -36,7 +39,6 @@ def prometheus_app_fixture(k8s_juju: jubilant.Juju): @pytest.fixture(scope="module", name="grafana_app") def grafana_app_fixture(k8s_juju: jubilant.Juju): """Deploy prometheus charm.""" - k8s_juju.model = f"microk8s:{k8s_juju.model}" k8s_juju.deploy("grafana-k8s", channel="1/stable") k8s_juju.integrate("grafana-k8s:grafana-source", "prometheus-k8s:grafana-source") k8s_juju.wait(lambda status: jubilant.all_active(status, "grafana-k8s", "prometheus-k8s")) @@ -47,7 +49,6 @@ def grafana_app_fixture(k8s_juju: jubilant.Juju): @pytest.fixture(scope="module", name="grafana_password") def grafana_password_fixture(k8s_juju: jubilant.Juju, grafana_app: AppStatus): """Get Grafana dashboard password.""" - k8s_juju.model = f"microk8s:{k8s_juju.model}" unit = next(iter(grafana_app.units.keys())) result = k8s_juju.run(unit, "get-admin-password") return result.results["admin-password"] @@ -77,11 +78,11 @@ def test_prometheus_metrics( act: when GitHub runner is integrated. assert: the datasource is registered and basic metrics are available. """ - k8s_juju.model = f"microk8s:{k8s_juju.model}" prometheus_offer_name = "prometheus-k8s" grafana_offer_name = "grafana-k8s" - juju.cli(f"consume microk8s:{k8s_juju.model}.{prometheus_offer_name}") - juju.cli(f"consume microk8s:{k8s_juju.model}.{grafana_offer_name}") + # k8s_juju.model already has : prefixed. + juju.cli(f"consume {k8s_juju.model}.{prometheus_offer_name}") + juju.cli(f"consume {k8s_juju.model}.{grafana_offer_name}") juju.integrate("grafana-agent", prometheus_offer_name) juju.integrate("grafana-agent", grafana_offer_name) juju.wait( From ea2cb4fde01f090cf122c52d745c4e1f54356e63 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 26 Jun 2025 08:45:38 +0000 Subject: [PATCH 39/75] test: add fixture dependency --- tests/integration/test_prometheus_metrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index 2ed38efed2..c45abc14e1 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -37,10 +37,10 @@ def prometheus_app_fixture(k8s_juju: jubilant.Juju): @pytest.fixture(scope="module", name="grafana_app") -def grafana_app_fixture(k8s_juju: jubilant.Juju): +def grafana_app_fixture(k8s_juju: jubilant.Juju, prometheus_app: AppStatus): """Deploy prometheus charm.""" k8s_juju.deploy("grafana-k8s", channel="1/stable") - k8s_juju.integrate("grafana-k8s:grafana-source", "prometheus-k8s:grafana-source") + k8s_juju.integrate("grafana-k8s:grafana-source", f"{prometheus_app.charm_name}:grafana-source") k8s_juju.wait(lambda status: jubilant.all_active(status, "grafana-k8s", "prometheus-k8s")) k8s_juju.offer("grafana-k8s", endpoint="grafana-dashboard") return k8s_juju.status().apps["grafana-k8s"] From 2b021a7f17b6844619ee027e8723d9abbd2a2830 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 26 Jun 2025 10:10:37 +0000 Subject: [PATCH 40/75] test: switch controller fixture for juju offers --- tests/integration/test_prometheus_metrics.py | 22 ++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index c45abc14e1..bf8647d68f 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -27,6 +27,27 @@ def k8s_juju_fixture(request: pytest.FixtureRequest) -> Generator[jubilant.Juju, yield juju +# juju.offer is not controller aware, we should manually switch to the microk8s controller. +@pytest.fixture(scope="function", name="switch_microk8s_controller") +def switch_microk8s_controller_fixture(k8s_juju: jubilant.Juju, juju: jubilant.Juju): + """Switch to the MicroK8s controller.""" + original_model_controller_name = juju.model or "" + assert ( + ":" in original_model_controller_name + ), f"model & controller name not set: {original_model_controller_name}" + original_controller_name = original_model_controller_name.split(":")[0] + + model_controller_name = k8s_juju.model or "" + assert ( + ":" in model_controller_name + ), f"model & controller name not set: {model_controller_name}" + controller_name = model_controller_name.split(":")[0] + k8s_juju.cli("switch", controller_name) + yield + k8s_juju.cli("switch", original_controller_name) + + +@pytest.mark.usefixtures("switch_microk8s_controller") @pytest.fixture(scope="module", name="prometheus_app") def prometheus_app_fixture(k8s_juju: jubilant.Juju): """Deploy prometheus charm.""" @@ -36,6 +57,7 @@ def prometheus_app_fixture(k8s_juju: jubilant.Juju): return k8s_juju.status().apps["prometheus-k8s"] +@pytest.mark.usefixtures("switch_microk8s_controller") @pytest.fixture(scope="module", name="grafana_app") def grafana_app_fixture(k8s_juju: jubilant.Juju, prometheus_app: AppStatus): """Deploy prometheus charm.""" From c85ac087da0020e3a25f331e215052fb206a0998 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 26 Jun 2025 11:16:27 +0000 Subject: [PATCH 41/75] test: switch controller select model --- tests/integration/test_prometheus_metrics.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index bf8647d68f..ce41727cfd 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -31,20 +31,16 @@ def k8s_juju_fixture(request: pytest.FixtureRequest) -> Generator[jubilant.Juju, @pytest.fixture(scope="function", name="switch_microk8s_controller") def switch_microk8s_controller_fixture(k8s_juju: jubilant.Juju, juju: jubilant.Juju): """Switch to the MicroK8s controller.""" - original_model_controller_name = juju.model or "" + original_model_controller_name = juju.model assert ( - ":" in original_model_controller_name + original_model_controller_name ), f"model & controller name not set: {original_model_controller_name}" - original_controller_name = original_model_controller_name.split(":")[0] - model_controller_name = k8s_juju.model or "" - assert ( - ":" in model_controller_name - ), f"model & controller name not set: {model_controller_name}" - controller_name = model_controller_name.split(":")[0] - k8s_juju.cli("switch", controller_name) + model_controller_name = k8s_juju.model + assert model_controller_name, f"model & controller name not set: {model_controller_name}" + k8s_juju.cli("switch", model_controller_name) yield - k8s_juju.cli("switch", original_controller_name) + k8s_juju.cli("switch", original_model_controller_name) @pytest.mark.usefixtures("switch_microk8s_controller") From 2788589e8397c8cecbfe7d554b43025d58291742 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 26 Jun 2025 12:33:17 +0000 Subject: [PATCH 42/75] test: modify juju controller & model env vars --- tests/integration/test_prometheus_metrics.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index ce41727cfd..f3ffdb825c 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -4,6 +4,7 @@ """Module for collecting metrics related to the reconciliation process.""" import logging +import os from typing import Generator, cast import jubilant @@ -38,9 +39,14 @@ def switch_microk8s_controller_fixture(k8s_juju: jubilant.Juju, juju: jubilant.J model_controller_name = k8s_juju.model assert model_controller_name, f"model & controller name not set: {model_controller_name}" + controller, model = model_controller_name.split(":") k8s_juju.cli("switch", model_controller_name) + os.environ["JUJU_CONTROLLER"] = controller + os.environ["JUJU_MODEL"] = model yield k8s_juju.cli("switch", original_model_controller_name) + os.environ.pop("JUJU_CONTROLLER") + os.environ.pop("JUJU_MODEL") @pytest.mark.usefixtures("switch_microk8s_controller") From abd07a4c93fc5079d3f8cbd710a80ff99aeb4549 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 26 Jun 2025 13:56:10 +0000 Subject: [PATCH 43/75] test: try direct cli_bin call --- tests/integration/test_prometheus_metrics.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index f3ffdb825c..4df4901784 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -5,6 +5,7 @@ import logging import os +import subprocess from typing import Generator, cast import jubilant @@ -55,7 +56,13 @@ def prometheus_app_fixture(k8s_juju: jubilant.Juju): """Deploy prometheus charm.""" k8s_juju.deploy("prometheus-k8s", channel="1/stable") k8s_juju.wait(lambda status: jubilant.all_active(status, "prometheus-k8s")) - k8s_juju.offer("prometheus-k8s", endpoint="receive-remote-write") + model_controller_name = k8s_juju.model + assert model_controller_name, f"model & controller name not set: {model_controller_name}" + controller, model = model_controller_name.split(":") + subprocess.run( + [k8s_juju.cli_binary, "offer", "prometheus-k8s:receive-remote-write"], + env={"JUJU_CONTROLLER": controller, "JUJU_MODEL": model}, + ) return k8s_juju.status().apps["prometheus-k8s"] @@ -66,7 +73,13 @@ def grafana_app_fixture(k8s_juju: jubilant.Juju, prometheus_app: AppStatus): k8s_juju.deploy("grafana-k8s", channel="1/stable") k8s_juju.integrate("grafana-k8s:grafana-source", f"{prometheus_app.charm_name}:grafana-source") k8s_juju.wait(lambda status: jubilant.all_active(status, "grafana-k8s", "prometheus-k8s")) - k8s_juju.offer("grafana-k8s", endpoint="grafana-dashboard") + model_controller_name = k8s_juju.model + assert model_controller_name, f"model & controller name not set: {model_controller_name}" + controller, model = model_controller_name.split(":") + subprocess.run( + [k8s_juju.cli_binary, "offer", "grafana-k8s:grafana-dashboard"], + env={"JUJU_CONTROLLER": controller, "JUJU_MODEL": model}, + ) return k8s_juju.status().apps["grafana-k8s"] From fe67e92b1b02ce53ada514144060faa722a7515b Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 26 Jun 2025 16:52:49 +0000 Subject: [PATCH 44/75] test: try no env patch --- tests/integration/test_prometheus_metrics.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index 4df4901784..a4cacfe3c7 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -56,13 +56,7 @@ def prometheus_app_fixture(k8s_juju: jubilant.Juju): """Deploy prometheus charm.""" k8s_juju.deploy("prometheus-k8s", channel="1/stable") k8s_juju.wait(lambda status: jubilant.all_active(status, "prometheus-k8s")) - model_controller_name = k8s_juju.model - assert model_controller_name, f"model & controller name not set: {model_controller_name}" - controller, model = model_controller_name.split(":") - subprocess.run( - [k8s_juju.cli_binary, "offer", "prometheus-k8s:receive-remote-write"], - env={"JUJU_CONTROLLER": controller, "JUJU_MODEL": model}, - ) + subprocess.run([k8s_juju.cli_binary, "offer", "prometheus-k8s:receive-remote-write"]) return k8s_juju.status().apps["prometheus-k8s"] @@ -73,13 +67,7 @@ def grafana_app_fixture(k8s_juju: jubilant.Juju, prometheus_app: AppStatus): k8s_juju.deploy("grafana-k8s", channel="1/stable") k8s_juju.integrate("grafana-k8s:grafana-source", f"{prometheus_app.charm_name}:grafana-source") k8s_juju.wait(lambda status: jubilant.all_active(status, "grafana-k8s", "prometheus-k8s")) - model_controller_name = k8s_juju.model - assert model_controller_name, f"model & controller name not set: {model_controller_name}" - controller, model = model_controller_name.split(":") - subprocess.run( - [k8s_juju.cli_binary, "offer", "grafana-k8s:grafana-dashboard"], - env={"JUJU_CONTROLLER": controller, "JUJU_MODEL": model}, - ) + subprocess.run([k8s_juju.cli_binary, "offer", "grafana-k8s:grafana-dashboard"]) return k8s_juju.status().apps["grafana-k8s"] From 9a852af7d7f5cff5a66927c1fefe68fea5b6bb61 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 26 Jun 2025 18:03:27 +0000 Subject: [PATCH 45/75] test: assert subprocess result --- tests/integration/test_prometheus_metrics.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index a4cacfe3c7..c834c116e8 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -56,7 +56,10 @@ def prometheus_app_fixture(k8s_juju: jubilant.Juju): """Deploy prometheus charm.""" k8s_juju.deploy("prometheus-k8s", channel="1/stable") k8s_juju.wait(lambda status: jubilant.all_active(status, "prometheus-k8s")) - subprocess.run([k8s_juju.cli_binary, "offer", "prometheus-k8s:receive-remote-write"]) + result = subprocess.run([k8s_juju.cli_binary, "offer", "prometheus-k8s:receive-remote-write"]) + assert ( + result.returncode == 0 + ), f"failed to create prometheus offer: {result.stdout} {result.stderr}" return k8s_juju.status().apps["prometheus-k8s"] @@ -67,7 +70,10 @@ def grafana_app_fixture(k8s_juju: jubilant.Juju, prometheus_app: AppStatus): k8s_juju.deploy("grafana-k8s", channel="1/stable") k8s_juju.integrate("grafana-k8s:grafana-source", f"{prometheus_app.charm_name}:grafana-source") k8s_juju.wait(lambda status: jubilant.all_active(status, "grafana-k8s", "prometheus-k8s")) - subprocess.run([k8s_juju.cli_binary, "offer", "grafana-k8s:grafana-dashboard"]) + result = subprocess.run([k8s_juju.cli_binary, "offer", "grafana-k8s:grafana-dashboard"]) + assert ( + result.returncode == 0 + ), f"failed to create grafana offer: {result.stdout} {result.stderr}" return k8s_juju.status().apps["grafana-k8s"] @@ -106,8 +112,8 @@ def test_prometheus_metrics( prometheus_offer_name = "prometheus-k8s" grafana_offer_name = "grafana-k8s" # k8s_juju.model already has : prefixed. - juju.cli(f"consume {k8s_juju.model}.{prometheus_offer_name}") - juju.cli(f"consume {k8s_juju.model}.{grafana_offer_name}") + juju.cli(f"consume {k8s_juju.model}.{prometheus_offer_name}", include_model=False) + juju.cli(f"consume {k8s_juju.model}.{grafana_offer_name}", include_model=False) juju.integrate("grafana-agent", prometheus_offer_name) juju.integrate("grafana-agent", grafana_offer_name) juju.wait( From 5b5a7d5b4f4749d4e0a10804a18d777cc5d151a5 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Thu, 26 Jun 2025 18:49:13 +0000 Subject: [PATCH 46/75] test: set additional juju envs --- tests/integration/test_prometheus_metrics.py | 23 ++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index c834c116e8..1fa5496bc6 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -56,7 +56,17 @@ def prometheus_app_fixture(k8s_juju: jubilant.Juju): """Deploy prometheus charm.""" k8s_juju.deploy("prometheus-k8s", channel="1/stable") k8s_juju.wait(lambda status: jubilant.all_active(status, "prometheus-k8s")) - result = subprocess.run([k8s_juju.cli_binary, "offer", "prometheus-k8s:receive-remote-write"]) + env = os.environ.copy() + model_controller_name = k8s_juju.model + logger.info("Model controller: %s", model_controller_name) + assert model_controller_name, f"model & controller name not set: {model_controller_name}" + controller, model = model_controller_name.split(":") + logger.info("Controller: %s, Model: %s", controller, model) + env["JUJU_CONTROLLER"] = controller + env["JUJU_MODEL"] = model + result = subprocess.run( + [k8s_juju.cli_binary, "offer", "prometheus-k8s:receive-remote-write"], env=env + ) assert ( result.returncode == 0 ), f"failed to create prometheus offer: {result.stdout} {result.stderr}" @@ -70,7 +80,16 @@ def grafana_app_fixture(k8s_juju: jubilant.Juju, prometheus_app: AppStatus): k8s_juju.deploy("grafana-k8s", channel="1/stable") k8s_juju.integrate("grafana-k8s:grafana-source", f"{prometheus_app.charm_name}:grafana-source") k8s_juju.wait(lambda status: jubilant.all_active(status, "grafana-k8s", "prometheus-k8s")) - result = subprocess.run([k8s_juju.cli_binary, "offer", "grafana-k8s:grafana-dashboard"]) + env = os.environ.copy() + model_controller_name = k8s_juju.model + assert model_controller_name, f"model & controller name not set: {model_controller_name}" + controller, model = model_controller_name.split(":") + logger.info("Controller: %s, Model: %s", controller, model) + env["JUJU_CONTROLLER"] = controller + env["JUJU_MODEL"] = model + result = subprocess.run( + [k8s_juju.cli_binary, "offer", "grafana-k8s:grafana-dashboard"], env=env + ) assert ( result.returncode == 0 ), f"failed to create grafana offer: {result.stdout} {result.stderr}" From 933b9f1e5aff1b4c2cae355b6020240caf50ab15 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Fri, 27 Jun 2025 02:49:36 +0000 Subject: [PATCH 47/75] test: use controller & model params --- tests/integration/test_prometheus_metrics.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index 1fa5496bc6..1ee4253276 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -65,7 +65,14 @@ def prometheus_app_fixture(k8s_juju: jubilant.Juju): env["JUJU_CONTROLLER"] = controller env["JUJU_MODEL"] = model result = subprocess.run( - [k8s_juju.cli_binary, "offer", "prometheus-k8s:receive-remote-write"], env=env + [ + k8s_juju.cli_binary, + "offer", + "-c", + controller, + f"{model}.prometheus-k8s:receive-remote-write", + ], + env=env, ) assert ( result.returncode == 0 @@ -88,7 +95,8 @@ def grafana_app_fixture(k8s_juju: jubilant.Juju, prometheus_app: AppStatus): env["JUJU_CONTROLLER"] = controller env["JUJU_MODEL"] = model result = subprocess.run( - [k8s_juju.cli_binary, "offer", "grafana-k8s:grafana-dashboard"], env=env + [k8s_juju.cli_binary, "offer", "-c", controller, f"{model}.grafana-k8s:grafana-dashboard"], + env=env, ) assert ( result.returncode == 0 From 14cf50e7cb44a915516ea560191879f4933b784d Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Fri, 27 Jun 2025 03:41:31 +0000 Subject: [PATCH 48/75] test: try consume model w/ model params --- tests/integration/test_prometheus_metrics.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index 1ee4253276..cf0c3a0990 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -38,16 +38,7 @@ def switch_microk8s_controller_fixture(k8s_juju: jubilant.Juju, juju: jubilant.J original_model_controller_name ), f"model & controller name not set: {original_model_controller_name}" - model_controller_name = k8s_juju.model - assert model_controller_name, f"model & controller name not set: {model_controller_name}" - controller, model = model_controller_name.split(":") - k8s_juju.cli("switch", model_controller_name) - os.environ["JUJU_CONTROLLER"] = controller - os.environ["JUJU_MODEL"] = model yield - k8s_juju.cli("switch", original_model_controller_name) - os.environ.pop("JUJU_CONTROLLER") - os.environ.pop("JUJU_MODEL") @pytest.mark.usefixtures("switch_microk8s_controller") @@ -64,6 +55,7 @@ def prometheus_app_fixture(k8s_juju: jubilant.Juju): logger.info("Controller: %s, Model: %s", controller, model) env["JUJU_CONTROLLER"] = controller env["JUJU_MODEL"] = model + # juju.offer has no controller parameter. Use the cli directly. result = subprocess.run( [ k8s_juju.cli_binary, @@ -94,6 +86,7 @@ def grafana_app_fixture(k8s_juju: jubilant.Juju, prometheus_app: AppStatus): logger.info("Controller: %s, Model: %s", controller, model) env["JUJU_CONTROLLER"] = controller env["JUJU_MODEL"] = model + # juju.offer has no controller parameter. Use the cli directly. result = subprocess.run( [k8s_juju.cli_binary, "offer", "-c", controller, f"{model}.grafana-k8s:grafana-dashboard"], env=env, @@ -139,8 +132,8 @@ def test_prometheus_metrics( prometheus_offer_name = "prometheus-k8s" grafana_offer_name = "grafana-k8s" # k8s_juju.model already has : prefixed. - juju.cli(f"consume {k8s_juju.model}.{prometheus_offer_name}", include_model=False) - juju.cli(f"consume {k8s_juju.model}.{grafana_offer_name}", include_model=False) + juju.cli(f"consume {k8s_juju.model}.{prometheus_offer_name}") + juju.cli(f"consume {k8s_juju.model}.{grafana_offer_name}") juju.integrate("grafana-agent", prometheus_offer_name) juju.integrate("grafana-agent", grafana_offer_name) juju.wait( From 37d2bb6c143a5889bd7312ec0db71ef168f4c8d2 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Fri, 27 Jun 2025 04:26:43 +0000 Subject: [PATCH 49/75] test: use direct consume command --- tests/integration/test_prometheus_metrics.py | 30 ++++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index cf0c3a0990..70810e488f 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -131,9 +131,32 @@ def test_prometheus_metrics( """ prometheus_offer_name = "prometheus-k8s" grafana_offer_name = "grafana-k8s" - # k8s_juju.model already has : prefixed. - juju.cli(f"consume {k8s_juju.model}.{prometheus_offer_name}") - juju.cli(f"consume {k8s_juju.model}.{grafana_offer_name}") + # k8s_juju.model and juju.model already has : prefixed. + result = subprocess.run( + [ + k8s_juju.cli_binary, + "consume", + "-m", + str(juju.model), + f"{str(k8s_juju.model)}.prometheus-k8s", + ] + ) + assert ( + result.returncode == 0 + ), f"failed to consume prometheus offer: {result.stdout} {result.stderr}" + result = subprocess.run( + [ + k8s_juju.cli_binary, + "consume", + "-m", + str(juju.model), + f"{str(k8s_juju.model)}.grafana-k8s", + ] + ) + assert ( + result.returncode == 0 + ), f"failed to consume grafana offer: {result.stdout} {result.stderr}" + juju.integrate("grafana-agent", prometheus_offer_name) juju.integrate("grafana-agent", grafana_offer_name) juju.wait( @@ -141,5 +164,6 @@ def test_prometheus_metrics( status, openstack_app_cos_agent.name, "grafana-agent" ) ) + grafana_ip = grafana_app.units["grafana-k8s/0"].address assert False, f"admin:{grafana_password}@{grafana_ip}:3000/" From 9a8f67c5c8ae15d301a56ba7257dcf1bc173c2af Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Fri, 27 Jun 2025 06:17:31 +0000 Subject: [PATCH 50/75] test: setup microstack metallb for ingress --- tests/integration/setup-integration-tests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/setup-integration-tests.sh b/tests/integration/setup-integration-tests.sh index 5504571582..fc319254e1 100755 --- a/tests/integration/setup-integration-tests.sh +++ b/tests/integration/setup-integration-tests.sh @@ -18,6 +18,8 @@ if [ $(id -gn) != $GROUP ]; then exec sg $GROUP "$0 $*" fi sudo microk8s enable hostpath-storage +IPADDR=$(ip -4 -j route get 2.2.2.2 | jq -r '.[] | .prefsrc') +sudo microk8s enable metallb:$IPADDR-$IPADDR microk8s status --wait-ready unset JUJU_CONTROLLER From 649597a071c78f569c5641b99251f455b792eef9 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Fri, 27 Jun 2025 06:17:49 +0000 Subject: [PATCH 51/75] test: check for metrics availability --- tests/integration/test_prometheus_metrics.py | 66 ++++++++++++++++---- 1 file changed, 53 insertions(+), 13 deletions(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index 70810e488f..2c8d11cffd 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -4,15 +4,16 @@ """Module for collecting metrics related to the reconciliation process.""" import logging -import os import subprocess -from typing import Generator, cast +from typing import Any, Generator, cast import jubilant import pytest import pytest_asyncio +import requests from jubilant.statustypes import AppStatus from juju.application import Application +from tenacity import retry, stop_after_attempt, wait_exponential logger = logging.getLogger(__name__) @@ -47,14 +48,11 @@ def prometheus_app_fixture(k8s_juju: jubilant.Juju): """Deploy prometheus charm.""" k8s_juju.deploy("prometheus-k8s", channel="1/stable") k8s_juju.wait(lambda status: jubilant.all_active(status, "prometheus-k8s")) - env = os.environ.copy() model_controller_name = k8s_juju.model logger.info("Model controller: %s", model_controller_name) assert model_controller_name, f"model & controller name not set: {model_controller_name}" controller, model = model_controller_name.split(":") logger.info("Controller: %s, Model: %s", controller, model) - env["JUJU_CONTROLLER"] = controller - env["JUJU_MODEL"] = model # juju.offer has no controller parameter. Use the cli directly. result = subprocess.run( [ @@ -63,8 +61,7 @@ def prometheus_app_fixture(k8s_juju: jubilant.Juju): "-c", controller, f"{model}.prometheus-k8s:receive-remote-write", - ], - env=env, + ] ) assert ( result.returncode == 0 @@ -79,17 +76,13 @@ def grafana_app_fixture(k8s_juju: jubilant.Juju, prometheus_app: AppStatus): k8s_juju.deploy("grafana-k8s", channel="1/stable") k8s_juju.integrate("grafana-k8s:grafana-source", f"{prometheus_app.charm_name}:grafana-source") k8s_juju.wait(lambda status: jubilant.all_active(status, "grafana-k8s", "prometheus-k8s")) - env = os.environ.copy() model_controller_name = k8s_juju.model assert model_controller_name, f"model & controller name not set: {model_controller_name}" controller, model = model_controller_name.split(":") logger.info("Controller: %s, Model: %s", controller, model) - env["JUJU_CONTROLLER"] = controller - env["JUJU_MODEL"] = model # juju.offer has no controller parameter. Use the cli directly. result = subprocess.run( - [k8s_juju.cli_binary, "offer", "-c", controller, f"{model}.grafana-k8s:grafana-dashboard"], - env=env, + [k8s_juju.cli_binary, "offer", "-c", controller, f"{model}.grafana-k8s:grafana-dashboard"] ) assert ( result.returncode == 0 @@ -97,6 +90,16 @@ def grafana_app_fixture(k8s_juju: jubilant.Juju, prometheus_app: AppStatus): return k8s_juju.status().apps["grafana-k8s"] +@pytest.fixture(scope="module", name="traefik_ingress") +def traefik_ingress_fixture( + k8s_juju: jubilant.Juju, prometheus_app: AppStatus, grafana_app: AppStatus +): + """Ingress for cross controller communication.""" + k8s_juju.deploy("traefik-k8s", channel="latest/stable") + k8s_juju.integrate("traefik-k8s", f"{prometheus_app.charm_name}:ingress") + k8s_juju.integrate("traefik-k8s", f"{grafana_app.charm_name}:ingress") + + @pytest.fixture(scope="module", name="grafana_password") def grafana_password_fixture(k8s_juju: jubilant.Juju, grafana_app: AppStatus): """Get Grafana dashboard password.""" @@ -116,6 +119,7 @@ def openstack_app_cos_agent_fixture(juju: jubilant.Juju, app_openstack_runner: A return app_openstack_runner +@pytest.mark.usefixtures("traefik_ingress") @pytest.mark.openstack def test_prometheus_metrics( juju: jubilant.Juju, @@ -123,6 +127,7 @@ def test_prometheus_metrics( openstack_app_cos_agent: Application, grafana_app: AppStatus, grafana_password: str, + prometheus_app: AppStatus, ): """ arrange: given a prometheus charm application. @@ -166,4 +171,39 @@ def test_prometheus_metrics( ) grafana_ip = grafana_app.units["grafana-k8s/0"].address - assert False, f"admin:{grafana_password}@{grafana_ip}:3000/" + _patiently_wait_for_prometheus_datasource( + grafana_ip=grafana_ip, grafana_password=grafana_password + ) + prometheus_ip = prometheus_app.units["prometheus-k8s/0"].address + _patiently_wait_for_prometheus_metrics( + prometheus_ip=prometheus_ip, + metric_names=( + "openstack_http_requests_total", + "reconcile_duration_seconds", + "expected_runners_count", + "busy_runners_count", + "idle_runners_count", + "cleaned_runners_total", + ), + ) + + +@retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=2, max=60), reraise=True) +def _patiently_wait_for_prometheus_datasource(grafana_ip: str, grafana_password: str): + """Wait for prometheus datasource to come up.""" + response = requests.get(f"admin:{grafana_password}@{grafana_ip}:3000/api/datasources") + response.raise_for_status() + datasources: list[dict[str, Any]] = response.json() + assert any(datasource["type"] == "prometheus" for datasource in datasources) + + +@retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=2, max=60), reraise=True) +def _patiently_wait_for_prometheus_metrics(prometheus_ip: str, *metric_names: str): + """Wait for the prometheus metrics to be available.""" + for metric_name in metric_names: + response = requests.get( + f"{prometheus_ip}:9090/api/v1/series", params={"match[]": metric_name} + ) + response.raise_for_status() + query_result = response.json()["data"] + assert len(query_result), f"No data found for metric: {metric_name}" From e32efb0559e65b217a1d4ed4b6120e87f07bce92 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Fri, 27 Jun 2025 07:43:27 +0000 Subject: [PATCH 52/75] test: add schema to request --- tests/integration/test_prometheus_metrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index 2c8d11cffd..c388c5886c 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -191,7 +191,7 @@ def test_prometheus_metrics( @retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=2, max=60), reraise=True) def _patiently_wait_for_prometheus_datasource(grafana_ip: str, grafana_password: str): """Wait for prometheus datasource to come up.""" - response = requests.get(f"admin:{grafana_password}@{grafana_ip}:3000/api/datasources") + response = requests.get(f"http://admin:{grafana_password}@{grafana_ip}:3000/api/datasources") response.raise_for_status() datasources: list[dict[str, Any]] = response.json() assert any(datasource["type"] == "prometheus" for datasource in datasources) @@ -202,7 +202,7 @@ def _patiently_wait_for_prometheus_metrics(prometheus_ip: str, *metric_names: st """Wait for the prometheus metrics to be available.""" for metric_name in metric_names: response = requests.get( - f"{prometheus_ip}:9090/api/v1/series", params={"match[]": metric_name} + f"http://{prometheus_ip}:9090/api/v1/series", params={"match[]": metric_name} ) response.raise_for_status() query_result = response.json()["data"] From 62cf23fc3b30f922859dafd6fd70b67344f5b129 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Fri, 27 Jun 2025 09:19:44 +0000 Subject: [PATCH 53/75] test: fix syntax err --- tests/integration/test_prometheus_metrics.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index c388c5886c..0ca8aef885 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -176,15 +176,13 @@ def test_prometheus_metrics( ) prometheus_ip = prometheus_app.units["prometheus-k8s/0"].address _patiently_wait_for_prometheus_metrics( - prometheus_ip=prometheus_ip, - metric_names=( - "openstack_http_requests_total", - "reconcile_duration_seconds", - "expected_runners_count", - "busy_runners_count", - "idle_runners_count", - "cleaned_runners_total", - ), + prometheus_ip, + "openstack_http_requests_total", + "reconcile_duration_seconds", + "expected_runners_count", + "busy_runners_count", + "idle_runners_count", + "cleaned_runners_total", ) From 09f404238448204a2274147d18df6824cf3f6fee Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Fri, 27 Jun 2025 10:52:42 +0000 Subject: [PATCH 54/75] test: fix prometheus ip --- tests/integration/test_prometheus_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index 0ca8aef885..d7bbb82d46 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -174,7 +174,7 @@ def test_prometheus_metrics( _patiently_wait_for_prometheus_datasource( grafana_ip=grafana_ip, grafana_password=grafana_password ) - prometheus_ip = prometheus_app.units["prometheus-k8s/0"].address + prometheus_ip = prometheus_app.address _patiently_wait_for_prometheus_metrics( prometheus_ip, "openstack_http_requests_total", From 1f71e5e102fbc92e460d0d02c857653fd6df951b Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Fri, 27 Jun 2025 12:16:49 +0000 Subject: [PATCH 55/75] test: wait for openstack metrics longer --- tests/integration/test_prometheus_metrics.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index d7bbb82d46..f55338c11c 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -195,7 +195,9 @@ def _patiently_wait_for_prometheus_datasource(grafana_ip: str, grafana_password: assert any(datasource["type"] == "prometheus" for datasource in datasources) -@retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=2, max=60), reraise=True) +@retry( + stop=stop_after_attempt(10), wait=wait_exponential(multiplier=2, min=10, max=60), reraise=True +) def _patiently_wait_for_prometheus_metrics(prometheus_ip: str, *metric_names: str): """Wait for the prometheus metrics to be available.""" for metric_name in metric_names: From 8546817e3aab66e4b5adb346915ff53c6dea76f7 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Sun, 29 Jun 2025 05:32:45 +0000 Subject: [PATCH 56/75] test: test only metrics that have been generated --- tests/integration/test_prometheus_metrics.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index f55338c11c..d574473c23 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -178,10 +178,8 @@ def test_prometheus_metrics( _patiently_wait_for_prometheus_metrics( prometheus_ip, "openstack_http_requests_total", - "reconcile_duration_seconds", + "reconcile_duration_seconds_sum", "expected_runners_count", - "busy_runners_count", - "idle_runners_count", "cleaned_runners_total", ) From 6271b74d45b0f519e7268cad58ae10a1fce63101 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Sun, 29 Jun 2025 05:36:44 +0000 Subject: [PATCH 57/75] test: restore tests --- .github/workflows/auto_update_libs.yaml | 10 +++ .github/workflows/bot_pr_approval.yaml | 9 ++ .github/workflows/comment.yaml | 12 +++ .github/workflows/comment_contributing.yaml | 13 +++ .github/workflows/e2e_test.yaml | 36 ++++++++ .github/workflows/e2e_test_run.yaml | 89 +++++++++++++++++++ .github/workflows/integration_test.yaml | 43 +++++---- .github/workflows/promote_charm.yaml | 32 +++++++ .github/workflows/publish_charm.yaml | 12 +++ .github/workflows/push_test.yaml | 14 +++ .github/workflows/schedule_issues_test.yaml | 26 ++++++ .github/workflows/test.yaml | 13 +++ .../workflows/test_github_runner_manager.yaml | 13 +++ .../workflow_dispatch_crash_test.yaml | 17 ++++ .../workflow_dispatch_failure_test.yaml | 23 +++++ .../workflow_dispatch_ssh_debug.yaml | 17 ++++ .github/workflows/workflow_dispatch_test.yaml | 20 +++++ .../workflow_dispatch_wait_test.yaml | 28 ++++++ 18 files changed, 405 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/auto_update_libs.yaml create mode 100644 .github/workflows/bot_pr_approval.yaml create mode 100644 .github/workflows/comment.yaml create mode 100644 .github/workflows/comment_contributing.yaml create mode 100644 .github/workflows/e2e_test.yaml create mode 100644 .github/workflows/e2e_test_run.yaml create mode 100644 .github/workflows/promote_charm.yaml create mode 100644 .github/workflows/publish_charm.yaml create mode 100644 .github/workflows/push_test.yaml create mode 100644 .github/workflows/schedule_issues_test.yaml create mode 100644 .github/workflows/test.yaml create mode 100644 .github/workflows/test_github_runner_manager.yaml create mode 100644 .github/workflows/workflow_dispatch_crash_test.yaml create mode 100644 .github/workflows/workflow_dispatch_failure_test.yaml create mode 100644 .github/workflows/workflow_dispatch_ssh_debug.yaml create mode 100644 .github/workflows/workflow_dispatch_test.yaml create mode 100644 .github/workflows/workflow_dispatch_wait_test.yaml diff --git a/.github/workflows/auto_update_libs.yaml b/.github/workflows/auto_update_libs.yaml new file mode 100644 index 0000000000..02b7b204e7 --- /dev/null +++ b/.github/workflows/auto_update_libs.yaml @@ -0,0 +1,10 @@ +name: Auto-update charm libraries + +on: + schedule: + - cron: "0 1 * * *" + +jobs: + auto-update-libs: + uses: canonical/operator-workflows/.github/workflows/auto_update_charm_libs.yaml@main + secrets: inherit diff --git a/.github/workflows/bot_pr_approval.yaml b/.github/workflows/bot_pr_approval.yaml new file mode 100644 index 0000000000..e38c5841dd --- /dev/null +++ b/.github/workflows/bot_pr_approval.yaml @@ -0,0 +1,9 @@ +name: Provide approval for bot PRs + +on: + pull_request: + +jobs: + bot_pr_approval: + uses: canonical/operator-workflows/.github/workflows/bot_pr_approval.yaml@main + secrets: inherit diff --git a/.github/workflows/comment.yaml b/.github/workflows/comment.yaml new file mode 100644 index 0000000000..26ac226df8 --- /dev/null +++ b/.github/workflows/comment.yaml @@ -0,0 +1,12 @@ +name: Comment on the pull request + +on: + workflow_run: + workflows: ["Tests"] + types: + - completed + +jobs: + comment-on-pr: + uses: canonical/operator-workflows/.github/workflows/comment.yaml@main + secrets: inherit diff --git a/.github/workflows/comment_contributing.yaml b/.github/workflows/comment_contributing.yaml new file mode 100644 index 0000000000..c51f873143 --- /dev/null +++ b/.github/workflows/comment_contributing.yaml @@ -0,0 +1,13 @@ +name: Comment on the pull request + +on: + pull_request: + types: + - opened + branches: + - 'track/**' + +jobs: + comment-on-pr: + uses: canonical/operator-workflows/.github/workflows/comment_contributing.yaml@main + secrets: inherit diff --git a/.github/workflows/e2e_test.yaml b/.github/workflows/e2e_test.yaml new file mode 100644 index 0000000000..60fdbacbdb --- /dev/null +++ b/.github/workflows/e2e_test.yaml @@ -0,0 +1,36 @@ +name: End-to-End tests + +on: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + # test option values defined at test/conftest.py are passed on via repository secret + # INTEGRATION_TEST_ARGS to operator-workflows automatically. + openstack-integration-end-to-end-test: + name: end-to-end test using private-endpoint + uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main + secrets: inherit + with: + juju-channel: 3.6/stable + pre-run-script: scripts/setup-integration-tests.sh + provider: lxd + test-tox-env: integration-juju3.6 + modules: '["test_e2e"]' + extra-arguments: "-m openstack" + self-hosted-runner: true + self-hosted-runner-label: stg-private-endpoint + + required_status_checks: + name: Required E2E Test Status Checks + runs-on: ubuntu-latest + needs: + - openstack-integration-end-to-end-test + if: always() && !cancelled() + timeout-minutes: 5 + steps: + - run: | + [ '${{ needs.openstack-integration-end-to-end-test.result }}' = 'success' ] || (echo openstack-integration-end-to-end-test failed && false) diff --git a/.github/workflows/e2e_test_run.yaml b/.github/workflows/e2e_test_run.yaml new file mode 100644 index 0000000000..bebbc1a2fc --- /dev/null +++ b/.github/workflows/e2e_test_run.yaml @@ -0,0 +1,89 @@ +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +name: Run End-to-End test + +on: + # The inputs are the same but they cannot be shared between triggers. + # See https://github.com/orgs/community/discussions/39357 + workflow_call: + inputs: + runner-tag: + description: The e2e test runner tag to run the workflow on. + type: string + required: true + workflow_dispatch: + inputs: + runner-tag: + description: The e2e test runner tag to run the workflow on. + type: string + required: true + +jobs: + e2e-test: + name: End-to-End Test Run + runs-on: [self-hosted, linux, "${{ inputs.runner-tag }}"] + steps: + - name: Hostname is set to "github-runner" + run: sudo hostnamectl hostname | grep github-runner + # Below is a series of simple tests to assess the functionality of the newly spawned runner. + - name: Echo hello world + run: echo "hello world" + - name: File permission for /usr/local/bin + run: ls -ld /usr/local/bin | grep drwxrwxrwx + - name: Test file permission for /usr/local/bin + run: touch /usr/local/bin/test_file + # "Install microk8s" step will test if the proxies settings are correct. + - name: Proxy set in /etc/environment + run: cat /etc/environment + # "Update apt in python docker container" step will test docker default proxy settings due to + # pulling the python image. + - name: Proxy set in docker daemon + run: | + [[ -z "${http_proxy}" && -z "${HTTP_PROXY}" ]] \ + || sudo cat /etc/systemd/system/docker.service.d/http-proxy.conf | grep HTTP_PROXY + # "Update apt in python docker container" step will test docker client default proxy settings. + - name: Proxy set in docker client + run: | + [[ -z "${http_proxy}" && -z "${HTTP_PROXY}" ]] \ + || cat /home/ubuntu/.docker/config.json | grep httpProxy + - name: Install microk8s + run: sudo snap install microk8s --classic + - name: Wait for microk8s + timeout-minutes: 10 + run: microk8s status --wait-ready + - name: Deploy nginx for testing + run: microk8s kubectl create deployment nginx --image=nginx + - name: Wait for nginx to be ready + run: microk8s kubectl rollout status deployment/nginx --timeout=30m + - name: Update apt in python docker container + run: docker run python:3.10-slim apt-get update + - name: Docker version + run: docker version + - name: Check python alias for python3 + run: python --version + - name: pip version + run: python3 -m pip --version + - name: npm version + run: npm --version + - name: shellcheck version + run: shellcheck --version + - name: jq version + run: jq --version + - name: yq version + run: yq --version + - name: apt update + run: sudo apt-get update -y + # Use pipx for 24.04 noble, check-jsonschema breaks OS system packages. + - name: install pipx + run: sudo apt-get install -y pipx + - name: install check-jsonschema + run: python3 -m pip install check-jsonschema || pipx install check-jsonschema + - name: unzip version + run: unzip -v + - name: gh version + run: gh --version + # `check-jsonschema` is installed using pip. The directory `~/.local/bin` needs to be added to PATH. + # ~/.local/bin is added to path runner env through in scripts/env.j2 + - name: test check-jsonschema + run: check-jsonschema --version diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 47b0e1ae48..a771b21c76 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -12,36 +12,35 @@ concurrency: cancel-in-progress: true jobs: - # openstack-interface-tests-private-endpoint: - # name: openstack interface test using private-endpoint - # uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main - # secrets: inherit - # with: - # juju-channel: 3.6/stable - # provider: lxd - # test-tox-env: integration-juju3.6 - # modules: '["test_runner_manager_openstack"]' - # extra-arguments: '--log-format="%(asctime)s %(levelname)s %(message)s"' - # self-hosted-runner: true - # self-hosted-runner-label: stg-private-endpoint + openstack-interface-tests-private-endpoint: + name: openstack interface test using private-endpoint + uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main + secrets: inherit + with: + juju-channel: 3.6/stable + pre-run-script: scripts/setup-integration-tests.sh + provider: lxd + test-tox-env: integration-juju3.6 + modules: '["test_runner_manager_openstack"]' + extra-arguments: '--log-format="%(asctime)s %(levelname)s %(message)s"' + self-hosted-runner: true + self-hosted-runner-label: stg-private-endpoint openstack-integration-tests-private-endpoint: name: Integration test using private-endpoint uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main secrets: inherit with: juju-channel: 3.6/stable - pre-run-script: tests/integration/setup-integration-tests.sh + pre-run-script: scripts/setup-integration-tests.sh provider: lxd test-tox-env: integration-juju3.6 - modules: '["test_prometheus_metrics"]' + modules: '["test_charm_metrics_failure", "test_charm_metrics_success", "test_charm_fork_repo", "test_charm_fork_path_change", "test_charm_no_runner", "test_charm_runner", "test_debug_ssh", "test_charm_upgrade", "test_reactive", "test_jobmanager_prespawned", "test_jobmanager_reactive", "test_prometheus_metrics.py"]' extra-arguments: '-m openstack --log-format="%(asctime)s %(levelname)s %(message)s"' self-hosted-runner: true self-hosted-runner-label: stg-private-endpoint - tmate-debug: true - tmate-timeout: 90 - # allure-report: - # if: ${{ (success() || failure()) && github.event_name == 'schedule' }} - # needs: - # - openstack-interface-tests-private-endpoint - # - openstack-integration-tests-private-endpoint - # uses: canonical/operator-workflows/.github/workflows/allure_report.yaml@main + allure-report: + if: ${{ (success() || failure()) && github.event_name == 'schedule' }} + needs: + - openstack-interface-tests-private-endpoint + - openstack-integration-tests-private-endpoint + uses: canonical/operator-workflows/.github/workflows/allure_report.yaml@main diff --git a/.github/workflows/promote_charm.yaml b/.github/workflows/promote_charm.yaml new file mode 100644 index 0000000000..d1a0ac0ffb --- /dev/null +++ b/.github/workflows/promote_charm.yaml @@ -0,0 +1,32 @@ +name: Promote charm + +on: + workflow_dispatch: + inputs: + origin-channel: + type: choice + description: 'Origin Channel' + options: + - latest/beta + - latest/edge + - 1/edge + destination-channel: + type: choice + description: 'Destination Channel' + options: + - latest/stable + - latest/beta + - 1/stable + secrets: + CHARMHUB_TOKEN: + required: true + +jobs: + promote-charm: + uses: canonical/operator-workflows/.github/workflows/promote_charm.yaml@main + with: + origin-channel: ${{ github.event.inputs.origin-channel }} + destination-channel: ${{ github.event.inputs.destination-channel }} + doc-automation-disabled: false + base-channel: '22.04' + secrets: inherit diff --git a/.github/workflows/publish_charm.yaml b/.github/workflows/publish_charm.yaml new file mode 100644 index 0000000000..a84cba1f56 --- /dev/null +++ b/.github/workflows/publish_charm.yaml @@ -0,0 +1,12 @@ +name: Publish to edge + +on: + push: + branches: + - main + - track/* + +jobs: + publish-to-edge: + uses: canonical/operator-workflows/.github/workflows/publish_charm.yaml@main + secrets: inherit diff --git a/.github/workflows/push_test.yaml b/.github/workflows/push_test.yaml new file mode 100644 index 0000000000..e19f300c28 --- /dev/null +++ b/.github/workflows/push_test.yaml @@ -0,0 +1,14 @@ +name: Push Event Tests + +on: + push: + branches: + - push-e2e-* + +jobs: + push-event-tests: + runs-on: [self-hosted, linux, "${{ github.ref_name }}"] + steps: + - name: Echo runner name + run: | + echo "Hello, runner: ${{ github.ref_name }}" diff --git a/.github/workflows/schedule_issues_test.yaml b/.github/workflows/schedule_issues_test.yaml new file mode 100644 index 0000000000..5140353f1e --- /dev/null +++ b/.github/workflows/schedule_issues_test.yaml @@ -0,0 +1,26 @@ +name: Issues and Schedule Event Tests + +on: + workflow_dispatch: # Replaced by end-to-end test to match 'issues' or 'schedule' + +jobs: + + extract-runner-name: + runs-on: ubuntu-latest + outputs: + name: ${{ steps.extract-runner-name.outputs.name }} + steps: + - name: Extract runner name + id: extract-runner-name + run: | + REPO=${{ github.repository }} + RUNNER_NAME=${REPO#${{ github.repository_owner }}/} + echo name=$RUNNER_NAME >> $GITHUB_OUTPUT + + issues-schedule-event-tests: + needs: [ extract-runner-name ] + runs-on: [ self-hosted, linux, "${{ needs.extract-runner-name.outputs.name }}" ] + steps: + - name: Echo runner name + run: | + echo "Hello, runner: ${{ needs.extract-runner-name.outputs.name }}" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000000..165fae6d27 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,13 @@ +name: Tests + +on: + pull_request: + +jobs: + unit-tests: + uses: canonical/operator-workflows/.github/workflows/test.yaml@main + secrets: inherit + with: + self-hosted-runner: true + self-hosted-runner-label: edge + vale-style-check: true diff --git a/.github/workflows/test_github_runner_manager.yaml b/.github/workflows/test_github_runner_manager.yaml new file mode 100644 index 0000000000..3f7eff3758 --- /dev/null +++ b/.github/workflows/test_github_runner_manager.yaml @@ -0,0 +1,13 @@ +name: Tests for github-runner-manager + +on: + pull_request: + +jobs: + unit-tests-github-runner-manager: + uses: canonical/operator-workflows/.github/workflows/test.yaml@main + secrets: inherit + with: + self-hosted-runner: true + self-hosted-runner-label: edge + working-directory: ./github-runner-manager/ diff --git a/.github/workflows/workflow_dispatch_crash_test.yaml b/.github/workflows/workflow_dispatch_crash_test.yaml new file mode 100644 index 0000000000..253513b641 --- /dev/null +++ b/.github/workflows/workflow_dispatch_crash_test.yaml @@ -0,0 +1,17 @@ +name: Workflow Dispatch Crash Tests + +on: + # Manually dispatched workflow action + workflow_dispatch: + inputs: + runner: + description: 'Self hosted gh runner' + required: true + +jobs: + workflow-dispatch-tests: + runs-on: [self-hosted, linux, "${{ inputs.runner }}"] + steps: + - name: Sleep for an hour to give the integration test some time to crash the runner. + run: | + sleep 3600 diff --git a/.github/workflows/workflow_dispatch_failure_test.yaml b/.github/workflows/workflow_dispatch_failure_test.yaml new file mode 100644 index 0000000000..ef5c6ac8a7 --- /dev/null +++ b/.github/workflows/workflow_dispatch_failure_test.yaml @@ -0,0 +1,23 @@ +name: Workflow Dispatch Failure Tests 2a34f8b1-41e4-4bcb-9bbf-7a74e6c482f7 + +# This workflow is for testing only. +# Self-hosted runner managed by the charm is setup to fail this workflow at pre-job step. + +on: + # Manually dispatched workflow action + workflow_dispatch: + inputs: + runner: + description: 'Self hosted gh runner' + required: true + +jobs: + workflow-dispatch-tests: + runs-on: [self-hosted, linux, "${{ inputs.runner }}"] + steps: + - name: Echo input variable and message + run: | + echo "Hello, runner: ${{ inputs.runner }}" + - name: Always echo a message + if: always() + run: echo "Should not echo if pre-job script failed" diff --git a/.github/workflows/workflow_dispatch_ssh_debug.yaml b/.github/workflows/workflow_dispatch_ssh_debug.yaml new file mode 100644 index 0000000000..edbfb39206 --- /dev/null +++ b/.github/workflows/workflow_dispatch_ssh_debug.yaml @@ -0,0 +1,17 @@ +name: Workflow Dispatch Tests (ssh-debug/tmate) + +on: + # Manually dispatched workflow action + workflow_dispatch: + inputs: + runner: + description: 'Self hosted gh runner' + required: true + +jobs: + workflow-dispatch-tests: + runs-on: [self-hosted, linux, "${{ inputs.runner }}"] + steps: + - name: Setup tmate session + uses: canonical/action-tmate@main + timeout-minutes: 5 diff --git a/.github/workflows/workflow_dispatch_test.yaml b/.github/workflows/workflow_dispatch_test.yaml new file mode 100644 index 0000000000..513dc4232e --- /dev/null +++ b/.github/workflows/workflow_dispatch_test.yaml @@ -0,0 +1,20 @@ +name: Workflow Dispatch Tests + +on: + # Manually dispatched workflow action + workflow_dispatch: + inputs: + runner: + description: 'Self hosted gh runner' + required: true + +jobs: + workflow-dispatch-tests: + runs-on: [self-hosted, linux, "${{ inputs.runner }}"] + steps: + - name: Echo input variable and message + run: | + echo "Hello, runner: ${{ inputs.runner }}" + - name: Always echo a message + if: always() + run: echo "Should not echo if pre-job script failed" diff --git a/.github/workflows/workflow_dispatch_wait_test.yaml b/.github/workflows/workflow_dispatch_wait_test.yaml new file mode 100644 index 0000000000..f2470bec4b --- /dev/null +++ b/.github/workflows/workflow_dispatch_wait_test.yaml @@ -0,0 +1,28 @@ +name: Workflow Dispatch Wait Tests + +on: + # Manually dispatched workflow action + workflow_dispatch: + inputs: + runner: + description: 'Self hosted gh runner' + required: true + minutes: + description: 'Number of minutes to wait' + # Number type not supported in workflow dispatch: https://github.com/orgs/community/discussions/67182 + # Seems to be by design: https://github.blog/changelog/2021-11-10-github-actions-input-types-for-manual-workflows/ + default: '2' + +jobs: + workflow-dispatch-tests: + runs-on: [self-hosted, linux, "${{ inputs.runner }}"] + steps: + - name: Echo input variable and message + run: | + echo "Hello, runner: ${{ inputs.runner }}" + - name: Wait + run: | + sleep ${{ inputs.minutes }}m + - name: Always echo a message + if: always() + run: echo "Should not echo if pre-job script failed" From 28e7e99e895ea16280c15860e76df61722d62415 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Sun, 29 Jun 2025 09:25:53 +0000 Subject: [PATCH 58/75] chore: move integration test script --- .github/workflows/integration_test.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index a771b21c76..ef5659bc64 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -18,7 +18,6 @@ jobs: secrets: inherit with: juju-channel: 3.6/stable - pre-run-script: scripts/setup-integration-tests.sh provider: lxd test-tox-env: integration-juju3.6 modules: '["test_runner_manager_openstack"]' @@ -31,7 +30,7 @@ jobs: secrets: inherit with: juju-channel: 3.6/stable - pre-run-script: scripts/setup-integration-tests.sh + pre-run-script: tests/integration/setup-integration-tests.sh provider: lxd test-tox-env: integration-juju3.6 modules: '["test_charm_metrics_failure", "test_charm_metrics_success", "test_charm_fork_repo", "test_charm_fork_path_change", "test_charm_no_runner", "test_charm_runner", "test_debug_ssh", "test_charm_upgrade", "test_reactive", "test_jobmanager_prespawned", "test_jobmanager_reactive", "test_prometheus_metrics.py"]' From c5e96ec0afb60bd1c73cb51c7123f0bb895a3ea5 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Sun, 29 Jun 2025 15:25:05 +0000 Subject: [PATCH 59/75] ci: separate tests depending on k8s --- .github/workflows/integration_test.yaml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index ef5659bc64..6233d646ec 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -24,6 +24,18 @@ jobs: extra-arguments: '--log-format="%(asctime)s %(levelname)s %(message)s"' self-hosted-runner: true self-hosted-runner-label: stg-private-endpoint + openstack-integration-tests-private-endpoint: + name: Integration test using private-endpoint + uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main + secrets: inherit + with: + juju-channel: 3.6/stable + provider: lxd + test-tox-env: integration-juju3.6 + modules: '["test_charm_metrics_failure", "test_charm_metrics_success", "test_charm_fork_repo", "test_charm_fork_path_change", "test_charm_no_runner", "test_charm_runner", "test_debug_ssh", "test_charm_upgrade", "test_reactive", "test_jobmanager_prespawned", "test_jobmanager_reactive"]' + extra-arguments: '-m openstack --log-format="%(asctime)s %(levelname)s %(message)s"' + self-hosted-runner: true + self-hosted-runner-label: stg-private-endpoint openstack-integration-tests-private-endpoint: name: Integration test using private-endpoint uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main @@ -33,7 +45,7 @@ jobs: pre-run-script: tests/integration/setup-integration-tests.sh provider: lxd test-tox-env: integration-juju3.6 - modules: '["test_charm_metrics_failure", "test_charm_metrics_success", "test_charm_fork_repo", "test_charm_fork_path_change", "test_charm_no_runner", "test_charm_runner", "test_debug_ssh", "test_charm_upgrade", "test_reactive", "test_jobmanager_prespawned", "test_jobmanager_reactive", "test_prometheus_metrics.py"]' + modules: '["test_prometheus_metrics.py"]' extra-arguments: '-m openstack --log-format="%(asctime)s %(levelname)s %(message)s"' self-hosted-runner: true self-hosted-runner-label: stg-private-endpoint From 80edf02acf8add3deaf8e464d5d0471fa01d7d55 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Sun, 29 Jun 2025 16:04:11 +0000 Subject: [PATCH 60/75] test: fix lints --- tests/integration/helpers/common.py | 1 - tests/integration/setup-integration-tests.sh | 2 +- tests/integration/test_prometheus_metrics.py | 8 ++++---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/integration/helpers/common.py b/tests/integration/helpers/common.py index 1f0c5f5d1e..a453b6ebe3 100644 --- a/tests/integration/helpers/common.py +++ b/tests/integration/helpers/common.py @@ -22,7 +22,6 @@ from github.WorkflowRun import WorkflowRun from juju.action import Action from juju.application import Application -from juju.client._definitions import ApplicationStatus, FullStatus, UnitStatus from juju.model import Model from juju.unit import Unit diff --git a/tests/integration/setup-integration-tests.sh b/tests/integration/setup-integration-tests.sh index fc319254e1..94568ce51f 100755 --- a/tests/integration/setup-integration-tests.sh +++ b/tests/integration/setup-integration-tests.sh @@ -14,7 +14,7 @@ echo "bootstrapping microk8s juju controller" sudo snap install microk8s --channel=1.32-strict/stable GROUP=snap_microk8s sudo usermod -a -G $GROUP $USER -if [ $(id -gn) != $GROUP ]; then +if [ $(id -gn) != "$GROUP" ]; then exec sg $GROUP "$0 $*" fi sudo microk8s enable hostpath-storage diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index d574473c23..c626ea7ffe 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -65,7 +65,7 @@ def prometheus_app_fixture(k8s_juju: jubilant.Juju): ) assert ( result.returncode == 0 - ), f"failed to create prometheus offer: {result.stdout} {result.stderr}" + ), f"failed to create prometheus offer: {str(result.stdout)} {str(result.stderr)}" return k8s_juju.status().apps["prometheus-k8s"] @@ -86,7 +86,7 @@ def grafana_app_fixture(k8s_juju: jubilant.Juju, prometheus_app: AppStatus): ) assert ( result.returncode == 0 - ), f"failed to create grafana offer: {result.stdout} {result.stderr}" + ), f"failed to create grafana offer: {str(result.stdout)} {str(result.stderr)}" return k8s_juju.status().apps["grafana-k8s"] @@ -148,7 +148,7 @@ def test_prometheus_metrics( ) assert ( result.returncode == 0 - ), f"failed to consume prometheus offer: {result.stdout} {result.stderr}" + ), f"failed to consume prometheus offer: {str(result.stdout)} {str(result.stderr)}" result = subprocess.run( [ k8s_juju.cli_binary, @@ -160,7 +160,7 @@ def test_prometheus_metrics( ) assert ( result.returncode == 0 - ), f"failed to consume grafana offer: {result.stdout} {result.stderr}" + ), f"failed to consume grafana offer: {str(result.stdout)} {str(result.stderr)}" juju.integrate("grafana-agent", prometheus_offer_name) juju.integrate("grafana-agent", grafana_offer_name) From 121795c977583cdce41e1e19f10ac0c691c950ee Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Sun, 29 Jun 2025 18:12:44 +0000 Subject: [PATCH 61/75] fix: lint --- .../src/github_runner_manager/http_server.py | 6 +++++- tests/integration/setup-integration-tests.sh | 16 +++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/github-runner-manager/src/github_runner_manager/http_server.py b/github-runner-manager/src/github_runner_manager/http_server.py index 6b852d651a..58b6cbcb9f 100644 --- a/github-runner-manager/src/github_runner_manager/http_server.py +++ b/github-runner-manager/src/github_runner_manager/http_server.py @@ -106,7 +106,11 @@ def _get_lock() -> Lock: @app.route("/metrics", methods=["GET"]) def metrics(): - """Return prometheus metrics from default registry.""" + """Return prometheus metrics from default registry. + + Returns: + The latest metrics from the default Prometheus registry. + """ return generate_latest() diff --git a/tests/integration/setup-integration-tests.sh b/tests/integration/setup-integration-tests.sh index 94568ce51f..bf9e47942f 100755 --- a/tests/integration/setup-integration-tests.sh +++ b/tests/integration/setup-integration-tests.sh @@ -10,19 +10,21 @@ # save original controller that is used for testing ORIGINAL_CONTROLLER=$(juju controllers --format json | jq -r '.controllers | keys | .[0]') + echo "bootstrapping microk8s juju controller" sudo snap install microk8s --channel=1.32-strict/stable GROUP=snap_microk8s -sudo usermod -a -G $GROUP $USER -if [ $(id -gn) != "$GROUP" ]; then - exec sg $GROUP "$0 $*" +sudo usermod -a -G "$GROUP" "$USER" +if [ "$(id -gn)" != "$GROUP" ]; then + exec sg "$GROUP" "$0" "$*" fi -sudo microk8s enable hostpath-storage -IPADDR=$(ip -4 -j route get 2.2.2.2 | jq -r '.[] | .prefsrc') -sudo microk8s enable metallb:$IPADDR-$IPADDR + +# Get preferred source IP address for metallb +IPADDR=$( { ip -4 -j route get 2.2.2.2; jq -r '.[] | .prefsrc'; } ) +sudo microk8s enable "metallb:$IPADDR-$IPADDR" "hostpath-storage" microk8s status --wait-ready unset JUJU_CONTROLLER unset JUJU_MODEL juju bootstrap microk8s microk8s -juju switch $ORIGINAL_CONTROLLER +juju switch "$ORIGINAL_CONTROLLER" From f7214a297ab7bf308fdbc4663cfd3bb89b38ddfa Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Sun, 29 Jun 2025 18:28:55 +0000 Subject: [PATCH 62/75] fix: lint --- github-runner-manager/src/github_runner_manager/http_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github-runner-manager/src/github_runner_manager/http_server.py b/github-runner-manager/src/github_runner_manager/http_server.py index 58b6cbcb9f..5c687c9615 100644 --- a/github-runner-manager/src/github_runner_manager/http_server.py +++ b/github-runner-manager/src/github_runner_manager/http_server.py @@ -105,7 +105,7 @@ def _get_lock() -> Lock: @app.route("/metrics", methods=["GET"]) -def metrics(): +def metrics() -> bytes: """Return prometheus metrics from default registry. Returns: From 7ab740dfa305114cb13149539c1b4f12a5359c27 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Mon, 30 Jun 2025 02:06:54 +0000 Subject: [PATCH 63/75] ci: remove unused setup script --- .github/workflows/e2e_test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/e2e_test.yaml b/.github/workflows/e2e_test.yaml index 60fdbacbdb..ca84e62bae 100644 --- a/.github/workflows/e2e_test.yaml +++ b/.github/workflows/e2e_test.yaml @@ -16,7 +16,6 @@ jobs: secrets: inherit with: juju-channel: 3.6/stable - pre-run-script: scripts/setup-integration-tests.sh provider: lxd test-tox-env: integration-juju3.6 modules: '["test_e2e"]' From 022b3a87b22341f999d9dccfeba4978f697eb600 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Mon, 30 Jun 2025 02:09:46 +0000 Subject: [PATCH 64/75] docs: add changelog & increment version --- docs/changelog.md | 4 ++++ github-runner-manager/pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index 68d45edcbd..3eed6d38e9 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,10 @@ This changelog documents user-relevant changes to the GitHub runner charm. +## 2025-06-30 + +- Added prometheus metrics to the GitHub runner manager application. + ## 2025-06-26 - Fix a process leak internal to the charm. diff --git a/github-runner-manager/pyproject.toml b/github-runner-manager/pyproject.toml index 86f71efcc9..f55569f29f 100644 --- a/github-runner-manager/pyproject.toml +++ b/github-runner-manager/pyproject.toml @@ -3,7 +3,7 @@ [project] name = "github-runner-manager" -version = "0.5.0" +version = "0.6.0" authors = [ { name = "Canonical IS DevOps", email = "is-devops-team@canonical.com" }, ] From 549c61d87f58424c98533285b57c528793dc86e2 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Mon, 30 Jun 2025 06:44:10 +0000 Subject: [PATCH 65/75] ci: separate test names --- .github/workflows/integration_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 6233d646ec..99c21f905d 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -12,7 +12,7 @@ concurrency: cancel-in-progress: true jobs: - openstack-interface-tests-private-endpoint: + openstack-interface-tests-runner-manager-private-endpoint: name: openstack interface test using private-endpoint uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main secrets: inherit @@ -36,7 +36,7 @@ jobs: extra-arguments: '-m openstack --log-format="%(asctime)s %(levelname)s %(message)s"' self-hosted-runner: true self-hosted-runner-label: stg-private-endpoint - openstack-integration-tests-private-endpoint: + openstack-integration-tests-cross-controller-private-endpoint: name: Integration test using private-endpoint uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main secrets: inherit From 7152672473e7e1ca29cb410a821514e2a9057d18 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Mon, 30 Jun 2025 06:44:59 +0000 Subject: [PATCH 66/75] chore: refactor the metrics to be fetched without side effects --- .../src/github_runner_manager/http_server.py | 5 ++++- .../src/github_runner_manager/manager/runner_manager.py | 2 +- .../src/github_runner_manager/manager/runner_scaler.py | 4 ---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/github-runner-manager/src/github_runner_manager/http_server.py b/github-runner-manager/src/github_runner_manager/http_server.py index 5c687c9615..a382b51959 100644 --- a/github-runner-manager/src/github_runner_manager/http_server.py +++ b/github-runner-manager/src/github_runner_manager/http_server.py @@ -17,6 +17,7 @@ from github_runner_manager.configuration import ApplicationConfiguration from github_runner_manager.errors import CloudError, LockError from github_runner_manager.manager.runner_manager import FlushMode +from github_runner_manager.metrics.reconcile import BUSY_RUNNERS_COUNT, IDLE_RUNNERS_COUNT from github_runner_manager.reconcile_service import get_runner_scaler APP_CONFIG_NAME = "app_config" @@ -45,11 +46,13 @@ def check_runner() -> tuple[str, int]: Returns: Information on the runners in JSON format. """ - app_config = app.config[APP_CONFIG_NAME] + app_config: ApplicationConfiguration = app.config[APP_CONFIG_NAME] app.logger.info("Checking runners...") runner_scaler = get_runner_scaler(app_config) try: runner_info = runner_scaler.get_runner_info() + BUSY_RUNNERS_COUNT.labels(app_config.name).set(runner_info.busy) + IDLE_RUNNERS_COUNT.labels(app_config.name).set(runner_info.online) except CloudError as err: app.logger.exception("Cloud error encountered while getting runner info") return (str(err), 500) diff --git a/github-runner-manager/src/github_runner_manager/manager/runner_manager.py b/github-runner-manager/src/github_runner_manager/manager/runner_manager.py index b6e143a9f9..dfd5b51de5 100644 --- a/github-runner-manager/src/github_runner_manager/manager/runner_manager.py +++ b/github-runner-manager/src/github_runner_manager/manager/runner_manager.py @@ -301,7 +301,6 @@ def _cleanup_resources( ) cloud_runners_to_delete = cloud_runners_to_delete[:maximum_runners_to_delete] - CLEANED_RUNNERS_TOTAL.labels(self.manager_name).inc(len(cloud_runners_to_delete)) return self._delete_cloud_runners( cloud_runners_to_delete, runners_health_response.requested_runners, @@ -354,6 +353,7 @@ def _delete_cloud_runners( logger.error("No metrics returned after deleting %s", cloud_runner.instance_id) else: extracted_runner_metrics.append(runner_metric) + CLEANED_RUNNERS_TOTAL.labels(self.manager_name).inc(len(cloud_runners)) return extracted_runner_metrics def _clean_platform_runners(self, runners: list[RunnerIdentity]) -> None: diff --git a/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py b/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py index 50eb77b85c..f57d7104b5 100644 --- a/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py +++ b/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py @@ -26,9 +26,7 @@ ) from github_runner_manager.metrics import events as metric_events from github_runner_manager.metrics.reconcile import ( - BUSY_RUNNERS_COUNT, EXPECTED_RUNNERS_COUNT, - IDLE_RUNNERS_COUNT, RECONCILE_DURATION_SECONDS, ) from github_runner_manager.openstack_cloud.models import OpenStackServerConfig @@ -248,8 +246,6 @@ def get_runner_info(self) -> RunnerInfo: offline += 1 case _: unknown += 1 - BUSY_RUNNERS_COUNT.labels(self._manager.manager_name).set(busy) - IDLE_RUNNERS_COUNT.labels(self._manager.manager_name).set(online) return RunnerInfo( online=online, busy=busy, From 3e55bc1305821de9cdf3d2d8f51501c552083851 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Mon, 30 Jun 2025 07:47:49 +0000 Subject: [PATCH 67/75] chore: increment metrics by 1 --- .../src/github_runner_manager/manager/runner_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github-runner-manager/src/github_runner_manager/manager/runner_manager.py b/github-runner-manager/src/github_runner_manager/manager/runner_manager.py index dfd5b51de5..aecd28ba39 100644 --- a/github-runner-manager/src/github_runner_manager/manager/runner_manager.py +++ b/github-runner-manager/src/github_runner_manager/manager/runner_manager.py @@ -349,11 +349,11 @@ def _delete_cloud_runners( logging.info("Delete runner in cloud: %s", cloud_runner.instance_id) runner_metric = self._cloud.delete_runner(cloud_runner.instance_id) + CLEANED_RUNNERS_TOTAL.labels(self.manager_name).inc(1) if not runner_metric: logger.error("No metrics returned after deleting %s", cloud_runner.instance_id) else: extracted_runner_metrics.append(runner_metric) - CLEANED_RUNNERS_TOTAL.labels(self.manager_name).inc(len(cloud_runners)) return extracted_runner_metrics def _clean_platform_runners(self, runners: list[RunnerIdentity]) -> None: From af86e5148f70d80573ba74fe330bd5d4bebcd88b Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Mon, 30 Jun 2025 08:20:56 +0000 Subject: [PATCH 68/75] chore: move code --- .../src/github_runner_manager/http_server.py | 3 --- .../src/github_runner_manager/manager/runner_scaler.py | 9 +++++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/github-runner-manager/src/github_runner_manager/http_server.py b/github-runner-manager/src/github_runner_manager/http_server.py index a382b51959..9ea7c4751f 100644 --- a/github-runner-manager/src/github_runner_manager/http_server.py +++ b/github-runner-manager/src/github_runner_manager/http_server.py @@ -17,7 +17,6 @@ from github_runner_manager.configuration import ApplicationConfiguration from github_runner_manager.errors import CloudError, LockError from github_runner_manager.manager.runner_manager import FlushMode -from github_runner_manager.metrics.reconcile import BUSY_RUNNERS_COUNT, IDLE_RUNNERS_COUNT from github_runner_manager.reconcile_service import get_runner_scaler APP_CONFIG_NAME = "app_config" @@ -51,8 +50,6 @@ def check_runner() -> tuple[str, int]: runner_scaler = get_runner_scaler(app_config) try: runner_info = runner_scaler.get_runner_info() - BUSY_RUNNERS_COUNT.labels(app_config.name).set(runner_info.busy) - IDLE_RUNNERS_COUNT.labels(app_config.name).set(runner_info.online) except CloudError as err: app.logger.exception("Cloud error encountered while getting runner info") return (str(err), 500) diff --git a/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py b/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py index f57d7104b5..5198381da5 100644 --- a/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py +++ b/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py @@ -26,7 +26,9 @@ ) from github_runner_manager.metrics import events as metric_events from github_runner_manager.metrics.reconcile import ( + BUSY_RUNNERS_COUNT, EXPECTED_RUNNERS_COUNT, + IDLE_RUNNERS_COUNT, RECONCILE_DURATION_SECONDS, ) from github_runner_manager.openstack_cloud.models import OpenStackServerConfig @@ -327,7 +329,7 @@ def reconcile(self) -> int: RECONCILE_DURATION_SECONDS.labels(self._manager.manager_name).observe( end_timestamp - start_timestamp ) - _issue_reconciliation_metric(reconcile_metric_data) + _issue_reconciliation_metric(reconcile_metric_data, self._manager.manager_name) logger.info("Finished reconciliation.") @@ -409,7 +411,7 @@ def _log_runners(runner_list: tuple[RunnerInstance]) -> None: def _issue_reconciliation_metric( - reconcile_metric_data: _ReconcileMetricData, + reconcile_metric_data: _ReconcileMetricData, manager_name: str ) -> None: """Issue the reconciliation metric. @@ -437,6 +439,9 @@ def _issue_reconciliation_metric( logger.info("Current available runners (idle + healthy offline): %s", available_runners) logger.info("Current active runners: %s", active_runners) + BUSY_RUNNERS_COUNT.labels(manager_name).set(len(active_runners)) + IDLE_RUNNERS_COUNT.labels(manager_name).set(len(idle_runners)) + try: metric_events.issue_event( From 1e7b781711053cafbc958665bd805f00f89c7d57 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Mon, 30 Jun 2025 10:03:33 +0000 Subject: [PATCH 69/75] fix: lint fixe --- .../src/github_runner_manager/manager/runner_scaler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py b/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py index 5198381da5..f33920edb1 100644 --- a/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py +++ b/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py @@ -417,6 +417,7 @@ def _issue_reconciliation_metric( Args: reconcile_metric_data: The data used to issue the reconciliation metric. + manager_name: The name of the manager. """ idle_runners = { runner.name From b7be50b0a6d5f238913d5392b2a8922a3d9963c7 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Tue, 1 Jul 2025 07:47:47 +0200 Subject: [PATCH 70/75] fix integration test wf file --- .github/workflows/integration_test.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 99c21f905d..5dc41d4bf1 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -52,6 +52,7 @@ jobs: allure-report: if: ${{ (success() || failure()) && github.event_name == 'schedule' }} needs: - - openstack-interface-tests-private-endpoint + - openstack-interface-tests-runner-manager-private-endpoint - openstack-integration-tests-private-endpoint + - openstack-integration-tests-cross-controller-private-endpoint uses: canonical/operator-workflows/.github/workflows/allure_report.yaml@main From bd463bb99954e0d4b4dba015bdaeca4323ebcbc3 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 2 Jul 2025 02:28:18 +0000 Subject: [PATCH 71/75] ci: update module name for prometheus metrics test --- .github/workflows/integration_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 5dc41d4bf1..cc8d9b4f33 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -45,7 +45,7 @@ jobs: pre-run-script: tests/integration/setup-integration-tests.sh provider: lxd test-tox-env: integration-juju3.6 - modules: '["test_prometheus_metrics.py"]' + modules: '["test_prometheus_metrics"]' extra-arguments: '-m openstack --log-format="%(asctime)s %(levelname)s %(message)s"' self-hosted-runner: true self-hosted-runner-label: stg-private-endpoint From 7ff9c70dea60af3fc02ad2d977179d350dfce779 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 2 Jul 2025 02:40:25 +0000 Subject: [PATCH 72/75] ci: update cross controller test name --- .github/workflows/integration_test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index cc8d9b4f33..48f47ef637 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -37,7 +37,7 @@ jobs: self-hosted-runner: true self-hosted-runner-label: stg-private-endpoint openstack-integration-tests-cross-controller-private-endpoint: - name: Integration test using private-endpoint + name: Cross controller integration test using private-endpoint uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main secrets: inherit with: From f53e31e6199118e427fc672f054c56025d987ec6 Mon Sep 17 00:00:00 2001 From: yanksyoon Date: Wed, 2 Jul 2025 20:50:52 +0800 Subject: [PATCH 73/75] Update setup-integration-tests.sh --- tests/integration/setup-integration-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/setup-integration-tests.sh b/tests/integration/setup-integration-tests.sh index bf9e47942f..c905cf305b 100755 --- a/tests/integration/setup-integration-tests.sh +++ b/tests/integration/setup-integration-tests.sh @@ -20,7 +20,7 @@ if [ "$(id -gn)" != "$GROUP" ]; then fi # Get preferred source IP address for metallb -IPADDR=$( { ip -4 -j route get 2.2.2.2; jq -r '.[] | .prefsrc'; } ) +IPADDR=$( { ip -4 -j route get 2.2.2.2 | jq -r '.[] | .prefsrc'; } ) sudo microk8s enable "metallb:$IPADDR-$IPADDR" "hostpath-storage" microk8s status --wait-ready From 6bf81a71c8d4efdf3cf9729687699e1cb0b26b44 Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 9 Jul 2025 02:37:39 +0000 Subject: [PATCH 74/75] test: test for busy/idle runner count (cleanup not happen when no vm) --- tests/integration/test_prometheus_metrics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index c626ea7ffe..8c871372b1 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -180,7 +180,8 @@ def test_prometheus_metrics( "openstack_http_requests_total", "reconcile_duration_seconds_sum", "expected_runners_count", - "cleaned_runners_total", + "busy_runners_count", + "idle_runners_count", ) From 6ae9a0d496a2727b03ab9524b9f8b97c0d9c65be Mon Sep 17 00:00:00 2001 From: charlie4284 Date: Wed, 9 Jul 2025 09:06:49 +0000 Subject: [PATCH 75/75] test: remove duplicate test module --- .github/workflows/integration_test.yaml | 13 - .../test_runner_manager_openstack.py | 584 ------------------ 2 files changed, 597 deletions(-) delete mode 100644 tests/integration/test_runner_manager_openstack.py diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 48f47ef637..649d47df23 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -12,18 +12,6 @@ concurrency: cancel-in-progress: true jobs: - openstack-interface-tests-runner-manager-private-endpoint: - name: openstack interface test using private-endpoint - uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main - secrets: inherit - with: - juju-channel: 3.6/stable - provider: lxd - test-tox-env: integration-juju3.6 - modules: '["test_runner_manager_openstack"]' - extra-arguments: '--log-format="%(asctime)s %(levelname)s %(message)s"' - self-hosted-runner: true - self-hosted-runner-label: stg-private-endpoint openstack-integration-tests-private-endpoint: name: Integration test using private-endpoint uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main @@ -52,7 +40,6 @@ jobs: allure-report: if: ${{ (success() || failure()) && github.event_name == 'schedule' }} needs: - - openstack-interface-tests-runner-manager-private-endpoint - openstack-integration-tests-private-endpoint - openstack-integration-tests-cross-controller-private-endpoint uses: canonical/operator-workflows/.github/workflows/allure_report.yaml@main diff --git a/tests/integration/test_runner_manager_openstack.py b/tests/integration/test_runner_manager_openstack.py deleted file mode 100644 index 2058a7d2fe..0000000000 --- a/tests/integration/test_runner_manager_openstack.py +++ /dev/null @@ -1,584 +0,0 @@ -# Copyright 2025 Canonical Ltd. -# See LICENSE file for licensing details. - -"""Testing the RunnerManager class with OpenStackRunnerManager as CloudManager. -It is assumed that the test runs in the CI under the ubuntu user. -""" - - -import asyncio -import json -import logging -from pathlib import Path -from secrets import token_hex -from typing import AsyncGenerator, Iterator - -import pytest -import pytest_asyncio -import yaml -from github.Branch import Branch -from github.Repository import Repository -from github.Workflow import Workflow -from github_runner_manager.configuration import ProxyConfig, SupportServiceConfig, UserInfo -from github_runner_manager.configuration.github import GitHubPath, parse_github_path -from github_runner_manager.github_client import GithubClient -from github_runner_manager.manager.cloud_runner_manager import CloudRunnerState -from github_runner_manager.manager.models import RunnerMetadata -from github_runner_manager.manager.runner_manager import FlushMode, RunnerManager -from github_runner_manager.metrics import events -from github_runner_manager.openstack_cloud import constants -from github_runner_manager.openstack_cloud.models import ( - OpenStackCredentials, - OpenStackRunnerManagerConfig, - OpenStackServerConfig, -) -from github_runner_manager.openstack_cloud.openstack_runner_manager import OpenStackRunnerManager -from github_runner_manager.platform.github_provider import ( - GitHubRunnerPlatform, - PlatformRunnerState, -) -from github_runner_manager.types_.github import GitHubRunnerStatus -from openstack.connection import Connection as OpenstackConnection - -from tests.integration.helpers.common import ( - DISPATCH_WAIT_TEST_WORKFLOW_FILENAME, - dispatch_workflow, - wait_for, -) - -logger = logging.getLogger(__name__) - -# A higher create server timeout is reasonable for integration tests, -# as only one machine that stays for more than the default time in BUILD, -# will break the tests -constants.CREATE_SERVER_TIMEOUT = 900 - - -@pytest.fixture(autouse=True, scope="module", name="runner_manager_user") -def runner_manager_user(): - """Mock the RUNNER_MANAGER_USER and RUNNER_MANAGER_GROUP constants. - - Yields: - None, just to be in a scope. - """ - with pytest.MonkeyPatch.context() as monkeypatch: - # we assume the test runs as ubuntu user - monkeypatch.setattr("github_runner_manager.constants.RUNNER_MANAGER_USER", "ubuntu") - monkeypatch.setattr("github_runner_manager.constants.RUNNER_MANAGER_GROUP", "ubuntu") - # monkeypatch is a scope function fixture, so this trick - yield None - - -@pytest.fixture(scope="module", name="runner_label") -def runner_label(): - return f"test-{token_hex(6)}" - - -@pytest.fixture(scope="module", name="log_dir_base_path") -def log_dir_base_path_fixture( - tmp_path_factory: pytest.TempPathFactory, -) -> Iterator[dict[str, Path]]: - """Mock the log directory path and return it.""" - with pytest.MonkeyPatch.context() as monkeypatch: - temp_log_dir = tmp_path_factory.mktemp("log") - - metric_log_path = temp_log_dir / "metric_log" - - monkeypatch.setattr(events, "METRICS_LOG_PATH", metric_log_path) - - yield { - "metric_log": metric_log_path, - } - - -@pytest.fixture(scope="module", name="prefix") -def prefix_fixture(app_name: str) -> str: - return f"{app_name}-0" - - -@pytest.fixture(scope="module", name="github_path") -def github_path_fixture(path: str) -> GitHubPath: - return parse_github_path(path, "Default") - - -@pytest.fixture(scope="module", name="proxy_config") -def openstack_proxy_config_fixture( - openstack_http_proxy: str, openstack_https_proxy: str, openstack_no_proxy: str -) -> ProxyConfig: - http_proxy = openstack_http_proxy if openstack_http_proxy else None - https_proxy = openstack_https_proxy if openstack_https_proxy else None - return ProxyConfig( - http=http_proxy, - https=https_proxy, - no_proxy=openstack_no_proxy, - ) - - -@pytest_asyncio.fixture(scope="module", name="openstack_runner_manager") -async def openstack_runner_manager_fixture( - app_name: str, - prefix: str, - private_endpoint_clouds_yaml: str, - openstack_test_image: str, - flavor_name: str, - network_name: str, - github_path: GitHubPath, - proxy_config: ProxyConfig, - runner_label: str, - openstack_connection: OpenstackConnection, -) -> AsyncGenerator[OpenStackRunnerManager, None]: - """Create OpenstackRunnerManager instance. - - The prefix args of OpenstackRunnerManager set to app_name to let openstack_connection_fixture - perform the cleanup of openstack resources. - """ - clouds_config = yaml.safe_load(private_endpoint_clouds_yaml) - - try: - # Pick the first cloud in the clouds.yaml - cloud = tuple(clouds_config["clouds"].values())[0] - print("============================================") - print(cloud) - print("============================================") - - credentials = OpenStackCredentials( - auth_url=cloud["auth"]["auth_url"], - project_name=cloud["auth"]["project_name"], - username=cloud["auth"]["username"], - password=cloud["auth"]["password"], - user_domain_name=cloud["auth"]["user_domain_name"], - project_domain_name=cloud["auth"]["project_domain_name"], - region_name=cloud["region_name"], - ) - except KeyError as err: - raise AssertionError("Issue with the format of the clouds.yaml used in test") from err - - server_config = OpenStackServerConfig( - image=openstack_test_image, - flavor=flavor_name, - network=network_name, - ) - - use_aproxy = bool(proxy_config.proxy_address) - - service_config = SupportServiceConfig( - proxy_config=proxy_config, - runner_proxy_config=proxy_config, - dockerhub_mirror=None, - ssh_debug_connections=[], - repo_policy_compliance=None, - use_aproxy=use_aproxy, - ) - - openstack_runner_manager_config = OpenStackRunnerManagerConfig( - name=app_name, - prefix=prefix, - credentials=credentials, - server_config=server_config, - service_config=service_config, - ) - user = UserInfo("ubuntu", "ubuntu") - - yield OpenStackRunnerManager( - config=openstack_runner_manager_config, - user=user, - ) - - -@pytest.fixture(scope="module", name="github_client_for_manager") -def github_client_for_manager_fixture(token: str) -> GithubClient: - github_client = GithubClient(token) - return github_client - - -@pytest.fixture(scope="module", name="github_platform") -def github_platform_fixture( - token: str, - prefix: str, - github_path: GitHubPath, - github_client_for_manager: GithubClient, -) -> GitHubRunnerPlatform: - github_platform = GitHubRunnerPlatform( - prefix=prefix, - path=github_path, - github_client=github_client_for_manager, - ) - return github_platform - - -@pytest_asyncio.fixture(scope="module", name="runner_manager") -async def runner_manager_fixture( - openstack_runner_manager: OpenStackRunnerManager, - log_dir_base_path: dict[str, Path], - runner_label: str, - github_platform: GitHubRunnerPlatform, -) -> AsyncGenerator[RunnerManager, None]: - """Get RunnerManager instance. - - Import of log_dir_base_path to monkeypatch the runner logs path with tmp_path. - """ - yield RunnerManager( - manager_name="test_runner", - platform_provider=github_platform, - cloud_runner_manager=openstack_runner_manager, - labels=["openstack_test", runner_label], - ) - - -@pytest_asyncio.fixture(scope="function", name="runner_manager_with_one_runner") -async def runner_manager_with_one_runner_fixture(runner_manager: RunnerManager) -> RunnerManager: - runner_manager.flush_runners(flush_mode=FlushMode.FLUSH_BUSY) - await wait_runner_amount(runner_manager, 0) - runner_manager.create_runners(1, RunnerMetadata()) - try: - await wait_runner_amount(runner_manager, 1) - except TimeoutError as err: - raise AssertionError("Test arrange failed: Expect one runner") from err - - runner_list = runner_manager.get_runners() - runner = runner_list[0] - assert ( - runner.cloud_state == CloudRunnerState.ACTIVE - ), "Test arrange failed: Expect runner in active state" - try: - await wait_for( - lambda: runner_manager.get_runners()[0].platform_state == PlatformRunnerState.IDLE, - timeout=1200, - check_interval=10, - ) - except TimeoutError as err: - raise AssertionError("Test arrange failed: Expect runner in idle state") from err - return runner_manager - - -def workflow_is_status(workflow: Workflow, status: str) -> bool: - """Check if workflow in provided status. - - Args: - workflow: The workflow to check. - status: The status to check for. - - Returns: - Whether the workflow is in the status. - """ - workflow.update() - return workflow.status == status - - -async def wait_runner_amount( - runner_manager: RunnerManager, num: int, timeout: int = 600, check_interval: int = 60 -) -> None: - """Wait until the runner manager has the number of runners. - - A TimeoutError will be thrown if runners amount is not correct after timeout. - - Args: - runner_manager: The RunnerManager to check. - num: Number of runner to check for. - timeout: The timeout in seconds. - check_interval: The interval to check in seconds. - """ - # The openstack server can take sometime to fully clean up or create. - await wait_for( - lambda: check_runners_amount_and_active(runner_manager, num), - timeout=timeout, - check_interval=check_interval, - ) - - -def check_runners_amount_and_active(runner_manager: RunnerManager, num: int) -> bool: - """Check if the number of runners match the expected amount and all runners are active. - - Args: - runner_manager: The RunnerManager instance to use. - num: The expected number of runners. - - Returns: - Whether the expected number of runner is spawned and active. - """ - runners = runner_manager.get_runners() - active_runners = [ - runner for runner in runners if runner.cloud_state == CloudRunnerState.ACTIVE - ] - if len(runners) == len(active_runners) and len(runners) == num: - return True - return False - - -@pytest.mark.openstack -@pytest.mark.asyncio -@pytest.mark.abort_on_fail -async def test_get_no_runner(runner_manager: RunnerManager) -> None: - """ - Arrange: RunnerManager instance with no runners. - Act: Get runners. - Assert: Empty tuple returned. - """ - runner_list = runner_manager.get_runners() - assert isinstance(runner_list, tuple) - assert not runner_list - - -@pytest.mark.openstack -@pytest.mark.asyncio -@pytest.mark.abort_on_fail -async def test_runner_normal_idle_lifecycle( - runner_manager: RunnerManager, - openstack_runner_manager: OpenStackRunnerManager, - github_client_for_manager: GithubClient, - github_path: GitHubPath, -) -> None: - """ - Arrange: RunnerManager instance with no runners. - Act: - 1. Create one runner. - 2. Run health check on the runner. - 3. Run cleanup. - 4. Delete all idle runner. - Assert: - 1. An active idle runner. - 2. Health check passes. - 3. One idle runner remains. - 4. No runners. - """ - # 1. - runner_id_list = runner_manager.create_runners(1, RunnerMetadata()) - assert isinstance(runner_id_list, tuple) - assert len(runner_id_list) == 1 - runner_id = runner_id_list[0] - - try: - await wait_runner_amount(runner_manager, 1) - except TimeoutError as err: - raise AssertionError("Test arrange failed: Expect one runner") from err - - runner_list = runner_manager.get_runners() - assert isinstance(runner_list, tuple) - assert len(runner_list) == 1 - runner = runner_list[0] - assert runner.instance_id == runner_id - assert runner.cloud_state == CloudRunnerState.ACTIVE - assert runner.metadata.platform_name == "github" - # Update on GitHub-side can take a bit of time. - await wait_for( - lambda: runner_manager.get_runners()[0].platform_state == PlatformRunnerState.IDLE, - timeout=120, - check_interval=10, - ) - - # 2. - openstack_instances = openstack_runner_manager._openstack_cloud.get_instances() - - assert len(openstack_instances) == 1, "Test arrange failed: Needs one runner." - runner = openstack_instances[0] - - self_hosted_runner = github_client_for_manager.get_runner( - github_path, runner.instance_id.prefix, int(runner.metadata.runner_id) - ) - assert self_hosted_runner.status == GitHubRunnerStatus.ONLINE - - # 3. - runner_manager.cleanup() - runner_list = runner_manager.get_runners() - assert isinstance(runner_list, tuple) - assert len(runner_list) == 1 - runner = runner_list[0] - assert runner.instance_id == runner_id - assert runner.cloud_state == CloudRunnerState.ACTIVE - - # 4. - runner_manager.flush_runners(flush_mode=FlushMode.FLUSH_IDLE) - await wait_runner_amount(runner_manager, 0) - - -@pytest.mark.openstack -@pytest.mark.asyncio -@pytest.mark.abort_on_fail -async def test_runner_flush_busy_lifecycle( - runner_manager_with_one_runner: RunnerManager, - test_github_branch: Branch, - github_repository: Repository, - runner_label: str, -): - """ - Arrange: RunnerManager with one idle runner. - Act: - 1. Run a long workflow. - 3. Run flush idle runner. - 4. Run flush busy runner. - Assert: - 1. Runner takes the job and become busy. - 3. Busy runner still exists. - 4. No runners exists. - """ - # 1. - workflow = await dispatch_workflow( - app=None, - branch=test_github_branch, - github_repository=github_repository, - conclusion="success", - workflow_id_or_name=DISPATCH_WAIT_TEST_WORKFLOW_FILENAME, - dispatch_input={"runner": runner_label, "minutes": "30"}, - wait=False, - ) - await wait_for(lambda: workflow_is_status(workflow, "in_progress")) - - runner_list = runner_manager_with_one_runner.get_runners() - assert len(runner_list) == 1 - busy_runner = runner_list[0] - assert busy_runner.cloud_state == CloudRunnerState.ACTIVE - assert busy_runner.platform_state == PlatformRunnerState.BUSY - - # 2. - runner_manager_with_one_runner.cleanup() - runner_list = runner_manager_with_one_runner.get_runners() - assert isinstance(runner_list, tuple) - assert len(runner_list) == 1 - runner = runner_list[0] - assert runner.cloud_state == CloudRunnerState.ACTIVE - assert busy_runner.platform_state == PlatformRunnerState.BUSY - - # 3. - runner_manager_with_one_runner.flush_runners(flush_mode=FlushMode.FLUSH_IDLE) - runner_list = runner_manager_with_one_runner.get_runners() - assert len(runner_list) == 1 - busy_runner = runner_list[0] - assert busy_runner.cloud_state == CloudRunnerState.ACTIVE - assert busy_runner.platform_state == PlatformRunnerState.BUSY - - # 4. - runner_manager_with_one_runner.flush_runners(flush_mode=FlushMode.FLUSH_BUSY) - # It takes a bit for the github agent to die, and it may not be cleaned - # in the first run. Just do it twice. - await asyncio.sleep(10) - runner_manager_with_one_runner.flush_runners(flush_mode=FlushMode.FLUSH_BUSY) - await wait_runner_amount(runner_manager_with_one_runner, 0) - - -@pytest.mark.openstack -@pytest.mark.asyncio -@pytest.mark.abort_on_fail -async def test_runner_normal_lifecycle( - runner_manager_with_one_runner: RunnerManager, - test_github_branch: Branch, - github_repository: Repository, - runner_label: str, - log_dir_base_path: dict[str, Path], -): - """ - Arrange: RunnerManager with one runner. Clean metric logs. - Act: - 1. Start a test workflow for the runner. - 2. Run cleanup. - Assert: - 1. The workflow complete successfully. - 2. The runner should be deleted. The metrics should be recorded. - """ - logger.info("Starting test_runner_normal_lifecycle") - metric_log_path = log_dir_base_path["metric_log"] - try: - metric_log_existing_content = metric_log_path.read_text(encoding="utf-8") - except FileNotFoundError: - metric_log_existing_content = "" - - workflow = await dispatch_workflow( - app=None, - branch=test_github_branch, - github_repository=github_repository, - conclusion="success", - workflow_id_or_name=DISPATCH_WAIT_TEST_WORKFLOW_FILENAME, - dispatch_input={"runner": runner_label, "minutes": "0"}, - wait=False, - ) - await wait_for(lambda: workflow_is_status(workflow, "completed")) - - # We encountered a race condition where runner_manager.cleanup was called while - # there was no runner process, but the post-metrics still had not yet been issued. - # Make the test more robust by waiting for the runner to go offline - # to reduce the race condition. - def is_runner_offline() -> bool: - """Check if the runner is offline. - - Returns: - True if the runner is offline, False otherwise. - """ - runners = runner_manager_with_one_runner.get_runners() - assert len(runners) == 1 - return runners[0].platform_state in (PlatformRunnerState.OFFLINE, None) - - await wait_for(is_runner_offline, check_interval=60, timeout=600) - - def have_metrics_been_issued() -> bool: - """Check if the expected metrics have been issued. - - Returns: - True if the expected metrics have been issued, False otherwise. - """ - issued_metrics_events = runner_manager_with_one_runner.cleanup() - logger.info("issued_metrics_events: %s", issued_metrics_events) - return ( - {events.RunnerInstalled, events.RunnerStart, events.RunnerStop} - == set(issued_metrics_events) - and issued_metrics_events[events.RunnerInstalled] == 1 - and issued_metrics_events[events.RunnerStart] == 1 - and issued_metrics_events[events.RunnerStop] == 1 - ) - - try: - await wait_for(have_metrics_been_issued, check_interval=60, timeout=600) - except TimeoutError: - assert False, "The expected metrics were not issued" - - metric_log_full_content = metric_log_path.read_text(encoding="utf-8") - assert metric_log_full_content.startswith( - metric_log_existing_content - ), "The metric log was modified in ways other than appending" - metric_log_new_content = metric_log_full_content[len(metric_log_existing_content) :] - metric_logs = [json.loads(metric) for metric in metric_log_new_content.splitlines()] - assert len(metric_logs) == 3, ( - "Assuming three events " - "should be runner_installed, runner_start and runner_stop, " - "modify this if new events are added" - ) - assert metric_logs[0]["event"] == "runner_installed" - assert metric_logs[0]["flavor"] == runner_manager_with_one_runner.manager_name - assert metric_logs[1]["event"] == "runner_start" - assert metric_logs[1]["workflow"] == "Workflow Dispatch Wait Tests" - assert metric_logs[2]["event"] == "runner_stop" - assert metric_logs[2]["workflow"] == "Workflow Dispatch Wait Tests" - - await wait_runner_amount(runner_manager_with_one_runner, 0) - - -@pytest.mark.openstack -@pytest.mark.asyncio -@pytest.mark.abort_on_fail -async def test_runner_spawn_two( - runner_manager: RunnerManager, openstack_runner_manager: OpenStackRunnerManager -) -> None: - """ - Arrange: RunnerManager instance with no runners. - Act: - 1. Create two runner. - 2. Delete all idle runner. - Assert: - 1. Two active idle runner. - 2. No runners. - """ - # 1. - runner_id_list = runner_manager.create_runners(2, RunnerMetadata()) - assert isinstance(runner_id_list, tuple) - assert len(runner_id_list) == 2 - - try: - await wait_runner_amount(runner_manager, 2) - except TimeoutError as err: - raise AssertionError("Test arrange failed: Expect two runner") from err - - runner_list = runner_manager.get_runners() - assert isinstance(runner_list, tuple) - assert len(runner_list) == 2 - - # 3. - runner_manager.flush_runners(flush_mode=FlushMode.FLUSH_IDLE) - await wait_runner_amount(runner_manager, 0)