diff --git a/.github/workflows/e2e_test.yaml b/.github/workflows/e2e_test.yaml index 60fdbacbdb..ca84e62bae 100644 --- a/.github/workflows/e2e_test.yaml +++ b/.github/workflows/e2e_test.yaml @@ -16,7 +16,6 @@ jobs: secrets: inherit with: juju-channel: 3.6/stable - pre-run-script: scripts/setup-integration-tests.sh provider: lxd test-tox-env: integration-juju3.6 modules: '["test_e2e"]' diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index b274e4d925..649d47df23 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -12,35 +12,34 @@ concurrency: cancel-in-progress: true jobs: - openstack-interface-tests-private-endpoint: - name: openstack interface test using private-endpoint + openstack-integration-tests-private-endpoint: + name: Integration test using private-endpoint uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main secrets: inherit with: juju-channel: 3.6/stable - pre-run-script: scripts/setup-integration-tests.sh provider: lxd test-tox-env: integration-juju3.6 - modules: '["test_runner_manager_openstack"]' - extra-arguments: '--log-format="%(asctime)s %(levelname)s %(message)s"' + modules: '["test_charm_metrics_failure", "test_charm_metrics_success", "test_charm_fork_repo", "test_charm_fork_path_change", "test_charm_no_runner", "test_charm_runner", "test_debug_ssh", "test_charm_upgrade", "test_reactive", "test_jobmanager_prespawned", "test_jobmanager_reactive"]' + extra-arguments: '-m openstack --log-format="%(asctime)s %(levelname)s %(message)s"' self-hosted-runner: true self-hosted-runner-label: stg-private-endpoint - openstack-integration-tests-private-endpoint: - name: Integration test using private-endpoint + openstack-integration-tests-cross-controller-private-endpoint: + name: Cross controller integration test using private-endpoint uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main secrets: inherit with: juju-channel: 3.6/stable - pre-run-script: scripts/setup-integration-tests.sh + pre-run-script: tests/integration/setup-integration-tests.sh provider: lxd test-tox-env: integration-juju3.6 - modules: '["test_charm_metrics_failure", "test_charm_metrics_success", "test_charm_fork_repo", "test_charm_fork_path_change", "test_charm_no_runner", "test_charm_runner", "test_debug_ssh", "test_charm_upgrade", "test_reactive", "test_jobmanager_prespawned", "test_jobmanager_reactive"]' + modules: '["test_prometheus_metrics"]' extra-arguments: '-m openstack --log-format="%(asctime)s %(levelname)s %(message)s"' self-hosted-runner: true self-hosted-runner-label: stg-private-endpoint allure-report: if: ${{ (success() || failure()) && github.event_name == 'schedule' }} needs: - - openstack-interface-tests-private-endpoint - openstack-integration-tests-private-endpoint + - openstack-integration-tests-cross-controller-private-endpoint uses: canonical/operator-workflows/.github/workflows/allure_report.yaml@main diff --git a/docs/changelog.md b/docs/changelog.md index e34be1901d..5d9604d59f 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -4,6 +4,7 @@ This changelog documents user-relevant changes to the GitHub runner charm. ### 2025-06-30 - New configuration options aproxy-exclude-addresses and aproxy-redirect-ports for allowing aproxy to redirect arbitrary TCP traffic +- Added prometheus metrics to the GitHub runner manager application. ## 2025-06-26 diff --git a/github-runner-manager/pyproject.toml b/github-runner-manager/pyproject.toml index 86f71efcc9..f55569f29f 100644 --- a/github-runner-manager/pyproject.toml +++ b/github-runner-manager/pyproject.toml @@ -3,7 +3,7 @@ [project] name = "github-runner-manager" -version = "0.5.0" +version = "0.6.0" authors = [ { name = "Canonical IS DevOps", email = "is-devops-team@canonical.com" }, ] diff --git a/github-runner-manager/requirements.txt b/github-runner-manager/requirements.txt index 38bb457784..a11a02eac5 100644 --- a/github-runner-manager/requirements.txt +++ b/github-runner-manager/requirements.txt @@ -1,9 +1,10 @@ +click==8.2.1 fabric >=3,<4 +flask==3.1.1 ghapi jinja2 kombu==5.5.3 openstacksdk==4.5.0 +prometheus-client==0.22.1 pydantic < 2 pymongo==4.13.0 -click==8.2.1 -flask==3.1.1 diff --git a/github-runner-manager/src/github_runner_manager/http_server.py b/github-runner-manager/src/github_runner_manager/http_server.py index be806a88fe..9ea7c4751f 100644 --- a/github-runner-manager/src/github_runner_manager/http_server.py +++ b/github-runner-manager/src/github_runner_manager/http_server.py @@ -12,6 +12,7 @@ from threading import Lock from flask import Flask, request +from prometheus_client import generate_latest from github_runner_manager.configuration import ApplicationConfiguration from github_runner_manager.errors import CloudError, LockError @@ -44,7 +45,7 @@ def check_runner() -> tuple[str, int]: Returns: Information on the runners in JSON format. """ - app_config = app.config[APP_CONFIG_NAME] + app_config: ApplicationConfiguration = app.config[APP_CONFIG_NAME] app.logger.info("Checking runners...") runner_scaler = get_runner_scaler(app_config) try: @@ -72,7 +73,7 @@ def flush_runner() -> tuple[str, int]: if flush_busy_str in ("True", "true"): flush_busy = True - lock = get_lock() + lock = _get_lock() with lock: app.logger.info("Flushing runners...") runner_scaler = get_runner_scaler(app_config) @@ -87,7 +88,7 @@ def flush_runner() -> tuple[str, int]: return ("", 204) -def get_lock() -> Lock: +def _get_lock() -> Lock: """Get the lock representing modification access to the set of runners. Raises: @@ -103,6 +104,16 @@ def get_lock() -> Lock: raise LockError("Lock not configured") +@app.route("/metrics", methods=["GET"]) +def metrics() -> bytes: + """Return prometheus metrics from default registry. + + Returns: + The latest metrics from the default Prometheus registry. + """ + return generate_latest() + + @dataclass class FlaskArgs: """Arguments for Flask HTTP server. diff --git a/github-runner-manager/src/github_runner_manager/manager/runner_manager.py b/github-runner-manager/src/github_runner_manager/manager/runner_manager.py index 8f20fc0183..aecd28ba39 100644 --- a/github-runner-manager/src/github_runner_manager/manager/runner_manager.py +++ b/github-runner-manager/src/github_runner_manager/manager/runner_manager.py @@ -23,6 +23,7 @@ from github_runner_manager.metrics import events as metric_events from github_runner_manager.metrics import github as github_metrics from github_runner_manager.metrics import runner as runner_metrics +from github_runner_manager.metrics.reconcile import CLEANED_RUNNERS_TOTAL from github_runner_manager.metrics.runner import RunnerMetrics from github_runner_manager.openstack_cloud.constants import CREATE_SERVER_TIMEOUT from github_runner_manager.platform.platform_provider import ( @@ -348,6 +349,7 @@ def _delete_cloud_runners( logging.info("Delete runner in cloud: %s", cloud_runner.instance_id) runner_metric = self._cloud.delete_runner(cloud_runner.instance_id) + CLEANED_RUNNERS_TOTAL.labels(self.manager_name).inc(1) if not runner_metric: logger.error("No metrics returned after deleting %s", cloud_runner.instance_id) else: diff --git a/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py b/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py index 34efa53e79..f33920edb1 100644 --- a/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py +++ b/github-runner-manager/src/github_runner_manager/manager/runner_scaler.py @@ -8,10 +8,7 @@ from dataclasses import dataclass import github_runner_manager.reactive.runner_manager as reactive_runner_manager -from github_runner_manager.configuration import ( - ApplicationConfiguration, - UserInfo, -) +from github_runner_manager.configuration import ApplicationConfiguration, UserInfo from github_runner_manager.constants import GITHUB_SELF_HOSTED_ARCH_LABELS from github_runner_manager.errors import ( CloudError, @@ -28,6 +25,12 @@ RunnerMetadata, ) from github_runner_manager.metrics import events as metric_events +from github_runner_manager.metrics.reconcile import ( + BUSY_RUNNERS_COUNT, + EXPECTED_RUNNERS_COUNT, + IDLE_RUNNERS_COUNT, + RECONCILE_DURATION_SECONDS, +) from github_runner_manager.openstack_cloud.models import OpenStackServerConfig from github_runner_manager.openstack_cloud.openstack_runner_manager import ( OpenStackRunnerManager, @@ -216,6 +219,8 @@ def __init__( # pylint: disable=too-many-arguments, too-many-positional-argumen self._platform_name = platform_name self._python_path = python_path + EXPECTED_RUNNERS_COUNT.labels(self._manager.manager_name).set(self._base_quantity) + def get_runner_info(self) -> RunnerInfo: """Get information on the runners. @@ -321,7 +326,10 @@ def reconcile(self) -> int: flavor=self._manager.manager_name, expected_runner_quantity=expected_runner_quantity, ) - _issue_reconciliation_metric(reconcile_metric_data) + RECONCILE_DURATION_SECONDS.labels(self._manager.manager_name).observe( + end_timestamp - start_timestamp + ) + _issue_reconciliation_metric(reconcile_metric_data, self._manager.manager_name) logger.info("Finished reconciliation.") @@ -403,12 +411,13 @@ def _log_runners(runner_list: tuple[RunnerInstance]) -> None: def _issue_reconciliation_metric( - reconcile_metric_data: _ReconcileMetricData, + reconcile_metric_data: _ReconcileMetricData, manager_name: str ) -> None: """Issue the reconciliation metric. Args: reconcile_metric_data: The data used to issue the reconciliation metric. + manager_name: The name of the manager. """ idle_runners = { runner.name @@ -431,6 +440,9 @@ def _issue_reconciliation_metric( logger.info("Current available runners (idle + healthy offline): %s", available_runners) logger.info("Current active runners: %s", active_runners) + BUSY_RUNNERS_COUNT.labels(manager_name).set(len(active_runners)) + IDLE_RUNNERS_COUNT.labels(manager_name).set(len(idle_runners)) + try: metric_events.issue_event( diff --git a/github-runner-manager/src/github_runner_manager/metrics/reconcile.py b/github-runner-manager/src/github_runner_manager/metrics/reconcile.py new file mode 100644 index 0000000000..dbd8df267e --- /dev/null +++ b/github-runner-manager/src/github_runner_manager/metrics/reconcile.py @@ -0,0 +1,34 @@ +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Module for collecting metrics related to the reconciliation process.""" + +from prometheus_client import Gauge, Histogram + +LABEL_FLAVOR = "flavor" + +RECONCILE_DURATION_SECONDS = Histogram( + name="reconcile_duration_seconds", + documentation="Duration of reconciliation (seconds)", + labelnames=[LABEL_FLAVOR], +) +EXPECTED_RUNNERS_COUNT = Gauge( + name="expected_runners_count", + documentation="Expected number of runners", + labelnames=[LABEL_FLAVOR], +) +BUSY_RUNNERS_COUNT = Gauge( + name="busy_runners_count", + documentation="Number of busy runners", + labelnames=[LABEL_FLAVOR], +) +IDLE_RUNNERS_COUNT = Gauge( + name="idle_runners_count", + documentation="Number of idle runners", + labelnames=[LABEL_FLAVOR], +) +CLEANED_RUNNERS_TOTAL = Gauge( + name="cleaned_runners_total", + documentation="Total number of runners cleaned up", + labelnames=[LABEL_FLAVOR], +) diff --git a/scripts/setup-integration-tests.sh b/scripts/setup-integration-tests.sh deleted file mode 100644 index c6a86227db..0000000000 --- a/scripts/setup-integration-tests.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2025 Canonical Ltd. -# See LICENSE file for licensing details. - -# Script to setup localhost for integration tests diff --git a/src/charm.py b/src/charm.py index 66a6e29283..419b40938d 100755 --- a/src/charm.py +++ b/src/charm.py @@ -186,7 +186,12 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) self._log_charm_status() - self._grafana_agent = COSAgentProvider(self) + self._grafana_agent = COSAgentProvider( + self, + metrics_endpoints=[ + {"path": "/metrics", "port": int(manager_service.GITHUB_RUNNER_MANAGER_PORT)} + ], + ) self._stored.set_default( path=self.config[PATH_CONFIG_NAME], # for detecting changes diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index eb85180957..088abe63f0 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -9,7 +9,7 @@ import string from pathlib import Path from time import sleep -from typing import Any, AsyncIterator, Generator, Iterator, Optional, cast +from typing import Any, AsyncGenerator, AsyncIterator, Generator, Iterator, Optional, cast import jubilant import nest_asyncio @@ -750,8 +750,10 @@ async def instance_helper_fixture(request: pytest.FixtureRequest) -> OpenStackIn return OpenStackInstanceHelper(openstack_connection=openstack_connection) -@pytest.fixture(scope="module") -def juju(request: pytest.FixtureRequest, model: Model) -> Generator[jubilant.Juju, None, None]: +@pytest_asyncio.fixture(scope="module") +async def juju( + request: pytest.FixtureRequest, model: Model +) -> AsyncGenerator[jubilant.Juju, None]: """Pytest fixture that wraps :meth:`jubilant.with_model`.""" def show_debug_log(juju: jubilant.Juju): @@ -764,14 +766,19 @@ def show_debug_log(juju: jubilant.Juju): log = juju.debug_log(limit=1000) print(log, end="") + controller = await model.get_controller() if model: - juju = jubilant.Juju(model=model.name) + # Currently juju has no way of switching controller context, this is required to operate + # in the right controller's right model when using multiple controllers. + # See: https://github.com/canonical/jubilant/issues/158 + juju = jubilant.Juju(model=f"{controller.controller_name}:{model.name}") yield juju show_debug_log(juju) return keep_models = cast(bool, request.config.getoption("--keep-models")) - with jubilant.temp_model(keep=keep_models) as juju: + with jubilant.temp_model(keep=keep_models, controller=controller.controller_name) as juju: + juju.model = f"{controller.controller_name}:{juju.model}" juju.wait_timeout = 10 * 60 yield juju show_debug_log(juju) diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt index 524736dde0..062f66c0ba 100644 --- a/tests/integration/requirements.txt +++ b/tests/integration/requirements.txt @@ -1,5 +1,6 @@ GitPython>3,<4 -pygithub +jubilant==1.1.* kombu==5.* +pygithub pymongo==4.* -jubilant==1.1.* +tenacity==9.1.2 diff --git a/tests/integration/setup-integration-tests.sh b/tests/integration/setup-integration-tests.sh new file mode 100755 index 0000000000..c905cf305b --- /dev/null +++ b/tests/integration/setup-integration-tests.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +# Pre-run script for integration test operator-workflows action. +# https://github.com/canonical/operator-workflows/blob/main/.github/workflows/integration_test.yaml + +# The COS observability stack are deployed on K8s models. + +# save original controller that is used for testing +ORIGINAL_CONTROLLER=$(juju controllers --format json | jq -r '.controllers | keys | .[0]') + +echo "bootstrapping microk8s juju controller" +sudo snap install microk8s --channel=1.32-strict/stable +GROUP=snap_microk8s +sudo usermod -a -G "$GROUP" "$USER" +if [ "$(id -gn)" != "$GROUP" ]; then + exec sg "$GROUP" "$0" "$*" +fi + +# Get preferred source IP address for metallb +IPADDR=$( { ip -4 -j route get 2.2.2.2 | jq -r '.[] | .prefsrc'; } ) +sudo microk8s enable "metallb:$IPADDR-$IPADDR" "hostpath-storage" +microk8s status --wait-ready + +unset JUJU_CONTROLLER +unset JUJU_MODEL +juju bootstrap microk8s microk8s +juju switch "$ORIGINAL_CONTROLLER" diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py new file mode 100644 index 0000000000..8c871372b1 --- /dev/null +++ b/tests/integration/test_prometheus_metrics.py @@ -0,0 +1,208 @@ +# Copyright 2025 Canonical Ltd. +# See LICENSE file for licensing details. + +"""Module for collecting metrics related to the reconciliation process.""" + +import logging +import subprocess +from typing import Any, Generator, cast + +import jubilant +import pytest +import pytest_asyncio +import requests +from jubilant.statustypes import AppStatus +from juju.application import Application +from tenacity import retry, stop_after_attempt, wait_exponential + +logger = logging.getLogger(__name__) + + +@pytest_asyncio.fixture(scope="module", name="k8s_juju") +def k8s_juju_fixture(request: pytest.FixtureRequest) -> Generator[jubilant.Juju, None, None]: + """The machine model for K8s charms.""" + keep_models = cast(bool, request.config.getoption("--keep-models")) + with jubilant.temp_model(keep=keep_models, controller="microk8s") as juju: + # Currently juju has no way of switching controller context, this is required to operate + # in the right controller's right model when using multiple controllers. + # See: https://github.com/canonical/jubilant/issues/158 + juju.model = f"microk8s:{juju.model}" + yield juju + + +# juju.offer is not controller aware, we should manually switch to the microk8s controller. +@pytest.fixture(scope="function", name="switch_microk8s_controller") +def switch_microk8s_controller_fixture(k8s_juju: jubilant.Juju, juju: jubilant.Juju): + """Switch to the MicroK8s controller.""" + original_model_controller_name = juju.model + assert ( + original_model_controller_name + ), f"model & controller name not set: {original_model_controller_name}" + + yield + + +@pytest.mark.usefixtures("switch_microk8s_controller") +@pytest.fixture(scope="module", name="prometheus_app") +def prometheus_app_fixture(k8s_juju: jubilant.Juju): + """Deploy prometheus charm.""" + k8s_juju.deploy("prometheus-k8s", channel="1/stable") + k8s_juju.wait(lambda status: jubilant.all_active(status, "prometheus-k8s")) + model_controller_name = k8s_juju.model + logger.info("Model controller: %s", model_controller_name) + assert model_controller_name, f"model & controller name not set: {model_controller_name}" + controller, model = model_controller_name.split(":") + logger.info("Controller: %s, Model: %s", controller, model) + # juju.offer has no controller parameter. Use the cli directly. + result = subprocess.run( + [ + k8s_juju.cli_binary, + "offer", + "-c", + controller, + f"{model}.prometheus-k8s:receive-remote-write", + ] + ) + assert ( + result.returncode == 0 + ), f"failed to create prometheus offer: {str(result.stdout)} {str(result.stderr)}" + return k8s_juju.status().apps["prometheus-k8s"] + + +@pytest.mark.usefixtures("switch_microk8s_controller") +@pytest.fixture(scope="module", name="grafana_app") +def grafana_app_fixture(k8s_juju: jubilant.Juju, prometheus_app: AppStatus): + """Deploy prometheus charm.""" + k8s_juju.deploy("grafana-k8s", channel="1/stable") + k8s_juju.integrate("grafana-k8s:grafana-source", f"{prometheus_app.charm_name}:grafana-source") + k8s_juju.wait(lambda status: jubilant.all_active(status, "grafana-k8s", "prometheus-k8s")) + model_controller_name = k8s_juju.model + assert model_controller_name, f"model & controller name not set: {model_controller_name}" + controller, model = model_controller_name.split(":") + logger.info("Controller: %s, Model: %s", controller, model) + # juju.offer has no controller parameter. Use the cli directly. + result = subprocess.run( + [k8s_juju.cli_binary, "offer", "-c", controller, f"{model}.grafana-k8s:grafana-dashboard"] + ) + assert ( + result.returncode == 0 + ), f"failed to create grafana offer: {str(result.stdout)} {str(result.stderr)}" + return k8s_juju.status().apps["grafana-k8s"] + + +@pytest.fixture(scope="module", name="traefik_ingress") +def traefik_ingress_fixture( + k8s_juju: jubilant.Juju, prometheus_app: AppStatus, grafana_app: AppStatus +): + """Ingress for cross controller communication.""" + k8s_juju.deploy("traefik-k8s", channel="latest/stable") + k8s_juju.integrate("traefik-k8s", f"{prometheus_app.charm_name}:ingress") + k8s_juju.integrate("traefik-k8s", f"{grafana_app.charm_name}:ingress") + + +@pytest.fixture(scope="module", name="grafana_password") +def grafana_password_fixture(k8s_juju: jubilant.Juju, grafana_app: AppStatus): + """Get Grafana dashboard password.""" + unit = next(iter(grafana_app.units.keys())) + result = k8s_juju.run(unit, "get-admin-password") + return result.results["admin-password"] + + +@pytest.fixture(scope="module", name="openstack_app_cos_agent") +def openstack_app_cos_agent_fixture(juju: jubilant.Juju, app_openstack_runner: Application): + """Deploy cos-agent subordinate charm on OpenStack runner application.""" + juju.deploy("grafana-agent", channel="1/stable", base="ubuntu@22.04") + juju.integrate(app_openstack_runner.name, "grafana-agent") + juju.wait( + lambda status: jubilant.all_agents_idle(status, app_openstack_runner.name, "grafana-agent") + ) + return app_openstack_runner + + +@pytest.mark.usefixtures("traefik_ingress") +@pytest.mark.openstack +def test_prometheus_metrics( + juju: jubilant.Juju, + k8s_juju: jubilant.Juju, + openstack_app_cos_agent: Application, + grafana_app: AppStatus, + grafana_password: str, + prometheus_app: AppStatus, +): + """ + arrange: given a prometheus charm application. + act: when GitHub runner is integrated. + assert: the datasource is registered and basic metrics are available. + """ + prometheus_offer_name = "prometheus-k8s" + grafana_offer_name = "grafana-k8s" + # k8s_juju.model and juju.model already has : prefixed. + result = subprocess.run( + [ + k8s_juju.cli_binary, + "consume", + "-m", + str(juju.model), + f"{str(k8s_juju.model)}.prometheus-k8s", + ] + ) + assert ( + result.returncode == 0 + ), f"failed to consume prometheus offer: {str(result.stdout)} {str(result.stderr)}" + result = subprocess.run( + [ + k8s_juju.cli_binary, + "consume", + "-m", + str(juju.model), + f"{str(k8s_juju.model)}.grafana-k8s", + ] + ) + assert ( + result.returncode == 0 + ), f"failed to consume grafana offer: {str(result.stdout)} {str(result.stderr)}" + + juju.integrate("grafana-agent", prometheus_offer_name) + juju.integrate("grafana-agent", grafana_offer_name) + juju.wait( + lambda status: jubilant.all_agents_idle( + status, openstack_app_cos_agent.name, "grafana-agent" + ) + ) + + grafana_ip = grafana_app.units["grafana-k8s/0"].address + _patiently_wait_for_prometheus_datasource( + grafana_ip=grafana_ip, grafana_password=grafana_password + ) + prometheus_ip = prometheus_app.address + _patiently_wait_for_prometheus_metrics( + prometheus_ip, + "openstack_http_requests_total", + "reconcile_duration_seconds_sum", + "expected_runners_count", + "busy_runners_count", + "idle_runners_count", + ) + + +@retry(stop=stop_after_attempt(5), wait=wait_exponential(multiplier=2, max=60), reraise=True) +def _patiently_wait_for_prometheus_datasource(grafana_ip: str, grafana_password: str): + """Wait for prometheus datasource to come up.""" + response = requests.get(f"http://admin:{grafana_password}@{grafana_ip}:3000/api/datasources") + response.raise_for_status() + datasources: list[dict[str, Any]] = response.json() + assert any(datasource["type"] == "prometheus" for datasource in datasources) + + +@retry( + stop=stop_after_attempt(10), wait=wait_exponential(multiplier=2, min=10, max=60), reraise=True +) +def _patiently_wait_for_prometheus_metrics(prometheus_ip: str, *metric_names: str): + """Wait for the prometheus metrics to be available.""" + for metric_name in metric_names: + response = requests.get( + f"http://{prometheus_ip}:9090/api/v1/series", params={"match[]": metric_name} + ) + response.raise_for_status() + query_result = response.json()["data"] + assert len(query_result), f"No data found for metric: {metric_name}" diff --git a/tests/integration/test_runner_manager_openstack.py b/tests/integration/test_runner_manager_openstack.py deleted file mode 100644 index 2058a7d2fe..0000000000 --- a/tests/integration/test_runner_manager_openstack.py +++ /dev/null @@ -1,584 +0,0 @@ -# Copyright 2025 Canonical Ltd. -# See LICENSE file for licensing details. - -"""Testing the RunnerManager class with OpenStackRunnerManager as CloudManager. -It is assumed that the test runs in the CI under the ubuntu user. -""" - - -import asyncio -import json -import logging -from pathlib import Path -from secrets import token_hex -from typing import AsyncGenerator, Iterator - -import pytest -import pytest_asyncio -import yaml -from github.Branch import Branch -from github.Repository import Repository -from github.Workflow import Workflow -from github_runner_manager.configuration import ProxyConfig, SupportServiceConfig, UserInfo -from github_runner_manager.configuration.github import GitHubPath, parse_github_path -from github_runner_manager.github_client import GithubClient -from github_runner_manager.manager.cloud_runner_manager import CloudRunnerState -from github_runner_manager.manager.models import RunnerMetadata -from github_runner_manager.manager.runner_manager import FlushMode, RunnerManager -from github_runner_manager.metrics import events -from github_runner_manager.openstack_cloud import constants -from github_runner_manager.openstack_cloud.models import ( - OpenStackCredentials, - OpenStackRunnerManagerConfig, - OpenStackServerConfig, -) -from github_runner_manager.openstack_cloud.openstack_runner_manager import OpenStackRunnerManager -from github_runner_manager.platform.github_provider import ( - GitHubRunnerPlatform, - PlatformRunnerState, -) -from github_runner_manager.types_.github import GitHubRunnerStatus -from openstack.connection import Connection as OpenstackConnection - -from tests.integration.helpers.common import ( - DISPATCH_WAIT_TEST_WORKFLOW_FILENAME, - dispatch_workflow, - wait_for, -) - -logger = logging.getLogger(__name__) - -# A higher create server timeout is reasonable for integration tests, -# as only one machine that stays for more than the default time in BUILD, -# will break the tests -constants.CREATE_SERVER_TIMEOUT = 900 - - -@pytest.fixture(autouse=True, scope="module", name="runner_manager_user") -def runner_manager_user(): - """Mock the RUNNER_MANAGER_USER and RUNNER_MANAGER_GROUP constants. - - Yields: - None, just to be in a scope. - """ - with pytest.MonkeyPatch.context() as monkeypatch: - # we assume the test runs as ubuntu user - monkeypatch.setattr("github_runner_manager.constants.RUNNER_MANAGER_USER", "ubuntu") - monkeypatch.setattr("github_runner_manager.constants.RUNNER_MANAGER_GROUP", "ubuntu") - # monkeypatch is a scope function fixture, so this trick - yield None - - -@pytest.fixture(scope="module", name="runner_label") -def runner_label(): - return f"test-{token_hex(6)}" - - -@pytest.fixture(scope="module", name="log_dir_base_path") -def log_dir_base_path_fixture( - tmp_path_factory: pytest.TempPathFactory, -) -> Iterator[dict[str, Path]]: - """Mock the log directory path and return it.""" - with pytest.MonkeyPatch.context() as monkeypatch: - temp_log_dir = tmp_path_factory.mktemp("log") - - metric_log_path = temp_log_dir / "metric_log" - - monkeypatch.setattr(events, "METRICS_LOG_PATH", metric_log_path) - - yield { - "metric_log": metric_log_path, - } - - -@pytest.fixture(scope="module", name="prefix") -def prefix_fixture(app_name: str) -> str: - return f"{app_name}-0" - - -@pytest.fixture(scope="module", name="github_path") -def github_path_fixture(path: str) -> GitHubPath: - return parse_github_path(path, "Default") - - -@pytest.fixture(scope="module", name="proxy_config") -def openstack_proxy_config_fixture( - openstack_http_proxy: str, openstack_https_proxy: str, openstack_no_proxy: str -) -> ProxyConfig: - http_proxy = openstack_http_proxy if openstack_http_proxy else None - https_proxy = openstack_https_proxy if openstack_https_proxy else None - return ProxyConfig( - http=http_proxy, - https=https_proxy, - no_proxy=openstack_no_proxy, - ) - - -@pytest_asyncio.fixture(scope="module", name="openstack_runner_manager") -async def openstack_runner_manager_fixture( - app_name: str, - prefix: str, - private_endpoint_clouds_yaml: str, - openstack_test_image: str, - flavor_name: str, - network_name: str, - github_path: GitHubPath, - proxy_config: ProxyConfig, - runner_label: str, - openstack_connection: OpenstackConnection, -) -> AsyncGenerator[OpenStackRunnerManager, None]: - """Create OpenstackRunnerManager instance. - - The prefix args of OpenstackRunnerManager set to app_name to let openstack_connection_fixture - perform the cleanup of openstack resources. - """ - clouds_config = yaml.safe_load(private_endpoint_clouds_yaml) - - try: - # Pick the first cloud in the clouds.yaml - cloud = tuple(clouds_config["clouds"].values())[0] - print("============================================") - print(cloud) - print("============================================") - - credentials = OpenStackCredentials( - auth_url=cloud["auth"]["auth_url"], - project_name=cloud["auth"]["project_name"], - username=cloud["auth"]["username"], - password=cloud["auth"]["password"], - user_domain_name=cloud["auth"]["user_domain_name"], - project_domain_name=cloud["auth"]["project_domain_name"], - region_name=cloud["region_name"], - ) - except KeyError as err: - raise AssertionError("Issue with the format of the clouds.yaml used in test") from err - - server_config = OpenStackServerConfig( - image=openstack_test_image, - flavor=flavor_name, - network=network_name, - ) - - use_aproxy = bool(proxy_config.proxy_address) - - service_config = SupportServiceConfig( - proxy_config=proxy_config, - runner_proxy_config=proxy_config, - dockerhub_mirror=None, - ssh_debug_connections=[], - repo_policy_compliance=None, - use_aproxy=use_aproxy, - ) - - openstack_runner_manager_config = OpenStackRunnerManagerConfig( - name=app_name, - prefix=prefix, - credentials=credentials, - server_config=server_config, - service_config=service_config, - ) - user = UserInfo("ubuntu", "ubuntu") - - yield OpenStackRunnerManager( - config=openstack_runner_manager_config, - user=user, - ) - - -@pytest.fixture(scope="module", name="github_client_for_manager") -def github_client_for_manager_fixture(token: str) -> GithubClient: - github_client = GithubClient(token) - return github_client - - -@pytest.fixture(scope="module", name="github_platform") -def github_platform_fixture( - token: str, - prefix: str, - github_path: GitHubPath, - github_client_for_manager: GithubClient, -) -> GitHubRunnerPlatform: - github_platform = GitHubRunnerPlatform( - prefix=prefix, - path=github_path, - github_client=github_client_for_manager, - ) - return github_platform - - -@pytest_asyncio.fixture(scope="module", name="runner_manager") -async def runner_manager_fixture( - openstack_runner_manager: OpenStackRunnerManager, - log_dir_base_path: dict[str, Path], - runner_label: str, - github_platform: GitHubRunnerPlatform, -) -> AsyncGenerator[RunnerManager, None]: - """Get RunnerManager instance. - - Import of log_dir_base_path to monkeypatch the runner logs path with tmp_path. - """ - yield RunnerManager( - manager_name="test_runner", - platform_provider=github_platform, - cloud_runner_manager=openstack_runner_manager, - labels=["openstack_test", runner_label], - ) - - -@pytest_asyncio.fixture(scope="function", name="runner_manager_with_one_runner") -async def runner_manager_with_one_runner_fixture(runner_manager: RunnerManager) -> RunnerManager: - runner_manager.flush_runners(flush_mode=FlushMode.FLUSH_BUSY) - await wait_runner_amount(runner_manager, 0) - runner_manager.create_runners(1, RunnerMetadata()) - try: - await wait_runner_amount(runner_manager, 1) - except TimeoutError as err: - raise AssertionError("Test arrange failed: Expect one runner") from err - - runner_list = runner_manager.get_runners() - runner = runner_list[0] - assert ( - runner.cloud_state == CloudRunnerState.ACTIVE - ), "Test arrange failed: Expect runner in active state" - try: - await wait_for( - lambda: runner_manager.get_runners()[0].platform_state == PlatformRunnerState.IDLE, - timeout=1200, - check_interval=10, - ) - except TimeoutError as err: - raise AssertionError("Test arrange failed: Expect runner in idle state") from err - return runner_manager - - -def workflow_is_status(workflow: Workflow, status: str) -> bool: - """Check if workflow in provided status. - - Args: - workflow: The workflow to check. - status: The status to check for. - - Returns: - Whether the workflow is in the status. - """ - workflow.update() - return workflow.status == status - - -async def wait_runner_amount( - runner_manager: RunnerManager, num: int, timeout: int = 600, check_interval: int = 60 -) -> None: - """Wait until the runner manager has the number of runners. - - A TimeoutError will be thrown if runners amount is not correct after timeout. - - Args: - runner_manager: The RunnerManager to check. - num: Number of runner to check for. - timeout: The timeout in seconds. - check_interval: The interval to check in seconds. - """ - # The openstack server can take sometime to fully clean up or create. - await wait_for( - lambda: check_runners_amount_and_active(runner_manager, num), - timeout=timeout, - check_interval=check_interval, - ) - - -def check_runners_amount_and_active(runner_manager: RunnerManager, num: int) -> bool: - """Check if the number of runners match the expected amount and all runners are active. - - Args: - runner_manager: The RunnerManager instance to use. - num: The expected number of runners. - - Returns: - Whether the expected number of runner is spawned and active. - """ - runners = runner_manager.get_runners() - active_runners = [ - runner for runner in runners if runner.cloud_state == CloudRunnerState.ACTIVE - ] - if len(runners) == len(active_runners) and len(runners) == num: - return True - return False - - -@pytest.mark.openstack -@pytest.mark.asyncio -@pytest.mark.abort_on_fail -async def test_get_no_runner(runner_manager: RunnerManager) -> None: - """ - Arrange: RunnerManager instance with no runners. - Act: Get runners. - Assert: Empty tuple returned. - """ - runner_list = runner_manager.get_runners() - assert isinstance(runner_list, tuple) - assert not runner_list - - -@pytest.mark.openstack -@pytest.mark.asyncio -@pytest.mark.abort_on_fail -async def test_runner_normal_idle_lifecycle( - runner_manager: RunnerManager, - openstack_runner_manager: OpenStackRunnerManager, - github_client_for_manager: GithubClient, - github_path: GitHubPath, -) -> None: - """ - Arrange: RunnerManager instance with no runners. - Act: - 1. Create one runner. - 2. Run health check on the runner. - 3. Run cleanup. - 4. Delete all idle runner. - Assert: - 1. An active idle runner. - 2. Health check passes. - 3. One idle runner remains. - 4. No runners. - """ - # 1. - runner_id_list = runner_manager.create_runners(1, RunnerMetadata()) - assert isinstance(runner_id_list, tuple) - assert len(runner_id_list) == 1 - runner_id = runner_id_list[0] - - try: - await wait_runner_amount(runner_manager, 1) - except TimeoutError as err: - raise AssertionError("Test arrange failed: Expect one runner") from err - - runner_list = runner_manager.get_runners() - assert isinstance(runner_list, tuple) - assert len(runner_list) == 1 - runner = runner_list[0] - assert runner.instance_id == runner_id - assert runner.cloud_state == CloudRunnerState.ACTIVE - assert runner.metadata.platform_name == "github" - # Update on GitHub-side can take a bit of time. - await wait_for( - lambda: runner_manager.get_runners()[0].platform_state == PlatformRunnerState.IDLE, - timeout=120, - check_interval=10, - ) - - # 2. - openstack_instances = openstack_runner_manager._openstack_cloud.get_instances() - - assert len(openstack_instances) == 1, "Test arrange failed: Needs one runner." - runner = openstack_instances[0] - - self_hosted_runner = github_client_for_manager.get_runner( - github_path, runner.instance_id.prefix, int(runner.metadata.runner_id) - ) - assert self_hosted_runner.status == GitHubRunnerStatus.ONLINE - - # 3. - runner_manager.cleanup() - runner_list = runner_manager.get_runners() - assert isinstance(runner_list, tuple) - assert len(runner_list) == 1 - runner = runner_list[0] - assert runner.instance_id == runner_id - assert runner.cloud_state == CloudRunnerState.ACTIVE - - # 4. - runner_manager.flush_runners(flush_mode=FlushMode.FLUSH_IDLE) - await wait_runner_amount(runner_manager, 0) - - -@pytest.mark.openstack -@pytest.mark.asyncio -@pytest.mark.abort_on_fail -async def test_runner_flush_busy_lifecycle( - runner_manager_with_one_runner: RunnerManager, - test_github_branch: Branch, - github_repository: Repository, - runner_label: str, -): - """ - Arrange: RunnerManager with one idle runner. - Act: - 1. Run a long workflow. - 3. Run flush idle runner. - 4. Run flush busy runner. - Assert: - 1. Runner takes the job and become busy. - 3. Busy runner still exists. - 4. No runners exists. - """ - # 1. - workflow = await dispatch_workflow( - app=None, - branch=test_github_branch, - github_repository=github_repository, - conclusion="success", - workflow_id_or_name=DISPATCH_WAIT_TEST_WORKFLOW_FILENAME, - dispatch_input={"runner": runner_label, "minutes": "30"}, - wait=False, - ) - await wait_for(lambda: workflow_is_status(workflow, "in_progress")) - - runner_list = runner_manager_with_one_runner.get_runners() - assert len(runner_list) == 1 - busy_runner = runner_list[0] - assert busy_runner.cloud_state == CloudRunnerState.ACTIVE - assert busy_runner.platform_state == PlatformRunnerState.BUSY - - # 2. - runner_manager_with_one_runner.cleanup() - runner_list = runner_manager_with_one_runner.get_runners() - assert isinstance(runner_list, tuple) - assert len(runner_list) == 1 - runner = runner_list[0] - assert runner.cloud_state == CloudRunnerState.ACTIVE - assert busy_runner.platform_state == PlatformRunnerState.BUSY - - # 3. - runner_manager_with_one_runner.flush_runners(flush_mode=FlushMode.FLUSH_IDLE) - runner_list = runner_manager_with_one_runner.get_runners() - assert len(runner_list) == 1 - busy_runner = runner_list[0] - assert busy_runner.cloud_state == CloudRunnerState.ACTIVE - assert busy_runner.platform_state == PlatformRunnerState.BUSY - - # 4. - runner_manager_with_one_runner.flush_runners(flush_mode=FlushMode.FLUSH_BUSY) - # It takes a bit for the github agent to die, and it may not be cleaned - # in the first run. Just do it twice. - await asyncio.sleep(10) - runner_manager_with_one_runner.flush_runners(flush_mode=FlushMode.FLUSH_BUSY) - await wait_runner_amount(runner_manager_with_one_runner, 0) - - -@pytest.mark.openstack -@pytest.mark.asyncio -@pytest.mark.abort_on_fail -async def test_runner_normal_lifecycle( - runner_manager_with_one_runner: RunnerManager, - test_github_branch: Branch, - github_repository: Repository, - runner_label: str, - log_dir_base_path: dict[str, Path], -): - """ - Arrange: RunnerManager with one runner. Clean metric logs. - Act: - 1. Start a test workflow for the runner. - 2. Run cleanup. - Assert: - 1. The workflow complete successfully. - 2. The runner should be deleted. The metrics should be recorded. - """ - logger.info("Starting test_runner_normal_lifecycle") - metric_log_path = log_dir_base_path["metric_log"] - try: - metric_log_existing_content = metric_log_path.read_text(encoding="utf-8") - except FileNotFoundError: - metric_log_existing_content = "" - - workflow = await dispatch_workflow( - app=None, - branch=test_github_branch, - github_repository=github_repository, - conclusion="success", - workflow_id_or_name=DISPATCH_WAIT_TEST_WORKFLOW_FILENAME, - dispatch_input={"runner": runner_label, "minutes": "0"}, - wait=False, - ) - await wait_for(lambda: workflow_is_status(workflow, "completed")) - - # We encountered a race condition where runner_manager.cleanup was called while - # there was no runner process, but the post-metrics still had not yet been issued. - # Make the test more robust by waiting for the runner to go offline - # to reduce the race condition. - def is_runner_offline() -> bool: - """Check if the runner is offline. - - Returns: - True if the runner is offline, False otherwise. - """ - runners = runner_manager_with_one_runner.get_runners() - assert len(runners) == 1 - return runners[0].platform_state in (PlatformRunnerState.OFFLINE, None) - - await wait_for(is_runner_offline, check_interval=60, timeout=600) - - def have_metrics_been_issued() -> bool: - """Check if the expected metrics have been issued. - - Returns: - True if the expected metrics have been issued, False otherwise. - """ - issued_metrics_events = runner_manager_with_one_runner.cleanup() - logger.info("issued_metrics_events: %s", issued_metrics_events) - return ( - {events.RunnerInstalled, events.RunnerStart, events.RunnerStop} - == set(issued_metrics_events) - and issued_metrics_events[events.RunnerInstalled] == 1 - and issued_metrics_events[events.RunnerStart] == 1 - and issued_metrics_events[events.RunnerStop] == 1 - ) - - try: - await wait_for(have_metrics_been_issued, check_interval=60, timeout=600) - except TimeoutError: - assert False, "The expected metrics were not issued" - - metric_log_full_content = metric_log_path.read_text(encoding="utf-8") - assert metric_log_full_content.startswith( - metric_log_existing_content - ), "The metric log was modified in ways other than appending" - metric_log_new_content = metric_log_full_content[len(metric_log_existing_content) :] - metric_logs = [json.loads(metric) for metric in metric_log_new_content.splitlines()] - assert len(metric_logs) == 3, ( - "Assuming three events " - "should be runner_installed, runner_start and runner_stop, " - "modify this if new events are added" - ) - assert metric_logs[0]["event"] == "runner_installed" - assert metric_logs[0]["flavor"] == runner_manager_with_one_runner.manager_name - assert metric_logs[1]["event"] == "runner_start" - assert metric_logs[1]["workflow"] == "Workflow Dispatch Wait Tests" - assert metric_logs[2]["event"] == "runner_stop" - assert metric_logs[2]["workflow"] == "Workflow Dispatch Wait Tests" - - await wait_runner_amount(runner_manager_with_one_runner, 0) - - -@pytest.mark.openstack -@pytest.mark.asyncio -@pytest.mark.abort_on_fail -async def test_runner_spawn_two( - runner_manager: RunnerManager, openstack_runner_manager: OpenStackRunnerManager -) -> None: - """ - Arrange: RunnerManager instance with no runners. - Act: - 1. Create two runner. - 2. Delete all idle runner. - Assert: - 1. Two active idle runner. - 2. No runners. - """ - # 1. - runner_id_list = runner_manager.create_runners(2, RunnerMetadata()) - assert isinstance(runner_id_list, tuple) - assert len(runner_id_list) == 2 - - try: - await wait_runner_amount(runner_manager, 2) - except TimeoutError as err: - raise AssertionError("Test arrange failed: Expect two runner") from err - - runner_list = runner_manager.get_runners() - assert isinstance(runner_list, tuple) - assert len(runner_list) == 2 - - # 3. - runner_manager.flush_runners(flush_mode=FlushMode.FLUSH_IDLE) - await wait_runner_amount(runner_manager, 0)