diff --git a/ci/README.md b/ci/README.md index 46d6cdcc..96a0dc5b 100644 --- a/ci/README.md +++ b/ci/README.md @@ -4,8 +4,8 @@ Rattan now contains two parts of tests: -* Standard Tests: run on each commit in all branches. (see .github/workflows/test.yaml) -* Verification Tests: run **only in the main branch** every day. (see .github/workflows/verify.yaml) +- Standard Tests: run on each commit in all branches. (see .github/workflows/test.yaml) +- Verification Tests: run **only in the main branch** every day. (see .github/workflows/verify.yaml) ## Kernels @@ -13,28 +13,95 @@ Rattan is currently tested on the latest four LTS kernels, that are 5.4 (the def This directory contains artifacts setting up four test machines with equipped aforementioned four kernels as our development and CI environments. -| Machine Nickname | Kernel Version | Distribution | Cloud Image | -| :---: | :---: | :---: | :---: | -| focal-0505 | 5.4 | Ubuntu 20.04 (focal) | [focal/release-20240606](https://cloud-images.ubuntu.com/releases/focal/release-20230908/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img) | -| jammy-0515 | 5.15 | Ubuntu 22.04 (jammy) | [jammy/release-20240612](https://cloud-images.ubuntu.com/releases/22.04/release-20230914/ubuntu-22.04-server-cloudimg-amd64-disk-kvm.img) | -| noble-0608 | 6.8 | Ubuntu 24.04 (noble) | [noble/20240521](https://cloud-images.ubuntu.com/releases/24.04/release-20240608/ubuntu-24.04-server-cloudimg-amd64.img) | -| noble-0612 | 6.12 | Ubuntu 24.04 (noble) | [noble/20241119](https://cloud-images.ubuntu.com/releases/24.04/release-20241119/ubuntu-24.04-server-cloudimg-amd64.img) | +| Machine Nickname | Kernel Version | Distribution | Cloud Image | +| :--------------: | :------------: | :------------------: | :---------------------------------------------------------------------------------------------------------------------------------------: | +| focal-0505 | 5.4 | Ubuntu 20.04 (focal) | [focal/release-20240606](https://cloud-images.ubuntu.com/releases/focal/release-20230908/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img) | +| jammy-0515 | 5.15 | Ubuntu 22.04 (jammy) | [jammy/release-20240612](https://cloud-images.ubuntu.com/releases/22.04/release-20230914/ubuntu-22.04-server-cloudimg-amd64-disk-kvm.img) | +| noble-0608 | 6.8 | Ubuntu 24.04 (noble) | [noble/20240521](https://cloud-images.ubuntu.com/releases/24.04/release-20240608/ubuntu-24.04-server-cloudimg-amd64.img) | +| noble-0612 | 6.12 | Ubuntu 24.04 (noble) | [noble/20241119](https://cloud-images.ubuntu.com/releases/24.04/release-20241119/ubuntu-24.04-server-cloudimg-amd64.img) | ## Prepare Virtual Machines -* Build machine images using [Packer by HashiCorp](https://www.packer.io/) with [libvirtd plugin](https://developer.hashicorp.com/packer/plugins/builders/libvirt) on a machine with `libvirtd` available. +- Build machine images using [Packer by HashiCorp](https://www.packer.io/) with [libvirtd plugin](https://developer.hashicorp.com/packer/plugins/builders/libvirt) on a machine with `libvirtd` available. + +First, install libvirtd using apt, and install Packer by following the [installation instruction](https://developer.hashicorp.com/packer/install). + +```shell +# Installs libvirtd +sudo apt update +sudo apt install -y qemu-kvm libvirt-daemon-system libvirt-clients bridge-utils virtinst +sudo systemctl enable --now libvirtd +sudo usermod -aG libvirt,kvm $USER # re-login to make group settings take effect +``` + +Ansible is also required. You can install it by following [the official guide](https://docs.ansible.com/projects/ansible/latest/installation_guide/intro_installation.html#installing-and-upgrading-ansible-with-pipx), or you can install it using `uv`. The version is pinned to 2.18 so it still supports the Python 3.8 target on focal/20.04. + +```shell +uv tool install --with ansible 'ansible-core>=2.18,<2.19' +``` + +Then, create the default pool to store the images created by packer: + +```shell +virsh pool-define-as default dir --target /var/lib/libvirt/images +virsh pool-build default +virsh pool-start default +virsh pool-autostart default +virsh pool-list --all +``` + +Then, install required packer plugins. In `ci/images` directory: + +```shell +# In ci/images +packer init . +``` + +Now, run `packer build` command for each distribution-kernel combination. + +```shell +# In ci/images +# Note the dot at the last line +packer build \ + -var "github_access_key=" \ + -var "release_name=" \ + -var "kernel_version=" \ + -var "key_import_user=" \ + -var "authorized_key_file=" \ + -var "http_proxy=" \ + . +``` + +- `` should be granted _Read and Write access to the `stack-rs/rattan` repository's self-hosted runners_ (repo `Administration` for a fine-grained PAT, or the `repo` scope for a classic PAT). +- `` should be `jammy` (for Ubuntu 22.04), `focal` (for Ubutnu 20.04) or `noble` (Ubuntu 24.04). +- `` should be `5.4`, `5.15`, `6.8`, or `6.12`. +- `` (optional) is a user whose public keys are pulled from GitHub and imported to the VM. +- `` (optional) is a local public key file injected into the VM's `authorized_keys`, e.g. `~/.ssh/ubuntu_rsa.pub`. Supports `~` expansion; a bare relative path resolves against `ci/images`. + +Both `key_import_user` and `authorized_key_file` are optional and can be combined, but provide **at least one** of them to ensure ssh access. You then log in as the `rattan` user with the matching private key. + +If the guest must reach the internet through an HTTP proxy, add `-var "http_proxy=http://:"`. +This routes the mainline kernel downloads, and Docker (daemon image pulls, the runner +container, and containers spawned by CI jobs) through the proxy. Apt connections +are not proxied, if needed, uncomment the line at `image.pkr.hcl` by searching +for `apt`. Omit it for a direct connection. + +If `packer build` command failed with error `Cloud not open '/var/lib/libvirt/images/-`, refer to [this issue](https://github.com/dmacvicar/terraform-provider-libvirt/issues/1163), and follow user dylanf9797's solution on configuring apparmor to allow access to `/var/lib/libvirt/images`. + +`packer build` refuses to overwrite an existing image, so to rebuild a combination you must first delete its volume. Images are stored in the `default` pool under the name `-` (e.g. `focal-5.4`): ```shell -packer build -var "github_access_key=" -var "release_name=" -var "kernel_version=" -var "key_import_user=" . +# List the current images +virsh vol-list default +# Delete one before rebuilding it +virsh vol-delete --pool default - ``` -* `` should be granted *Read and Write access to organization self hosted runners*. -* `` should be `jammy` (for Ubuntu 22.04), `focal` (for Ubutnu 20.04) or `noble` (Ubuntu 24.04). -* `` should be `5.4`, `5.15` or `6.8`. -* `` should be the user whose public keys will be imported to the VM. +## Run Virtual Machines -Run VMs on a mchine with `libvirtd` available. +Run VMs on a machine with `libvirtd` available. Must first create the image using packer. ```shell +# In ci/libvirt virsh create ./libvirt/-.xml ``` diff --git a/ci/images/ansible/configure.yml b/ci/images/ansible/configure.yml index 1a9bfe2b..df1efb03 100644 --- a/ci/images/ansible/configure.yml +++ b/ci/images/ansible/configure.yml @@ -1,6 +1,13 @@ --- - name: Install and Run Github Self-hosted Runner hosts: all + environment: + http_proxy: "{{ http_proxy | default('') }}" + https_proxy: "{{ http_proxy | default('') }}" + HTTP_PROXY: "{{ http_proxy | default('') }}" + HTTPS_PROXY: "{{ http_proxy | default('') }}" + no_proxy: "localhost,127.0.0.1,::1" + NO_PROXY: "localhost,127.0.0.1,::1" tasks: - name: install pip ansible.builtin.include_role: diff --git a/ci/images/ansible/docker-compose.yml.j2 b/ci/images/ansible/docker-compose.yml.j2 index aed34be4..4d785b35 100644 --- a/ci/images/ansible/docker-compose.yml.j2 +++ b/ci/images/ansible/docker-compose.yml.j2 @@ -6,10 +6,16 @@ services: privileged: true environment: RUNNER_NAME: {{ runner_name }} - RUNNER_SCOPE: org - ORG_NAME: stack-rs + RUNNER_SCOPE: repo + REPO_URL: https://github.com/stack-rs/rattan ACCESS_TOKEN: {{ github_access_key }} LABELS: linux,x64,{{ kernel_version }} + HTTP_PROXY: "{{ http_proxy | default('') }}" + HTTPS_PROXY: "{{ http_proxy | default('') }}" + NO_PROXY: "localhost,127.0.0.1,::1" + http_proxy: "{{ http_proxy | default('') }}" + https_proxy: "{{ http_proxy | default('') }}" + no_proxy: "localhost,127.0.0.1,::1" cap_add: - NET_ADMIN - NET_RAW @@ -23,3 +29,4 @@ services: volumes: - '/var/run/docker.sock:/var/run/docker.sock' - '/sys:/sys' + - '/etc/runner/docker-config.json:/root/.docker/config.json:ro' diff --git a/ci/images/ansible/github-runner.yml b/ci/images/ansible/github-runner.yml index 02bedd6b..591835a2 100644 --- a/ci/images/ansible/github-runner.yml +++ b/ci/images/ansible/github-runner.yml @@ -4,6 +4,21 @@ path: /etc/runner state: directory +- name: write docker client proxy config for job-spawned containers + become: true + ansible.builtin.copy: + dest: /etc/runner/docker-config.json + content: | + { + "proxies": { + "default": { + "httpProxy": "{{ http_proxy | default('') }}", + "httpsProxy": "{{ http_proxy | default('') }}", + "noProxy": "localhost,127.0.0.1,::1" + } + } + } + - name: copy github-runner docker-compose file become: true ansible.builtin.template: diff --git a/ci/images/image.pkr.hcl b/ci/images/image.pkr.hcl index 37c44b86..99ba467c 100644 --- a/ci/images/image.pkr.hcl +++ b/ci/images/image.pkr.hcl @@ -77,12 +77,13 @@ source "libvirt" "image" { shell = "/bin/bash" lock_passwd = false hashed_passwd = "$6$rounds=4096$InVTnQ3fjMCSbc$ryRQrcU7ym0mvl.d7YxmR4HINu8/9u3XfG0KS4Ie59Pi8P5Xc9QoMRXOSVnEfpC4vJQn6Xa.2MHpBY6TeFZMH." - ssh_import_id = [ + ssh_import_id = var.key_import_user != "" ? [ "gh:${var.key_import_user}" - ] - ssh_authorized_keys = [ - data.sshkey.install.public_key, - ] + ] : [] + ssh_authorized_keys = concat( + [data.sshkey.install.public_key], + var.authorized_key_file != "" ? [trimspace(file(pathexpand(var.authorized_key_file)))] : [] + ) } ] @@ -90,10 +91,29 @@ source "libvirt" "image" { { path = "/etc/modules-load.d/bbr.conf" content = "tcp_bbr" + }, + # System-wide proxy for login/SSH sessions and generic tools. + { + path = "/etc/environment" + append = true + content = "http_proxy=${var.http_proxy}\nhttps_proxy=${var.http_proxy}\nHTTP_PROXY=${var.http_proxy}\nHTTPS_PROXY=${var.http_proxy}\nno_proxy=localhost,127.0.0.1,::1\nNO_PROXY=localhost,127.0.0.1,::1\n" + }, + # Preserve proxy vars across sudo (env_reset would otherwise drop them). + { + path = "/etc/sudoers.d/proxy" + permissions = "0440" + content = "Defaults env_keep += \"http_proxy https_proxy HTTP_PROXY HTTPS_PROXY no_proxy NO_PROXY ftp_proxy FTP_PROXY\"\n" + }, + # Proxy for the Docker daemon itself (image pulls: the runner image and any images CI jobs pull). + { + path = "/etc/systemd/system/docker.service.d/http-proxy.conf" + content = "[Service]\nEnvironment=\"HTTP_PROXY=${var.http_proxy}\"\nEnvironment=\"HTTPS_PROXY=${var.http_proxy}\"\nEnvironment=\"NO_PROXY=localhost,127.0.0.1,::1\"\n" } ] apt = { + # http_proxy = "${var.http_proxy}" + # https_proxy = "${var.http_proxy}" sources = { mainline = { source = "ppa:cappelikan/ppa" @@ -140,6 +160,11 @@ build { note = "You can examine the created domain with virt-manager, virsh or via SSH" } provisioner "shell" { + environment_vars = [ + "http_proxy=${var.http_proxy}", + "https_proxy=${var.http_proxy}", + "no_proxy=localhost,127.0.0.1,::1", + ] inline = [ "set -xo pipefail", "sudo mainline list | grep -E \"^[0-9]+\\.[0-9]+\" | grep -E \"^${var.kernel_version}\" | head -n 1 | tr -d ' ' | sed -e 's/Installed//' | xargs -I {} sudo mainline install {}" @@ -161,6 +186,6 @@ build { provisioner "ansible" { playbook_file = "./ansible/configure.yml" galaxy_file = "./ansible/requirements.yml" - extra_arguments = [ "--extra-vars", "github_access_key=${var.github_access_key} kernel_version=${var.kernel_version} runner_name=${var.release_name}-${var.kernel_version} tag=ubuntu-${var.release_name} install_runner=${var.install_runner}" ] + extra_arguments = [ "--extra-vars", "github_access_key=${var.github_access_key} kernel_version=${var.kernel_version} runner_name=${var.release_name}-${var.kernel_version} tag=ubuntu-${var.release_name} install_runner=${var.install_runner} http_proxy=${var.http_proxy}" ] } } diff --git a/ci/images/variables.pkr.hcl b/ci/images/variables.pkr.hcl index 2f37b7f6..ae6574b2 100644 --- a/ci/images/variables.pkr.hcl +++ b/ci/images/variables.pkr.hcl @@ -41,10 +41,21 @@ variable "kernel_version" { } variable "key_import_user" { - type = string + type = string + default = "" +} + +variable "authorized_key_file" { + type = string + default = "" } variable "install_runner" { type = bool default = true } + +variable "http_proxy" { + type = string + default = "" +}