From badb0f496db087cc55e0c7dbf700fa0b5568534b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=87a=C4=9Fatay=20Yi=C4=9Fit=20=C5=9Eahin?= Date: Sun, 17 Aug 2025 22:31:36 +0200 Subject: [PATCH 1/2] ci: create builder image The Dockerfile defines an image that contains the tools necessary to build the kernel and modules, and builds the appropriate version of the kernel to be used for CI. The workflow builds and pushes the image when the image definition is changed. --- .github/workflows/container.yaml | 43 ++++++++++++++++++++++++++++++++ Dockerfile | 17 +++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 .github/workflows/container.yaml create mode 100644 Dockerfile diff --git a/.github/workflows/container.yaml b/.github/workflows/container.yaml new file mode 100644 index 0000000..2c6b707 --- /dev/null +++ b/.github/workflows/container.yaml @@ -0,0 +1,43 @@ +name: Create and publish a Docker image + +on: + push: + paths: + - "Dockerfile" + +env: + REGISTRY: ghcr.io + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ github.repository }}-builder + + - name: Build and push Docker image + id: push + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..774dd79 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +# The virtme-ng present in the latest LTS at the time of writing has a bug that causes +# the VM to shutdown unexpectedly. +FROM ubuntu:25.04 + +# deb-src repositories are required for `apt-get build-dep`. +RUN sed -i 's/Types: deb/Types: deb deb-src/g' /etc/apt/sources.list.d/ubuntu.sources + +# iproute2, kbd and udev are required by virtme-ng +# nodejs is needed for the GitHub Actions cache scripts +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y virtme-ng iproute2 kbd udev fio nodejs && \ + apt-get build-dep -y --no-install-recommends linux +WORKDIR /usr/src/ +ADD https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-6.5.7.tar.xz /usr/src/ +RUN tar xf linux-6.5.7.tar.xz +WORKDIR /usr/src/linux-6.5.7/ +RUN vng --configitem CONFIG_DEBUG_KMEMLEAK=y --build From fe77eef3fbf6c848226343c041e7e2b77de233e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=87a=C4=9Fatay=20Yi=C4=9Fit=20=C5=9Eahin?= Date: Sun, 17 Aug 2025 22:41:27 +0200 Subject: [PATCH 2/2] ci: add checks Use an adapted version of the provided evaluation script to run various checks. --- .github/workflows/check.yaml | 54 ++++ evaluation.sh | 526 +++++++++++++++++++++++++++++++++++ 2 files changed, 580 insertions(+) create mode 100644 .github/workflows/check.yaml create mode 100644 evaluation.sh diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml new file mode 100644 index 0000000..99f55a5 --- /dev/null +++ b/.github/workflows/check.yaml @@ -0,0 +1,54 @@ +name: Check + +on: [push] + +jobs: + build: + runs-on: ubuntu-latest + container: + image: ghcr.io/${{ github.repository }}-builder:ci_pr + steps: + - name: Check out repository code + uses: actions/checkout@v4 + + - name: Build ouichefs + run: make -C /usr/src/linux-6.5.7/ M=$PWD + + - name: Prepare disk image + run: make -C mkfs/ img IMG=a.img + + - name: Cache built module + uses: actions/cache/save@v4 + with: + path: ./ + key: ${{ github.sha }} + + check: + needs: build + runs-on: ubuntu-latest + container: + image: ghcr.io/${{ github.repository }}-builder:ci-pr + strategy: + matrix: + checks: [ouichefs_basic_behavior, ouichefs_fio] + timeout-minutes: 10 + steps: + - name: Restore built module + uses: actions/cache/restore@v4 + with: + key: ${{ github.sha }} + path: ./ + + - name: Prepare images + run: cp mkfs/a.img mkfs/b.img + + - name: Check ${{ matrix.checks }}... + run: > + vng + --verbose + --run=/usr/src/linux-6.5.7 + --qemu-opts='-drive file=mkfs/a.img,format=raw,if=virtio' + --qemu-opts='-drive file=mkfs/b.img,format=raw,if=virtio' + --exec='make -C /lib/modules/\`uname -r\`/build M=$PWD modules_install + && source ./evaluation.sh + && check_${{ matrix.checks }}' diff --git a/evaluation.sh b/evaluation.sh new file mode 100644 index 0000000..7ceb85d --- /dev/null +++ b/evaluation.sh @@ -0,0 +1,526 @@ +#!/bin/bash -e +modulename=ouichefs + +sysfs="/sys/fs/ouichefs" +# sysfs="/sys/ouichefs" + +exit_fail() { + echo "test failed: $1" + exit 1 +} + +# this script requires two drives to be attached to the VM +# under /dev/vda and /dev/vdb +mnta=/tmp/mnt1 +mntb=/tmp/mnt2 +mkdir -p $mnta +mkdir -p $mntb + +check_ouichefs_basic_behavior() { + set -x + modprobe $modulename + + mount /dev/vda $mnta + cd $mnta + rm -fr $mnta/* + + mkdir dir + echo "some data" > dir/file + echo "some data" > file + echo "some data" > file2 + mv file2 dir + echo "some more data" > file + + ls -Ril > /dev/null + rm file + rm -fr dir + ls -Ril > /dev/null + + rm -fr $mnta/* + touch file{1..65} + mkdir dir2 + mv file{1..65} dir2 + rm -fr $mnta/* + + echo "data" > file + echo "data" >> file + echo "data" >> file + echo "data" >> file + echo "data" >> file + echo "data" >> file + echo "data" >> file + echo "data" >> file + echo "data" >> file + echo "data" >> file + echo "data" >> file + rm file + + cd + umount $mnta + rmmod $modulename +} + +check_ouichefs_fio() { + set -x + modprobe $modulename + + mount /dev/vda $mnta + cd $mnta + rm -fr $mnta/* + + echo "[global] +nrfiles=10 +directory=$mnta +size=16M +runtime=20s +ioengine=sync +time_based +bs=4k +verify=sha1 +verify_fatal=1 + +group_reporting + +[rw] +rw=randrw +numjobs=1 + +[rwappend] +file_append=1 +rw=randrw +numjobs=1" > /tmp/workload.fio + + fio -f /tmp/workload.fio > /dev/null + cd + rm -fr $mnta/* + umount $mnta + rmmod $modulename +} + +check_sysfs_structure() { + set -x + modprobe $modulename + + mount /dev/vda $mnta + mount /dev/vdb $mntb + + for dev in vda vdb; do + + if ! [ -d "$sysfs/$dev" ]; then + exit_fail "$sysfs/$dev not created!" + fi + + for file in free_blocks used_blocks sliced_blocks total_free_slices files small_files total_data_size total_used_size efficiency ; do + variable="$sysfs/$dev/$file" + if ! [ -f "$variable" ]; then + exit_fail "$variable not created!" + fi + if ! cat "$variable" > /dev/null; then + exit_fail "error while reading ${variable}!" + fi + done + done + + umount $mnta + umount $mntb + + for dev in vda vdb; do + if [ -d "$sysfs/$dev" ]; then + exit_fail "$sysfs/$dev still present after unmount!" + fi + done + + rmmod $modulename +} + +check_simple_write() { + set -x + modprobe $modulename + + mount /dev/vda $mnta + mount /dev/vdb $mntb + + variable_used="$sysfs/vda/used_blocks" + variable_small_files="$sysfs/vda/small_files" + variable_free_slices="$sysfs/vda/total_free_slices" + variable_slices="$sysfs/vda/sliced_blocks" + + # make sure there's at least 1 filled block allocated + dd if=/dev/urandom of=$mnta/file0 bs=48 count=1 2>/dev/null + + if ! used_before=$(cat $variable_used); then + exit_fail "error while reading ${variable_used}!" + fi + if ! free_slices_before=$(cat $variable_free_slices); then + exit_fail "error while reading ${variable_free_slices}!" + fi + if ! slices_before=$(cat $variable_slices); then + exit_fail "error while reading ${variable_slices}!" + fi + if ! small_files_before=$(cat $variable_small_files); then + exit_fail "error while reading ${variable_small_files}!" + fi + + # allocate a new file + dd if=/dev/urandom of="$mnta/file1" bs=4k count=1 2>/dev/null + + if ! used_after=$(cat $variable_used); then + exit_fail "error while reading ${variable_used}!" + fi + if ! [ "$((used_before + 2))" -eq "$used_after" ]; then + exit_fail ">error while counting $variable_used: $used_before $used_after" + fi + + if ! used_before=$(cat $variable_used); then + exit_fail "error while reading ${variable_used}!" + fi + + dd if=/dev/urandom of=$mnta/file2 bs=18 count=1 2>/dev/null + dd if=/dev/urandom of=$mnta/file3 bs=48 count=1 2>/dev/null + dd if=/dev/urandom of=$mnta/file4 bs=127 count=1 2>/dev/null + + if ! used_after=$(cat $variable_used); then + exit_fail "error while reading ${variable_used}!" + fi + if ! slices_after=$(cat $variable_slices); then + exit_fail "error while reading ${variable_slices}!" + fi + if ! free_slices_after=$(cat $variable_free_slices); then + exit_fail "error while reading ${variable_free_slices}!" + fi + if ! small_files_after=$(cat $variable_small_files); then + exit_fail "error while reading ${variable_small_files}!" + fi + if ! [ "$((free_slices_before - 3))" -eq "$free_slices_after" ]; then + exit_fail ">>error while counting $variable_free_slices: $free_slices_before $free_slices_after" + fi + if ! [ "$used_before" -eq "$used_after" ]; then + exit_fail ">>error while counting $variable_used: $used_before $used_after" + fi + if ! [ "$((small_files_before + 3))" -eq "$small_files_after" ]; then + exit_fail ">>error while counting $variable_small_files: $small_files_before $small_files_after" + fi + if ! [ "$((slices_after))" -eq "$slices_before" ]; then + exit_fail ">>error while counting $variable_slices: $slices_before $slices_after" + fi + + if ! slices_before=$(cat $variable_slices); then + exit_fail "error while reading ${variable_slices}!" + fi + for i in $(seq 45); do + dd if=/dev/urandom of="$mnta/_file$i" bs=128 count=1 2>/dev/null + done + if ! slices_after=$(cat $variable_slices); then + exit_fail "error while reading ${variable_slices}!" + fi + if ! [ "$((slices_after))" -eq "$((slices_before + 1))" ]; then + exit_fail ">>error while counting $variable_slices: $slices_before $slices_after" + fi + + umount $mnta + umount $mntb + + for dev in vda vdb; do + if [ -d "$sysfs/$dev" ]; then + exit_fail "$sysfs/$dev still present after unmount!" + fi + done + + rmmod $modulename +} + +check_remove() { + set -x + modprobe $modulename + + mount /dev/vda $mnta + mount /dev/vdb $mntb + + variable_used="$sysfs/vda/used_blocks" + variable_small_files="$sysfs/vda/small_files" + variable_free_slices="$sysfs/vda/total_free_slices" + variable_slices="$sysfs/vda/sliced_blocks" + + if ! used_before=$(cat $variable_used); then + exit_fail "error while reading ${variable_used}!" + fi + if ! slices_before=$(cat $variable_slices); then + exit_fail "error while reading ${variable_slices}!" + fi + if ! free_slices_before=$(cat $variable_free_slices); then + exit_fail "error while reading ${variable_free_slices}!" + fi + if ! small_files_before=$(cat $variable_small_files); then + exit_fail "error while reading ${variable_small_files}!" + fi + + rm "$mnta/"file{0,2,3,4} + + if ! used_after=$(cat $variable_used); then + exit_fail "error while reading ${variable_used}!" + fi + if ! slices_after=$(cat $variable_slices); then + exit_fail "error while reading ${variable_slices}!" + fi + if ! free_slices_after=$(cat $variable_free_slices); then + exit_fail "error while reading ${variable_free_slices}!" + fi + if ! small_files_after=$(cat $variable_small_files); then + exit_fail "error while reading ${variable_small_files}!" + fi + + if ! [ "$used_before" -eq "$used_after" ]; then + exit_fail ">>error while counting $variable_used: $used_before $used_after" + fi + if ! [ "$((small_files_before - 4))" -eq "$small_files_after" ]; then + exit_fail ">>error while counting $variable_small_files: $small_files_before $small_files_after" + fi + if ! [ "$((slices_after))" -eq "$slices_before" ]; then + exit_fail ">>error while counting $variable_slices: $slices_before $slices_after" + fi + if ! [ "$((free_slices_before + 4))" -eq "$free_slices_after" ]; then + exit_fail ">>error while counting $variable_free_slices: $free_slices_before $free_slices_after" + fi + + if ! slices_before=$(cat $variable_slices); then + exit_fail "error while reading ${variable_slices}!" + fi + for i in $(seq 1 44); do + rm "$mnta/_file$i" + done + if ! slices_after=$(cat $variable_slices); then + exit_fail "error while reading ${variable_slices}!" + fi + if ! [ "$((slices_after))" -eq "$((slices_before - 1))" ]; then + exit_fail ">>error while counting $variable_slices: $slices_before $slices_after" + fi + + + umount $mnta + umount $mntb + + for dev in vda vdb; do + if [ -d "$sysfs/$dev" ]; then + exit_fail "$sysfs/$dev still present after unmount!" + fi + done + + rmmod $modulename +} + +check_coexistence_trad_fs() { + set -x + modprobe $modulename + + mount /dev/vda $mnta + mount /dev/vdb $mntb + + dd if=/dev/urandom of=$mntb/file1 bs=128 count=1 2>/dev/null + + variable_used="$sysfs/vdb/used_blocks" + variable_small_files="$sysfs/vdb/small_files" + variable_free_slices="$sysfs/vdb/total_free_slices" + variable_slices="$sysfs/vdb/sliced_blocks" + + if ! used_before=$(cat $variable_used); then + exit_fail "error while reading ${variable_used}!" + fi + if ! slices_before=$(cat $variable_slices); then + exit_fail "error while reading ${variable_slices}!" + fi + if ! free_slices_before=$(cat $variable_free_slices); then + exit_fail "error while reading ${variable_free_slices}!" + fi + if ! small_files_before=$(cat $variable_small_files); then + exit_fail "error while reading ${variable_small_files}!" + fi + + dd if=/dev/urandom of=$mntb/file3 bs=128 count=1 2>/dev/null + + if ! used_after=$(cat $variable_used); then + exit_fail "error while reading ${variable_used}!" + fi + if ! slices_after=$(cat $variable_slices); then + exit_fail "error while reading ${variable_slices}!" + fi + if ! free_slices_after=$(cat $variable_free_slices); then + exit_fail "error while reading ${variable_free_slices}!" + fi + if ! small_files_after=$(cat $variable_small_files); then + exit_fail "error while reading ${variable_small_files}!" + fi + if ! [ "$((free_slices_before - 1))" -eq "$free_slices_after" ]; then + exit_fail ">>error while counting $variable_free_slices: $free_slices_before $free_slices_after" + fi + if ! [ "$used_before" -eq "$used_after" ]; then + exit_fail ">>error while counting $variable_used: $used_before $used_after" + fi + if ! [ "$((small_files_before + 1))" -eq "$small_files_after" ]; then + exit_fail ">>error while counting $variable_small_files: $small_files_before $small_files_after" + fi + if ! [ "$((slices_after))" -eq "$slices_before" ]; then + exit_fail ">>error while counting $variable_slices: $slices_before $slices_after" + fi + + + if ! slices_before=$(cat $variable_slices); then + exit_fail "error while reading ${variable_slices}!" + fi + if ! free_slices_before=$(cat $variable_free_slices); then + exit_fail "error while reading ${variable_free_slices}!" + fi + if ! small_files_before=$(cat $variable_small_files); then + exit_fail "error while reading ${variable_small_files}!" + fi + + dd if=/dev/urandom of=$mntb/file3 bs=4k count=1 2>/dev/null + + if ! slices_after=$(cat $variable_slices); then + exit_fail "error while reading ${variable_slices}!" + fi + if ! free_slices_after=$(cat $variable_free_slices); then + exit_fail "error while reading ${variable_free_slices}!" + fi + if ! small_files_after=$(cat $variable_small_files); then + exit_fail "error while reading ${variable_small_files}!" + fi + if ! [ "$((small_files_before - 1))" -eq "$small_files_after" ]; then + exit_fail ">>error while counting $variable_small_files: $small_files_before $small_files_after" + fi + if ! [ "$((free_slices_before + 1))" -eq "$free_slices_after" ]; then + exit_fail ">>error while counting $variable_free_slices: $free_slices_before $free_slices_after" + fi + if ! [ "$((slices_after))" -eq "$slices_before" ]; then + exit_fail ">>error while counting $variable_slices: $slices_before $slices_after" + fi + + umount $mnta + umount $mntb + + for dev in vda vdb; do + if [ -d "$sysfs/$dev" ]; then + exit_fail "$sysfs/$dev still present after unmount!" + fi + done + + rmmod $modulename +} + +check_spanning_multiple_slices() { + set -x + modprobe $modulename + + mount /dev/vda $mnta + mount /dev/vdb $mntb + + variable_used="$sysfs/vdb/used_blocks" + variable_small_files="$sysfs/vdb/small_files" + variable_free_slices="$sysfs/vdb/total_free_slices" + variable_slices="$sysfs/vdb/sliced_blocks" + + dd if=/dev/urandom of=$mntb/file1 bs=100 count=1 2>/dev/null + + if ! used_before=$(cat $variable_used); then + exit_fail "error while reading ${variable_used}!" + fi + if ! slices_before=$(cat $variable_slices); then + exit_fail "error while reading ${variable_slices}!" + fi + if ! free_slices_before=$(cat $variable_free_slices); then + exit_fail "error while reading ${variable_free_slices}!" + fi + if ! small_files_before=$(cat $variable_small_files); then + exit_fail "error while reading ${variable_small_files}!" + fi + + dd if=/dev/urandom of=$mntb/file2 bs=100 count=1 oflag=append conv=notrunc 2>/dev/null + dd if=/dev/urandom of=$mntb/file2 bs=128 count=1 oflag=append conv=notrunc 2>/dev/null + + if ! used_after=$(cat $variable_used); then + exit_fail "error while reading ${variable_used}!" + fi + if ! slices_after=$(cat $variable_slices); then + exit_fail "error while reading ${variable_slices}!" + fi + if ! free_slices_after=$(cat $variable_free_slices); then + exit_fail "error while reading ${variable_free_slices}!" + fi + if ! small_files_after=$(cat $variable_small_files); then + exit_fail "error while reading ${variable_small_files}!" + fi + if ! [ "$((free_slices_before - 2))" -eq "$free_slices_after" ]; then + exit_fail ">>error while counting $variable_free_slices: $free_slices_before $free_slices_after" + fi + if ! [ "$used_before" -eq "$used_after" ]; then + exit_fail ">>error while counting $variable_used: $used_before $used_after" + fi + if ! [ "$((small_files_before + 1))" -eq "$small_files_after" ]; then + exit_fail ">>error while counting $variable_small_files: $small_files_before $small_files_after" + fi + if ! [ "$((slices_after))" -eq "$slices_before" ]; then + exit_fail ">>error while counting $variable_slices: $slices_before $slices_after" + fi + + if ! slices_before=$(cat $variable_slices); then + exit_fail "error while reading ${variable_slices}!" + fi + if ! free_slices_before=$(cat $variable_free_slices); then + exit_fail "error while reading ${variable_free_slices}!" + fi + if ! small_files_before=$(cat $variable_small_files); then + exit_fail "error while reading ${variable_small_files}!" + fi + + dd if=/dev/urandom of=$mntb/file2 bs=128 count=1 oflag=append conv=notrunc 2>/dev/null + dd if=/dev/urandom of=$mntb/file2 bs=128 count=1 oflag=append conv=notrunc 2>/dev/null + dd if=/dev/urandom of=$mntb/file2 bs=128 count=1 oflag=append conv=notrunc 2>/dev/null + + if ! slices_after=$(cat $variable_slices); then + exit_fail "error while reading ${variable_slices}!" + fi + if ! free_slices_after=$(cat $variable_free_slices); then + exit_fail "error while reading ${variable_free_slices}!" + fi + if ! small_files_after=$(cat $variable_small_files); then + exit_fail "error while reading ${variable_small_files}!" + fi + if ! [ "$((small_files_before))" -eq "$small_files_after" ]; then + exit_fail ">>error while counting $variable_small_files: $small_files_before $small_files_after" + fi + if ! [ "$((free_slices_before - 3))" -eq "$free_slices_after" ]; then + exit_fail ">>error while counting $variable_free_slices: $free_slices_before $free_slices_after" + fi + if ! [ "$((slices_after))" -eq "$slices_before" ]; then + exit_fail ">>error while counting $variable_slices: $slices_before $slices_after" + fi + + umount $mnta + umount $mntb + + for dev in vda vdb; do + if [ -d "$sysfs/$dev" ]; then + exit_fail "$sysfs/$dev still present after unmount!" + fi + done + + rmmod $modulename + +} +check_memory_leak() { + set -x + check_ouichefs_basic_behavior + check_ouichefs_fio + check_sysfs_structure + check_simple_write + check_remove + check_coexistence_trad_fs + check_spanning_multiple_slices + check_ouichefs_basic_behavior + + echo scan > /sys/kernel/debug/kmemleak + sleep 60 + if [ -s /sys/kernel/debug/kmemleak ]; then + cat /sys/kernel/debug/kmemleak + exit 1 + fi +}