diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 3302d067..7a3fd0ce 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -30,7 +30,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install pymarkdownlnt==0.9.36 + pip install $(cat config/requirements.txt | grep -E ^pymarkdownlnt) - name: Check ProperDocs YAML configuration uses: ibiqlik/action-yamllint@v3 with: diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 151c63cb..6cc38c7b 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -1,37 +1,37 @@ -# documentation: https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax -# -name: Test build NCC course material website -on: - pull_request: - push: - branches-ignore: - - main - - production - workflow_dispatch: -permissions: - contents: read - pages: write - id-token: write -jobs: - check: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - lfs: true - - name: Read Python version - id: pyversion - run: echo "PYTHON_VERSION=$(cat .python)" >> $GITHUB_ENV - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '${{ env.PYTHON_VERSION }}' - cache: 'pip' - - name: Install dependencies - run: | + # documentation: https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax + # + name: Test build NCC course material website + on: + pull_request: + push: + branches-ignore: + - main + - production + workflow_dispatch: + permissions: + contents: read + pages: write + id-token: write + jobs: + check: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + lfs: true + - name: Read Python version + id: pyversion + run: echo "PYTHON_VERSION=$(cat .python)" >> $GITHUB_ENV + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '${{ env.PYTHON_VERSION }}' + cache: 'pip' + - name: Install dependencies + run: | python -m pip install --upgrade pip - pip install pymarkdownlnt==0.9.36 + pip install $(cat config/requirements.txt | grep -E ^pymarkdownlnt) - name: Check ProperDocs YAML configuration uses: ibiqlik/action-yamllint@v3 with: diff --git a/README.md b/README.md index 6195e8a9..0e230cdd 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,22 @@ source ~/environments/ncc/bin/activate uv pip install --requirements requirements.txt ``` +## Deploying the page locally + +The project is compiled and deployed locally with `properdocs`. Start by loading the development environment. + +```bash +source ~/environments/ncc/bin/activate +``` + +The typeset and deploy the page with `properdocs`. + +```bash +properdocs serve --strict +``` + +The `--strict` flag causes the deployment to fail in case of errors. + ### Compiling the images in resource directories There is code for LaTeX generated figures in some sections. The source code is located in directories named `resources`. The compilation of the code requires [TeX Live](https://tug.org/texlive/) and the [`convert`](https://linux.die.net/man/1/convert) tool. Compile the images manually by running the command @@ -24,3 +40,25 @@ make inside the `resource` directories. +### Linting + +To lint the documentation locally, install in the environment the linting packages. + +```console +source ~/environments/ncc/bin/activate +uv pip install --requirements config/requirements.txt +``` + +Then, lint the markdown files with the following command. + +```console +pymarkdown \ + --config config/pymarkdown.yml --strict-config scan \ + --recurse docs +``` + +Lint the ProperDocs configuration with the following command. + +```console +yamllint --config-file config/yamllint.yml properdocs.yml +``` diff --git a/config/requirements.txt b/config/requirements.txt new file mode 100644 index 00000000..55153fa1 --- /dev/null +++ b/config/requirements.txt @@ -0,0 +1,2 @@ +pymarkdownlnt==0.9.38 +yamllint==1.38.0 diff --git a/docs/courses/containerization/README.txt b/docs/courses/containerization/README.txt index 196295c9..d4626968 100644 --- a/docs/courses/containerization/README.txt +++ b/docs/courses/containerization/README.txt @@ -283,3 +283,98 @@ echo 'Welcome to chroot jail!' > ${HOME}/jail/home/myusername/hello.txt ## Building a container * References [1, Shipping Your Machine: Building a Container in 50 Lines of Code](https://dev.to/yechielk/shipping-your-machine-building-a-container-in-60-lines-of-code-part-1-14ma), [2, Build Your Own Docker with Linux Namespaces, cgroups, and chroot: Hands-on Guide ](https://akashrajpurohit.com/blog/build-your-own-docker-with-linux-namespaces-cgroups-and-chroot-handson-guide/) * + +## Running Apptainer in Grid' 5000 + +```console +sudo-g5k +sudo apt install libseccomp-dev squashfuse cryptsetup +module use /home/gkafanas/ulhpc/easybuild/release/2025a/modules/all +sudo mount -o remount,suid /srv/storage/ulhpc@storage1.luxembourg.grid5000.fr +``` + +## References + +- https://snarky.ca/how-virtual-environments-work/ + +- Source of DIA-NN examples: https://mbite.mdhs.unimelb.edu.au/intro-to-proteomics/02diann.html + +- https://en.wikipedia.org/wiki/Linux_namespaces + +- https://www.redhat.com/en/blog/7-linux-namespaces +- https://www.redhat.com/en/blog/pid-namespace +- https://www.redhat.com/en/blog/linux-pid-namespaces + +- https://www.hackerstack.org/understanding-linux-namespaces/ + +- https://www.toptal.com/developers/linux/separation-anxiety-isolating-your-system-with-linux-namespaces + +## Virtual environments + +### The `venv` in Python + +### The `renv` in R + +## Containers + +An [introduction to Docker](https://carpentries-incubator.github.io/docker-introduction/advanced-containers.html) is available in a carpentry course. + +### Singularity type containers + +### Using Singularity containers + +#### Existing files and artifacts + +#### Fetching from SHPC registries + +- https://docs.alcf.anl.gov/crux/containers/containers/ +- https://github.com/argonne-lcf/container-registry/tree/main/containers/shpc + +### Building Singularity containers + +#### Builds that do not require elevated privileges + +#### Builds requiring `fakeroot` or id mapping + +- https://apptainer.org/docs/admin/latest/user_namespace.html +- https://apptainer.org/docs/admin/latest/installation.html#fakeroot-with-uid-gid-mapping-on-network-filesystems +- https://apptainer.org/docs/admin/latest/configfiles.html + +## Containerization + +### Creating a `chroot` environment + +### Modern containers and `namegroups` + +### Creating a containerized environment from first principle + +## Advanced containerization concepts + +### Connecting with system libraries + +#### MPI network libraries + +#### Offloading libraries and drivers + +### Reproducibility + +### Building a reproducible containers + +### Balancing reproducibility and usage of efficient system libraries + +For instance, imagine that there is an MPI library that ensures binary reproducibility. Your container could ensure binary reproducibility if it linked with this external MPI library, and still ensure native level MPI performance in every system that supports this MPI library. + +- Introductory example + - chroot base example + - the linux hier & file system exporters +- containers in HPC + - basic usage + - building contaiers + - simple + - requiring root +- modern containers + - namespaces and cgroups + - advanced bindings + - using mpi: container, native, on the fly compiled + - using accelerators +- reproducible builds diff --git a/docs/courses/containerization/containers.md b/docs/courses/containerization/containers.md index 2c9125eb..3ba5d4a3 100644 --- a/docs/courses/containerization/containers.md +++ b/docs/courses/containerization/containers.md @@ -1,146 +1,149 @@ -# Containers +# Basic principles of containers -Operating systems like GNU/Linux consists of 2 components, the kernel and the user space. The kernel contains the hardware drivers and provides interfaces for applications to access the hardware. For instance, the kernel provides functions for applications to write to storage devices. The user space contains libraries and executables required for programs to run. For instance the `glibc` package of Debian provides some version of the `libc` library that defines functions such as `malloc`. If you work on a RHEL based user space, you cannot use an application build for Debian based systems are they require a different version of `libc`. Containers solve this problem by creating isolated and swappable user spaces. +Containers use modern kernel features to enable running applications in isolated environments. The techniques used by container allow much better isolation of the environment from the underlying system that what is offered by virtual environments. Operating systems like GNU/Linux usually separate processes 2 groups, the kernel and the user space, that run in different memory regions. The kernel is a privileged process that schedules processes, manages resources such as CPU and memory usage, and handles devices such as storage, network, and I/O devices. Processes request services from the kernel invoking _system calls_ (see [fig. 1](#fig-1)). For instance, the kernel provides functions for applications to write to storage devices. The user space contains user processes and operating system libraries and executables required for user programs to run. For instance the `glibc` package of Debian provides some version of the `libc` library as part of the operating system. The `libc` library defines functions such as `malloc`, that provides an interface for user programs to access memory in a manner that hides the complexities of memory management by the kernel.
- ![Container architecture](images/container.png){ align=left width="400" } -
Fig. 1: Architecture of a simple container
-
- -## Container technologies + ![Containerization techniques](resources/linux_interface.png){ align=left width="800" } -The term _container_ describes a wide and loose set of technologies that enable software applications to run in isolated user spaces. The 3 main systems used in containers are the following. +
Fig. 1: Linux application interface and containerization techniques. Containerization methods control the interface available between the users process, the operating system, and the operating system kernel.
+ -- [_Chroot_](https://en.wikipedia.org/wiki/Chroot): a system operation that changes the apparent root directory for the current running process and its children. -- [_Namespaces_](https://en.wikipedia.org/wiki/Linux_namespaces): a Linux kernel feature that limits the resources that a set of processes can access; examples of resource include process IDs, user IDs, and interprocess communication mechanisms and groups. -- [_Control groups_ (`cgroups`)](https://en.wikipedia.org/wiki/Cgroups): a Linux kernel feature that limits, accounts for, and isolates the resource usage for collections of processes; resource include CPU cores, memory, and disk IOPs [1]. +Typically, each container has its own `\bin` and `\lib` directories for instance that completely override the host system directories inside the container. Since operating system binaries and libraries reside in `\bin` and `\lib` respectively, containers can run any operating system whose kernel is compatible with the host kernel. Despite each container running its own operating system, all containers use the kernel of the host system, as depicted in [fig. 2](#fig-2).
- ![Overview of Linux containers](images/linux-containers.png){ align=left width="400" } -
Fig. 2: Overview of technologies used in Linux containers
-
+ ![Container architecture](resources/container_architecture.png){ align=left width="800" } -## A toy example with Chroot +
Fig. 2: A diagram of a container engine running 2 container, one container with 2 processes and one container with a single process.
+ -Before working on containers for HPC systems let's explore how `chroot`, a fundamental technology for containers, works. You should run this example in your personal machine, as the `chroot` command requires super user privileges (`sudo`). Modern containers use more sophisticated methods to control the access to resources, so no special privileges are required for most operations. +??? info "Differences between containers and virtual machines" -1. Create the root of the `chroot` file system: + Containers perform similar but distinct functions to virtual machines. In virtualisation systems, multiple kernels run on top of a hypervisor, a lightweight operating system presenting guest operating systems with a virtual operating platform. Each virtual machine requires its own kernel, and thus is significantly more resource intensive than a container (see [fig (a)](#fig-a1)). On the other hand, virtual machines can use different kernels on the same machine. - ```bash - mkdir --parents ${HOME}/jail/{bin,lib,lib64,home/myusername} - ``` + Running multiple kernels on the same machine is useful for instance in applications that require different device drivers. Device drivers are part of the kernel. Hypervisors typically expose physical devices to virtual machines. If a machine has 2 GPUs for instance, it can host one virtual machine per GPU, and each machine can use a different device driver for the GPU. -2. Copy the executables that should be available in the `chroot` system: +
+ ![Comparison of virtual machines and containers](resources/container_VM_comparison.png){ align=left width="800" } - ```bash - for binary in /bin/bash /bin/ls /bin/cat; do - cp ${binary} ${HOME}/jail/bin/ - done; unset binary - ``` +
Fig. (a): A diagram of a container engine running 2 container, and a virtualisation server running the same workload on 2 virtual machines. Note that the virtualisation server must run 2 kernel instances even if the 2 machines use the exact same kernel.
+
-3. Copy the linker: +## Example: chroot jail environment - ```bash - for linker in /lib64/ld-linux-x86-64.so*; do - cp ${linker} ${HOME}/jail/lib64/ - done; unset linker - ``` +Before working on containers for HPC systems let's explore one of the first technologies that allowed the creation of isolated process environments, the `chroot` command. You should run this example in a machine where you have super user privileges (`sudo`), as the command required `sudo` access. -4. Copy the libraries required by the executables: +1. Create the root of the `chroot` file system. - ```bash - while IFS="" read -r library; do - cp ${library} ${HOME}/jail/lib/ - done < <(ldd /bin/bash /bin/ls /bin/cat | grep -E '=>' | awk 'BEGIN {FS="(=>)|( +)"} {print $4}' | sort | uniq); unset library - ``` + ```bash + mkdir --parents ${HOME}/jail/{bin,lib,lib64,home/jailuser} + ``` -5. Create a text file to test the executables: +1. Copy the executables that should be available in the `chroot` system. - ```bash - echo 'Welcome to chroot jail!' > ${HOME}/jail/home/myusername/hello.txt - ``` + ```bash + for binary in /bin/bash /bin/sh /bin/ls /bin/cat; do + cp ${binary} ${HOME}/jail/bin/ + done; unset binary + cp $(realpath /bin/which) ${HOME}/jail/bin/which + ``` -Your isolated environment is now created in `${HOME}/jail`. You can change to the `jail` environment with the command: +1. Copy the linker. -```console -$ sudo chroot ${HOME}/jail /bin/bash -bash-5.2# -``` + ```bash + for linker in /lib64/ld-linux-x86-64.so*; do + cp ${linker} ${HOME}/jail/lib64/ + done; unset linker + ``` -Any program running in the isolated environment cannot name and thus cannot access files outside the environment. Thus the term _jail_ is often used to describe an isolated environment created with `chroot`. +1. Copy the libraries required by the executables. -??? info "Accessing host system resources" + ```bash + while IFS="" read -r library; do + cp ${library} ${HOME}/jail/lib/ + done < <(ldd /bin/bash /bin/sh /bin/ls /bin/cat $(/bin/which) | grep -E '=>' | awk 'BEGIN {FS="(=>)|( +)"} {print $4}' | sort | uniq); unset library + ``` - In a typical system you may want to access more resources, like: +1. Create a text file to test the executables. - - `/proc`, the process information pseudo-filesystem; - - `/sys`, a mount point for the sysfs filesystem, which provides information about the kernel like `/proc`, but better structured; - - `/dev`, special or device files, which refer to physical devices; - - `/run`, a directory that contains information about processes that describes the system state; - - your home directory under `/home`. + ```bash + echo 'Welcome to chroot jail!' > ${HOME}/jail/home/jailuser/hello.txt + ``` - You can use `man hier` to see more of the resource you may need to access inside a container. +1. Your isolated environment is now created in `${HOME}/jail`. To use the chroot jail environment, simply call `chroot` on the root of the jail. - You can mount any resource files using bindings: - ```bash - for fs in dev dev/pts proc sys sys/firmware/efi/efivars run; do - sudo mount --bind "/${fs}" "${system_root}/${fs}" - done; unset fs - ``` + ```console + sudo chroot ${HOME}/jail /bin/bash + ``` - However, this process involves a lot of manual work, and exposes system components to the jailed system. Containers automate this process and use systems such as namespaces to resolve the security issues. +Any program running in the isolated environment cannot name and thus cannot access paths outside the environment. Thus the term _chroot jail_ is often used to describe an isolated environment created with `chroot`. -## Containers for HPC systems +!!! question "Using a chroot jail" -The main container systems used in HPC platforms are [Apptainer](https://apptainer.org/docs/user/latest/) and [Singularity](https://docs.sylabs.io/guides/latest/user-guide/). Both these container systems are built around the Singularity Image File (SIF) format of containers, and are interchangeable in most cases. There are other container format such as [Docker](https://www.docker.com/), but these systems are often not well suited for HPC applications. + === "Question" -??? info "Relation between Apptainer and Singularity" - Singularity began as an open-source project in 2015, when a team of researchers at Lawrence Berkeley National Laboratory. The original project joined the [Linux Foundation](https://www.linuxfoundation.org/) and was renamed to Apptainer, where as the original Singularity code has been forked by the company Sylabs and distributed under various licenses and names. + Can you list the content of the home directory of the chroot jail (`/home/jailuser`), and display the contents of `hello.txt` which is located in the home directory? What are the contents of root directory viewed from inside the jailed system? Can you use the `which` command to locate the `ls` executable inside the chroot jail, and to which file does this executable correspond in the host system? -### Why Singularity containers + === "Answer" -Docker in particular contains some components that do not fit HPC system architecture well. Docker is composed by a daemon service (dockerd) and a client application (Docker client). This is a client-server architecture, where the client must run in the underlying system with root privileges and shares resources to the clients that run the containers. This posses 2 problems. + To list the contents of the home directory of the chroot jail (`/home/jailuser`), and display the contents of `hello.txt` which is located in the home directory, change to the chroot jail, and use the `ls` and `cat` commands whose executables were copied into the jail during step 2. -- Deployment: A daemon service with root access must run in each node of the cluster where a Docker container can run. -- Security: The daemon service must control which privileges are available to each container and user on top of the typical access privileges of each user. Otherwise, users can access unsanctioned resource through their containers since the daemon service has root privileges. + ```console + $ sudo chroot ${HOME}/jail /bin/bash + bash-5.1# ls /home/myusername/ + hello.txt + bash-5.1# cat /home/myusername/hello.txt + Welcome to chroot jail! + ``` -Singularity containers solve the problems of Docker type containers by using a simple service architecture. Each Singularity container runs in a simple Apptainer of Singularity process without the need for any services. Furthermore, the process running the container is launched by the user, so the user has inside the container the same privileges as in the underlying system and privilege escalation is not possible. Docker has introduced a daemonless and rootless mode in 2020, but Singularity type systems are much more mature in this aspect and offer other features that are more attractive for HPC systems. + The contents of the chroot jail root are displayed using the `ls` command. -### Container image formats + ```console + bash-5.1# ls -lahF / + total 0 + drwxr-xr-x 1 0 0 30 Jun 14 20:07 ./ + drwxr-xr-x 1 0 0 30 Jun 14 20:07 ../ + drwxr-xr-x 1 0 0 66 Jun 14 20:11 bin/ + drwxr-xr-x 1 0 0 16 Jun 14 20:07 home/ + drwxr-xr-x 1 0 0 126 Jun 14 20:07 lib/ + drwxr-xr-x 1 0 0 40 Jun 14 20:07 lib64/ + ``` -The containers are effectively a directory structure into which the system `chroot` together with a set of configuration files. These files can be stored in a number of formats. The SIF format is the format used natively but Apptainer and Singularity. The native formats for Docker are Docker tarballs and [OCI tarballs](https://github.com/opencontainers/image-spec/blob/main/image-layout.md) of the [Open Container Initiative](https://opencontainers.org/). + The path for `ls` reported from inside the chroot jail is the following. -The main difference between SIF and tarball based formats is that tarball formats are a stack of layers. For instance, when building a OCI tarball one layer may include the Debian base image with `libc`, a second level may install Python, and a third level may install a python `venv`. The stack of layers is stored in a single tar ball and assembled dynamically in runtime. + ```console + bash-5.1# which ls + /bin/ls + ``` -A SIF container in a single file that is simply extracted and read at runtime, and more importantly SIF containers are static. For instance, an OCI tarball may require root privileges in a layer that installs Python system-wide at runtime, and since OCI tarballs are dynamic they require root privileges every time the container is run. The static nature of SFI container does away with the need for root privileges; Python is installed once when the container is created, and then the container is loaded with Python installed. + This corresponds to the following path in the host system. -## Executing HPC containers + ```text + ${HOME}/jail/bin/ls + ``` -To run containers in HPC systems, you need access to Apptainer or Singularity executables and your container files. Containers for various applications can be found in [container registries](#container-registries), collections of repositories with container images. +The chroot jail we created follows the filesystem hierarchy standard inside the `${HOME}/jail` path, and this allows programs to operate as if they were operating in a system with `${HOME}/jail` as root. -Singularity based systems use SIF as their native format, but they can also load other container types such as Docker and OCI tarballs. On top of loading containers of multiple formats, SIF based container systems can also convert and store these containers in SIF format. +The `chroot` executable is still used to create containers. However, chroot jails require root permission which is not available in share systems. Modern containers use namespaces to control the access to resources, and with the introduction of user namespaces no special privileges are required for most operations in containers. -Singularity container applications can also work with sandboxes. Sandboxes are expansions of the binary SIF image file into a regular directory of the file system. The user may access and modify the sandbox form the host system or can launch the sandbox as a container and modify it from withing. Sandboxes is the primary mean of creating and modifying SIF containers. The operation of building a SIF container from a sandbox is monolithic and creates a single layer in contrast to tarball containers like OCI that may be built in layers. +### Accessing host system resources -## Further resources +In a typical system you may want to access resources other than simple executables, like network and storage devices. Linux operating systems expose such resources in the root path using pseudo filesystems like _sysfs_ and _procfs_. In many Linux operating systems, -### Popular container registries +- `/proc` contains the process information pseudo-filesystem, +- `/sys` is a mount point for the sysfs filesystem that provides information about the kernel like `/proc`, but better structured, +- `/dev` contains special or device files that refer to physical devices, and +- `/run` is a directory that contains information about processes that describes the system state. -- [Docker Hub](https://hub.docker.com/) -- [NVIDIA NGC Catalog](https://catalog.ngc.nvidia.com/containers) -- [AMD Infinity Hub](https://www.amd.com/en/developer/resources/infinity-hub.html) -- [Sylabs Cloud Library](https://cloud.sylabs.io/library) -- [RED HAT Quay.io](https://quay.io/) -- [BioContainers](https://biocontainers.pro/) +You can use `man hier` to see more of the resource you may need to access inside an isolated environment. You can mount any resource files using bindings: -### Extra reading +```bash +for fs in dev dev/pts proc sys sys/firmware/efi/efivars run; do + sudo mount --bind "/${fs}" "${system_root}/${fs}" +done; unset fs +``` -- [Apptainer and MPI applications](https://apptainer.org/docs/user/latest/mpi.html) -- [GPU Support (NVIDIA CUDA & AMD ROCm)](https://apptainer.org/docs/user/latest/gpu.html) -- [Portable MPI containerization with the Process Management Interface (PMI)](https://ciq.com/blog/a-new-approach-to-mpi-in-apptainer/) -- [Docker vs Apptainer](https://www.linkedin.com/pulse/docker-vs-apptainer-anup-khanal-vxvxf/) -- [Container image formats under the hood](https://snyk.io/blog/container-image-formats/) +However, this process involves a lot of manual work, and exposes system components to the jailed system. Containers automate this process and use systems such as namespace and cgroup to expose resources in a more secure manner. -### References +## Differences between containers and virtual environments -1. Kurtzer, Gregory M., Vanessa, Sochat, and Michael W., Bauer. "Singularity: Scientific containers for mobility of compute"._PLOS ONE_ 12, no.5 (2017): e0177459, doi: [10.1371/journal.pone.0177459](https://www.doi.org/10.1371/journal.pone.0177459). +Containers and virtual environments both modify the environment where applications run. However, they generally differ in the manner the modification is made and thus the aspects of the environment they can modify. Virtual environment often rely on features of the operating system, such as the `PATH` variable used by executables and the `LD_LIBRARY_PATH` used by the operating system dynamic linker. If your application needs a specific version of `glibc` for instance, then it is much safer to use a container that overrides the default linker (`/lib64/ld-linux-x86-64.so.2`) and `glibc` (`/lib64/libc.so.6`). diff --git a/docs/courses/containerization/containers_in_hpc_systems.md b/docs/courses/containerization/containers_in_hpc_systems.md new file mode 100644 index 00000000..d0620313 --- /dev/null +++ b/docs/courses/containerization/containers_in_hpc_systems.md @@ -0,0 +1,446 @@ +# Containers in HPC systems + +There are multiple container standards and even more container engines and storage formats that support them. Singularity type containers [[1](#ref_Kurtzer_2017a)] are popular in HPC systems due to the simplicity with which they integrate to HPC environments [[2](#ref_Mosciatti_2020a)]. The main container engines supporting Singularity type containers are are [Apptainer](https://apptainer.org/docs/user/latest/) and [Singularity](https://docs.sylabs.io/guides/latest/user-guide/). Both these container engines are built around the Singularity Image File (SIF) format of containers, and are interchangeable. + +??? info "Relation between Apptainer and Singularity" + + Singularity began as an open-source project in 2015, when a team of researchers at Lawrence Berkeley National Laboratory. The original project joined the [Linux Foundation](https://www.linuxfoundation.org/) and was renamed to Apptainer, where as the original Singularity code has been forked by the company Sylabs and distributed under various licenses and names. + +??? info "Why Singularity type containers are favored in HPC systems" + + The initial versions of [Docker](https://docs.docker.com/) needed significant adjustments to work well in HPC systems due to some design choices. Docker is a client-server architecture composed of a daemon service (dockerd) and client applications (Docker clients). The client must run in the host system with root privileges and shares resources to the clients that run the containers. + + While this designs works well in cloud architectures where the containers often run on VMs, provide services that are designed to run for an indefinite amount of time, it also poses difficulties in HPC systems for 2 main reasons. + + - Deployment: for every Docker container, the container process is spawned as a child of a root owned Docker daemon; while Docker has introduced a rootless mode in 2020, a Docker daemon is still necessary to spawn containers. + + - Security: As the user is able to directly interact with and control the Docker daemon, it is theoretically possible to coerce the daemon process into granting the users escalated privileges; this is particularly problematic for root owned daemons. + + Singularity avoid these issues by launching containers directly as a user process, in the same user namespace with [minimal isolation](https://apptainer.org/docs/user/main/docker_and_oci.html#namespace-device-isolation). + + - There is no daemon process controlling the container, so there no need to start an extra daemon process. Even daemons requiring no root privileges add extra complexity, as they add extra layers of resource management (namespaces for use mappings and cgroups for resource management) on top of the cluster scheduler (Slurm in Meluxina). Singularity containers directly inherit their resource constraints from the systems scheduler and run on the same namespace as the user process that launches the container. + + - There is no need for elevated user rights reducing the risk of privilege escalation, and the user does not interact with processes that have elevated user rights, reducing the attack surface of the system. + + A lot of the drawbacks of the Docker architecture are resolved in [Podman](https://podman.io/docs), a daemonless tool container engine. However, the minimal isolation that Singularity offers by default is better suited for HPC systems. Container engines like Docker and Podman that target cloud application tend to isolate the container environment by default, and apply extra layers of resource management which are not transparent to the system scheduler and must be explicitly deactivated. In contrast, Singularity container engines like Singularity and Apptainer, mounts host directories (like your home, `/tmp`, and device directories) inside the container by default, granting direct access to your local files and GPU hardware without extra configuration. + + A subtle advantage of the low complexity approach favored by Singularity type container engines is that the flat storage format of SIF images is often more efficient in HPC systems. While overlayed image formats where layers can be shared between container instance to save memory are useful in cloud applications where multiple containers are running on a single system, it is rarely useful in HPC where containers are used to package and run an application from a single container. Furthermore, large centers are often using CernMV-FS repositories that load container over a network file system in a lazy manner, only loading the components that are actually used. While storing deduplicated images and loading images in a lazy manner is easy for flat SIF files, extra processing is required to achieve the same for overlayed OCI Images, resulting in minor performance degradation [[2](#ref_Mosciatti_2020a)]. + +## Container image formats + +The containers are effectively a directory structure into which a chroot jail root together with a set of configuration files is written. However storing and shipping directories is cumbersome. There are image formats that allow serializing the container directory structure into a single binary tarball for easy distributions. However, not all image format are monolithic. + +Docker tarballs and [OCI Image](https://github.com/opencontainers/image-spec/blob/main/image-layout.md) of the [Open Container Initiative](https://opencontainers.org/) are using [ovelayFS](https://docs.kernel.org/filesystems/overlayfs.html) to build containers as a stack of layers. For instance, when building a OCI Image one layer may include the Debian base image with `libc`, a second level may install Python, and a third level may install a python `venv`. The layers are assembled dynamically during runtime. This saves space in container registries by storing a single copy of each shared layer, and runtime memory and computation in container engines by loading a single copy of the required shared libraries at runtime. + +The [Singularity Image Format (SIF)](https://github.com/apptainer/sif) is the format used natively by Singularity type containers. Similar to Singularity type container engines, SIF emphasizes simplicity. The is a single overlay layer in SIF files. Singularity type container engines can work with OCI Images and Docker tarballs, but the have to convert them in a SIF compatible format first. There is an OCI compatible format for SIF containers, [OCI-SIF](https://docs.sylabs.io/guides/latest/user-guide/oci_runtime.html#oci-sif-images) that stores the OCI image with its overlays in a SIF file. Launching a container from an OCI-SIF format directly is [supported in Singularity](https://docs.sylabs.io/guides/latest/user-guide/oci_runtime.html#oci-mode-oci), but [not in Apptainer](https://github.com/sylabs/singularity/discussions/2948) yet. Apptainer users must convert their containers to a format compatible with conventional SIF file to run them. + +## Using Apptiner in HPC systems + +To run containers in HPC systems, you need to access the Apptainer or Singularity executables and and a container file. Apptainer is already installed on Meluxina, and is available as a module. To use it, load the Apptainer module. + +```console +module load env/release/2025.1 +module load Apptainer +``` + +There are multiple [container registries](#container-registries), collections container images, where you can find images for many popular applications. We now demonstrate how you can fetch, store, and run an existing container image with Apptainer. + +### Example: Pulling a container into a SIF file + +Registries are data collections that can be accessed with programs designed to fetch data from the registry. Apptainer provides the `pull` function that fetches containers from registries and store them locally into a SIF file. In this example we fetch a container and store it into a SIF file locally. + +1. Create a directory to store your container images. + + ```console + mkdir -p ${HOME}/containers + ``` + +1. Fetch the [`rancher/cowsay`](https://hub.docker.com/r/rancher/cowsay) from Docker hub and store it locally. + + ```console + apptainer pull ${HOME}/containers/cow.sif docker://rancher/cowsay:latest + ``` + +??? question "Managing container images" + + === "Question" + + What operations did the pull command perform? + + === "Answer" + + Apptainer fetched the image for `rancher/cowsay`. + + - The argument `pull` instructs the Apptainer to fetch the container image and store it locally. We could have used the commands `run` or `exec` to run the image without storing it locally. + + - The argument `${HOME}/containers/cow.sif` is the location where the container image is stored. By default, the container is stored as a SIF image. + + - The argument `docker://rancher/cowsay:latest` indicates the source of the container images, in this case the [Docke hub](https://hub.docker.com/) registry. Since the images in Docker hub are not stored in SIF format, Apptainer will convert the image into a SIF file before executing any further steps. + + Artifacts of image conversions are stored in `${HOME}/.apptainer/cache/`. + +### Example: Running containers + +You can now launch the container stored locally. Singularity type container engines provide two launch commands, + +- the `exec` command that takes as argument the executable to execute inside the container, and +- the `run` command that run a default executable defined during the container creation. + +!!! question "Run the `cowsay` command" + + === "Question" + + The cowsay container is designed to run the cowsay command. Run the container with the `run` command and argument `Hellow, world!`, and with the `exec` command and calling `cowsay` explicitly. + + === "Answer" + + Executing the container with the run command produces the following result. + + ```console + $ apptainer run ${HOME}/containers/cow.sif 'Hello, world!' + _______________ + < Hello, world! > + --------------- + \ ^__^ + \ (oo)\_______ + (__)\ )\/\ + ||----w | + || || + ``` + + To run the same command with `exec` we call the container as follows. + + ```console + $ apptainer exec ${HOME}/containers/cow.sif cowsay 'Hello, world!' + _______________ + < Hello, world! > + --------------- + \ ^__^ + \ (oo)\_______ + (__)\ )\/\ + ||----w | + || || + ``` + +### Example: Building a container from a definition file + +In this example we build Singularity type containers with Apptainer from a Singularity definition files. The definition file (`.def`) a list of instruction that a container builder program like Apptainer can use to compose a container binary file. + +The first container we build, is a container for [DIA-NN](https://github.com/vdemichev/DiaNN), an automated software suite for data-independent acquisition (DIA) proteomics data processing. The software is [supported for a specific version of Linux Mint](https://github.com/vdemichev/DiaNN#installation), and thus requires a container to run in Meluxina that uses a RHEL based system. The `.def` contains the following instructions. + +!!! abstract "DIA-NN-2.5.1.def" + + ```singularity + BootStrap: docker + From: linuxmintd/mint21.2-amd64:latest + + %arguments + VERSION=2.5.1 + RELEASE=2.0 + + %post + export DIA_NN='DIA-NN-{{VERSION}}-Academia-Linux.zip' + if [ -f "/tmp/${DIA_NN}" ]; then + rm "/tmp/${DIA_NN}" + fi + wget "https://github.com/vdemichev/DiaNN/releases/download/{{RELEASE}}/${DIA_NN}" --directory-prefix=/tmp + unzip "/tmp/${DIA_NN}" -d /opt + chmod ugo+x /opt/diann-{{VERSION}}/diann-linux + rm "/tmp/${DIA_NN}" + + %environment + export LC_ALL=C + export PATH=/opt/diann-{{VERSION}}:${PATH} + + %runscript + /opt/diann-{{VERSION}}/diann-linux + + %labels + Container DIA-NN + Version {{VERSION}} + Author hpc-team@uni.lu + + %help + A a universal software suite for data-independent acquisition (DIA) proteomics data processing. Conceived at the University of Cambridge, UK, in the laboratory of Kathryn Lilley (Cambridge Centre for Proteomics), DIA-NN opened a new chapter in proteomics, introducing a number of algorithms which enabled reliable, robust and quantitatively accurate large-scale experiments using high-throughput methods. DIA-NN is currently being further developed in the laboratory of Vadim Demichev at the Charité (University Medicine Berlin, Germany). + + Source: https://github.com/vdemichev/DiaNN + ``` + +The definition file, is text file with a description on how to build the _DIA-NN_ container: + +1. bootstrap the container by pulling from the Docker image `linuxmintd/mint21.2-amd64:latest`; +1. download and extract the DIA-NN binaries, and cleanup any artifacts leftover; +1. set some environment variables to access the binaries; +1. define a command to run when executing the container. + +A more complete list of section names for the definition is available in the [Apptainer documentation](https://apptainer.org/docs/user/main/definition_files.html). + +From this definition file, you can build the container with the following command. + +```bash +apptainer build ${HOME}/containers/DIA-NN.sif DIA-NN.def +``` + +This command takes the definition file `DIA-NN.def` to create a new container image in the file `DIA-NN.sif`. + +!!! question "Building and running containers from definition files" + + === "Question" + + Build the container from the definition file. Then, fetch the example input file [`Aug2022_Mouse Tissue Contaminants.fasta`](https://github.com/HaoGroup-ProtContLib/Protein-Contaminant-Libraries-for-DDA-and-DIA-Proteomics/raw/refs/heads/main/Sample-type%20specific%20contaminant%20FASTA/Aug2022_Mouse%20Tissue%20Contaminants.fasta), and run the following command in the container. + + ```bash + diann-linux + --lib "" \ + --out "/tmp/report.parquet" \ + --out-lib "/tmp/report-lib.parquet" \ + --fasta "Aug2022_Mouse Tissue Contaminants.fasta" \ + --threads 32 \ + --verbose 1 \ + --qvalue 0.01 \ + --matrices \ + --gen-spec-lib \ + --predictor \ + --reannotate \ + --fasta-search \ + --min-fr-mz 200 \ + --max-fr-mz 1800 \ + --met-excision \ + --min-pep-len 7 \ + --max-pep-len 30 \ + --min-pr-mz 300 \ + --max-pr-mz 1800 \ + --min-pr-charge 1 \ + --max-pr-charge 4 \ + --cut K*,R* \ + --missed-cleavages 1 \ + --unimod4 \ + --rt-profiling + ``` + + === "Answer" + + From this definition file, you can build the container with the following command. + + ```bash + apptainer build ${HOME}/containers/DIA-NN.sif DIA-NN.def + ``` + + Then, execute the container with the following command. + + ```bash + apptainer exec ${HOME}/containers/DIA-NN.sif diann-linux + --lib "" \ + --out "/tmp/report.parquet" \ + --out-lib "/tmp/report-lib.parquet" \ + --fasta "Aug2022_Mouse Tissue Contaminants.fasta" \ + --threads 32 \ + --verbose 1 \ + --qvalue 0.01 \ + --matrices \ + --gen-spec-lib \ + --predictor \ + --reannotate \ + --fasta-search \ + --min-fr-mz 200 \ + --max-fr-mz 1800 \ + --met-excision \ + --min-pep-len 7 \ + --max-pep-len 30 \ + --min-pr-mz 300 \ + --max-pr-mz 1800 \ + --min-pr-charge 1 \ + --max-pr-charge 4 \ + --cut K*,R* \ + --missed-cleavages 1 \ + --unimod4 \ + --rt-profiling + ``` + +!!! question "Provide an interface for using the DIA-NN container" + + === "Question" + + The current version of `DIA-NN.def` creates a container where the `run` argument does not accept any arguments and thus it is not very useful. Modify the container definition file so that it access arguments for the DIA-NN executable. + + === "Answer" + + We modify the `runscript` section to pass the arguments to the `run` command through to the call to `diann-linux` executable. The resulting definition file is the following. + + ```singularity + BootStrap: docker + From: linuxmintd/mint21.2-amd64:latest + + %arguments + VERSION=2.5.1 + RELEASE=2.0 + + %post + export DIA_NN='DIA-NN-{{VERSION}}-Academia-Linux.zip' + if [ -f "/tmp/${DIA_NN}" ]; then + rm "/tmp/${DIA_NN}" + fi + wget "https://github.com/vdemichev/DiaNN/releases/download/{{RELEASE}}/${DIA_NN}" --directory-prefix=/tmp + unzip "/tmp/${DIA_NN}" -d /opt + chmod ugo+x /opt/diann-{{VERSION}}/diann-linux + rm "/tmp/${DIA_NN}" + + %environment + export LC_ALL=C + export PATH=/opt/diann-{{VERSION}}:${PATH} + + %runscript + /opt/diann-{{VERSION}}/diann-linux "$@" + + %labels + Container DIA-NN + Version {{VERSION}} + Author hpc-team@uni.lu + + %help + A a universal software suite for data-independent acquisition (DIA) proteomics data processing. Conceived at the University of Cambridge, UK, in the laboratory of Kathryn Lilley (Cambridge Centre for Proteomics), DIA-NN opened a new chapter in proteomics, introducing a number of algorithms which enabled reliable, robust and quantitatively accurate large-scale experiments using high-throughput methods. DIA-NN is currently being further developed in the laboratory of Vadim Demichev at the Charité (University Medicine Berlin, Germany). + + Source: https://github.com/vdemichev/DiaNN + ``` + + Observe the extra "$@" argument to the call to the `diann-linux` executable in the `runscript` section. + +### Example: Containers that require elevated right to be built + +The second example is based on a definition file from the [documentation of Apptainer](https://apptainer.org/docs/user/latest/build_a_container.html#building-containers-from-apptainer-definition-files). The example creates a _lolcow_ container which is a combination of the [cowsay](https://en.wikipedia.org/wiki/Cowsay), [lolcat](https://github.com/busyloop/lolcat), and [fortune](https://en.wikipedia.org/wiki/Fortune_(Unix)) tools. The tools are installed from the repositories of Ubuntu using `apt-get`, and thus require elevated privileges to install in the container, either root access or Apptainer setup to run with `fakeroot`. Thus, you want be able to build this container in Meluxina, and you need to build it in a machine where you have the appropriate rights. + +The definition file `lolcow.def` for this container is the following. + +!!! abstract "lolcow.def" + + ```singularity + Bootstrap: docker + From: ubuntu:20.04 + + %post + apt-get -y update + apt-get -y install cowsay lolcat fortunes + + %environment + export LC_ALL=C + export PATH=/usr/games:$PATH + + %runscript + if [ "$#" -gt 0 ]; then + cowsay "$@" | lolcat + else + fortune | cowsay | lolcat + fi + ``` + +The definition file describes the procedure to build the _lolcow_ container: + +1. bootstrap the container by pulling from the Docker image `ubuntu:20.04`; +1. install the package `cowsay` and `lolcat`; +1. set some environment variables; +1. define a command to run when executing the container. + +From this definition file, you can build the container with this command: + +```bash +apptainer build lolcow.sif lolcow.def +``` + +This command takes the definition file `lolcow.def` to create a new container image in the file `lolcow.sif`. + +??? info "The `runscript` section" + + The optional `runscript` section is used to define a script that is called when apptainer is called with the `run` argument. The contents of the `runscript` section are copied in a bash script `/singularity` in the container file system, and the `/singularity` script is executed when the `run` argument is used. + + In the `lolcow.sif` container for instance, the contents of the `/singularity` are the following. + + !!! abstract "/singularity" + ```bash + #!/bin/bash + + if [ "$#" -gt 0 ]; then + cowsay "$@" | lolcat + else + fortune | cowsay | lolcat + fi + ``` + + Furthermore, a call to the `run` command + + ```console + apptainer run lolcow.sif + ``` + + is equivalent to the more verbose + + ```console + apptainer exec lolcow.sif /singularity + ``` + + call to the `exec` command. Thus the `runscript` section provides the means of defining a interface for using the container. + +### Example: Unpacking containers into sandboxes + +Sandboxes are expansions of image files into a regular directory on the file system. The user may access and modify the sandbox form the host system or they can launch the sandbox as a container and modify it from withing. Sandboxes can be packages into SIF images with the container application. + +!!! question "Modifying a container in a sandbox" + + === "Question" + + The [`rancher/cowsay`](https://hub.docker.com/r/rancher/cowsay) container hangs when no argument is given to the `run` command. Modify the `/singularity` script of the container to print the date (output of the `date` command) when no argument is given. + + === "Answer" + + Pull and unpack the container in the working directory. + + ```console + apptainer build --sandbox cowsay/ docker://rancher/cowsay:latest + ``` + + The modify the container in the host filesystem. + + ```console + cat > cowsay/singularity <Kurtzer, Gregory M., Vanessa, Sochat, and Michael W., Bauer. "Singularity: Scientific containers for mobility of compute". _PLOS ONE_ 12, no.5 (2017): e0177459, doi: [10.1371/journal.pone.0177459](https://www.doi.org/10.1371/journal.pone.0177459). + +1. Mosciatti, S. and Blomer, J. and Ganis, G. and Popescu, R. "CernVM-FS Container Image Integration". _Journal of Physics: Conference Series_ 1525, no. 1 (2020): 012058, doi: [10.1088/1742-6596/1525/1/012058](https://www.doi.org/10.1088/1742-6596/1525/1/012058). + +### Popular container registries { #container-registries } + +- [Docker Hub](https://hub.docker.com/) +- [NVIDIA NGC Catalog](https://catalog.ngc.nvidia.com/containers) +- [AMD Infinity Hub](https://www.amd.com/en/developer/resources/infinity-hub.html) +- [Sylabs Cloud Library](https://cloud.sylabs.io/library) +- [RED HAT Quay.io](https://quay.io/) +- [BioContainers](https://biocontainers.pro/) diff --git a/docs/courses/containerization/images/container.png b/docs/courses/containerization/images/container.png deleted file mode 100644 index acc38d08..00000000 --- a/docs/courses/containerization/images/container.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ab92a49141be74979e2d40ec5552f0d69f4636f6306ad5700ebe872897e4a54f -size 4298 diff --git a/docs/courses/containerization/index.md b/docs/courses/containerization/index.md index 923cefa1..bf296b3b 100644 --- a/docs/courses/containerization/index.md +++ b/docs/courses/containerization/index.md @@ -1,5 +1,8 @@ # Environment management and containers for HPC systems +> Author: [Gergios Kafanas](https://www.uni.lu/fstm-en/people/georgios-kafanas/)
+> email: [georgios.kafanas@uni.lu](mailto:georgios.kafanas@uni.lu) + Environment management and containerization in shared computing resources like HPC systems and workstations allow the installation and execution of programs in isolation of other system and user applications. Environment management and containerization encompass a collection of techniques base on features of the operating system and the kernel which are used for a diverse set of tasks such as - sourcing and installing software diff --git a/docs/courses/containerization/introduction.md b/docs/courses/containerization/introduction.md index 35b92312..105c7872 100644 --- a/docs/courses/containerization/introduction.md +++ b/docs/courses/containerization/introduction.md @@ -1,21 +1,72 @@ # The Linux environment -Environment management in shared computing resources like HPC systems and workstations involves a collection of techniques that allow user processes to run in isolation from other user and system processes. The isolation allows +Environment management in shared computing resources, like HPC systems and workstations, involves a collection of techniques that allow user processes to run in isolation from other user and system processes. This isolation allows -- multiple processes to share the same system without interfering with each other and with system processes with respect to resource usage, and -- each processes can configure its own software environment without interfering with the other user space and operating system processes. +- non-kernel processes to run on environments isolated from other processes, and +- multiple processes to share system resources without interfering with each other. -The isolation methods can be categorized in methods using the +This isolation is particularly useful in HPC systems where multiple users install and use software simultaneously. + +## The Linux process interface + +Processes in a GNU/Linux system are categorized into + +- user processes, +- operating system processes, and +- the system kernel. + +A user process interacts with other process, through function calls ([fig. 1](#fig-1)) and interprocess communication methods, such as streams and message queues.
- ![Container architecture](resources/linux_interface.png){ align=left width="400" } -
Fig. 1: The linux application interface
+ ![Linux process types and interactions](resources/linux_structure.png){ align=left width="400" } + +
Fig. 1: Linux process types and interactions.
-## Terms +The Linux operating system provides mechanisms to modify the view that a user process has of the operating system and the processes that are running on it. Containers and related methods utilize these mechanisms to modify the view that a process has of the system. + +## Containerization technologies + +The term _container_ describes a wide and loose set of technologies that enable software applications to run in isolated user spaces. The 3 main technologies used in containers are the following. + +- [_Chroot_](https://en.wikipedia.org/wiki/Chroot): a system operation that changes the apparent root directory for the current running process and its children. +- [_Namespaces_](https://en.wikipedia.org/wiki/Linux_namespaces): a Linux kernel feature that limits the resources that a set of processes can access; examples of resource include process IDs, user IDs, and interprocess communication mechanisms and groups. +- [_Control group_ (`cgroup`)](https://en.wikipedia.org/wiki/Cgroups): a Linux kernel feature that limits, accounts for, and isolates the resource usage for collections of processes; resource include CPU cores, memory, and disk IOPs. + +
+ ![Overview of Linux containers](images/linux-containers.png){ align=left width="400" } + +
Fig. 2: Overview of technologies used in Linux containers.
+
+ +The complete set of technologies used in containers and virtual environments involves a few more features of the operating system, like + +- `PATH`, an environment variable used to set the location where applications look for executables, +- `LD_LIBRARY_PATH`, an environment variable used by the linker to determine the locations where to look for shared libraries, and +- tools such as `fakeroot` and `chroot` used to build isolated environments. + +In the following sessions we will look into the details of these techniques, and how they are used in practice. An overview the various methods and their relations is presented in ([fig. 3](#fig-3)). + +
+ ![Containerization techniques](resources/linux_interface.png){ align=left width="800" } + +
Fig. 3: Linux application interface and containerization techniques. Containerization methods control the interface available between the users process, the operating system, and the operating system kernel.
+
+ +## Further resources + +### Extra reading + +- [Apptainer and MPI applications](https://apptainer.org/docs/user/latest/mpi.html) +- [GPU Support (NVIDIA CUDA & AMD ROCm)](https://apptainer.org/docs/user/latest/gpu.html) +- [Portable MPI containerization with the Process Management Interface (PMI)](https://ciq.com/blog/a-new-approach-to-mpi-in-apptainer/) +- [Docker vs Apptainer](https://www.linkedin.com/pulse/docker-vs-apptainer-anup-khanal-vxvxf/) +- [Container image formats under the hood](https://snyk.io/blog/container-image-formats/) + +### Terms -_Environments_ modify the user space to provide access to different sets of executables and libraries without modifying globally installed tools. Environment management tools such as Environment Modules, Conda, and application specific tools such Python Virtual Environments (`venv`) rely on environment variables such as `PATH` and `LD_LIBRARY_PATH`. +- _Environments_ modify the user space to provide access to different sets of executables and libraries without modifying globally installed tools. Environment management tools such as Environment Modules, Conda, and application specific tools such Python Virtual Environments (`venv`) rely on environment variables such as `PATH` and `LD_LIBRARY_PATH`. -_Containers_ on top of user space modifications can also modify aspects of the operating system. Singularity containers for instance, can use a custom GNU C Library (`glibc`), a wrapper around the system calls of the Linux kernel. Containers rely on a variety of tools to control the environment, access to files, and access to resources such as cores and memory. +- _Containers_ on top of user space modifications can also modify aspects of the operating system. Singularity containers for instance, can use a custom GNU C Library (`glibc`), a wrapper around the system calls of the Linux kernel. Containers rely on a variety of tools to control the environment, access to files, and access to resources such as cores and memory. -Anything that requires customization of the kernel and drivers must use _virtualisation_, which is beyond the scope of most HPC applications. +- Anything that requires customization of the kernel and drivers must use _virtualisation_, which is beyond the scope of most HPC applications. diff --git a/docs/courses/containerization/resources/Makefile b/docs/courses/containerization/resources/Makefile index 7b50f624..ea82066b 100644 --- a/docs/courses/containerization/resources/Makefile +++ b/docs/courses/containerization/resources/Makefile @@ -16,6 +16,7 @@ $(patsubst %, %.png, $(TARGETS)): %.png: sources/build/output/%.pdf sources/build/output/%.pdf: sources/%.tex $(MAKE) --directory=sources $(patsubst sources/build/output/%.pdf, %, $@) touch $@ + sources/build/output/%.tex: ; diff --git a/docs/courses/containerization/resources/container_VM_comparison.png b/docs/courses/containerization/resources/container_VM_comparison.png new file mode 100644 index 00000000..c4c70a25 --- /dev/null +++ b/docs/courses/containerization/resources/container_VM_comparison.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bc2fcd53cf998a45f49f2d01d657da26e103ea9118d9d7fef7d06b441eef2de +size 20925 diff --git a/docs/courses/containerization/resources/container_architecture.png b/docs/courses/containerization/resources/container_architecture.png new file mode 100644 index 00000000..3aad2e82 --- /dev/null +++ b/docs/courses/containerization/resources/container_architecture.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91700366d3de81735f1d9169ef95d48e3a1413c10d90532372b92573a9332a5f +size 14345 diff --git a/docs/courses/containerization/resources/linux_interface.png b/docs/courses/containerization/resources/linux_interface.png index f8259e27..c4c24623 100644 --- a/docs/courses/containerization/resources/linux_interface.png +++ b/docs/courses/containerization/resources/linux_interface.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4ac803812ee1a5fddec3f171dd81e0edb680f1ca83b47f0ead3441f5674cdb62 -size 101968 +oid sha256:cec6c983c07f7b9db606c4b999bdf3f5cb879f0fb6657625899bf094db1283a9 +size 101636 diff --git a/docs/courses/containerization/resources/sources/container_VM_comparison.tex b/docs/courses/containerization/resources/sources/container_VM_comparison.tex new file mode 100644 index 00000000..f6f47acb --- /dev/null +++ b/docs/courses/containerization/resources/sources/container_VM_comparison.tex @@ -0,0 +1,118 @@ +\documentclass[border=12pt]{standalone} + \usepackage{tikz} + \usepackage{xcolor} + \usepackage[T1]{fontenc} + + \usetikzlibrary{ + shapes.symbols, + positioning, + calc + } + +\definecolor{hardwarecolor}{RGB}{64,64,64} +\definecolor{hvcolor}{RGB}{185,185,185} +\definecolor{appcolor}{RGB}{67,114,196} % blue +\definecolor{oscolor}{RGB}{82,153,53} % green +\definecolor{kernelcolor}{RGB}{192,39,39} % red + +\NewDocumentCommand\component{mmmmmm}{ + \path[draw,fill=#4] + let + \p{#1_crd_bottomleft} = ($#2+(0, 0)$), + \p{#1_crd_topright} = ($#2+(#3*2.2-0.2, 1)$) + in + (\x{#1_crd_bottomleft}, \y{#1_crd_bottomleft}) coordinate (#1_bottomleft) -- + (\x{#1_crd_topright},\y{#1_crd_bottomleft}) coordinate (#1_bottomright) -- + (\x{#1_crd_topright},\y{#1_crd_topright}) coordinate (#1_topright) -- + (\x{#1_crd_bottomleft},\y{#1_crd_topright}) coordinate (#1_topleft) -- + cycle + ; + \path[text=#5] + ($(#1_bottomright)!0.5!(#1_topleft)$) node[label={#6}] (#1_label) {} + ; +} + +\tikzset{ + vm/.pic={ + \component{vm_machine}{(0,0)}{3}{hardwarecolor}{white}{ + [align=center]center:Infrastructure + } + + \component{vm_hypervisor}{(vm_machine_topleft)+(0,0.05)}{3}{hvcolor}{black}{ + [align=center]center:Hypervisor + } + + \component{vm_kernel_0}{(vm_hypervisor_topleft)+(0,0.05)}{2}{kernelcolor}{black}{ + [align=center]center:OS 0 kernel + } + \component{vm_kernel_1}{(vm_kernel_0_bottomright)+(0.2,0)}{1}{kernelcolor}{black}{ + [align=center]center:OS 1 kernel + } + + \component{vm_os_0}{(vm_kernel_0_topleft)+(0,0.05)}{2}{oscolor}{black}{ + [align=center]center:OS 0\\{\ttfamily\small (\textbackslash bin:\textbackslash lib)} + } + \component{vm_os_1}{(vm_os_0_bottomright)+(0.2,0)}{1}{oscolor}{black}{ + [align=center]center:OS 1\\{\ttfamily\small (\textbackslash bin:\textbackslash lib)} + } + + \component{vm_app_0}{(vm_os_0_topleft)+(0,0.05)}{1}{appcolor}{black}{ + [align=center]center:App 0 + } + \component{vm_app_1}{(vm_app_0_bottomright)+(0.2,0)}{1}{appcolor}{black}{ + [align=center]center:App 1 + } + \component{vm_app_2}{(vm_app_1_bottomright)+(0.2,0)}{1}{appcolor}{black}{ + [align=center]center:App 2 + } + + \coordinate (-bottomleft) at (vm_machine_bottomleft); + \coordinate (-bottomright) at (vm_machine_bottomright); + \coordinate (-topleft) at (vm_app_0_topleft); + \coordinate (-topright) at (vm_app_2_topright); + } +} + +\tikzset{ + container/.pic={ + \component{cnt_machine}{(0,0)}{3}{hardwarecolor}{white}{ + [align=center]center:Infrastructure + } + + \component{cnt_kernel}{(cnt_machine_topleft)+(0,0.05)}{3}{kernelcolor}{black}{ + [align=center]center:Host OS kernel + } + + \component{cnt_os_0}{(cnt_kernel_topleft)+(0,0.05)}{2}{oscolor}{black}{ + [align=center]center:OS 0\\{\ttfamily\small (\textbackslash bin:\textbackslash lib)} + } + \component{cnt_os_1}{(cnt_os_0_bottomright)+(0.2,0)}{1}{oscolor}{black}{ + [align=center]center:OS 1\\{\ttfamily\small (\textbackslash bin:\textbackslash lib)} + } + + \component{cnt_app_0}{(cnt_os_0_topleft)+(0,0.05)}{1}{appcolor}{black}{ + [align=center]center:App 0 + } + \component{cnt_app_1}{(cnt_app_0_bottomright)+(0.2,0)}{1}{appcolor}{black}{ + [align=center]center:App 1 + } + \component{cnt_app_2}{(cnt_app_1_bottomright)+(0.2,0)}{1}{appcolor}{black}{ + [align=center]center:App 2 + } + + \coordinate (-bottomleft) at (cnt_machine_bottomleft); + \coordinate (-bottomright) at (cnt_machine_bottomright); + \coordinate (-topleft) at (cnt_app_0_topleft); + \coordinate (-topright) at (cnt_app_2_topright); + } +} + +\begin{document} +\begin{tikzpicture}[x=1cm, y=1cm] + +\path[draw] (0,0) coordinate pic (vm) {vm}; +\path[draw] ($(vm-bottomright)+(1,0)$) pic {container}; + +\end{tikzpicture} +\end{document} + diff --git a/docs/courses/containerization/resources/sources/container_architecture.tex b/docs/courses/containerization/resources/sources/container_architecture.tex new file mode 100644 index 00000000..71273579 --- /dev/null +++ b/docs/courses/containerization/resources/sources/container_architecture.tex @@ -0,0 +1,125 @@ +\documentclass[border=12pt]{standalone} + \usepackage{tikz} + \usepackage{xparse} + \usepackage{xcolor} + \usepackage[T1]{fontenc} + + \usetikzlibrary{ + shapes.symbols, + decorations.pathreplacing, + positioning, + calc + } + +\definecolor{hardwarecolor}{RGB}{64,64,64} +\definecolor{hvcolor}{RGB}{185,185,185} +\definecolor{appcolor}{RGB}{67,114,196} % blue +\definecolor{oscolor}{RGB}{82,153,53} % green +\definecolor{kernelcolor}{RGB}{192,39,39} % red + +\tikzset{ + pics/container_app/.style args={#1,#2}{ + code={ + \path[draw,fill=appcolor] + (0,0) coordinate (topleft_app_#1_#2) + -- ++(2,0) coordinate (topright_app_#1_#2) + -- ++(0,-1) coordinate (bottomright_app_#1_#2) + -- ++(-2,0) coordinate (bottomleft_app_#1_#2) + -- cycle + ; + \path[] + ($(topleft_app_#1_#2)!0.5!(bottomright_app_#1_#2)$) node[] (label_app_#1_#2) {Apps #1#2} + ; + } + } +} + +\NewDocumentCommand\component{mmmmmm}{ + \path[draw,fill=#4] + let + \p{#1_crd_topleft} = ($(#2)+(0,-0.05)$), + \p{#1_crd_bottomright} = ($(#3)+(0,-1.05)$) + in + (\x{#1_crd_topleft}, \y{#1_crd_topleft}) coordinate (#1_topleft) -- + (\x{#1_crd_bottomright},\y{#1_crd_topleft}) coordinate (#1_topright) -- + (\x{#1_crd_bottomright},\y{#1_crd_bottomright}) coordinate (#1_bottomright) -- + (\x{#1_crd_topleft},\y{#1_crd_bottomright}) coordinate (#1_bottomleft) -- + cycle + ; + \path[text=#5] + ($(#1_topleft)!0.5!(#1_bottomright)$) node[label={#6}] (#1_label) {} + ; +} + +\tikzset{ + pics/container/.style args={#1,#2,#3}{ + code={ + \component{container_#1}{#2}{#3}{oscolor}{black}{ + [align=center]center:OS #1\\{\ttfamily\small (\textbackslash bin:\textbackslash lib)} + } + } + } +} + +\tikzset{ + pics/hostos/.style args={#1,#2}{ + code={ + \component{hostos}{#1}{#2}{oscolor}{black}{ + [align=center]center:Host OS \\{\ttfamily\small (\textbackslash bin:\textbackslash lib)} + } + } + } +} + +\tikzset{ + pics/kernel/.style args={#1,#2}{ + code={ + \component{kernel}{#1}{#2}{kernelcolor}{black}{ + [align=center]center:Host OS kernel + } + } + } +} + +\tikzset{ + pics/machine/.style args={#1,#2}{ + code={ + \component{machine}{#1}{#2}{hardwarecolor}{white}{ + [align=center]center:Infrastructure + } + } + } +} + +\begin{document} +\begin{tikzpicture}[x=1cm, y=1cm] + +% Container A +\path[draw] (0,0) pic {container_app={A,0}}; +\path[draw] ($(topright_app_A_0)+(0.2,0)$) pic {container_app={A,1}}; +% Container B +\path[draw] ($(topright_app_A_1)+(0.2,0)$) pic {container_app={B,0}}; +%\path[draw] ($(topright_app_B_0)+(0.2,0)$) pic {container_app={B,1}}; +%\path[draw] ($(topright_app_B_1)+(0.2,0)$) pic {container_app={B,2}}; + +\path[draw] (0,0) pic {container={A,bottomleft_app_A_0,bottomright_app_A_1}}; +\path[draw] (0,0) pic {container={B,bottomleft_app_B_0,bottomright_app_B_0}}; + +%\path[draw] (0,0) pic {hostos={container_A_bottomleft,container_B_bottomright}}; +% +%\path[draw] (0,0) pic {kernel={hostos_bottomleft,hostos_bottomright}}; + +\path[draw] (0,0) pic {kernel={container_A_bottomleft,container_B_bottomright}}; + +\path[draw] (0,0) pic {machine={kernel_bottomleft,kernel_bottomright}}; + +\path[draw=gray,decorate,decoration={brace,mirror},line width=2pt] + ($(topleft_app_A_0)+(-0.2,0)$) coordinate (note_top) + -- ($(container_A_bottomleft)+(-0.2,0)$) coordinate (note_bottom) +; +\coordinate (container_midpoint) at ($(note_top)!0.5!(note_bottom)$); +\node[anchor=east,left=0.4 of container_midpoint,draw=black,fill=white] (container_annotation) {\Large Containers}; + +\end{tikzpicture} +\end{document} + diff --git a/docs/courses/containerization/resources/sources/linux_interface.tex b/docs/courses/containerization/resources/sources/linux_interface.tex index 78e74d12..f0af382e 100644 --- a/docs/courses/containerization/resources/sources/linux_interface.tex +++ b/docs/courses/containerization/resources/sources/linux_interface.tex @@ -176,7 +176,7 @@ text align={center, right indent=0em}, raise=0pt, text format delimiters={[}{]}, - text={Subsystems: [\ttfamily]namespaces,cgroups} + text={Subsystems: [\ttfamily]cgroup, namespace} } }] let @@ -191,7 +191,7 @@ text align={center, right indent=0em}, raise=0pt, text format delimiters={[}{]}, - text={Tools: [\ttfamily]chroot,fakeroot} + text={Tools: [\ttfamily]chroot, fakeroot} } }] let diff --git a/docs/courses/containerization/virtual_environments.md b/docs/courses/containerization/virtual_environments.md new file mode 100644 index 00000000..72fe3ba7 --- /dev/null +++ b/docs/courses/containerization/virtual_environments.md @@ -0,0 +1,139 @@ +# Virtual environments + +Environments typically rely on features of the operating system to modify the environment where processes run. + +- The [`PATH` variable](https://pubs.opengroup.org/onlinepubs/9799919799/) is defined in the [POSIX standard](https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/V1_chap01.html) of operating system interface and environment, as the sequence of path prefixes that functions and utilities apply in searching for an executable file. + +- Similarly, the `LD_LIBRARY_PATH` is a list of paths where the dynamic linker (see [`ld.so`](https://man7.org/linux/man-pages/man8/ld.so.8.html)) searches for shared libraries that are linked dynamically to executables at runtime. + +Environment systems typically install a partial filesystem hierarchy in some path, and prepend the `\bin` and `\lib` subpataths of the hierarchy to the `PATH` and `LD_LIBRARY_PATH` lists respectively. Thus, the programs and libraries available of the environment are added to the ones available in the `\bin` and `\lib` of the root directory. + +## Filesystem hierarchy for isolated environments + +The Linux operating systems rely on the [Filesystem Hierarchy Standard](https://refspecs.linuxfoundation.org/FHS_3.0/fhs-3.0.html), that places programs, drivers, and their components in specific paths. For instance, + +- `\bin` contains commands that may be used by both the system administrator and by users, +- `\lib` contains those shared library images needed to run the commands in the root filesystem, and +- `\home` contains user home directories. + +Every isolated environment and containerization system creates a file system hierarchy in some path of the file system that forms the _root_ of the isolated environment. You can use + +- `man hier` to access a succinct description of the filesystem hierarchy of your system, and +- `man file-hierarchy` to access the systemd file system hierarchy requirements. + +The latter is useful in container environment that need to support system services. + +## Example: Python virtual environment (`venv`) + +Start by loading the latest software release, and from the software release load the Python module to access the Python executable. + +```console +module load env/release/2025.1 +module load Python +``` + +You can now use the python `venv` module to create and setup the virtual environment as a non-privileged user. + +1. Create the virtual environment. + + ```console + python -m venv ${HOME}/environments/env-numpy + ``` + +1. Activate the virtual environment. + + ```console + source ${HOME}/environments/env-numpy/bin/activate + ``` + +1. Install the required packages. For instance, in this example we install `numpy` and `pyyaml`. + + ```console + (env-numpy) $ pip install numpy pyyaml + ``` + +After installing the packages the environment is ready for use. In future sessions, load the environment with the command + +```console +source ${HOME}/environments/env-numpy/bin/activate +``` + +and unload the environment with the command: + +```console +deactivate +``` + +To remove the environment, simply delete the environment directory. + +```console +rm -r ${HOME}/environments/env-numpy +``` + +!!! question "Virtual environment setup and contents" + + === "Question" + + What are the contents of the virtual environment directory? Can you detect any of the path to the virtual environment directory in the `PATH` and `LD_LIBRARY_PATH` variables? + + === "Answer" + + The contents of the virtual environment directory follow the filesystem hierarchy structure. + + ```console + $ ls -lahF $VIRTUAL_ENV + total 28K + drwxr-x--- 5 u101064 hpcusers 4.0K Jun 14 18:12 ./ + drwx------ 15 u101064 hpcusers 4.0K Jun 14 18:12 ../ + drwxr-x--- 2 u101064 hpcusers 4.0K Jun 14 18:12 bin/ + -rw-r----- 1 u101064 hpcusers 69 Jun 14 18:12 .gitignore + drwxr-x--- 3 u101064 hpcusers 4.0K Jun 14 18:12 include/ + drwxr-x--- 3 u101064 hpcusers 4.0K Jun 14 18:12 lib/ + lrwxrwxrwx 1 u101064 hpcusers 3 Jun 14 18:12 lib64 -> lib/ + -rw-r----- 1 u101064 hpcusers 381 Jun 14 18:12 pyvenv.cfg + ``` + The virtual environment sets the convenient variable `VIRTUAL_ENV` that points to the root of the virtual environment, `${HOME}/environments/env-numpy`. The `PATH` variable points to `$VIRTUAL_ENV/bin`. + + ```console + $ echo $PATH | tr ':' '\n' + /home/users/u101064/env-numpy/bin + /apps/USE/easybuild/release/2025.1/software/Python/3.13.1-GCCcore-14.2.0/bin + /apps/USE/easybuild/release/2025.1/software/OpenSSL/3/bin + /apps/USE/easybuild/release/2025.1/software/XZ/5.6.3-GCCcore-14.2.0/bin + /apps/USE/easybuild/release/2025.1/software/SQLite/3.47.2-GCCcore-14.2.0/bin + /apps/USE/easybuild/release/2025.1/software/Tcl/8.6.16-GCCcore-14.2.0/bin + /apps/USE/easybuild/release/2025.1/software/ncurses/6.5-GCCcore-14.2.0/bin + /apps/USE/easybuild/release/2025.1/software/bzip2/1.0.8-GCCcore-14.2.0/bin + /apps/USE/easybuild/release/2025.1/software/binutils/2.42-GCCcore-14.2.0/bin + /apps/USE/easybuild/release/2025.1/software/GCCcore/14.2.0/bin + /home/users/u101064/.local/bin + /home/users/u101064/bin + /usr/local/bin + /usr/bin + /usr/local/sbin + /usr/sbin + ``` + + However, the `lib` directory of the environment does not appear in the `LD_LIBRARY_PATH`. The environment is a Python environment, and all external libraries are loaded by Python executable of the environment and not the system linker, so setting up the linker environment variable, `LD_LIBRARY_PATH`, is redundant. The Python executable in the environment, + + ```text + /home/users/u101064/env-numpy/bin/python + ``` + + will look into the `pyvenv.cfg` at the root of the virtual environment to determine the location where Python will look for share libraries. The shared libraries of the environment are still stored in `$VIRTUAL_ENV/lib`. For instance the mathematical libraries used by NumPy are located in the following directory. + + ```console + $ ls -lahF $VIRTUAL_ENV/lib/python3.13/site-packages/numpy.libs + total 28M + drwxr-x--- 2 u101064 hpcusers 4.0K Jun 14 18:12 ./ + drwxr-x--- 7 u101064 hpcusers 4.0K Jun 14 18:12 ../ + -rwxr-x--x 1 u101064 hpcusers 2.8M Jun 14 18:12 libgfortran-040039e1-0352e75f.so.5.0.0* + -rwxr-x--x 1 u101064 hpcusers 246K Jun 14 18:12 libquadmath-96973f99-934c22de.so.0.0.0* + -rwxr-x--x 1 u101064 hpcusers 25M Jun 14 18:12 libscipy_openblas64_-32a4b2a6.so* + ``` + + The get a list of paths where the Python interpreter is looking for shared libraries, you can use the following command. + + ```console + python -c "import sys; from pathlib import PureWindowsPath; print('\n'.join(PureWindowsPath(p).as_posix() for p in sys.path))" + ``` diff --git a/docs/courses/electronic_structure_methods/index.md b/docs/courses/electronic_structure_methods/index.md index 1592d581..99541031 100644 --- a/docs/courses/electronic_structure_methods/index.md +++ b/docs/courses/electronic_structure_methods/index.md @@ -1,7 +1,7 @@ # Electronic structure methods used in industry from pharmaceuticals materials and beyond -> Author: [Ariadni Boziki](https://www.uni.lu/fstm-en/people/ariadni-boziki/)
-> email: [ariadni.boziki@uni.lu](mailto:ariadni.boziki@uni.lu) +> Author: [Ariadni Boziki](https://researchportal.list.lu/scientific-community/detail/boziki-ariadni/)
+> email: [ariadni.boziki@uni.lu](mailto:ariadni.boziki@list.lu) ## Lecture slides diff --git a/docs/organization/instructions_for_educators.md b/docs/organization/instructions_for_educators.md index b926c6a0..23a400ac 100644 --- a/docs/organization/instructions_for_educators.md +++ b/docs/organization/instructions_for_educators.md @@ -52,6 +52,6 @@ Furthermore the following steps would summarize the steps for instructors. 6. **Technical Support:** - Use the [LXP service ticket system](https://servicedesk.lxp.lu/servicedesk/customer/portal/4) to request compute node resources, specifying the expected number of participants. Make these requests at least two weeks before the event starts. Alternatively, you can forward the list of participants to the [Julie DEL DUCA](mailto:julie.delduca@lxp.lu), the project coordinator in LuxProvide to create the user accounts. + Use the [LXP service ticket system](https://servicedesk.lxp.lu/servicedesk/customer/portal/4) to request compute node resources, specifying the expected number of participants. Make these requests at least two weeks before the event starts. Alternatively, you can forward the list of participants to [Julie DEL DUCA](mailto:julie.delduca@lxp.lu), the project coordinator in LuxProvide to create the user accounts. Make sure to follow these guidelines to ensure a successful training event. diff --git a/properdocs.yml b/properdocs.yml index a50b6634..5c902462 100644 --- a/properdocs.yml +++ b/properdocs.yml @@ -65,7 +65,10 @@ nav: - Environment management and containers: - Course description: 'courses/containerization/index.md' - Introduction: 'courses/containerization/introduction.md' - - Containers: 'courses/containerization/containers.md' + - Virtual environments: 'courses/containerization/virtual_environments.md' + - Containers: + - Basic principles: 'courses/containerization/containers.md' + - Containers in HPC systems: 'courses/containerization/containers_in_hpc_systems.md' # - Environment managers: 'courses/containerization/conda.md' # - Package management in R: 'courses/containerization/R.md' # - Best practices in environment management: 'environment/best-practises.md' @@ -147,6 +150,7 @@ markdown_extensions: - attr_list - md_in_html - mdx_truly_sane_lists + - admonition - pymdownx.superfences: custom_fences: - name: mermaid @@ -159,10 +163,8 @@ markdown_extensions: pygments_lang_class: true - pymdownx.inlinehilite - pymdownx.snippets - - pymdownx.superfences - tables - footnotes - - admonition - codehilite: guess_lang: false - pymdownx.betterem: diff --git a/requirements.txt b/requirements.txt index 90d8d838..4241cb95 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ properdocs==1.6.7 mdx-truly-sane-lists==1.3 markdown-include==0.8.1 -mkdocs-document-dates==3.8.2 +mkdocs-document-dates==3.8.6 mkdocs-glightbox==0.5.2 -mkdocs-materialx==10.1.5 +mkdocs-materialx==10.1.7 mkdocs-minify-plugin==0.8.0 mkdocs-include-markdown-plugin==7.3.0 mkdocs-site-urls==0.3.1