Build: Add compiler hardening flags and gate them in CI

.dockerignore

@@ -2,6 +2,7 @@
 .git/
 .gitignore
 .github/
+!.github/scripts/ci/check-hardening.sh
 .clang-format
 .claude/
 .ci-local/

.github/path_filters.yml

@@ -213,14 +213,16 @@ base_build:
   - 'tools/sch_smi_emulate/**'
 
 # docker_build.yml, ubuntu-docker-build. The build context is the repository
-# root less .dockerignore, which drops .github/, so the hash_<name> filters do
-# not apply: they hold .github/scripts/setup_environment.sh. Each image builds
-# DPDK, runs build.sh and the unit suite, and copies script/ into the image.
+# root less .dockerignore, which drops .github/ except check-hardening.sh, run
+# by the rocky9 build, so the hash_<name> filters do not apply: they hold
+# .github/scripts/setup_environment.sh. Each image builds DPDK, runs build.sh
+# and the unit suite, and copies script/ into the image.
 docker_build:
   - .github/path_filters.yml
   - .github/workflows/docker_build.yml
   - 'docker/*.dockerfile'
   - .dockerignore
+  - .github/scripts/ci/check-hardening.sh
   - 'script/**'
   - 'patches/dpdk/**'
   - versions.env

.github/scripts/ci/check-hardening.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2026 Intel Corporation
+#
+# Fails when an x86-64 executable or shared object under the given directories
+# lacks a mark the hardening flags leave, see doc/build.md: full RELRO
+# (-z relro -z now), a non-executable stack, PIE and CET IBT and SHSTK
+# (-fcf-protection=full, except for libopenh264). The stack protector and
+# _FORTIFY_SOURCE leave no reliable mark in a binary, so they are not checked.
+# Every check needs a positive match, so a readelf that prints something else
+# fails the file instead of passing it, and every given path must hold at least
+# one x86-64 ELF file, so a missing or empty one fails too.
+
+set -euo pipefail
+
+(($#)) || {
+	echo "usage: check-hardening.sh DIR..." >&2
+	exit 2
+}
+
+checked=0
+failed=0
+for path; do
+	before=$checked
+	while IFS= read -r -d '' file; do
+		elf=$(readelf -W -h -l -d -n "$file" 2>/dev/null) || continue
+		grep -q 'Machine: *Advanced Micro Devices X86-64' <<<"$elf" || continue
+		grep -qE 'Type: *(DYN|EXEC) ' <<<"$elf" || continue
+		checked=$((checked + 1))
+		missing=()
+		grep -q ' GNU_RELRO ' <<<"$elf" || missing+=(RELRO)
+		grep -qE '\(FLAGS\) .*BIND_NOW|\(FLAGS_1\) .*Flags:.* NOW' <<<"$elf" || missing+=(BIND_NOW)
+		grep -qE ' GNU_STACK .* RW ' <<<"$elf" || missing+=(NX)
+		grep -qE 'Type: *DYN ' <<<"$elf" || missing+=(PIE)
+		# The openh264 assembly has no CET mark, see doc/build.md
+		if [[ ${file##*/} != libopenh264.so* ]]; then
+			grep -q 'x86 feature: IBT, SHSTK' <<<"$elf" || missing+=(IBT/SHSTK)
+		fi
+		if ((${#missing[@]})); then
+			echo "::error::${file} is not hardened: no ${missing[*]}" >&2
+			failed=$((failed + 1))
+		fi
+	done < <(find "$path" -type f -print0)
+	((checked > before)) || {
+		echo "::error::no x86-64 ELF file found under $path" >&2
+		exit 1
+	}
+done
+
+echo "hardening: ${checked} ELF files checked under $*, ${failed} failed"
+((failed == 0))

.github/scripts/ci/validate-cache.sh

@@ -38,4 +38,19 @@ ice) ICE_BUNDLE_ROOT="$component_root" bash "${root_dir}/.github/scripts/ci/vali
 	;;
 esac
 
+# Compiler hardening of what MTL builds, see doc/build.md. In branch mode
+# (the input branch of build.yml and the pytest workflows) the trees come
+# from HEAD and this script from the workflow commit, so the check is the
+# one HEAD has: a HEAD that predates it is not checked. A git error fails
+# the validation.
+case "$component" in
+dpdk | mtl | ffmpeg | gstreamer | plugins)
+	head_check=$(git -C "$root_dir" ls-tree --name-only HEAD .github/scripts/ci/check-hardening.sh)
+	if [[ -n $head_check ]]; then
+		checker=$(git -C "$root_dir" show "HEAD:${head_check}")
+		bash -c "$checker" check-hardening.sh "$component_root"
+	fi
+	;;
+esac
+
 echo "${component} cache: valid"

app/meson.build

@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright 2022 Intel Corporation
 
-project('mtl_app', 'c', default_options: ['buildtype=release'],
+project('mtl_app', 'c', default_options: ['buildtype=release', 'b_pie=true'],
   version: run_command(find_program('cat'), files('../VERSION'), check: true).stdout().strip(),)
 
 gpu_direct = dependency('mtl_gpu_direct', required: false)
@@ -56,6 +56,21 @@ subdir('v4l2_to_ip')
 app_c_args = []
 app_ld_args = []
 
+# Compiler hardening, see doc/build.md
+if not is_windows
+  add_project_arguments(cc.get_supported_arguments(['-fstack-protector-strong',
+    '-fstack-clash-protection', '-fcf-protection=full']),
+    '-Wformat', '-Wformat-security', '-Werror=format-security', language : 'c')
+  # keep the level a compiler (Ubuntu 24.04 gcc: 3) or CFLAGS already set, asking the
+  # compiler itself: meson >= 1.11 runs its checks with -U_FORTIFY_SOURCE
+  fortify = run_command(cc.cmd_array(), get_option('c_args'), '-O2', '-dM', '-E', '-x', 'c', '/dev/null',
+    check: true).stdout()
+  if get_option('optimization') not in ['0', 'plain'] and not fortify.contains('_FORTIFY_SOURCE')
+    add_project_arguments('-D_FORTIFY_SOURCE=2', language : 'c')
+  endif
+  add_project_link_arguments('-Wl,-z,relro', '-Wl,-z,now', '-Wl,-z,noexecstack', language : 'c')
+endif
+
 # enable warning as error for non debug build
 if get_option('buildtype') != 'debug'
   app_c_args += ['-Werror']

doc/build.md

@@ -163,6 +163,8 @@ git am $mtl_source_code/patches/dpdk/${DPDK_VER}/*.patch
 
 ### 2.3. Build and install DPDK library
 
+`script/build_dpdk.sh` builds DPDK with the options of [4.5. Compiler hardening](#45-compiler-hardening); pass them to a manual build as that section shows.
+
 ```bash
 meson setup build
 ninja -C build
@@ -314,5 +316,38 @@ For older kernel version on Red Hat, the issue is that Red Hat uses vault repos.
 
 [The exemplary location for the kernel-devel package for Rocky Linux 9.3](https://dl.rockylinux.org/vault/rocky/9.3/BaseOS/x86_64/os/Packages/k/)
 
+### 4.5. Compiler hardening
+
+The Linux build compiles the shipped MTL components, DPDK, openh264 and FFmpeg with the hardening options below. The `meson.build` of each MTL component sets them, `script/build_dpdk.sh` passes them to DPDK, and `ecosystem/ffmpeg_plugin/build.sh` passes them to openh264 and FFmpeg. SVT-JPEG-XS and the libbpf and libxdp from `script/build_ebpf_xdp.sh` keep their own build flags.
+
+| Option | Protection |
+| --- | --- |
+| `-fstack-protector-strong` | Stack buffer overflow canary |
+| `-fstack-clash-protection` | Stack clash |
+| `-fcf-protection=full` | Intel CET shadow stack (SHSTK) and indirect branch tracking (IBT) |
+| `-D_FORTIFY_SOURCE=2` | Buffer overflow checks in libc calls. Not added when the compiler already sets a level (Ubuntu 24.04 gcc sets 3) or without optimization (`-O0`, buildtype `plain`) |
+| `-Wformat -Wformat-security -Werror=format-security` | Format string attacks |
+| `-Wl,-z,relro -Wl,-z,now` | Full RELRO, a read-only GOT |
+| `-Wl,-z,noexecstack` | Non-executable stack |
+| `b_pie=true`, FFmpeg `-pie` | Position independent executables for ASLR |
+
+For a manual DPDK build, pass them to `meson setup`, and add `-D_FORTIFY_SOURCE=2` to `c_args` if `echo | cc -O2 -dM -E - | grep _FORTIFY_SOURCE` prints nothing:
+
+```bash
+meson setup build -Db_pie=true \
+  -Dc_args="-fstack-protector-strong -fstack-clash-protection -fcf-protection=full -Wformat -Wformat-security -Werror=format-security" \
+  -Dc_link_args="-Wl,-z,relro -Wl,-z,now -Wl,-z,noexecstack"
+```
+
+The FFmpeg 7.0 assembly is marked SHSTK only, so the build applies `ecosystem/ffmpeg_plugin/7.0/0002-x86-add-Intel-CET-IBT-support.patch` to add IBT. The FFmpeg 6.1 and 4.4 assembly has no CET mark, so those versions build without IBT and SHSTK.
+
+The openh264 assembly has no CET mark, so `libopenh264` has neither IBT nor SHSTK, and a process that loads it runs without CET.
+
+To check an install tree, run the check CI runs on the DPDK, MTL, FFmpeg, GStreamer and plugins caches and in the Rocky Linux 9 image build. Under `/usr/local` it also reports other software installed there.
+
+```bash
+.github/scripts/ci/check-hardening.sh /usr/local/lib /usr/local/bin
+```
+
 ## Next Steps
 Proceed to [Running MTL](./run.md) for further instructions.

docker/rocky9.dockerfile

@@ -55,7 +55,8 @@ RUN ./build.sh unit && \
     ./build.sh && \
     ninja -C build install && \
     DESTDIR=/install ninja -C build install && \
-    setcap 'cap_net_raw+ep' tests/tools/RxTxApp/build/RxTxApp
+    setcap 'cap_net_raw+ep' tests/tools/RxTxApp/build/RxTxApp && \
+    .github/scripts/ci/check-hardening.sh /install /usr/local/bin/RxTxApp /usr/local/bin/MtlManager /usr/local/bin/KahawaiTest
 
 # Rocky Linux 9, runtime/final stage
 FROM "${IMAGE_CACHE_REGISTRY}/library/rockylinux:9" AS final

ecosystem/ffmpeg_plugin/7.0/0002-x86-add-Intel-CET-IBT-support.patch

@@ -0,0 +1,194 @@
+From 64afcb153972f62840c92818ba097781ba179271 Mon Sep 17 00:00:00 2001
+From: "Wilczynski, Andrzej" <andrzej.wilczynski@intel.com>
+Date: Fri, 25 Sep 2026 12:00:00 +0200
+Subject: [PATCH] x86: add Intel CET IBT support
+
+x86inc.asm marks every NASM object with the SHSTK property only, so a
+build compiled with -fcf-protection=full still links libavcodec,
+libavfilter, libavutil, libswresample and libswscale without IBT: the
+linker ANDs the property of all objects.
+
+Emit endbr64 at every cglobal entry and mark the objects IBT and SHSTK
+when NASM knows the CET instructions (2.15.01 and later). Make the
+indirect branch targets that are not function entries valid too: the
+vvc_mc jump table targets get an endbr64, the mlpdsp inline asm jumps
+into its unrolled filter with notrack, as GCC does for switch tables,
+and the swscale MMXEXT run-time generated code starts with endbr64.
+
+endbr64 is a NOP on CPUs without CET.
+
+Signed-off-by: Wilczynski, Andrzej <andrzej.wilczynski@intel.com>
+---
+ libavcodec/x86/mlpdsp_init.c               |  4 ++--
+ libavcodec/x86/vvc/vvc_mc.asm              | 16 +++++++++++++++-
+ libavutil/x86/x86inc.asm                   | 14 +++++++++++++-
+ libswscale/x86/hscale_fast_bilinear_simd.c |  8 ++++++++
+ 4 files changed, 38 insertions(+), 4 deletions(-)
+
+diff --git a/libavcodec/x86/mlpdsp_init.c b/libavcodec/x86/mlpdsp_init.c
+index 950f996..941a5f5 100644
+--- a/libavcodec/x86/mlpdsp_init.c
++++ b/libavcodec/x86/mlpdsp_init.c
+@@ -145,7 +145,7 @@ static void mlp_filter_channel_x86(int32_t *state, const int32_t *coeff,
+     __asm__ volatile(
+         "1:                           \n\t"
+         CLEAR_ACCUM
+-        "jmp  *%5                     \n\t"
++        "notrack jmp *%5              \n\t"
+         FIRMUL   (ff_mlp_firorder_8, 0x1c   )
+         FIRMUL   (ff_mlp_firorder_7, 0x18   )
+         FIRMUL   (ff_mlp_firorder_6, 0x14   )
+@@ -155,7 +155,7 @@ static void mlp_filter_channel_x86(int32_t *state, const int32_t *coeff,
+         FIRMUL   (ff_mlp_firorder_2, 0x04   )
+         FIRMULREG(ff_mlp_firorder_1, 0x00, 8)
+         LABEL_MANGLE(ff_mlp_firorder_0)":\n\t"
+-        "jmp  *%6                     \n\t"
++        "notrack jmp *%6              \n\t"
+         IIRMUL   (ff_mlp_iirorder_4, 0x0c   )
+         IIRMUL   (ff_mlp_iirorder_3, 0x08   )
+         IIRMUL   (ff_mlp_iirorder_2, 0x04   )
+diff --git a/libavcodec/x86/vvc/vvc_mc.asm b/libavcodec/x86/vvc/vvc_mc.asm
+index 30aa97c..9126583 100644
+--- a/libavcodec/x86/vvc/vvc_mc.asm
++++ b/libavcodec/x86/vvc/vvc_mc.asm
+@@ -47,7 +47,7 @@ pw_256  times 2 dw 256
+     %xdefine %%prefix mangle(private_prefix %+ _vvc_%1_%2bpc_%3)
+     %%table:
+     %rep %0 - 3
+-        dd %%prefix %+ .w%4 - %%base
++        dd %%prefix %+ .w%4_ibt - %%base
+         %rotate 1
+     %endrep
+ %endmacro
+@@ -79,6 +79,8 @@ SECTION .text
+ %macro AVG_FN 2 ; bpc, op
+    jmp                  wq
+
++.w2_ibt:
++    _CET_ENDBR
+ .w2:
+     movd                xm0, [src0q]
+     pinsrd              xm0, [src0q + AVG_SRC_STRIDE], 1
+@@ -88,6 +90,8 @@ SECTION .text
+     AVG_SAVE_W2          %1
+     AVG_LOOP_END        .w2
+
++.w4_ibt:
++    _CET_ENDBR
+ .w4:
+     movq                xm0, [src0q]
+     pinsrq              xm0, [src0q + AVG_SRC_STRIDE], 1
+@@ -98,6 +102,8 @@ SECTION .text
+
+     AVG_LOOP_END        .w4
+
++.w8_ibt:
++    _CET_ENDBR
+ .w8:
+     vinserti128         m0, m0, [src0q], 0
+     vinserti128         m0, m0, [src0q + AVG_SRC_STRIDE], 1
+@@ -108,21 +114,29 @@ SECTION .text
+
+     AVG_LOOP_END       .w8
+
++.w16_ibt:
++    _CET_ENDBR
+ .w16:
+     AVG_W16_FN          %1, %2, 1
+
+     AVG_LOOP_END       .w16
+
++.w32_ibt:
++    _CET_ENDBR
+ .w32:
+     AVG_W16_FN          %1, %2, 2
+
+     AVG_LOOP_END       .w32
+
++.w64_ibt:
++    _CET_ENDBR
+ .w64:
+     AVG_W16_FN          %1, %2, 4
+
+     AVG_LOOP_END       .w64
+
++.w128_ibt:
++    _CET_ENDBR
+ .w128:
+     AVG_W16_FN          %1, %2, 8
+
+diff --git a/libavutil/x86/x86inc.asm b/libavutil/x86/x86inc.asm
+index e61d924..f1b2678 100644
+--- a/libavutil/x86/x86inc.asm
++++ b/libavutil/x86/x86inc.asm
+@@ -849,6 +849,7 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae,
+     %endif
+     align function_align
+     %2:
++    _CET_ENDBR
+     RESET_MM_PERMUTATION        ; needed for x86-64, also makes disassembly somewhat nicer
+     %xdefine rstk rsp           ; copy of the original stack pointer, used when greater alignment than the known stack alignment is required
+     %assign stack_offset 0      ; stack pointer offset relative to the return address
+@@ -870,6 +871,7 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae,
+         global current_function %+ %1
+     %endif
+     %1:
++    _CET_ENDBR
+ %endmacro
+
+ %macro cextern 1
+@@ -899,6 +901,10 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae,
+     %1: %2
+ %endmacro
+
++; Indirect branch target marker (Intel CET IBT). Only emitted when the
++; GNU property note below advertises IBT; empty everywhere else.
++%define _CET_ENDBR
++
+ %if FORMAT_ELF
+     ; The GNU linker assumes the stack is executable by default.
+     [SECTION .note.GNU-stack noalloc noexec nowrite progbits]
+@@ -907,6 +913,12 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae,
+         %if __NASM_VERSION_ID__ >= 0x020e0300 ; 2.14.03
+             %if ARCH_X86_64
+                 ; Control-flow Enforcement Technology (CET) properties.
++                %if __NASM_VERSION_ID__ >= 0x020f0100 ; 2.15.01 added CET instructions
++                    %define _CET_ENDBR endbr64
++                    %define CET_FEATURE_1 3 ; GNU_PROPERTY_X86_FEATURE_1_IBT | GNU_PROPERTY_X86_FEATURE_1_SHSTK
++                %else
++                    %define CET_FEATURE_1 2 ; GNU_PROPERTY_X86_FEATURE_1_SHSTK
++                %endif
+                 [SECTION .note.gnu.property alloc noexec nowrite note align=gprsize]
+                 dd 0x00000004  ; n_namesz
+                 dd gprsize + 8 ; n_descsz
+@@ -914,7 +926,7 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae,
+                 db "GNU",0     ; n_name
+                 dd 0xc0000002  ; pr_type = GNU_PROPERTY_X86_FEATURE_1_AND
+                 dd 0x00000004  ; pr_datasz
+-                dd 0x00000002  ; pr_data = GNU_PROPERTY_X86_FEATURE_1_SHSTK
++                dd CET_FEATURE_1 ; pr_data
+                 dd 0x00000000  ; pr_padding
+             %endif
+         %endif
+diff --git a/libswscale/x86/hscale_fast_bilinear_simd.c b/libswscale/x86/hscale_fast_bilinear_simd.c
+index f6409b4..f7c587b 100644
+--- a/libswscale/x86/hscale_fast_bilinear_simd.c
++++ b/libswscale/x86/hscale_fast_bilinear_simd.c
+@@ -132,6 +132,14 @@ av_cold int ff_init_hscaler_mmxext(int dstW, int xInc, uint8_t *filterCode,
+     xpos        = 0; // lumXInc/2 - 0x8000; // difference between pixel centers
+     fragmentPos = 0;
+
++#if ARCH_X86_64
++    // The generated code is entered via an indirect call, so it must start
++    // with endbr64 to be a valid CET IBT target (a NOP on non-CET CPUs).
++    if (filterCode)
++        memcpy(filterCode, "\xf3\x0f\x1e\xfa", 4);
++    fragmentPos = 4;
++#endif
++
+     for (i = 0; i < dstW / numSplits; i++) {
+         int xx = xpos >> 16;
+
+-- 
+2.34.1
+