From f2013c65c7206b21ac9e64848317102094e85b96 Mon Sep 17 00:00:00 2001 From: Mehmet Samet Duman Date: Wed, 4 Nov 2026 17:57:42 +0300 Subject: [PATCH] Added KDE Deps installation steps to CI Signed-off-by: Mehmet Samet Duman --- .github/actions/setup-dependencies/action.yml | 21 +- .../setup-dependencies/linux/action.yml | 14 +- .../setup-dependencies/macos/action.yml | 10 +- .../setup-dependencies/windows/action.yml | 172 ++ .github/actions/setup-kf6/README.md | 181 +++ .github/actions/setup-kf6/action.yml | 298 ++++ .github/actions/setup-kf6/kf6.py | 1447 +++++++++++++++++ .github/workflows/build.yml | 45 +- 8 files changed, 2143 insertions(+), 45 deletions(-) create mode 100644 .github/actions/setup-kf6/README.md create mode 100644 .github/actions/setup-kf6/action.yml create mode 100644 .github/actions/setup-kf6/kf6.py diff --git a/.github/actions/setup-dependencies/action.yml b/.github/actions/setup-dependencies/action.yml index 80e3cf92f..d1a838005 100644 --- a/.github/actions/setup-dependencies/action.yml +++ b/.github/actions/setup-dependencies/action.yml @@ -99,7 +99,6 @@ runs: vcpkg_platform=linux ;; "macOS") - vcpkg_arch=universal vcpkg_platform=osx ;; "Windows") @@ -132,9 +131,27 @@ runs: with: aqtsource: "git+https://github.com/miurahr/aqtinstall.git@076e1659807d0b362a3ed684d54c2e9c775eb9c7" version: ${{ inputs.qt-version }} - modules: qtimageformats qtnetworkauth + modules: qtimageformats qtnetworkauth qtshadertools qtspeech qtmultimedia qttasktree cache: ${{ inputs.build-type == 'Debug' }} + - name: Setup KDE Frameworks 6 + if: ${{ inputs.msystem == '' && inputs.qt-major == '6' }} + uses: ./.github/actions/setup-kf6 + env: + # The frameworks are cached by the action itself; keep them out of sccache. + CMAKE_C_COMPILER_LAUNCHER: "" + CMAKE_CXX_COMPILER_LAUNCHER: "" + with: + version: 6.31.0 + # Always Release: like the old vcpkg triplets, dependencies don't need a debug half. + build-type: Release + linkage: shared + # Windows (MSVC) only: where the DocBook DTD/stylesheets were unpacked, see setup-dependencies/windows. + framework-cmake-args: "${{ env.KF6_KDOCTOOLS_ARGS && format('kdoctools: {0}', env.KF6_KDOCTOOLS_ARGS) || '' }}" + # TODO: set to false once KF6 builds cleanly everywhere. While bringing up a platform, + # this reports every failing framework in one run instead of only the first. + keep-going: true + - name: Setup bundled OpenSSL uses: ./.github/actions/setup-openssl with: diff --git a/.github/actions/setup-dependencies/linux/action.yml b/.github/actions/setup-dependencies/linux/action.yml index e4a275f2a..fe07667b2 100644 --- a/.github/actions/setup-dependencies/linux/action.yml +++ b/.github/actions/setup-dependencies/linux/action.yml @@ -10,12 +10,20 @@ runs: run: | sudo apt-get -y update sudo apt-get -y install \ - dpkg-dev \ + dpkg-dev build-essential \ ninja-build pkg-config scdoc gamemode-dev \ - libxcb-cursor-dev libtomlplusplus-dev libvulkan-dev \ + libxcb-cursor-dev libxcb-randr0-dev libxcb-shape0-dev \ + libxcb-xfixes0-dev libxcb-xkb-dev libxcb-icccm4-dev \ + libxcb-keysyms1-dev libxcb-image0-dev libxcb-render-util0-dev \ + libxcb-xinerama0-dev libxcb-sync-dev libtomlplusplus-dev libvulkan-dev \ libgpg-error-dev libassuan-dev libksba-dev \ libnpth0-dev libbotan-3-dev autoconf automake libtool gettext \ - autoconf-archive + autoconf-archive cmake gperf zlib1g-dev libbz2-dev liblzma-dev \ + libxml2-dev libx11-dev libxcb1-dev libxkbcommon-dev \ + libxkbcommon-x11-dev plasma-wayland-protocols libxcb-res0-dev \ + libwayland-dev wayland-protocols libwayland-egl1 libx11-dev \ + libxfixes-dev docbook docbook-xml docbook-xsl docbook-utils \ + xsltproc libxml2-utils libaspell-dev libxslt1-dev - name: Setup Java uses: actions/setup-java@v6 diff --git a/.github/actions/setup-dependencies/macos/action.yml b/.github/actions/setup-dependencies/macos/action.yml index 6bcd63d1b..14840ce21 100644 --- a/.github/actions/setup-dependencies/macos/action.yml +++ b/.github/actions/setup-dependencies/macos/action.yml @@ -17,8 +17,14 @@ runs: shell: bash run: | brew update - brew install ninja extra-cmake-modules temurin@17 autoconf pkg-config \ - autoconf-archive automake libtool + brew install cmake ninja temurin@17 autoconf pkg-config \ + autoconf-archive automake libtool gettext libxml2 zlib \ + bzip2 xz gperf cmark docbook docbook-xsl cpanminus bison \ + libical qrencode libdmtx zxing-cpp zstd + + cpanm URI + + echo "$(brew --prefix bison)/bin" >> "$GITHUB_PATH" - name: Install Qt5 if: ${{ inputs.qt-major == '5' }} diff --git a/.github/actions/setup-dependencies/windows/action.yml b/.github/actions/setup-dependencies/windows/action.yml index 90b966646..b26abb12c 100644 --- a/.github/actions/setup-dependencies/windows/action.yml +++ b/.github/actions/setup-dependencies/windows/action.yml @@ -40,8 +40,135 @@ runs: if: ${{ inputs.msystem == '' }} shell: pwsh run: | + $ErrorActionPreference = "Stop" + choco install pkgconfiglite -y --allow-empty-checksums + $vcpkgRoot = $env:VCPKG_INSTALLATION_ROOT + if (-not $vcpkgRoot) { + $vcpkgRoot = $env:VCPKG_ROOT + } + if (-not $vcpkgRoot) { + throw "VCPKG_INSTALLATION_ROOT/VCPKG_ROOT is not set on this runner." + } + + $triplet = if ($env:RUNNER_ARCH -eq "ARM64") { + "arm64-windows" + } else { + "x64-windows" + } + + & "$vcpkgRoot\vcpkg.exe" install --classic ` + gettext-libintl gperf zlib bzip2 liblzma libxml2 iso-codes cmark libarchive libqrencode libxslt zstd ` + libical libdmtx nu-book-zxing-cpp ` + "--triplet=$triplet" + + if ($LASTEXITCODE -ne 0) { + throw "vcpkg dependency installation failed." + } + + $tripletRoot = Join-Path $vcpkgRoot "installed\$triplet" + $toolsRoot = Join-Path $tripletRoot "tools" + + $pathEntries = [System.Collections.Generic.HashSet[string]]::new( + [System.StringComparer]::OrdinalIgnoreCase + ) + + # Executables installed directly in the triplet's bin directory. + $binPath = Join-Path $tripletRoot "bin" + if (Test-Path $binPath) { + [void]$pathEntries.Add($binPath) + } + + # vcpkg commonly installs build tools in tools\\. + if (Test-Path $toolsRoot) { + Get-ChildItem -Path $toolsRoot -Directory -Recurse | + ForEach-Object { + $directory = $_.FullName + + if ( + Get-ChildItem -LiteralPath $directory -File ` + -ErrorAction SilentlyContinue | + Where-Object { + $_.Extension -in @(".exe", ".cmd", ".bat", ".com") + } | + Select-Object -First 1 + ) { + [void]$pathEntries.Add($directory) + } + } + + # Also include the tools root itself. + [void]$pathEntries.Add($toolsRoot) + } + + foreach ($entry in $pathEntries) { + Add-Content -Path $env:GITHUB_PATH -Value $entry + Write-Host "Added to PATH: $entry" + } + + # PATH is not enough: CMake does not look for libraries/headers next to the tools on + # PATH, so zlib, libintl, libical, ... in this tree were reported as "Could NOT find". + # CMAKE_PREFIX_PATH makes find_package/find_library/find_path look into it. + # (The SnoreToast step below extends it and exports the final value.) + Add-Content -Path $env:GITHUB_ENV -Value "VCPKG_TRIPLET_ROOT=$tripletRoot" + Add-Content -Path $env:GITHUB_ENV -Value "CMAKE_PREFIX_PATH=$tripletRoot" + # Some KDE Frameworks find libraries (zstd) only through pkg-config. + Add-Content -Path $env:GITHUB_ENV -Value "PKG_CONFIG_PATH=$(Join-Path $tripletRoot 'lib\pkgconfig')" + + # Verify that the next GitHub Actions step can find gperf. + $gperfCandidates = @( + (Join-Path $toolsRoot "gperf\gperf.exe"), + (Join-Path $binPath "gperf.exe") + ) + + $gperf = Get-Command gperf -ErrorAction SilentlyContinue + if (-not $gperf) { + foreach ($candidate in $gperfCandidates) { + if (Test-Path $candidate) { + $gperf = Get-Item $candidate + break + } + } + } + + if ($gperf) { + Write-Host "Found gperf: $($gperf.Source)" + } else { + Write-Warning "gperf.exe was not found in the expected vcpkg locations." + Get-ChildItem -Path $toolsRoot -Filter "gperf.exe" ` + -File -Recurse -ErrorAction SilentlyContinue | + ForEach-Object { Write-Host "Found candidate: $($_.FullName)" } + } + + # KNotifications on Windows needs either SnoreToast (Windows toast notifications) or + # libcanberra, which does not exist here. KIO, KJobWidgets and KWallet need KNotifications. + - name: Build SnoreToast (MSVC) + if: ${{ inputs.msystem == '' }} + shell: pwsh + run: | + $ErrorActionPreference = "Stop" + + $depsRoot = Join-Path $env:RUNNER_TEMP "deps" + $src = Join-Path $env:RUNNER_TEMP "snoretoast-src" + $bld = Join-Path $env:RUNNER_TEMP "snoretoast-build" + + git clone --depth 1 --branch v0.9.1 https://github.com/KDE/snoretoast.git $src + if ($LASTEXITCODE -ne 0) { throw "cloning snoretoast failed" } + + # BUILD_STATIC_RUNTIME=OFF: the frameworks use the dynamic CRT (/MD); a static (/MT) + # library linked into KNotifications would mix the two. + cmake -S $src -B $bld -G Ninja -DCMAKE_BUILD_TYPE=Release ` + "-DCMAKE_INSTALL_PREFIX=$depsRoot" -DBUILD_STATIC_RUNTIME=OFF + if ($LASTEXITCODE -ne 0) { throw "configuring snoretoast failed" } + cmake --build $bld + if ($LASTEXITCODE -ne 0) { throw "building snoretoast failed" } + cmake --install $bld + if ($LASTEXITCODE -ne 0) { throw "installing snoretoast failed" } + + # Final CMAKE_PREFIX_PATH for the following steps: vcpkg tree + these tools. + Add-Content -Path $env:GITHUB_ENV -Value "CMAKE_PREFIX_PATH=$env:VCPKG_TRIPLET_ROOT;$depsRoot" + - name: Setup MSYS2 (MinGW) if: ${{ inputs.msystem != '' }} uses: msys2/setup-msys2@v2 @@ -50,6 +177,7 @@ runs: update: true install: >- git + gperf pacboy: >- toolchain:p ccache:p @@ -60,9 +188,53 @@ runs: qt${{ inputs.qt-major }}-imageformats:p qt${{ inputs.qt-major }}-networkauth:p qt${{ inputs.qt-major }}-tools:p + qt${{ inputs.qt-major }}-declarative:p gpgme:p qgpgme:p gpgmepp:p + gettext:p + zlib:p + bzip2:p + xz:p + libxml2:p + cmark:p + gpgme:p + qrencode:p + libarchive:p + libxslt:p + docbook-xml:p + docbook5-xml:p + docbook-xsl:p + kf6:p + + # kdoctools needs the DocBook XML DTD and the DocBook XSL stylesheets. MSVC has no package + # for them, and KDE's CMake modules only search system prefixes, so the locations are handed + # to kdoctools explicitly (see framework-cmake-args in setup-dependencies/action.yml). + - name: Install DocBook (MSVC) + if: ${{ inputs.msystem == '' }} + shell: pwsh + run: | + $ErrorActionPreference = "Stop" + $tmp = $env:RUNNER_TEMP + + Invoke-WebRequest -Uri "https://www.oasis-open.org/docbook/xml/4.5/docbook-xml-4.5.zip" -OutFile "$tmp\docbook-xml-4.5.zip" + Expand-Archive -Path "$tmp\docbook-xml-4.5.zip" -DestinationPath "C:\dbk45" -Force + + Invoke-WebRequest -Uri "https://github.com/docbook/xslt10-stylesheets/releases/download/snapshot%2F2020-06-03/docbook-xsl-snapshot.zip" -OutFile "$tmp\docbook-xsl-snapshot.zip" + Expand-Archive -Path "$tmp\docbook-xsl-snapshot.zip" -DestinationPath "C:\dbksl" -Force + + # The stylesheets' root is the directory that contains lib\lib.xsl. + $libXsl = Get-ChildItem -Path "C:\dbksl" -Recurse -Filter "lib.xsl" | + Where-Object { $_.Directory.Name -eq "lib" } | Select-Object -First 1 + if (-not $libXsl) { throw "lib\lib.xsl not found in the DocBook XSL archive" } + $xslDir = $libXsl.Directory.Parent.FullName + if (-not (Test-Path "C:\dbk45\docbookx.dtd")) { throw "docbookx.dtd not found in C:\dbk45" } + + Add-Content -Path $env:GITHUB_ENV -Value "XML_CATALOG_FILES=C:\dbk45\catalog.xml" + Add-Content -Path $env:GITHUB_ENV -Value "DocBookXML4_ROOT=C:\dbk45" + $kdocArgs = "-DDocBookXML4_DTD_DIR=C:/dbk45 -DDocBookXSL_DIR=$($xslDir -replace '\\','/')" + Add-Content -Path $env:GITHUB_ENV -Value "KF6_KDOCTOOLS_ARGS=$kdocArgs" + Write-Host "kdoctools arguments: $kdocArgs" - name: List pacman packages (MinGW) if: ${{ inputs.msystem != '' }} diff --git a/.github/actions/setup-kf6/README.md b/.github/actions/setup-kf6/README.md new file mode 100644 index 000000000..276db88a6 --- /dev/null +++ b/.github/actions/setup-kf6/README.md @@ -0,0 +1,181 @@ +# setup-kf6 + +A general purpose composite action that builds and caches +[KDE Frameworks 6](https://api.kde.org/frameworks/) from the official release +tarballs against a Qt 6 you already installed (for example with +[`jurplel/install-qt-action`](https://github.com/jurplel/install-qt-action) / +aqtinstall). It does not depend on vcpkg, Conan, distro packages or anything +else in this repository, so it can be used from other projects as +`uses: //.github/actions/setup-kf6@`. + +Works on GitHub-hosted Linux, macOS and Windows (MSVC) runners. + +## Usage + +```yaml +- uses: jurplel/install-qt-action@v4 + with: + version: 6.12.0 + modules: qtshadertools qt5compat # see "Qt requirements" + +# Everything KDE Frameworks offers on this platform: +- uses: ./.github/actions/setup-kf6 + +# ...or only what you need. Dependencies are found and added for you: +- uses: ./.github/actions/setup-kf6 + with: + frameworks: kconfig kcoreaddons ki18n kio +``` + +On Windows enter the Visual Studio developer environment first +(`ilammy/msvc-dev-cmd`); the action checks that `cl` is on `PATH`. + +Afterwards `CMAKE_PREFIX_PATH`, `PATH`, `QT_PLUGIN_PATH`, `QML_IMPORT_PATH`, +`XDG_DATA_DIRS` and `KF6_PREFIX` are exported (turn off with +`export-env: false`), so a plain `find_package(KF6Config REQUIRED)` works. + +## How it works + +1. **Plan.** The release listing for the requested version is read from + download.kde.org, tarballs are downloaded and verified against the published + SHA-256. Nothing is hard-coded per framework; the plan is derived from what + ships in each tarball: + + | source | used for | + |---|---| + | the CMake files (`find_package(KF6... REQUIRED)`) | **hard** dependencies: what the framework really needs | + | `.kde-ci.yml` and optional `find_package`s | **soft** dependencies: only affect build order | + | `metainfo.yaml` | declared tier, supported platforms | + + `.kde-ci.yml` alone is not reliable: it only lists what KDE's CI exercises per + platform, so it sometimes lists more than is needed (documentation tooling + such as `kdoctools` for `kpackage`) and sometimes less (`kcmutils` has no + macOS entry although it requires KIO there). Hard dependencies are pulled + in and a failure blocks whatever needs them; a failed soft dependency does + not. A hard dependency that does not exist on the platform is treated as soft + (it is nearly always guarded by a platform check). + + * No `frameworks` input: every framework supported on the runner's platform + is built (frameworks KDE does not support there, e.g. `baloo` on Windows, + are skipped, as is anything that hard-depends on a skipped framework). + * `frameworks: a b c`: the hard dependency tree is walked from those and only + the closure is built. Asking for something unsupported on the platform is + an error with a message saying who needed it. +2. **Tiers.** A framework's tier is the one KDE declares, raised to the tier of + anything it depends on if the declaration is missing or inconsistent. Tier 1 + is built before tier 2 before tier 3 (`frameworkintegration` is the only + tier 4 framework). Within a tier frameworks are ordered by dependency depth, + so every dependency is always installed before it is needed. +3. **System packages.** Build tools and libraries needed by the planned + frameworks (gperf, gettext, bison, X11/Wayland headers, libical, ...) are + installed with apt / brew / choco. Only packages for frameworks in the plan + are installed. Disable with `system-deps: false`. +4. **Build.** Each framework is configured, built and installed into a single + prefix, tier by tier, with `BUILD_TESTING=OFF`, `BUILD_QCH=OFF`, shared libs + (`linkage: shared`) and `Release`. +5. **Cache.** Every tier is cached on its own as the *files that tier added* to + the prefix, together with a manifest saying which frameworks it contains and + how each was built (a *fingerprint*: tarball, its CMake arguments and the + fingerprints of what it depends on). The key of tier N covers KF version, + Qt version, OS/arch, build type, linkage, CMake arguments, the install prefix + and exactly the frameworks of tiers 1..N. Consequences: + * if tier 3 fails, tiers 1 and 2 are already saved and the next run starts at + tier 3; + * **a tier in which something failed is saved too** (under a `-partial-` key), + and the next run restores it and builds only what is missing. A tier can take + half an hour on a Windows runner, so fixing one framework does not redo the rest; + * the cache is found even if settings changed (restore-keys), but a framework + is only trusted if its fingerprint still matches, anything else is rebuilt. + Changing the arguments of one framework therefore only rebuilds it and what + depends on it; + * asking for a different subset of frameworks reuses every tier that is + identical. + +## Inputs + +| input | default | | +|---|---|---| +| `version` | `6.31.0` | KDE Frameworks release, or `latest` | +| `frameworks` | _(all)_ | what to build; names like `kio`, `KF6KIO`, `KF6::KIO`; dependencies are added | +| `exclude` | | frameworks to leave out | +| `qt-prefix` | `$QT_ROOT_DIR` / `$Qt6_DIR` | Qt 6 install (set by install-qt-action) | +| `qt-host-path` | | host Qt for cross builds (`QT_HOST_PATH`) | +| `install-prefix` | `/kf6` | where KDE Frameworks is installed | +| `build-type` | `Release` | | +| `linkage` | `shared` | `shared` or `static` | +| `generator` | `Ninja` | | +| `osx-architectures` | | e.g. `x86_64;arm64` (see Limitations) | +| `cmake-args` | | extra arguments for every framework (come after the defaults) | +| `framework-cmake-args` | | `: ` per line | +| `system-deps` | `true` | install needed system packages | +| `extra-system-packages` | | additional native package names | +| `cache` | `true` | per-tier caching | +| `cache-key-suffix` | | change to force a rebuild | +| `keep-going` | `false` | build everything that does not depend on a failure, report all failures at the end | +| `ignore-failures` | `false` | downgrade failures to a warning | +| `export-env` | `true` | export `CMAKE_PREFIX_PATH`, `PATH`, ... | +| `jobs` | CPU count | parallel jobs | +| `verbose` | `false` | stream full build output into the job log | +| `upload-logs` | `true` | upload per-framework logs as an artifact on failure | +| `base-url`, `work-dir` | | mirror and scratch directory | + +Outputs: `prefix`, `frameworks` (what was built), `cache-hit`, `failed`, +`kf-version`. + +The job summary shows the plan and a per-framework result table +(built / cached / failed / skipped). When something fails the CMake/compiler +error lines and the end of the log are printed in the job log, and all logs are +uploaded as an artifact. + +> While bringing up a new platform set `keep-going: true`: one run then tells +> you every framework that fails instead of only the first. + +## Qt requirements + +The frameworks need these Qt modules; with aqtinstall the base install covers +most of them, add the rest with `modules:`: + +* the base install: Core, Gui, Widgets, Network, DBus, Xml, Concurrent, Sql, Test, + PrintSupport, Qml, Quick, QuickControls2, QuickWidgets, Svg, LinguistTools +* `qtshadertools`: `kirigami`, `kdeclarative`, `kquickcharts` compile Qt Quick shaders +* `qtspeech` (Qt TextToSpeech): required by `ktexteditor`; enables speech support + in `ktextwidgets` +* `qtmultimedia`: only enables barcode *scanning* in `prison` + +Optional features that need a Qt module are switched off automatically when that +module is not installed (`ktextwidgets` without `qtspeech`, `prison` without +`qtmultimedia`). If a framework cannot be configured because a Qt module is +missing, the log says which one (with its aqtinstall name). + +## Limitations / things to know + +* **macOS universal builds do not work with Homebrew libraries.** They are + single-architecture (`ld: ignoring file libintl.dylib: found architecture 'arm64', + required architecture 'x86_64'`), so with `osx-architectures: x86_64;arm64` every + framework that links one fails: `ki18n` (libintl), `kcalendarcore` (libical), + `prison` (qrencode, libdmtx, zxing-cpp), `karchive` (zstd). Build those libraries as + universal binaries yourself and point `CMAKE_PREFIX_PATH` at them, or build native. +* **macOS install names** embed the absolute install prefix; bundling tools + (macdeployqt, dylibbundler) take care of that when you package an app. +* **Windows (MSVC).** The libraries are not found by name, so the caller has to provide + them and put their prefix into `CMAKE_PREFIX_PATH` (an exported `CMAKE_PREFIX_PATH` is + honoured; `PATH` alone is *not* enough for CMake to find libraries). Needed: + gettext/`libintl`, gperf, bison/flex, libical, libqrencode + libdmtx + zxing-cpp + (for `prison`), libxml2 + libxslt, and the DocBook DTD/XSL (for `kdoctools`; pass + their location with `framework-cmake-args`). `knotifications` needs + [SnoreToast](https://github.com/KDE/snoretoast) (`LibSnoreToast`) there, and `kio`, + `kjobwidgets` and `kwallet` need `knotifications`. The action itself handles the + Windows specifics it can: breeze-icons' symlinks are unpacked the way Git for Windows + checks them out without symlink support (small files holding the target's name, which + is what its tools expect; real links and copies both break them), other tarballs get + copies, `\bin` is put on `PATH` while building (programs run during the + build, e.g. `meinproc6`, need the DLLs of the frameworks installed so far), Windows' + own `tar` is used for app templates (Git's GNU tar mistakes `D:/...` for a host), and + zstd support in `karchive` is switched off when pkg-config cannot see `libzstd`. The + installed breeze theme then lacks the alias icons (the placeholder files are not + icons); the embedded icon library does include them. +* Frameworks whose CMake project needs libraries that are not packaged on the + runner will fail in `all` mode; use `exclude`, `frameworks`, or + `extra-system-packages`. +* Optional features of a framework (image formats, spell checkers, ...) are + enabled only if their libraries are present when it is configured. diff --git a/.github/actions/setup-kf6/action.yml b/.github/actions/setup-kf6/action.yml new file mode 100644 index 000000000..a104c9763 --- /dev/null +++ b/.github/actions/setup-kf6/action.yml @@ -0,0 +1,298 @@ +name: Setup KDE Frameworks 6 +description: >- + Build and cache KDE Frameworks 6 from the official release tarballs against an + already installed Qt 6 (e.g. from aqtinstall). With no inputs it builds every + framework the runner's platform supports; with `frameworks` it analyses the + dependency tree and builds only what is needed. Tier by tier, each tier cached. + +inputs: + version: + description: >- + KDE Frameworks release to build, e.g. 6.31.0, or `latest`. + required: false + default: 6.31.0 + frameworks: + description: >- + Frameworks to build (comma/space/newline separated). Names are accepted as + `kio`, `KIO`, `KF6KIO` or `KF6::KIO`. Their dependencies are resolved and + added automatically, using the dependency data KDE ships in each release + (`.kde-ci.yml`) for the runner's platform. Empty or `all` builds every + framework available on this platform. + required: false + default: "" + exclude: + description: >- + Frameworks to leave out. In `all` mode, anything that depends on an excluded + framework is dropped too; with an explicit `frameworks` list it is an error + if a required framework is excluded. + required: false + default: "" + qt-prefix: + description: >- + Qt 6 installation (the directory containing lib/cmake/Qt6). Defaults to + $QT_ROOT_DIR / $Qt6_DIR, which jurplel/install-qt-action sets. + required: false + default: "" + qt-host-path: + description: Host Qt for cross builds (passed as QT_HOST_PATH), e.g. Windows on ARM. + required: false + default: "" + install-prefix: + description: Where KDE Frameworks is installed. Defaults to `/kf6`. + required: false + default: "" + build-type: + description: CMake build type for the frameworks. + required: false + default: Release + linkage: + description: "`shared` or `static`." + required: false + default: shared + generator: + description: CMake generator. The frameworks are single-config builds, so use Ninja. + required: false + default: Ninja + osx-architectures: + description: >- + CMAKE_OSX_ARCHITECTURES, e.g. `x86_64;arm64` for a universal build. Empty + uses the compiler's default (or $CMAKE_OSX_ARCHITECTURES). Note that + Homebrew libraries are single-architecture, which a universal link of + frameworks needing them (ki18n -> libintl) cannot use. + required: false + default: "" + cmake-args: + description: >- + Extra CMake arguments for every framework, one or more per line (shell + quoting; use forward slashes in paths). They come after the defaults so + they can override them. + required: false + default: "" + framework-cmake-args: + description: >- + Extra CMake arguments for single frameworks, one `: ` per + line, e.g. `kio: -DKIO_FOO=ON`. + required: false + default: "" + system-deps: + description: >- + Install the system packages the selected frameworks need (apt / brew / + choco, chosen from the dependency analysis). Set to false if your + environment already provides them. + required: false + default: "true" + extra-system-packages: + description: Additional packages (native package names) to install along with the system dependencies. + required: false + default: "" + cache: + description: >- + Cache each tier separately (key covers KF version, Qt version, OS/arch, + build options and everything built up to that tier). + required: false + default: "true" + cache-key-suffix: + description: Extra text mixed into the cache keys, to force a rebuild. + required: false + default: "" + keep-going: + description: >- + Do not stop at the first framework that fails: build everything that does + not depend on it, then fail at the end with a full report. Useful while + bringing up a new platform. + required: false + default: "false" + ignore-failures: + description: Report failed frameworks as a warning instead of failing the action. + required: false + default: "false" + export-env: + description: >- + Export CMAKE_PREFIX_PATH, PATH, QT_PLUGIN_PATH, QML_IMPORT_PATH, + XDG_DATA_DIRS and KF6_PREFIX for the following steps. + required: false + default: "true" + jobs: + description: Parallel build jobs. 0 means the number of CPUs. + required: false + default: "0" + base-url: + description: Mirror/base URL of the release archive. + required: false + default: https://download.kde.org/stable/frameworks + work-dir: + description: Scratch directory for downloads, sources, builds and logs. + required: false + default: ${{ runner.temp }}/setup-kf6 + verbose: + description: Stream the full configure/build output into the job log instead of only writing it to the log files. + required: false + default: "false" + upload-logs: + description: Upload the per-framework build logs as an artifact when the action fails. + required: false + default: "true" + +outputs: + prefix: + description: Installation prefix of KDE Frameworks. + value: ${{ steps.finish.outputs.prefix }} + frameworks: + description: Comma separated list of the frameworks that were built (selection plus dependencies). + value: ${{ steps.plan.outputs.frameworks }} + cache-hit: + description: "`true` if every tier was restored from cache." + value: ${{ steps.finish.outputs.cache-hit }} + failed: + description: Comma separated list of frameworks that failed or were skipped (only with keep-going/ignore-failures). + value: ${{ steps.finish.outputs.failed }} + kf-version: + description: The KDE Frameworks version that was built. + value: ${{ steps.plan.outputs.kf-version }} + +runs: + using: composite + + steps: + - name: Locate Python + shell: bash + run: | + PY="$(command -v python3 || command -v python)" + if [ -z "$PY" ]; then + echo "::error::setup-kf6 needs Python 3 on PATH" + exit 1 + fi + echo "KF6_PYTHON=$PY" >> "$GITHUB_ENV" + echo "KF6_SCRIPT=${{ github.action_path }}/kf6.py" >> "$GITHUB_ENV" + + - name: Plan KDE Frameworks build + id: plan + shell: bash + env: + KF6_INPUTS: ${{ toJSON(inputs) }} + run: '"$KF6_PYTHON" "$KF6_SCRIPT" plan' + + - name: Install system dependencies + if: ${{ inputs.system-deps == 'true' }} + shell: bash + env: + KF6_INPUTS: ${{ toJSON(inputs) }} + run: '"$KF6_PYTHON" "$KF6_SCRIPT" sysdeps' + + # ---------------------------------------------------------------- Tier 1 + - name: Restore KDE Frameworks tier 1 + if: ${{ inputs.cache == 'true' && steps.plan.outputs.count-1 != '0' }} + uses: actions/cache/restore@v6.1.0 + with: + path: ${{ steps.plan.outputs.work-dir }}/delta/tier1 + key: ${{ steps.plan.outputs.key-1 }} + # Fall back to any earlier cache of this tier (finished or partial); kf6.py then only + # trusts the frameworks that were built exactly like they would be now. + restore-keys: ${{ steps.plan.outputs.restore-1 }} + + - name: Build KDE Frameworks tier 1 + id: tier1 + if: ${{ steps.plan.outputs.count-1 != '0' }} + shell: bash + env: + KF6_INPUTS: ${{ toJSON(inputs) }} + run: '"$KF6_PYTHON" "$KF6_SCRIPT" build 1' + + - name: Save KDE Frameworks tier 1 + if: ${{ !cancelled() && steps.tier1.outputs.save == 'true' }} + uses: actions/cache/save@v6.1.0 + with: + path: ${{ steps.plan.outputs.work-dir }}/delta/tier1 + key: ${{ steps.tier1.outputs.save-key }} + + # ---------------------------------------------------------------- Tier 2 + - name: Restore KDE Frameworks tier 2 + if: ${{ inputs.cache == 'true' && steps.plan.outputs.count-2 != '0' }} + uses: actions/cache/restore@v6.1.0 + with: + path: ${{ steps.plan.outputs.work-dir }}/delta/tier2 + key: ${{ steps.plan.outputs.key-2 }} + restore-keys: ${{ steps.plan.outputs.restore-2 }} + + - name: Build KDE Frameworks tier 2 + id: tier2 + if: ${{ steps.plan.outputs.count-2 != '0' }} + shell: bash + env: + KF6_INPUTS: ${{ toJSON(inputs) }} + run: '"$KF6_PYTHON" "$KF6_SCRIPT" build 2' + + - name: Save KDE Frameworks tier 2 + if: ${{ !cancelled() && steps.tier2.outputs.save == 'true' }} + uses: actions/cache/save@v6.1.0 + with: + path: ${{ steps.plan.outputs.work-dir }}/delta/tier2 + key: ${{ steps.tier2.outputs.save-key }} + + # ---------------------------------------------------------------- Tier 3 + - name: Restore KDE Frameworks tier 3 + if: ${{ inputs.cache == 'true' && steps.plan.outputs.count-3 != '0' }} + uses: actions/cache/restore@v6.1.0 + with: + path: ${{ steps.plan.outputs.work-dir }}/delta/tier3 + key: ${{ steps.plan.outputs.key-3 }} + restore-keys: ${{ steps.plan.outputs.restore-3 }} + + - name: Build KDE Frameworks tier 3 + id: tier3 + if: ${{ steps.plan.outputs.count-3 != '0' }} + shell: bash + env: + KF6_INPUTS: ${{ toJSON(inputs) }} + run: '"$KF6_PYTHON" "$KF6_SCRIPT" build 3' + + - name: Save KDE Frameworks tier 3 + if: ${{ !cancelled() && steps.tier3.outputs.save == 'true' }} + uses: actions/cache/save@v6.1.0 + with: + path: ${{ steps.plan.outputs.work-dir }}/delta/tier3 + key: ${{ steps.tier3.outputs.save-key }} + + # ---------------------------------------------------------------- Tier 4 + # KDE declares a single framework (frameworkintegration) in tier 4. + - name: Restore KDE Frameworks tier 4 + if: ${{ inputs.cache == 'true' && steps.plan.outputs.count-4 != '0' }} + uses: actions/cache/restore@v6.1.0 + with: + path: ${{ steps.plan.outputs.work-dir }}/delta/tier4 + key: ${{ steps.plan.outputs.key-4 }} + restore-keys: ${{ steps.plan.outputs.restore-4 }} + + - name: Build KDE Frameworks tier 4 + id: tier4 + if: ${{ steps.plan.outputs.count-4 != '0' }} + shell: bash + env: + KF6_INPUTS: ${{ toJSON(inputs) }} + run: '"$KF6_PYTHON" "$KF6_SCRIPT" build 4' + + - name: Save KDE Frameworks tier 4 + if: ${{ !cancelled() && steps.tier4.outputs.save == 'true' }} + uses: actions/cache/save@v6.1.0 + with: + path: ${{ steps.plan.outputs.work-dir }}/delta/tier4 + key: ${{ steps.tier4.outputs.save-key }} + + # ---------------------------------------------------------------- Finish + - name: Finish KDE Frameworks setup + id: finish + if: ${{ !cancelled() && steps.plan.outcome == 'success' }} + shell: bash + env: + KF6_INPUTS: ${{ toJSON(inputs) }} + run: '"$KF6_PYTHON" "$KF6_SCRIPT" finish' + + - name: Upload KDE Frameworks build logs + if: ${{ failure() && inputs.upload-logs == 'true' && steps.plan.outcome == 'success' }} + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: kf6-logs-${{ steps.plan.outputs.key-base }} + path: ${{ steps.plan.outputs.work-dir }}/logs + overwrite: true + if-no-files-found: ignore diff --git a/.github/actions/setup-kf6/kf6.py b/.github/actions/setup-kf6/kf6.py new file mode 100644 index 000000000..6665e25ee --- /dev/null +++ b/.github/actions/setup-kf6/kf6.py @@ -0,0 +1,1447 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Driver for the ``setup-kf6`` composite action. + +Builds KDE Frameworks 6 from the official release tarballs against an already +installed Qt 6 (for example one installed by aqtinstall), in dependency order. + +Sub-commands (each is one workflow step, they share ``/plan.json``): + + plan resolve the selection, download + verify tarballs, analyse the + dependency tree, group frameworks into tiers, compute cache keys + sysdeps install the system packages the planned frameworks need + build build one tier (or restore it from its cache delta) + finish export the environment, write the summary, set outputs + +Dependency analysis +------------------- +KDE publishes the dependency data its own CI uses inside every release +tarball, so nothing here is hard-coded per framework: + +* ``.kde-ci.yml`` build-time dependencies, per platform (Linux/Windows/macOS) +* ``metainfo.yaml`` declared tier + supported platforms + +The tier of a framework (the unit that is cached) is the tier KDE declares, +raised to the tier of anything it depends on if the declaration is missing or +inconsistent. Inside a tier, frameworks are ordered by dependency depth, so +(tier, depth, name) is always a valid topological build order. + +Inputs arrive as JSON in ``$KF6_INPUTS`` (``${{ toJSON(inputs) }}``), see +``action.yml`` for their documentation. The script has no third party +dependencies and runs on the Linux, macOS and Windows GitHub runners. +""" + +from __future__ import annotations + +import concurrent.futures +import contextlib +import difflib +import hashlib +import json +import os +import platform +import re +import shlex +import shutil +import subprocess +import sys +import tarfile +import time +import urllib.error +import urllib.request +from dataclasses import asdict, dataclass +from pathlib import Path + +# Tiers are cached separately. KDE declares tiers 1..3 (frameworkintegration +# declares 4); anything deeper is folded into the last group, which is safe +# because order inside a group is still topological. +MAX_GROUP = 4 + +# Part of every cache key, see cmd_plan. Bump it when cached data must not be reused. +CACHE_FORMAT = 3 + +# Release tarballs that are not CMake projects. +NOT_BUILDABLE = {"kapidox"} + +ECM = "extra-cmake-modules" +ALIASES = {"ecm": ECM} + +PLATFORM_ALIASES = { + "Linux": {"Linux"}, + "macOS": {"macOS", "MacOSX", "Mac OS X"}, + "Windows": {"Windows"}, +} + +# Defaults every framework is configured with. User supplied ``cmake-args`` +# come afterwards, so they can override anything here. +BASE_CMAKE_DEFS = [ + "-DBUILD_TESTING=OFF", + "-DBUILD_QCH=OFF", + "-DBUILD_EXAMPLES=OFF", + "-DBUILD_PYTHON_BINDINGS=OFF", + # Keep plugins/QML inside our prefix instead of writing into the Qt install. + "-DKDE_INSTALL_USE_QT_SYS_PATHS=OFF", +] + +# Per-framework defaults (before user supplied ``framework-cmake-args``). +FRAMEWORK_CMAKE_DEFS = { + # 24x24 icon generation needs Python + the lxml module; the 22px/32px + # icons are shipped pre-rendered so nothing essential is lost. + "breeze-icons": ["-DWITH_ICON_GENERATION=OFF"], + # The secret service daemons need Qca, which distro/Homebrew packages build against + # *their* Qt, not the one used here, and a daemon is not something a bundled prefix + # can run anyway. The KF6Wallet client library is unaffected. + "kwallet": ["-DBUILD_KSECRETD=OFF", "-DBUILD_KWALLETD=OFF"], +} + +# Optional features that need a Qt module. Switched off when that module is not +# installed, so a base aqtinstall Qt works without extra modules: +# framework -> (CMake option, Qt module). +QT_MODULE_FEATURES = { + "ktextwidgets": ("WITH_TEXT_TO_SPEECH", "TextToSpeech"), # qtspeech + "prison": ("WITH_MULTIMEDIA", "Multimedia"), # qtmultimedia +} + +# System packages by platform. "*" is always installed, other keys only when +# the framework is part of the plan. Names are apt / brew / choco packages. +_X11 = [ + "libx11-dev", "libx11-xcb-dev", "libxcb1-dev", "libxcb-keysyms1-dev", + "libxcb-xkb-dev", "libxcb-icccm4-dev", "libxcb-image0-dev", "libxcb-util-dev", + "libxcb-res0-dev", "libxcb-record0-dev", "libxcb-screensaver0-dev", + "libxcb-sync-dev", "libxcb-randr0-dev", "libxcb-xtest0-dev", + "libxkbcommon-x11-dev", "libxext-dev", "libxfixes-dev", "libxrender-dev", +] +_WAYLAND = ["libwayland-dev", "wayland-protocols", "plasma-wayland-protocols"] +_XML = ["libxml2-dev", "libxslt1-dev"] +SYSTEM_DEPS = { + "Linux": { + "*": ["gperf", "gettext", "bison", "flex", "pkg-config", "perl", + "libgl1-mesa-dev", "libxkbcommon-dev"], + "kcoreaddons": ["libmount-dev", "libudev-dev"], + "solid": ["libudev-dev", "libmount-dev"], + "kwindowsystem": _X11 + _WAYLAND, + "kguiaddons": _X11 + _WAYLAND, + "kidletime": _X11 + _WAYLAND, + "kglobalaccel": _X11, + "kcrash": _X11, + "kdesu": _X11 + ["libutempter-dev"], + "kstatusnotifieritem": _X11, + "qqc2-desktop-style": _X11, + "kpty": ["libutempter-dev"], + "knotifications": ["libcanberra-dev"], + "kdoctools": _XML + ["docbook-xml", "docbook-xsl", "libxml2-utils", "xsltproc", "liburi-perl"], + "kio": _XML + ["libacl1-dev", "libmount-dev"], + "kcalendarcore": ["libical-dev"], + "kdnssd": ["libavahi-client-dev"], + "kfilemetadata": ["libattr1-dev"], + "baloo": ["liblmdb-dev"], + "networkmanager-qt": ["libnm-dev"], + "modemmanager-qt": ["libmm-glib-dev"], + "prison": ["libqrencode-dev", "libdmtx-dev", "libzxing-dev"], + # Deliberately absent: libqca-qt6-dev, libpolkit-qt6-1-dev, ... Packages that + # link Qt are built against the distro's Qt, which must not be mixed with the + # aqtinstall Qt used here. KAuth falls back to its own backends without polkit. + }, + "macOS": { + "*": ["gettext", "gperf", "bison", "flex", "pkgconf"], + "kcalendarcore": ["libical"], + # libxml2/libxslt come with the macOS SDK. kdoctools' own CMake module knows + # Homebrew's DocBook layout (opt/docbook/docbook/xml/4.5). + "kdoctools": ["docbook", "docbook-xsl"], + "prison": ["qrencode", "libdmtx", "zxing-cpp"], + }, + "Windows": { + "*": ["gperf", "winflexbison3", "gettext"], + }, +} + + +# -------------------------------------------------------------------------- +# small helpers +# -------------------------------------------------------------------------- + +IN_CI = os.environ.get("GITHUB_ACTIONS") == "true" + + +def log(msg: str = "") -> None: + print(msg, flush=True) + + +def notice(msg: str) -> None: + log(f"::notice::{msg}" if IN_CI else f"notice: {msg}") + + +def warning(msg: str) -> None: + log(f"::warning::{msg}" if IN_CI else f"warning: {msg}") + + +def die(msg: str) -> None: + log(f"::error::{msg}" if IN_CI else f"error: {msg}") + sys.exit(1) + + +@contextlib.contextmanager +def group(title: str): + log(f"::group::{title}" if IN_CI else f"--- {title}") + try: + yield + finally: + if IN_CI: + log("::endgroup::") + + +def append_file(env_var: str, text: str) -> None: + target = os.environ.get(env_var) + if not target: + log(f"[{env_var}] {text.rstrip()}") + return + with open(target, "a", encoding="utf-8") as fh: + fh.write(text if text.endswith("\n") else text + "\n") + + +def set_output(key: str, value: str) -> None: + append_file("GITHUB_OUTPUT", f"{key}={value}") + + +def truthy(value) -> bool: + return str(value).strip().lower() in {"1", "true", "yes", "on"} + + +def current_platform() -> str: + override = os.environ.get("KF6_PLATFORM_OVERRIDE") # for testing the planner + if override: + return override + return {"Linux": "Linux", "Darwin": "macOS", "Windows": "Windows"}.get( + platform.system(), platform.system() + ) + + +def normalized_arch() -> str: + machine = platform.machine().lower() + return {"amd64": "x86_64", "aarch64": "arm64"}.get(machine, machine) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def http_get(url: str, retries: int = 4, fatal: bool = True) -> bytes | None: + last: Exception | None = None + for attempt in range(retries): + try: + req = urllib.request.Request(url, headers={"User-Agent": "setup-kf6"}) + with urllib.request.urlopen(req, timeout=120) as resp: + return resp.read() + except (urllib.error.URLError, TimeoutError, ConnectionError) as exc: + last = exc + time.sleep(2 * (attempt + 1)) + if fatal: + die(f"Could not download {url}: {last}") + return None + + +def split_list(value: str) -> list[str]: + return [t for t in re.split(r"[\s,;]+", value or "") if t] + + +# -------------------------------------------------------------------------- +# configuration +# -------------------------------------------------------------------------- + + +class Config: + """Typed view over the action inputs.""" + + def __init__(self) -> None: + raw = os.environ.get("KF6_INPUTS", "{}") + self.raw: dict = json.loads(raw) + self.version = self.get("version", "6.31.0") + self.frameworks = self.get("frameworks") + self.exclude = split_list(self.get("exclude")) + self.build_type = self.get("build-type", "Release") + self.linkage = self.get("linkage", "shared").lower() + self.cmake_args = self.get("cmake-args") + self.framework_cmake_args = self.get("framework-cmake-args") + self.osx_architectures = self.get("osx-architectures") + self.generator = self.get("generator", "Ninja") + self.cache = truthy(self.get("cache", "true")) + self.cache_key_suffix = self.get("cache-key-suffix") + self.system_deps = truthy(self.get("system-deps", "true")) + self.extra_system_packages = split_list(self.get("extra-system-packages")) + self.keep_going = truthy(self.get("keep-going", "false")) + self.ignore_failures = truthy(self.get("ignore-failures", "false")) + self.export_env = truthy(self.get("export-env", "true")) + self.verbose = truthy(self.get("verbose", "false")) + self.base_url = self.get("base-url", "https://download.kde.org/stable/frameworks").rstrip("/") + self.jobs = int(self.get("jobs", "0") or 0) or (os.cpu_count() or 2) + self.qt_prefix_input = self.get("qt-prefix") + self.qt_host_path = self.get("qt-host-path") + + work = self.get("work-dir") or os.path.join( + os.environ.get("RUNNER_TEMP", str(Path.home() / ".cache")), "setup-kf6" + ) + self.work = Path(work).resolve() + prefix = self.get("install-prefix") or str(self.work.parent / "kf6") + self.prefix = Path(prefix).resolve() + + if self.linkage not in {"shared", "static"}: + die(f"linkage must be 'shared' or 'static', got '{self.linkage}'") + + def get(self, name: str, default: str = "") -> str: + value = self.raw.get(name) + return default if value is None or str(value).strip() == "" else str(value).strip() + + +# -------------------------------------------------------------------------- +# release metadata +# -------------------------------------------------------------------------- + + +@dataclass +class Framework: + name: str + tarball: str + sha256: str + src: str # extracted source directory + declared_tier: int | None + platforms: list[str] + deps: list[str] # hard: find_package(... REQUIRED) in CMake (incl. ECM). Pulled in, failure blocks us. + soft_deps: list[str] # optional in CMake, or only named by .kde-ci.yml. Ordering only. + external: list[str] # non-framework KDE dependencies named by .kde-ci.yml + buildable: bool + cmakename: str = "" # e.g. KF6KIO + fingerprint: str = "" # identifies how this framework is built (see cmd_plan) + tier: int = 0 # computed + group: int = 0 # cache group, 1..MAX_GROUP + + def supports(self, plat: str) -> bool: + wanted = PLATFORM_ALIASES.get(plat, {plat}) + return any(p == "All" or p in wanted for p in self.platforms) + + +def parse_metainfo(text: str) -> tuple[int | None, list[str]]: + tier = None + m = re.search(r"^tier:\s*(\d+)", text, re.M) + if m: + tier = int(m.group(1)) + platforms: list[str] = [] + in_platforms = False + for line in text.splitlines(): + if re.match(r"^platforms:\s*$", line): + in_platforms = True + continue + if in_platforms: + if line and not line[0].isspace(): + break + m = re.match(r"^\s*-\s*name:\s*(.+?)\s*$", line) + if m: + platforms.append(m.group(1).strip("'\"")) + return tier, platforms + + +def parse_kde_ci(text: str, plat: str) -> tuple[list[str], list[str]]: + """Return (framework deps, other KDE deps) that apply to ``plat``. + + Understands the subset of YAML KDE uses for the ``Dependencies`` block:: + + Dependencies: + - 'on': ['Linux', 'Windows'] + 'require': + 'frameworks/kconfig': '@same' + """ + wanted = PLATFORM_ALIASES.get(plat, {plat}) + frameworks: list[str] = [] + external: list[str] = [] + in_deps = False + applies = False + in_require = False + for raw in text.splitlines(): + line = re.sub(r"\s+#.*$", "", raw).rstrip() + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + if not line[0].isspace(): # top level key + in_deps = stripped.startswith("Dependencies:") + in_require = False + applies = False + continue + if not in_deps: + continue + m = re.match(r"-\s*['\"]?on['\"]?\s*:\s*\[(.*)\]\s*$", stripped) + if m: + targets = {t.strip().strip("'\"") for t in m.group(1).split(",") if t.strip()} + applies = bool(targets & (wanted | {"@all", "All"})) + in_require = False + continue + if re.match(r"['\"]?require['\"]?\s*:\s*$", stripped): + in_require = True + continue + m = re.match(r"['\"]([^'\"]+)['\"]\s*:\s*['\"][^'\"]*['\"]\s*$", stripped) + if m and in_require: + if applies: + dep = m.group(1) + (frameworks if dep.startswith("frameworks/") else external).append(dep) + continue + warning(f"unrecognised line in .kde-ci.yml Dependencies block: {stripped!r}") + return [d.split("/", 1)[1] for d in frameworks], external + + +# Directories whose CMake projects are not part of the framework itself. +SKIP_DIRS = {"autotests", "tests", "test", "examples", "example", "demo", "demos", "templates", "po", "poqm", "doc", "docs"} +# find_package() keywords that are not component names. +_FP_KEYWORDS = { + "REQUIRED", "QUIET", "CONFIG", "NO_MODULE", "MODULE", "EXACT", "GLOBAL", "NO_POLICY_SCOPE", + "NO_DEFAULT_PATH", "CMAKE_FIND_ROOT_PATH_BOTH", "ONLY_CMAKE_FIND_ROOT_PATH", +} +_FIND_PACKAGE = re.compile(r"\bfind_package\s*\(([^)]*)\)", re.I) + + +def scan_cmake_kf_deps(src: Path) -> tuple[set[str], set[str]]: + """KF6 packages (without the ``KF6`` prefix) the CMake project asks for. + + Returns ``(required, optional)``. Understands both forms:: + + find_package(KF6Config 6.31.0 REQUIRED) + find_package(KF6 6.31.0 REQUIRED COMPONENTS CoreAddons I18n OPTIONAL_COMPONENTS Foo) + + The CMake files are the source of truth for what a framework really needs: + ``.kde-ci.yml`` only lists what KDE's CI exercises per platform, which is + sometimes more (optional documentation tooling) and sometimes less (it has + no macOS entry for kcmutils although it requires KIO there too). + """ + required: set[str] = set() + optional: set[str] = set() + for root, dirs, files in os.walk(src): + dirs[:] = [d for d in dirs if d.lower() not in SKIP_DIRS and not d.startswith(".")] + if "CMakeLists.txt" not in files: + continue + text = (Path(root) / "CMakeLists.txt").read_text(encoding="utf-8", errors="replace") + text = re.sub(r"(?m)#.*$", "", text) + for m in _FIND_PACKAGE.finditer(text): + tokens = m.group(1).split() + if not tokens: + continue + name, is_required = tokens[0], "REQUIRED" in tokens + if name == "KF6": + mode = None + for tok in tokens[1:]: + if tok == "COMPONENTS": + mode = "components" + elif tok == "OPTIONAL_COMPONENTS": + mode = "optional" + elif tok in _FP_KEYWORDS or "$" in tok or tok[0].isdigit(): + continue + elif mode == "components": + (required if is_required else optional).add(tok) + elif mode == "optional": + optional.add(tok) + elif name.startswith("KF6") and len(name) > 3 and "$" not in name: + (required if is_required else optional).add(name[3:]) + return required, optional + + +# CMake package names that do not follow the "KF6" + repository name pattern: +# KF6Su is kdesu, and some frameworks ship extra find_package()-able sub-libraries. +PACKAGE_ALIASES = { + "su": "kdesu", + "newstuffcore": "knewstuff", + "newstuffwidgets": "knewstuff", + "kirigamiplatform": "kirigami", +} + + +class Releases: + """Download, verify, extract and parse framework release tarballs.""" + + def __init__(self, cfg: Config, plat: str) -> None: + self.cfg = cfg + self.plat = plat + self.version = cfg.version + self.dir_url = "" + self.catalog: dict[str, str] = {} + self._fw: dict[str, Framework] = {} + self.downloads = cfg.work / "downloads" + self.sources = cfg.work / "src" + self.downloads.mkdir(parents=True, exist_ok=True) + self.sources.mkdir(parents=True, exist_ok=True) + self._load_catalog() + + def _load_catalog(self) -> None: + base = self.cfg.base_url + version = self.version + if version.lower() == "latest": + index = http_get(base + "/").decode("utf-8", "replace") + series = sorted( + {tuple(map(int, m.groups())) for m in re.finditer(r'href="(\d+)\.(\d+)/"', index)} + ) + if not series: + die(f"Could not find any release series at {base}/") + version = "%d.%d.0" % series[-1] + major_minor = ".".join(version.split(".")[:2]) + self.dir_url = f"{base}/{major_minor}" + index = http_get(self.dir_url + "/").decode("utf-8", "replace") + found: dict[str, set[str]] = {} + for m in re.finditer(r'href="([A-Za-z0-9._+-]+?)-(\d+\.\d+\.\d+)\.tar\.xz"', index): + found.setdefault(m.group(2), set()).add(m.group(1)) + if self.version.lower() == "latest": + version = max(found, key=lambda v: (len(found[v]), tuple(map(int, v.split(".")))), default="") + if version not in found: + avail = ", ".join(sorted(found)) or "none" + die(f"KDE Frameworks {version} not found at {self.dir_url}/ (available there: {avail})") + self.version = version + self.catalog = {n: f"{n}-{version}.tar.xz" for n in sorted(found[version])} + + def resolve_name(self, token: str) -> str: + """Map ``kio``, ``KIO``, ``KF6KIO``, ``KF6::KIO``, ``Config``, ``ecm`` ... to a repo name.""" + + def norm(s: str) -> str: + return re.sub(r"[^a-z0-9]", "", s.lower()) + + index = {norm(n): n for n in self.catalog} + for alias, target in ALIASES.items(): + index.setdefault(alias, target) + stripped = re.sub(r"^kf6(::)?", "", token, flags=re.I) + for candidate in (norm(token), norm(stripped), "k" + norm(stripped)): + if candidate in index: + return index[candidate] + close = difflib.get_close_matches(norm(stripped), list(index), n=3) + hint = f" Did you mean: {', '.join(index[c] for c in close)}?" if close else "" + die(f"Unknown framework '{token}' for KDE Frameworks {self.version}.{hint}") + raise AssertionError + + def get(self, name: str) -> Framework: + if name in self._fw: + return self._fw[name] + tarball = self.catalog[name] + path = self.downloads / tarball + url = f"{self.dir_url}/{tarball}" + if not path.exists(): + part = path.with_suffix(".part") + part.write_bytes(http_get(url)) + part.replace(path) + digest = sha256_file(path) + published = http_get(url + ".sha256", retries=2, fatal=False) + if published: + expected = published.decode().split()[0].lower() + if expected != digest: + path.unlink(missing_ok=True) + die(f"SHA-256 mismatch for {tarball}: expected {expected}, got {digest}") + else: + warning(f"no .sha256 published for {tarball}; skipping verification") + + root = f"{name}-{self.version}" + src = self.sources / root + if not (src / ".extracted").exists(): + shutil.rmtree(src, ignore_errors=True) + extract_tarball(path, self.sources, pseudo_svg_links=(name == "breeze-icons")) + (src / ".extracted").write_text("ok") + + meta = src / "metainfo.yaml" + meta_text = meta.read_text(encoding="utf-8") if meta.exists() else "" + tier, platforms = parse_metainfo(meta_text) + m = re.search(r"^cmakename:\s*(\S+)", meta_text, re.M) + cmakename = m.group(1) if m else "" + ci = src / ".kde-ci.yml" + ci_deps, external = parse_kde_ci(ci.read_text(encoding="utf-8"), self.plat) if ci.exists() else ([], []) + + required, optional = scan_cmake_kf_deps(src) if (src / "CMakeLists.txt").exists() else (set(), set()) + hard: set[str] = set() + for pkg in sorted(required): + dep = self.package_to_repo(pkg) + if dep: + hard.add(dep) + else: + log(f" note: {name} requires KF6{pkg}, which is not a KDE Frameworks release tarball (assumed provided)") + soft = {self.package_to_repo(p) for p in optional} | set(ci_deps) + if name != ECM: + hard.add(ECM) + soft -= hard | {name, None} + hard.discard(name) + fw = Framework( + name=name, + tarball=tarball, + sha256=digest, + src=str(src), + declared_tier=tier, + platforms=platforms or ["All"], + deps=sorted(hard, key=lambda d: (d != ECM, d)), + soft_deps=sorted(soft), + external=sorted(set(external)), + buildable=(src / "CMakeLists.txt").exists() and name not in NOT_BUILDABLE, + cmakename=cmakename, + ) + self._fw[name] = fw + return fw + + def package_to_repo(self, pkg: str) -> str | None: + """``Config`` / ``KIO`` / ``DocTools`` (from find_package(KF6)) -> ``kconfig`` / ``kio`` / ``kdoctools``.""" + + def norm(s: str) -> str: + return re.sub(r"[^a-z0-9]", "", s.lower()) + + key = norm(pkg) + for fw in list(self._fw.values()): # exact answers from metainfo.yaml's cmakename where known + if fw.cmakename and norm(fw.cmakename[3:] if fw.cmakename.startswith("KF6") else fw.cmakename) == key: + return fw.name + index = {norm(n): n for n in self.catalog} + for candidate in (key, "k" + key): + if candidate in index: + return index[candidate] + return PACKAGE_ALIASES.get(key) if PACKAGE_ALIASES.get(key) in self.catalog else None + + def load_all(self) -> None: + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool: + list(pool.map(self.get, list(self.catalog))) + + +def materialize_symlinks(links: list[tuple[Path, str]], pseudo_svg_links: bool = False) -> None: + """Windows: replace the symlinks of a tarball by something that works there. + + Real symlinks are unreliable on Windows (they need a privilege, and some tools mishandle them). + + * ``pseudo_svg_links``: every link (files and directories, like Git does) becomes what Git + for Windows checks symlinks out as without symlink support: a small file holding just the + target's name. breeze-icons' tools are written for exactly that. Its icon generator skips + such files, and ``qrcAlias`` resolves ``*.svg`` ones as aliases. Neither works with real + links (the generator turns them into bogus ``.lnk`` shortcuts) nor with copies (``qrcAlias`` + rejects the duplicated icons, ``places/16@3x`` is a link to the directory ``16``). + * otherwise a link is replaced by a copy of what it points to (links to links are resolved + by retrying until what they need exists). + """ + pending: list[tuple[Path, str]] = [] + for link, target in links: + if pseudo_svg_links: + link.parent.mkdir(parents=True, exist_ok=True) + link.write_text(target, encoding="utf-8") + else: + pending.append((link, target)) + for _ in range(8): + still: list[tuple[Path, str]] = [] + for link, target in pending: + resolved = link.parent / target + if not resolved.exists(): + still.append((link, target)) + continue + if resolved.is_dir(): + shutil.copytree(resolved, link, dirs_exist_ok=True) + else: + link.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(resolved, link) + pending = still + if not pending: + return + for link, target in pending: + warning(f"dangling link {link} -> {target} (skipped)") + + +def extract_tarball(path: Path, dest: Path, windows: bool | None = None, pseudo_svg_links: bool = False) -> None: + windows = (os.name == "nt") if windows is None else windows + links: list[tuple[Path, str]] = [] + with tarfile.open(path) as tar: + for member in tar: + if windows and member.issym(): + links.append((dest / member.name, member.linkname)) + continue + try: + try: + tar.extract(member, dest, filter="data") + except TypeError: # Python < 3.12 without the filter backport + tar.extract(member, dest) + except (OSError, tarfile.TarError) as exc: + if member.issym() or member.islnk(): + warning(f"could not create link {member.name}: {exc}") + else: + die(f"could not extract {member.name} from {path.name}: {exc}") + if links: + materialize_symlinks(links, pseudo_svg_links) + + +# -------------------------------------------------------------------------- +# planning +# -------------------------------------------------------------------------- + + +def resolve_selection(rel: Releases, cfg: Config) -> list[Framework]: + plat = rel.plat + explicit = split_list(cfg.frameworks) + select_all = not explicit or any(t.lower() == "all" for t in explicit) + exclude = {rel.resolve_name(t) for t in cfg.exclude} + + if select_all: + rel.load_all() + for fw in rel._fw.values(): + demote_unavailable_deps(rel, fw) + chosen = { + n: fw + for n, fw in rel._fw.items() + if fw.buildable and fw.supports(plat) and n not in exclude + } + for n, fw in sorted(rel._fw.items()): + if n in chosen: + continue + why = ( + "excluded" if n in exclude else + "not a CMake project" if not fw.buildable else + f"not supported on {plat}" + ) + log(f" skip {n}: {why}") + # Drop frameworks whose dependencies are not available. + changed = True + while changed: + changed = False + for n in list(chosen): + missing = [d for d in chosen[n].deps if d not in chosen] + if missing: + log(f" skip {n}: needs {', '.join(missing)} which is not built") + del chosen[n] + changed = True + return list(chosen.values()) + + wanted = [rel.resolve_name(t) for t in explicit] + picked: dict[str, Framework] = {} + queue = list(wanted) + required_by: dict[str, str] = {} + while queue: + name = queue.pop(0) + if name in picked: + continue + who = f" (needed by {required_by[name]})" if name in required_by else "" + if name not in rel.catalog: + die(f"'{name}'{who} is not part of KDE Frameworks {rel.version}") + fw = rel.get(name) + if name in exclude: + die(f"'{name}'{who} is excluded but required") + if not fw.buildable: + die(f"'{name}'{who} is not a CMake project and cannot be built") + if not fw.supports(plat): + die(f"'{name}'{who} is not supported on {plat} (supported: {', '.join(fw.platforms)})") + demote_unavailable_deps(rel, fw) + picked[name] = fw + # Only hard dependencies are pulled in; optional ones (docs tooling, ...) are + # used only if they were asked for too. + for dep in fw.deps: + required_by.setdefault(dep, name) + queue.append(dep) + return list(picked.values()) + + +def demote_unavailable_deps(rel: Releases, fw: Framework) -> None: + """A REQUIRED dependency that cannot exist on this platform is almost always + guarded by a platform check in the CMake file; treat it as optional.""" + keep: list[str] = [] + for dep in fw.deps: + target = rel.get(dep) if dep in rel.catalog else None + if target is None or (target.buildable and target.supports(rel.plat)): + keep.append(dep) + else: + log(f" note: {fw.name} lists {dep} as required, but it is not available on {rel.plat}; treating it as optional") + fw.soft_deps = sorted(set(fw.soft_deps) | {dep}) + fw.deps = keep + + +def assign_tiers(fws: list[Framework]) -> None: + """Compute ``group`` (the tier, used for caching) and ``tier`` (build order). + + * ``group`` follows the tier KDE declares, but is never lower than the group + of anything the framework depends on, so every dependency sits in the same + or an earlier group. KDE's own "Tier 3" frameworks depend on each other + (kio -> kiconthemes), so same-group dependencies are normal. + * ``tier`` is the dependency depth (ECM = 0). Sorting by (group, tier) is a + valid topological order, including inside a group. + """ + by_name = {f.name: f for f in fws} + depth: dict[str, int] = {} + grp: dict[str, int] = {} + visiting: list[str] = [] + + def visit(name: str) -> None: + if name in depth: + return + if name in visiting: + die("dependency cycle: " + " -> ".join(visiting[visiting.index(name):] + [name])) + visiting.append(name) + fw = by_name[name] + deps = [d for d in fw.deps if d in by_name] + for d in deps: + visit(d) + # Optional dependencies only constrain the order if they are built too, + # and are dropped rather than failing if they would form a cycle. + for d in fw.soft_deps: + if d in by_name and d not in deps: + if d in visiting: + log(f" note: ignoring optional dependency {name} -> {d} (would form a cycle)") + continue + visit(d) + deps.append(d) + if name == ECM: + depth[name], grp[name] = 0, 1 + else: + depth[name] = 1 + max([depth[d] for d in deps] + [0]) + grp[name] = max([fw.declared_tier or 1] + [grp[d] for d in deps]) + visiting.pop() + + for fw in fws: + visit(fw.name) + fw.tier = depth[fw.name] + fw.group = max(1, min(grp[fw.name], MAX_GROUP)) + + +def detect_qt(cfg: Config) -> dict: + candidates: list[str] = [cfg.qt_prefix_input, os.environ.get("QT_ROOT_DIR", "")] + qt6_dir = os.environ.get("Qt6_DIR", "") + if qt6_dir: + candidates.append(str(Path(qt6_dir).parents[2])) # /lib/cmake/Qt6 + candidates.append(os.environ.get("QTDIR", "")) + for cand in candidates: + if not cand: + continue + root = Path(cand) + cmake_dir = root / "lib" / "cmake" / "Qt6Core" + if (cmake_dir / "Qt6CoreConfig.cmake").exists(): + version = "unknown" + # Newer Qt keeps the version in ...ConfigVersionImpl.cmake, older in ...ConfigVersion.cmake. + for fname in ("Qt6CoreConfigVersionImpl.cmake", "Qt6CoreConfigVersion.cmake"): + f = cmake_dir / fname + m = re.search(r'set\(PACKAGE_VERSION\s+"([0-9][^"\s]*)"', f.read_text()) if f.exists() else None + if m: + version = m.group(1) + break + return {"prefix": str(root.resolve()), "version": version, "name": root.name} + die( + "Could not find a Qt 6 installation. Install Qt first (e.g. jurplel/install-qt-action) " + "or pass the 'qt-prefix' input. Looked at: " + + (", ".join(c for c in candidates if c) or "QT_ROOT_DIR, Qt6_DIR, QTDIR (none set)") + ) + raise AssertionError + + +def user_cmake_args(cfg: Config) -> list[str]: + args: list[str] = [] + for line in cfg.cmake_args.splitlines(): + args += shlex.split(line, posix=True) + return args + + +def per_framework_args(cfg: Config) -> dict[str, list[str]]: + out: dict[str, list[str]] = {} + for line in cfg.framework_cmake_args.splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + name, sep, rest = line.partition(":") + if not sep: + die(f"framework-cmake-args: expected ': ', got '{line}'") + out.setdefault(name.strip(), []).extend(shlex.split(rest, posix=True)) + return out + + +def cmd_plan(cfg: Config) -> None: + plat = current_platform() + cfg.work.mkdir(parents=True, exist_ok=True) + qt = detect_qt(cfg) + + with group("Resolving KDE Frameworks release"): + rel = Releases(cfg, plat) + log(f"KDE Frameworks {rel.version} for {plat}/{normalized_arch()}, Qt {qt['version']} ({qt['prefix']})") + fws = resolve_selection(rel, cfg) + if not fws: + die("Nothing to build: the selection is empty for this platform.") + assign_tiers(fws) + # Validate user per-framework arguments early. + names = {f.name for f in fws} + for n in per_framework_args(cfg): + if n not in rel.catalog: + die(f"framework-cmake-args: unknown framework '{n}'") + if n not in names: + warning(f"framework-cmake-args given for '{n}', which is not part of the plan") + fws.sort(key=lambda f: (f.group, f.tier, f.name)) + + # ---- cache keys: each tier's key covers everything built up to it + ctx = { + "kf": rel.version, + "os": plat, + "arch": normalized_arch(), + "image": os.environ.get("ImageOS", ""), + "qt": qt["version"], + "qt_name": qt["name"], + "build_type": cfg.build_type, + "linkage": cfg.linkage, + "generator": cfg.generator, + # macOS install names and RPATHs embed the absolute prefix. + "prefix": cfg.prefix.as_posix(), + "osx_arch": cfg.osx_architectures or os.environ.get("CMAKE_OSX_ARCHITECTURES", ""), + "args": user_cmake_args(cfg), # per-framework arguments are part of that framework's entry below + "suffix": cfg.cache_key_suffix, + # Bumped by hand when the layout or meaning of cached data changes. Deliberately not a + # hash of this file: editing the script must not throw away hours of cached builds. What a + # framework was built with is covered by its fingerprint instead. + "cache_format": CACHE_FORMAT, + } + key_base = f"kf6-{rel.version}-{plat}-{normalized_arch()}-qt{qt['version']}-{cfg.build_type}-{cfg.linkage}" + + # A framework's fingerprint says how it is built: its tarball, its own arguments and the + # fingerprints of everything it depends on (hard or soft, since optional features are + # picked up when the dependency is there). `fws` is in build order, so dependencies come first. + fw_args = per_framework_args(cfg) + fingerprints: dict[str, str] = {} + for f in fws: + parts = [f.name, f.sha256, fw_args.get(f.name, []), + sorted(fingerprints[d] for d in f.deps + f.soft_deps if d in fingerprints)] + f.fingerprint = hashlib.sha256(json.dumps(parts).encode()).hexdigest()[:16] + fingerprints[f.name] = f.fingerprint + + # Cache keys. The part before the final hash only describes the environment, so a cache from an + # earlier run of the same tier can always be found (restore-keys) and is then used framework by + # framework: only those whose fingerprint still matches are trusted, the rest is rebuilt. The + # exact key additionally covers every framework up to this tier and finds a finished tier directly. + ctx_hash = hashlib.sha256(json.dumps(ctx, sort_keys=True).encode()).hexdigest()[:12] + keys: dict[int, str] = {} + restore_prefix: dict[int, str] = {} + for g in range(1, MAX_GROUP + 1): + upto = [(f.name, f.fingerprint) for f in fws if f.group <= g] + chain = hashlib.sha256(json.dumps(upto).encode()).hexdigest()[:12] + restore_prefix[g] = f"{key_base}-t{g}-{ctx_hash}-" + keys[g] = f"{restore_prefix[g]}{chain}" + + plan = { + "version": rel.version, + "platform": plat, + "qt": qt, + "prefix": str(cfg.prefix), + "work": str(cfg.work), + "key_base": key_base, + "keys": keys, + "restore": restore_prefix, + "frameworks": [asdict(f) for f in fws], + } + (cfg.work / "plan.json").write_text(json.dumps(plan, indent=2)) + (cfg.work / "status.json").write_text("{}") + + # ---- report + with group("Build plan"): + for g in range(1, MAX_GROUP + 1): + members = [f for f in fws if f.group == g] + if members: + log(f"Tier {g}: " + ", ".join(f.name for f in members)) + for f in fws: + if f.external: + log(f" note: {f.name} also lists non-framework KDE dependencies: {', '.join(f.external)}") + + lines = [f"### KDE Frameworks {rel.version} build plan ({plat}/{normalized_arch()}, Qt {qt['version']})", "", + "| Tier | Frameworks |", "|---|---|"] + for g in range(1, MAX_GROUP + 1): + members = [f.name for f in fws if f.group == g] + if members: + lines.append(f"| {g} | {', '.join(members)} |") + append_file("GITHUB_STEP_SUMMARY", "\n".join(lines) + "\n") + + for g in range(1, MAX_GROUP + 1): + set_output(f"count-{g}", str(sum(1 for f in fws if f.group == g))) + set_output(f"key-{g}", keys[g]) + set_output(f"restore-{g}", restore_prefix[g]) + set_output("work-dir", str(cfg.work)) + set_output("key-base", key_base) + set_output("prefix", str(cfg.prefix)) + set_output("frameworks", ",".join(f.name for f in fws)) + set_output("kf-version", rel.version) + set_output("qt-version", qt["version"]) + + +# -------------------------------------------------------------------------- +# state shared between steps +# -------------------------------------------------------------------------- + + +def load_plan(cfg: Config) -> dict: + path = cfg.work / "plan.json" + if not path.exists(): + die(f"{path} not found: the 'plan' step has not run") + return json.loads(path.read_text()) + + +def load_status(cfg: Config) -> dict: + path = cfg.work / "status.json" + return json.loads(path.read_text()) if path.exists() else {} + + +def save_status(cfg: Config, status: dict) -> None: + (cfg.work / "status.json").write_text(json.dumps(status, indent=2)) + + +# -------------------------------------------------------------------------- +# system packages +# -------------------------------------------------------------------------- + + +def run_quiet(cmd: list[str], **kw) -> subprocess.CompletedProcess: + return subprocess.run(cmd, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw) + + +def cmd_sysdeps(cfg: Config) -> None: + plan = load_plan(cfg) + plat = plan["platform"] + table = SYSTEM_DEPS.get(plat, {}) + packages: list[str] = list(table.get("*", [])) + for fw in plan["frameworks"]: + packages += table.get(fw["name"], []) + packages += cfg.extra_system_packages + packages = list(dict.fromkeys(packages)) + sysdeps_state: dict[str, list[str]] = {"path": [], "prefix": []} + + with group(f"Installing system packages ({plat}): {' '.join(packages)}"): + if plat == "Linux": + sudo = [] if os.geteuid() == 0 else ["sudo"] + env = dict(os.environ, DEBIAN_FRONTEND="noninteractive") + subprocess.run(sudo + ["apt-get", "-y", "update"], env=env, check=False) + base = sudo + ["apt-get", "-y", "install", "--no-install-recommends"] + if subprocess.run(base + packages, env=env).returncode != 0: + warning("bulk apt-get install failed, retrying package by package") + for pkg in packages: + if subprocess.run(base + [pkg], env=env).returncode != 0: + warning(f"apt package '{pkg}' is unavailable; frameworks needing it may fail or lose features") + elif plat == "macOS": + env = dict(os.environ, HOMEBREW_NO_AUTO_UPDATE="1", HOMEBREW_NO_INSTALL_CLEANUP="1") + for pkg in packages: + if subprocess.run(["brew", "install", pkg], env=env).returncode != 0: + warning(f"brew package '{pkg}' could not be installed") + continue + res = run_quiet(["brew", "--prefix", pkg], env=env) + prefix = res.stdout.strip().splitlines()[-1] if res.returncode == 0 and res.stdout.strip() else "" + if prefix and Path(prefix).is_dir(): + # Many of these are keg-only: neither bin/ nor the libraries are + # reachable without being named explicitly. + sysdeps_state["prefix"].append(prefix) + if (Path(prefix) / "bin").is_dir(): + sysdeps_state["path"].append(str(Path(prefix) / "bin")) + elif plat == "Windows": + for pkg in packages: + res = subprocess.run(["choco", "install", pkg, "-y", "--no-progress", "--allow-empty-checksums"]) + if res.returncode not in (0, 1641, 3010): + warning(f"choco package '{pkg}' could not be installed") + else: + warning(f"no system package installer known for {plat}") + + (cfg.work / "sysdeps.json").write_text(json.dumps(sysdeps_state)) + if IN_CI: + for p in sysdeps_state["path"]: + append_file("GITHUB_PATH", p) + + +def load_sysdeps(cfg: Config) -> dict: + path = cfg.work / "sysdeps.json" + return json.loads(path.read_text()) if path.exists() else {"path": [], "prefix": []} + + +# -------------------------------------------------------------------------- +# building +# -------------------------------------------------------------------------- + + +def snapshot(prefix: Path) -> dict[str, tuple[int, int]]: + snap: dict[str, tuple[int, int]] = {} + if not prefix.exists(): + return snap + stack = [prefix] + while stack: + current = stack.pop() + with os.scandir(current) as it: + for entry in it: + if entry.is_dir(follow_symlinks=False): + stack.append(Path(entry.path)) + else: + st = entry.stat(follow_symlinks=False) + snap[os.path.relpath(entry.path, prefix)] = (st.st_size, st.st_mtime_ns) + return snap + + +def copy_entry(src: Path, dst: Path) -> None: + dst.parent.mkdir(parents=True, exist_ok=True) + if os.path.lexists(dst): + dst.unlink() + if src.is_symlink(): + os.symlink(os.readlink(src), dst) + else: + shutil.copy2(src, dst) + + +MANIFEST = ".manifest.json" + + +def read_manifest(delta: Path) -> dict | None: + path = delta / MANIFEST + try: + return json.loads(path.read_text()) + except (OSError, ValueError): + return None + + +def write_delta(prefix: Path, before: dict, delta: Path, merge: bool = False) -> int: + """Copy what changed in ``prefix`` since ``before`` into ``delta``. + + With ``merge`` the existing delta (a partial tier restored from cache) is kept. + """ + if not merge: + shutil.rmtree(delta, ignore_errors=True) + delta.mkdir(parents=True, exist_ok=True) + after = snapshot(prefix) + changed = [rel for rel, meta in after.items() if before.get(rel) != meta] + for rel in changed: + copy_entry(prefix / rel, delta / rel) + return len(changed) + + +def apply_delta(delta: Path, prefix: Path) -> int: + count = 0 + for root, _dirs, files in os.walk(delta): + for name in files: + src = Path(root) / name + rel = src.relative_to(delta) + if str(rel) in {MANIFEST, ".complete"}: + continue + copy_entry(src, prefix / rel) + count += 1 + return count + + +def build_env(cfg: Config, sysdeps: dict) -> dict: + env = dict(os.environ) + if sysdeps["path"]: + env["PATH"] = os.pathsep.join(sysdeps["path"] + [env.get("PATH", "")]) + if current_platform() == "Windows": + if not shutil.which("cl") and not env.get("CXX"): + die("No MSVC compiler on PATH. Run ilammy/msvc-dev-cmd (or open a VS developer shell) before this action.") + # Windows has no RPATH: programs run during the build (meinproc6, ...) find the DLLs of the + # frameworks installed so far only through PATH. Without this they exit with 0xC0000135. + env["PATH"] = os.pathsep.join([str(cfg.prefix / "bin"), env.get("PATH", "")]) + return env + + +def tail(path: Path, lines: int = 80) -> str: + try: + return "\n".join(path.read_text(errors="replace").splitlines()[-lines:]) + except OSError: + return "" + + +def error_context(path: Path) -> str: + """The CMake / compiler error lines, so the cause is visible without scrolling.""" + try: + text = path.read_text(errors="replace").splitlines() + except OSError: + return "" + picked: list[str] = [] + for i, line in enumerate(text): + # Harmless: the aqtinstall Qt's QML plugin metadata mentions a module that is not + # installed, and Vulkan headers are optional. + if "Qt6TaskTree" in line or "WrapVulkanHeaders" in line: + continue + if re.search(r"CMake Error|FAILED:|error:|fatal error|Could NOT find|Could not find", line): + picked += text[i:i + 4] + if len(picked) > 60: + break + return "\n".join(picked) + + +def missing_qt_module_hint(path: Path) -> str: + """Name the aqtinstall module when a configure failed for a missing Qt component.""" + try: + text = path.read_text(errors="replace") + except OSError: + return "" + modules = sorted(set(re.findall(r'Failed to find required Qt component "(\w+)"', text))) + if not modules: + return "" + special = {"TextToSpeech": "qtspeech"} # aqtinstall name differs from the Qt module name + names = ", ".join(f"{m} (aqtinstall module: {special.get(m, 'qt' + m.lower())})" for m in modules) + return f"Missing Qt module(s): {names}. Install them with the Qt installer (e.g. `modules:` of install-qt-action)." + + +def run_logged(cfg: Config, cmd: list[str], logfile: Path, env: dict) -> int: + logfile.parent.mkdir(parents=True, exist_ok=True) + with open(logfile, "w", encoding="utf-8", errors="replace") as fh: + fh.write("$ " + " ".join(shlex.quote(c) for c in cmd) + "\n\n") + fh.flush() + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env, text=True, + encoding="utf-8", errors="replace") + assert proc.stdout is not None + for line in proc.stdout: + fh.write(line) + if cfg.verbose: + sys.stdout.write(line) + return proc.wait() + + +def pkg_config_has(module: str) -> bool: + exe = shutil.which("pkg-config") or shutil.which("pkgconf") + if not exe: + return False + try: + return subprocess.run([exe, "--exists", module], stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=30).returncode == 0 + except (OSError, subprocess.SubprocessError): + return False + + +def cmake_defs(cfg: Config, plan: dict, sysdeps: dict, fw: dict) -> list[str]: + prefix = Path(plan["prefix"]).as_posix() + # Prefixes the caller already exported (e.g. a vcpkg tree or tools built by an earlier + # step) are honoured by CMake anyway; listing them makes that visible and explicit. + inherited = [p for p in os.environ.get("CMAKE_PREFIX_PATH", "").split(os.pathsep) if p] + prefix_path = ";".join(dict.fromkeys( + [prefix, Path(plan["qt"]["prefix"]).as_posix()] + + [Path(p).as_posix() for p in sysdeps["prefix"]] + + [Path(p).as_posix() for p in inherited] + )) + defs = [ + f"-DCMAKE_BUILD_TYPE={cfg.build_type}", + f"-DCMAKE_INSTALL_PREFIX={prefix}", + f"-DCMAKE_PREFIX_PATH={prefix_path}", + f"-DBUILD_SHARED_LIBS={'ON' if cfg.linkage == 'shared' else 'OFF'}", + ] + if cfg.qt_host_path: + defs.append(f"-DQT_HOST_PATH={Path(cfg.qt_host_path).as_posix()}") + if cfg.osx_architectures: + defs.append(f"-DCMAKE_OSX_ARCHITECTURES={cfg.osx_architectures}") + defs += BASE_CMAKE_DEFS + defs += FRAMEWORK_CMAKE_DEFS.get(fw["name"], []) + feature = QT_MODULE_FEATURES.get(fw["name"]) + if feature and not (Path(plan["qt"]["prefix"]) / "lib" / "cmake" / f"Qt6{feature[1]}").is_dir(): + log(f" {fw['name']}: Qt{feature[1]} is not installed, building with -D{feature[0]}=OFF") + defs.append(f"-D{feature[0]}=OFF") + if fw["name"] == "karchive" and not pkg_config_has("libzstd"): + # karchive finds zstd only through pkg-config and treats it as REQUIRED by default. + log(" karchive: libzstd is not visible to pkg-config, building with -DWITH_LIBZSTD=OFF") + defs.append("-DWITH_LIBZSTD=OFF") + if current_platform() == "Windows": + # ECM's app template packaging runs the first `tar` it finds. On the runners that is + # Git's GNU tar, which reads "D:/x" as host "D", so point it at Windows' own bsdtar + # (it then falls back to `cmake -E tar`). + win_tar = Path(os.environ.get("SystemRoot", r"C:\Windows")) / "System32" / "tar.exe" + if win_tar.exists(): + defs.append(f"-D_tar_executable={win_tar.as_posix()}") + defs += user_cmake_args(cfg) + defs += per_framework_args(cfg).get(fw["name"], []) + return defs + + +def build_one(cfg: Config, plan: dict, sysdeps: dict, fw: dict, env: dict) -> bool: + name = fw["name"] + src = Path(fw["src"]) + build_dir = cfg.work / "build" / name + logs = cfg.work / "logs" + shutil.rmtree(build_dir, ignore_errors=True) + + steps = [ + ("configure", ["cmake", "-S", str(src), "-B", str(build_dir), "-G", cfg.generator] + + cmake_defs(cfg, plan, sysdeps, fw)), + ("build", ["cmake", "--build", str(build_dir), "--parallel", str(cfg.jobs)]), + ("install", ["cmake", "--install", str(build_dir)]), + ] + for label, cmd in steps: + logfile = logs / f"{name}-{label}.log" + rc = run_logged(cfg, cmd, logfile, env) + if rc != 0: + with group(f"{name}: {label} FAILED (exit {rc}) - log: {logfile}"): + ctx = error_context(logfile) + hint = missing_qt_module_hint(logfile) + if hint: + log("HINT: " + hint) + if ctx: + log("---- errors ----\n" + ctx + "\n---- end of errors ----") + log("---- last lines of the log ----\n" + tail(logfile)) + if hint: + log(f"::error title=KDE Frameworks: {name} needs another Qt module::{hint}" if IN_CI else f"error: {hint}") + log(f"::error title=KDE Frameworks: {name} {label} failed::see the '{name}: {label} FAILED' group above" + if IN_CI else f"error: {name} {label} failed") + return False + shutil.rmtree(build_dir, ignore_errors=True) # keep the runner's disk free + return True + + +def cmd_build(cfg: Config, tier: int) -> None: + plan = load_plan(cfg) + status = load_status(cfg) + sysdeps = load_sysdeps(cfg) + prefix = Path(plan["prefix"]) + delta = cfg.work / "delta" / f"tier{tier}" + members = [f for f in plan["frameworks"] if f["group"] == tier] + prefix.mkdir(parents=True, exist_ok=True) + set_output("save", "false") + + if not members: + return + + # A restored cache holds a manifest listing what it contains: either the whole tier, + # or (after an earlier run in which something failed) the frameworks that did build. + already: set[str] = set() + manifest = read_manifest(delta) + if manifest: + count = apply_delta(delta, prefix) + # The cache may come from a run with different settings (restore-keys). Only trust a + # framework if it was built exactly the way it would be built now. + cached = manifest.get("frameworks", {}) + already = {m["name"] for m in members if cached.get(m["name"]) == m["fingerprint"]} + stale = sorted(m["name"] for m in members if m["name"] in cached and m["name"] not in already) + for fw in members: + if fw["name"] in already: + status[fw["name"]] = {"state": "cached", "seconds": 0} + save_status(cfg, status) + if stale: + log(f"Tier {tier}: rebuilding {', '.join(stale)} (built earlier with different settings)") + if len(already) == len(members): + notice(f"Tier {tier}: restored {len(members)} framework(s) from cache ({count} files)") + return + notice(f"Tier {tier}: resuming, {len(already)}/{len(members)} framework(s) restored from cache ({count} files)") + else: + shutil.rmtree(delta, ignore_errors=True) # an incomplete leftover + + env = build_env(cfg, sysdeps) + before = snapshot(prefix) # after restoring, so the new delta only holds what we add + failed = False + built_now: list[str] = [] + t_tier = time.time() + todo = [m for m in members if m["name"] not in already] + log(f"Tier {tier}: building {len(todo)} framework(s): {', '.join(m['name'] for m in todo)}") + for fw in todo: + name = fw["name"] + blockers = [d for d in fw["deps"] if status.get(d, {}).get("state") in {"failed", "skipped"}] + if blockers: + status[name] = {"state": "skipped", "seconds": 0, "reason": f"depends on failed {', '.join(blockers)}"} + log(f" - {name}: skipped (depends on failed {', '.join(blockers)})") + failed = True + continue + start = time.time() + ok = build_one(cfg, plan, sysdeps, fw, env) + seconds = round(time.time() - start) + status[name] = {"state": "built" if ok else "failed", "seconds": seconds} + save_status(cfg, status) + log(f" {'✔' if ok else '✘'} {name} ({seconds}s)") + if ok: + built_now.append(name) + else: + failed = True + if not cfg.keep_going: + break + + save_status(cfg, status) + log(f"Tier {tier} finished in {round(time.time() - t_tier)}s") + # Save what was built even if something failed, so the next run does not start from + # scratch (a tier can take half an hour). A fully built tier gets the exact key and is + # reused as a whole; a partial one gets a unique "-partial-" key that later runs resume from. + if cfg.cache and built_now: + files = write_delta(prefix, before, delta, merge=bool(manifest)) + complete = len(already) + len(built_now) == len(members) + fingerprints = {m["name"]: m["fingerprint"] for m in members if m["name"] in already | set(built_now)} + (delta / MANIFEST).write_text(json.dumps({"frameworks": fingerprints, "complete": complete})) + key = plan["keys"][str(tier)] + if not complete: + run = f"{os.environ.get('GITHUB_RUN_ID', int(time.time()))}-{os.environ.get('GITHUB_RUN_ATTEMPT', '1')}" + key = f"{plan['restore'][str(tier)]}partial-{run}" + log(f"Tier {tier}: cache {'complete' if complete else 'partial'}, {files} new file(s), key {key}") + set_output("save", "true") + set_output("save-key", key) + if failed and not cfg.keep_going: + sys.exit(1) + # With keep-going the failure is reported by the 'finish' step, so later + # tiers can still show everything that is broken in one run. + + +# -------------------------------------------------------------------------- +# finishing +# -------------------------------------------------------------------------- + + +def lib_subdirs(prefix: Path, leaf: str) -> list[str]: + found: list[str] = [] + for libdir in [prefix / "lib", prefix / "lib64"] + sorted((prefix / "lib").glob("*-linux-gnu")): + cand = libdir / leaf + if cand.is_dir(): + found.append(str(cand)) + return found + + +def join_paths(new: list[str], existing: str) -> str: + return os.pathsep.join([p for p in new + [existing] if p]) + + +def cmd_finish(cfg: Config) -> None: + if not (cfg.work / "plan.json").exists(): + return # planning itself failed; the error was already reported + plan = load_plan(cfg) + status = load_status(cfg) + prefix = Path(plan["prefix"]) + sysdeps = load_sysdeps(cfg) + + rows = ["", "| Framework | Tier | Result | Time |", "|---|---|---|---|"] + icons = {"built": "✅ built", "cached": "📦 cached", "failed": "❌ failed", "skipped": "⏭️ skipped"} + failures: list[str] = [] + for fw in plan["frameworks"]: + st = status.get(fw["name"], {"state": "not run", "seconds": 0}) + if st["state"] in {"failed", "skipped", "not run"}: + failures.append(fw["name"]) + note = f" ({st['reason']})" if st.get("reason") else "" + seconds = f"{st['seconds']}s" if st["seconds"] else "" + rows.append(f"| {fw['name']} | {fw['group']} | {icons.get(st['state'], st['state'])}{note} | {seconds} |") + append_file("GITHUB_STEP_SUMMARY", "\n".join(rows) + "\n") + + all_cached = all(status.get(f["name"], {}).get("state") == "cached" for f in plan["frameworks"]) + set_output("prefix", str(prefix)) + set_output("cache-hit", "true" if all_cached else "false") + set_output("failed", ",".join(failures)) + + if failures: + message = "KDE Frameworks failed/skipped: " + ", ".join(failures) + if cfg.ignore_failures: + warning(message) + else: + die(message) + + if cfg.export_env: + export: dict[str, str] = {} + export["KF6_PREFIX"] = str(prefix) + export["CMAKE_PREFIX_PATH"] = join_paths( + [str(prefix)] + sysdeps["prefix"], os.environ.get("CMAKE_PREFIX_PATH", "") + ) + plugins = lib_subdirs(prefix, "plugins") + qml = lib_subdirs(prefix, "qml") + if plugins: + export["QT_PLUGIN_PATH"] = join_paths(plugins, os.environ.get("QT_PLUGIN_PATH", "")) + if qml: + export["QML_IMPORT_PATH"] = join_paths(qml, os.environ.get("QML_IMPORT_PATH", "")) + export["QML2_IMPORT_PATH"] = join_paths(qml, os.environ.get("QML2_IMPORT_PATH", "")) + if plan["platform"] != "Windows": + default = os.environ.get("XDG_DATA_DIRS", "/usr/local/share:/usr/share") + export["XDG_DATA_DIRS"] = join_paths([str(prefix / "share")], default) + for key, value in export.items(): + append_file("GITHUB_ENV", f"{key}={value}") + for p in [str(prefix / "bin")] + ([str(prefix / "libexec")] if (prefix / "libexec").is_dir() else []): + append_file("GITHUB_PATH", p) + log("Exported: " + ", ".join(export) + ", PATH") + + +# -------------------------------------------------------------------------- + + +def main(argv: list[str]) -> None: + if len(argv) < 2 or argv[1] not in {"plan", "sysdeps", "build", "finish"}: + die("usage: kf6.py plan|sysdeps|build |finish") + cfg = Config() + command = argv[1] + if command == "plan": + cmd_plan(cfg) + elif command == "sysdeps": + cmd_sysdeps(cfg) + elif command == "build": + if len(argv) < 3: + die("usage: kf6.py build ") + cmd_build(cfg, int(argv[2])) + else: + cmd_finish(cfg) + + +if __name__ == "__main__": + main(sys.argv) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1601bf984..62e17bf0e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -86,50 +86,19 @@ jobs: - os: macos-26 artifact-name: macOS - cmake-preset: macos_universal - macosx-deployment-target: 14.0 + cmake-preset: macos + macosx-deployment-target: 14.4 qt-version: 6.12.0 qt-major: 6 - build-name: macOS Universal Qt6 + build-name: macOS Qt6 - os: macos-26 artifact-name: Legacy-macOS - cmake-preset: macos_universal - macosx-deployment-target: 12.0 - qt-version: 6.9.3 - qt-major: 6 - build-name: macOS Universal Legacy Qt6 - - - os: ubuntu-26.04 - artifact-name: Linux - cmake-preset: linux - qt-version: 5.15.2 - qt-major: 5 - build-name: Linux x86_64 Qt5 - - - os: windows-2025 - artifact-name: Windows-MinGW-w64 - cmake-preset: windows_mingw - msystem: CLANG64 - vcvars-arch: amd64_x86 - qt-major: 5 - build-name: Windows MinGW x86_64 Qt5 - - - os: windows-2022 - artifact-name: Windows-MSVC - cmake-preset: windows_msvc - # TODO(@YongDo-Hyun): This is the default in setup-dependencies/windows. Why isn't it working?!?! - vcvars-arch: amd64 - qt-version: 5.15.2 - qt-major: 5 - build-name: Windows MSVC x86_64 Qt5 - - - os: macos-26 - artifact-name: macOS-ARM cmake-preset: macos - macosx-deployment-target: 12.0 - qt-major: 5 - build-name: macOS aarch64 Qt5 + macosx-deployment-target: 13.0 + qt-version: 6.11.3 + qt-major: 6 + build-name: macOS Legacy Qt6 runs-on: ${{ matrix.os }}