From f57fe0729a543e2d8c0238767918197f5295eb3d Mon Sep 17 00:00:00 2001 From: Kai Henseler Date: Thu, 6 Nov 2025 10:04:23 +0100 Subject: [PATCH 01/16] IONOS(theming): fix user bubble color Signed-off-by: Kai Henseler --- apps/theming/css/ionos/files.css | 8 ++++++++ apps/theming/lib/Themes/IonosTheme.php | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/apps/theming/css/ionos/files.css b/apps/theming/css/ionos/files.css index bb9cc032896a1..01d6f4eca6522 100644 --- a/apps/theming/css/ionos/files.css +++ b/apps/theming/css/ionos/files.css @@ -480,3 +480,11 @@ div.v-popper--theme-dropdown.v-popper__popper:has(.v-popper__wrapper), .header-m min-width: 0; } } + +/* Sidebar */ +aside#app-sidebar-vue { + div.user-bubble__content { + background-color: var(--ion-surface-secondary); + color: var(--ion-text); + } +} diff --git a/apps/theming/lib/Themes/IonosTheme.php b/apps/theming/lib/Themes/IonosTheme.php index cc325edd1d961..c62ca67d18025 100644 --- a/apps/theming/lib/Themes/IonosTheme.php +++ b/apps/theming/lib/Themes/IonosTheme.php @@ -214,7 +214,7 @@ public function getCSSVariables(): array { // used for different active/hover/focus/disabled states '--color-background-hover' => 'light-dark( var(--ion-color-blue-b1), var(--ion-color-blue-b8))', - '--color-background-dark' => 'light-dark( ' . $this->util->darken($colorMainBackground, 7) . ', var(--ion-color-blue-b6))', + '--color-background-dark' => $this->util->darken($colorMainBackground, 7), '--color-background-darker' => $this->util->darken($colorMainBackground, 14), '--color-placeholder-light' => $this->util->darken($colorMainBackground, 10), From e1d641dc576814725ebfb6bbb1577e620b03a106 Mon Sep 17 00:00:00 2001 From: Kai Henseler Date: Thu, 6 Nov 2025 10:04:50 +0100 Subject: [PATCH 02/16] IONOS(theming): adjust tasks app theming Signed-off-by: Kai Henseler --- apps/theming/css/ionos/tasks.css | 50 ++++++++++++++++++++++++++ apps/theming/lib/Themes/IonosTheme.php | 5 +-- 2 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 apps/theming/css/ionos/tasks.css diff --git a/apps/theming/css/ionos/tasks.css b/apps/theming/css/ionos/tasks.css new file mode 100644 index 0000000000000..1621839c9edaa --- /dev/null +++ b/apps/theming/css/ionos/tasks.css @@ -0,0 +1,50 @@ +div.app-tasks { + main#app-content-vue { + background-color: var(--color-main-background) !important; + + .header { + background-color: var(--color-main-background) !important; + } + + .task-item__body .task-body__icons .date { + color: var(--ion-text); + } + } + aside.app-sidebar { + ul.vs__dropdown-menu { + background-color: var(--ion-context-menu-background); + border: 1px solid var(--ion-context-menu-border); + + li.vs__dropdown-option { + &:hover { + background-color: var(--ion-context-menu-item-background-hover); + } + + &:active { + background-color: var(--ion-context-menu-item-background-active); + } + + div.multiselect-picker-option { + color: var(--ion-context-menu-item-text); + } + } + } + } + + .property__item .item__content .content__input input[type=number] { + background-color: var(--color-main-background); + color: var(--ion-text); + } + + nav#app-navigation-vue { + div#nc-vue-3 li select { + background-color: var(--ion-context-menu-background); + color: var(--ion-text); + + option { + background-color: var(--ion-context-menu-background); + color: var(--ion-text); + } + } + } +} diff --git a/apps/theming/lib/Themes/IonosTheme.php b/apps/theming/lib/Themes/IonosTheme.php index c62ca67d18025..b78c55c97c49e 100644 --- a/apps/theming/lib/Themes/IonosTheme.php +++ b/apps/theming/lib/Themes/IonosTheme.php @@ -16,7 +16,7 @@ class IonosTheme extends DefaultTheme implements ITheme { private const FONT_PATH_PREFIX = 'fonts/OpenSans/'; // CSS file paths for custom styling - private const CSS_FILES = [ + private const array CSS_FILES = [ 'variables.css', 'buttons.css', 'sidebar.css', @@ -25,7 +25,8 @@ class IonosTheme extends DefaultTheme implements ITheme { 'files.css', 'settings.css', '_layout.css', - 'tables.css' + 'tables.css', + 'tasks.css' ]; public function getId(): string { From 77e1072509a4d4265c6aa7ba7f9e035c8117d105 Mon Sep 17 00:00:00 2001 From: "Misha M.-Kupriyanov" Date: Wed, 12 Nov 2025 10:58:00 +0100 Subject: [PATCH 03/16] IONOS(build): copy build-artifact.yml as build-artifact-original.yml for later performance comparison Signed-off-by: Misha M.-Kupriyanov --- .github/workflows/build-artifact-original.yml | 924 ++++++++++++++++++ 1 file changed, 924 insertions(+) create mode 100644 .github/workflows/build-artifact-original.yml diff --git a/.github/workflows/build-artifact-original.yml b/.github/workflows/build-artifact-original.yml new file mode 100644 index 0000000000000..a2cbfdb849649 --- /dev/null +++ b/.github/workflows/build-artifact-original.yml @@ -0,0 +1,924 @@ +name: Build Nextcloud Workspace artifact + +# SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors +# SPDX-FileCopyrightText: 2025 STRATO AG +# SPDX-License-Identifier: AGPL-3.0-or-later + +# The Nextcloud Workspace source is packaged as a container image. +# This is a workaround because releases cannot be created without tags, +# and we want to be able to create snapshots from branches. + +on: + pull_request: + paths: + - '.github/workflows/**' + - 'src/**' + - 'apps/**' + - 'apps/**/appinfo/info.xml' + - 'apps-external/**' + - 'IONOS' + - 'package.json' + - 'package-lock.json' + - 'themes/**' + - 'lib/**' + - 'tsconfig.json' + - '**.js' + - '**.ts' + - '**.vue' + - '.gitmodules' + push: + branches: + - ionos-dev + - ionos-stable + +concurrency: + group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/ionos-dev' && github.run_id || github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + TARGET_PACKAGE_NAME: nc-workspace.zip + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + ARTIFACTORY_REPOSITORY_SNAPSHOT: ionos-productivity-ncwserver-snapshot + +permissions: + contents: read + +jobs: + prepare-matrix: + runs-on: ubuntu-latest + outputs: + external-apps-matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + submodules: true + fetch-depth: '1' + + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y make jq + + - name: Set matrix + id: set-matrix + run: | + # Create matrix configuration as a compact JSON string + matrix='[ + { + "name": "activity", + "path": "apps-external/activity", + "has_npm": true, + "has_composer": true, + "makefile_target": "build_activity_app" + }, + { + "name": "assistant", + "path": "apps-external/assistant", + "has_npm": true, + "has_composer": true, + "makefile_target": "build_assistant_app" + }, + { + "name": "calendar", + "path": "apps-external/calendar", + "has_npm": true, + "has_composer": true, + "makefile_target": "build_calendar_app" + }, + { + "name": "circles", + "path": "apps-external/circles", + "has_npm": false, + "has_composer": true, + "makefile_target": "build_circles_app" + }, + { + "name": "collectives", + "path": "apps-external/collectives", + "has_npm": true, + "has_composer": true, + "makefile_target": "build_collectives_app" + }, + { + "name": "contacts", + "path": "apps-external/contacts", + "has_npm": true, + "has_composer": true, + "makefile_target": "build_contacts_app" + }, + { + "name": "deck", + "path": "apps-external/deck", + "has_npm": true, + "has_composer": true, + "makefile_target": "build_deck_app" + }, + { + "name": "end_to_end_encryption", + "path": "apps-external/end_to_end_encryption", + "has_npm": true, + "has_composer": true, + "makefile_target": "build_end_to_end_encryption_app" + }, + { + "name": "forms", + "path": "apps-external/forms", + "has_npm": true, + "has_composer": true, + "makefile_target": "build_forms_app" + }, + { + "name": "groupfolders", + "path": "apps-external/groupfolders", + "has_npm": true, + "has_composer": true, + "makefile_target": "build_groupfolders_app" + }, + { + "name": "integration_openai", + "path": "apps-external/integration_openai", + "has_npm": true, + "has_composer": true, + "makefile_target": "build_integration_openai_app" + }, + { + "name": "mail", + "path": "apps-external/mail", + "has_npm": true, + "has_composer": true, + "makefile_target": "build_mail_app" + }, + { + "name": "ncw_apps_menu", + "path": "apps-external/ncw_apps_menu", + "has_npm": true, + "has_composer": true, + "makefile_target": "build_ncw_apps_menu_app" + }, + { + "name": "ncw_mailtemplate", + "path": "apps-external/ncw_mailtemplate", + "has_npm": true, + "has_composer": true, + "makefile_target": "build_ncw_mailtemplate_app" + }, + { + "name": "notes", + "path": "apps-external/notes", + "has_npm": true, + "has_composer": true, + "makefile_target": "build_notes_app" + }, + { + "name": "notifications", + "path": "apps-external/notifications", + "has_npm": true, + "has_composer": true, + "makefile_target": "build_notifications_app" + }, + { + "name": "notify_push", + "path": "apps-external/notify_push", + "has_npm": false, + "has_composer": true, + "makefile_target": "build_notify_push_app" + }, + { + "name": "password_policy", + "path": "apps-external/password_policy", + "has_npm": true, + "has_composer": true, + "makefile_target": "build_password_policy_app" + }, + { + "name": "richdocuments", + "path": "apps-external/richdocuments", + "has_npm": true, + "has_composer": true, + "makefile_target": "build_richdocuments_app" + }, + { + "name": "spreed", + "path": "apps-external/spreed", + "has_npm": true, + "has_composer": true, + "makefile_target": "build_spreed_app" + }, + { + "name": "tables", + "path": "apps-external/tables", + "has_npm": true, + "has_composer": true, + "makefile_target": "build_tables_app" + }, + { + "name": "tasks", + "path": "apps-external/tasks", + "has_npm": true, + "has_composer": true, + "makefile_target": "build_tasks_app" + }, + { + "name": "text", + "path": "apps-external/text", + "has_npm": true, + "has_composer": true, + "makefile_target": "build_text_app" + }, + { + "name": "twofactor_totp", + "path": "apps-external/twofactor_totp", + "has_npm": true, + "has_composer": true, + "makefile_target": "build_twofactor_totp_app" + }, + { + "name": "user_oidc", + "path": "apps-external/user_oidc", + "has_npm": true, + "has_composer": true, + "makefile_target": "build_user_oidc_app" + }, + { + "name": "viewer", + "path": "apps-external/viewer", + "has_npm": true, + "has_composer": true, + "makefile_target": "build_viewer_app" + }, + { + "name": "whiteboard", + "path": "apps-external/whiteboard", + "has_npm": true, + "has_composer": true, + "makefile_target": "build_whiteboard_app" + } + ]' + + # Validate JSON and output as compact format + if echo "$matrix" | jq empty 2>/dev/null; then + echo "matrix=$(echo "$matrix" | jq -c '.')" >> $GITHUB_OUTPUT + echo "Matrix configuration set successfully" + else + echo "Error: Invalid JSON in matrix configuration" + exit 1 + fi + + - name: Validate matrix against Makefile + run: | + set +e # Intentionally allow script to continue on error for custom error handling and reporting to GITHUB_STEP_SUMMARY + set -u # Exit on undefined variable + + echo "### 🔍 Matrix Validation" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + # Debug: Check if apps-external exists + echo "Checking apps-external directory..." + if [ ! -d "apps-external" ]; then + echo "❌ **Error:** apps-external directory does not exist!" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Directory listing:" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + ls -la >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + exit 1 + fi + + echo "Apps-external directory exists. Listing contents:" + ls -la apps-external/ | head -10 + + # Check if jq is available + echo "Checking if jq is installed..." + if ! command -v jq &> /dev/null; then + echo "❌ **Error:** jq is not installed!" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "jq is required for matrix generation but was not found in PATH." >> $GITHUB_STEP_SUMMARY + exit 1 + fi + echo "jq version: $(jq --version)" + + echo "Generating matrix from Makefile..." + # Capture both stdout and stderr separately to better diagnose issues + makefile_output=$(make -f IONOS/Makefile generate_external_apps_matrix_json 2>&1) + makefile_exit_code=$? + + echo "Makefile exit code: ${makefile_exit_code}" + echo "Makefile output length: ${#makefile_output}" + + # Debug: Check if GITHUB_STEP_SUMMARY is set + echo "GITHUB_STEP_SUMMARY: ${GITHUB_STEP_SUMMARY:-NOT SET}" + + # If the Makefile command failed, show the error + if [ ${makefile_exit_code} -ne 0 ]; then + echo "" + echo "=== MAKEFILE ERROR ===" + echo "Exit code: ${makefile_exit_code}" + echo "Output:" + echo "$makefile_output" + echo "=====================" + echo "" + + # Write to summary + echo "❌ **Error:** Makefile command failed with exit code ${makefile_exit_code}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "
" >> $GITHUB_STEP_SUMMARY + echo "Makefile error output" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "$makefile_output" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "
" >> $GITHUB_STEP_SUMMARY + + echo "Error written to summary file: ${GITHUB_STEP_SUMMARY}" + exit 1 + fi + + # Filter out the info message to get just the JSON + # The Makefile outputs "[i] Generating..." to stderr, but we captured everything with 2>&1 + # So we need to extract just the JSON part + generated_matrix=$(echo "$makefile_output" | grep -v '^\[i\]' || echo "$makefile_output") + + workflow_matrix='${{ steps.set-matrix.outputs.matrix }}' + + # Debug output + echo "Generated matrix length: ${#generated_matrix}" + echo "Workflow matrix length: ${#workflow_matrix}" + + # Show first 200 chars of generated matrix for debugging + if [ -n "$generated_matrix" ]; then + echo "Generated matrix preview: ${generated_matrix:0:200}..." + fi + + # Validate that we got valid JSON + if ! echo "$generated_matrix" | jq empty 2>/dev/null; then + echo "❌ **Error:** Generated matrix is not valid JSON" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "
" >> $GITHUB_STEP_SUMMARY + echo "Invalid JSON output" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "$generated_matrix" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "
" >> $GITHUB_STEP_SUMMARY + exit 1 + fi + + # Validate that we got data + if [ -z "$generated_matrix" ] || [ -z "$workflow_matrix" ]; then + echo "❌ **Error:** Failed to load matrices" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- Generated matrix empty: $([ -z "$generated_matrix" ] && echo "yes" || echo "no")" >> $GITHUB_STEP_SUMMARY + echo "- Workflow matrix empty: $([ -z "$workflow_matrix" ] && echo "yes" || echo "no")" >> $GITHUB_STEP_SUMMARY + echo "- Makefile exit code: ${makefile_exit_code}" >> $GITHUB_STEP_SUMMARY + + exit 1 + fi + + # Sort both matrices for comparison + generated_sorted=$(echo "$generated_matrix" | jq -S '.' 2>&1 || echo "ERROR") + workflow_sorted=$(echo "$workflow_matrix" | jq -S '.' 2>&1 || echo "ERROR") + + echo "Sorted matrix lengths - generated: ${#generated_sorted}, workflow: ${#workflow_sorted}" + + # Compare the two matrices + if [ "$generated_sorted" = "$workflow_sorted" ]; then + echo "✅ **Validation passed!** The workflow matrix matches the Makefile configuration." >> $GITHUB_STEP_SUMMARY + echo "" + echo "✅ Matrix validation passed!" + else + echo "❌ **Validation failed!** The workflow matrix does not match the Makefile configuration." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "Starting detailed comparison..." + + # Extract app names from both matrices + generated_apps=$(echo "$generated_matrix" | jq -r '.[].name' 2>/dev/null | sort || echo "") + workflow_apps=$(echo "$workflow_matrix" | jq -r '.[].name' 2>/dev/null | sort || echo "") + + echo "Generated apps count: $(echo "$generated_apps" | wc -l)" + echo "Workflow apps count: $(echo "$workflow_apps" | wc -l)" + + # Find missing apps (in Makefile but not in workflow) + missing_apps=$(comm -23 <(echo "$generated_apps") <(echo "$workflow_apps")) + if [ $? -ne 0 ]; then + echo "Error: comm command failed when finding missing apps." >&2 + exit 1 + fi + # Find extra apps (in workflow but not in Makefile) + extra_apps=$(comm -13 <(echo "$generated_apps") <(echo "$workflow_apps")) + if [ $? -ne 0 ]; then + echo "Error: comm command failed when finding extra apps." >&2 + exit 1 + fi + + echo "Missing apps: ${missing_apps:-none}" + echo "Extra apps: ${extra_apps:-none}" + + if [ -n "$missing_apps" ]; then + echo "#### ⚠️ Missing Apps" >> $GITHUB_STEP_SUMMARY + echo "The following apps are configured in the Makefile but missing from the workflow:" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "$missing_apps" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + fi + + if [ -n "$extra_apps" ]; then + echo "#### ⚠️ Extra Apps" >> $GITHUB_STEP_SUMMARY + echo "The following apps are in the workflow but not configured in the Makefile:" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "$extra_apps" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + fi + + # Check for configuration mismatches in common apps + common_apps=$(comm -12 <(echo "$generated_apps") <(echo "$workflow_apps") 2>/dev/null || echo "") + + echo "Common apps count: $(echo "$common_apps" | wc -l)" + + if [ -n "$common_apps" ]; then + mismatched_apps="" + + while IFS= read -r app; do + [ -z "$app" ] && continue + gen_config=$(echo "$generated_matrix" | jq -c --arg app "$app" '.[] | select(.name == $app)' 2>/dev/null || echo "") + wf_config=$(echo "$workflow_matrix" | jq -c --arg app "$app" '.[] | select(.name == $app)' 2>/dev/null || echo "") + + if [ -n "$gen_config" ] && [ -n "$wf_config" ] && [ "$gen_config" != "$wf_config" ]; then + mismatched_apps="${mismatched_apps}${app}"$'\n' + fi + done <<< "$common_apps" + + echo "Mismatched apps: ${mismatched_apps:-none}" + + if [ -n "$mismatched_apps" ]; then + echo "#### ⚠️ Configuration Mismatches" >> $GITHUB_STEP_SUMMARY + echo "The following apps have different configurations (has_npm, has_composer, etc.):" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "$mismatched_apps" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "
" >> $GITHUB_STEP_SUMMARY + echo "📋 Detailed differences" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo '```diff' >> $GITHUB_STEP_SUMMARY + + while IFS= read -r app; do + [ -z "$app" ] && continue + echo "=== $app ===" >> $GITHUB_STEP_SUMMARY + diff -u --label "Workflow" --label "Makefile" \ + <(echo "$workflow_matrix" | jq --arg app "$app" '.[] | select(.name == $app)' 2>/dev/null || echo "{}") \ + <(echo "$generated_matrix" | jq --arg app "$app" '.[] | select(.name == $app)' 2>/dev/null || echo "{}") \ + >> $GITHUB_STEP_SUMMARY 2>&1 || true + done <<< "$mismatched_apps" + + echo '```' >> $GITHUB_STEP_SUMMARY + echo "
" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + fi + fi + + # Provide fix instructions + echo "#### 🔧 How to Fix" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Run this command locally to generate the correct matrix:" >> $GITHUB_STEP_SUMMARY + echo '```bash' >> $GITHUB_STEP_SUMMARY + echo "make -f IONOS/Makefile generate_external_apps_matrix_json" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Then update the \`matrix\` variable in \`.github/workflows/build-artifact.yml\` in the set-matrix step with the generated output." >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + # Show full diff in expandable section + echo "
" >> $GITHUB_STEP_SUMMARY + echo "📄 Full matrix comparison" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Workflow Matrix:**" >> $GITHUB_STEP_SUMMARY + echo '```json' >> $GITHUB_STEP_SUMMARY + echo "$workflow_matrix" | jq '.' 2>/dev/null >> $GITHUB_STEP_SUMMARY || echo "$workflow_matrix" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Makefile Matrix:**" >> $GITHUB_STEP_SUMMARY + echo '```json' >> $GITHUB_STEP_SUMMARY + echo "$generated_matrix" | jq '.' 2>/dev/null >> $GITHUB_STEP_SUMMARY || echo "$generated_matrix" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "
" >> $GITHUB_STEP_SUMMARY + + echo "" + echo "❌ ERROR: Matrix validation failed!" + echo "See the job summary for details on what's wrong and how to fix it." + echo "Summary file size: $(wc -c < $GITHUB_STEP_SUMMARY || echo 0) bytes" + exit 1 + fi + + build-external-apps: + runs-on: ubuntu-latest + needs: prepare-matrix + + permissions: + contents: read + + name: build-external-apps + strategy: + max-parallel: 20 + matrix: + app: ${{ fromJson(needs.prepare-matrix.outputs.external-apps-matrix) }} + + steps: + - name: Checkout server + uses: actions/checkout@v5 + with: + submodules: true + fetch-depth: '1' + + - name: Set up node with version from package.json's engines + if: matrix.app.has_npm + uses: actions/setup-node@v5 + with: + node-version-file: "package.json" + cache: 'npm' + cache-dependency-path: ${{ matrix.app.path }}/package-lock.json + + - name: Setup PHP with PECL extension + if: matrix.app.has_composer + uses: shivammathur/setup-php@c541c155eee45413f5b09a52248675b1a2575231 #v2.31.1 + with: + tools: composer:v2 + extensions: gd, zip, curl, xml, xmlrpc, mbstring, sqlite, xdebug, pgsql, intl, imagick, gmp, apcu, bcmath, redis, soap, imap, opcache + env: + runner: ubuntu-latest + + - name: Cache Composer dependencies for ${{ matrix.app.name }} + if: matrix.app.has_composer + uses: actions/cache@v4 + with: + path: ${{ matrix.app.path }}/vendor + key: ${{ runner.os }}-composer-${{ matrix.app.name }}-${{ hashFiles(format('{0}/composer.lock', matrix.app.path)) }} + restore-keys: | + ${{ runner.os }}-composer-${{ matrix.app.name }}- + + - name: Build ${{ matrix.app.name }} app + run: make -f IONOS/Makefile ${{ matrix.app.makefile_target }} + + - name: Upload ${{ matrix.app.name }} build artifacts + uses: actions/upload-artifact@v4 + with: + retention-days: 1 + name: external-app-build-${{ matrix.app.name }} + path: | + ${{ matrix.app.path }} + !${{ matrix.app.path }}/node_modules + + build-artifact: + runs-on: ubuntu-latest + needs: [prepare-matrix, build-external-apps] + + permissions: + contents: read + + outputs: + NC_VERSION: ${{ steps.get_nc_version.outputs.NC_VERSION }} + + name: build-artifact + steps: + - name: Checkout server + uses: actions/checkout@v5 + with: + submodules: true + fetch-depth: '1' + + - name: Download build external apps + uses: actions/download-artifact@v5 + with: + pattern: external-app-build-* + path: apps-external/ + + - name: Reorganize downloaded apps-external artifacts + run: | + cd apps-external/ + + echo "Initial structure:" + ls -la + + # Move contents from external-app-build-* directories to their target directories + for artifact_dir in external-app-build-*; do + if [ -d "$artifact_dir" ]; then + # Extract app name from artifact directory name + app_name=${artifact_dir#external-app-build-} + + echo "Processing artifact: $artifact_dir -> $app_name" + + # If target directory exists, merge the contents from the artifact directory containing build artifacts + if [ -d "$app_name" ]; then + echo "Target directory $app_name exists, merging contents from $artifact_dir" + # Copy contents from artifact directory to target directory + cp -r "$artifact_dir"/* "$app_name"/ + # Remove the now-empty artifact directory + rm -rf "$artifact_dir" + else + # Move the artifact directory to the proper app name + echo "Moving $artifact_dir to $app_name" + mv "$artifact_dir" "$app_name" + fi + fi + done + + echo "Reorganization complete. Final structure:" + ls -la + + - name: Verify downloaded artifacts structure + run: | + echo "External apps structure:" + ls -la apps-external/ + for app_dir in apps-external/*/; do + if [ -d "$app_dir" ]; then + echo "Contents of $app_dir:" + ls -la "$app_dir" + fi + done + + - name: Set up node with version from package.json's engines + uses: actions/setup-node@v5 + with: + node-version-file: "package.json" + cache: 'npm' + + - name: Install Dependencies + run: sudo apt-get update && sudo apt-get install -y make zip unzip + + - name: Print dependencies versions + run: make --version && node --version && npm --version + + - name: Setup PHP with PECL extension + uses: shivammathur/setup-php@c541c155eee45413f5b09a52248675b1a2575231 #v2.31.1 + with: + tools: composer:v2 + extensions: gd, zip, curl, xml, xmlrpc, mbstring, sqlite, xdebug, pgsql, intl, imagick, gmp, apcu, bcmath, redis, soap, imap, opcache + env: + runner: ubuntu-latest + + - name: Cache Composer dependencies + uses: actions/cache@v4 + with: + path: vendor + key: ${{ runner.os }}-composer-${{ hashFiles('**/composer.lock') }} + restore-keys: | + ${{ runner.os }}-composer- + + - name: Print PHP install + run: php -i && php -m + + - name: Build Nextcloud + run: make -f IONOS/Makefile build_ncw + + - name: Add config partials + run: make -f IONOS/Makefile add_config_partials + + - name: Zip dependencies + run: make -f IONOS/Makefile zip_dependencies TARGET_PACKAGE_NAME=${{ env.TARGET_PACKAGE_NAME }} + + - name: Get NC version + id: get_nc_version + continue-on-error: false + run: | + NC_VERSION=$(jq -r '.ncVersion' version.json) + echo "NC_VERSION: $NC_VERSION" + + if [ -z "$NC_VERSION" ]; then + echo "NC_VERSION is empty" + exit 1 + fi + + echo "NC_VERSION=$NC_VERSION" >> $GITHUB_OUTPUT + + - name: Upload artifact result for job build-artifact + uses: actions/upload-artifact@v4 + with: + retention-days: 30 + name: nextcloud_workspace_build_artifact + path: ${{ env.TARGET_PACKAGE_NAME }} + + - name: Show changes on failure + if: failure() + run: | + git status + git --no-pager diff + exit 1 # make it red to grab attention + + upload-to-artifactory: + runs-on: self-hosted + # Upload the artifact to the Artifactory repository on PR *OR* on "ionos-dev|ionos-stable" branch push defined in the on:push:branches + if: github.event_name == 'pull_request' || github.ref_name == 'ionos-dev' || github.ref_name == 'ionos-stable' + + name: Push to artifactory + needs: [prepare-matrix, build-external-apps, build-artifact] + + outputs: + ARTIFACTORY_LAST_BUILD_PATH: ${{ steps.artifactory_upload.outputs.ARTIFACTORY_LAST_BUILD_PATH }} + + env: + BUILD_NAME: "nextcloud-workspace-snapshot" + + steps: + - name: Check prerequisites + run: | + # count the number of secrets that are set + echo "Checking if required secrets are set..." + error_count=0 + + if [ -z "${{ secrets.JF_ARTIFACTORY_URL }}" ]; then + # output error to github actions log + echo "::error::JF_ARTIFACTORY_URL secret is not set" + error_count=$((error_count + 1)) + fi + + if [ -z "${{ secrets.JF_ARTIFACTORY_USER }}" ]; then + echo "::error::JF_ARTIFACTORY_USER secret is not set" + error_count=$((error_count + 1)) + fi + + if [ -z "${{ secrets.JF_ACCESS_TOKEN }}" ]; then + echo "::error::JF_ACCESS_TOKEN secret is not set" + error_count=$((error_count + 1)) + fi + + # abort if any of the required secrets are not set + if [ $error_count -ne 0 ]; then + echo "::error::Required secrets are not set. Aborting." + exit 1 + fi + + - name: Download artifact zip + uses: actions/download-artifact@v5 + with: + name: nextcloud_workspace_build_artifact + + # This action sets up the JFrog CLI with the Artifactory URL and access token + - uses: jfrog/setup-jfrog-cli@7c95feb32008765e1b4e626b078dfd897c4340ad # v4.4.1 + env: + JF_URL: ${{ secrets.JF_ARTIFACTORY_URL }} + JF_USER: ${{ secrets.JF_ARTIFACTORY_USER }} + JF_ACCESS_TOKEN: ${{ secrets.JF_ACCESS_TOKEN }} + + - name: Ping the JF server + run: | + # Ping the server + jf rt ping + + - name: Upload build to artifactory + id: artifactory_upload + run: | + # PR builds are stored in a separate directory as "dev/pr/nextcloud-workspace-pr-.zip" + # Push to "ionos-dev" branch is stored as "dev/nextcloud-workspace-.zip" + + ARTIFACTORY_STAGE_PREFIX="dev" + + # set ARTIFACTORY_STAGE_PREFIX=stable on ionos-stable branch + if [ "${{ github.ref_name }}" == "ionos-stable" ]; then + ARTIFACTORY_STAGE_PREFIX="stable" + fi + + export PATH_TO_DIRECTORY="${{ env.ARTIFACTORY_REPOSITORY_SNAPSHOT }}/${ARTIFACTORY_STAGE_PREFIX}" + PATH_TO_FILE="pr/nextcloud-workspace-pr-${{ github.event.pull_request.number }}.zip" + + if [ -z "${{ github.event.pull_request.number }}" ]; then + PATH_TO_FILE="nextcloud-workspace-${{ needs.build-artifact.outputs.NC_VERSION }}.zip" + fi + + export PATH_TO_LATEST_ARTIFACT="${PATH_TO_DIRECTORY}/${PATH_TO_FILE}" + + # Promote current build to the "latest" dev build + jf rt upload "${{ env.TARGET_PACKAGE_NAME }}" \ + --build-name "${{ env.BUILD_NAME }}" \ + --build-number ${{ github.run_number }} \ + --target-props "build.nc_version=${{ needs.build-artifact.outputs.NC_VERSION }};vcs.branch=${{ github.ref }};vcs.revision=${{ github.sha }}" \ + $PATH_TO_LATEST_ARTIFACT + + echo "ARTIFACTORY_LAST_BUILD_PATH=${PATH_TO_LATEST_ARTIFACT}" >> $GITHUB_OUTPUT + + - name: Show changes on failure + if: failure() + run: | + git status + git --no-pager diff + exit 1 # make it red to grab attention + + nextcloud-workspace-artifact-to-ghcr_io: + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + name: Push artifact to ghcr.io + needs: [prepare-matrix, build-external-apps, build-artifact] + + steps: + - name: Download artifact zip + uses: actions/download-artifact@v5 + with: + name: nextcloud_workspace_build_artifact + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}" + + - name: Create Dockerfile + run: | + cat >Dockerfile << EOF + FROM busybox as builder + COPY ./${{ env.TARGET_PACKAGE_NAME }} / + WORKDIR /builder + RUN unzip /${{ env.TARGET_PACKAGE_NAME }} -d /builder + + FROM scratch + WORKDIR /app + VOLUME /app + COPY --from=builder /builder /app + EOF + + - name: Build and push Docker image + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Show changes on failure + if: failure() + run: | + echo "Git status:" + git status + echo "Git diff:" + git diff + exit 1 # make it red to grab attention + + trigger-remote-dev-worflow: + runs-on: self-hosted + + name: Trigger remote workflow + needs: [ build-artifact, upload-to-artifactory ] + # Trigger remote build on "ionos-dev|ionos-stable" branch *push* defined in the on:push:branches + if: github.event_name == 'push' && ( github.ref_name == 'ionos-dev' || github.ref_name == 'ionos-stable' ) + steps: + - name: Trigger remote workflow + run: | + # Enable command echo for debugging purposes + set -x + + # Determine build type based on branch: + # - 'ionos-dev' branch triggers 'dev' build type + # - 'ionos-stable' branch triggers 'stable' build type + BUILD_TYPE="dev" + + # Override build type for stable branch + if [ "${{ github.ref_name }}" == "ionos-stable" ]; then + BUILD_TYPE="stable" + fi + + # Trigger GitLab pipeline via webhook with build artifacts and metadata + # Passes GitHub context variables to remote GitLab workflow + curl \ + --silent \ + --insecure \ + --request POST \ + --fail-with-body \ + -o response.json \ + --form token=${{ secrets.GITLAB_TOKEN }} \ + --form ref="stable" \ + --form "variables[GITHUB_SHA]=${{ github.sha }}" \ + --form "variables[ARTIFACTORY_LAST_BUILD_PATH]=${{ needs.upload-to-artifactory.outputs.ARTIFACTORY_LAST_BUILD_PATH }}" \ + --form "variables[NC_VERSION]=${{ needs.build-artifact.outputs.NC_VERSION }}" \ + --form "variables[BUILD_ID]=${{ github.run_id }}" \ + --form "variables[BUILD_TYPE]=${BUILD_TYPE}" \ + "${{ secrets.GITLAB_TRIGGER_URL }}" || ( RETCODE="$?"; jq . response.json; exit "$RETCODE" ) + + # Disable command echo + set +x + + # Print and parse json + # jq . response.json + echo "json<> $GITHUB_OUTPUT + cat response.json >> $GITHUB_OUTPUT + echo "END" >> $GITHUB_OUTPUT + echo "web_url<> $GITHUB_OUTPUT + cat response.json | jq --raw-output '.web_url' >> $GITHUB_OUTPUT + echo "END" >> $GITHUB_OUTPUT + + - name: Show changes on failure + if: failure() + run: | + git status + git --no-pager diff + exit 1 # make it red to grab attention From 02858e2d039f55276de11bc187b60e745717bb21 Mon Sep 17 00:00:00 2001 From: "Misha M.-Kupriyanov" Date: Wed, 12 Nov 2025 12:14:41 +0100 Subject: [PATCH 04/16] IONOS(build): update artifact filename to include '-original' suffix in order not to overwrite optimized artifact Signed-off-by: Misha M.-Kupriyanov --- .github/workflows/build-artifact-original.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-artifact-original.yml b/.github/workflows/build-artifact-original.yml index a2cbfdb849649..8d76945d5b7d4 100644 --- a/.github/workflows/build-artifact-original.yml +++ b/.github/workflows/build-artifact-original.yml @@ -780,10 +780,10 @@ jobs: fi export PATH_TO_DIRECTORY="${{ env.ARTIFACTORY_REPOSITORY_SNAPSHOT }}/${ARTIFACTORY_STAGE_PREFIX}" - PATH_TO_FILE="pr/nextcloud-workspace-pr-${{ github.event.pull_request.number }}.zip" + PATH_TO_FILE="pr/nextcloud-workspace-pr-${{ github.event.pull_request.number }}-original.zip" if [ -z "${{ github.event.pull_request.number }}" ]; then - PATH_TO_FILE="nextcloud-workspace-${{ needs.build-artifact.outputs.NC_VERSION }}.zip" + PATH_TO_FILE="nextcloud-workspace-${{ needs.build-artifact.outputs.NC_VERSION }}-original.zip" fi export PATH_TO_LATEST_ARTIFACT="${PATH_TO_DIRECTORY}/${PATH_TO_FILE}" From 4cc5c2f590ce4099f08598b7c8c86ed69ff1da74 Mon Sep 17 00:00:00 2001 From: "Misha M.-Kupriyanov" Date: Tue, 11 Nov 2025 15:33:09 +0100 Subject: [PATCH 05/16] IONOS(build): Optimize build workflow - only rebuild changed apps Signed-off-by: Misha M.-Kupriyanov --- .github/workflows/build-artifact.yml | 168 +++++++++++++++++++++++++-- 1 file changed, 158 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-artifact.yml b/.github/workflows/build-artifact.yml index a2cbfdb849649..a8e3bc3e8f8da 100644 --- a/.github/workflows/build-artifact.yml +++ b/.github/workflows/build-artifact.yml @@ -1,12 +1,13 @@ -name: Build Nextcloud Workspace artifact +name: Build Nextcloud Workspace artifact (Optimized) # SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors # SPDX-FileCopyrightText: 2025 STRATO AG # SPDX-License-Identifier: AGPL-3.0-or-later -# The Nextcloud Workspace source is packaged as a container image. -# This is a workaround because releases cannot be created without tags, -# and we want to be able to create snapshots from branches. +# Optimized build workflow that uses cache-based detection +# - Checks cache for each app's current SHA +# - Only builds apps with no cached build +# - Significantly reduces build time through smart caching on: pull_request: @@ -48,21 +49,23 @@ jobs: prepare-matrix: runs-on: ubuntu-latest outputs: + apps-to-build: ${{ steps.detect.outputs.apps-to-build }} external-apps-matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - name: Checkout repository uses: actions/checkout@v5 with: submodules: true - fetch-depth: '1' + fetch-depth: 1 # Shallow clone is fine - we only need current state - name: Install dependencies run: sudo apt-get update && sudo apt-get install -y make jq - - name: Set matrix + - name: Generate full apps matrix id: set-matrix run: | # Create matrix configuration as a compact JSON string + # This is the same as the original workflow matrix='[ { "name": "activity", @@ -512,6 +515,91 @@ jobs: exit 1 fi + - name: Collect apps and their SHAs for cache-based building + id: detect + run: | + set -e # Exit on error + set -u # Exit on undefined variable + set -o pipefail # Exit if any command in pipeline fails + + echo "Collecting app SHAs for cache-based building..." + echo "(Cache checking happens in parallel build jobs)" + echo "" + + # Get the matrix from previous step + MATRIX='${{ steps.set-matrix.outputs.matrix }}' + + # Build JSON array for apps that actually need building + APPS_TO_BUILD="[]" + APPS_CHECKED=0 + APPS_CACHED=0 + APPS_TO_BUILD_COUNT=0 + + # Create temporary file for cache check results + CACHE_CHECK_RESULTS=$(mktemp) + + # Iterate through each app in the matrix + while IFS= read -r app_json; do + APP_NAME=$(echo "$app_json" | jq -r '.name') + APP_PATH=$(echo "$app_json" | jq -r '.path') + + APPS_CHECKED=$((APPS_CHECKED + 1)) + + # Get current submodule SHA + if [ -d "$APP_PATH" ]; then + CURRENT_SHA=$(git -C "$APP_PATH" rev-parse HEAD 2>/dev/null || echo "") + else + echo "⊘ $APP_NAME - directory not found, will build" + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c --arg app "$APP_NAME" --arg sha "unknown" '. + [{name: $app, sha: $sha}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + continue + fi + + if [ -z "$CURRENT_SHA" ]; then + echo "⊘ $APP_NAME - not a git repo, will build" + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c --arg app "$APP_NAME" --arg sha "unknown" '. + [{name: $app, sha: $sha}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + continue + fi + + # Cache key that would be used for this app + CACHE_KEY="app-build-${APP_NAME}-${CURRENT_SHA}" + + # Check if cache exists using GitHub API + # We'll use a simple approach: try to restore with lookup-only via actions/cache API + # For now, we'll mark all apps to build and let the build job check cache + # (GitHub Actions doesn't provide easy cache lookup in bash) + + echo " $APP_NAME (SHA: ${CURRENT_SHA:0:8}) - cache key: $CACHE_KEY" + + # Since we can't easily check cache existence here, we'll add all apps + # The build job will use lookup-only to check and skip if cached + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" '. + [{name: $app, sha: $sha}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + + done < <(echo "$MATRIX" | jq -c '.[]') + + echo "" + echo "Summary:" + echo " Apps checked: $APPS_CHECKED" + echo " Apps to build/check: $APPS_TO_BUILD_COUNT" + + # Validate that we built valid JSON + if ! echo "$APPS_TO_BUILD" | jq empty 2>/dev/null; then + echo "ERROR: Failed to build valid JSON for apps-to-build" + echo "Content: $APPS_TO_BUILD" + exit 1 + fi + + # Output app list with SHAs for the build job to use + # Use proper multiline output format for GitHub Actions + echo "apps-to-build<> $GITHUB_OUTPUT + echo "$APPS_TO_BUILD" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + echo "" + echo "Build jobs will check cache and skip if cached build exists" + build-external-apps: runs-on: ubuntu-latest needs: prepare-matrix @@ -526,14 +614,65 @@ jobs: app: ${{ fromJson(needs.prepare-matrix.outputs.external-apps-matrix) }} steps: + # Get current SHA for this app's submodule + - name: Get submodule SHA + id: get-sha + run: | + # We'll check cache based on the SHA stored in detect output + APPS_TO_BUILD='${{ needs.prepare-matrix.outputs.apps-to-build }}' + APP_NAME='${{ matrix.app.name }}' + + # Extract SHA for this app from the apps-to-build list + APP_SHA=$(echo "$APPS_TO_BUILD" | jq -r --arg name "$APP_NAME" '.[] | select(.name == $name) | .sha') + + if [ -z "$APP_SHA" ]; then + echo "⚠️ Could not find SHA for $APP_NAME, will build" + echo "should-build=true" >> $GITHUB_OUTPUT + echo "app-sha=unknown" >> $GITHUB_OUTPUT + else + echo "app-sha=$APP_SHA" >> $GITHUB_OUTPUT + echo "SHA for $APP_NAME: ${APP_SHA:0:12}" + fi + + # Try to restore from cache based on SHA + - name: Restore cached build artifacts + id: cache-restore + uses: actions/cache/restore@v4 + with: + path: ${{ matrix.app.path }} + key: app-build-${{ matrix.app.name }}-${{ steps.get-sha.outputs.app-sha }} + lookup-only: true # Just check if cache exists, don't restore yet + + # Determine if we need to build + - name: Check if build needed + id: should-build + run: | + if [ "${{ steps.cache-restore.outputs.cache-hit }}" = "true" ]; then + echo "should-build=false" >> $GITHUB_OUTPUT + echo "✓ Using cached build for ${{ matrix.app.name }} (SHA: ${{ steps.get-sha.outputs.app-sha }})" + else + echo "should-build=true" >> $GITHUB_OUTPUT + echo "⚡ Building ${{ matrix.app.name }} (SHA: ${{ steps.get-sha.outputs.app-sha }}) - no cache found" + fi + + # Checkout only if we need to build - name: Checkout server + if: steps.should-build.outputs.should-build == 'true' uses: actions/checkout@v5 with: submodules: true - fetch-depth: '1' + fetch-depth: 1 + + # Restore cache for real if building (to get the built files) + - name: Restore cached build for upload + if: steps.should-build.outputs.should-build == 'false' + uses: actions/cache/restore@v4 + with: + path: ${{ matrix.app.path }} + key: app-build-${{ matrix.app.name }}-${{ steps.get-sha.outputs.app-sha }} - name: Set up node with version from package.json's engines - if: matrix.app.has_npm + if: steps.should-build.outputs.should-build == 'true' && matrix.app.has_npm uses: actions/setup-node@v5 with: node-version-file: "package.json" @@ -541,7 +680,7 @@ jobs: cache-dependency-path: ${{ matrix.app.path }}/package-lock.json - name: Setup PHP with PECL extension - if: matrix.app.has_composer + if: steps.should-build.outputs.should-build == 'true' && matrix.app.has_composer uses: shivammathur/setup-php@c541c155eee45413f5b09a52248675b1a2575231 #v2.31.1 with: tools: composer:v2 @@ -550,7 +689,7 @@ jobs: runner: ubuntu-latest - name: Cache Composer dependencies for ${{ matrix.app.name }} - if: matrix.app.has_composer + if: steps.should-build.outputs.should-build == 'true' && matrix.app.has_composer uses: actions/cache@v4 with: path: ${{ matrix.app.path }}/vendor @@ -559,8 +698,17 @@ jobs: ${{ runner.os }}-composer-${{ matrix.app.name }}- - name: Build ${{ matrix.app.name }} app + if: steps.should-build.outputs.should-build == 'true' run: make -f IONOS/Makefile ${{ matrix.app.makefile_target }} + # Save built app to cache for future runs + - name: Save build artifacts to cache + if: steps.should-build.outputs.should-build == 'true' + uses: actions/cache/save@v4 + with: + path: ${{ matrix.app.path }} + key: app-build-${{ matrix.app.name }}-${{ steps.get-sha.outputs.app-sha }} + - name: Upload ${{ matrix.app.name }} build artifacts uses: actions/upload-artifact@v4 with: From 49f2ec9743b0fa63bfde003d143f4ec2f3686853 Mon Sep 17 00:00:00 2001 From: "Misha M.-Kupriyanov" Date: Tue, 11 Nov 2025 17:55:52 +0100 Subject: [PATCH 06/16] IONOS(build): enhance cache reporting and cleanup in build workflow Signed-off-by: Misha M.-Kupriyanov --- .github/workflows/build-artifact.yml | 41 +++++++++++++++++++--------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-artifact.yml b/.github/workflows/build-artifact.yml index a8e3bc3e8f8da..ceac84ad0132f 100644 --- a/.github/workflows/build-artifact.yml +++ b/.github/workflows/build-artifact.yml @@ -523,7 +523,6 @@ jobs: set -o pipefail # Exit if any command in pipeline fails echo "Collecting app SHAs for cache-based building..." - echo "(Cache checking happens in parallel build jobs)" echo "" # Get the matrix from previous step @@ -532,11 +531,15 @@ jobs: # Build JSON array for apps that actually need building APPS_TO_BUILD="[]" APPS_CHECKED=0 - APPS_CACHED=0 APPS_TO_BUILD_COUNT=0 - # Create temporary file for cache check results - CACHE_CHECK_RESULTS=$(mktemp) + # Create temporary directory for cache check results + TEMP_DIR=$(mktemp -d) + + echo "### 📦 Cache Status Report" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "| App | SHA | Cache Key | Status |" >> $GITHUB_STEP_SUMMARY + echo "|-----|-----|-----------|--------|" >> $GITHUB_STEP_SUMMARY # Iterate through each app in the matrix while IFS= read -r app_json; do @@ -550,6 +553,7 @@ jobs: CURRENT_SHA=$(git -C "$APP_PATH" rev-parse HEAD 2>/dev/null || echo "") else echo "⊘ $APP_NAME - directory not found, will build" + echo "| $APP_NAME | N/A | N/A | ⊘ Directory not found |" >> $GITHUB_STEP_SUMMARY APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c --arg app "$APP_NAME" --arg sha "unknown" '. + [{name: $app, sha: $sha}]') APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) continue @@ -557,6 +561,7 @@ jobs: if [ -z "$CURRENT_SHA" ]; then echo "⊘ $APP_NAME - not a git repo, will build" + echo "| $APP_NAME | N/A | N/A | ⊘ Not a git repo |" >> $GITHUB_STEP_SUMMARY APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c --arg app "$APP_NAME" --arg sha "unknown" '. + [{name: $app, sha: $sha}]') APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) continue @@ -564,25 +569,35 @@ jobs: # Cache key that would be used for this app CACHE_KEY="app-build-${APP_NAME}-${CURRENT_SHA}" + SHORT_SHA="${CURRENT_SHA:0:8}" - # Check if cache exists using GitHub API - # We'll use a simple approach: try to restore with lookup-only via actions/cache API - # For now, we'll mark all apps to build and let the build job check cache - # (GitHub Actions doesn't provide easy cache lookup in bash) + echo " Checking $APP_NAME (SHA: $SHORT_SHA)..." - echo " $APP_NAME (SHA: ${CURRENT_SHA:0:8}) - cache key: $CACHE_KEY" + # Note: We collect all apps here. The actual cache existence check + # happens in the build job using actions/cache/restore with lookup-only + # because GitHub Actions doesn't provide a simple way to check cache + # existence in bash without using the cache action. + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$CACHE_KEY\` | ⏳ Will check in build job |" >> $GITHUB_STEP_SUMMARY - # Since we can't easily check cache existence here, we'll add all apps - # The build job will use lookup-only to check and skip if cached APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" '. + [{name: $app, sha: $sha}]') APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) done < <(echo "$MATRIX" | jq -c '.[]') + # Clean up temp directory + rm -rf "$TEMP_DIR" + + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Summary:**" >> $GITHUB_STEP_SUMMARY + echo "- Apps checked: $APPS_CHECKED" >> $GITHUB_STEP_SUMMARY + echo "- Apps to build/verify: $APPS_TO_BUILD_COUNT" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "> **Note:** Cache existence is verified in parallel during the build job for each app." >> $GITHUB_STEP_SUMMARY + echo "" echo "Summary:" echo " Apps checked: $APPS_CHECKED" - echo " Apps to build/check: $APPS_TO_BUILD_COUNT" + echo " Apps to build/verify: $APPS_TO_BUILD_COUNT" # Validate that we built valid JSON if ! echo "$APPS_TO_BUILD" | jq empty 2>/dev/null; then @@ -598,7 +613,7 @@ jobs: echo "EOF" >> $GITHUB_OUTPUT echo "" - echo "Build jobs will check cache and skip if cached build exists" + echo "✓ Build jobs will check cache and skip if cached build exists" build-external-apps: runs-on: ubuntu-latest From 5dc867aade4bb8dcda6eeacdf5434aee51217fee Mon Sep 17 00:00:00 2001 From: "Misha M.-Kupriyanov" Date: Tue, 11 Nov 2025 18:01:40 +0100 Subject: [PATCH 07/16] IONOS(build): improve cache handling and reporting in build workflow Signed-off-by: Misha M.-Kupriyanov --- .github/workflows/build-artifact.yml | 255 ++++++++++++++++++--------- 1 file changed, 168 insertions(+), 87 deletions(-) diff --git a/.github/workflows/build-artifact.yml b/.github/workflows/build-artifact.yml index ceac84ad0132f..2ae4d2a62a7a5 100644 --- a/.github/workflows/build-artifact.yml +++ b/.github/workflows/build-artifact.yml @@ -51,6 +51,7 @@ jobs: outputs: apps-to-build: ${{ steps.detect.outputs.apps-to-build }} external-apps-matrix: ${{ steps.set-matrix.outputs.matrix }} + has-cached-apps: ${{ steps.detect.outputs.has-cached-apps }} steps: - name: Checkout repository uses: actions/checkout@v5 @@ -517,12 +518,14 @@ jobs: - name: Collect apps and their SHAs for cache-based building id: detect + env: + GH_TOKEN: ${{ github.token }} run: | set -e # Exit on error set -u # Exit on undefined variable set -o pipefail # Exit if any command in pipeline fails - echo "Collecting app SHAs for cache-based building..." + echo "Collecting app SHAs and checking cache status..." echo "" # Get the matrix from previous step @@ -531,11 +534,9 @@ jobs: # Build JSON array for apps that actually need building APPS_TO_BUILD="[]" APPS_CHECKED=0 + APPS_CACHED=0 APPS_TO_BUILD_COUNT=0 - # Create temporary directory for cache check results - TEMP_DIR=$(mktemp -d) - echo "### 📦 Cache Status Report" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "| App | SHA | Cache Key | Status |" >> $GITHUB_STEP_SUMMARY @@ -571,33 +572,42 @@ jobs: CACHE_KEY="app-build-${APP_NAME}-${CURRENT_SHA}" SHORT_SHA="${CURRENT_SHA:0:8}" - echo " Checking $APP_NAME (SHA: $SHORT_SHA)..." - - # Note: We collect all apps here. The actual cache existence check - # happens in the build job using actions/cache/restore with lookup-only - # because GitHub Actions doesn't provide a simple way to check cache - # existence in bash without using the cache action. - echo "| $APP_NAME | \`$SHORT_SHA\` | \`$CACHE_KEY\` | ⏳ Will check in build job |" >> $GITHUB_STEP_SUMMARY + echo -n " Checking $APP_NAME (SHA: $SHORT_SHA)... " - APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" '. + [{name: $app, sha: $sha}]') - APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + # Check if cache exists using GitHub CLI + CACHE_EXISTS="false" + if gh cache list --key "$CACHE_KEY" --json key --jq ".[].key" 2>/dev/null | grep -q "^${CACHE_KEY}$"; then + CACHE_EXISTS="true" + APPS_CACHED=$((APPS_CACHED + 1)) + echo "✓ cached" + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$CACHE_KEY\` | ✅ Cached |" >> $GITHUB_STEP_SUMMARY + else + echo "⚡ needs build" + echo "| $APP_NAME | \`$SHORT_SHA\` | \`$CACHE_KEY\` | 🔨 Needs build |" >> $GITHUB_STEP_SUMMARY + APPS_TO_BUILD=$(echo "$APPS_TO_BUILD" | jq -c --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" '. + [{name: $app, sha: $sha}]') + APPS_TO_BUILD_COUNT=$((APPS_TO_BUILD_COUNT + 1)) + fi done < <(echo "$MATRIX" | jq -c '.[]') - # Clean up temp directory - rm -rf "$TEMP_DIR" - echo "" >> $GITHUB_STEP_SUMMARY echo "**Summary:**" >> $GITHUB_STEP_SUMMARY - echo "- Apps checked: $APPS_CHECKED" >> $GITHUB_STEP_SUMMARY - echo "- Apps to build/verify: $APPS_TO_BUILD_COUNT" >> $GITHUB_STEP_SUMMARY + echo "- Total apps checked: $APPS_CHECKED" >> $GITHUB_STEP_SUMMARY + echo "- ✅ Apps with cached builds: $APPS_CACHED" >> $GITHUB_STEP_SUMMARY + echo "- 🔨 Apps needing build: $APPS_TO_BUILD_COUNT" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "> **Note:** Cache existence is verified in parallel during the build job for each app." >> $GITHUB_STEP_SUMMARY + + if [ $APPS_CACHED -gt 0 ]; then + CACHE_HIT_PERCENT=$((APPS_CACHED * 100 / APPS_CHECKED)) + echo "**Cache hit rate: ${CACHE_HIT_PERCENT}%** 🎯" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + fi echo "" echo "Summary:" - echo " Apps checked: $APPS_CHECKED" - echo " Apps to build/verify: $APPS_TO_BUILD_COUNT" + echo " Total apps: $APPS_CHECKED" + echo " Cached: $APPS_CACHED" + echo " To build: $APPS_TO_BUILD_COUNT" # Validate that we built valid JSON if ! echo "$APPS_TO_BUILD" | jq empty 2>/dev/null; then @@ -612,12 +622,26 @@ jobs: echo "$APPS_TO_BUILD" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT + # Determine if there are cached apps by comparing counts + # If apps-to-build count is less than total apps, some are cached + if [ $APPS_TO_BUILD_COUNT -lt $APPS_CHECKED ]; then + echo "has-cached-apps=true" >> $GITHUB_OUTPUT + else + echo "has-cached-apps=false" >> $GITHUB_OUTPUT + fi + echo "" - echo "✓ Build jobs will check cache and skip if cached build exists" + if [ $APPS_TO_BUILD_COUNT -eq 0 ]; then + echo "🎉 All apps are cached! No builds needed." + else + echo "✓ Will build $APPS_TO_BUILD_COUNT app(s)" + fi build-external-apps: runs-on: ubuntu-latest needs: prepare-matrix + # Only run if there are apps to build + if: needs.prepare-matrix.outputs.apps-to-build != '[]' permissions: contents: read @@ -626,76 +650,54 @@ jobs: strategy: max-parallel: 20 matrix: - app: ${{ fromJson(needs.prepare-matrix.outputs.external-apps-matrix) }} + # Use the filtered list of apps that need building (not in cache) + app_info: ${{ fromJson(needs.prepare-matrix.outputs.apps-to-build) }} steps: - # Get current SHA for this app's submodule - - name: Get submodule SHA - id: get-sha + - name: Get app configuration from full matrix + id: app-config run: | - # We'll check cache based on the SHA stored in detect output - APPS_TO_BUILD='${{ needs.prepare-matrix.outputs.apps-to-build }}' - APP_NAME='${{ matrix.app.name }}' + # Get the full matrix to look up app configuration + FULL_MATRIX='${{ needs.prepare-matrix.outputs.external-apps-matrix }}' + APP_NAME='${{ matrix.app_info.name }}' - # Extract SHA for this app from the apps-to-build list - APP_SHA=$(echo "$APPS_TO_BUILD" | jq -r --arg name "$APP_NAME" '.[] | select(.name == $name) | .sha') + # Find the app configuration in the full matrix + APP_CONFIG=$(echo "$FULL_MATRIX" | jq -c --arg name "$APP_NAME" '.[] | select(.name == $name)') - if [ -z "$APP_SHA" ]; then - echo "⚠️ Could not find SHA for $APP_NAME, will build" - echo "should-build=true" >> $GITHUB_OUTPUT - echo "app-sha=unknown" >> $GITHUB_OUTPUT - else - echo "app-sha=$APP_SHA" >> $GITHUB_OUTPUT - echo "SHA for $APP_NAME: ${APP_SHA:0:12}" + if [ -z "$APP_CONFIG" ]; then + echo "ERROR: Could not find configuration for $APP_NAME" + exit 1 fi - # Try to restore from cache based on SHA - - name: Restore cached build artifacts - id: cache-restore - uses: actions/cache/restore@v4 - with: - path: ${{ matrix.app.path }} - key: app-build-${{ matrix.app.name }}-${{ steps.get-sha.outputs.app-sha }} - lookup-only: true # Just check if cache exists, don't restore yet + # Extract configuration values + APP_PATH=$(echo "$APP_CONFIG" | jq -r '.path') + HAS_NPM=$(echo "$APP_CONFIG" | jq -r '.has_npm') + HAS_COMPOSER=$(echo "$APP_CONFIG" | jq -r '.has_composer') + MAKEFILE_TARGET=$(echo "$APP_CONFIG" | jq -r '.makefile_target') - # Determine if we need to build - - name: Check if build needed - id: should-build - run: | - if [ "${{ steps.cache-restore.outputs.cache-hit }}" = "true" ]; then - echo "should-build=false" >> $GITHUB_OUTPUT - echo "✓ Using cached build for ${{ matrix.app.name }} (SHA: ${{ steps.get-sha.outputs.app-sha }})" - else - echo "should-build=true" >> $GITHUB_OUTPUT - echo "⚡ Building ${{ matrix.app.name }} (SHA: ${{ steps.get-sha.outputs.app-sha }}) - no cache found" - fi + # Set outputs + echo "path=$APP_PATH" >> $GITHUB_OUTPUT + echo "has-npm=$HAS_NPM" >> $GITHUB_OUTPUT + echo "has-composer=$HAS_COMPOSER" >> $GITHUB_OUTPUT + echo "makefile-target=$MAKEFILE_TARGET" >> $GITHUB_OUTPUT + echo "Building $APP_NAME from $APP_PATH (SHA: ${{ matrix.app_info.sha }})" - # Checkout only if we need to build - name: Checkout server - if: steps.should-build.outputs.should-build == 'true' uses: actions/checkout@v5 with: submodules: true fetch-depth: 1 - # Restore cache for real if building (to get the built files) - - name: Restore cached build for upload - if: steps.should-build.outputs.should-build == 'false' - uses: actions/cache/restore@v4 - with: - path: ${{ matrix.app.path }} - key: app-build-${{ matrix.app.name }}-${{ steps.get-sha.outputs.app-sha }} - - name: Set up node with version from package.json's engines - if: steps.should-build.outputs.should-build == 'true' && matrix.app.has_npm + if: steps.app-config.outputs.has-npm == 'true' uses: actions/setup-node@v5 with: node-version-file: "package.json" cache: 'npm' - cache-dependency-path: ${{ matrix.app.path }}/package-lock.json + cache-dependency-path: ${{ steps.app-config.outputs.path }}/package-lock.json - name: Setup PHP with PECL extension - if: steps.should-build.outputs.should-build == 'true' && matrix.app.has_composer + if: steps.app-config.outputs.has-composer == 'true' uses: shivammathur/setup-php@c541c155eee45413f5b09a52248675b1a2575231 #v2.31.1 with: tools: composer:v2 @@ -703,28 +705,94 @@ jobs: env: runner: ubuntu-latest - - name: Cache Composer dependencies for ${{ matrix.app.name }} - if: steps.should-build.outputs.should-build == 'true' && matrix.app.has_composer + - name: Cache Composer dependencies for ${{ matrix.app_info.name }} + if: steps.app-config.outputs.has-composer == 'true' uses: actions/cache@v4 with: - path: ${{ matrix.app.path }}/vendor - key: ${{ runner.os }}-composer-${{ matrix.app.name }}-${{ hashFiles(format('{0}/composer.lock', matrix.app.path)) }} + path: ${{ steps.app-config.outputs.path }}/vendor + key: ${{ runner.os }}-composer-${{ matrix.app_info.name }}-${{ hashFiles(format('{0}/composer.lock', steps.app-config.outputs.path)) }} restore-keys: | - ${{ runner.os }}-composer-${{ matrix.app.name }}- + ${{ runner.os }}-composer-${{ matrix.app_info.name }}- - - name: Build ${{ matrix.app.name }} app - if: steps.should-build.outputs.should-build == 'true' - run: make -f IONOS/Makefile ${{ matrix.app.makefile_target }} + - name: Build ${{ matrix.app_info.name }} app + run: make -f IONOS/Makefile ${{ steps.app-config.outputs.makefile-target }} # Save built app to cache for future runs - name: Save build artifacts to cache - if: steps.should-build.outputs.should-build == 'true' uses: actions/cache/save@v4 + with: + path: ${{ steps.app-config.outputs.path }} + key: app-build-${{ matrix.app_info.name }}-${{ matrix.app_info.sha }} + + - name: Upload ${{ matrix.app_info.name }} build artifacts + uses: actions/upload-artifact@v4 + with: + retention-days: 1 + name: external-app-build-${{ matrix.app_info.name }} + path: | + ${{ steps.app-config.outputs.path }} + !${{ steps.app-config.outputs.path }}/node_modules + + restore-cached-apps: + runs-on: ubuntu-latest + needs: prepare-matrix + # Only run if there are cached apps (when not all apps need to be built) + if: needs.prepare-matrix.outputs.has-cached-apps == 'true' + + permissions: + contents: read + + name: restore-cached-apps + strategy: + max-parallel: 20 + matrix: + app: ${{ fromJson(needs.prepare-matrix.outputs.external-apps-matrix) }} + + steps: + - name: Check if app needs restoring from cache + id: check-cache + run: | + APPS_TO_BUILD='${{ needs.prepare-matrix.outputs.apps-to-build }}' + APP_NAME='${{ matrix.app.name }}' + + # Check if this app is in the apps-to-build list + IN_BUILD_LIST=$(echo "$APPS_TO_BUILD" | jq -r --arg name "$APP_NAME" 'any(.[]; .name == $name)') + + if [ "$IN_BUILD_LIST" = "true" ]; then + echo "skip=true" >> $GITHUB_OUTPUT + echo "⏭️ Skipping $APP_NAME - will be built by build job" + else + echo "skip=false" >> $GITHUB_OUTPUT + echo "📦 Will restore $APP_NAME from cache" + fi + + - name: Checkout server (sparse) + if: steps.check-cache.outputs.skip == 'false' + uses: actions/checkout@v5 + with: + sparse-checkout: | + ${{ matrix.app.path }} + fetch-depth: 1 + + - name: Get cached app SHA + if: steps.check-cache.outputs.skip == 'false' + id: get-sha + run: | + cd ${{ matrix.app.path }} + APP_SHA=$(git rev-parse HEAD 2>/dev/null || echo "unknown") + echo "app-sha=$APP_SHA" >> $GITHUB_OUTPUT + echo "SHA for ${{ matrix.app.name }}: ${APP_SHA:0:12}" + + - name: Restore cached build from cache + if: steps.check-cache.outputs.skip == 'false' + uses: actions/cache/restore@v4 with: path: ${{ matrix.app.path }} - key: app-build-${{ matrix.app.name }}-${{ steps.get-sha.outputs.app-sha }} + key: app-build-${{ matrix.app.name }}-${{ matrix.app.sha }} + fail-on-cache-miss: true - - name: Upload ${{ matrix.app.name }} build artifacts + - name: Upload cached ${{ matrix.app.name }} build artifacts + if: steps.check-cache.outputs.skip == 'false' uses: actions/upload-artifact@v4 with: retention-days: 1 @@ -735,7 +803,12 @@ jobs: build-artifact: runs-on: ubuntu-latest - needs: [prepare-matrix, build-external-apps] + needs: [prepare-matrix, build-external-apps, restore-cached-apps] + # Always run this job, even if restore-cached-apps is skipped + if: | + always() && + (needs.build-external-apps.result == 'success' || needs.build-external-apps.result == 'skipped') && + (needs.restore-cached-apps.result == 'success' || needs.restore-cached-apps.result == 'skipped') permissions: contents: read @@ -872,10 +945,12 @@ jobs: upload-to-artifactory: runs-on: self-hosted # Upload the artifact to the Artifactory repository on PR *OR* on "ionos-dev|ionos-stable" branch push defined in the on:push:branches - if: github.event_name == 'pull_request' || github.ref_name == 'ionos-dev' || github.ref_name == 'ionos-stable' + if: | + (github.event_name == 'pull_request' || github.ref_name == 'ionos-dev' || github.ref_name == 'ionos-stable') && + needs.build-artifact.result == 'success' name: Push to artifactory - needs: [prepare-matrix, build-external-apps, build-artifact] + needs: [prepare-matrix, build-external-apps, restore-cached-apps, build-artifact] outputs: ARTIFACTORY_LAST_BUILD_PATH: ${{ steps.artifactory_upload.outputs.ARTIFACTORY_LAST_BUILD_PATH }} @@ -969,13 +1044,15 @@ jobs: nextcloud-workspace-artifact-to-ghcr_io: runs-on: ubuntu-latest + # Only run if build-artifact succeeded + if: needs.build-artifact.result == 'success' permissions: contents: read packages: write name: Push artifact to ghcr.io - needs: [prepare-matrix, build-external-apps, build-artifact] + needs: [prepare-matrix, build-external-apps, restore-cached-apps, build-artifact] steps: - name: Download artifact zip @@ -1031,9 +1108,13 @@ jobs: runs-on: self-hosted name: Trigger remote workflow - needs: [ build-artifact, upload-to-artifactory ] + needs: [build-artifact, upload-to-artifactory] # Trigger remote build on "ionos-dev|ionos-stable" branch *push* defined in the on:push:branches - if: github.event_name == 'push' && ( github.ref_name == 'ionos-dev' || github.ref_name == 'ionos-stable' ) + if: | + github.event_name == 'push' && + (github.ref_name == 'ionos-dev' || github.ref_name == 'ionos-stable') && + needs.build-artifact.result == 'success' && + needs.upload-to-artifactory.result == 'success' steps: - name: Trigger remote workflow run: | From 128828722e86bcfe5fdec2f7e245b2480adfa70b Mon Sep 17 00:00:00 2001 From: "Misha M.-Kupriyanov" Date: Wed, 12 Nov 2025 10:01:49 +0100 Subject: [PATCH 08/16] IONOS(build): improve cache handling and reporting in build workflow Signed-off-by: Misha M.-Kupriyanov --- .github/workflows/build-artifact.yml | 130 ++++++++++++++++++++++----- 1 file changed, 107 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build-artifact.yml b/.github/workflows/build-artifact.yml index 2ae4d2a62a7a5..e962456cd4a43 100644 --- a/.github/workflows/build-artifact.yml +++ b/.github/workflows/build-artifact.yml @@ -717,6 +717,15 @@ jobs: - name: Build ${{ matrix.app_info.name }} app run: make -f IONOS/Makefile ${{ steps.app-config.outputs.makefile-target }} + - name: Report build completion + if: success() + run: | + echo "### ✅ Built ${{ matrix.app_info.name }}" >> $GITHUB_STEP_SUMMARY + echo "- **SHA:** \`${{ steps.app-config.outputs.app-sha }}\`" >> $GITHUB_STEP_SUMMARY + echo "- **Path:** ${{ steps.app-config.outputs.path }}" >> $GITHUB_STEP_SUMMARY + echo "- **Status:** Success" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + # Save built app to cache for future runs - name: Save build artifacts to cache uses: actions/cache/save@v4 @@ -788,9 +797,28 @@ jobs: uses: actions/cache/restore@v4 with: path: ${{ matrix.app.path }} - key: app-build-${{ matrix.app.name }}-${{ matrix.app.sha }} + key: app-build-${{ matrix.app.name }}-${{ steps.get-sha.outputs.app-sha }} fail-on-cache-miss: true + - name: Validate cached build + if: steps.check-cache.outputs.skip == 'false' + run: | + APP_PATH="${{ matrix.app.path }}" + + # Check that the directory exists and is not empty + if [ ! -d "$APP_PATH" ] || [ -z "$(ls -A $APP_PATH)" ]; then + echo "❌ Cache validation failed: Directory is empty or missing" + exit 1 + fi + + # Check for appinfo/info.xml (required for all Nextcloud apps) + if [ ! -f "$APP_PATH/appinfo/info.xml" ]; then + echo "❌ Cache validation failed: Missing appinfo/info.xml" + exit 1 + fi + + echo "✅ Cache validation passed for ${{ matrix.app.name }}" + - name: Upload cached ${{ matrix.app.name }} build artifacts if: steps.check-cache.outputs.skip == 'false' uses: actions/upload-artifact@v4 @@ -1026,12 +1054,37 @@ jobs: export PATH_TO_LATEST_ARTIFACT="${PATH_TO_DIRECTORY}/${PATH_TO_FILE}" - # Promote current build to the "latest" dev build - jf rt upload "${{ env.TARGET_PACKAGE_NAME }}" \ - --build-name "${{ env.BUILD_NAME }}" \ - --build-number ${{ github.run_number }} \ - --target-props "build.nc_version=${{ needs.build-artifact.outputs.NC_VERSION }};vcs.branch=${{ github.ref }};vcs.revision=${{ github.sha }}" \ - $PATH_TO_LATEST_ARTIFACT + # Upload with retry logic (3 attempts with 30s delay) + MAX_ATTEMPTS=3 + ATTEMPT=1 + UPLOAD_SUCCESS=false + + while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do + echo "Upload attempt $ATTEMPT of $MAX_ATTEMPTS..." + + if jf rt upload "${{ env.TARGET_PACKAGE_NAME }}" \ + --build-name "${{ env.BUILD_NAME }}" \ + --build-number ${{ github.run_number }} \ + --target-props "build.nc_version=${{ needs.build-artifact.outputs.NC_VERSION }};vcs.branch=${{ github.ref }};vcs.revision=${{ github.sha }}" \ + $PATH_TO_LATEST_ARTIFACT; then + UPLOAD_SUCCESS=true + echo "✅ Upload successful on attempt $ATTEMPT" + break + else + echo "⚠️ Upload attempt $ATTEMPT failed" + if [ $ATTEMPT -lt $MAX_ATTEMPTS ]; then + echo "Waiting 30 seconds before retry..." + sleep 30 + fi + fi + + ATTEMPT=$((ATTEMPT + 1)) + done + + if [ "$UPLOAD_SUCCESS" != "true" ]; then + echo "❌ Upload failed after $MAX_ATTEMPTS attempts" + exit 1 + fi echo "ARTIFACTORY_LAST_BUILD_PATH=${PATH_TO_LATEST_ARTIFACT}" >> $GITHUB_OUTPUT @@ -1131,22 +1184,53 @@ jobs: BUILD_TYPE="stable" fi - # Trigger GitLab pipeline via webhook with build artifacts and metadata - # Passes GitHub context variables to remote GitLab workflow - curl \ - --silent \ - --insecure \ - --request POST \ - --fail-with-body \ - -o response.json \ - --form token=${{ secrets.GITLAB_TOKEN }} \ - --form ref="stable" \ - --form "variables[GITHUB_SHA]=${{ github.sha }}" \ - --form "variables[ARTIFACTORY_LAST_BUILD_PATH]=${{ needs.upload-to-artifactory.outputs.ARTIFACTORY_LAST_BUILD_PATH }}" \ - --form "variables[NC_VERSION]=${{ needs.build-artifact.outputs.NC_VERSION }}" \ - --form "variables[BUILD_ID]=${{ github.run_id }}" \ - --form "variables[BUILD_TYPE]=${BUILD_TYPE}" \ - "${{ secrets.GITLAB_TRIGGER_URL }}" || ( RETCODE="$?"; jq . response.json; exit "$RETCODE" ) + # Trigger GitLab pipeline via webhook with retry logic (3 attempts with 30s delay) + MAX_ATTEMPTS=3 + ATTEMPT=1 + TRIGGER_SUCCESS=false + + while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do + echo "Trigger attempt $ATTEMPT of $MAX_ATTEMPTS..." + + if curl \ + --silent \ + --insecure \ + --request POST \ + --fail-with-body \ + -o response.json \ + --form token=${{ secrets.GITLAB_TOKEN }} \ + --form ref="stable" \ + --form "variables[GITHUB_SHA]=${{ github.sha }}" \ + --form "variables[ARTIFACTORY_LAST_BUILD_PATH]=${{ needs.upload-to-artifactory.outputs.ARTIFACTORY_LAST_BUILD_PATH }}" \ + --form "variables[NC_VERSION]=${{ needs.build-artifact.outputs.NC_VERSION }}" \ + --form "variables[BUILD_ID]=${{ github.run_id }}" \ + --form "variables[BUILD_TYPE]=${BUILD_TYPE}" \ + "${{ secrets.GITLAB_TRIGGER_URL }}"; then + TRIGGER_SUCCESS=true + echo "✅ Trigger successful on attempt $ATTEMPT" + break + else + RETCODE="$?" + echo "⚠️ Trigger attempt $ATTEMPT failed with code $RETCODE" + if [ -f response.json ]; then + jq . response.json || cat response.json + fi + if [ $ATTEMPT -lt $MAX_ATTEMPTS ]; then + echo "Waiting 30 seconds before retry..." + sleep 30 + fi + fi + + ATTEMPT=$((ATTEMPT + 1)) + done + + if [ "$TRIGGER_SUCCESS" != "true" ]; then + echo "❌ Trigger failed after $MAX_ATTEMPTS attempts" + if [ -f response.json ]; then + jq . response.json || cat response.json + fi + exit 1 + fi # Disable command echo set +x From 3f84a3889db1e16a3fb2a0e3bbe671140324d741 Mon Sep 17 00:00:00 2001 From: "Misha M.-Kupriyanov" Date: Wed, 12 Nov 2025 11:17:30 +0100 Subject: [PATCH 09/16] IONOS(build): refine conditions for artifact push and nextcloud workspace artifact Signed-off-by: Misha M.-Kupriyanov --- .github/workflows/build-artifact.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-artifact.yml b/.github/workflows/build-artifact.yml index e962456cd4a43..5bea4415601f4 100644 --- a/.github/workflows/build-artifact.yml +++ b/.github/workflows/build-artifact.yml @@ -974,7 +974,10 @@ jobs: runs-on: self-hosted # Upload the artifact to the Artifactory repository on PR *OR* on "ionos-dev|ionos-stable" branch push defined in the on:push:branches if: | + always() && (github.event_name == 'pull_request' || github.ref_name == 'ionos-dev' || github.ref_name == 'ionos-stable') && + (needs.build-external-apps.result == 'success' || needs.build-external-apps.result == 'skipped') && + (needs.restore-cached-apps.result == 'success' || needs.restore-cached-apps.result == 'skipped') && needs.build-artifact.result == 'success' name: Push to artifactory @@ -1098,7 +1101,11 @@ jobs: nextcloud-workspace-artifact-to-ghcr_io: runs-on: ubuntu-latest # Only run if build-artifact succeeded - if: needs.build-artifact.result == 'success' + if: | + always() && + (needs.build-external-apps.result == 'success' || needs.build-external-apps.result == 'skipped') && + (needs.restore-cached-apps.result == 'success' || needs.restore-cached-apps.result == 'skipped') && + needs.build-artifact.result == 'success' permissions: contents: read From 2537f6711236786d0f7045348b9ce81ad55ac71f Mon Sep 17 00:00:00 2001 From: "Misha M.-Kupriyanov" Date: Wed, 12 Nov 2025 12:02:16 +0100 Subject: [PATCH 10/16] IONOS(build): add cache versioning to improve cache management Signed-off-by: Misha M.-Kupriyanov --- .github/workflows/build-artifact.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-artifact.yml b/.github/workflows/build-artifact.yml index 5bea4415601f4..33f68021fbc63 100644 --- a/.github/workflows/build-artifact.yml +++ b/.github/workflows/build-artifact.yml @@ -41,6 +41,10 @@ env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} ARTIFACTORY_REPOSITORY_SNAPSHOT: ionos-productivity-ncwserver-snapshot + # Cache version - increment this to invalidate all caches when build process changes + # Update when: Node.js version changes, PHP version changes, build scripts modified, etc. + # Format: v. (e.g., v1.0, v1.1, v2.0) + CACHE_VERSION: v1.0 permissions: contents: read @@ -569,7 +573,8 @@ jobs: fi # Cache key that would be used for this app - CACHE_KEY="app-build-${APP_NAME}-${CURRENT_SHA}" + # Format: -app-build-- + CACHE_KEY="${{ env.CACHE_VERSION }}-app-build-${APP_NAME}-${CURRENT_SHA}" SHORT_SHA="${CURRENT_SHA:0:8}" echo -n " Checking $APP_NAME (SHA: $SHORT_SHA)... " @@ -731,7 +736,7 @@ jobs: uses: actions/cache/save@v4 with: path: ${{ steps.app-config.outputs.path }} - key: app-build-${{ matrix.app_info.name }}-${{ matrix.app_info.sha }} + key: ${{ env.CACHE_VERSION }}-app-build-${{ matrix.app_info.name }}-${{ matrix.app_info.sha }} - name: Upload ${{ matrix.app_info.name }} build artifacts uses: actions/upload-artifact@v4 @@ -797,7 +802,7 @@ jobs: uses: actions/cache/restore@v4 with: path: ${{ matrix.app.path }} - key: app-build-${{ matrix.app.name }}-${{ steps.get-sha.outputs.app-sha }} + key: ${{ env.CACHE_VERSION }}-app-build-${{ matrix.app.name }}-${{ steps.get-sha.outputs.app-sha }} fail-on-cache-miss: true - name: Validate cached build From 5b196cf3b45d210e9aa5c37ccad71388c01a1c85 Mon Sep 17 00:00:00 2001 From: "Misha M.-Kupriyanov" Date: Wed, 12 Nov 2025 12:17:03 +0100 Subject: [PATCH 11/16] IONOS(build): fix SHA reference in build summary output Signed-off-by: Misha M.-Kupriyanov --- .github/workflows/build-artifact.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-artifact.yml b/.github/workflows/build-artifact.yml index 33f68021fbc63..868d7ccba1ffb 100644 --- a/.github/workflows/build-artifact.yml +++ b/.github/workflows/build-artifact.yml @@ -726,7 +726,7 @@ jobs: if: success() run: | echo "### ✅ Built ${{ matrix.app_info.name }}" >> $GITHUB_STEP_SUMMARY - echo "- **SHA:** \`${{ steps.app-config.outputs.app-sha }}\`" >> $GITHUB_STEP_SUMMARY + echo "- **SHA:** \`${{ matrix.app_info.sha }}\`" >> $GITHUB_STEP_SUMMARY echo "- **Path:** ${{ steps.app-config.outputs.path }}" >> $GITHUB_STEP_SUMMARY echo "- **Status:** Success" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY From 799864a33a7a98aa914c93cf48b15a739da686f6 Mon Sep 17 00:00:00 2001 From: "Misha M.-Kupriyanov" Date: Wed, 12 Nov 2025 12:33:54 +0100 Subject: [PATCH 12/16] IONOS(build): add SHA mapping for apps to improve cache restoration Signed-off-by: Misha M.-Kupriyanov --- .github/workflows/build-artifact.yml | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-artifact.yml b/.github/workflows/build-artifact.yml index 868d7ccba1ffb..4a4935ebc4305 100644 --- a/.github/workflows/build-artifact.yml +++ b/.github/workflows/build-artifact.yml @@ -56,6 +56,7 @@ jobs: apps-to-build: ${{ steps.detect.outputs.apps-to-build }} external-apps-matrix: ${{ steps.set-matrix.outputs.matrix }} has-cached-apps: ${{ steps.detect.outputs.has-cached-apps }} + apps-sha-map: ${{ steps.detect.outputs.apps-sha-map }} steps: - name: Checkout repository uses: actions/checkout@v5 @@ -540,6 +541,7 @@ jobs: APPS_CHECKED=0 APPS_CACHED=0 APPS_TO_BUILD_COUNT=0 + APPS_SHA_MAP="{}" echo "### 📦 Cache Status Report" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY @@ -572,6 +574,9 @@ jobs: continue fi + # Add SHA to the map for all apps (regardless of cache status) + APPS_SHA_MAP=$(echo "$APPS_SHA_MAP" | jq -c --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" '.[$app] = $sha') + # Cache key that would be used for this app # Format: -app-build-- CACHE_KEY="${{ env.CACHE_VERSION }}-app-build-${APP_NAME}-${CURRENT_SHA}" @@ -627,6 +632,11 @@ jobs: echo "$APPS_TO_BUILD" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT + # Output the SHA map for all apps + echo "apps-sha-map<> $GITHUB_OUTPUT + echo "$APPS_SHA_MAP" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + # Determine if there are cached apps by comparing counts # If apps-to-build count is less than total apps, some are cached if [ $APPS_TO_BUILD_COUNT -lt $APPS_CHECKED ]; then @@ -768,6 +778,7 @@ jobs: run: | APPS_TO_BUILD='${{ needs.prepare-matrix.outputs.apps-to-build }}' APP_NAME='${{ matrix.app.name }}' + APPS_SHA_MAP='${{ needs.prepare-matrix.outputs.apps-sha-map }}' # Check if this app is in the apps-to-build list IN_BUILD_LIST=$(echo "$APPS_TO_BUILD" | jq -r --arg name "$APP_NAME" 'any(.[]; .name == $name)') @@ -776,8 +787,11 @@ jobs: echo "skip=true" >> $GITHUB_OUTPUT echo "⏭️ Skipping $APP_NAME - will be built by build job" else + # Get the SHA from the SHA map + APP_SHA=$(echo "$APPS_SHA_MAP" | jq -r --arg name "$APP_NAME" '.[$name] // "unknown"') + echo "app-sha=$APP_SHA" >> $GITHUB_OUTPUT echo "skip=false" >> $GITHUB_OUTPUT - echo "📦 Will restore $APP_NAME from cache" + echo "📦 Will restore $APP_NAME from cache (SHA: ${APP_SHA:0:12})" fi - name: Checkout server (sparse) @@ -788,21 +802,13 @@ jobs: ${{ matrix.app.path }} fetch-depth: 1 - - name: Get cached app SHA - if: steps.check-cache.outputs.skip == 'false' - id: get-sha - run: | - cd ${{ matrix.app.path }} - APP_SHA=$(git rev-parse HEAD 2>/dev/null || echo "unknown") - echo "app-sha=$APP_SHA" >> $GITHUB_OUTPUT - echo "SHA for ${{ matrix.app.name }}: ${APP_SHA:0:12}" - name: Restore cached build from cache if: steps.check-cache.outputs.skip == 'false' uses: actions/cache/restore@v4 with: path: ${{ matrix.app.path }} - key: ${{ env.CACHE_VERSION }}-app-build-${{ matrix.app.name }}-${{ steps.get-sha.outputs.app-sha }} + key: ${{ env.CACHE_VERSION }}-app-build-${{ matrix.app.name }}-${{ steps.check-cache.outputs.app-sha }} fail-on-cache-miss: true - name: Validate cached build From 2619432b545fadc7c12b6547591d344a41db515e Mon Sep 17 00:00:00 2001 From: "Misha M.-Kupriyanov" Date: Wed, 12 Nov 2025 12:50:04 +0100 Subject: [PATCH 13/16] IONOS(build): update action versions for improved functionality and compatibility Signed-off-by: Misha M.-Kupriyanov --- .github/workflows/build-artifact.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-artifact.yml b/.github/workflows/build-artifact.yml index 4a4935ebc4305..2eb5c742105b5 100644 --- a/.github/workflows/build-artifact.yml +++ b/.github/workflows/build-artifact.yml @@ -705,7 +705,7 @@ jobs: - name: Set up node with version from package.json's engines if: steps.app-config.outputs.has-npm == 'true' - uses: actions/setup-node@v5 + uses: actions/setup-node@v6 with: node-version-file: "package.json" cache: 'npm' @@ -749,7 +749,7 @@ jobs: key: ${{ env.CACHE_VERSION }}-app-build-${{ matrix.app_info.name }}-${{ matrix.app_info.sha }} - name: Upload ${{ matrix.app_info.name }} build artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: retention-days: 1 name: external-app-build-${{ matrix.app_info.name }} @@ -832,7 +832,7 @@ jobs: - name: Upload cached ${{ matrix.app.name }} build artifacts if: steps.check-cache.outputs.skip == 'false' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: retention-days: 1 name: external-app-build-${{ matrix.app.name }} @@ -864,7 +864,7 @@ jobs: fetch-depth: '1' - name: Download build external apps - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v6 with: pattern: external-app-build-* path: apps-external/ @@ -914,7 +914,7 @@ jobs: done - name: Set up node with version from package.json's engines - uses: actions/setup-node@v5 + uses: actions/setup-node@v6 with: node-version-file: "package.json" cache: 'npm' @@ -968,7 +968,7 @@ jobs: echo "NC_VERSION=$NC_VERSION" >> $GITHUB_OUTPUT - name: Upload artifact result for job build-artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: retention-days: 30 name: nextcloud_workspace_build_artifact @@ -1030,7 +1030,7 @@ jobs: fi - name: Download artifact zip - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v6 with: name: nextcloud_workspace_build_artifact @@ -1127,7 +1127,7 @@ jobs: steps: - name: Download artifact zip - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v6 with: name: nextcloud_workspace_build_artifact From 946026933afd7af681e309fdc0448d12c74a4dc3 Mon Sep 17 00:00:00 2001 From: "Misha M.-Kupriyanov" Date: Wed, 12 Nov 2025 12:51:03 +0100 Subject: [PATCH 14/16] IONOS(build): remove sparse checkout step to streamline cache restoration Signed-off-by: Misha M.-Kupriyanov --- .github/workflows/build-artifact.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/build-artifact.yml b/.github/workflows/build-artifact.yml index 2eb5c742105b5..c7395f83f470d 100644 --- a/.github/workflows/build-artifact.yml +++ b/.github/workflows/build-artifact.yml @@ -794,14 +794,6 @@ jobs: echo "📦 Will restore $APP_NAME from cache (SHA: ${APP_SHA:0:12})" fi - - name: Checkout server (sparse) - if: steps.check-cache.outputs.skip == 'false' - uses: actions/checkout@v5 - with: - sparse-checkout: | - ${{ matrix.app.path }} - fetch-depth: 1 - - name: Restore cached build from cache if: steps.check-cache.outputs.skip == 'false' From 0f28ef4cfc30038adb4dd4e54f6172ec77c5f172 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 12 Nov 2025 13:58:37 +0000 Subject: [PATCH 15/16] Initial plan From 3df69652e513230f7edceb18ad336512cefb1aa3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 12 Nov 2025 14:03:37 +0000 Subject: [PATCH 16/16] IONOS(build): optimize restore-cached-apps to use filtered matrix Co-authored-by: printminion-co <145785698+printminion-co@users.noreply.github.com> --- .github/workflows/build-artifact.yml | 73 ++++++++++++++++------------ 1 file changed, 43 insertions(+), 30 deletions(-) diff --git a/.github/workflows/build-artifact.yml b/.github/workflows/build-artifact.yml index c7395f83f470d..86311f7d72d99 100644 --- a/.github/workflows/build-artifact.yml +++ b/.github/workflows/build-artifact.yml @@ -54,6 +54,7 @@ jobs: runs-on: ubuntu-latest outputs: apps-to-build: ${{ steps.detect.outputs.apps-to-build }} + apps-to-restore: ${{ steps.detect.outputs.apps-to-restore }} external-apps-matrix: ${{ steps.set-matrix.outputs.matrix }} has-cached-apps: ${{ steps.detect.outputs.has-cached-apps }} apps-sha-map: ${{ steps.detect.outputs.apps-sha-map }} @@ -538,6 +539,8 @@ jobs: # Build JSON array for apps that actually need building APPS_TO_BUILD="[]" + # Build JSON array for apps that need restoration from cache + APPS_TO_RESTORE="[]" APPS_CHECKED=0 APPS_CACHED=0 APPS_TO_BUILD_COUNT=0 @@ -591,6 +594,8 @@ jobs: APPS_CACHED=$((APPS_CACHED + 1)) echo "✓ cached" echo "| $APP_NAME | \`$SHORT_SHA\` | \`$CACHE_KEY\` | ✅ Cached |" >> $GITHUB_STEP_SUMMARY + # Add to restore list with SHA for cache restoration + APPS_TO_RESTORE=$(echo "$APPS_TO_RESTORE" | jq -c --arg app "$APP_NAME" --arg sha "$CURRENT_SHA" '. + [{name: $app, sha: $sha}]') else echo "⚡ needs build" echo "| $APP_NAME | \`$SHORT_SHA\` | \`$CACHE_KEY\` | 🔨 Needs build |" >> $GITHUB_STEP_SUMMARY @@ -626,12 +631,23 @@ jobs: exit 1 fi + if ! echo "$APPS_TO_RESTORE" | jq empty 2>/dev/null; then + echo "ERROR: Failed to build valid JSON for apps-to-restore" + echo "Content: $APPS_TO_RESTORE" + exit 1 + fi + # Output app list with SHAs for the build job to use # Use proper multiline output format for GitHub Actions echo "apps-to-build<> $GITHUB_OUTPUT echo "$APPS_TO_BUILD" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT + # Output app list with SHAs for the restore job to use + echo "apps-to-restore<> $GITHUB_OUTPUT + echo "$APPS_TO_RESTORE" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + # Output the SHA map for all apps echo "apps-sha-map<> $GITHUB_OUTPUT echo "$APPS_SHA_MAP" >> $GITHUB_OUTPUT @@ -760,8 +776,8 @@ jobs: restore-cached-apps: runs-on: ubuntu-latest needs: prepare-matrix - # Only run if there are cached apps (when not all apps need to be built) - if: needs.prepare-matrix.outputs.has-cached-apps == 'true' + # Only run if there are cached apps to restore + if: needs.prepare-matrix.outputs.apps-to-restore != '[]' permissions: contents: read @@ -770,43 +786,41 @@ jobs: strategy: max-parallel: 20 matrix: - app: ${{ fromJson(needs.prepare-matrix.outputs.external-apps-matrix) }} + # Use the filtered list of apps that need restoration from cache + app_info: ${{ fromJson(needs.prepare-matrix.outputs.apps-to-restore) }} steps: - - name: Check if app needs restoring from cache - id: check-cache + - name: Get app configuration from full matrix + id: app-config run: | - APPS_TO_BUILD='${{ needs.prepare-matrix.outputs.apps-to-build }}' - APP_NAME='${{ matrix.app.name }}' - APPS_SHA_MAP='${{ needs.prepare-matrix.outputs.apps-sha-map }}' + # Get the full matrix to look up app configuration + FULL_MATRIX='${{ needs.prepare-matrix.outputs.external-apps-matrix }}' + APP_NAME='${{ matrix.app_info.name }}' - # Check if this app is in the apps-to-build list - IN_BUILD_LIST=$(echo "$APPS_TO_BUILD" | jq -r --arg name "$APP_NAME" 'any(.[]; .name == $name)') + # Find the app configuration in the full matrix + APP_CONFIG=$(echo "$FULL_MATRIX" | jq -c --arg name "$APP_NAME" '.[] | select(.name == $name)') - if [ "$IN_BUILD_LIST" = "true" ]; then - echo "skip=true" >> $GITHUB_OUTPUT - echo "⏭️ Skipping $APP_NAME - will be built by build job" - else - # Get the SHA from the SHA map - APP_SHA=$(echo "$APPS_SHA_MAP" | jq -r --arg name "$APP_NAME" '.[$name] // "unknown"') - echo "app-sha=$APP_SHA" >> $GITHUB_OUTPUT - echo "skip=false" >> $GITHUB_OUTPUT - echo "📦 Will restore $APP_NAME from cache (SHA: ${APP_SHA:0:12})" + if [ -z "$APP_CONFIG" ]; then + echo "ERROR: Could not find app '$APP_NAME' in full matrix" + exit 1 fi + # Extract and output app properties + echo "path=$(echo "$APP_CONFIG" | jq -r '.path')" >> $GITHUB_OUTPUT + echo "name=$(echo "$APP_CONFIG" | jq -r '.name')" >> $GITHUB_OUTPUT + + echo "📦 Restoring $APP_NAME from cache (SHA: ${{ matrix.app_info.sha }})" - name: Restore cached build from cache - if: steps.check-cache.outputs.skip == 'false' uses: actions/cache/restore@v4 with: - path: ${{ matrix.app.path }} - key: ${{ env.CACHE_VERSION }}-app-build-${{ matrix.app.name }}-${{ steps.check-cache.outputs.app-sha }} + path: ${{ steps.app-config.outputs.path }} + key: ${{ env.CACHE_VERSION }}-app-build-${{ matrix.app_info.name }}-${{ matrix.app_info.sha }} fail-on-cache-miss: true - name: Validate cached build - if: steps.check-cache.outputs.skip == 'false' run: | - APP_PATH="${{ matrix.app.path }}" + APP_PATH="${{ steps.app-config.outputs.path }}" # Check that the directory exists and is not empty if [ ! -d "$APP_PATH" ] || [ -z "$(ls -A $APP_PATH)" ]; then @@ -820,17 +834,16 @@ jobs: exit 1 fi - echo "✅ Cache validation passed for ${{ matrix.app.name }}" + echo "✅ Cache validation passed for ${{ matrix.app_info.name }}" - - name: Upload cached ${{ matrix.app.name }} build artifacts - if: steps.check-cache.outputs.skip == 'false' + - name: Upload cached ${{ matrix.app_info.name }} build artifacts uses: actions/upload-artifact@v5 with: retention-days: 1 - name: external-app-build-${{ matrix.app.name }} + name: external-app-build-${{ matrix.app_info.name }} path: | - ${{ matrix.app.path }} - !${{ matrix.app.path }}/node_modules + ${{ steps.app-config.outputs.path }} + !${{ steps.app-config.outputs.path }}/node_modules build-artifact: runs-on: ubuntu-latest