Skip to content
d2e Docker Build

create download dir for SupabaseStorageAPI (#3258) #14581

Sign in to view logs

create download dir for SupabaseStorageAPI (#3258)

create download dir for SupabaseStorageAPI (#3258) #14581

Workflow file for this run

.github/workflows/docker-build-push.yaml at 0d466e6
name: d2e Docker Build
permissions:
contents: read
packages: write
on:
pull_request:
types: [opened, ready_for_review, reopened, synchronize]
merge_group:
push:
branches:
- develop
- "release/*"
workflow_dispatch:
inputs:
release_tag:
description: Enter version for release
required: true
type: string
artifacttype:
type: choice
required: true
default: "OSS-develop"
options:
- "OSS-develop"
- "OSS-release"
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }}-${{ github.event_name }}
cancel-in-progress: true
env:
REG_URL: ghcr.io/ohdsi
DOCKER_BUILD_ARG1: GITHUB_PAT=${{ secrets.GITHUB_TOKEN }}
PREPULL_FLOW_IMAGES: "" # legacy flow images retired: flows run on the pixi process worker
jobs:
setup:
runs-on: ubuntu-latest
if: github.event_name != 'pull_request' || !github.event.pull_request.draft
outputs:
random_runner: ${{ steps.shuffle.outputs.random_runner }}
run_tag: ${{ steps.tag.outputs.run_tag }}
canonical_tag: ${{ steps.tag.outputs.canonical_tag }}
fallback_tag: ${{ steps.tag.outputs.fallback_tag }}
is_develop_push: ${{ steps.tag.outputs.is_develop_push }}
is_release_push: ${{ steps.tag.outputs.is_release_push }}
is_publish_push: ${{ steps.tag.outputs.is_publish_push }}
plugins_tag: ${{ steps.tag.outputs.plugins_tag }}
is_pr_or_mg: ${{ steps.tag.outputs.is_pr_or_mg }}
changes_src: ${{ steps.file_changes.outputs.src }}
changes_dataflow: ${{ steps.file_changes.outputs.dataflow }}
changes_plugins: ${{ steps.file_changes.outputs.plugins }}
changes_http_tests: ${{ steps.file_changes.outputs.http_tests }}
changes_ui: ${{ steps.file_changes.outputs.ui }}
changes_flow_package_json: ${{ steps.file_changes.outputs.flow_package_json }}
changes_e2e: ${{ steps.file_changes.outputs.e2e }}
changes_regression: ${{ steps.file_changes.outputs.regression }}
steps:
- id: shuffle
run: echo "random_runner=$(shuf -e ubuntu-22.04 ubuntu-24.04 | head -n 1)" >> "$GITHUB_OUTPUT"
- uses: actions/checkout@v4
with:
submodules: recursive
- id: tag
run: |
SHORT_SHA=$(git rev-parse --short HEAD)
IS_PR_OR_MG=false
IS_DEVELOP_PUSH=false
IS_RELEASE_PUSH=false
CANONICAL_TAG=""
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
RUN_TAG="pr-${{ github.event.pull_request.number }}-${SHORT_SHA}"
IS_PR_OR_MG=true
elif [[ "${{ github.event_name }}" == "merge_group" ]]; then
RUN_TAG="mg-${SHORT_SHA}"
IS_PR_OR_MG=true
elif [[ "${{ github.event_name }}" == "push" && "${{ github.ref_name }}" == "develop" ]]; then
RUN_TAG="develop-${SHORT_SHA}"
IS_DEVELOP_PUSH=true
CANONICAL_TAG="develop"
elif [[ "${{ github.event_name }}" == "push" && "${{ github.ref_name }}" == release/* ]]; then
REL_SLUG="${GITHUB_REF_NAME//\//-}"
RUN_TAG="${REL_SLUG}-${SHORT_SHA}"
IS_RELEASE_PUSH=true
CANONICAL_TAG="${REL_SLUG}"
PLUGINS_TAG=""
elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
RUN_TAG="${{ github.event.inputs.release_tag }}-beta"
IS_RELEASE_PUSH=true
CANONICAL_TAG="${{ github.event.inputs.release_tag }}"
PLUGINS_TAG="${{ github.event.inputs.release_tag }}"
else
RUN_TAG="dispatch-${SHORT_SHA}"
PLUGINS_TAG=""
fi
IS_PUBLISH_PUSH=false
if [[ "$IS_DEVELOP_PUSH" == "true" || "$IS_RELEASE_PUSH" == "true" ]]; then
IS_PUBLISH_PUSH=true
fi
# Source tag a skipped build copies its RUN_TAG from: the canonical
# tag of the branch this run targets.
BASE_REF="${{ github.base_ref }}"
FALLBACK_TAG="${BASE_REF:-develop}"
FALLBACK_TAG="${FALLBACK_TAG//\//-}"
{
echo "run_tag=${RUN_TAG}"
echo "canonical_tag=${CANONICAL_TAG}"
echo "fallback_tag=${FALLBACK_TAG}"
echo "plugins_tag=${PLUGINS_TAG}"
echo "is_pr_or_mg=${IS_PR_OR_MG}"
echo "is_develop_push=${IS_DEVELOP_PUSH}"
echo "is_release_push=${IS_RELEASE_PUSH}"
echo "is_publish_push=${IS_PUBLISH_PUSH}"
} >> "$GITHUB_OUTPUT"
{
echo "RUN_TAG=${RUN_TAG}"
echo "CANONICAL_TAG=${CANONICAL_TAG}"
} >> "$GITHUB_STEP_SUMMARY"
- uses: dorny/paths-filter@v3
id: file_changes
with:
initial-fetch-depth: 1
filters: |
src:
- "plugins/flows/**"
- "services/**"
- "package.json"
- "docker-compose.yml"
- ".github/workflows/docker-build-push.yaml"
# Everything the dataflow-gen-worker image bakes. Its Dockerfile
# COPYs only plugins/flows/* even though the build context is the
# repo root, so a change anywhere else cannot alter the result — and
# rebuilding it is the slowest job in the matrix (~1h50m).
dataflow:
- "plugins/flows/**"
- "services/alp-dataflow-gen-worker/**"
- ".github/workflows/docker-build-push.yaml"
plugins:
- "plugins/flows/**"
- "package.json"
- "docker-compose.yml"
- "plugins/ui/**"
- "plugins/atlas/**"
- "plugins/functions/**"
- "plugins/fhir_functions/**"
- "plugins/sibyl_functions/**"
- "services/trex/ext/**"
- ".github/workflows/docker-build-push.yaml"
http_tests:
- "plugins/functions/_shared/alp-libs/nodejs/**"
- "plugins/functions/analytics-svc/**"
- "plugins/functions/bookmark-svc/**"
- "plugins/functions/cdw-svc/**"
- "plugins/functions/mri-pa-config/**"
- "plugins/functions/mri-pg-config/**"
- "plugins/functions/query-gen-svc/**"
- "services/mri-db/**"
- "services/alp-logto/**"
- "services/trex/**"
- "package.json"
- "docker-compose.yml"
- "docker-compose-local.yml"
- "scripts/**"
- "tests/backend_integration_tests/**"
- ".github/workflows/docker-build-push.yaml"
ui:
- "plugins/ui/**"
- ".github/workflows/docker-build-push.yaml"
flow_package_json:
- "plugins/flows/base/package.json"
- ".github/workflows/docker-build-push.yaml"
e2e:
- "tests/e2e/**"
- "docker-compose-local.yml"
- "services/trex/**"
- ".github/workflows/docker-build-push.yaml"
regression:
- "tests/regression/**"
- "docker-compose-local.yml"
- ".github/workflows/docker-build-push.yaml"
build_images:
needs: [setup]
if: |
needs.setup.outputs.is_publish_push == 'true' ||
contains(github.base_ref, 'release/') ||
github.event_name == 'workflow_dispatch' ||
github.event_name == 'merge_group' ||
(github.event_name == 'pull_request' && (needs.setup.outputs.changes_src == 'true' || needs.setup.outputs.changes_plugins == 'true' || needs.setup.outputs.changes_e2e == 'true'))
runs-on: ${{ needs.setup.outputs.random_runner }}
env:
RUN_TAG: ${{ needs.setup.outputs.run_tag }}
FALLBACK_TAG: ${{ needs.setup.outputs.fallback_tag }}
# branch-scoped registry-cache suffix (tag-safe)
CACHE_SLUG: ${{ github.head_ref && format('pr-{0}', github.event.pull_request.number) || 'develop' }}
strategy:
fail-fast: false
matrix:
include:
- AZ_REG_REPOSITORY: d2e-logto
DOCKER_BUILD_FILE_PATH: ./services/alp-logto/Dockerfile
DOCKER_BUILD_CONTEXT_PATH: ./services/alp-logto
- AZ_REG_REPOSITORY: d2e-logto-post-init
DOCKER_BUILD_FILE_PATH: ./services/alp-logto/post-init/Dockerfile
DOCKER_BUILD_CONTEXT_PATH: ./services/alp-logto/post-init
- AZ_REG_REPOSITORY: d2e-dataflow-gen
DOCKER_BUILD_FILE_PATH: ./services/alp-dataflow-gen/Dockerfile
DOCKER_BUILD_CONTEXT_PATH: ./services/alp-dataflow-gen
- AZ_REG_REPOSITORY: d2e-dataflow-gen-worker
# Slowest job in the matrix (~1h50m) and it rebuilt on every run,
# including changes it cannot possibly bake. Gate it on the paths it
# actually COPYs; a skipped run still publishes the tag from the
# canonical image, so the image set stays complete.
build_disable: ${{ (needs.setup.outputs.changes_dataflow != 'true') && 'true' || 'false' }}
DOCKER_BUILD_FILE_PATH: ./services/alp-dataflow-gen-worker/Dockerfile
# repo-root context: the image bakes plugins/flows (pixi envs + R)
DOCKER_BUILD_CONTEXT_PATH: .
# Registry build cache (mode=max exports every stage, including the
# expensive env stages): unchanged groups become blob downloads
# instead of renv compiles. PRs write to a per-branch cache tag and
# fall back to reading develop's; develop pushes write the shared
# tag. The hades stage reads GITHUB_PAT from a BuildKit secret so
# the rotating token cannot poison its cache key.
cache_enabled: "true"
needs_github_pat_secret: "true"
- AZ_REG_REPOSITORY: d2e-supabase-storage
DOCKER_BUILD_FILE_PATH: ./services/supabase-storage/Dockerfile
DOCKER_BUILD_CONTEXT_PATH: ./services/supabase-storage
- AZ_REG_REPOSITORY: d2e-materialize-cohorts
DOCKER_BUILD_FILE_PATH: ./services/materialize-cohorts/Dockerfile
DOCKER_BUILD_CONTEXT_PATH: .
steps:
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
with:
tool-cache: true
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- uses: actions/checkout@v4
with:
submodules: recursive
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- uses: docker/setup-buildx-action@v3
id: builder
- name: Build and Push
id: build
if: ${{ matrix.build_disable != 'true' || github.event_name == 'workflow_dispatch' || needs.setup.outputs.is_develop_push == 'true' || contains(github.ref_name, 'release/') || github.event_name == 'merge_group' }}
uses: docker/build-push-action@v6
with:
builder: ${{ steps.builder.outputs.name }}
context: ${{ matrix.DOCKER_BUILD_CONTEXT_PATH }}
file: ${{ matrix.DOCKER_BUILD_FILE_PATH }}
platforms: linux/amd64
push: true
pull: ${{ matrix.pull_base == 'true' || false }}
tags: ${{ env.REG_URL }}/${{ matrix.AZ_REG_REPOSITORY }}:${{ env.RUN_TAG }}
provenance: false
sbom: false
cache-from: |
${{ matrix.cache_enabled == 'true' && format('type=registry,ref={0}/{1}:buildcache-{2}', env.REG_URL, matrix.AZ_REG_REPOSITORY, env.CACHE_SLUG) || '' }}
${{ matrix.cache_enabled == 'true' && format('type=registry,ref={0}/{1}:buildcache', env.REG_URL, matrix.AZ_REG_REPOSITORY) || '' }}
cache-to: ${{ matrix.cache_enabled == 'true' && format('type=registry,ref={0}/{1}:buildcache{2},mode=max', env.REG_URL, matrix.AZ_REG_REPOSITORY, needs.setup.outputs.is_publish_push == 'true' && '' || format('-{0}', env.CACHE_SLUG)) || '' }}
secrets: ${{ matrix.needs_github_pat_secret == 'true' && format('github_pat={0}', secrets.GITHUB_TOKEN) || '' }}
build-args: |
GIT_COMMIT_ARG=${{ github.sha }}
BUILD_TYPE=OSS
${{ env.DOCKER_BUILD_ARG1 }}
# A skipped build (build_disable) pushes no RUN_TAG, but consumers ask for
# ONE tag across the whole image set — docker compose pulls every service
# at DOCKER_TAG_NAME — so a single hole fails the whole pull with
# "manifest unknown" on an image nobody touched. imagetools copies the
# manifest server-side, so this costs a registry call, not a rebuild, and
# keeps multi-arch lists intact.
- name: Publish run tag for skipped build
if: steps.build.outcome == 'skipped'
run: |
DST=${REG_URL}/${{ matrix.AZ_REG_REPOSITORY }}:${RUN_TAG}
for SRC_TAG in "${FALLBACK_TAG}" develop; do
SRC=${REG_URL}/${{ matrix.AZ_REG_REPOSITORY }}:${SRC_TAG}
if docker manifest inspect "$SRC" >/dev/null 2>&1; then
echo "Build skipped; publishing $DST from $SRC"
docker buildx imagetools create -t "$DST" "$SRC"
exit 0
fi
done
echo "::error::no source tag available to publish ${DST} from"
exit 1
- name: Prune system
if: always()
run: docker system prune -af
# Lockfile discipline for pixi-managed flow plugin environments: manifests
# must not change without their committed pixi.lock, and locks must stay
# binary-only (see _pixi-lock-check.yml).
pixi_lock_check:
needs: [setup]
if: |
github.event_name == 'merge_group' ||
needs.setup.outputs.is_publish_push == 'true' ||
(github.event_name == 'pull_request' && needs.setup.outputs.changes_plugins == 'true')
uses: ./.github/workflows/_pixi-lock-check.yml
build_plugins:
needs: [setup]
if: |
needs.setup.outputs.is_publish_push == 'true' ||
contains(github.base_ref, 'release/') ||
github.event_name == 'workflow_dispatch' ||
github.event_name == 'merge_group' ||
(github.event_name == 'pull_request' && (needs.setup.outputs.changes_plugins == 'true' || needs.setup.outputs.changes_e2e == 'true'))
strategy:
fail-fast: false
matrix:
include:
- PKGPATH: ./plugins/flows/base/
- PKGPATH: ./plugins/flows/i2b2/
- PKGPATH: ./plugins/flows/search_embedding/
- PKGPATH: ./plugins/flows/data_management/
- PKGPATH: ./plugins/flows/hades/
- PKGPATH: ./plugins/flows/loyalty_score/
- PKGPATH: ./plugins/flows/data_transformation/
- PKGPATH: ./plugins/ui
NPM: bun
NODE_VERSION: "20.x"
- PKGPATH: ./plugins/atlas
NODE_VERSION: "20.x"
ATLAS: true
- PKGPATH: ./plugins/functions/
DESTPATH: /usr/src/data/plugins/@data2evidence/d2e-functions
RUNNER: ubuntu-24.04
- PKGPATH: ./plugins/fhir_functions/
DESTPATH: /usr/src/data/plugins/@data2evidence/fhir
RUNNER: ubuntu-24.04
- PKGPATH: ./plugins/sibyl_functions/
DESTPATH: /usr/src/data/plugins/@data2evidence/sibyl
RUNNER: ubuntu-24.04
runs-on: ${{ matrix.RUNNER || needs.setup.outputs.random_runner }}
env:
NPM_ARTIFACT_TYPE: ${{ github.event.inputs.artifacttype || 'OSS-develop'}}
steps:
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
with:
tool-cache: true
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- uses: actions/checkout@v4
with:
submodules: recursive
fetch-depth: 0
- name: Use Node.js
uses: actions/setup-node@v4
with:
node-version: ${{ matrix.NODE_VERSION || '18.x' }}
registry-url: "https://npm.pkg.github.com"
scope: "@data2evidence"
- name: Setup bun
if: ${{ matrix.NPM == 'bun' }}
uses: oven-sh/setup-bun@v2
with:
bun-version: 1.2.23
- name: Setup NPM
if: ${{ matrix.NPM && matrix.NPM != 'bun' }}
run: npm install -g ${{ matrix.NPM }}
- name: Setup Java
if: ${{ matrix.DESTPATH }}
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '17'
- name: Install dependencies
if: ${{ matrix.DESTPATH }}
run: |
npm install -g deno@2.5.6
node ./install-deno-deps.js ${{ matrix.PKGPATH }}
- name: Update version
run: |
cd ${{ matrix.PKGPATH }}
if [[ $GITHUB_EVENT_NAME == 'workflow_dispatch' ]]; then
RELEASE_VERSION=${{ needs.setup.outputs.plugins_tag }}
jq --arg v $RELEASE_VERSION '.version=$v' package.json > tmppkg; mv tmppkg package.json
else
jq --arg v "-$(date +%s)-$GITHUB_SHA" '.version+=$v' package.json > tmppkg; mv tmppkg package.json
fi
- name: Build
if: ${{ !matrix.ATLAS }}
run: |
cd ${{ matrix.PKGPATH }}
${{ matrix.NPM == 'bun' && 'bun install' ||'npm install --ignore-scripts' }}
env:
CI: ${{ matrix.NPM != 'bun' }}
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Atlas3 sub-plugins (network/notebook-plugin/strategus/studies) live in the
# trex-notebook submodule at plugins/atlas/trex-notebook and are unpublished;
# build them from source so plugins/atlas's file: deps resolve during its
# install/postinstall. This is a second checkout of trex-notebook, pinned to the
# restructured monorepo; plugins/ui/libs/react-notebook stays on the older
# single-package commit that the portal's webr-notebook builds against.
# results-viewer is NOT built here: it is consumed prebuilt from GitHub
# Packages (@ohdsi/results-viewer) with its WebR/shinylive runtime included,
# so no R toolchain is needed on this runner.
- name: Build Atlas sub-plugins (trex-notebook submodule)
if: ${{ matrix.ATLAS }}
env:
CI: "true"
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
SUB=plugins/atlas/trex-notebook/plugins
# notebook-plugin bundles the notebook app's source, which imports webr.
( cd "$SUB/notebook" && npm install )
# Prefer build:pkg (targets dist/); the plain build targets the sibyl dev host.
for p in network notebook-plugin strategus studies; do
( cd "$SUB/$p" && npm install \
&& npm run "$(node -e "process.stdout.write(require('./package.json').scripts['build:pkg'] ? 'build:pkg' : 'build')")" )
done
# data-quality lives in this repo (plugins/atlas/subplugins), not the
# submodule, but is a file: dep of plugins/atlas — so unlike
# patient-analytics it must be built BEFORE the atlas install, whose
# postinstall stages its dist/. Mirrors scripts/build-atlas.sh.
DQ_DIR=plugins/atlas/subplugins/data-quality
( cd "$DQ_DIR" && npm install && npm run build )
if [ ! -f "$DQ_DIR/dist/index.system.js" ]; then
echo "::error::data-quality did not produce dist/index.system.js"
exit 1
fi
# Atlas plugin: postinstall bakes Atlas3 + the Pythia UI into resources/;
# build the portal entry. Packs only resources/ (see package.json "files").
- name: Build Atlas plugin
if: ${{ matrix.ATLAS }}
working-directory: ${{ matrix.PKGPATH }}
env:
CI: "true"
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
npm install
npm run build:portal
# Patient Analytics (Data Exploration) is an Atlas3 sub-plugin too, but it lives
# in the UI monorepo instead of the trex-notebook submodule, so it is built and
# staged here rather than by the atlas postinstall — and it has to run AFTER the
# step above, whose postinstall recreates resources/atlas from scratch.
# --workspaces=false keeps npm out of the surrounding bun-managed monorepo.
# Mirrors the same steps in scripts/build-atlas.sh; keep the two in sync.
- name: Build Atlas sub-plugin (patient-analytics)
if: ${{ matrix.ATLAS }}
env:
CI: "true"
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
PA_DIR=plugins/ui/apps/vue-mri-ui-lib
( cd "$PA_DIR" && npm install --workspaces=false --legacy-peer-deps && npm run build:atlas )
if [ ! -f "$PA_DIR/dist-atlas/index.system.js" ]; then
echo "::error::patient-analytics did not produce dist-atlas/index.system.js"
exit 1
fi
PA_DEST=plugins/atlas/resources/atlas/plugins/patient-analytics
rm -rf "$PA_DEST"
mkdir -p "$PA_DEST"
cp -r "$PA_DIR/dist-atlas/." "$PA_DEST/"
echo "Staged patient-analytics at /atlas/plugins/patient-analytics"
- name: Patch Package
if: ${{ matrix.NPM }}
working-directory: ${{ matrix.PKGPATH }}
run: jq '.private=false' package.json > tmppkg; mv tmppkg package.json
- name: DEST
working-directory: ${{ matrix.PKGPATH }}
if: ${{ matrix.DESTPATH }}
run: |
cp package.json package.org.json
sudo mkdir -p ${{ matrix.DESTPATH }}
sudo chown runner:docker ${{ matrix.DESTPATH }}
cp -a . ${{ matrix.DESTPATH }}
cd ${{ matrix.DESTPATH }}
export DENO_DIR="$HOME/.cache/deno"
node ${{ github.workspace }}/install-deno-deps.js .
npm run build
- name: Use Node.js - OSS Develop
uses: actions/setup-node@v4
if: env.NPM_ARTIFACT_TYPE == 'OSS-develop'
with:
node-version: ${{ matrix.NODE_VERSION || '18.x' }}
registry-url: "https://pkgs.dev.azure.com/data2evidence/d2e/_packaging/d2e/npm/registry/"
scope: "@data2evidence"
- name: Use Node.js - OSS Release
uses: actions/setup-node@v4
if: env.NPM_ARTIFACT_TYPE == 'OSS-release'
with:
node-version: ${{ matrix.NODE_VERSION || '18.x' }}
registry-url: "https://pkgs.dev.azure.com/data2evidence/d2e/_packaging/stable/npm/registry/"
scope: "@data2evidence"
- name: Publish or Pack
env:
CI: ${{ matrix.NPM != 'bun' }}
SHOULD_PUBLISH: ${{ needs.setup.outputs.is_publish_push == 'true' || github.event_name == 'workflow_dispatch' }}
NODE_AUTH_TOKEN: ${{ secrets.NPM_PROJECT_TOKEN }}
run: |
cd ${{ matrix.DESTPATH || matrix.PKGPATH }}
${{ matrix.NPM == 'bun' && 'bun pm pack' || format('{0} pack', matrix.NPM || 'npm') }}
FAILED=0
MAX_SIZE_MIB=500
MAX_SIZE=$((MAX_SIZE_MIB * 1024 * 1024))
shopt -s nullglob
TGZ_FILES=(*.tgz)
if [ ${#TGZ_FILES[@]} -eq 0 ]; then
echo "ERROR: no .tgz files found after pack"
exit 1
fi
for PACK_FILE in "${TGZ_FILES[@]}"; do
SIZE=$(wc -c < "$PACK_FILE")
SIZE_MIB=$(awk "BEGIN {printf \"%.2f\", $SIZE / 1048576}")
if [ "$SIZE" -gt "$MAX_SIZE" ]; then
echo "FAIL $PACK_FILE — ${SIZE_MIB}MiB (exceeds ${MAX_SIZE_MIB}MiB Azure limit)"
echo "::error::$PACK_FILE is ${SIZE_MIB}MiB, exceeds ${MAX_SIZE_MIB}MiB Azure limit"
FAILED=1
else
echo "OK $PACK_FILE — ${SIZE_MIB}MiB"
fi
done
[ $FAILED -eq 0 ] || exit $FAILED
if [[ $SHOULD_PUBLISH == true ]]; then
for PACK_FILE in "${TGZ_FILES[@]}"; do
npm publish "$PACK_FILE"
done
fi
- name: Slugify PKGPATH for artifact name
id: slug
run: |
slug=$(echo "${{ matrix.PKGPATH }}" | sed 's|^\./||; s|/$||; s|/|-|g')
echo "slug=$slug" >> "$GITHUB_OUTPUT"
- name: Upload plugin tarball artifact
uses: actions/upload-artifact@v4
with:
name: plugin-tgz-${{ steps.slug.outputs.slug }}
path: ${{ matrix.DESTPATH || matrix.PKGPATH }}/*.tgz
if-no-files-found: error
retention-days: 7
- name: Upload UI resources
if: matrix.PKGPATH == './plugins/ui'
uses: actions/upload-artifact@v4
with:
name: ui-resources
path: ./plugins/ui/resources
if-no-files-found: error
retention-days: 7
- name: Upload UI package tarball
if: matrix.PKGPATH == './plugins/ui'
uses: actions/upload-artifact@v4
with:
name: ui-tgz
path: ./plugins/ui/*.tgz
if-no-files-found: error
retention-days: 7
build_trex_image:
needs: [setup, build_plugins]
if: |
needs.setup.outputs.is_publish_push == 'true' ||
contains(github.base_ref, 'release/') ||
github.event_name == 'workflow_dispatch' ||
github.event_name == 'merge_group' ||
(github.event_name == 'pull_request' && (needs.setup.outputs.changes_src == 'true' || needs.setup.outputs.changes_plugins == 'true' || needs.setup.outputs.changes_e2e == 'true'))
runs-on: ${{ needs.setup.outputs.random_runner }}
env:
RUN_TAG: ${{ needs.setup.outputs.run_tag }}
steps:
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
with:
tool-cache: true
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- uses: actions/checkout@v4
with:
submodules: recursive
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Download plugin artifacts
uses: actions/download-artifact@v4
with:
pattern: plugin-tgz-*
path: ./services/trex/plugin-artifacts
merge-multiple: true
- name: Inspect bundled tarballs
run: |
ls -lh ./services/trex/plugin-artifacts/
echo "---"
for f in ./services/trex/plugin-artifacts/*.tgz; do
echo "$(basename "$f"): $(tar -xzOf "$f" package/package.json | jq -r '.name + " " + .version')"
done
- uses: docker/setup-buildx-action@v3
id: builder
# Two images off one Dockerfile, differing only in the trexsql base.
# The default is trexsql's lean prod target and is what docker-compose.yml
# runs; the -devx variant below uses the full base and is what
# docker-compose-local.yml runs. They share no layers, so this is a second
# full build rather than a cache resolve — the dev tooling lives in the
# base, and a later layer cannot delete it out of the image.
- name: Build and Push trex image
uses: docker/build-push-action@v6
with:
builder: ${{ steps.builder.outputs.name }}
context: ./services/trex
file: ./services/trex/Dockerfile.v2
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ env.REG_URL }}/d2e-trex:${{ env.RUN_TAG }}
provenance: false
sbom: false
build-args: |
GIT_COMMIT_ARG=${{ github.sha }}
BUILD_TYPE=OSS
PLUGINS_FEED=${{ needs.setup.outputs.is_release_push == 'true' && 'stable' || 'd2e' }}
PLUGINS_FROM_REGISTRY=@ohdsi/pythia-agent@0.1.0-develop.18.g80bf4aa
secrets: |
"PLUGINS_REGISTRY=${{ secrets.PLUGINS_REGISTRY }}"
"github_token=${{ secrets.GITHUB_TOKEN }}"
# Built on every run, not just develop, so the tag exists for whatever
# DOCKER_TAG_NAME a local stack is pointed at — release tags included.
- name: Build and Push trex devx image
uses: docker/build-push-action@v6
with:
builder: ${{ steps.builder.outputs.name }}
context: ./services/trex
file: ./services/trex/Dockerfile.v2
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ env.REG_URL }}/d2e-trex:${{ env.RUN_TAG }}-devx
provenance: false
sbom: false
build-args: |
GIT_COMMIT_ARG=${{ github.sha }}
BUILD_TYPE=OSS
PLUGINS_FEED=${{ needs.setup.outputs.is_release_push == 'true' && 'stable' || 'd2e' }}
PLUGINS_FROM_REGISTRY=@ohdsi/pythia-agent@0.1.0-develop.18.g80bf4aa
TREXSQL_REF=sha-5ce42757279f969bca123f0128a5c967ff2e3bd4@sha256:bad73e8323fc07fdd2dba7a25e6f7f1c2b3d6e86410b039f5e8629c0d0214cc6
secrets: |
"PLUGINS_REGISTRY=${{ secrets.PLUGINS_REGISTRY }}"
"github_token=${{ secrets.GITHUB_TOKEN }}"
- name: Prune system
if: always()
run: docker system prune -af
build_e2e_image:
needs: [setup]
if: |
needs.setup.outputs.is_publish_push == 'true' ||
contains(github.base_ref, 'release/') ||
github.event_name == 'workflow_dispatch' ||
github.event_name == 'merge_group' ||
(github.event_name == 'pull_request' && (needs.setup.outputs.changes_e2e == 'true' || needs.setup.outputs.changes_plugins == 'true'))
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
env:
RUN_TAG: ${{ needs.setup.outputs.run_tag }}
steps:
- uses: actions/checkout@v4
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- uses: docker/setup-buildx-action@v3
id: builder
- name: Build and Push e2e image
uses: docker/build-push-action@v6
with:
builder: ${{ steps.builder.outputs.name }}
context: ./tests/e2e
file: ./tests/e2e/Dockerfile
platforms: linux/amd64
push: true
tags: ${{ env.REG_URL }}/d2e-e2e:${{ env.RUN_TAG }}
provenance: false
sbom: false
helm_lint:
needs: [setup]
runs-on: ubuntu-latest
env:
CHART_DIR: charts/d2e-services
CI_VALUES: charts/d2e-services/ci/test-values.yaml
HELM_VERSION: v3.16.2
steps:
- uses: actions/checkout@v4
- uses: azure/setup-helm@v4
with:
version: ${{ env.HELM_VERSION }}
- name: Helm lint
run: helm lint "$CHART_DIR" -f "$CI_VALUES"
- name: Render templates
run: |
mkdir -p /tmp/rendered
helm template d2e "$CHART_DIR" \
-f "$CI_VALUES" \
--namespace d2e \
> /tmp/rendered/all.yaml
- name: Install kubeconform
run: |
curl -sSL -o /tmp/kubeconform.tgz \
https://github.com/yannh/kubeconform/releases/download/v0.6.7/kubeconform-linux-amd64.tar.gz
tar -xzf /tmp/kubeconform.tgz -C /tmp
sudo mv /tmp/kubeconform /usr/local/bin/
- name: Validate rendered manifests
run: |
kubeconform -summary -strict \
-skip CustomResourceDefinition \
-ignore-missing-schemas \
/tmp/rendered/all.yaml
test_helm_kind:
# TEMPORARILY DISABLED: the Helm chart does not yet support the in-engine
# (trexsql-base) trex image; running it on helm needs the trex/WebAPI runtime
# env + DB-bootstrap gating wired into the chart. Re-enabled in a follow-up PR.
# When skipped, test_helm_kind_gate skips and docker-success treats it as OK.
if: false
needs: [setup, build_images, build_trex_image]
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
packages: read
env:
RUN_TAG: ${{ needs.setup.outputs.run_tag }}
PLUGINS_TAG: ${{ needs.setup.outputs.plugins_tag }}
CHART_DIR: charts/d2e-services
CI_VALUES: charts/d2e-services/ci/test-values.yaml
HELM_VERSION: v3.16.2
RELEASE: d2e
NAMESPACE: d2e
steps:
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
with:
tool-cache: true
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Recreate /opt/hostedtoolcache
# free-disk-space deletes /opt/hostedtoolcache; helm/kind-action needs it.
run: sudo mkdir -p /opt/hostedtoolcache
- uses: actions/checkout@v4
- uses: helm/kind-action@v1
with:
cluster_name: helm-test
- uses: azure/setup-helm@v4
with:
version: ${{ env.HELM_VERSION }}
- name: Show cluster info
run: |
kubectl version
kubectl get nodes -o wide
- name: Create ghcr image-pull secret
run: |
kubectl create namespace "$NAMESPACE"
kubectl create secret docker-registry ghcr-pull \
--docker-server=ghcr.io \
--docker-username="${{ github.actor }}" \
--docker-password="${{ secrets.GITHUB_TOKEN }}" \
-n "$NAMESPACE"
kubectl patch serviceaccount default -n "$NAMESPACE" \
-p '{"imagePullSecrets":[{"name":"ghcr-pull"}]}'
- name: Deploy Postgres for CI
run: |
kubectl apply -n "$NAMESPACE" -f "$CHART_DIR/ci/postgres.yaml"
kubectl rollout status deployment/postgres -n "$NAMESPACE" --timeout=3m
- name: Generate throwaway TLS material
run: |
set -euo pipefail
D=/tmp/ci-tls
mkdir -p "$D"
openssl req -x509 -newkey rsa:2048 -nodes \
-keyout "$D/ca.key" -out "$D/ca.crt" \
-days 30 -subj "/CN=d2e-ci-ca" 2>/dev/null
openssl req -newkey rsa:2048 -nodes \
-keyout "$D/srv.key" -out "$D/srv.csr" \
-subj "/CN=d2e.local" 2>/dev/null
# The SAN has to go on the signing step: `openssl x509 -req` drops CSR
# extensions unless told otherwise. Without a SAN the cert validates
# against nothing at all, since clients ignore CN when no SAN is
# present — which only became visible once the chart started verifying.
# A wildcard matches exactly one label, so `*.d2e.cluster.local` does
# NOT cover `idp.d2e.svc.cluster.local`. The chart addresses every
# internal service by its cluster DNS name, so the SAN needs the
# namespace-scoped entries too or every hop fails hostname
# verification. The `*.d2e.local` entries are kept so one cert still
# serves docker-compose.
printf '%s\n' \
"subjectAltName=DNS:*.${NAMESPACE}.svc.cluster.local,DNS:*.${NAMESPACE}.svc,DNS:*.d2e.local,DNS:d2e.local,DNS:*.d2e.cluster.local,DNS:d2e.cluster.local" \
'keyUsage=critical,digitalSignature' \
'extendedKeyUsage=serverAuth,clientAuth' > "$D/srv.ext"
openssl x509 -req -in "$D/srv.csr" \
-CA "$D/ca.crt" -CAkey "$D/ca.key" -CAcreateserial \
-out "$D/srv.crt" -days 30 -extfile "$D/srv.ext" 2>/dev/null
# Assert, don't just print: `openssl x509 -noout -ext subjectAltName`
# exits 0 and prints "No extensions in certificate" when the SAN is
# missing, so on its own it cannot fail the job it exists to guard.
openssl x509 -in "$D/srv.crt" -noout -ext subjectAltName \
| tee /dev/stderr | grep -q "DNS:\*\.${NAMESPACE}\.svc\.cluster\.local"
openssl verify -CAfile "$D/ca.crt" "$D/srv.crt"
- name: Helm install (using just-built images)
run: |
D=/tmp/ci-tls
helm install "$RELEASE" "$CHART_DIR" \
-f "$CI_VALUES" \
--set "global.image.tag=${RUN_TAG}" \
--set-file "d2e-core.secrets.ROOT__PKI__CA_CRT=$D/ca.crt" \
--set-file "d2e-core.secrets.ROOT__PKI__CA_KEY=$D/ca.key" \
--set-file "d2e-core.secrets.SERVER__CRT=$D/srv.crt" \
--set-file "d2e-core.secrets.SERVER__KEY=$D/srv.key" \
--set-file "global.secrets.TLS__INTERNAL__CA_CRT=$D/ca.crt" \
--set-file "global.secrets.TLS__INTERNAL__CRT=$D/srv.crt" \
--set-file "global.secrets.TLS__INTERNAL__KEY=$D/srv.key" \
--namespace "$NAMESPACE" \
--wait --timeout 20m
- name: Wait for pods Ready
run: |
kubectl wait \
--for=condition=Ready pod \
--all \
-n "$NAMESPACE" \
--timeout=5m
- name: Show final state
if: always()
run: |
echo "=== pods ==="
kubectl get pods -n "$NAMESPACE" -o wide || true
echo "=== services ==="
kubectl get svc -n "$NAMESPACE" || true
- name: Failure diagnostics
if: failure()
run: |
echo "=== events ==="
kubectl get events -A --sort-by=.lastTimestamp | tail -100 || true
for pod in $(kubectl get pods -n "$NAMESPACE" -o name); do
echo "------------------------------"
echo "DESCRIBE $pod"
echo "------------------------------"
kubectl describe -n "$NAMESPACE" "$pod" || true
echo "------------------------------"
echo "LOGS $pod"
echo "------------------------------"
kubectl logs -n "$NAMESPACE" "$pod" --all-containers --tail=200 || true
done
- name: Mark helm kind attempt success
if: success()
run: |
mkdir -p ./_attempt
touch ./_attempt/success
- name: Upload helm kind attempt outcome
if: always()
uses: actions/upload-artifact@v4
with:
name: helm-kind-attempt
path: ./_attempt/
overwrite: true
- name: Cleanup
if: always()
run: helm uninstall "$RELEASE" -n "$NAMESPACE" || true
test_demosetup_dev:
needs: [setup, build_images, build_trex_image, build_plugins, build_e2e_image]
runs-on: ubuntu-24.04
strategy:
fail-fast: false
matrix:
attempt: [1, 2]
env:
VERSION: develop
DOCKER_TAG_NAME: ${{ needs.setup.outputs.run_tag }}
PLUGINS_IMAGE_TAG: ${{ needs.setup.outputs.run_tag }}
steps:
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
with:
tool-cache: true
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Increase swapfile to 15G
run: |
sudo swapoff -a
sudo fallocate -l 15G /swapfile
sudo chmod 600 /swapfile
sudo mkswap /swapfile
sudo swapon /swapfile
sudo swapon --show
- uses: actions/checkout@v4
with:
submodules: recursive
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: npm install
if: success() || failure()
run: npm install
- name: Install Atlas plugin resources
if: success() || failure()
working-directory: ./plugins/atlas
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# --legacy-peer-deps: @trex/notebook (reached through the nested file:
# link chain atlas -> notebook-plugin -> notebook) pulls webr, which
# pulls react-data-grid@^7.0.0-beta.44. That floats to a beta whose
# peers moved to react@^19.2, conflicting with the react@^18.2.0 pinned
# here. Resolving that conflict across the link chain crashes npm 10.8.2
# with "Cannot read properties of null (reading 'edgesOut')". Skipping
# peer resolution avoids the code path entirely. Same flag, same reason
# as the patient-analytics install above.
run: npm install --legacy-peer-deps
- name: Patch seed config for e2e tests
run: node ./scripts/patch-seed-for-e2e.js
- name: Download UI package tarball
uses: actions/download-artifact@v4
with:
name: ui-tgz
path: ./plugins/ui
- name: Extract UI package
run: |
TGZ=$(ls ./plugins/ui/data2evidence-d2e-ui-*.tgz | head -1)
rm -rf ./plugins/ui-built
mkdir -p ./plugins/ui-built
tar -xzf "$TGZ" -C ./plugins/ui-built --strip-components=1
- name: Initialise d2e
if: success() || failure()
id: init
run: |
init_choice=y ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v develop init
- name: Pre-pull flow images
run: |
for img in $PREPULL_FLOW_IMAGES; do
docker pull "${REG_URL}/d2e/${img}:${DOCKER_TAG_NAME}"
done
- name: Pull docker compose images
if: success() || failure()
run: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v develop pull
docker images --format "table {{.ID}}\t{{.Repository}}\t{{.Tag}}"
- name: Start services
if: success() || failure()
uses: nick-fields/retry@v3
id: start
with:
timeout_seconds: 1800
retry_wait_seconds: 100
max_attempts: 2
retry_on: any
command: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} -c ./docker-compose-ci.yml start
DC_EXIT_CODE=$?
DOCKER_STATUS_FILE=../docker-status-${GITHUB_RUN_ID:-ghRunId}-$(date '+%Y%m%dT%H%M%S').txt | tee -a $GITHUB_ENV
docker ps --format {{.Names}},{{.Status}} 2> /dev/null | sort | tee $DOCKER_STATUS_FILE | tee -a $GITHUB_STEP_SUMMARY
[ $DC_EXIT_CODE = 0 ] || { echo EXIT DC_EXIT_CODE=$DC_EXIT_CODE && exit $DC_EXIT_CODE; }
- name: Log status
if: success() || failure()
run: |
echo DC_EXIT_CODE=$DC_EXIT_CODE | tee -a $GITHUB_STEP_SUMMARY
echo RETRY_EXIT_CODE=${{ steps.start.outputs.exit_code }} | tee -a $GITHUB_STEP_SUMMARY
echo RETRY_OUTCOME=${{ steps.start.outcome }} | tee -a $GITHUB_STEP_SUMMARY
echo RETRY_TOTAL_ATTEMPTS=${{ steps.start.outputs.total_attempts }} | tee -a $GITHUB_STEP_SUMMARY
DC_FAIL_COUNT=$(grep -icE "starting|unhealthy" $DOCKER_STATUS_FILE|| true)
echo DC_FAIL_COUNT=$DC_FAIL_COUNT | tee -a $GITHUB_STEP_SUMMARY
- name: Logs
if: success() || failure()
run: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} logs
- name: Wait for d2e-trex healthy
if: success() || failure()
run: |
status=missing
for i in $(seq 1 60); do
status=$(docker inspect -f '{{.State.Health.Status}}' d2e-trex 2>/dev/null || echo "missing")
echo "[$i] d2e-trex health: $status"
[ "$status" = "healthy" ] && break
sleep 5
done
docker ps --format '{{.Names}},{{.Status}}' | sort
if [ "$status" != "healthy" ]; then
echo "::error::d2e-trex did not become healthy (last status: $status)"
exit 1
fi
# demodb (postgres:16-alpine, docker-compose.yml) is behind the demodb profile and
# is otherwise pulled from Docker Hub lazily inside Setupdemo, where a transient
# registry timeout aborts the whole demo setup. Pre-pull it with retries so the
# image is cached before Setupdemo runs.
- name: Pre-pull demo db image
if: success() || failure()
uses: nick-fields/retry@v3
with:
timeout_seconds: 600
retry_wait_seconds: 30
max_attempts: 5
retry_on: any
command: docker pull postgres:16-alpine
- name: Setupdemo
if: success() || failure()
uses: nick-fields/retry@v3
id: setupdemo
with:
timeout_seconds: 1800
retry_wait_seconds: 100
max_attempts: 3
retry_on: any
command: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} setupdemo
DC_EXIT_CODE=$?
DOCKER_STATUS_FILE=../docker-status-${GITHUB_RUN_ID:-ghRunId}-$(date '+%Y%m%dT%H%M%S').txt | tee -a $GITHUB_ENV
docker ps --format {{.Names}},{{.Status}} 2> /dev/null | sort | tee $DOCKER_STATUS_FILE | tee -a $GITHUB_STEP_SUMMARY
[ $DC_EXIT_CODE = 0 ] || { echo EXIT DC_EXIT_CODE=$DC_EXIT_CODE && exit $DC_EXIT_CODE; }
- name: Logs
if: success() || failure()
run: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} logs
- name: Checkflow
if: success() || failure()
uses: nick-fields/retry@v3
id: checkflow
with:
timeout_seconds: 1800
retry_wait_seconds: 100
max_attempts: 1
retry_on: any
command: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} checkflow
DC_EXIT_CODE=$?
[ $DC_EXIT_CODE = 0 ] || { echo EXIT DC_EXIT_CODE=$DC_EXIT_CODE && exit $DC_EXIT_CODE; }
- name: Restart d2e services
if: success() || failure()
run: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} -c ./docker-compose-ci.yml stop
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} -c ./docker-compose-ci.yml start
- name: Wait for trex to finish the atlas-db-init seeding
run: |
# trex applies atlas-db-init in d2eBoot() before it starts listening, so a
# healthy trex means the seeding (including the logto admin user) has run.
echo "Waiting for trex to become healthy..."
healthy=0
for i in $(seq 1 60); do
status=$(docker ps -a --filter "name=${PROJECT_NAME:-d2e}-trex" --format '{{.Status}}')
if echo "$status" | grep -q "(healthy)"; then
echo "trex is healthy; atlas-db-init has run"
healthy=1
break
fi
if echo "$status" | grep -qE "Exited \("; then
echo "trex failed to start!"
docker logs ${PROJECT_NAME:-d2e}-trex || true
exit 1
fi
echo "Waiting for trex... ($i/60)"
sleep 5
done
if [ "$healthy" != "1" ]; then
echo "trex never reported healthy; atlas-db-init cannot be assumed to have run"
docker logs ${PROJECT_NAME:-d2e}-trex || true
exit 1
fi
- name: Pull e2e Docker image
run: |
docker pull ${REG_URL}/d2e-e2e:${DOCKER_TAG_NAME}
docker tag ${REG_URL}/d2e-e2e:${DOCKER_TAG_NAME} d2e-e2e
- name: Verify app is reachable
run: |
curl -k -s -o /dev/null -w "%{http_code}" https://localhost:443/d2e/portal || echo "App not reachable"
curl -k -I https://localhost:443/d2e/portal || true
- name: Disable IPv6 to prevent ERR_NETWORK_CHANGED
run: sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1
- name: e2e tests
run: |
cd ./tests/e2e
docker run --rm \
--network=host \
--ipc=host \
-v "$(pwd)/test-results:/work/test-results" \
-v "$(pwd)/ctrf:/work/ctrf" \
-e D2E_BASE_URL=https://localhost:443 \
-e CI=true \
-e GITHUB_ACTIONS=true \
d2e-e2e npm test
- name: Mark e2e attempt success
if: success()
run: |
mkdir -p ./_attempt
touch ./_attempt/success
- name: Upload e2e attempt outcome
if: always()
uses: actions/upload-artifact@v4
with:
name: e2e-attempt-${{ matrix.attempt }}
path: ./_attempt/
overwrite: true
- name: Upload Error Context
if: always()
uses: actions/upload-artifact@v4
with:
name: error-context-attempt-${{ matrix.attempt }}
path: tests/e2e/test-results/**/*
overwrite: true
- name: Logs
if: success() || failure()
run: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} logs
- name: Publish Test Report
if: always()
uses: ctrf-io/github-test-reporter@v1
with:
report-path: "./tests/e2e/ctrf/*.json"
file-report: true
- name: Stop d2e services
if: success() || failure()
run: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} stop
- name: Prune system
if: always()
run: docker system prune -af
test_demosetup_hana:
needs: [setup, build_images, build_trex_image, build_plugins]
runs-on: ${{ needs.setup.outputs.random_runner }}
strategy:
fail-fast: false
matrix:
attempt: [1, 2]
env:
VERSION: develop
DOCKER_TAG_NAME: ${{ needs.setup.outputs.run_tag }}
PLUGINS_IMAGE_TAG: ${{ needs.setup.outputs.run_tag }}
steps:
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
with:
tool-cache: true
android: true
dotnet: true
haskell: true
large-packages: false
docker-images: true
swap-storage: true
- uses: actions/checkout@v4
with:
submodules: recursive
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: npm install
if: success() || failure()
run: npm install
- name: Install Atlas plugin resources
if: success() || failure()
working-directory: ./plugins/atlas
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: npm install --legacy-peer-deps
- name: Initialise d2e
if: success() || failure()
run: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v develop -n .env init
# HANA data-model flows need the sqlalchemy-hana dialect. #2824 moved its
# install to the INSTALL_SQLALCHEMY workpool command (blank by default), so
# set that in this test's .env — the dev/DuckDB path doesn't need it.
echo "INSTALL_SQLALCHEMY=bash -c 'uv pip install sqlalchemy-hana==2.2.0 && prefect flow-run execute'" >> .env
env:
init_choice: y
- name: Initialise HANA
if: success() || failure()
run: |
timeout 5m bash -c "ACCEPT_SAP_LICENSE=true DOTENV_FILE=.env ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v develop -n .env --hana inithana" || echo "HANA initialization timed out or failed, continuing..."
- name: Pre-pull flow images
run: |
for img in $PREPULL_FLOW_IMAGES; do
docker pull "${REG_URL}/d2e/${img}:${DOCKER_TAG_NAME}"
done
- name: Pull docker compose images
if: success() || failure()
run: |
DOTENV_FILE=.env ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v develop --hana -d ./plugins/functions pull
docker images --format "table {{.ID}}\t{{.Repository}}\t{{.Tag}}"
- name: Start services
if: success() || failure()
uses: nick-fields/retry@v3
id: start
with:
timeout_seconds: 1800
retry_wait_seconds: 100
max_attempts: 2
retry_on: any
command: |
DOTENV_FILE=.env ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} -d ./plugins/functions --hana start
DC_EXIT_CODE=$?
DOCKER_STATUS_FILE=../docker-status-${GITHUB_RUN_ID:-ghRunId}-$(date '+%Y%m%dT%H%M%S').txt
echo "DOCKER_STATUS_FILE=$DOCKER_STATUS_FILE" >> $GITHUB_ENV
docker ps --format {{.Names}},{{.Status}} 2> /dev/null | sort | tee $DOCKER_STATUS_FILE | tee -a $GITHUB_STEP_SUMMARY
[ $DC_EXIT_CODE = 0 ] || { echo EXIT DC_EXIT_CODE=$DC_EXIT_CODE && exit $DC_EXIT_CODE; }
- name: Log status
if: success() || failure()
run: |
echo DC_EXIT_CODE=$DC_EXIT_CODE | tee -a $GITHUB_STEP_SUMMARY
echo RETRY_EXIT_CODE=${{ steps.start.outputs.exit_code }} | tee -a $GITHUB_STEP_SUMMARY
echo RETRY_OUTCOME=${{ steps.start.outcome }} | tee -a $GITHUB_STEP_SUMMARY
echo RETRY_TOTAL_ATTEMPTS=${{ steps.start.outputs.total_attempts }} | tee -a $GITHUB_STEP_SUMMARY
DC_FAIL_COUNT=$(grep -icE "starting|unhealthy" $DOCKER_STATUS_FILE || true)
echo DC_FAIL_COUNT=$DC_FAIL_COUNT | tee -a $GITHUB_STEP_SUMMARY
- name: Logs
if: success() || failure()
run: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} -d ./plugins/functions logs
- name: Setupdemohana
if: success() || failure()
uses: nick-fields/retry@v3
id: setupdemo
with:
timeout_seconds: 1800
retry_wait_seconds: 100
max_attempts: 1
retry_on: any
command: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} -d ./plugins/functions setupdemohana
DC_EXIT_CODE=$?
DOCKER_STATUS_FILE=../docker-status-${GITHUB_RUN_ID:-ghRunId}-$(date '+%Y%m%dT%H%M%S').txt | tee -a $GITHUB_ENV
docker ps --format {{.Names}},{{.Status}} 2> /dev/null | sort | tee $DOCKER_STATUS_FILE | tee -a $GITHUB_STEP_SUMMARY
[ $DC_EXIT_CODE = 0 ] || { echo EXIT DC_EXIT_CODE=$DC_EXIT_CODE && exit $DC_EXIT_CODE; }
- name: Logs
if: success() || failure()
run: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} -d ./plugins/functions logs
- name: Checkflow
if: success() || failure()
uses: nick-fields/retry@v3
with:
timeout_seconds: 1800
retry_wait_seconds: 100
max_attempts: 1
retry_on: any
command: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} -d ./plugins/functions checkflow
DC_EXIT_CODE=$?
[ $DC_EXIT_CODE = 0 ] || { echo EXIT DC_EXIT_CODE=$DC_EXIT_CODE && exit $DC_EXIT_CODE; }
- name: Mark demo hana attempt success
if: success()
run: |
mkdir -p ./_attempt
touch ./_attempt/success
- name: Upload demo hana attempt outcome
if: always()
uses: actions/upload-artifact@v4
with:
name: demo-hana-attempt-${{ matrix.attempt }}
path: ./_attempt/
overwrite: true
- name: Stop d2e services
if: success() || failure()
run: |
ENV_TYPE=remote CADDY__CONFIG=./deploy/caddy-config node ./scripts/dist/cli.js -e -v ${{ env.VERSION }} -d ./plugins/functions stop
- name: Prune system
if: always()
run: docker system prune -af
test_http_duckdb:
needs: [setup, build_images, build_trex_image, build_plugins]
if: |
needs.setup.outputs.is_publish_push == 'true' ||
contains(github.base_ref, 'release/') ||
github.event_name == 'workflow_dispatch' ||
github.event_name == 'merge_group' ||
(github.event_name == 'pull_request' && needs.setup.outputs.changes_http_tests == 'true')
uses: ./.github/workflows/_test-http-duckdb.yml
with:
attempt: "1"
runner: ${{ needs.setup.outputs.random_runner }}
run_tag: ${{ needs.setup.outputs.run_tag }}
changes_ui: ${{ needs.setup.outputs.changes_ui }}
reg_url: ghcr.io/ohdsi
prepull_flow_images: ""
secrets: inherit
test_http_duckdb_retry:
needs: [setup, build_images, build_trex_image, build_plugins, test_http_duckdb]
if: |
always() && needs.test_http_duckdb.result == 'failure' && (
needs.setup.outputs.is_publish_push == 'true' ||
contains(github.base_ref, 'release/') ||
github.event_name == 'workflow_dispatch' ||
github.event_name == 'merge_group' ||
(github.event_name == 'pull_request' && needs.setup.outputs.changes_http_tests == 'true')
)
uses: ./.github/workflows/_test-http-duckdb.yml
with:
attempt: "2"
runner: ${{ needs.setup.outputs.random_runner }}
run_tag: ${{ needs.setup.outputs.run_tag }}
changes_ui: ${{ needs.setup.outputs.changes_ui }}
reg_url: ghcr.io/ohdsi
prepull_flow_images: ""
secrets: inherit
test_http_hana:
needs: [setup, build_images, build_trex_image, build_plugins]
if: |
needs.setup.outputs.is_publish_push == 'true' ||
contains(github.base_ref, 'release/') ||
github.event_name == 'workflow_dispatch' ||
github.event_name == 'merge_group' ||
(github.event_name == 'pull_request' && needs.setup.outputs.changes_http_tests == 'true')
uses: ./.github/workflows/_test-http-hana.yml
with:
attempt: "1"
runner: ${{ needs.setup.outputs.random_runner }}
run_tag: ${{ needs.setup.outputs.run_tag }}
reg_url: ghcr.io/ohdsi
prepull_flow_images: ""
secrets: inherit
test_http_hana_retry:
needs: [setup, build_images, build_trex_image, build_plugins, test_http_hana]
if: |
always() && needs.test_http_hana.result == 'failure' && (
needs.setup.outputs.is_publish_push == 'true' ||
contains(github.base_ref, 'release/') ||
github.event_name == 'workflow_dispatch' ||
github.event_name == 'merge_group' ||
(github.event_name == 'pull_request' && needs.setup.outputs.changes_http_tests == 'true')
)
uses: ./.github/workflows/_test-http-hana.yml
with:
attempt: "2"
runner: ${{ needs.setup.outputs.random_runner }}
run_tag: ${{ needs.setup.outputs.run_tag }}
reg_url: ghcr.io/ohdsi
prepull_flow_images: ""
secrets: inherit
test_regression:
needs: [setup, build_images, build_trex_image, build_plugins]
if: |
needs.setup.outputs.is_publish_push == 'true' ||
contains(github.base_ref, 'release/') ||
github.event_name == 'workflow_dispatch' ||
github.event_name == 'merge_group' ||
(github.event_name == 'pull_request' && (needs.setup.outputs.changes_regression == 'true' || needs.setup.outputs.changes_http_tests == 'true'))
uses: ./.github/workflows/_test-regression.yml
with:
attempt: "1"
runner: ${{ needs.setup.outputs.random_runner }}
run_tag: ${{ needs.setup.outputs.run_tag }}
reg_url: ghcr.io/ohdsi
prepull_flow_images: ""
secrets: inherit
test_regression_gate:
needs: [test_regression]
if: always() && needs.test_regression.result != 'skipped'
runs-on: ubuntu-latest
steps:
- uses: actions/download-artifact@v4
with:
pattern: regression-attempt-*
path: ./_attempts
- name: Pass if regression attempt succeeded
run: |
if find ./_attempts -name 'success' -type f | grep -q .; then
echo "Regression performance test attempt succeeded."
exit 0
fi
echo "Regression performance test attempt failed."
exit 1
test_demosetup_dev_gate:
needs: [test_demosetup_dev]
if: always() && needs.test_demosetup_dev.result != 'skipped'
runs-on: ubuntu-latest
steps:
- uses: actions/download-artifact@v4
with:
pattern: e2e-attempt-*
path: ./_attempts
- name: Pass if any attempt succeeded
run: |
if find ./_attempts -name 'success' -type f | grep -q .; then
echo "At least one e2e attempt succeeded."
exit 0
fi
echo "All e2e attempts failed."
exit 1
test_demosetup_hana_gate:
needs: [test_demosetup_hana]
if: always() && needs.test_demosetup_hana.result != 'skipped'
runs-on: ubuntu-latest
steps:
- uses: actions/download-artifact@v4
with:
pattern: demo-hana-attempt-*
path: ./_attempts
- name: Pass if any attempt succeeded
run: |
if find ./_attempts -name 'success' -type f | grep -q .; then
echo "At least one demo hana attempt succeeded."
exit 0
fi
echo "All demo hana attempts failed."
exit 1
test_http_duckdb_gate:
needs: [test_http_duckdb, test_http_duckdb_retry]
if: always() && needs.test_http_duckdb.result != 'skipped'
runs-on: ubuntu-latest
steps:
- uses: actions/download-artifact@v4
with:
pattern: http-duckdb-attempt-*
path: ./_attempts
- name: Pass if any attempt succeeded
run: |
if find ./_attempts -name 'success' -type f | grep -q .; then
echo "At least one HTTP duckdb attempt succeeded."
exit 0
fi
echo "All HTTP duckdb attempts failed."
exit 1
test_http_hana_gate:
needs: [test_http_hana, test_http_hana_retry]
if: always() && needs.test_http_hana.result != 'skipped'
runs-on: ubuntu-latest
steps:
- uses: actions/download-artifact@v4
with:
pattern: http-hana-attempt-*
path: ./_attempts
- name: Pass if any attempt succeeded
run: |
if find ./_attempts -name 'success' -type f | grep -q .; then
echo "At least one HTTP hana attempt succeeded."
exit 0
fi
echo "All HTTP hana attempts failed."
exit 1
test_helm_kind_gate:
needs: [test_helm_kind]
if: always() && needs.test_helm_kind.result != 'skipped'
runs-on: ubuntu-latest
steps:
- uses: actions/download-artifact@v4
with:
pattern: helm-kind-attempt
path: ./_attempts
- name: Pass if helm kind attempt succeeded
run: |
if find ./_attempts -name 'success' -type f | grep -q .; then
echo "Helm kind attempt succeeded."
exit 0
fi
echo "Helm kind attempt failed."
exit 1
promote:
needs: [setup, build_images, build_trex_image]
if: |
always() &&
needs.build_images.result == 'success' &&
needs.build_trex_image.result == 'success' &&
needs.setup.outputs.is_publish_push == 'true'
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
env:
RUN_TAG: ${{ needs.setup.outputs.run_tag }}
CANONICAL_TAG: ${{ needs.setup.outputs.canonical_tag }}
strategy:
fail-fast: false
matrix:
image:
- d2e-logto
- d2e-logto-post-init
- d2e-trex
- d2e-dataflow-gen
- d2e-dataflow-gen-worker
- d2e-supabase-storage
- d2e-materialize-cohorts
steps:
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Re-tag run image as canonical
run: |
SRC=${REG_URL}/${{ matrix.image }}:${RUN_TAG}
DST=${REG_URL}/${{ matrix.image }}:${CANONICAL_TAG}
if [[ -z "$CANONICAL_TAG" ]]; then
echo "CANONICAL_TAG empty; nothing to promote."
exit 0
fi
if docker manifest inspect "$SRC" >/dev/null 2>&1; then
docker buildx imagetools create -t "$DST" "$SRC"
elif docker manifest inspect "$DST" >/dev/null 2>&1; then
# The image was skipped this run (build_disable), so no RUN_TAG was
# pushed for it. Publish one from the canonical tag instead of
# leaving a hole: consumers ask for ONE tag across the whole set —
# docker-compose pulls every service at DOCKER_TAG_NAME, and trex's
# e2e resolves the newest tag that ALL images share. A single
# missing tag makes the entire set unusable, which surfaces far from
# here as "manifest unknown" on an image nobody touched.
#
# imagetools create copies the manifest server-side, so this costs a
# registry call rather than a rebuild, and keeps multi-arch lists
# intact.
echo "$SRC not present (build_disable); publishing it from $DST"
docker buildx imagetools create -t "$SRC" "$DST"
else
echo "Neither $SRC nor $DST present; nothing to publish."
fi
# Multi-arch-aware cleanup: the per-arch child manifests of a manifest
# list are stored as untagged package versions in GHCR, so a blind
# delete-only-untagged sweep (actions/delete-package-versions) removes
# the children of freshly pushed multi-arch tags and leaves the tag
# unpullable ("manifest unknown"). This action resolves manifest-list
# references and only deletes truly orphaned versions.
- name: Delete orphaned untagged versions
uses: dataaxiom/ghcr-cleanup-action@v1
continue-on-error: true
with:
package: ${{ matrix.image }}
delete-untagged: true
validate: true
docker-success:
needs: [build_images, build_trex_image, build_plugins, build_e2e_image, helm_lint, test_helm_kind_gate, test_demosetup_dev_gate, test_demosetup_hana_gate, test_http_duckdb_gate, test_http_hana_gate, test_regression_gate]
if: always()
runs-on: ubuntu-latest
steps:
- name: Check upstream job results
env:
R_BUILD_IMAGES: ${{ needs.build_images.result }}
R_BUILD_TREX: ${{ needs.build_trex_image.result }}
R_BUILD_PLUGINS: ${{ needs.build_plugins.result }}
R_BUILD_E2E: ${{ needs.build_e2e_image.result }}
R_HELM_LINT: ${{ needs.helm_lint.result }}
R_HELM_KIND: ${{ needs.test_helm_kind_gate.result }}
R_DEMO_DEV: ${{ needs.test_demosetup_dev_gate.result }}
R_DEMO_HANA: ${{ needs.test_demosetup_hana_gate.result }}
R_HTTP_DUCKDB: ${{ needs.test_http_duckdb_gate.result }}
R_HTTP_HANA: ${{ needs.test_http_hana_gate.result }}
R_REGRESSION: ${{ needs.test_regression_gate.result }}
run: |
fail=0
for r in "$R_BUILD_IMAGES" "$R_BUILD_TREX" "$R_BUILD_PLUGINS" "$R_BUILD_E2E" "$R_HELM_LINT" "$R_HELM_KIND" "$R_DEMO_DEV" "$R_DEMO_HANA" "$R_HTTP_DUCKDB" "$R_HTTP_HANA" "$R_REGRESSION"; do
case "$r" in
success|skipped) ;;
*) fail=1 ;;
esac
done
exit $fail