From fb5c4dc5642b15715125803d4b8b5162a6101f1c Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Tue, 14 Apr 2026 14:33:33 -0700 Subject: [PATCH 1/3] init 1.4.0 --- airflow/versions/1.4.0/Chart.yaml | 12 ++ airflow/versions/1.4.0/README.md | 178 ++++++++++++++++++ airflow/versions/1.4.0/templates/_helpers.tpl | 104 ++++++++++ airflow/versions/1.4.0/templates/gvc.yaml | 15 ++ .../versions/1.4.0/templates/identity.yaml | 6 + airflow/versions/1.4.0/templates/policy.yaml | 15 ++ airflow/versions/1.4.0/templates/secret.yaml | 15 ++ .../1.4.0/templates/volumeset-airflow.yaml | 12 ++ .../1.4.0/templates/volumeset-postgres.yaml | 10 + .../1.4.0/templates/volumeset-redis.yaml | 10 + .../1.4.0/templates/workload-celery.yaml | 100 ++++++++++ .../1.4.0/templates/workload-postgresql.yaml | 52 +++++ .../1.4.0/templates/workload-redis.yaml | 38 ++++ .../1.4.0/templates/workload-webserver.yaml | 151 +++++++++++++++ airflow/versions/1.4.0/values.yaml | 90 +++++++++ 15 files changed, 808 insertions(+) create mode 100644 airflow/versions/1.4.0/Chart.yaml create mode 100644 airflow/versions/1.4.0/README.md create mode 100644 airflow/versions/1.4.0/templates/_helpers.tpl create mode 100644 airflow/versions/1.4.0/templates/gvc.yaml create mode 100644 airflow/versions/1.4.0/templates/identity.yaml create mode 100644 airflow/versions/1.4.0/templates/policy.yaml create mode 100644 airflow/versions/1.4.0/templates/secret.yaml create mode 100644 airflow/versions/1.4.0/templates/volumeset-airflow.yaml create mode 100644 airflow/versions/1.4.0/templates/volumeset-postgres.yaml create mode 100644 airflow/versions/1.4.0/templates/volumeset-redis.yaml create mode 100644 airflow/versions/1.4.0/templates/workload-celery.yaml create mode 100644 airflow/versions/1.4.0/templates/workload-postgresql.yaml create mode 100644 airflow/versions/1.4.0/templates/workload-redis.yaml create mode 100644 airflow/versions/1.4.0/templates/workload-webserver.yaml create mode 100644 airflow/versions/1.4.0/values.yaml diff --git a/airflow/versions/1.4.0/Chart.yaml b/airflow/versions/1.4.0/Chart.yaml new file mode 100644 index 00000000..5d12cf81 --- /dev/null +++ b/airflow/versions/1.4.0/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +name: airflow +description: Airflow with Celery using Keda, Redis and Postgres +type: application +version: 1.3.0 +appVersion: "3.0.3" + +annotations: + created: "2025-08-14" + lastModified: "2026-03-10" + category: "app" + createsGvc: true \ No newline at end of file diff --git a/airflow/versions/1.4.0/README.md b/airflow/versions/1.4.0/README.md new file mode 100644 index 00000000..94473207 --- /dev/null +++ b/airflow/versions/1.4.0/README.md @@ -0,0 +1,178 @@ +# Apache Airflow + +Deploys Apache Airflow 3.x with CeleryExecutor using Redis as the message broker and PostgreSQL as the metadata database, with optional KEDA autoscaling and git-sync DAG delivery. + +## What's Included + +- **Airflow Webserver** – Hosts the web UI, API server, scheduler, dag-processor, and triggerer +- **Celery Workers** – Distributed task execution workers that process DAG tasks +- **Redis** – Message broker for the Celery task queue (persistent volume) +- **PostgreSQL** – Metadata database for Airflow state (persistent volume) +- **KEDA Autoscaling** (optional) – Scales Celery workers automatically based on Redis queue length +- **git-sync** (optional) – Sidecar that continuously syncs DAGs from a Git repository + +## Pre-Deployment Checklist + +Before deploying, update the following **required** values in `values.yaml`: + +| Value | How to generate | +|-------|----------------| +| `airflow.auth.jwtSecret` | `openssl rand -base64 48` | +| `airflow.auth.fernetKey` | `python3 -c 'from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())'` | +| `airflow.admin.password` | Choose a strong password | +| `postgres.config.password` | Choose a strong password | + +## Configuration Reference + +### GVC Settings + +| Property | Description | +|----------|-------------| +| `gvc.name` | Name of the GVC (must be unique per deployment) | +| `gvc.locations` | Cloud locations to deploy to (e.g. `aws-eu-central-1`) | + +### PostgreSQL + +| Property | Description | +|----------|-------------| +| `postgres.image` | PostgreSQL Docker image | +| `postgres.resources.minCpu` / `maxCpu` | CPU request / limit | +| `postgres.resources.minMemory` / `maxMemory` | Memory request / limit | +| `postgres.config.username` | Database username | +| `postgres.config.password` | Database password | +| `postgres.config.database` | Database name | +| `postgres.volumeset.capacity` | Storage capacity in GiB (minimum 10) | + +### Redis + +| Property | Description | +|----------|-------------| +| `redis.image` | Redis Docker image | +| `redis.resources.cpu` / `memory` | CPU and memory allocation | +| `redis.volumeset.capacity` | Storage capacity in GiB (minimum 10) | + +### Airflow + +| Property | Description | +|----------|-------------| +| `airflow.webserver.image` | Airflow Docker image (e.g. `apache/airflow:3.0.3`) | +| `airflow.webserver.resources.cpu` / `memory` | Webserver CPU and memory | +| `airflow.celeryWorker.image` | Celery worker Docker image | +| `airflow.celeryWorker.resources.cpu` / `memory` | Worker CPU and memory | +| `airflow.webPort` | Port for the Airflow web interface (default `8080`) | + +#### Authentication + +| Property | Description | +|----------|-------------| +| `airflow.auth.jwtSecret` | **Required.** Secret for signing JWT tokens | +| `airflow.auth.jwtExpirationDelta` | JWT token lifetime in seconds | +| `airflow.auth.jwtRefreshThreshold` | Seconds before expiry to allow token refresh | +| `airflow.auth.fernetKey` | **Required.** Encrypts connections and variables stored in the database | + +#### Admin User + +| Property | Description | +|----------|-------------| +| `airflow.admin.username` | Initial admin username | +| `airflow.admin.password` | **Required.** Initial admin password | + +The admin user is created on first startup using Airflow's `SimpleAuthManager`. The credentials are written to a password file on the shared volume and re-applied on every container restart, so the password always reflects the current value in `values.yaml`. + +> **Note:** `SimpleAuthManager` is the default auth manager in Airflow 3.x and is suitable for development and internal deployments. For production deployments requiring SSO or LDAP, consider integrating an external auth provider via OAuth/OIDC. + +#### Scheduler + +| Property | Description | +|----------|-------------| +| `airflow.scheduler.dagDirListInterval` | How often to scan the DAG folder (seconds) | +| `airflow.scheduler.minFileProcessInterval` | Minimum interval between DAG file processing (seconds) | + +#### Celery + +| Property | Description | +|----------|-------------| +| `airflow.celery.workerConcurrency` | Tasks each worker runs concurrently (when KEDA is enabled) | + +### Volumes + +| Property | Description | +|----------|-------------| +| `volumeset.airflow.capacity` | Shared volume for Airflow home directory in GiB (minimum 10) | + +The Airflow volume uses a shared (`NFS-style`) filesystem, allowing both the webserver and Celery workers to access DAGs and logs from the same volume. + +### Firewall + +| Property | Description | +|----------|-------------| +| `firewallConfig.inboundAllowCIDR` | List of CIDRs allowed to access the Airflow UI. Defaults to `0.0.0.0/0` (public). Restrict in production. | + +### DAG Delivery (git-sync) + +DAGs are delivered via a `git-sync` sidecar that continuously polls a Git repository and syncs files to the shared Airflow volume. This is the recommended approach for managing DAGs in production. + +| Property | Description | +|----------|-------------| +| `gitSync.enabled` | Enable git-sync DAG delivery (default `false`) | +| `gitSync.repo` | Git repository URL (e.g. `https://github.com/org/dags`) | +| `gitSync.branch` | Branch to sync (default `main`) | +| `gitSync.period` | Sync interval (default `60s`) | +| `gitSync.subPath` | Subfolder within the repo containing DAGs (leave empty if DAGs are at the repo root) | +| `gitSync.auth.token` | Personal access token for private repos (leave empty for public repos) | + +When git-sync is disabled, DAGs can be placed manually in the `/opt/airflow/dags` directory on the Airflow volume. + +### KEDA Autoscaling + +| Property | Description | +|----------|-------------| +| `keda.enabled` | Enable KEDA autoscaling (default `true`) | +| `keda.minScale` | Minimum number of Celery workers | +| `keda.maxScale` | Maximum number of Celery workers | +| `keda.scaleToZeroDelay` | Seconds of idle time before scaling to zero | +| `keda.listLength` | Redis queue length threshold that triggers scaling | +| `keda.cooldownPeriod` | Cooldown between scaling events (seconds) | +| `keda.initialCooldownPeriod` | Cooldown after startup before scaling begins (seconds) | +| `keda.pollingInterval` | How often KEDA queries Redis metrics (seconds) | + +## Accessing Airflow + +Once deployed, the Airflow UI is available at the canonical endpoint of the webserver workload: + +``` +https://-airflow-webserver..cpln.app +``` + +Log in with the `airflow.admin.username` and `airflow.admin.password` set in `values.yaml`. + +### API Access + +Airflow 3.x uses JWT-based authentication. To obtain a token: + +```bash +curl -X POST https:///auth/token \ + -H "Content-Type: application/json" \ + -d '{"username": "admin", "password": "your-password"}' +``` + +Use the returned token for subsequent API requests: + +```bash +curl https:///api/v2/dags \ + -H "Authorization: Bearer " +``` + +## Production Considerations + +- **Change all `CHANGE_ME` values** before deploying — `jwtSecret`, `fernetKey`, and `admin.password` are all required +- **Restrict `firewallConfig.inboundAllowCIDR`** to trusted IP ranges to limit access to the Airflow UI +- **Enable git-sync** for reliable, version-controlled DAG delivery +- **Auth**: `SimpleAuthManager` is not recommended for deployments requiring enterprise SSO. Evaluate an OAuth/OIDC integration for those use cases + +## References + +- [Apache Airflow Documentation](https://airflow.apache.org/docs/) +- [Redis Documentation](https://redis.io/docs/latest/) +- [KEDA Documentation](https://keda.sh/docs/) +- [git-sync Documentation](https://github.com/kubernetes/git-sync) diff --git a/airflow/versions/1.4.0/templates/_helpers.tpl b/airflow/versions/1.4.0/templates/_helpers.tpl new file mode 100644 index 00000000..69d38fee --- /dev/null +++ b/airflow/versions/1.4.0/templates/_helpers.tpl @@ -0,0 +1,104 @@ +{{/* Resource Naming */}} + +{{/* +Airflow Celery Workload Name +*/}} +{{- define "airflow.celery.name" -}} +{{- printf "%s-airflow-celery-worker" .Release.Name }} +{{- end }} + +{{/* +Airflow Webserver Workload Name +*/}} +{{- define "airflow.webserver.name" -}} +{{- printf "%s-airflow-webserver" .Release.Name }} +{{- end }} + +{{/* +Airflow Postgres Workload Name +*/}} +{{- define "airflow.postgres.name" -}} +{{- printf "%s-airflow-postgres" .Release.Name }} +{{- end }} + +{{/* +Postgres Volume Set Name +*/}} +{{- define "airflow.postgresVolume.name" -}} +{{- printf "%s-airflow-postgres-vs" .Release.Name }} +{{- end }} + +{{/* +Airflow Redis Workload Name +*/}} +{{- define "airflow.redis.name" -}} +{{- printf "%s-airflow-redis" .Release.Name }} +{{- end }} + +{{/* +Redis Volume Set Name +*/}} +{{- define "airflow.redisVolume.name" -}} +{{- printf "%s-airflow-redis-vs" .Release.Name }} +{{- end }} + +{{/* +Airflow Secret Name +*/}} +{{- define "airflow.secret.name" -}} +{{- printf "%s-airflow-config" .Release.Name }} +{{- end }} + +{{/* +Airflow Identity Name +*/}} +{{- define "airflow.identity.name" -}} +{{- printf "%s-airflow-identity" .Release.Name }} +{{- end }} + +{{/* +Airflow Policy Name +*/}} +{{- define "airflow.policy.name" -}} +{{- printf "%s-airflow-policy" .Release.Name }} +{{- end }} + +{{/* +Airflow Volume Set Name +*/}} +{{- define "airflow.volume.name" -}} +{{- printf "%s-airflow-vs" .Release.Name }} +{{- end }} + + +{{/* Labeling */}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airflow.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "airflow.tags" -}} +helm.sh/chart: {{ include "airflow.chart" . }} +{{ include "airflow.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.cpln.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.cpln.io/managed-by: {{ .Release.Service }} +cpln/marketplace: "true" +cpln/marketplace-template: airflow +cpln/marketplace-template-version: {{ .Chart.Version }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "airflow.selectorLabels" -}} +app.cpln.io/name: {{ .Release.Name }} +app.cpln.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/airflow/versions/1.4.0/templates/gvc.yaml b/airflow/versions/1.4.0/templates/gvc.yaml new file mode 100644 index 00000000..0e6ce524 --- /dev/null +++ b/airflow/versions/1.4.0/templates/gvc.yaml @@ -0,0 +1,15 @@ +kind: gvc +name: {{ .Values.gvc.name }} +description: Airflow GVC +tags: {{- include "airflow.tags" . | nindent 4 }} +spec: + endpointNamingFormat: org + {{- if .Values.keda.enabled }} + keda: + enabled: true + {{- end }} + staticPlacement: + locationLinks: + {{- range .Values.gvc.locations }} + - //location/{{ .name }} + {{- end }} diff --git a/airflow/versions/1.4.0/templates/identity.yaml b/airflow/versions/1.4.0/templates/identity.yaml new file mode 100644 index 00000000..049ce42e --- /dev/null +++ b/airflow/versions/1.4.0/templates/identity.yaml @@ -0,0 +1,6 @@ +--- +kind: identity +name: {{ include "airflow.identity.name" . }} +description: Airflow identity +tags: {{- include "airflow.tags" . | nindent 4 }} +gvc: {{ .Values.gvc.name }} diff --git a/airflow/versions/1.4.0/templates/policy.yaml b/airflow/versions/1.4.0/templates/policy.yaml new file mode 100644 index 00000000..c1bd42f1 --- /dev/null +++ b/airflow/versions/1.4.0/templates/policy.yaml @@ -0,0 +1,15 @@ +# Helm template for Airflow policies +--- +kind: policy +name: {{ include "airflow.policy.name" . }} +description: Airflow policy +tags: + {{- include "airflow.tags" . | nindent 4 }} +bindings: + - permissions: + - reveal + principalLinks: + - //gvc/{{ .Values.gvc.name }}/identity/{{ include "airflow.identity.name" . }} +targetKind: secret +targetLinks: + - //secret/{{ include "airflow.secret.name" . }} \ No newline at end of file diff --git a/airflow/versions/1.4.0/templates/secret.yaml b/airflow/versions/1.4.0/templates/secret.yaml new file mode 100644 index 00000000..31446c63 --- /dev/null +++ b/airflow/versions/1.4.0/templates/secret.yaml @@ -0,0 +1,15 @@ +kind: secret +name: {{ include "airflow.secret.name" . }} +description: Airflow config +tags: + {{- include "airflow.tags" . | nindent 4 }} +type: dictionary +data: + username: {{ .Values.postgres.config.username | quote }} + password: {{ .Values.postgres.config.password | quote }} + jwtSecret: {{ .Values.airflow.auth.jwtSecret | quote }} + fernetKey: {{ .Values.airflow.auth.fernetKey | quote }} + adminPassword: {{ .Values.airflow.admin.password | quote }} + {{- if .Values.gitSync.auth.token }} + gitSyncToken: {{ .Values.gitSync.auth.token | quote }} + {{- end }} \ No newline at end of file diff --git a/airflow/versions/1.4.0/templates/volumeset-airflow.yaml b/airflow/versions/1.4.0/templates/volumeset-airflow.yaml new file mode 100644 index 00000000..c021dd2c --- /dev/null +++ b/airflow/versions/1.4.0/templates/volumeset-airflow.yaml @@ -0,0 +1,12 @@ +kind: volumeset +name: {{ include "airflow.volume.name" . }} +description: Airflow volume set +gvc: {{ .Values.gvc.name }} +tags: + {{- include "airflow.tags" . | nindent 4 }} +spec: + fileSystemType: shared + initialCapacity: {{ .Values.volumeset.airflow.capacity }} + performanceClass: shared + + diff --git a/airflow/versions/1.4.0/templates/volumeset-postgres.yaml b/airflow/versions/1.4.0/templates/volumeset-postgres.yaml new file mode 100644 index 00000000..0cccceef --- /dev/null +++ b/airflow/versions/1.4.0/templates/volumeset-postgres.yaml @@ -0,0 +1,10 @@ +kind: volumeset +name: {{ include "airflow.postgresVolume.name" . }} +description: Airflow Postgres volume set +gvc: {{ .Values.gvc.name }} +tags: + {{- include "airflow.tags" . | nindent 4 }} +spec: + fileSystemType: ext4 + initialCapacity: {{ .Values.postgres.volumeset.capacity }} + performanceClass: general-purpose-ssd diff --git a/airflow/versions/1.4.0/templates/volumeset-redis.yaml b/airflow/versions/1.4.0/templates/volumeset-redis.yaml new file mode 100644 index 00000000..22235ff8 --- /dev/null +++ b/airflow/versions/1.4.0/templates/volumeset-redis.yaml @@ -0,0 +1,10 @@ +kind: volumeset +name: {{ include "airflow.redisVolume.name" . }} +description: Airflow Redis volume set +gvc: {{ .Values.gvc.name }} +tags: + {{- include "airflow.tags" . | nindent 4 }} +spec: + fileSystemType: ext4 + initialCapacity: {{ .Values.redis.volumeset.capacity }} + performanceClass: general-purpose-ssd diff --git a/airflow/versions/1.4.0/templates/workload-celery.yaml b/airflow/versions/1.4.0/templates/workload-celery.yaml new file mode 100644 index 00000000..fe7da923 --- /dev/null +++ b/airflow/versions/1.4.0/templates/workload-celery.yaml @@ -0,0 +1,100 @@ +kind: workload +name: {{ include "airflow.celery.name" . }} +description: Airflow Celery worker +gvc: {{ .Values.gvc.name }} +tags: {{- include "airflow.tags" . | nindent 4 }} +spec: + type: stateful + identityLink: //identity/{{ include "airflow.identity.name" . }} + containers: + - name: celery-worker + cpu: {{ .Values.airflow.celeryWorker.resources.cpu | quote }} + memory: {{ .Values.airflow.celeryWorker.resources.memory | quote }} + env: + - name: AIRFLOW_HOME + value: /opt/airflow + - name: AIRFLOW__API_AUTH__JWT_EXPIRATION_DELTA + value: "{{ .Values.airflow.auth.jwtExpirationDelta }}" + - name: AIRFLOW__API_AUTH__JWT_REFRESH_THRESHOLD + value: "{{ .Values.airflow.auth.jwtRefreshThreshold }}" + - name: POSTGRES_USERNAME + value: cpln://secret/{{ include "airflow.secret.name" . }}.username + - name: POSTGRES_PASSWORD + value: cpln://secret/{{ include "airflow.secret.name" . }}.password + - name: AIRFLOW__API_AUTH__JWT_SECRET + value: cpln://secret/{{ include "airflow.secret.name" . }}.jwtSecret + - name: AIRFLOW__CORE__FERNET_KEY + value: cpln://secret/{{ include "airflow.secret.name" . }}.fernetKey + - name: AIRFLOW__API__BASE_URL + value: http://{{ include "airflow.webserver.name" . }}:{{ .Values.airflow.webPort }} + - name: AIRFLOW__CELERY__BROKER_URL + value: redis://{{ include "airflow.redis.name" . }}.{{ .Values.gvc.name }}.cpln.local + - name: AIRFLOW__CELERY__RESULT_BACKEND + value: db+postgresql://$(POSTGRES_USERNAME):$(POSTGRES_PASSWORD)@{{ include "airflow.postgres.name" . }}:5432/{{ .Values.postgres.config.database }} + {{- if .Values.keda.enabled }} + - name: AIRFLOW__CELERY__WORKER_CONCURRENCY + value: "{{ .Values.airflow.celery.workerConcurrency }}" + {{- end }} + {{- if .Values.gitSync.enabled }} + - name: AIRFLOW__CORE__DAGS_FOLDER + value: /opt/airflow/dags-sync/current{{ if .Values.gitSync.subPath }}/{{ .Values.gitSync.subPath }}{{ end }} + {{- else }} + - name: AIRFLOW__CORE__DAGS_FOLDER + value: /opt/airflow/dags + {{- end }} + - name: AIRFLOW__CORE__EXECUTOR + value: CeleryExecutor + - name: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN + value: postgresql+psycopg2://$(POSTGRES_USERNAME):$(POSTGRES_PASSWORD)@{{ include "airflow.postgres.name" . }}:5432/{{ .Values.postgres.config.database }} + - name: AIRFLOW__LOGGING__REMOTE_LOGGING + value: 'False' + image: {{ .Values.airflow.celeryWorker.image }} + inheritEnv: false + ports: [] + volumes: + - path: /opt/airflow + recoveryPolicy: retain + uri: cpln://volumeset/{{ include "airflow.volume.name" . }} + {{- if .Values.keda.enabled }} + defaultOptions: + autoscaling: + keda: + cooldownPeriod: {{ .Values.keda.cooldownPeriod }} + initialCooldownPeriod: {{ .Values.keda.initialCooldownPeriod }} + pollingInterval: {{ .Values.keda.pollingInterval }} + triggers: + - name: redis-trigger + type: redis + metadata: + address: {{ include "airflow.redis.name" . }}.{{ .Values.gvc.name }}.cpln.local:6379 + listLength: "{{ .Values.keda.listLength }}" + listName: default + maxConcurrency: 0 + maxScale: {{ .Values.keda.maxScale }} + metric: keda + minScale: {{ .Values.keda.minScale }} + scaleToZeroDelay: {{ .Values.keda.scaleToZeroDelay }} + capacityAI: false + debug: false + suspend: false + timeoutSeconds: 5 + {{- end }} + firewallConfig: + external: + inboundAllowCIDR: + - 0.0.0.0/0 + inboundBlockedCIDR: [] + outboundAllowCIDR: + - 0.0.0.0/0 + outboundAllowHostname: [] + outboundAllowPort: [] + outboundBlockedCIDR: [] + internal: + inboundAllowType: same-gvc + inboundAllowWorkload: [] + loadBalancer: + direct: + enabled: false + ports: [] + replicaDirect: false + supportDynamicTags: false diff --git a/airflow/versions/1.4.0/templates/workload-postgresql.yaml b/airflow/versions/1.4.0/templates/workload-postgresql.yaml new file mode 100644 index 00000000..923c7a88 --- /dev/null +++ b/airflow/versions/1.4.0/templates/workload-postgresql.yaml @@ -0,0 +1,52 @@ +kind: workload +name: {{ include "airflow.postgres.name" . }} +description: Airflow Postgres +gvc: {{ .Values.gvc.name }} +tags: + {{- include "airflow.tags" . | nindent 2 }} +spec: + type: stateful + identityLink: //identity/{{ include "airflow.identity.name" . }} + containers: + - name: postgresql + env: + - name: PGDATA + value: "/var/lib/postgresql/data/pg_data" + - name: POSTGRES_DB + value: {{ .Values.postgres.config.database }} + - name: POSTGRES_PASSWORD + value: 'cpln://secret/{{ include "airflow.secret.name" . }}.password' + - name: POSTGRES_USER + value: 'cpln://secret/{{ include "airflow.secret.name" . }}.username' + image: {{ .Values.postgres.image }} + inheritEnv: false + minCpu: {{ .Values.postgres.resources.minCpu | quote }} + cpu: {{ .Values.postgres.resources.maxCpu | quote }} + minMemory: {{ .Values.postgres.resources.minMemory | quote }} + memory: {{ .Values.postgres.resources.maxMemory | quote }} + ports: + - number: 5432 + protocol: tcp + readinessProbe: + exec: + command: + - sh + - '-c' + - 'pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}' + failureThreshold: 10 + initialDelaySeconds: 17 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 3 + volumes: + - path: /var/lib/postgresql/data + uri: 'cpln://volumeset/{{ include "airflow.postgresVolume.name" . }}' + defaultOptions: + autoscaling: + metric: disabled + minScale: 1 + maxScale: 1 + capacityAI: false + firewallConfig: + internal: + inboundAllowType: same-gvc diff --git a/airflow/versions/1.4.0/templates/workload-redis.yaml b/airflow/versions/1.4.0/templates/workload-redis.yaml new file mode 100644 index 00000000..c037fd6c --- /dev/null +++ b/airflow/versions/1.4.0/templates/workload-redis.yaml @@ -0,0 +1,38 @@ +kind: workload +name: {{ include "airflow.redis.name" . }} +description: Airflow Redis +gvc: {{ .Values.gvc.name }} +tags: + {{- include "airflow.tags" . | nindent 2 }} +spec: + type: stateful + containers: + - name: redis + cpu: {{ .Values.redis.resources.cpu | quote }} + memory: {{ .Values.redis.resources.memory | quote }} + image: {{ .Values.redis.image }} + inheritEnv: false + ports: + - number: 6379 + protocol: tcp + volumes: + - path: /data + uri: cpln://volumeset/{{ include "airflow.redisVolume.name" . }} + firewallConfig: + external: + inboundAllowCIDR: [] + inboundBlockedCIDR: [] + outboundAllowCIDR: [] + outboundAllowHostname: [] + outboundAllowPort: [] + outboundBlockedCIDR: [] + internal: + inboundAllowType: same-gvc + inboundAllowWorkload: [] + loadBalancer: + direct: + enabled: false + ports: [] + replicaDirect: false + + supportDynamicTags: false diff --git a/airflow/versions/1.4.0/templates/workload-webserver.yaml b/airflow/versions/1.4.0/templates/workload-webserver.yaml new file mode 100644 index 00000000..4467855e --- /dev/null +++ b/airflow/versions/1.4.0/templates/workload-webserver.yaml @@ -0,0 +1,151 @@ +kind: workload +name: {{ include "airflow.webserver.name" . }} +description: Airflow webserver +gvc: {{ .Values.gvc.name }} +tags: + {{- include "airflow.tags" . | nindent 2 }} +spec: + type: stateful + identityLink: //identity/{{ include "airflow.identity.name" . }} + containers: + - name: airflow + command: bash + args: + - '-c' + - |- + mkdir -p /opt/airflow/dags; + python3 -c "import json,os; print(json.dumps({os.environ['AIRFLOW_ADMIN_USERNAME']:os.environ['AIRFLOW_ADMIN_PASSWORD']}))" > /opt/airflow/simple_auth_manager_passwords.json.generated; + airflow db migrate; + airflow dag-processor & + airflow scheduler & + airflow triggerer & + exec airflow api-server + cpu: {{ .Values.airflow.webserver.resources.cpu | quote }} + memory: {{ .Values.airflow.webserver.resources.memory | quote }} + image: {{ .Values.airflow.webserver.image }} + inheritEnv: false + lifecycle: + preStop: + exec: + command: + - bash + - '-c' + - sleep 0 + env: + - name: AIRFLOW_HOME + value: /opt/airflow + - name: AIRFLOW__API_AUTH__JWT_EXPIRATION_DELTA + value: {{ .Values.airflow.auth.jwtExpirationDelta | quote }} + - name: AIRFLOW__API_AUTH__JWT_REFRESH_THRESHOLD + value: {{ .Values.airflow.auth.jwtRefreshThreshold | quote }} + - name: POSTGRES_USERNAME + value: cpln://secret/{{ include "airflow.secret.name" . }}.username + - name: POSTGRES_PASSWORD + value: cpln://secret/{{ include "airflow.secret.name" . }}.password + - name: AIRFLOW__API_AUTH__JWT_SECRET + value: cpln://secret/{{ include "airflow.secret.name" . }}.jwtSecret + - name: AIRFLOW__CORE__FERNET_KEY + value: cpln://secret/{{ include "airflow.secret.name" . }}.fernetKey + - name: AIRFLOW__API__BASE_URL + value: http://{{ include "airflow.webserver.name" . }}:{{ .Values.airflow.webPort }} + - name: AIRFLOW__CELERY__BROKER_URL + value: redis://{{ include "airflow.redis.name" . }}.{{ .Values.gvc.name }}.cpln.local + - name: AIRFLOW__CELERY__RESULT_BACKEND + value: db+postgresql://$(POSTGRES_USERNAME):$(POSTGRES_PASSWORD)@{{ include "airflow.postgres.name" . }}:5432/{{ .Values.postgres.config.database }} + {{- if .Values.gitSync.enabled }} + - name: AIRFLOW__CORE__DAGS_FOLDER + value: /opt/airflow/dags-sync/current{{ if .Values.gitSync.subPath }}/{{ .Values.gitSync.subPath }}{{ end }} + {{- else }} + - name: AIRFLOW__CORE__DAGS_FOLDER + value: /opt/airflow/dags + {{- end }} + - name: AIRFLOW__CORE__EXECUTOR + value: CeleryExecutor + - name: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN + value: postgresql+psycopg2://$(POSTGRES_USERNAME):$(POSTGRES_PASSWORD)@{{ include "airflow.postgres.name" . }}:5432/{{ .Values.postgres.config.database }} + - name: AIRFLOW__LOGGING__REMOTE_LOGGING + value: 'False' + - name: AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL + value: "{{ .Values.airflow.scheduler.dagDirListInterval }}" + - name: AIRFLOW__SCHEDULER__MIN_FILE_PROCESS_INTERVAL + value: "{{ .Values.airflow.scheduler.minFileProcessInterval }}" + - name: AIRFLOW_ADMIN_USERNAME + value: {{ .Values.airflow.admin.username | quote }} + - name: AIRFLOW_ADMIN_PASSWORD + value: cpln://secret/{{ include "airflow.secret.name" . }}.adminPassword + - name: AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS + value: {{ printf "%s:ADMIN" .Values.airflow.admin.username | quote }} + - name: AIRFLOW__API__HOST + value: 0.0.0.0 + - name: AIRFLOW__API__PORT + value: "{{ .Values.airflow.webPort }}" + ports: + - number: {{ .Values.airflow.webPort }} + protocol: http + readinessProbe: + httpGet: + path: /api/v2/monitor/health + port: {{ .Values.airflow.webPort }} + failureThreshold: 5 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + volumes: + - path: /opt/airflow + recoveryPolicy: retain + uri: cpln://volumeset/{{ include "airflow.volume.name" . }} + {{- if .Values.gitSync.enabled }} + - name: git-sync + image: registry.k8s.io/git-sync/git-sync:v4.4.0 + cpu: 100m + memory: 128Mi + inheritEnv: false + env: + - name: GITSYNC_REPO + value: {{ .Values.gitSync.repo | quote }} + - name: GITSYNC_BRANCH + value: {{ .Values.gitSync.branch | quote }} + - name: GITSYNC_ROOT + value: /opt/airflow/dags-sync + - name: GITSYNC_LINK + value: current + - name: GITSYNC_PERIOD + value: {{ .Values.gitSync.period | quote }} + {{- if .Values.gitSync.auth.token }} + - name: GITSYNC_USERNAME + value: git + - name: GITSYNC_PASSWORD + value: cpln://secret/{{ include "airflow.secret.name" . }}.gitSyncToken + {{- end }} + volumes: + - path: /opt/airflow + recoveryPolicy: retain + uri: cpln://volumeset/{{ include "airflow.volume.name" . }} + {{- end }} + firewallConfig: + external: + inboundAllowCIDR: + {{- toYaml .Values.firewallConfig.inboundAllowCIDR | nindent 8 }} + inboundBlockedCIDR: [] + outboundAllowCIDR: + - 0.0.0.0/0 + outboundAllowHostname: [] + outboundAllowPort: [] + outboundBlockedCIDR: [] + internal: + inboundAllowType: same-gvc + inboundAllowWorkload: [] + loadBalancer: + direct: + enabled: false + ports: [] + replicaDirect: false + rolloutOptions: + maxSurgeReplicas: 25% + minReadySeconds: 0 + scalingPolicy: OrderedReady + terminationGracePeriodSeconds: 1 + securityOptions: + filesystemGroupId: 50000 + supportDynamicTags: false \ No newline at end of file diff --git a/airflow/versions/1.4.0/values.yaml b/airflow/versions/1.4.0/values.yaml new file mode 100644 index 00000000..176d9f8a --- /dev/null +++ b/airflow/versions/1.4.0/values.yaml @@ -0,0 +1,90 @@ +# Global Virtual Cloud (GVC) settings +gvc: + name: airflow + locations: + - name: aws-eu-central-1 + +# Postgres database configuration +postgres: + image: postgres:18 + resources: + minCpu: 250m + maxCpu: 500m + minMemory: 512Mi + maxMemory: 1024Mi + config: + username: username + password: password + database: airflow + volumeset: + capacity: 10 # initial capacity in GiB (minimum is 10) + +# Redis cache configuration +redis: + image: redis:7.4 + resources: + cpu: 250m + memory: 512Mi + volumeset: + capacity: 10 # initial capacity in GiB (minimum is 10) + +# Apache Airflow configuration +airflow: + webserver: + image: apache/airflow:3.0.3 + resources: + cpu: 2000m + memory: 3Gi + celeryWorker: + image: controlplanecorporation/celery:v1 + resources: + cpu: 256m + memory: 512Mi + webPort: 8080 # Port for accessing the Airflow web interface + + auth: + jwtSecret: CHANGE_ME # REQUIRED: generate with "openssl rand -base64 48" + jwtExpirationDelta: 3600 # JWT token expiration time in seconds + jwtRefreshThreshold: 300 # Threshold before token expires to allow refresh in seconds + fernetKey: CHANGE_ME # REQUIRED: generate with "python3 -c 'from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())'" + + admin: + username: admin + password: CHANGE_ME # REQUIRED: change before deploying to production + + scheduler: + dagDirListInterval: 10 # How often to check DAG folder (seconds) + minFileProcessInterval: 10 # Minimum interval to process DAG files (seconds) + + celery: + workerConcurrency: 1 # Number of tasks each worker can run concurrently when KEDA is enabled. + +volumeset: + airflow: + capacity: 10 # initial capacity in GiB (minimum is 10) + +# Firewall configuration +firewallConfig: + inboundAllowCIDR: + - 0.0.0.0/0 # Restrict to specific IPs in production (e.g. - 203.0.113.0/24) + +# Git-sync configuration for DAG delivery +gitSync: + enabled: false + repo: "" # Git repository URL (e.g. https://github.com/org/dags) + branch: main # Branch to sync + period: 60s # How often to sync + subPath: "" # Optional subfolder within the repo containing DAGs + auth: + token: "" # Personal access token for private repos (leave empty for public repos) + +# KEDA (Kubernetes Event-driven Autoscaling) configuration +keda: + enabled: true # Enable or disable KEDA autoscaling + minScale: 1 # Minimum number of Celery workers + maxScale: 3 # Maximum number of Celery workers + scaleToZeroDelay: 300 # Time before scaling to zero (seconds) + listLength: 3 # Queue length threshold to trigger scaling + cooldownPeriod: 1 # Cooldown between scaling events (seconds) + initialCooldownPeriod: 1 # Cooldown after startup before scaling (seconds) + pollingInterval: 4 # Interval at which KEDA queries metrics for scaling(seconds) \ No newline at end of file From 84d2e38ce11c91570d4cfb9e59ced94035b3073e Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Tue, 14 Apr 2026 14:39:48 -0700 Subject: [PATCH 2/3] added cpln-common tagging --- airflow/versions/1.4.0/Chart.yaml | 9 +++++-- airflow/versions/1.4.0/templates/_helpers.tpl | 27 ++----------------- 2 files changed, 9 insertions(+), 27 deletions(-) diff --git a/airflow/versions/1.4.0/Chart.yaml b/airflow/versions/1.4.0/Chart.yaml index 5d12cf81..7e82b6c9 100644 --- a/airflow/versions/1.4.0/Chart.yaml +++ b/airflow/versions/1.4.0/Chart.yaml @@ -2,11 +2,16 @@ apiVersion: v2 name: airflow description: Airflow with Celery using Keda, Redis and Postgres type: application -version: 1.3.0 +version: 1.4.0 appVersion: "3.0.3" annotations: created: "2025-08-14" lastModified: "2026-03-10" category: "app" - createsGvc: true \ No newline at end of file + createsGvc: true + +dependencies: + - name: cpln-common + version: 1.0.0 + repository: "oci://ghcr.io/controlplane-com/templates" \ No newline at end of file diff --git a/airflow/versions/1.4.0/templates/_helpers.tpl b/airflow/versions/1.4.0/templates/_helpers.tpl index 69d38fee..8f3c8bf9 100644 --- a/airflow/versions/1.4.0/templates/_helpers.tpl +++ b/airflow/versions/1.4.0/templates/_helpers.tpl @@ -74,31 +74,8 @@ Airflow Volume Set Name {{/* Labeling */}} {{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "airflow.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels +Common labels - delegated to cpln-common */}} {{- define "airflow.tags" -}} -helm.sh/chart: {{ include "airflow.chart" . }} -{{ include "airflow.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.cpln.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.cpln.io/managed-by: {{ .Release.Service }} -cpln/marketplace: "true" -cpln/marketplace-template: airflow -cpln/marketplace-template-version: {{ .Chart.Version }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "airflow.selectorLabels" -}} -app.cpln.io/name: {{ .Release.Name }} -app.cpln.io/instance: {{ .Release.Name }} +{{- include "cpln-common.tags" . }} {{- end }} From 075cc945a3e94dc535055bcc85a8610298eecf41 Mon Sep 17 00:00:00 2001 From: Jacob <163480591+jacobecox@users.noreply.github.com> Date: Tue, 14 Apr 2026 14:48:30 -0700 Subject: [PATCH 3/3] switched gvc tagging to be conditional if createGvc is true (#235) --- cpln-common/versions/1.0.0/templates/_helpers.tpl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpln-common/versions/1.0.0/templates/_helpers.tpl b/cpln-common/versions/1.0.0/templates/_helpers.tpl index ec97c01e..8f9f6a18 100644 --- a/cpln-common/versions/1.0.0/templates/_helpers.tpl +++ b/cpln-common/versions/1.0.0/templates/_helpers.tpl @@ -28,5 +28,7 @@ app.cpln.io/managed-by: {{ .Release.Service }} cpln/marketplace: "true" cpln/marketplace-template: {{ .Chart.Name }} cpln/marketplace-template-version: {{ .Chart.Version }} +{{- if .Values.global.cpln.gvc }} cpln/marketplace-gvc: {{ .Values.global.cpln.gvc }} {{- end }} +{{- end }}