diff --git a/.github/kubeval.sh b/.github/kubeval.sh index 49807c59..0c24056f 100755 --- a/.github/kubeval.sh +++ b/.github/kubeval.sh @@ -1,7 +1,7 @@ #!/bin/bash set -euo pipefail -CHART_DIRS="$(git diff --find-renames --name-only "$(git rev-parse --abbrev-ref HEAD)" remotes/origin/main -- helm | grep '[cC]hart.yaml' | sed -e 's#/[Cc]hart.yaml##g')" +CHART_DIRS="$(git diff --find-renames --name-only "$(git rev-parse --abbrev-ref HEAD)" remotes/origin/main -- helm | grep '[cC]hart.yaml' | grep -v 'helm/airbyte-v1' | sed -e 's#/[Cc]hart.yaml##g')" KUBEVAL_VERSION="0.14.0" SCHEMA_LOCATION="https://raw.githubusercontent.com/instrumenta/kubernetes-json-schema/master/" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 282c54a2..90e41a64 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2,8 +2,8 @@ name: Lint and Test Charts on: pull_request: - paths: - - 'helm/**' + paths-ignore: + - 'helm/airbyte-v1/**' jobs: lint-chart: @@ -56,4 +56,4 @@ jobs: uses: helm/chart-testing-action@main with: command: install - config: .github/ct.yaml \ No newline at end of file + config: .github/ct.yaml diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index db38651d..2ca3e01e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -4,8 +4,10 @@ on: branches: - main - eks_upgrade - paths: - - 'helm/**' + - airbyte_v1 + - airbyte_v2 + # paths-ignore: + # - 'helm/airbyte-v1/**' release: types: [published, created] @@ -20,3 +22,5 @@ jobs: with: token: ${{ secrets.BO_CHARTS_HELM_TOKEN }} charts_dir: helm + # Due to airbyte-v1 version incompatility with the current config of gh actions it always fails on linting so we switched it off + linting: off diff --git a/helm/airbyte-v1/.helmignore b/helm/airbyte-v1/.helmignore new file mode 100644 index 00000000..2b1b66e6 --- /dev/null +++ b/helm/airbyte-v1/.helmignore @@ -0,0 +1,26 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ + +ci.sh +.cache/ diff --git a/helm/airbyte-v1/Chart.lock b/helm/airbyte-v1/Chart.lock new file mode 100644 index 00000000..c0718a1b --- /dev/null +++ b/helm/airbyte-v1/Chart.lock @@ -0,0 +1,51 @@ +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + version: 1.17.1 +- name: airbyte-bootloader + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- name: temporal + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- name: temporal-ui + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- name: webapp + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- name: server + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- name: worker + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- name: workload-api-server + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- name: workload-launcher + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- name: pod-sweeper + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- name: metrics + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- name: cron + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- name: connector-builder-server + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- name: connector-rollout-worker + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- name: keycloak + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- name: keycloak-setup + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +digest: sha256:f0a8ad517d58f33d56b16654e9b868a981e7dbac7af214473a8f486dc5a2e563 +generated: "2024-12-09T21:59:21.985259637Z" diff --git a/helm/airbyte-v1/Chart.yaml b/helm/airbyte-v1/Chart.yaml new file mode 100644 index 00000000..7c475372 --- /dev/null +++ b/helm/airbyte-v1/Chart.yaml @@ -0,0 +1,72 @@ +apiVersion: v2 +appVersion: 1.3.0 +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x +- condition: airbyte-bootloader.enabled + name: airbyte-bootloader + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- condition: temporal.enabled + name: temporal + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- condition: temporal-ui.enabled + name: temporal-ui + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- condition: webapp.enabled + name: webapp + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- condition: server.enabled + name: server + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- condition: worker.enabled + name: worker + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- condition: workload-api-server.enabled + name: workload-api-server + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- condition: workload-launcher.enabled + name: workload-launcher + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- condition: pod-sweeper.enabled + name: pod-sweeper + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- condition: metrics.enabled + name: metrics + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- condition: cron.enabled + name: cron + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- condition: connector-builder-server.enabled + name: connector-builder-server + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- condition: connector-rollout-worker.enabled + name: connector-rollout-worker + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- condition: keycloak.enabled + name: keycloak + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +- condition: keycloak-setup.enabled + name: keycloak-setup + repository: https://airbytehq.github.io/helm-charts/ + version: 1.3.0 +description: Helm chart to deploy airbyte +name: airbyte-v1 +type: application +version: "v1.3.12" diff --git a/helm/airbyte-v1/charts/airbyte-bootloader/.gitignore b/helm/airbyte-v1/charts/airbyte-bootloader/.gitignore new file mode 100644 index 00000000..88e91e8a --- /dev/null +++ b/helm/airbyte-v1/charts/airbyte-bootloader/.gitignore @@ -0,0 +1,2 @@ +# Charts are downloaded at install time with `helm dep build`. +charts diff --git a/helm/airbyte-v1/charts/airbyte-bootloader/Chart.lock b/helm/airbyte-v1/charts/airbyte-bootloader/Chart.lock new file mode 100644 index 00000000..fb39f2dc --- /dev/null +++ b/helm/airbyte-v1/charts/airbyte-bootloader/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + version: 1.17.1 +digest: sha256:dacc73770a5640c011e067ff8840ddf89631fc19016c8d0a9e5ea160e7da8690 +generated: "2022-10-17T18:35:11.238826963Z" diff --git a/helm/airbyte-v1/charts/airbyte-bootloader/Chart.yaml b/helm/airbyte-v1/charts/airbyte-bootloader/Chart.yaml new file mode 100644 index 00000000..1c903e66 --- /dev/null +++ b/helm/airbyte-v1/charts/airbyte-bootloader/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +appVersion: 1.3.0 +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x +description: Helm chart to deploy airbyte-bootloader +name: airbyte-bootloader +type: application +version: 1.3.0 diff --git a/helm/airbyte-v1/charts/airbyte-bootloader/README.md b/helm/airbyte-v1/charts/airbyte-bootloader/README.md new file mode 100644 index 00000000..ceaa5352 --- /dev/null +++ b/helm/airbyte-v1/charts/airbyte-bootloader/README.md @@ -0,0 +1,47 @@ +# airbyte-bootloader + +![Version: 0.67.17](https://img.shields.io/badge/Version-0.67.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) + +Helm chart to deploy airbyte-bootloader + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://charts.bitnami.com/bitnami | common | 1.x.x | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| containerSecurityContext | object | `{}` | | +| enabled | bool | `true` | | +| env_vars | object | `{}` | | +| extraContainers | list | `[]` | | +| extraEnv | list | `[]` | | +| extraInitContainers | list | `[]` | | +| extraLabels | object | `{}` | | +| extraSelectorLabels | object | `{}` | | +| extraVolumeMounts | list | `[]` | | +| extraVolumes | list | `[]` | | +| global.database.secretName | string | `""` | | +| global.database.secretValue | string | `""` | | +| global.deploymentMode | string | `"oss"` | | +| global.env_vars | object | `{}` | | +| global.extraContainers | list | `[]` | | +| global.extraLabels | object | `{}` | | +| global.extraSelectorLabels | object | `{}` | | +| global.secretName | string | `""` | | +| global.secrets | object | `{}` | | +| global.serviceAccountName | string | `"placeholderServiceAccount"` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"airbyte/bootloader"` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| podLabels | object | `{}` | | +| resources.limits | object | `{}` | | +| resources.requests | object | `{}` | | +| secrets | object | `{}` | | +| tolerations | list | `[]` | | + diff --git a/helm/airbyte-v1/charts/airbyte-bootloader/templates/_helpers.tpl b/helm/airbyte-v1/charts/airbyte-bootloader/templates/_helpers.tpl new file mode 100644 index 00000000..2b326d75 --- /dev/null +++ b/helm/airbyte-v1/charts/airbyte-bootloader/templates/_helpers.tpl @@ -0,0 +1,51 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "airbyte.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airbyte.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airbyte.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "airbyte.labels" -}} +helm.sh/chart: {{ include "airbyte.chart" . }} +{{ include "airbyte.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "airbyte.selectorLabels" -}} +app.kubernetes.io/name: {{ include "airbyte.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/helm/airbyte-v1/charts/airbyte-bootloader/templates/_images.tpl b/helm/airbyte-v1/charts/airbyte-bootloader/templates/_images.tpl new file mode 100644 index 00000000..6c32ec44 --- /dev/null +++ b/helm/airbyte-v1/charts/airbyte-bootloader/templates/_images.tpl @@ -0,0 +1,17 @@ +{{- define "imageUrl" -}} +{{- $img := index . 0 -}} +{{- $root := index . 1 -}} + +{{- $reg := $root.Values.global.image.registry -}} +{{/* ensure the registry has a trailing slash, if set */}} +{{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} + +{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{- if (eq (typeOf $img) "string") -}} +{{- printf "%s%s" $reg (tpl $img $root) | quote -}} +{{- else -}} +{{- $tag := coalesce $img.tag $root.Values.global.image.tag $root.Chart.AppVersion -}} +{{- printf "%s%s:%s" $reg $img.repository $tag | quote -}} +{{- end -}} + +{{- end -}} \ No newline at end of file diff --git a/helm/airbyte-v1/charts/airbyte-bootloader/templates/bootloader-secrets.yaml b/helm/airbyte-v1/charts/airbyte-bootloader/templates/bootloader-secrets.yaml new file mode 100644 index 00000000..202def0e --- /dev/null +++ b/helm/airbyte-v1/charts/airbyte-bootloader/templates/bootloader-secrets.yaml @@ -0,0 +1,18 @@ +# Create secrets only for the local deployment +{{- if .Values.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: bootloader-secrets + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" + +type: Opaque +data: + {{- range $k, $v := mergeOverwrite .Values.secrets .Values.global.secrets }} + {{ $k }}: {{ if $v }}{{ $v | b64enc }} {{else}}""{{end}} + {{- end }} + {{- end }} diff --git a/helm/airbyte-v1/charts/airbyte-bootloader/templates/pod.yaml b/helm/airbyte-v1/charts/airbyte-bootloader/templates/pod.yaml new file mode 100644 index 00000000..8a72fa90 --- /dev/null +++ b/helm/airbyte-v1/charts/airbyte-bootloader/templates/pod.yaml @@ -0,0 +1,130 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ include "common.names.fullname" . }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- if .Values.extraLabels }} + {{ toYaml (mergeOverwrite .Values.extraLabels .Values.global.extraLabels) | nindent 4 }} + {{- end }} + {{- if .Values.podLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 4 }} + {{- end }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "0" + {{- if .Values.podAnnotations }} + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 4 }} + {{- end }} +spec: + serviceAccountName: {{ .Values.global.serviceAccountName }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 2 }} + {{- end }} + {{- end }} + restartPolicy: Never + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.extraInitContainers }} + initContainers: + {{- toYaml .Values.extraInitContainers | nindent 4 }} + {{- end }} + containers: + - name: airbyte-bootloader-container + image: {{ include "imageUrl" (list .Values.image $) }} + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + env: + {{- if eq .Values.global.deploymentMode "oss" }} + - name: AIRBYTE_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Values.global.configMapName | default (printf "%s-airbyte-env" .Release.Name) }} + key: AIRBYTE_VERSION + - name: RUN_DATABASE_MIGRATION_ON_STARTUP + valueFrom: + configMapKeyRef: + name: {{ .Values.global.configMapName | default (printf "%s-airbyte-env" .Release.Name) }} + key: RUN_DATABASE_MIGRATION_ON_STARTUP + + # Airbyte auth secret keys and values + {{- if and (eq .Values.global.edition "community") .Values.global.auth.enabled }} + - name: AB_AUTH_SECRET_CREATION_ENABLED + value: "true" + - name: AB_KUBERNETES_SECRET_NAME + value: {{ .Values.global.auth.secretName | default "airbyte-auth-secrets" | quote }} + - name: AB_INSTANCE_ADMIN_PASSWORD_SECRET_KEY + value: {{ .Values.global.auth.instanceAdmin.passwordSecretKey | default "instance-admin-password" | quote }} + - name: AB_INSTANCE_ADMIN_CLIENT_ID_SECRET_KEY + value: {{ .Values.global.auth.instanceAdmin.clientIdSecretKey | default "instance-admin-client-id" | quote }} + - name: AB_INSTANCE_ADMIN_CLIENT_SECRET_SECRET_KEY + value: {{ .Values.global.auth.instanceAdmin.clientSecretSecretKey | default "instance-admin-client-secret" | quote }} + - name: AB_JWT_SIGNATURE_SECRET_KEY + value: {{ .Values.global.auth.jwtSignatureSecretKey | default "jwt-signature-secret" | quote }} + - name: AB_INSTANCE_ADMIN_PASSWORD + value: {{ .Values.global.auth.instanceAdmin.password | quote }} + - name: AB_INSTANCE_ADMIN_CLIENT_ID + value: {{ .Values.global.auth.instanceAdmin.clientId | quote }} + - name: AB_INSTANCE_ADMIN_CLIENT_SECRET + value: {{ .Values.global.auth.instanceAdmin.clientSecret | quote }} + - name: AB_JWT_SIGNATURE_SECRET + value: {{ .Values.global.auth.jwtSignatureSecret | quote }} + {{- end }} + + {{- include "airbyte.database.envs" . | nindent 8 }} + + {{- end }} + # Values from secret + {{- if .Values.secrets }} + {{- range $k, $v := .Values.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: bootloader-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if or .Values.env_vars .Values.global.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + # Values from extraEnv for more compability(if you want to use external secret source or other stuff) + {{- if .Values.extraEnv }} + {{- toYaml .Values.extraEnv | nindent 8 }} + {{- end }} + + {{- if .Values.resources }} + resources: {{- toYaml .Values.resources | nindent 8 }} + {{- end }} + {{- if .Values.extraVolumeMounts }} + volumeMounts: + {{- toYaml .Values.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.containerSecurityContext }} + securityContext: {{- toYaml .Values.containerSecurityContext | nindent 8 }} + {{- end }} + {{- if .Values.extraContainers }} + {{ toYaml .Values.extraContainers | nindent 4 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 4 }} + {{- end }} + securityContext: {{- toYaml .Values.podSecurityContext | nindent 6 }} + {{- if .Values.extraVolumes }} + volumes: + {{- toYaml .Values.extraVolumes | nindent 4 }} + {{- end }} diff --git a/helm/airbyte-v1/charts/airbyte-bootloader/values.yaml b/helm/airbyte-v1/charts/airbyte-bootloader/values.yaml new file mode 100644 index 00000000..ee3bc615 --- /dev/null +++ b/helm/airbyte-v1/charts/airbyte-bootloader/values.yaml @@ -0,0 +1,189 @@ +## global.serviceAccountName Name of service account to be associated with service +## deploymentMode Determines deployment mode of airbyte +## secretName Overrides the secret name with credentials to S3 Bucket and logging +## database.secretName Name of database secret +## database.secretValue Value of database password key stored in secret +global: + serviceAccountName: placeholderServiceAccount + deploymentMode: oss + secretName: "" + database: + secretName: "" + secretValue: "" + secrets: {} + env_vars: {} + extraContainers: [] + ## extraSelectorLabels [object] - use to specify own additional selector labels for deployment + extraSelectorLabels: {} + ## extraLabels [object] - use to specify own additional labels for deployment + extraLabels: {} + +# Docker image config that will apply to all images. + image: + # Docker registry to pull platform images from, e.g. http://my-registry:8000/ + registry: "" + # Image tag to use for airbyte images. + # Does not include non-airbyte images such as busybox, temporal, minio, etc. + tag: "" + + +enabled: true +## image.repository The repository to use for the airbyte bootloader image. +## image.pullPolicy the pull policy to use for the airbyte bootloader image +## image.tag The airbyte bootloader image tag. Defaults to the chart's AppVersion +image: + repository: airbyte/bootloader + pullPolicy: IfNotPresent + +## podAnnotations [object] Add extra annotations to the bootloader pod +## +podAnnotations: {} + +## podLabels [object] Add extra labels to the bootloader pod +## +podLabels: {} + +## containerSecurityContext Security context for the container +## Examples: +## containerSecurityContext: +## runAsNonRoot: true +## runAsUser: 1000 +## readOnlyRootFilesystem: true +containerSecurityContext: {} + +## nodeSelector [object] Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## tolerations [array] Tolerations for worker pod assignment. +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Bootloader resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## We usually recommend not to specify default resources and to leave this as a conscious +## choice for the user. This also increases chances charts run on environments with little +## resources, such as Minikube. If you do want to specify resources, uncomment the following +## lines, adjust them as necessary, and remove the curly braces after 'resources:'. +## bootloader.resources.limits [object] The resources limits for the airbyte bootloader image +## bootloader.resources.requests [object] The requested resources for the airbyte bootloader image +resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + +## affinity [object] Affinity and anti-affinity for bootloader pod assignment. +## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## extraEnv [array] Supply extra env variables to main container using full notation +## Example: (With default env vars and values taken from generated config map) +## extraEnv: +## - name: AIRBYTE_VERSION +## valueFrom: +## configMapKeyRef: +## name: airbyte-env +## key: AIRBYTE_VERSION +## - name: DATABASE_HOST +## valueFrom: +## configMapKeyRef: +## name: airbyte-env +## key: DATABASE_HOST +## - name: DATABASE_PORT +## valueFrom: +## configMapKeyRef: +## name: airbyte-env +## key: DATABASE_PORT +## - name: DATABASE_PASSWORD +## valueFrom: +## secretKeyRef: +## name: airbyte-secrets +## key: DATABASE_PASSWORD +## - name: DATABASE_URL +## valueFrom: +## configMapKeyRef: +## name: airbyte-env +## key: DATABASE_URL +## - name: DATABASE_USER +## valueFrom: +## secretKeyRef: +## name: airbyte-secrets +## key: DATABASE_USER +## +extraEnv: [] + +## secrets [object] Supply additional secrets to container +## Example: +## secrets: +## DATABASE_PASSWORD: strong-password +## DATABASE_USER: my-db-user +secrets: {} + +## extraContainer [array] Additional container for server pod(s) +## Example: +# extraContainers: +# - name: otel_collector +# image: somerepo/someimage:sometag +# args: [ +# "--important-args" +# ] +# ports: +# - containerPort: 443 +# volumeMounts: +# - name: volumeMountCool +# mountPath: /some/path +# readOnly: true +extraContainers: [] + +## extraInitContainers [array] Additional init containers for server pod(s) +## Example: +# extraInitContainers: +# - name: sleepy +# image: alpine +# command: ['sleep', '60'] + +extraInitContainers: [] + +## extraVolumeMounts [array] Additional volumeMounts for server container(s). +## Examples (when using `containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumeMounts: +## - name: tmpdir +## mountPath: /tmp +## +extraVolumeMounts: [] + +## extraVolumes [array] Additional volumes for server pod(s). +## Examples (when using `containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumes: +## - name: tmpdir +## emptyDir: {} +## +extraVolumes: [] + +## env_vars [object] Supply extra env variables to main container using simplified notation +## Example: +# env_vars: +# AIRBYTE_VERSION: 0.40.4 + +# # Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db +# DATABASE_HOST: airbyte-db-svc +# DATABASE_PORT: 5432 +# # translate manually DATABASE_URL: jdbc:postgresql://${DATABASE_HOST}:${DATABASE_PORT/${DATABASE_DB} +# DATABASE_URL: jdbc:postgresql://airbyte-db-svc:5432/airbyte +# DATABASE_PASSWORD: 12345 +# DATABASE_USER: airbyte +env_vars: {} + +## extraSelectorLabels [object] - use to specify own additional selector labels for deployment +extraSelectorLabels: {} +## extraLabels [object] - use to specify own additional labels for deployment +extraLabels: {} diff --git a/helm/airbyte-v1/charts/common/.helmignore b/helm/airbyte-v1/charts/common/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/helm/airbyte-v1/charts/common/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/airbyte-v1/charts/common/Chart.yaml b/helm/airbyte-v1/charts/common/Chart.yaml new file mode 100644 index 00000000..39edbb3c --- /dev/null +++ b/helm/airbyte-v1/charts/common/Chart.yaml @@ -0,0 +1,23 @@ +annotations: + category: Infrastructure +apiVersion: v2 +appVersion: 1.17.1 +description: A Library Helm Chart for grouping common logic between bitnami charts. + This chart is not deployable by itself. +home: https://github.com/bitnami/charts/tree/master/bitnami/common +icon: https://bitnami.com/downloads/logos/bitnami-mark.png +keywords: +- common +- helper +- template +- function +- bitnami +maintainers: +- name: Bitnami + url: https://github.com/bitnami/charts +name: common +sources: +- https://github.com/bitnami/charts +- https://www.bitnami.com/ +type: library +version: 1.17.1 diff --git a/helm/airbyte-v1/charts/common/README.md b/helm/airbyte-v1/charts/common/README.md new file mode 100644 index 00000000..a2ecd604 --- /dev/null +++ b/helm/airbyte-v1/charts/common/README.md @@ -0,0 +1,350 @@ +# Bitnami Common Library Chart + +A [Helm Library Chart](https://helm.sh/docs/topics/library_charts/#helm) for grouping common logic between bitnami charts. + +## TL;DR + +```yaml +dependencies: + - name: common + version: 1.x.x + repository: https://charts.bitnami.com/bitnami +``` + +```bash +$ helm dependency update +``` + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.names.fullname" . }} +data: + myvalue: "Hello World" +``` + +## Introduction + +This chart provides a common template helpers which can be used to develop new charts using [Helm](https://helm.sh) package manager. + +Bitnami charts can be used with [Kubeapps](https://kubeapps.dev/) for deployment and management of Helm Charts in clusters. + +## Prerequisites + +- Kubernetes 1.19+ +- Helm 3.2.0+ + +## Parameters + +The following table lists the helpers available in the library which are scoped in different sections. + +### Affinities + +| Helper identifier | Description | Expected Input | +|-------------------------------|------------------------------------------------------|------------------------------------------------| +| `common.affinities.nodes.soft` | Return a soft nodeAffinity definition | `dict "key" "FOO" "values" (list "BAR" "BAZ")` | +| `common.affinities.nodes.hard` | Return a hard nodeAffinity definition | `dict "key" "FOO" "values" (list "BAR" "BAZ")` | +| `common.affinities.pods.soft` | Return a soft podAffinity/podAntiAffinity definition | `dict "component" "FOO" "context" $` | +| `common.affinities.pods.hard` | Return a hard podAffinity/podAntiAffinity definition | `dict "component" "FOO" "context" $` | + +### Capabilities + +| Helper identifier | Description | Expected Input | +|------------------------------------------------|------------------------------------------------------------------------------------------------|-------------------| +| `common.capabilities.kubeVersion` | Return the target Kubernetes version (using client default if .Values.kubeVersion is not set). | `.` Chart context | +| `common.capabilities.cronjob.apiVersion` | Return the appropriate apiVersion for cronjob. | `.` Chart context | +| `common.capabilities.deployment.apiVersion` | Return the appropriate apiVersion for deployment. | `.` Chart context | +| `common.capabilities.statefulset.apiVersion` | Return the appropriate apiVersion for statefulset. | `.` Chart context | +| `common.capabilities.ingress.apiVersion` | Return the appropriate apiVersion for ingress. | `.` Chart context | +| `common.capabilities.rbac.apiVersion` | Return the appropriate apiVersion for RBAC resources. | `.` Chart context | +| `common.capabilities.crd.apiVersion` | Return the appropriate apiVersion for CRDs. | `.` Chart context | +| `common.capabilities.policy.apiVersion` | Return the appropriate apiVersion for podsecuritypolicy. | `.` Chart context | +| `common.capabilities.networkPolicy.apiVersion` | Return the appropriate apiVersion for networkpolicy. | `.` Chart context | +| `common.capabilities.apiService.apiVersion` | Return the appropriate apiVersion for APIService. | `.` Chart context | +| `common.capabilities.hpa.apiVersion` | Return the appropriate apiVersion for Horizontal Pod Autoscaler | `.` Chart context | +| `common.capabilities.supportsHelmVersion` | Returns true if the used Helm version is 3.3+ | `.` Chart context | + +### Errors + +| Helper identifier | Description | Expected Input | +|-----------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------| +| `common.errors.upgrade.passwords.empty` | It will ensure required passwords are given when we are upgrading a chart. If `validationErrors` is not empty it will throw an error and will stop the upgrade action. | `dict "validationErrors" (list $validationError00 $validationError01) "context" $` | + +### Images + +| Helper identifier | Description | Expected Input | +|-----------------------------|------------------------------------------------------|---------------------------------------------------------------------------------------------------------| +| `common.images.image` | Return the proper and full image name | `dict "imageRoot" .Values.path.to.the.image "global" $`, see [ImageRoot](#imageroot) for the structure. | +| `common.images.pullSecrets` | Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) | `dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global` | +| `common.images.renderPullSecrets` | Return the proper Docker Image Registry Secret Names (evaluates values as templates) | `dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "context" $` | + +### Ingress + +| Helper identifier | Description | Expected Input | +|-------------------------------------------|-------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.ingress.backend` | Generate a proper Ingress backend entry depending on the API version | `dict "serviceName" "foo" "servicePort" "bar"`, see the [Ingress deprecation notice](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/) for the syntax differences | +| `common.ingress.supportsPathType` | Prints "true" if the pathType field is supported | `.` Chart context | +| `common.ingress.supportsIngressClassname` | Prints "true" if the ingressClassname field is supported | `.` Chart context | +| `common.ingress.certManagerRequest` | Prints "true" if required cert-manager annotations for TLS signed certificates are set in the Ingress annotations | `dict "annotations" .Values.path.to.the.ingress.annotations` | + +### Labels + +| Helper identifier | Description | Expected Input | +|-----------------------------|-----------------------------------------------------------------------------|-------------------| +| `common.labels.standard` | Return Kubernetes standard labels | `.` Chart context | +| `common.labels.matchLabels` | Labels to use on `deploy.spec.selector.matchLabels` and `svc.spec.selector` | `.` Chart context | + +### Names + +| Helper identifier | Description | Expected Input | +|-----------------------------------|-----------------------------------------------------------------------|-------------------| +| `common.names.name` | Expand the name of the chart or use `.Values.nameOverride` | `.` Chart context | +| `common.names.fullname` | Create a default fully qualified app name. | `.` Chart context | +| `common.names.namespace` | Allow the release namespace to be overridden | `.` Chart context | +| `common.names.fullname.namespace` | Create a fully qualified app name adding the installation's namespace | `.` Chart context | +| `common.names.chart` | Chart name plus version | `.` Chart context | + +### Secrets + +| Helper identifier | Description | Expected Input | +|---------------------------|--------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.secrets.name` | Generate the name of the secret. | `dict "existingSecret" .Values.path.to.the.existingSecret "defaultNameSuffix" "mySuffix" "context" $` see [ExistingSecret](#existingsecret) for the structure. | +| `common.secrets.key` | Generate secret key. | `dict "existingSecret" .Values.path.to.the.existingSecret "key" "keyName"` see [ExistingSecret](#existingsecret) for the structure. | +| `common.passwords.manage` | Generate secret password or retrieve one if already created. | `dict "secret" "secret-name" "key" "keyName" "providedValues" (list "path.to.password1" "path.to.password2") "length" 10 "strong" false "chartName" "chartName" "context" $`, length, strong and chartNAme fields are optional. | +| `common.secrets.exists` | Returns whether a previous generated secret already exists. | `dict "secret" "secret-name" "context" $` | + +### Storage + +| Helper identifier | Description | Expected Input | +|-------------------------------|---------------------------------------|---------------------------------------------------------------------------------------------------------------------| +| `common.storage.class` | Return the proper Storage Class | `dict "persistence" .Values.path.to.the.persistence "global" $`, see [Persistence](#persistence) for the structure. | + +### TplValues + +| Helper identifier | Description | Expected Input | +|---------------------------|----------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.tplvalues.render` | Renders a value that contains template | `dict "value" .Values.path.to.the.Value "context" $`, value is the value should rendered as template, context frequently is the chart context `$` or `.` | + +### Utils + +| Helper identifier | Description | Expected Input | +|--------------------------------|------------------------------------------------------------------------------------------|------------------------------------------------------------------------| +| `common.utils.fieldToEnvVar` | Build environment variable name given a field. | `dict "field" "my-password"` | +| `common.utils.secret.getvalue` | Print instructions to get a secret value. | `dict "secret" "secret-name" "field" "secret-value-field" "context" $` | +| `common.utils.getValueFromKey` | Gets a value from `.Values` object given its key path | `dict "key" "path.to.key" "context" $` | +| `common.utils.getKeyFromList` | Returns first `.Values` key with a defined value or first of the list if all non-defined | `dict "keys" (list "path.to.key1" "path.to.key2") "context" $` | + +### Validations + +| Helper identifier | Description | Expected Input | +|--------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.validations.values.single.empty` | Validate a value must not be empty. | `dict "valueKey" "path.to.value" "secret" "secret.name" "field" "my-password" "subchart" "subchart" "context" $` secret, field and subchart are optional. In case they are given, the helper will generate a how to get instruction. See [ValidateValue](#validatevalue) | +| `common.validations.values.multiple.empty` | Validate a multiple values must not be empty. It returns a shared error for all the values. | `dict "required" (list $validateValueConf00 $validateValueConf01) "context" $`. See [ValidateValue](#validatevalue) | +| `common.validations.values.mariadb.passwords` | This helper will ensure required password for MariaDB are not empty. It returns a shared error for all the values. | `dict "secret" "mariadb-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use mariadb chart and the helper. | +| `common.validations.values.mysql.passwords` | This helper will ensure required password for MySQL are not empty. It returns a shared error for all the values. | `dict "secret" "mysql-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use mysql chart and the helper. | +| `common.validations.values.postgresql.passwords` | This helper will ensure required password for PostgreSQL are not empty. It returns a shared error for all the values. | `dict "secret" "postgresql-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use postgresql chart and the helper. | +| `common.validations.values.redis.passwords` | This helper will ensure required password for Redis® are not empty. It returns a shared error for all the values. | `dict "secret" "redis-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use redis chart and the helper. | +| `common.validations.values.cassandra.passwords` | This helper will ensure required password for Cassandra are not empty. It returns a shared error for all the values. | `dict "secret" "cassandra-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use cassandra chart and the helper. | +| `common.validations.values.mongodb.passwords` | This helper will ensure required password for MongoDB® are not empty. It returns a shared error for all the values. | `dict "secret" "mongodb-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use mongodb chart and the helper. | + +### Warnings + +| Helper identifier | Description | Expected Input | +|------------------------------|----------------------------------|------------------------------------------------------------| +| `common.warnings.rollingTag` | Warning about using rolling tag. | `ImageRoot` see [ImageRoot](#imageroot) for the structure. | + +## Special input schemas + +### ImageRoot + +```yaml +registry: + type: string + description: Docker registry where the image is located + example: docker.io + +repository: + type: string + description: Repository and image name + example: bitnami/nginx + +tag: + type: string + description: image tag + example: 1.16.1-debian-10-r63 + +pullPolicy: + type: string + description: Specify a imagePullPolicy. Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + +pullSecrets: + type: array + items: + type: string + description: Optionally specify an array of imagePullSecrets (evaluated as templates). + +debug: + type: boolean + description: Set to true if you would like to see extra information on logs + example: false + +## An instance would be: +# registry: docker.io +# repository: bitnami/nginx +# tag: 1.16.1-debian-10-r63 +# pullPolicy: IfNotPresent +# debug: false +``` + +### Persistence + +```yaml +enabled: + type: boolean + description: Whether enable persistence. + example: true + +storageClass: + type: string + description: Ghost data Persistent Volume Storage Class, If set to "-", storageClassName: "" which disables dynamic provisioning. + example: "-" + +accessMode: + type: string + description: Access mode for the Persistent Volume Storage. + example: ReadWriteOnce + +size: + type: string + description: Size the Persistent Volume Storage. + example: 8Gi + +path: + type: string + description: Path to be persisted. + example: /bitnami + +## An instance would be: +# enabled: true +# storageClass: "-" +# accessMode: ReadWriteOnce +# size: 8Gi +# path: /bitnami +``` + +### ExistingSecret + +```yaml +name: + type: string + description: Name of the existing secret. + example: mySecret +keyMapping: + description: Mapping between the expected key name and the name of the key in the existing secret. + type: object + +## An instance would be: +# name: mySecret +# keyMapping: +# password: myPasswordKey +``` + +#### Example of use + +When we store sensitive data for a deployment in a secret, some times we want to give to users the possibility of using theirs existing secrets. + +```yaml +# templates/secret.yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "common.names.fullname" . }} + labels: + app: {{ include "common.names.fullname" . }} +type: Opaque +data: + password: {{ .Values.password | b64enc | quote }} + +# templates/dpl.yaml +--- +... + env: + - name: PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "common.secrets.name" (dict "existingSecret" .Values.existingSecret "context" $) }} + key: {{ include "common.secrets.key" (dict "existingSecret" .Values.existingSecret "key" "password") }} +... + +# values.yaml +--- +name: mySecret +keyMapping: + password: myPasswordKey +``` + +### ValidateValue + +#### NOTES.txt + +```console +{{- $validateValueConf00 := (dict "valueKey" "path.to.value00" "secret" "secretName" "field" "password-00") -}} +{{- $validateValueConf01 := (dict "valueKey" "path.to.value01" "secret" "secretName" "field" "password-01") -}} + +{{ include "common.validations.values.multiple.empty" (dict "required" (list $validateValueConf00 $validateValueConf01) "context" $) }} +``` + +If we force those values to be empty we will see some alerts + +```console +$ helm install test mychart --set path.to.value00="",path.to.value01="" + 'path.to.value00' must not be empty, please add '--set path.to.value00=$PASSWORD_00' to the command. To get the current value: + + export PASSWORD_00=$(kubectl get secret --namespace default secretName -o jsonpath="{.data.password-00}" | base64 -d) + + 'path.to.value01' must not be empty, please add '--set path.to.value01=$PASSWORD_01' to the command. To get the current value: + + export PASSWORD_01=$(kubectl get secret --namespace default secretName -o jsonpath="{.data.password-01}" | base64 -d) +``` + +## Upgrading + +### To 1.0.0 + +[On November 13, 2020, Helm v2 support was formally finished](https://github.com/helm/charts#status-of-the-project), this major version is the result of the required changes applied to the Helm Chart to be able to incorporate the different features added in Helm v3 and to be consistent with the Helm project itself regarding the Helm v2 EOL. + +**What changes were introduced in this major version?** + +- Previous versions of this Helm Chart use `apiVersion: v1` (installable by both Helm 2 and 3), this Helm Chart was updated to `apiVersion: v2` (installable by Helm 3 only). [Here](https://helm.sh/docs/topics/charts/#the-apiversion-field) you can find more information about the `apiVersion` field. +- Use `type: library`. [Here](https://v3.helm.sh/docs/faq/#library-chart-support) you can find more information. +- The different fields present in the *Chart.yaml* file has been ordered alphabetically in a homogeneous way for all the Bitnami Helm Charts + +**Considerations when upgrading to this version** + +- If you want to upgrade to this version from a previous one installed with Helm v3, you shouldn't face any issues +- If you want to upgrade to this version using Helm v2, this scenario is not supported as this version doesn't support Helm v2 anymore +- If you installed the previous version with Helm v2 and wants to upgrade to this version with Helm v3, please refer to the [official Helm documentation](https://helm.sh/docs/topics/v2_v3_migration/#migration-use-cases) about migrating from Helm v2 to v3 + +**Useful links** + +- https://docs.bitnami.com/tutorials/resolve-helm2-helm3-post-migration-issues/ +- https://helm.sh/docs/topics/v2_v3_migration/ +- https://helm.sh/blog/migrate-from-helm-v2-to-helm-v3/ + +## License + +Copyright © 2022 Bitnami + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/helm/airbyte-v1/charts/common/templates/_affinities.tpl b/helm/airbyte-v1/charts/common/templates/_affinities.tpl new file mode 100644 index 00000000..2387be26 --- /dev/null +++ b/helm/airbyte-v1/charts/common/templates/_affinities.tpl @@ -0,0 +1,102 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Return a soft nodeAffinity definition +{{ include "common.affinities.nodes.soft" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.soft" -}} +preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . | quote }} + {{- end }} + weight: 1 +{{- end -}} + +{{/* +Return a hard nodeAffinity definition +{{ include "common.affinities.nodes.hard" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.hard" -}} +requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . | quote }} + {{- end }} +{{- end -}} + +{{/* +Return a nodeAffinity definition +{{ include "common.affinities.nodes" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.nodes.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.nodes.hard" . -}} + {{- end -}} +{{- end -}} + +{{/* +Return a soft podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.soft" (dict "component" "FOO" "extraMatchLabels" .Values.extraMatchLabels "context" $) -}} +*/}} +{{- define "common.affinities.pods.soft" -}} +{{- $component := default "" .component -}} +{{- $extraMatchLabels := default (dict) .extraMatchLabels -}} +preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" .context) | nindent 10 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := $extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + namespaces: + - {{ include "common.names.namespace" .context | quote }} + topologyKey: kubernetes.io/hostname + weight: 1 +{{- end -}} + +{{/* +Return a hard podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.hard" (dict "component" "FOO" "extraMatchLabels" .Values.extraMatchLabels "context" $) -}} +*/}} +{{- define "common.affinities.pods.hard" -}} +{{- $component := default "" .component -}} +{{- $extraMatchLabels := default (dict) .extraMatchLabels -}} +requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" .context) | nindent 8 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := $extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + namespaces: + - {{ include "common.names.namespace" .context | quote }} + topologyKey: kubernetes.io/hostname +{{- end -}} + +{{/* +Return a podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.pods" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.pods.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.pods.hard" . -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/common/templates/_capabilities.tpl b/helm/airbyte-v1/charts/common/templates/_capabilities.tpl new file mode 100644 index 00000000..9d9b7600 --- /dev/null +++ b/helm/airbyte-v1/charts/common/templates/_capabilities.tpl @@ -0,0 +1,154 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Return the target Kubernetes version +*/}} +{{- define "common.capabilities.kubeVersion" -}} +{{- if .Values.global }} + {{- if .Values.global.kubeVersion }} + {{- .Values.global.kubeVersion -}} + {{- else }} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} + {{- end -}} +{{- else }} +{{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for poddisruptionbudget. +*/}} +{{- define "common.capabilities.policy.apiVersion" -}} +{{- if semverCompare "<1.21-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "policy/v1beta1" -}} +{{- else -}} +{{- print "policy/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for networkpolicy. +*/}} +{{- define "common.capabilities.networkPolicy.apiVersion" -}} +{{- if semverCompare "<1.7-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for cronjob. +*/}} +{{- define "common.capabilities.cronjob.apiVersion" -}} +{{- if semverCompare "<1.21-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "batch/v1beta1" -}} +{{- else -}} +{{- print "batch/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for deployment. +*/}} +{{- define "common.capabilities.deployment.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for statefulset. +*/}} +{{- define "common.capabilities.statefulset.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apps/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "common.capabilities.ingress.apiVersion" -}} +{{- if .Values.ingress -}} +{{- if .Values.ingress.apiVersion -}} +{{- .Values.ingress.apiVersion -}} +{{- else if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end }} +{{- else if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for RBAC resources. +*/}} +{{- define "common.capabilities.rbac.apiVersion" -}} +{{- if semverCompare "<1.17-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "rbac.authorization.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "rbac.authorization.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for CRDs. +*/}} +{{- define "common.capabilities.crd.apiVersion" -}} +{{- if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiextensions.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiextensions.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for APIService. +*/}} +{{- define "common.capabilities.apiService.apiVersion" -}} +{{- if semverCompare "<1.10-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiregistration.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiregistration.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for Horizontal Pod Autoscaler. +*/}} +{{- define "common.capabilities.hpa.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .context) -}} +{{- if .beta2 -}} +{{- print "autoscaling/v2beta2" -}} +{{- else -}} +{{- print "autoscaling/v2beta1" -}} +{{- end -}} +{{- else -}} +{{- print "autoscaling/v2" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if the used Helm version is 3.3+. +A way to check the used Helm version was not introduced until version 3.3.0 with .Capabilities.HelmVersion, which contains an additional "{}}" structure. +This check is introduced as a regexMatch instead of {{ if .Capabilities.HelmVersion }} because checking for the key HelmVersion in <3.3 results in a "interface not found" error. +**To be removed when the catalog's minimun Helm version is 3.3** +*/}} +{{- define "common.capabilities.supportsHelmVersion" -}} +{{- if regexMatch "{(v[0-9])*[^}]*}}$" (.Capabilities | toString ) }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/common/templates/_errors.tpl b/helm/airbyte-v1/charts/common/templates/_errors.tpl new file mode 100644 index 00000000..a79cc2e3 --- /dev/null +++ b/helm/airbyte-v1/charts/common/templates/_errors.tpl @@ -0,0 +1,23 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Through error when upgrading using empty passwords values that must not be empty. + +Usage: +{{- $validationError00 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password00" "secret" "secretName" "field" "password-00") -}} +{{- $validationError01 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password01" "secret" "secretName" "field" "password-01") -}} +{{ include "common.errors.upgrade.passwords.empty" (dict "validationErrors" (list $validationError00 $validationError01) "context" $) }} + +Required password params: + - validationErrors - String - Required. List of validation strings to be return, if it is empty it won't throw error. + - context - Context - Required. Parent context. +*/}} +{{- define "common.errors.upgrade.passwords.empty" -}} + {{- $validationErrors := join "" .validationErrors -}} + {{- if and $validationErrors .context.Release.IsUpgrade -}} + {{- $errorString := "\nPASSWORDS ERROR: You must provide your current passwords when upgrading the release." -}} + {{- $errorString = print $errorString "\n Note that even after reinstallation, old credentials may be needed as they may be kept in persistent volume claims." -}} + {{- $errorString = print $errorString "\n Further information can be obtained at https://docs.bitnami.com/general/how-to/troubleshoot-helm-chart-issues/#credential-errors-while-upgrading-chart-releases" -}} + {{- $errorString = print $errorString "\n%s" -}} + {{- printf $errorString $validationErrors | fail -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/common/templates/_images.tpl b/helm/airbyte-v1/charts/common/templates/_images.tpl new file mode 100644 index 00000000..42ffbc72 --- /dev/null +++ b/helm/airbyte-v1/charts/common/templates/_images.tpl @@ -0,0 +1,75 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper image name +{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }} +*/}} +{{- define "common.images.image" -}} +{{- $registryName := .imageRoot.registry -}} +{{- $repositoryName := .imageRoot.repository -}} +{{- $tag := .imageRoot.tag | toString -}} +{{- if .global }} + {{- if .global.imageRegistry }} + {{- $registryName = .global.imageRegistry -}} + {{- end -}} +{{- end -}} +{{- if $registryName }} +{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- else -}} +{{- printf "%s:%s" $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) +{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} +*/}} +{{- define "common.images.pullSecrets" -}} + {{- $pullSecrets := list }} + + {{- if .global }} + {{- range .global.imagePullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names evaluating values as templates +{{ include "common.images.renderPullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "context" $) }} +*/}} +{{- define "common.images.renderPullSecrets" -}} + {{- $pullSecrets := list }} + {{- $context := .context }} + + {{- if $context.Values.global }} + {{- range $context.Values.global.imagePullSecrets -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" $context)) -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" $context)) -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/helm/airbyte-v1/charts/common/templates/_ingress.tpl b/helm/airbyte-v1/charts/common/templates/_ingress.tpl new file mode 100644 index 00000000..8caf73a6 --- /dev/null +++ b/helm/airbyte-v1/charts/common/templates/_ingress.tpl @@ -0,0 +1,68 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Generate backend entry that is compatible with all Kubernetes API versions. + +Usage: +{{ include "common.ingress.backend" (dict "serviceName" "backendName" "servicePort" "backendPort" "context" $) }} + +Params: + - serviceName - String. Name of an existing service backend + - servicePort - String/Int. Port name (or number) of the service. It will be translated to different yaml depending if it is a string or an integer. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.ingress.backend" -}} +{{- $apiVersion := (include "common.capabilities.ingress.apiVersion" .context) -}} +{{- if or (eq $apiVersion "extensions/v1beta1") (eq $apiVersion "networking.k8s.io/v1beta1") -}} +serviceName: {{ .serviceName }} +servicePort: {{ .servicePort }} +{{- else -}} +service: + name: {{ .serviceName }} + port: + {{- if typeIs "string" .servicePort }} + name: {{ .servicePort }} + {{- else if or (typeIs "int" .servicePort) (typeIs "float64" .servicePort) }} + number: {{ .servicePort | int }} + {{- end }} +{{- end -}} +{{- end -}} + +{{/* +Print "true" if the API pathType field is supported +Usage: +{{ include "common.ingress.supportsPathType" . }} +*/}} +{{- define "common.ingress.supportsPathType" -}} +{{- if (semverCompare "<1.18-0" (include "common.capabilities.kubeVersion" .)) -}} +{{- print "false" -}} +{{- else -}} +{{- print "true" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if the ingressClassname field is supported +Usage: +{{ include "common.ingress.supportsIngressClassname" . }} +*/}} +{{- define "common.ingress.supportsIngressClassname" -}} +{{- if semverCompare "<1.18-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "false" -}} +{{- else -}} +{{- print "true" -}} +{{- end -}} +{{- end -}} + +{{/* +Return true if cert-manager required annotations for TLS signed +certificates are set in the Ingress annotations +Ref: https://cert-manager.io/docs/usage/ingress/#supported-annotations +Usage: +{{ include "common.ingress.certManagerRequest" ( dict "annotations" .Values.path.to.the.ingress.annotations ) }} +*/}} +{{- define "common.ingress.certManagerRequest" -}} +{{ if or (hasKey .annotations "cert-manager.io/cluster-issuer") (hasKey .annotations "cert-manager.io/issuer") }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/common/templates/_labels.tpl b/helm/airbyte-v1/charts/common/templates/_labels.tpl new file mode 100644 index 00000000..252066c7 --- /dev/null +++ b/helm/airbyte-v1/charts/common/templates/_labels.tpl @@ -0,0 +1,18 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Kubernetes standard labels +*/}} +{{- define "common.labels.standard" -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +helm.sh/chart: {{ include "common.names.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Labels to use on deploy.spec.selector.matchLabels and svc.spec.selector +*/}} +{{- define "common.labels.matchLabels" -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} diff --git a/helm/airbyte-v1/charts/common/templates/_names.tpl b/helm/airbyte-v1/charts/common/templates/_names.tpl new file mode 100644 index 00000000..1bdac8b7 --- /dev/null +++ b/helm/airbyte-v1/charts/common/templates/_names.tpl @@ -0,0 +1,70 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "common.names.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "common.names.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "common.names.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a default fully qualified dependency name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +Usage: +{{ include "common.names.dependency.fullname" (dict "chartName" "dependency-chart-name" "chartValues" .Values.dependency-chart "context" $) }} +*/}} +{{- define "common.names.dependency.fullname" -}} +{{- if .chartValues.fullnameOverride -}} +{{- .chartValues.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .chartName .chartValues.nameOverride -}} +{{- if contains $name .context.Release.Name -}} +{{- .context.Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .context.Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts. +*/}} +{{- define "common.names.namespace" -}} +{{- if .Values.namespaceOverride -}} +{{- .Values.namespaceOverride -}} +{{- else -}} +{{- .Release.Namespace -}} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified app name adding the installation's namespace. +*/}} +{{- define "common.names.fullname.namespace" -}} +{{- printf "%s-%s" (include "common.names.fullname" .) (include "common.names.namespace" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/common/templates/_secrets.tpl b/helm/airbyte-v1/charts/common/templates/_secrets.tpl new file mode 100644 index 00000000..a53fb44f --- /dev/null +++ b/helm/airbyte-v1/charts/common/templates/_secrets.tpl @@ -0,0 +1,140 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Generate secret name. + +Usage: +{{ include "common.secrets.name" (dict "existingSecret" .Values.path.to.the.existingSecret "defaultNameSuffix" "mySuffix" "context" $) }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/master/bitnami/common#existingsecret + - defaultNameSuffix - String - Optional. It is used only if we have several secrets in the same deployment. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.secrets.name" -}} +{{- $name := (include "common.names.fullname" .context) -}} + +{{- if .defaultNameSuffix -}} +{{- $name = printf "%s-%s" $name .defaultNameSuffix | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- with .existingSecret -}} +{{- if not (typeIs "string" .) -}} +{{- with .name -}} +{{- $name = . -}} +{{- end -}} +{{- else -}} +{{- $name = . -}} +{{- end -}} +{{- end -}} + +{{- printf "%s" $name -}} +{{- end -}} + +{{/* +Generate secret key. + +Usage: +{{ include "common.secrets.key" (dict "existingSecret" .Values.path.to.the.existingSecret "key" "keyName") }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/master/bitnami/common#existingsecret + - key - String - Required. Name of the key in the secret. +*/}} +{{- define "common.secrets.key" -}} +{{- $key := .key -}} + +{{- if .existingSecret -}} + {{- if not (typeIs "string" .existingSecret) -}} + {{- if .existingSecret.keyMapping -}} + {{- $key = index .existingSecret.keyMapping $.key -}} + {{- end -}} + {{- end }} +{{- end -}} + +{{- printf "%s" $key -}} +{{- end -}} + +{{/* +Generate secret password or retrieve one if already created. + +Usage: +{{ include "common.secrets.passwords.manage" (dict "secret" "secret-name" "key" "keyName" "providedValues" (list "path.to.password1" "path.to.password2") "length" 10 "strong" false "chartName" "chartName" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - key - String - Required - Name of the key in the secret. + - providedValues - List - Required - The path to the validating value in the values.yaml, e.g: "mysql.password". Will pick first parameter with a defined value. + - length - int - Optional - Length of the generated random password. + - strong - Boolean - Optional - Whether to add symbols to the generated random password. + - chartName - String - Optional - Name of the chart used when said chart is deployed as a subchart. + - context - Context - Required - Parent context. + +The order in which this function returns a secret password: + 1. Already existing 'Secret' resource + (If a 'Secret' resource is found under the name provided to the 'secret' parameter to this function and that 'Secret' resource contains a key with the name passed as the 'key' parameter to this function then the value of this existing secret password will be returned) + 2. Password provided via the values.yaml + (If one of the keys passed to the 'providedValues' parameter to this function is a valid path to a key in the values.yaml and has a value, the value of the first key with a value will be returned) + 3. Randomly generated secret password + (A new random secret password with the length specified in the 'length' parameter will be generated and returned) + +*/}} +{{- define "common.secrets.passwords.manage" -}} + +{{- $password := "" }} +{{- $subchart := "" }} +{{- $chartName := default "" .chartName }} +{{- $passwordLength := default 10 .length }} +{{- $providedPasswordKey := include "common.utils.getKeyFromList" (dict "keys" .providedValues "context" $.context) }} +{{- $providedPasswordValue := include "common.utils.getValueFromKey" (dict "key" $providedPasswordKey "context" $.context) }} +{{- $secretData := (lookup "v1" "Secret" $.context.Release.Namespace .secret).data }} +{{- if $secretData }} + {{- if hasKey $secretData .key }} + {{- $password = index $secretData .key }} + {{- else }} + {{- printf "\nPASSWORDS ERROR: The secret \"%s\" does not contain the key \"%s\"\n" .secret .key | fail -}} + {{- end -}} +{{- else if $providedPasswordValue }} + {{- $password = $providedPasswordValue | toString | b64enc | quote }} +{{- else }} + + {{- if .context.Values.enabled }} + {{- $subchart = $chartName }} + {{- end -}} + + {{- $requiredPassword := dict "valueKey" $providedPasswordKey "secret" .secret "field" .key "subchart" $subchart "context" $.context -}} + {{- $requiredPasswordError := include "common.validations.values.single.empty" $requiredPassword -}} + {{- $passwordValidationErrors := list $requiredPasswordError -}} + {{- include "common.errors.upgrade.passwords.empty" (dict "validationErrors" $passwordValidationErrors "context" $.context) -}} + + {{- if .strong }} + {{- $subStr := list (lower (randAlpha 1)) (randNumeric 1) (upper (randAlpha 1)) | join "_" }} + {{- $password = randAscii $passwordLength }} + {{- $password = regexReplaceAllLiteral "\\W" $password "@" | substr 5 $passwordLength }} + {{- $password = printf "%s%s" $subStr $password | toString | shuffle | b64enc | quote }} + {{- else }} + {{- $password = randAlphaNum $passwordLength | b64enc | quote }} + {{- end }} +{{- end -}} +{{- printf "%s" $password -}} +{{- end -}} + +{{/* +Returns whether a previous generated secret already exists + +Usage: +{{ include "common.secrets.exists" (dict "secret" "secret-name" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - context - Context - Required - Parent context. +*/}} +{{- define "common.secrets.exists" -}} +{{- $secret := (lookup "v1" "Secret" $.context.Release.Namespace .secret) }} +{{- if $secret }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/common/templates/_storage.tpl b/helm/airbyte-v1/charts/common/templates/_storage.tpl new file mode 100644 index 00000000..60e2a844 --- /dev/null +++ b/helm/airbyte-v1/charts/common/templates/_storage.tpl @@ -0,0 +1,23 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper Storage Class +{{ include "common.storage.class" ( dict "persistence" .Values.path.to.the.persistence "global" $) }} +*/}} +{{- define "common.storage.class" -}} + +{{- $storageClass := .persistence.storageClass -}} +{{- if .global -}} + {{- if .global.storageClass -}} + {{- $storageClass = .global.storageClass -}} + {{- end -}} +{{- end -}} + +{{- if $storageClass -}} + {{- if (eq "-" $storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" $storageClass -}} + {{- end -}} +{{- end -}} + +{{- end -}} diff --git a/helm/airbyte-v1/charts/common/templates/_tplvalues.tpl b/helm/airbyte-v1/charts/common/templates/_tplvalues.tpl new file mode 100644 index 00000000..2db16685 --- /dev/null +++ b/helm/airbyte-v1/charts/common/templates/_tplvalues.tpl @@ -0,0 +1,13 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Renders a value that contains template. +Usage: +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $) }} +*/}} +{{- define "common.tplvalues.render" -}} + {{- if typeIs "string" .value }} + {{- tpl .value .context }} + {{- else }} + {{- tpl (.value | toYaml) .context }} + {{- end }} +{{- end -}} diff --git a/helm/airbyte-v1/charts/common/templates/_utils.tpl b/helm/airbyte-v1/charts/common/templates/_utils.tpl new file mode 100644 index 00000000..8c22b2a3 --- /dev/null +++ b/helm/airbyte-v1/charts/common/templates/_utils.tpl @@ -0,0 +1,62 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Print instructions to get a secret value. +Usage: +{{ include "common.utils.secret.getvalue" (dict "secret" "secret-name" "field" "secret-value-field" "context" $) }} +*/}} +{{- define "common.utils.secret.getvalue" -}} +{{- $varname := include "common.utils.fieldToEnvVar" . -}} +export {{ $varname }}=$(kubectl get secret --namespace {{ .context.Release.Namespace | quote }} {{ .secret }} -o jsonpath="{.data.{{ .field }}}" | base64 -d) +{{- end -}} + +{{/* +Build env var name given a field +Usage: +{{ include "common.utils.fieldToEnvVar" dict "field" "my-password" }} +*/}} +{{- define "common.utils.fieldToEnvVar" -}} + {{- $fieldNameSplit := splitList "-" .field -}} + {{- $upperCaseFieldNameSplit := list -}} + + {{- range $fieldNameSplit -}} + {{- $upperCaseFieldNameSplit = append $upperCaseFieldNameSplit ( upper . ) -}} + {{- end -}} + + {{ join "_" $upperCaseFieldNameSplit }} +{{- end -}} + +{{/* +Gets a value from .Values given +Usage: +{{ include "common.utils.getValueFromKey" (dict "key" "path.to.key" "context" $) }} +*/}} +{{- define "common.utils.getValueFromKey" -}} +{{- $splitKey := splitList "." .key -}} +{{- $value := "" -}} +{{- $latestObj := $.context.Values -}} +{{- range $splitKey -}} + {{- if not $latestObj -}} + {{- printf "please review the entire path of '%s' exists in values" $.key | fail -}} + {{- end -}} + {{- $value = ( index $latestObj . ) -}} + {{- $latestObj = $value -}} +{{- end -}} +{{- printf "%v" (default "" $value) -}} +{{- end -}} + +{{/* +Returns first .Values key with a defined value or first of the list if all non-defined +Usage: +{{ include "common.utils.getKeyFromList" (dict "keys" (list "path.to.key1" "path.to.key2") "context" $) }} +*/}} +{{- define "common.utils.getKeyFromList" -}} +{{- $key := first .keys -}} +{{- $reverseKeys := reverse .keys }} +{{- range $reverseKeys }} + {{- $value := include "common.utils.getValueFromKey" (dict "key" . "context" $.context ) }} + {{- if $value -}} + {{- $key = . }} + {{- end -}} +{{- end -}} +{{- printf "%s" $key -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/common/templates/_warnings.tpl b/helm/airbyte-v1/charts/common/templates/_warnings.tpl new file mode 100644 index 00000000..ae10fa41 --- /dev/null +++ b/helm/airbyte-v1/charts/common/templates/_warnings.tpl @@ -0,0 +1,14 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Warning about using rolling tag. +Usage: +{{ include "common.warnings.rollingTag" .Values.path.to.the.imageRoot }} +*/}} +{{- define "common.warnings.rollingTag" -}} + +{{- if and (contains "bitnami/" .repository) (not (.tag | toString | regexFind "-r\\d+$|sha256:")) }} +WARNING: Rolling tag detected ({{ .repository }}:{{ .tag }}), please note that it is strongly recommended to avoid using rolling tags in a production environment. ++info https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/ +{{- end }} + +{{- end -}} diff --git a/helm/airbyte-v1/charts/common/templates/validations/_cassandra.tpl b/helm/airbyte-v1/charts/common/templates/validations/_cassandra.tpl new file mode 100644 index 00000000..ded1ae3b --- /dev/null +++ b/helm/airbyte-v1/charts/common/templates/validations/_cassandra.tpl @@ -0,0 +1,72 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate Cassandra required passwords are not empty. + +Usage: +{{ include "common.validations.values.cassandra.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where Cassandra values are stored, e.g: "cassandra-passwords-secret" + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.cassandra.passwords" -}} + {{- $existingSecret := include "common.cassandra.values.existingSecret" . -}} + {{- $enabled := include "common.cassandra.values.enabled" . -}} + {{- $dbUserPrefix := include "common.cassandra.values.key.dbUser" . -}} + {{- $valueKeyPassword := printf "%s.password" $dbUserPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "cassandra-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.cassandra.values.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.cassandra.values.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.cassandra.dbUser.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.dbUser.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled cassandra. + +Usage: +{{ include "common.cassandra.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.cassandra.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.cassandra.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key dbUser + +Usage: +{{ include "common.cassandra.values.key.dbUser" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.cassandra.values.key.dbUser" -}} + {{- if .subchart -}} + cassandra.dbUser + {{- else -}} + dbUser + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/common/templates/validations/_mariadb.tpl b/helm/airbyte-v1/charts/common/templates/validations/_mariadb.tpl new file mode 100644 index 00000000..b6906ff7 --- /dev/null +++ b/helm/airbyte-v1/charts/common/templates/validations/_mariadb.tpl @@ -0,0 +1,103 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MariaDB required passwords are not empty. + +Usage: +{{ include "common.validations.values.mariadb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MariaDB values are stored, e.g: "mysql-passwords-secret" + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mariadb.passwords" -}} + {{- $existingSecret := include "common.mariadb.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mariadb.values.enabled" . -}} + {{- $architecture := include "common.mariadb.values.architecture" . -}} + {{- $authPrefix := include "common.mariadb.values.key.auth" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mariadb-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- if not (empty $valueUsername) -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mariadb-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replication") -}} + {{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mariadb-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mariadb.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mariadb. + +Usage: +{{ include "common.mariadb.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mariadb.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mariadb.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mariadb.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mariadb.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.key.auth" -}} + {{- if .subchart -}} + mariadb.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/common/templates/validations/_mongodb.tpl b/helm/airbyte-v1/charts/common/templates/validations/_mongodb.tpl new file mode 100644 index 00000000..f820ec10 --- /dev/null +++ b/helm/airbyte-v1/charts/common/templates/validations/_mongodb.tpl @@ -0,0 +1,108 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MongoDB® required passwords are not empty. + +Usage: +{{ include "common.validations.values.mongodb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MongoDB® values are stored, e.g: "mongodb-passwords-secret" + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mongodb.passwords" -}} + {{- $existingSecret := include "common.mongodb.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mongodb.values.enabled" . -}} + {{- $authPrefix := include "common.mongodb.values.key.auth" . -}} + {{- $architecture := include "common.mongodb.values.architecture" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyDatabase := printf "%s.database" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicaSetKey := printf "%s.replicaSetKey" $authPrefix -}} + {{- $valueKeyAuthEnabled := printf "%s.enabled" $authPrefix -}} + + {{- $authEnabled := include "common.utils.getValueFromKey" (dict "key" $valueKeyAuthEnabled "context" .context) -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") (eq $authEnabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mongodb-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- $valueDatabase := include "common.utils.getValueFromKey" (dict "key" $valueKeyDatabase "context" .context) }} + {{- if and $valueUsername $valueDatabase -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mongodb-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replicaset") -}} + {{- $requiredReplicaSetKey := dict "valueKey" $valueKeyReplicaSetKey "secret" .secret "field" "mongodb-replica-set-key" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicaSetKey -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mongodb.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDb is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mongodb.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mongodb. + +Usage: +{{ include "common.mongodb.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mongodb.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mongodb.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mongodb.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.key.auth" -}} + {{- if .subchart -}} + mongodb.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mongodb.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mongodb.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/common/templates/validations/_mysql.tpl b/helm/airbyte-v1/charts/common/templates/validations/_mysql.tpl new file mode 100644 index 00000000..74472a06 --- /dev/null +++ b/helm/airbyte-v1/charts/common/templates/validations/_mysql.tpl @@ -0,0 +1,103 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MySQL required passwords are not empty. + +Usage: +{{ include "common.validations.values.mysql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MySQL values are stored, e.g: "mysql-passwords-secret" + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mysql.passwords" -}} + {{- $existingSecret := include "common.mysql.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mysql.values.enabled" . -}} + {{- $architecture := include "common.mysql.values.architecture" . -}} + {{- $authPrefix := include "common.mysql.values.key.auth" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mysql-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- if not (empty $valueUsername) -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mysql-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replication") -}} + {{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mysql-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mysql.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mysql.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mysql. + +Usage: +{{ include "common.mysql.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mysql.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mysql.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mysql.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mysql.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mysql.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.key.auth" -}} + {{- if .subchart -}} + mysql.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/common/templates/validations/_postgresql.tpl b/helm/airbyte-v1/charts/common/templates/validations/_postgresql.tpl new file mode 100644 index 00000000..164ec0d0 --- /dev/null +++ b/helm/airbyte-v1/charts/common/templates/validations/_postgresql.tpl @@ -0,0 +1,129 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate PostgreSQL required passwords are not empty. + +Usage: +{{ include "common.validations.values.postgresql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where postgresql values are stored, e.g: "postgresql-passwords-secret" + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.postgresql.passwords" -}} + {{- $existingSecret := include "common.postgresql.values.existingSecret" . -}} + {{- $enabled := include "common.postgresql.values.enabled" . -}} + {{- $valueKeyPostgresqlPassword := include "common.postgresql.values.key.postgressPassword" . -}} + {{- $valueKeyPostgresqlReplicationEnabled := include "common.postgresql.values.key.replicationPassword" . -}} + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + {{- $requiredPostgresqlPassword := dict "valueKey" $valueKeyPostgresqlPassword "secret" .secret "field" "postgresql-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlPassword -}} + + {{- $enabledReplication := include "common.postgresql.values.enabled.replication" . -}} + {{- if (eq $enabledReplication "true") -}} + {{- $requiredPostgresqlReplicationPassword := dict "valueKey" $valueKeyPostgresqlReplicationEnabled "secret" .secret "field" "postgresql-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to decide whether evaluate global values. + +Usage: +{{ include "common.postgresql.values.use.global" (dict "key" "key-of-global" "context" $) }} +Params: + - key - String - Required. Field to be evaluated within global, e.g: "existingSecret" +*/}} +{{- define "common.postgresql.values.use.global" -}} + {{- if .context.Values.global -}} + {{- if .context.Values.global.postgresql -}} + {{- index .context.Values.global.postgresql .key | quote -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.postgresql.values.existingSecret" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.existingSecret" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "existingSecret" "context" .context) -}} + + {{- if .subchart -}} + {{- default (.context.Values.postgresql.existingSecret | quote) $globalValue -}} + {{- else -}} + {{- default (.context.Values.existingSecret | quote) $globalValue -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled postgresql. + +Usage: +{{ include "common.postgresql.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key postgressPassword. + +Usage: +{{ include "common.postgresql.values.key.postgressPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.postgressPassword" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "postgresqlUsername" "context" .context) -}} + + {{- if not $globalValue -}} + {{- if .subchart -}} + postgresql.postgresqlPassword + {{- else -}} + postgresqlPassword + {{- end -}} + {{- else -}} + global.postgresql.postgresqlPassword + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled.replication. + +Usage: +{{ include "common.postgresql.values.enabled.replication" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.enabled.replication" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.replication.enabled -}} + {{- else -}} + {{- printf "%v" .context.Values.replication.enabled -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key replication.password. + +Usage: +{{ include "common.postgresql.values.key.replicationPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.replicationPassword" -}} + {{- if .subchart -}} + postgresql.replication.password + {{- else -}} + replication.password + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/common/templates/validations/_redis.tpl b/helm/airbyte-v1/charts/common/templates/validations/_redis.tpl new file mode 100644 index 00000000..dcccfc1a --- /dev/null +++ b/helm/airbyte-v1/charts/common/templates/validations/_redis.tpl @@ -0,0 +1,76 @@ + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate Redis® required passwords are not empty. + +Usage: +{{ include "common.validations.values.redis.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where redis values are stored, e.g: "redis-passwords-secret" + - subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.redis.passwords" -}} + {{- $enabled := include "common.redis.values.enabled" . -}} + {{- $valueKeyPrefix := include "common.redis.values.keys.prefix" . -}} + {{- $standarizedVersion := include "common.redis.values.standarized.version" . }} + + {{- $existingSecret := ternary (printf "%s%s" $valueKeyPrefix "auth.existingSecret") (printf "%s%s" $valueKeyPrefix "existingSecret") (eq $standarizedVersion "true") }} + {{- $existingSecretValue := include "common.utils.getValueFromKey" (dict "key" $existingSecret "context" .context) }} + + {{- $valueKeyRedisPassword := ternary (printf "%s%s" $valueKeyPrefix "auth.password") (printf "%s%s" $valueKeyPrefix "password") (eq $standarizedVersion "true") }} + {{- $valueKeyRedisUseAuth := ternary (printf "%s%s" $valueKeyPrefix "auth.enabled") (printf "%s%s" $valueKeyPrefix "usePassword") (eq $standarizedVersion "true") }} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $useAuth := include "common.utils.getValueFromKey" (dict "key" $valueKeyRedisUseAuth "context" .context) -}} + {{- if eq $useAuth "true" -}} + {{- $requiredRedisPassword := dict "valueKey" $valueKeyRedisPassword "secret" .secret "field" "redis-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRedisPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled redis. + +Usage: +{{ include "common.redis.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.redis.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.redis.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right prefix path for the values + +Usage: +{{ include "common.redis.values.key.prefix" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false +*/}} +{{- define "common.redis.values.keys.prefix" -}} + {{- if .subchart -}}redis.{{- else -}}{{- end -}} +{{- end -}} + +{{/* +Checks whether the redis chart's includes the standarizations (version >= 14) + +Usage: +{{ include "common.redis.values.standarized.version" (dict "context" $) }} +*/}} +{{- define "common.redis.values.standarized.version" -}} + + {{- $standarizedAuth := printf "%s%s" (include "common.redis.values.keys.prefix" .) "auth" -}} + {{- $standarizedAuthValues := include "common.utils.getValueFromKey" (dict "key" $standarizedAuth "context" .context) }} + + {{- if $standarizedAuthValues -}} + {{- true -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/common/templates/validations/_validations.tpl b/helm/airbyte-v1/charts/common/templates/validations/_validations.tpl new file mode 100644 index 00000000..9a814cf4 --- /dev/null +++ b/helm/airbyte-v1/charts/common/templates/validations/_validations.tpl @@ -0,0 +1,46 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate values must not be empty. + +Usage: +{{- $validateValueConf00 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-00") -}} +{{- $validateValueConf01 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-01") -}} +{{ include "common.validations.values.empty" (dict "required" (list $validateValueConf00 $validateValueConf01) "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" +*/}} +{{- define "common.validations.values.multiple.empty" -}} + {{- range .required -}} + {{- include "common.validations.values.single.empty" (dict "valueKey" .valueKey "secret" .secret "field" .field "context" $.context) -}} + {{- end -}} +{{- end -}} + +{{/* +Validate a value must not be empty. + +Usage: +{{ include "common.validations.value.empty" (dict "valueKey" "mariadb.password" "secret" "secretName" "field" "my-password" "subchart" "subchart" "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" + - subchart - String - Optional - Name of the subchart that the validated password is part of. +*/}} +{{- define "common.validations.values.single.empty" -}} + {{- $value := include "common.utils.getValueFromKey" (dict "key" .valueKey "context" .context) }} + {{- $subchart := ternary "" (printf "%s." .subchart) (empty .subchart) }} + + {{- if not $value -}} + {{- $varname := "my-value" -}} + {{- $getCurrentValue := "" -}} + {{- if and .secret .field -}} + {{- $varname = include "common.utils.fieldToEnvVar" . -}} + {{- $getCurrentValue = printf " To get the current value:\n\n %s\n" (include "common.utils.secret.getvalue" .) -}} + {{- end -}} + {{- printf "\n '%s' must not be empty, please add '--set %s%s=$%s' to the command.%s" .valueKey $subchart .valueKey $varname $getCurrentValue -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/common/values.yaml b/helm/airbyte-v1/charts/common/values.yaml new file mode 100644 index 00000000..f2df68e5 --- /dev/null +++ b/helm/airbyte-v1/charts/common/values.yaml @@ -0,0 +1,5 @@ +## bitnami/common +## It is required by CI/CD tools and processes. +## @skip exampleValue +## +exampleValue: common-chart diff --git a/helm/airbyte-v1/charts/connector-builder-server/.gitignore b/helm/airbyte-v1/charts/connector-builder-server/.gitignore new file mode 100644 index 00000000..88e91e8a --- /dev/null +++ b/helm/airbyte-v1/charts/connector-builder-server/.gitignore @@ -0,0 +1,2 @@ +# Charts are downloaded at install time with `helm dep build`. +charts diff --git a/helm/airbyte-v1/charts/connector-builder-server/Chart.lock b/helm/airbyte-v1/charts/connector-builder-server/Chart.lock new file mode 100644 index 00000000..746fb41b --- /dev/null +++ b/helm/airbyte-v1/charts/connector-builder-server/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + version: 1.17.1 +digest: sha256:dacc73770a5640c011e067ff8840ddf89631fc19016c8d0a9e5ea160e7da8690 +generated: "2022-10-17T18:35:15.123937677Z" diff --git a/helm/airbyte-v1/charts/connector-builder-server/Chart.yaml b/helm/airbyte-v1/charts/connector-builder-server/Chart.yaml new file mode 100644 index 00000000..a4e8f766 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-builder-server/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +appVersion: 1.3.0 +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x +description: Helm chart to deploy airbyte-connector-builder-server +name: connector-builder-server +type: application +version: 1.3.0 diff --git a/helm/airbyte-v1/charts/connector-builder-server/README.md b/helm/airbyte-v1/charts/connector-builder-server/README.md new file mode 100644 index 00000000..9b49f69a --- /dev/null +++ b/helm/airbyte-v1/charts/connector-builder-server/README.md @@ -0,0 +1,64 @@ +# connector-builder-server + +![Version: 0.67.17](https://img.shields.io/badge/Version-0.67.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) + +Helm chart to deploy airbyte-connector-builder-server + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://charts.bitnami.com/bitnami | common | 1.x.x | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| containerSecurityContext | object | `{}` | | +| debug.enabled | bool | `false` | | +| debug.remoteDebugPort | int | `5005` | | +| enabled | bool | `true` | | +| env_vars | object | `{}` | | +| extraContainers | list | `[]` | | +| extraEnv | list | `[]` | | +| extraInitContainers | list | `[]` | | +| extraVolumeMounts | list | `[]` | | +| extraVolumes | list | `[]` | | +| global.configMapName | string | `""` | | +| global.credVolumeOverride | string | `""` | | +| global.database.secretName | string | `""` | | +| global.database.secretValue | string | `""` | | +| global.deploymentMode | string | `"oss"` | | +| global.deploymentStrategyType | string | `"RollingUpdate"` | | +| global.extraContainers | list | `[]` | | +| global.extraLabels | object | `{}` | | +| global.extraSelectorLabels | object | `{}` | | +| global.secretName | string | `""` | | +| global.serviceAccountName | string | `"placeholderServiceAccount"` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"airbyte/connector-builder-server"` | | +| livenessProbe.enabled | bool | `true` | | +| livenessProbe.failureThreshold | int | `3` | | +| livenessProbe.initialDelaySeconds | int | `30` | | +| livenessProbe.periodSeconds | int | `10` | | +| livenessProbe.successThreshold | int | `1` | | +| livenessProbe.timeoutSeconds | int | `1` | | +| log.level | string | `"INFO"` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| readinessProbe.enabled | bool | `true` | | +| readinessProbe.failureThreshold | int | `3` | | +| readinessProbe.initialDelaySeconds | int | `10` | | +| readinessProbe.periodSeconds | int | `10` | | +| readinessProbe.successThreshold | int | `1` | | +| readinessProbe.timeoutSeconds | int | `1` | | +| replicaCount | int | `1` | | +| resources.limits | object | `{}` | | +| resources.requests | object | `{}` | | +| secrets | object | `{}` | | +| service.annotations | object | `{}` | | +| service.port | int | `80` | | +| service.type | string | `"NodePort"` | | +| tolerations | list | `[]` | | + diff --git a/helm/airbyte-v1/charts/connector-builder-server/templates/_helpers.tpl b/helm/airbyte-v1/charts/connector-builder-server/templates/_helpers.tpl new file mode 100644 index 00000000..174aebfe --- /dev/null +++ b/helm/airbyte-v1/charts/connector-builder-server/templates/_helpers.tpl @@ -0,0 +1,59 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "airbyte.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airbyte.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airbyte.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "airbyte.labels" -}} +helm.sh/chart: {{ include "airbyte.chart" . }} +{{ include "airbyte.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "airbyte.selectorLabels" -}} +app.kubernetes.io/name: {{ include "airbyte.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Define db secret +*/}} + +{{- define "database.secret.name" -}} +{{- printf "%s-postgresql" .Release.Name }} +{{- end }} \ No newline at end of file diff --git a/helm/airbyte-v1/charts/connector-builder-server/templates/_images.tpl b/helm/airbyte-v1/charts/connector-builder-server/templates/_images.tpl new file mode 100644 index 00000000..6c32ec44 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-builder-server/templates/_images.tpl @@ -0,0 +1,17 @@ +{{- define "imageUrl" -}} +{{- $img := index . 0 -}} +{{- $root := index . 1 -}} + +{{- $reg := $root.Values.global.image.registry -}} +{{/* ensure the registry has a trailing slash, if set */}} +{{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} + +{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{- if (eq (typeOf $img) "string") -}} +{{- printf "%s%s" $reg (tpl $img $root) | quote -}} +{{- else -}} +{{- $tag := coalesce $img.tag $root.Values.global.image.tag $root.Chart.AppVersion -}} +{{- printf "%s%s:%s" $reg $img.repository $tag | quote -}} +{{- end -}} + +{{- end -}} \ No newline at end of file diff --git a/helm/airbyte-v1/charts/connector-builder-server/templates/deployment.yaml b/helm/airbyte-v1/charts/connector-builder-server/templates/deployment.yaml new file mode 100644 index 00000000..4eef212e --- /dev/null +++ b/helm/airbyte-v1/charts/connector-builder-server/templates/deployment.yaml @@ -0,0 +1,179 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.names.fullname" . }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "airbyte.selectorLabels" . | nindent 6 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 6 }} + {{- end }} + strategy: + type: {{ .Values.deploymentStrategyType }} + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 8 }} + {{- end }} + {{- if .Values.podLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ .Values.global.serviceAccountName }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.extraInitContainers }} + initContainers: + {{- toYaml .Values.extraInitContainers | nindent 6 }} + {{- end }} + containers: + - name: airbyte-connector-builder-server + image: {{ include "imageUrl" (list .Values.image $) }} + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + env: + {{- if .Values.debug.enabled }} + - name: JAVA_TOOL_OPTIONS + value: "-Xdebug -agentlib:jdwp=transport=dt_socket,address=0.0.0.0:{{ .Values.debug.remoteDebugPort }},server=y,suspend=n" + {{- end}} + {{- if eq .Values.global.deploymentMode "oss" }} + - name: AIRBYTE_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_VERSION + - name: MICROMETER_METRICS_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MICROMETER_METRICS_ENABLED + - name: MICROMETER_METRICS_STATSD_FLAVOR + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MICROMETER_METRICS_STATSD_FLAVOR + - name: SEGMENT_WRITE_KEY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: SEGMENT_WRITE_KEY + - name: STATSD_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: STATSD_HOST + - name: STATSD_PORT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: STATSD_PORT + - name: TRACKING_STRATEGY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TRACKING_STRATEGY + {{- end }} + + # Values from secret + {{- if .Values.secrets }} + {{- range $k, $v := .Values.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: server-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if or .Values.env_vars .Values.global.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + # Values from extraEnv for more compability(if you want to use external secret source or other stuff) + {{- if .Values.extraEnv }} + {{- toYaml .Values.extraEnv | nindent 8 }} + {{- end }} + + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /v1/health + port: http + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /v1/health + port: http + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + + ports: + - name: http + containerPort: 8080 + protocol: TCP + {{- if .Values.debug.enabled }} + - name: debug + containerPort: {{ .Values.debug.remoteDebugPort }} + protocol: TCP + {{- end}} + {{- if .Values.resources }} + resources: {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + {{- if .Values.containerSecurityContext }} + securityContext: {{- toYaml .Values.containerSecurityContext | nindent 10 }} + {{- end }} + volumeMounts: + {{- if .Values.extraVolumeMounts }} + {{ toYaml .Values.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.global.extraVolumeMounts }} + {{ toYaml .Values.global.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.extraContainers }} + {{ toYaml .Values.extraContainers | nindent 6 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 6 }} + {{- end }} + securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + volumes: + {{- if .Values.extraVolumes }} +{{ toYaml .Values.extraVolumes | nindent 6 }} + {{- end }} diff --git a/helm/airbyte-v1/charts/connector-builder-server/templates/secrets.yaml b/helm/airbyte-v1/charts/connector-builder-server/templates/secrets.yaml new file mode 100644 index 00000000..e00ae6ba --- /dev/null +++ b/helm/airbyte-v1/charts/connector-builder-server/templates/secrets.yaml @@ -0,0 +1,17 @@ +# Create secrets only for the local deployment +{{- if .Values.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: connector-builder-server-secrets + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" +type: Opaque +data: + {{- range $k, $v := mergeOverwrite .Values.secrets .Values.global.secrets }} + {{ $k }}: {{ if $v }}{{ $v | b64enc }} {{else}}""{{end}} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v1/charts/connector-builder-server/templates/service.yaml b/helm/airbyte-v1/charts/connector-builder-server/templates/service.yaml new file mode 100644 index 00000000..6f1cb26e --- /dev/null +++ b/helm/airbyte-v1/charts/connector-builder-server/templates/service.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{.Release.Name }}-airbyte-connector-builder-server-svc + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "airbyte.selectorLabels" . | nindent 4 }} diff --git a/helm/airbyte-v1/charts/connector-builder-server/values.yaml b/helm/airbyte-v1/charts/connector-builder-server/values.yaml new file mode 100644 index 00000000..64326ab9 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-builder-server/values.yaml @@ -0,0 +1,192 @@ +global: + serviceAccountName: placeholderServiceAccount + deploymentMode: oss + deploymentStrategyType: RollingUpdate + configMapName: "" + secretName: "" + credVolumeOverride: "" + extraContainers: [] + ## extraSelectorLabels [object] - use to specify own additional selector labels for deployment + extraSelectorLabels: {} + ## extraLabels [object] - use to specify own additional labels for deployment + extraLabels: {} + database: + secretName: "" + secretValue: "" + + # Docker image config that will apply to all images. + image: + # Docker registry to pull platform images from, e.g. http://my-registry:8000/ + registry: "" + # Image tag to use for airbyte images. + # Does not include non-airbyte images such as busybox, temporal, minio, etc. + tag: "" + +enabled: true +## replicaCount Number of server replicas +replicaCount: 1 + +## image.repository The repository to use for the airbyte server image. +## image.pullPolicy the pull policy to use for the airbyte server image +## image.tag The airbyte server image tag. Defaults to the chart's AppVersion +image: + repository: airbyte/connector-builder-server + pullPolicy: IfNotPresent + +## podAnnotations [object] Add extra annotations to the server pod +## +podAnnotations: {} + +## containerSecurityContext Security context for the container +## Examples: +## containerSecurityContext: +## runAsNonRoot: true +## runAsUser: 1000 +## readOnlyRootFilesystem: true +containerSecurityContext: {} + +## Configure extra options for the server containers' liveness and readiness probes +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes +## livenessProbe.enabled Enable livenessProbe on the server +## livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe +## livenessProbe.periodSeconds Period seconds for livenessProbe +## livenessProbe.timeoutSeconds Timeout seconds for livenessProbe +## livenessProbe.failureThreshold Failure threshold for livenessProbe +## livenessProbe.successThreshold Success threshold for livenessProbe +## +livenessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + +## readinessProbe.enabled Enable readinessProbe on the server +## readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe +## readinessProbe.periodSeconds Period seconds for readinessProbe +## readinessProbe.timeoutSeconds Timeout seconds for readinessProbe +## readinessProbe.failureThreshold Failure threshold for readinessProbe +## readinessProbe.successThreshold Success threshold for readinessProbe +## +readinessProbe: + enabled: true + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + +## Server app resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## We usually recommend not to specify default resources and to leave this as a conscious +## choice for the user. This also increases chances charts run on environments with little +## resources, such as Minikube. If you do want to specify resources, uncomment the following +## lines, adjust them as necessary, and remove the curly braces after 'resources:'. +## resources.limits [object] The resources limits for the server container +## resources.requests [object] The requested resources for the server container +resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + +## service.type The service type to use for the API server +## service.port The service port to expose the API server on +service: + type: NodePort + port: 80 + annotations: {} + +## nodeSelector [object] Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## tolerations [array] Tolerations for server pod assignment. +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## affinity [object] Affinity and anti-affinity for server pod assignment. +## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## log.level The log level to log at +log: + level: "INFO" + +## extraVolumeMounts [array] Additional volumeMounts for server container(s). +## Examples (when using `containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumeMounts: +## - name: tmpdir +## mountPath: /tmp +## +extraVolumeMounts: [] + +## extraVolumes [array] Additional volumes for server pod(s). +## Examples (when using `containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumes: +## - name: tmpdir +## emptyDir: {} +## +extraVolumes: [] + +## extraContainer [array] Additional container for server pod(s) +## Example: +# extraContainers: +# - name: otel_collector +# image: somerepo/someimage:sometag +# args: [ +# "--important-args" +# ] +# ports: +# - containerPort: 443 +# volumeMounts: +# - name: volumeMountCool +# mountPath: /some/path +# readOnly: true +extraContainers: [] + +## extraInitContainers [array] Additional init containers for server pod(s) +## Example: +# extraInitContainers: +# - name: sleepy +# image: alpine +# command: ['sleep', '60'] + +extraInitContainers: [] + +## extraEnv [array] Supply extra env variables to main container using full notation +## Example: (With default env vars and values taken from generated config map) +# extraEnv: +# - name: AIRBYTE_VERSION +# valueFrom: +# configMapKeyRef: +# name: airbyte-env +# key: AIRBYTE_VERSION +## +extraEnv: [] +## secrets [object] Supply additional secrets to container +## Example: +## secrets: +## DATABASE_PASSWORD: strong-password +## DATABASE_USER: my-db-user +secrets: {} + +## env_vars [object] Supply extra env variables to main container using simplified notation +## Example: +# env_vars: +# AIRBYTE_VERSION: 0.40.4 +env_vars: {} + +debug: + enabled: false + remoteDebugPort: 5005 diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/Chart.lock b/helm/airbyte-v1/charts/connector-rollout-worker/Chart.lock new file mode 100644 index 00000000..9778a41b --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + version: 1.17.1 +digest: sha256:dacc73770a5640c011e067ff8840ddf89631fc19016c8d0a9e5ea160e7da8690 +generated: "2024-12-09T21:57:06.251315255Z" diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/Chart.yaml b/helm/airbyte-v1/charts/connector-rollout-worker/Chart.yaml new file mode 100644 index 00000000..4c69c7f3 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +appVersion: 1.3.0 +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x +description: Helm chart to deploy airbyte-connector-rollout-worker +name: connector-rollout-worker +type: application +version: 1.3.0 diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/README.md b/helm/airbyte-v1/charts/connector-rollout-worker/README.md new file mode 100644 index 00000000..ad3e11f2 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/README.md @@ -0,0 +1,60 @@ +# connector-rollout-worker + +![Version: 0.67.17](https://img.shields.io/badge/Version-0.67.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) + +Helm chart to deploy airbyte-connector-rollout-worker + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://charts.bitnami.com/bitnami | common | 1.x.x | + +## Values + +| Key | Type | Default | Description | +|---------------------------------------------------|------|--------------------------------------|-------------| +| affinity | object | `{}` | | +| containerSecurityContext | object | `{}` | | +| debug.enabled | bool | `false` | | +| debug.remoteDebugPort | int | `5005` | | +| enabled | bool | `true` | | +| env_vars | object | `{}` | | +| extraContainers | list | `[]` | | +| extraEnv | list | `[]` | | +| extraInitContainers | list | `[]` | | +| extraLabels | object | `{}` | | +| extraSelectorLabels | object | `{}` | | +| extraVolumeMounts | list | `[]` | | +| extraVolumes | list | `[]` | | +| global.credVolumeOverride | string | `""` | | +| global.database.secretName | string | `""` | | +| global.database.secretValue | string | `""` | | +| global.extraContainers | list | `[]` | | +| global.extraLabels | object | `{}` | | +| global.extraSelectorLabels | object | `{}` | | +| global.secretName | string | `""` | | +| global.serviceAccountName | string | `"airbyte-admin"` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"airbyte/connector-rollout-worker"` | | +| livenessProbe.enabled | bool | `true` | | +| livenessProbe.failureThreshold | int | `3` | | +| livenessProbe.initialDelaySeconds | int | `50` | | +| livenessProbe.periodSeconds | int | `10` | | +| livenessProbe.successThreshold | int | `1` | | +| livenessProbe.timeoutSeconds | int | `1` | | +| log.level | string | `"INFO"` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| podLabels | object | `{}` | | +| readinessProbe.enabled | bool | `true` | | +| readinessProbe.failureThreshold | int | `3` | | +| readinessProbe.initialDelaySeconds | int | `10` | | +| readinessProbe.periodSeconds | int | `10` | | +| readinessProbe.successThreshold | int | `1` | | +| readinessProbe.timeoutSeconds | int | `1` | | +| replicaCount | int | `1` | | +| resources.limits | object | `{}` | | +| resources.requests | object | `{}` | | +| secrets | object | `{}` | | +| tolerations | list | `[]` | | diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/.helmignore b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/Chart.yaml b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/Chart.yaml new file mode 100644 index 00000000..39edbb3c --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/Chart.yaml @@ -0,0 +1,23 @@ +annotations: + category: Infrastructure +apiVersion: v2 +appVersion: 1.17.1 +description: A Library Helm Chart for grouping common logic between bitnami charts. + This chart is not deployable by itself. +home: https://github.com/bitnami/charts/tree/master/bitnami/common +icon: https://bitnami.com/downloads/logos/bitnami-mark.png +keywords: +- common +- helper +- template +- function +- bitnami +maintainers: +- name: Bitnami + url: https://github.com/bitnami/charts +name: common +sources: +- https://github.com/bitnami/charts +- https://www.bitnami.com/ +type: library +version: 1.17.1 diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/README.md b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/README.md new file mode 100644 index 00000000..a2ecd604 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/README.md @@ -0,0 +1,350 @@ +# Bitnami Common Library Chart + +A [Helm Library Chart](https://helm.sh/docs/topics/library_charts/#helm) for grouping common logic between bitnami charts. + +## TL;DR + +```yaml +dependencies: + - name: common + version: 1.x.x + repository: https://charts.bitnami.com/bitnami +``` + +```bash +$ helm dependency update +``` + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.names.fullname" . }} +data: + myvalue: "Hello World" +``` + +## Introduction + +This chart provides a common template helpers which can be used to develop new charts using [Helm](https://helm.sh) package manager. + +Bitnami charts can be used with [Kubeapps](https://kubeapps.dev/) for deployment and management of Helm Charts in clusters. + +## Prerequisites + +- Kubernetes 1.19+ +- Helm 3.2.0+ + +## Parameters + +The following table lists the helpers available in the library which are scoped in different sections. + +### Affinities + +| Helper identifier | Description | Expected Input | +|-------------------------------|------------------------------------------------------|------------------------------------------------| +| `common.affinities.nodes.soft` | Return a soft nodeAffinity definition | `dict "key" "FOO" "values" (list "BAR" "BAZ")` | +| `common.affinities.nodes.hard` | Return a hard nodeAffinity definition | `dict "key" "FOO" "values" (list "BAR" "BAZ")` | +| `common.affinities.pods.soft` | Return a soft podAffinity/podAntiAffinity definition | `dict "component" "FOO" "context" $` | +| `common.affinities.pods.hard` | Return a hard podAffinity/podAntiAffinity definition | `dict "component" "FOO" "context" $` | + +### Capabilities + +| Helper identifier | Description | Expected Input | +|------------------------------------------------|------------------------------------------------------------------------------------------------|-------------------| +| `common.capabilities.kubeVersion` | Return the target Kubernetes version (using client default if .Values.kubeVersion is not set). | `.` Chart context | +| `common.capabilities.cronjob.apiVersion` | Return the appropriate apiVersion for cronjob. | `.` Chart context | +| `common.capabilities.deployment.apiVersion` | Return the appropriate apiVersion for deployment. | `.` Chart context | +| `common.capabilities.statefulset.apiVersion` | Return the appropriate apiVersion for statefulset. | `.` Chart context | +| `common.capabilities.ingress.apiVersion` | Return the appropriate apiVersion for ingress. | `.` Chart context | +| `common.capabilities.rbac.apiVersion` | Return the appropriate apiVersion for RBAC resources. | `.` Chart context | +| `common.capabilities.crd.apiVersion` | Return the appropriate apiVersion for CRDs. | `.` Chart context | +| `common.capabilities.policy.apiVersion` | Return the appropriate apiVersion for podsecuritypolicy. | `.` Chart context | +| `common.capabilities.networkPolicy.apiVersion` | Return the appropriate apiVersion for networkpolicy. | `.` Chart context | +| `common.capabilities.apiService.apiVersion` | Return the appropriate apiVersion for APIService. | `.` Chart context | +| `common.capabilities.hpa.apiVersion` | Return the appropriate apiVersion for Horizontal Pod Autoscaler | `.` Chart context | +| `common.capabilities.supportsHelmVersion` | Returns true if the used Helm version is 3.3+ | `.` Chart context | + +### Errors + +| Helper identifier | Description | Expected Input | +|-----------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------| +| `common.errors.upgrade.passwords.empty` | It will ensure required passwords are given when we are upgrading a chart. If `validationErrors` is not empty it will throw an error and will stop the upgrade action. | `dict "validationErrors" (list $validationError00 $validationError01) "context" $` | + +### Images + +| Helper identifier | Description | Expected Input | +|-----------------------------|------------------------------------------------------|---------------------------------------------------------------------------------------------------------| +| `common.images.image` | Return the proper and full image name | `dict "imageRoot" .Values.path.to.the.image "global" $`, see [ImageRoot](#imageroot) for the structure. | +| `common.images.pullSecrets` | Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) | `dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global` | +| `common.images.renderPullSecrets` | Return the proper Docker Image Registry Secret Names (evaluates values as templates) | `dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "context" $` | + +### Ingress + +| Helper identifier | Description | Expected Input | +|-------------------------------------------|-------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.ingress.backend` | Generate a proper Ingress backend entry depending on the API version | `dict "serviceName" "foo" "servicePort" "bar"`, see the [Ingress deprecation notice](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/) for the syntax differences | +| `common.ingress.supportsPathType` | Prints "true" if the pathType field is supported | `.` Chart context | +| `common.ingress.supportsIngressClassname` | Prints "true" if the ingressClassname field is supported | `.` Chart context | +| `common.ingress.certManagerRequest` | Prints "true" if required cert-manager annotations for TLS signed certificates are set in the Ingress annotations | `dict "annotations" .Values.path.to.the.ingress.annotations` | + +### Labels + +| Helper identifier | Description | Expected Input | +|-----------------------------|-----------------------------------------------------------------------------|-------------------| +| `common.labels.standard` | Return Kubernetes standard labels | `.` Chart context | +| `common.labels.matchLabels` | Labels to use on `deploy.spec.selector.matchLabels` and `svc.spec.selector` | `.` Chart context | + +### Names + +| Helper identifier | Description | Expected Input | +|-----------------------------------|-----------------------------------------------------------------------|-------------------| +| `common.names.name` | Expand the name of the chart or use `.Values.nameOverride` | `.` Chart context | +| `common.names.fullname` | Create a default fully qualified app name. | `.` Chart context | +| `common.names.namespace` | Allow the release namespace to be overridden | `.` Chart context | +| `common.names.fullname.namespace` | Create a fully qualified app name adding the installation's namespace | `.` Chart context | +| `common.names.chart` | Chart name plus version | `.` Chart context | + +### Secrets + +| Helper identifier | Description | Expected Input | +|---------------------------|--------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.secrets.name` | Generate the name of the secret. | `dict "existingSecret" .Values.path.to.the.existingSecret "defaultNameSuffix" "mySuffix" "context" $` see [ExistingSecret](#existingsecret) for the structure. | +| `common.secrets.key` | Generate secret key. | `dict "existingSecret" .Values.path.to.the.existingSecret "key" "keyName"` see [ExistingSecret](#existingsecret) for the structure. | +| `common.passwords.manage` | Generate secret password or retrieve one if already created. | `dict "secret" "secret-name" "key" "keyName" "providedValues" (list "path.to.password1" "path.to.password2") "length" 10 "strong" false "chartName" "chartName" "context" $`, length, strong and chartNAme fields are optional. | +| `common.secrets.exists` | Returns whether a previous generated secret already exists. | `dict "secret" "secret-name" "context" $` | + +### Storage + +| Helper identifier | Description | Expected Input | +|-------------------------------|---------------------------------------|---------------------------------------------------------------------------------------------------------------------| +| `common.storage.class` | Return the proper Storage Class | `dict "persistence" .Values.path.to.the.persistence "global" $`, see [Persistence](#persistence) for the structure. | + +### TplValues + +| Helper identifier | Description | Expected Input | +|---------------------------|----------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.tplvalues.render` | Renders a value that contains template | `dict "value" .Values.path.to.the.Value "context" $`, value is the value should rendered as template, context frequently is the chart context `$` or `.` | + +### Utils + +| Helper identifier | Description | Expected Input | +|--------------------------------|------------------------------------------------------------------------------------------|------------------------------------------------------------------------| +| `common.utils.fieldToEnvVar` | Build environment variable name given a field. | `dict "field" "my-password"` | +| `common.utils.secret.getvalue` | Print instructions to get a secret value. | `dict "secret" "secret-name" "field" "secret-value-field" "context" $` | +| `common.utils.getValueFromKey` | Gets a value from `.Values` object given its key path | `dict "key" "path.to.key" "context" $` | +| `common.utils.getKeyFromList` | Returns first `.Values` key with a defined value or first of the list if all non-defined | `dict "keys" (list "path.to.key1" "path.to.key2") "context" $` | + +### Validations + +| Helper identifier | Description | Expected Input | +|--------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.validations.values.single.empty` | Validate a value must not be empty. | `dict "valueKey" "path.to.value" "secret" "secret.name" "field" "my-password" "subchart" "subchart" "context" $` secret, field and subchart are optional. In case they are given, the helper will generate a how to get instruction. See [ValidateValue](#validatevalue) | +| `common.validations.values.multiple.empty` | Validate a multiple values must not be empty. It returns a shared error for all the values. | `dict "required" (list $validateValueConf00 $validateValueConf01) "context" $`. See [ValidateValue](#validatevalue) | +| `common.validations.values.mariadb.passwords` | This helper will ensure required password for MariaDB are not empty. It returns a shared error for all the values. | `dict "secret" "mariadb-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use mariadb chart and the helper. | +| `common.validations.values.mysql.passwords` | This helper will ensure required password for MySQL are not empty. It returns a shared error for all the values. | `dict "secret" "mysql-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use mysql chart and the helper. | +| `common.validations.values.postgresql.passwords` | This helper will ensure required password for PostgreSQL are not empty. It returns a shared error for all the values. | `dict "secret" "postgresql-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use postgresql chart and the helper. | +| `common.validations.values.redis.passwords` | This helper will ensure required password for Redis® are not empty. It returns a shared error for all the values. | `dict "secret" "redis-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use redis chart and the helper. | +| `common.validations.values.cassandra.passwords` | This helper will ensure required password for Cassandra are not empty. It returns a shared error for all the values. | `dict "secret" "cassandra-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use cassandra chart and the helper. | +| `common.validations.values.mongodb.passwords` | This helper will ensure required password for MongoDB® are not empty. It returns a shared error for all the values. | `dict "secret" "mongodb-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use mongodb chart and the helper. | + +### Warnings + +| Helper identifier | Description | Expected Input | +|------------------------------|----------------------------------|------------------------------------------------------------| +| `common.warnings.rollingTag` | Warning about using rolling tag. | `ImageRoot` see [ImageRoot](#imageroot) for the structure. | + +## Special input schemas + +### ImageRoot + +```yaml +registry: + type: string + description: Docker registry where the image is located + example: docker.io + +repository: + type: string + description: Repository and image name + example: bitnami/nginx + +tag: + type: string + description: image tag + example: 1.16.1-debian-10-r63 + +pullPolicy: + type: string + description: Specify a imagePullPolicy. Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + +pullSecrets: + type: array + items: + type: string + description: Optionally specify an array of imagePullSecrets (evaluated as templates). + +debug: + type: boolean + description: Set to true if you would like to see extra information on logs + example: false + +## An instance would be: +# registry: docker.io +# repository: bitnami/nginx +# tag: 1.16.1-debian-10-r63 +# pullPolicy: IfNotPresent +# debug: false +``` + +### Persistence + +```yaml +enabled: + type: boolean + description: Whether enable persistence. + example: true + +storageClass: + type: string + description: Ghost data Persistent Volume Storage Class, If set to "-", storageClassName: "" which disables dynamic provisioning. + example: "-" + +accessMode: + type: string + description: Access mode for the Persistent Volume Storage. + example: ReadWriteOnce + +size: + type: string + description: Size the Persistent Volume Storage. + example: 8Gi + +path: + type: string + description: Path to be persisted. + example: /bitnami + +## An instance would be: +# enabled: true +# storageClass: "-" +# accessMode: ReadWriteOnce +# size: 8Gi +# path: /bitnami +``` + +### ExistingSecret + +```yaml +name: + type: string + description: Name of the existing secret. + example: mySecret +keyMapping: + description: Mapping between the expected key name and the name of the key in the existing secret. + type: object + +## An instance would be: +# name: mySecret +# keyMapping: +# password: myPasswordKey +``` + +#### Example of use + +When we store sensitive data for a deployment in a secret, some times we want to give to users the possibility of using theirs existing secrets. + +```yaml +# templates/secret.yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "common.names.fullname" . }} + labels: + app: {{ include "common.names.fullname" . }} +type: Opaque +data: + password: {{ .Values.password | b64enc | quote }} + +# templates/dpl.yaml +--- +... + env: + - name: PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "common.secrets.name" (dict "existingSecret" .Values.existingSecret "context" $) }} + key: {{ include "common.secrets.key" (dict "existingSecret" .Values.existingSecret "key" "password") }} +... + +# values.yaml +--- +name: mySecret +keyMapping: + password: myPasswordKey +``` + +### ValidateValue + +#### NOTES.txt + +```console +{{- $validateValueConf00 := (dict "valueKey" "path.to.value00" "secret" "secretName" "field" "password-00") -}} +{{- $validateValueConf01 := (dict "valueKey" "path.to.value01" "secret" "secretName" "field" "password-01") -}} + +{{ include "common.validations.values.multiple.empty" (dict "required" (list $validateValueConf00 $validateValueConf01) "context" $) }} +``` + +If we force those values to be empty we will see some alerts + +```console +$ helm install test mychart --set path.to.value00="",path.to.value01="" + 'path.to.value00' must not be empty, please add '--set path.to.value00=$PASSWORD_00' to the command. To get the current value: + + export PASSWORD_00=$(kubectl get secret --namespace default secretName -o jsonpath="{.data.password-00}" | base64 -d) + + 'path.to.value01' must not be empty, please add '--set path.to.value01=$PASSWORD_01' to the command. To get the current value: + + export PASSWORD_01=$(kubectl get secret --namespace default secretName -o jsonpath="{.data.password-01}" | base64 -d) +``` + +## Upgrading + +### To 1.0.0 + +[On November 13, 2020, Helm v2 support was formally finished](https://github.com/helm/charts#status-of-the-project), this major version is the result of the required changes applied to the Helm Chart to be able to incorporate the different features added in Helm v3 and to be consistent with the Helm project itself regarding the Helm v2 EOL. + +**What changes were introduced in this major version?** + +- Previous versions of this Helm Chart use `apiVersion: v1` (installable by both Helm 2 and 3), this Helm Chart was updated to `apiVersion: v2` (installable by Helm 3 only). [Here](https://helm.sh/docs/topics/charts/#the-apiversion-field) you can find more information about the `apiVersion` field. +- Use `type: library`. [Here](https://v3.helm.sh/docs/faq/#library-chart-support) you can find more information. +- The different fields present in the *Chart.yaml* file has been ordered alphabetically in a homogeneous way for all the Bitnami Helm Charts + +**Considerations when upgrading to this version** + +- If you want to upgrade to this version from a previous one installed with Helm v3, you shouldn't face any issues +- If you want to upgrade to this version using Helm v2, this scenario is not supported as this version doesn't support Helm v2 anymore +- If you installed the previous version with Helm v2 and wants to upgrade to this version with Helm v3, please refer to the [official Helm documentation](https://helm.sh/docs/topics/v2_v3_migration/#migration-use-cases) about migrating from Helm v2 to v3 + +**Useful links** + +- https://docs.bitnami.com/tutorials/resolve-helm2-helm3-post-migration-issues/ +- https://helm.sh/docs/topics/v2_v3_migration/ +- https://helm.sh/blog/migrate-from-helm-v2-to-helm-v3/ + +## License + +Copyright © 2022 Bitnami + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_affinities.tpl b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_affinities.tpl new file mode 100644 index 00000000..2387be26 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_affinities.tpl @@ -0,0 +1,102 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Return a soft nodeAffinity definition +{{ include "common.affinities.nodes.soft" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.soft" -}} +preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . | quote }} + {{- end }} + weight: 1 +{{- end -}} + +{{/* +Return a hard nodeAffinity definition +{{ include "common.affinities.nodes.hard" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.hard" -}} +requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . | quote }} + {{- end }} +{{- end -}} + +{{/* +Return a nodeAffinity definition +{{ include "common.affinities.nodes" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.nodes.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.nodes.hard" . -}} + {{- end -}} +{{- end -}} + +{{/* +Return a soft podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.soft" (dict "component" "FOO" "extraMatchLabels" .Values.extraMatchLabels "context" $) -}} +*/}} +{{- define "common.affinities.pods.soft" -}} +{{- $component := default "" .component -}} +{{- $extraMatchLabels := default (dict) .extraMatchLabels -}} +preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" .context) | nindent 10 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := $extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + namespaces: + - {{ include "common.names.namespace" .context | quote }} + topologyKey: kubernetes.io/hostname + weight: 1 +{{- end -}} + +{{/* +Return a hard podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.hard" (dict "component" "FOO" "extraMatchLabels" .Values.extraMatchLabels "context" $) -}} +*/}} +{{- define "common.affinities.pods.hard" -}} +{{- $component := default "" .component -}} +{{- $extraMatchLabels := default (dict) .extraMatchLabels -}} +requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" .context) | nindent 8 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := $extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + namespaces: + - {{ include "common.names.namespace" .context | quote }} + topologyKey: kubernetes.io/hostname +{{- end -}} + +{{/* +Return a podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.pods" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.pods.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.pods.hard" . -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_capabilities.tpl b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_capabilities.tpl new file mode 100644 index 00000000..9d9b7600 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_capabilities.tpl @@ -0,0 +1,154 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Return the target Kubernetes version +*/}} +{{- define "common.capabilities.kubeVersion" -}} +{{- if .Values.global }} + {{- if .Values.global.kubeVersion }} + {{- .Values.global.kubeVersion -}} + {{- else }} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} + {{- end -}} +{{- else }} +{{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for poddisruptionbudget. +*/}} +{{- define "common.capabilities.policy.apiVersion" -}} +{{- if semverCompare "<1.21-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "policy/v1beta1" -}} +{{- else -}} +{{- print "policy/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for networkpolicy. +*/}} +{{- define "common.capabilities.networkPolicy.apiVersion" -}} +{{- if semverCompare "<1.7-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for cronjob. +*/}} +{{- define "common.capabilities.cronjob.apiVersion" -}} +{{- if semverCompare "<1.21-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "batch/v1beta1" -}} +{{- else -}} +{{- print "batch/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for deployment. +*/}} +{{- define "common.capabilities.deployment.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for statefulset. +*/}} +{{- define "common.capabilities.statefulset.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apps/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "common.capabilities.ingress.apiVersion" -}} +{{- if .Values.ingress -}} +{{- if .Values.ingress.apiVersion -}} +{{- .Values.ingress.apiVersion -}} +{{- else if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end }} +{{- else if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for RBAC resources. +*/}} +{{- define "common.capabilities.rbac.apiVersion" -}} +{{- if semverCompare "<1.17-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "rbac.authorization.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "rbac.authorization.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for CRDs. +*/}} +{{- define "common.capabilities.crd.apiVersion" -}} +{{- if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiextensions.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiextensions.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for APIService. +*/}} +{{- define "common.capabilities.apiService.apiVersion" -}} +{{- if semverCompare "<1.10-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiregistration.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiregistration.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for Horizontal Pod Autoscaler. +*/}} +{{- define "common.capabilities.hpa.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .context) -}} +{{- if .beta2 -}} +{{- print "autoscaling/v2beta2" -}} +{{- else -}} +{{- print "autoscaling/v2beta1" -}} +{{- end -}} +{{- else -}} +{{- print "autoscaling/v2" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if the used Helm version is 3.3+. +A way to check the used Helm version was not introduced until version 3.3.0 with .Capabilities.HelmVersion, which contains an additional "{}}" structure. +This check is introduced as a regexMatch instead of {{ if .Capabilities.HelmVersion }} because checking for the key HelmVersion in <3.3 results in a "interface not found" error. +**To be removed when the catalog's minimun Helm version is 3.3** +*/}} +{{- define "common.capabilities.supportsHelmVersion" -}} +{{- if regexMatch "{(v[0-9])*[^}]*}}$" (.Capabilities | toString ) }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_errors.tpl b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_errors.tpl new file mode 100644 index 00000000..a79cc2e3 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_errors.tpl @@ -0,0 +1,23 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Through error when upgrading using empty passwords values that must not be empty. + +Usage: +{{- $validationError00 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password00" "secret" "secretName" "field" "password-00") -}} +{{- $validationError01 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password01" "secret" "secretName" "field" "password-01") -}} +{{ include "common.errors.upgrade.passwords.empty" (dict "validationErrors" (list $validationError00 $validationError01) "context" $) }} + +Required password params: + - validationErrors - String - Required. List of validation strings to be return, if it is empty it won't throw error. + - context - Context - Required. Parent context. +*/}} +{{- define "common.errors.upgrade.passwords.empty" -}} + {{- $validationErrors := join "" .validationErrors -}} + {{- if and $validationErrors .context.Release.IsUpgrade -}} + {{- $errorString := "\nPASSWORDS ERROR: You must provide your current passwords when upgrading the release." -}} + {{- $errorString = print $errorString "\n Note that even after reinstallation, old credentials may be needed as they may be kept in persistent volume claims." -}} + {{- $errorString = print $errorString "\n Further information can be obtained at https://docs.bitnami.com/general/how-to/troubleshoot-helm-chart-issues/#credential-errors-while-upgrading-chart-releases" -}} + {{- $errorString = print $errorString "\n%s" -}} + {{- printf $errorString $validationErrors | fail -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_images.tpl b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_images.tpl new file mode 100644 index 00000000..42ffbc72 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_images.tpl @@ -0,0 +1,75 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper image name +{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }} +*/}} +{{- define "common.images.image" -}} +{{- $registryName := .imageRoot.registry -}} +{{- $repositoryName := .imageRoot.repository -}} +{{- $tag := .imageRoot.tag | toString -}} +{{- if .global }} + {{- if .global.imageRegistry }} + {{- $registryName = .global.imageRegistry -}} + {{- end -}} +{{- end -}} +{{- if $registryName }} +{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- else -}} +{{- printf "%s:%s" $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) +{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} +*/}} +{{- define "common.images.pullSecrets" -}} + {{- $pullSecrets := list }} + + {{- if .global }} + {{- range .global.imagePullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names evaluating values as templates +{{ include "common.images.renderPullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "context" $) }} +*/}} +{{- define "common.images.renderPullSecrets" -}} + {{- $pullSecrets := list }} + {{- $context := .context }} + + {{- if $context.Values.global }} + {{- range $context.Values.global.imagePullSecrets -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" $context)) -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" $context)) -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_ingress.tpl b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_ingress.tpl new file mode 100644 index 00000000..8caf73a6 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_ingress.tpl @@ -0,0 +1,68 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Generate backend entry that is compatible with all Kubernetes API versions. + +Usage: +{{ include "common.ingress.backend" (dict "serviceName" "backendName" "servicePort" "backendPort" "context" $) }} + +Params: + - serviceName - String. Name of an existing service backend + - servicePort - String/Int. Port name (or number) of the service. It will be translated to different yaml depending if it is a string or an integer. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.ingress.backend" -}} +{{- $apiVersion := (include "common.capabilities.ingress.apiVersion" .context) -}} +{{- if or (eq $apiVersion "extensions/v1beta1") (eq $apiVersion "networking.k8s.io/v1beta1") -}} +serviceName: {{ .serviceName }} +servicePort: {{ .servicePort }} +{{- else -}} +service: + name: {{ .serviceName }} + port: + {{- if typeIs "string" .servicePort }} + name: {{ .servicePort }} + {{- else if or (typeIs "int" .servicePort) (typeIs "float64" .servicePort) }} + number: {{ .servicePort | int }} + {{- end }} +{{- end -}} +{{- end -}} + +{{/* +Print "true" if the API pathType field is supported +Usage: +{{ include "common.ingress.supportsPathType" . }} +*/}} +{{- define "common.ingress.supportsPathType" -}} +{{- if (semverCompare "<1.18-0" (include "common.capabilities.kubeVersion" .)) -}} +{{- print "false" -}} +{{- else -}} +{{- print "true" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if the ingressClassname field is supported +Usage: +{{ include "common.ingress.supportsIngressClassname" . }} +*/}} +{{- define "common.ingress.supportsIngressClassname" -}} +{{- if semverCompare "<1.18-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "false" -}} +{{- else -}} +{{- print "true" -}} +{{- end -}} +{{- end -}} + +{{/* +Return true if cert-manager required annotations for TLS signed +certificates are set in the Ingress annotations +Ref: https://cert-manager.io/docs/usage/ingress/#supported-annotations +Usage: +{{ include "common.ingress.certManagerRequest" ( dict "annotations" .Values.path.to.the.ingress.annotations ) }} +*/}} +{{- define "common.ingress.certManagerRequest" -}} +{{ if or (hasKey .annotations "cert-manager.io/cluster-issuer") (hasKey .annotations "cert-manager.io/issuer") }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_labels.tpl b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_labels.tpl new file mode 100644 index 00000000..252066c7 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_labels.tpl @@ -0,0 +1,18 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Kubernetes standard labels +*/}} +{{- define "common.labels.standard" -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +helm.sh/chart: {{ include "common.names.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Labels to use on deploy.spec.selector.matchLabels and svc.spec.selector +*/}} +{{- define "common.labels.matchLabels" -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_names.tpl b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_names.tpl new file mode 100644 index 00000000..1bdac8b7 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_names.tpl @@ -0,0 +1,70 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "common.names.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "common.names.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "common.names.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a default fully qualified dependency name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +Usage: +{{ include "common.names.dependency.fullname" (dict "chartName" "dependency-chart-name" "chartValues" .Values.dependency-chart "context" $) }} +*/}} +{{- define "common.names.dependency.fullname" -}} +{{- if .chartValues.fullnameOverride -}} +{{- .chartValues.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .chartName .chartValues.nameOverride -}} +{{- if contains $name .context.Release.Name -}} +{{- .context.Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .context.Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts. +*/}} +{{- define "common.names.namespace" -}} +{{- if .Values.namespaceOverride -}} +{{- .Values.namespaceOverride -}} +{{- else -}} +{{- .Release.Namespace -}} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified app name adding the installation's namespace. +*/}} +{{- define "common.names.fullname.namespace" -}} +{{- printf "%s-%s" (include "common.names.fullname" .) (include "common.names.namespace" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_secrets.tpl b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_secrets.tpl new file mode 100644 index 00000000..a53fb44f --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_secrets.tpl @@ -0,0 +1,140 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Generate secret name. + +Usage: +{{ include "common.secrets.name" (dict "existingSecret" .Values.path.to.the.existingSecret "defaultNameSuffix" "mySuffix" "context" $) }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/master/bitnami/common#existingsecret + - defaultNameSuffix - String - Optional. It is used only if we have several secrets in the same deployment. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.secrets.name" -}} +{{- $name := (include "common.names.fullname" .context) -}} + +{{- if .defaultNameSuffix -}} +{{- $name = printf "%s-%s" $name .defaultNameSuffix | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- with .existingSecret -}} +{{- if not (typeIs "string" .) -}} +{{- with .name -}} +{{- $name = . -}} +{{- end -}} +{{- else -}} +{{- $name = . -}} +{{- end -}} +{{- end -}} + +{{- printf "%s" $name -}} +{{- end -}} + +{{/* +Generate secret key. + +Usage: +{{ include "common.secrets.key" (dict "existingSecret" .Values.path.to.the.existingSecret "key" "keyName") }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/master/bitnami/common#existingsecret + - key - String - Required. Name of the key in the secret. +*/}} +{{- define "common.secrets.key" -}} +{{- $key := .key -}} + +{{- if .existingSecret -}} + {{- if not (typeIs "string" .existingSecret) -}} + {{- if .existingSecret.keyMapping -}} + {{- $key = index .existingSecret.keyMapping $.key -}} + {{- end -}} + {{- end }} +{{- end -}} + +{{- printf "%s" $key -}} +{{- end -}} + +{{/* +Generate secret password or retrieve one if already created. + +Usage: +{{ include "common.secrets.passwords.manage" (dict "secret" "secret-name" "key" "keyName" "providedValues" (list "path.to.password1" "path.to.password2") "length" 10 "strong" false "chartName" "chartName" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - key - String - Required - Name of the key in the secret. + - providedValues - List - Required - The path to the validating value in the values.yaml, e.g: "mysql.password". Will pick first parameter with a defined value. + - length - int - Optional - Length of the generated random password. + - strong - Boolean - Optional - Whether to add symbols to the generated random password. + - chartName - String - Optional - Name of the chart used when said chart is deployed as a subchart. + - context - Context - Required - Parent context. + +The order in which this function returns a secret password: + 1. Already existing 'Secret' resource + (If a 'Secret' resource is found under the name provided to the 'secret' parameter to this function and that 'Secret' resource contains a key with the name passed as the 'key' parameter to this function then the value of this existing secret password will be returned) + 2. Password provided via the values.yaml + (If one of the keys passed to the 'providedValues' parameter to this function is a valid path to a key in the values.yaml and has a value, the value of the first key with a value will be returned) + 3. Randomly generated secret password + (A new random secret password with the length specified in the 'length' parameter will be generated and returned) + +*/}} +{{- define "common.secrets.passwords.manage" -}} + +{{- $password := "" }} +{{- $subchart := "" }} +{{- $chartName := default "" .chartName }} +{{- $passwordLength := default 10 .length }} +{{- $providedPasswordKey := include "common.utils.getKeyFromList" (dict "keys" .providedValues "context" $.context) }} +{{- $providedPasswordValue := include "common.utils.getValueFromKey" (dict "key" $providedPasswordKey "context" $.context) }} +{{- $secretData := (lookup "v1" "Secret" $.context.Release.Namespace .secret).data }} +{{- if $secretData }} + {{- if hasKey $secretData .key }} + {{- $password = index $secretData .key }} + {{- else }} + {{- printf "\nPASSWORDS ERROR: The secret \"%s\" does not contain the key \"%s\"\n" .secret .key | fail -}} + {{- end -}} +{{- else if $providedPasswordValue }} + {{- $password = $providedPasswordValue | toString | b64enc | quote }} +{{- else }} + + {{- if .context.Values.enabled }} + {{- $subchart = $chartName }} + {{- end -}} + + {{- $requiredPassword := dict "valueKey" $providedPasswordKey "secret" .secret "field" .key "subchart" $subchart "context" $.context -}} + {{- $requiredPasswordError := include "common.validations.values.single.empty" $requiredPassword -}} + {{- $passwordValidationErrors := list $requiredPasswordError -}} + {{- include "common.errors.upgrade.passwords.empty" (dict "validationErrors" $passwordValidationErrors "context" $.context) -}} + + {{- if .strong }} + {{- $subStr := list (lower (randAlpha 1)) (randNumeric 1) (upper (randAlpha 1)) | join "_" }} + {{- $password = randAscii $passwordLength }} + {{- $password = regexReplaceAllLiteral "\\W" $password "@" | substr 5 $passwordLength }} + {{- $password = printf "%s%s" $subStr $password | toString | shuffle | b64enc | quote }} + {{- else }} + {{- $password = randAlphaNum $passwordLength | b64enc | quote }} + {{- end }} +{{- end -}} +{{- printf "%s" $password -}} +{{- end -}} + +{{/* +Returns whether a previous generated secret already exists + +Usage: +{{ include "common.secrets.exists" (dict "secret" "secret-name" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - context - Context - Required - Parent context. +*/}} +{{- define "common.secrets.exists" -}} +{{- $secret := (lookup "v1" "Secret" $.context.Release.Namespace .secret) }} +{{- if $secret }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_storage.tpl b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_storage.tpl new file mode 100644 index 00000000..60e2a844 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_storage.tpl @@ -0,0 +1,23 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper Storage Class +{{ include "common.storage.class" ( dict "persistence" .Values.path.to.the.persistence "global" $) }} +*/}} +{{- define "common.storage.class" -}} + +{{- $storageClass := .persistence.storageClass -}} +{{- if .global -}} + {{- if .global.storageClass -}} + {{- $storageClass = .global.storageClass -}} + {{- end -}} +{{- end -}} + +{{- if $storageClass -}} + {{- if (eq "-" $storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" $storageClass -}} + {{- end -}} +{{- end -}} + +{{- end -}} diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_tplvalues.tpl b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_tplvalues.tpl new file mode 100644 index 00000000..2db16685 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_tplvalues.tpl @@ -0,0 +1,13 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Renders a value that contains template. +Usage: +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $) }} +*/}} +{{- define "common.tplvalues.render" -}} + {{- if typeIs "string" .value }} + {{- tpl .value .context }} + {{- else }} + {{- tpl (.value | toYaml) .context }} + {{- end }} +{{- end -}} diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_utils.tpl b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_utils.tpl new file mode 100644 index 00000000..8c22b2a3 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_utils.tpl @@ -0,0 +1,62 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Print instructions to get a secret value. +Usage: +{{ include "common.utils.secret.getvalue" (dict "secret" "secret-name" "field" "secret-value-field" "context" $) }} +*/}} +{{- define "common.utils.secret.getvalue" -}} +{{- $varname := include "common.utils.fieldToEnvVar" . -}} +export {{ $varname }}=$(kubectl get secret --namespace {{ .context.Release.Namespace | quote }} {{ .secret }} -o jsonpath="{.data.{{ .field }}}" | base64 -d) +{{- end -}} + +{{/* +Build env var name given a field +Usage: +{{ include "common.utils.fieldToEnvVar" dict "field" "my-password" }} +*/}} +{{- define "common.utils.fieldToEnvVar" -}} + {{- $fieldNameSplit := splitList "-" .field -}} + {{- $upperCaseFieldNameSplit := list -}} + + {{- range $fieldNameSplit -}} + {{- $upperCaseFieldNameSplit = append $upperCaseFieldNameSplit ( upper . ) -}} + {{- end -}} + + {{ join "_" $upperCaseFieldNameSplit }} +{{- end -}} + +{{/* +Gets a value from .Values given +Usage: +{{ include "common.utils.getValueFromKey" (dict "key" "path.to.key" "context" $) }} +*/}} +{{- define "common.utils.getValueFromKey" -}} +{{- $splitKey := splitList "." .key -}} +{{- $value := "" -}} +{{- $latestObj := $.context.Values -}} +{{- range $splitKey -}} + {{- if not $latestObj -}} + {{- printf "please review the entire path of '%s' exists in values" $.key | fail -}} + {{- end -}} + {{- $value = ( index $latestObj . ) -}} + {{- $latestObj = $value -}} +{{- end -}} +{{- printf "%v" (default "" $value) -}} +{{- end -}} + +{{/* +Returns first .Values key with a defined value or first of the list if all non-defined +Usage: +{{ include "common.utils.getKeyFromList" (dict "keys" (list "path.to.key1" "path.to.key2") "context" $) }} +*/}} +{{- define "common.utils.getKeyFromList" -}} +{{- $key := first .keys -}} +{{- $reverseKeys := reverse .keys }} +{{- range $reverseKeys }} + {{- $value := include "common.utils.getValueFromKey" (dict "key" . "context" $.context ) }} + {{- if $value -}} + {{- $key = . }} + {{- end -}} +{{- end -}} +{{- printf "%s" $key -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_warnings.tpl b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_warnings.tpl new file mode 100644 index 00000000..ae10fa41 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/_warnings.tpl @@ -0,0 +1,14 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Warning about using rolling tag. +Usage: +{{ include "common.warnings.rollingTag" .Values.path.to.the.imageRoot }} +*/}} +{{- define "common.warnings.rollingTag" -}} + +{{- if and (contains "bitnami/" .repository) (not (.tag | toString | regexFind "-r\\d+$|sha256:")) }} +WARNING: Rolling tag detected ({{ .repository }}:{{ .tag }}), please note that it is strongly recommended to avoid using rolling tags in a production environment. ++info https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/ +{{- end }} + +{{- end -}} diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/validations/_cassandra.tpl b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/validations/_cassandra.tpl new file mode 100644 index 00000000..ded1ae3b --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/validations/_cassandra.tpl @@ -0,0 +1,72 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate Cassandra required passwords are not empty. + +Usage: +{{ include "common.validations.values.cassandra.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where Cassandra values are stored, e.g: "cassandra-passwords-secret" + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.cassandra.passwords" -}} + {{- $existingSecret := include "common.cassandra.values.existingSecret" . -}} + {{- $enabled := include "common.cassandra.values.enabled" . -}} + {{- $dbUserPrefix := include "common.cassandra.values.key.dbUser" . -}} + {{- $valueKeyPassword := printf "%s.password" $dbUserPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "cassandra-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.cassandra.values.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.cassandra.values.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.cassandra.dbUser.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.dbUser.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled cassandra. + +Usage: +{{ include "common.cassandra.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.cassandra.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.cassandra.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key dbUser + +Usage: +{{ include "common.cassandra.values.key.dbUser" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.cassandra.values.key.dbUser" -}} + {{- if .subchart -}} + cassandra.dbUser + {{- else -}} + dbUser + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/validations/_mariadb.tpl b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/validations/_mariadb.tpl new file mode 100644 index 00000000..b6906ff7 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/validations/_mariadb.tpl @@ -0,0 +1,103 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MariaDB required passwords are not empty. + +Usage: +{{ include "common.validations.values.mariadb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MariaDB values are stored, e.g: "mysql-passwords-secret" + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mariadb.passwords" -}} + {{- $existingSecret := include "common.mariadb.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mariadb.values.enabled" . -}} + {{- $architecture := include "common.mariadb.values.architecture" . -}} + {{- $authPrefix := include "common.mariadb.values.key.auth" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mariadb-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- if not (empty $valueUsername) -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mariadb-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replication") -}} + {{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mariadb-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mariadb.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mariadb. + +Usage: +{{ include "common.mariadb.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mariadb.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mariadb.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mariadb.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mariadb.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.key.auth" -}} + {{- if .subchart -}} + mariadb.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/validations/_mongodb.tpl b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/validations/_mongodb.tpl new file mode 100644 index 00000000..f820ec10 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/validations/_mongodb.tpl @@ -0,0 +1,108 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MongoDB® required passwords are not empty. + +Usage: +{{ include "common.validations.values.mongodb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MongoDB® values are stored, e.g: "mongodb-passwords-secret" + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mongodb.passwords" -}} + {{- $existingSecret := include "common.mongodb.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mongodb.values.enabled" . -}} + {{- $authPrefix := include "common.mongodb.values.key.auth" . -}} + {{- $architecture := include "common.mongodb.values.architecture" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyDatabase := printf "%s.database" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicaSetKey := printf "%s.replicaSetKey" $authPrefix -}} + {{- $valueKeyAuthEnabled := printf "%s.enabled" $authPrefix -}} + + {{- $authEnabled := include "common.utils.getValueFromKey" (dict "key" $valueKeyAuthEnabled "context" .context) -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") (eq $authEnabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mongodb-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- $valueDatabase := include "common.utils.getValueFromKey" (dict "key" $valueKeyDatabase "context" .context) }} + {{- if and $valueUsername $valueDatabase -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mongodb-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replicaset") -}} + {{- $requiredReplicaSetKey := dict "valueKey" $valueKeyReplicaSetKey "secret" .secret "field" "mongodb-replica-set-key" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicaSetKey -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mongodb.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDb is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mongodb.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mongodb. + +Usage: +{{ include "common.mongodb.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mongodb.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mongodb.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mongodb.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.key.auth" -}} + {{- if .subchart -}} + mongodb.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mongodb.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mongodb.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/validations/_mysql.tpl b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/validations/_mysql.tpl new file mode 100644 index 00000000..74472a06 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/validations/_mysql.tpl @@ -0,0 +1,103 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MySQL required passwords are not empty. + +Usage: +{{ include "common.validations.values.mysql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MySQL values are stored, e.g: "mysql-passwords-secret" + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mysql.passwords" -}} + {{- $existingSecret := include "common.mysql.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mysql.values.enabled" . -}} + {{- $architecture := include "common.mysql.values.architecture" . -}} + {{- $authPrefix := include "common.mysql.values.key.auth" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mysql-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- if not (empty $valueUsername) -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mysql-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replication") -}} + {{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mysql-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mysql.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mysql.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mysql. + +Usage: +{{ include "common.mysql.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mysql.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mysql.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mysql.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mysql.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mysql.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.key.auth" -}} + {{- if .subchart -}} + mysql.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/validations/_postgresql.tpl b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/validations/_postgresql.tpl new file mode 100644 index 00000000..164ec0d0 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/validations/_postgresql.tpl @@ -0,0 +1,129 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate PostgreSQL required passwords are not empty. + +Usage: +{{ include "common.validations.values.postgresql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where postgresql values are stored, e.g: "postgresql-passwords-secret" + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.postgresql.passwords" -}} + {{- $existingSecret := include "common.postgresql.values.existingSecret" . -}} + {{- $enabled := include "common.postgresql.values.enabled" . -}} + {{- $valueKeyPostgresqlPassword := include "common.postgresql.values.key.postgressPassword" . -}} + {{- $valueKeyPostgresqlReplicationEnabled := include "common.postgresql.values.key.replicationPassword" . -}} + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + {{- $requiredPostgresqlPassword := dict "valueKey" $valueKeyPostgresqlPassword "secret" .secret "field" "postgresql-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlPassword -}} + + {{- $enabledReplication := include "common.postgresql.values.enabled.replication" . -}} + {{- if (eq $enabledReplication "true") -}} + {{- $requiredPostgresqlReplicationPassword := dict "valueKey" $valueKeyPostgresqlReplicationEnabled "secret" .secret "field" "postgresql-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to decide whether evaluate global values. + +Usage: +{{ include "common.postgresql.values.use.global" (dict "key" "key-of-global" "context" $) }} +Params: + - key - String - Required. Field to be evaluated within global, e.g: "existingSecret" +*/}} +{{- define "common.postgresql.values.use.global" -}} + {{- if .context.Values.global -}} + {{- if .context.Values.global.postgresql -}} + {{- index .context.Values.global.postgresql .key | quote -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.postgresql.values.existingSecret" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.existingSecret" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "existingSecret" "context" .context) -}} + + {{- if .subchart -}} + {{- default (.context.Values.postgresql.existingSecret | quote) $globalValue -}} + {{- else -}} + {{- default (.context.Values.existingSecret | quote) $globalValue -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled postgresql. + +Usage: +{{ include "common.postgresql.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key postgressPassword. + +Usage: +{{ include "common.postgresql.values.key.postgressPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.postgressPassword" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "postgresqlUsername" "context" .context) -}} + + {{- if not $globalValue -}} + {{- if .subchart -}} + postgresql.postgresqlPassword + {{- else -}} + postgresqlPassword + {{- end -}} + {{- else -}} + global.postgresql.postgresqlPassword + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled.replication. + +Usage: +{{ include "common.postgresql.values.enabled.replication" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.enabled.replication" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.replication.enabled -}} + {{- else -}} + {{- printf "%v" .context.Values.replication.enabled -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key replication.password. + +Usage: +{{ include "common.postgresql.values.key.replicationPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.replicationPassword" -}} + {{- if .subchart -}} + postgresql.replication.password + {{- else -}} + replication.password + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/validations/_redis.tpl b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/validations/_redis.tpl new file mode 100644 index 00000000..dcccfc1a --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/validations/_redis.tpl @@ -0,0 +1,76 @@ + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate Redis® required passwords are not empty. + +Usage: +{{ include "common.validations.values.redis.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where redis values are stored, e.g: "redis-passwords-secret" + - subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.redis.passwords" -}} + {{- $enabled := include "common.redis.values.enabled" . -}} + {{- $valueKeyPrefix := include "common.redis.values.keys.prefix" . -}} + {{- $standarizedVersion := include "common.redis.values.standarized.version" . }} + + {{- $existingSecret := ternary (printf "%s%s" $valueKeyPrefix "auth.existingSecret") (printf "%s%s" $valueKeyPrefix "existingSecret") (eq $standarizedVersion "true") }} + {{- $existingSecretValue := include "common.utils.getValueFromKey" (dict "key" $existingSecret "context" .context) }} + + {{- $valueKeyRedisPassword := ternary (printf "%s%s" $valueKeyPrefix "auth.password") (printf "%s%s" $valueKeyPrefix "password") (eq $standarizedVersion "true") }} + {{- $valueKeyRedisUseAuth := ternary (printf "%s%s" $valueKeyPrefix "auth.enabled") (printf "%s%s" $valueKeyPrefix "usePassword") (eq $standarizedVersion "true") }} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $useAuth := include "common.utils.getValueFromKey" (dict "key" $valueKeyRedisUseAuth "context" .context) -}} + {{- if eq $useAuth "true" -}} + {{- $requiredRedisPassword := dict "valueKey" $valueKeyRedisPassword "secret" .secret "field" "redis-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRedisPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled redis. + +Usage: +{{ include "common.redis.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.redis.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.redis.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right prefix path for the values + +Usage: +{{ include "common.redis.values.key.prefix" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false +*/}} +{{- define "common.redis.values.keys.prefix" -}} + {{- if .subchart -}}redis.{{- else -}}{{- end -}} +{{- end -}} + +{{/* +Checks whether the redis chart's includes the standarizations (version >= 14) + +Usage: +{{ include "common.redis.values.standarized.version" (dict "context" $) }} +*/}} +{{- define "common.redis.values.standarized.version" -}} + + {{- $standarizedAuth := printf "%s%s" (include "common.redis.values.keys.prefix" .) "auth" -}} + {{- $standarizedAuthValues := include "common.utils.getValueFromKey" (dict "key" $standarizedAuth "context" .context) }} + + {{- if $standarizedAuthValues -}} + {{- true -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/validations/_validations.tpl b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/validations/_validations.tpl new file mode 100644 index 00000000..9a814cf4 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/templates/validations/_validations.tpl @@ -0,0 +1,46 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate values must not be empty. + +Usage: +{{- $validateValueConf00 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-00") -}} +{{- $validateValueConf01 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-01") -}} +{{ include "common.validations.values.empty" (dict "required" (list $validateValueConf00 $validateValueConf01) "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" +*/}} +{{- define "common.validations.values.multiple.empty" -}} + {{- range .required -}} + {{- include "common.validations.values.single.empty" (dict "valueKey" .valueKey "secret" .secret "field" .field "context" $.context) -}} + {{- end -}} +{{- end -}} + +{{/* +Validate a value must not be empty. + +Usage: +{{ include "common.validations.value.empty" (dict "valueKey" "mariadb.password" "secret" "secretName" "field" "my-password" "subchart" "subchart" "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" + - subchart - String - Optional - Name of the subchart that the validated password is part of. +*/}} +{{- define "common.validations.values.single.empty" -}} + {{- $value := include "common.utils.getValueFromKey" (dict "key" .valueKey "context" .context) }} + {{- $subchart := ternary "" (printf "%s." .subchart) (empty .subchart) }} + + {{- if not $value -}} + {{- $varname := "my-value" -}} + {{- $getCurrentValue := "" -}} + {{- if and .secret .field -}} + {{- $varname = include "common.utils.fieldToEnvVar" . -}} + {{- $getCurrentValue = printf " To get the current value:\n\n %s\n" (include "common.utils.secret.getvalue" .) -}} + {{- end -}} + {{- printf "\n '%s' must not be empty, please add '--set %s%s=$%s' to the command.%s" .valueKey $subchart .valueKey $varname $getCurrentValue -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/values.yaml b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/values.yaml new file mode 100644 index 00000000..f2df68e5 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/charts/common/values.yaml @@ -0,0 +1,5 @@ +## bitnami/common +## It is required by CI/CD tools and processes. +## @skip exampleValue +## +exampleValue: common-chart diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/templates/_helpers.tpl b/helm/airbyte-v1/charts/connector-rollout-worker/templates/_helpers.tpl new file mode 100644 index 00000000..7be7bc1a --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/templates/_helpers.tpl @@ -0,0 +1,51 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "airbyte.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airbyte.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airbyte.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "airbyte.labels" -}} +helm.sh/chart: {{ include "airbyte.chart" . }} +{{ include "airbyte.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "airbyte.selectorLabels" -}} +app.kubernetes.io/name: {{ include "airbyte.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} \ No newline at end of file diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/templates/_images.tpl b/helm/airbyte-v1/charts/connector-rollout-worker/templates/_images.tpl new file mode 100644 index 00000000..6c32ec44 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/templates/_images.tpl @@ -0,0 +1,17 @@ +{{- define "imageUrl" -}} +{{- $img := index . 0 -}} +{{- $root := index . 1 -}} + +{{- $reg := $root.Values.global.image.registry -}} +{{/* ensure the registry has a trailing slash, if set */}} +{{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} + +{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{- if (eq (typeOf $img) "string") -}} +{{- printf "%s%s" $reg (tpl $img $root) | quote -}} +{{- else -}} +{{- $tag := coalesce $img.tag $root.Values.global.image.tag $root.Chart.AppVersion -}} +{{- printf "%s%s:%s" $reg $img.repository $tag | quote -}} +{{- end -}} + +{{- end -}} \ No newline at end of file diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/templates/deployment.yaml b/helm/airbyte-v1/charts/connector-rollout-worker/templates/deployment.yaml new file mode 100644 index 00000000..f1094295 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/templates/deployment.yaml @@ -0,0 +1,216 @@ +{{- if or (eq .Values.global.edition "community") (eq .Values.global.enterprise.connectorRolloutOptIn true)}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.names.fullname" . }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- if .Values.extraLabels }} + {{ toYaml (mergeOverwrite .Values.extraLabels .Values.global.extraLabels) | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "airbyte.selectorLabels" . | nindent 6 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 6 }} + {{- end }} + strategy: + type: Recreate + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 8 }} + {{- end }} + {{- if .Values.podLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ .Values.global.serviceAccountName }} + automountServiceAccountToken: true + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.extraInitContainers }} + initContainers: + {{- toYaml .Values.extraInitContainers | nindent 6 }} + {{- end }} + containers: + - name: airbyte-connector-rollout-worker-container + image: {{ include "imageUrl" (list .Values.image $) }} + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + env: + {{- if .Values.debug.enabled }} + - name: JAVA_TOOL_OPTIONS + value: "-Xdebug -agentlib:jdwp=transport=dt_socket,address=0.0.0.0:{{ .Values.debug.remoteDebugPort }},server=y,suspend=n" + {{- end }} + {{- if eq .Values.global.deploymentMode "oss" }} + - name: AIRBYTE_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_VERSION + - name: INTERNAL_API_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: INTERNAL_API_HOST + - name: MICROMETER_METRICS_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MICROMETER_METRICS_ENABLED + - name: MICROMETER_METRICS_STATSD_FLAVOR + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MICROMETER_METRICS_STATSD_FLAVOR + - name: SEGMENT_WRITE_KEY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: SEGMENT_WRITE_KEY + - name: STATSD_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: STATSD_HOST + - name: STATSD_PORT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: STATSD_PORT + - name: TRACKING_STRATEGY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TRACKING_STRATEGY + - name: WEBAPP_URL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WEBAPP_URL + - name: TEMPORAL_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TEMPORAL_HOST + - name: TEMPORAL_WORKER_PORTS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TEMPORAL_WORKER_PORTS + - name: LOG_LEVEL + value: "{{ .Values.log.level }}" + {{- end }} + {{- if or (eq .Values.global.edition "pro") (eq .Values.global.edition "enterprise") }} + - name: AIRBYTE_API_AUTH_HEADER_NAME + value: "X-Airbyte-Auth" + - name: AIRBYTE_API_AUTH_HEADER_VALUE + value: "Internal worker" + {{- else if and (eq .Values.global.deploymentMode "oss") .Values.global.auth.enabled }} + # Self-Managed Enterprise and Community w/ auth enabled use the same auth header, just + # splitting into two separate blocks for readability. + - name: AIRBYTE_API_AUTH_HEADER_NAME + value: "X-Airbyte-Auth" + - name: AIRBYTE_API_AUTH_HEADER_VALUE + value: "Internal worker" + {{- end }} + + # Values from secret + {{- if .Values.secrets }} + {{- range $k, $v := .Values.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: connector-rollout-worker-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if or .Values.env_vars .Values.global.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + {{- if .Values.extraEnv }} + {{- toYaml .Values.extraEnv | nindent 8 }} + {{- end }} + + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /health/liveness + port: heartbeat + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /health/readiness + port: heartbeat + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + ports: + {{- if .Values.debug.enabled }} + - name: debug + containerPort: {{ .Values.debug.remoteDebugPort }} + protocol: TCP + {{- end}} + - name: heartbeat + containerPort: 8016 # for heartbeat server + + {{- if .Values.resources }} + resources: {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + {{- if .Values.containerSecurityContext }} + securityContext: {{- toYaml .Values.containerSecurityContext | nindent 10 }} + {{- end }} + volumeMounts: + {{- if .Values.extraVolumeMounts }} + {{ toYaml .Values.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.global.extraVolumeMounts }} + {{ toYaml .Values.global.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.extraContainers }} + {{ toYaml .Values.extraContainers | nindent 6 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 6 }} + {{- end }} + securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + volumes: + {{- if .Values.extraVolumes }} + {{ toYaml .Values.extraVolumes | nindent 6 }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/templates/secrets.yaml b/helm/airbyte-v1/charts/connector-rollout-worker/templates/secrets.yaml new file mode 100644 index 00000000..24e141b1 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/templates/secrets.yaml @@ -0,0 +1,17 @@ +# Create secrets only for the local deployment +{{- if .Values.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: connector-rollout-worker-secrets + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" +type: Opaque +data: + {{- range $k, $v := mergeOverwrite .Values.secrets .Values.global.secrets }} + {{ $k }}: {{ if $v }}{{ $v | b64enc }} {{else}}""{{end}} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v1/charts/connector-rollout-worker/values.yaml b/helm/airbyte-v1/charts/connector-rollout-worker/values.yaml new file mode 100644 index 00000000..85d6bf18 --- /dev/null +++ b/helm/airbyte-v1/charts/connector-rollout-worker/values.yaml @@ -0,0 +1,273 @@ +global: + + serviceAccountName: &service-account-name "airbyte-admin" + edition: community + credVolumeOverride: "" + secretName: "" + extraContainers: [] + ## extraSelectorLabels [object] - use to specify own additional selector labels for deployment + extraSelectorLabels: {} + ## extraLabels [object] - use to specify own additional labels for deployment + extraLabels: {} + database: + secretName: "" + secretValue: "" + + # Docker image config that will apply to all images. + image: + # Docker registry to pull platform images from, e.g. http://my-registry:8000/ + registry: "" + # Image tag to use for airbyte images. + # Does not include non-airbyte images such as busybox, temporal, minio, etc. + tag: "" + + enterprise: + connectorRolloutOptIn: false + + jobs: + ## Jobs resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + ## jobs.resources.limits [object] The resources limits for jobs + ## jobs.resources.requests [object] The requested resources for jobs + resources: + ## Example: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + + kube: + ## JOB_KUBE_ANNOTATIONS + ## pod annotations of the sync job and the default pod annotations fallback for others jobs + ## jobs.kube.annotations [object] key/value annotations applied to kube jobs + annotations: {} + + ## JOB_KUBE_LABELS + ## pod labels of the sync job and the default pod labels fallback for others jobs + ## jobs.kube.labels [object] key/value labels applied to kube jobs + labels: {} + + ## JOB_KUBE_NODE_SELECTORS + ## pod node selector of the sync job and the default pod node selector fallback for others jobs + ## jobs.kube.nodeSelector [object] key/value node selector applied to kube jobs + nodeSelector: {} + + ## JOB_KUBE_TOLERATIONS + ## jobs.kube.tolerations [array] Tolerations for jobs.kube pod assignment. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## any boolean values should be quoted to ensure the value is passed through as a string, eg: + ## - key: airbyte-server + ## operator: Equal + ## value: "true" + ## effect: NoSchedule + tolerations: [] + + ## JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET + ## image pull secret to use for job pod + ## jobs.kube.main_container_image_pull_secret [string] image pull secret to use for job pod + main_container_image_pull_secret: "" + + images: + ## JOB_KUBE_BUSYBOX_IMAGE + ## busybox image used by the job pod + ## jobs.kube.images.busybox [string] busybox image used by the job pod + busybox: "" + ## JOB_KUBE_SOCAT_IMAGE + ## socat image used by the job pod + ## jobs.kube.images.socat [string] socat image used by the job pod + socat: "" + ## JOB_KUBE_CURL_IMAGE + ## curl image used by the job pod + ## jobs.kube.images.curl [string] curl image used by the job pod + curl: "" + +enabled: true +## connector-rollout-worker.replicaCount Number of connector-rollout-worker replicas +replicaCount: 1 + +## connector-rollout-worker.image.repository The repository to use for the airbyte connector-rollout-worker image. +## connector-rollout-worker.image.pullPolicy the pull policy to use for the airbyte connector-rollout-worker image +## connector-rollout-worker.image.tag The airbyte connector-rollout-worker image tag. Defaults to the chart's AppVersion +image: + repository: airbyte/connector-rollout-worker + pullPolicy: IfNotPresent + +## connector-rollout-worker.podAnnotations [object] Add extra annotations to the connector-rollout-worker pod(s) +## +podAnnotations: {} + +## connector-rollout-worker.podLabels [object] Add extra labels to the connector-rollout-worker pod(s) +## +podLabels: {} + +## connector-rollout-worker.containerSecurityContext Security context for the container +## Examples: +## containerSecurityContext: +## runAsNonRoot: true +## runAsUser: 1000 +## readOnlyRootFilesystem: true +containerSecurityContext: {} + +## Configure extra options for the connector-rollout-worker containers' liveness and readiness probes +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes +## connector-rollout-worker.livenessProbe.enabled Enable livenessProbe on the connector-rollout-worker +## connector-rollout-worker.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe +## connector-rollout-worker.livenessProbe.periodSeconds Period seconds for livenessProbe +## connector-rollout-worker.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe +## connector-rollout-worker.livenessProbe.failureThreshold Failure threshold for livenessProbe +## connector-rollout-worker.livenessProbe.successThreshold Success threshold for livenessProbe +## +livenessProbe: + enabled: true + initialDelaySeconds: 50 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + +## connector-rollout-worker.readinessProbe.enabled Enable readinessProbe on the connector-rollout-worker +## connector-rollout-worker.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe +## connector-rollout-worker.readinessProbe.periodSeconds Period seconds for readinessProbe +## connector-rollout-worker.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe +## connector-rollout-worker.readinessProbe.failureThreshold Failure threshold for readinessProbe +## connector-rollout-worker.readinessProbe.successThreshold Success threshold for readinessProbe +## +readinessProbe: + enabled: true + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + +## connector-rollout-worker resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## We usually recommend not to specify default resources and to leave this as a conscious +## choice for the user. This also increases chances charts run on environments with little +## resources, such as Minikube. If you do want to specify resources, uncomment the following +## lines, adjust them as necessary, and remove the curly braces after 'resources:'. +## connector-rollout-worker.resources.limits [object] The resources limits for the connector-rollout-worker container +## connector-rollout-worker.resources.requests [object] The requested resources for the connector-rollout-worker container +resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + +## connector-rollout-worker.nodeSelector [object] Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## connector-rollout-worker.tolerations [array] Tolerations for connector-rollout-worker pod assignment. +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## connector-rollout-worker.affinity [object] Affinity and anti-affinity for connector-rollout-worker pod assignment. +## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## connector-rollout-worker.log.level The log level to log at. +log: + level: "INFO" + + + +## connector-rollout-worker.extraVolumeMounts [array] Additional volumeMounts for connector-rollout-worker container(s). +## Examples (when using `connector-rollout-worker.containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumeMounts: +## - name: tmpdir +## mountPath: /tmp +## +extraVolumeMounts: [] + +## connector-rollout-worker.extraVolumes [array] Additional volumes for connector-rollout-worker pod(s). +## Examples (when using `connector-rollout-worker.containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumes: +## - name: tmpdir +## emptyDir: {} +## +extraVolumes: [] + +## extraContainer [array] Additional container for server pod(s) +## Example: +# extraContainers: +# - name: otel_collector +# image: somerepo/someimage:sometag +# args: [ +# "--important-args" +# ] +# ports: +# - containerPort: 443 +# volumeMounts: +# - name: volumeMountCool +# mountPath: /some/path +# readOnly: true +extraContainers: [] + +## extraInitContainers [array] Additional init containers for server pod(s) +## Example: +# extraInitContainers: +# - name: sleepy +# image: alpine +# command: ['sleep', '60'] + +extraInitContainers: [] + +## extraEnv [array] Supply extra env variables to main container using full notation +## Example: (With default env vars and values taken from generated config map) +# extraEnv: +# - name: AIRBYTE_VERSION +# valueFrom: +# configMapKeyRef: +# name: airbyte-env +# key: AIRBYTE_VERSION +## +extraEnv: [] +## secrets [object] Supply additional secrets to container +## Example: +## secrets: +## DATABASE_PASSWORD: strong-password +## DATABASE_USER: my-db-user +secrets: {} + +## env_vars [object] Supply extra env variables to main container using simplified notation +## Example with default values for connector-rollout-worker: +# env_vars: +# AIRBYTE_VERSION: 0.40.4 + +# # Workspace storage for running jobs (logs, etc) +# WORKSPACE_ROOT: /workspace +# WORKSPACE_DOCKER_MOUNT: airbyte_workspace + +# # Miscellaneous +# TRACKING_STRATEGY: segment +# WEBAPP_URL: airbyte-webapp-svc:80 +# INTERNAL_API_HOST: http://airbyte-server-svc:8001 +env_vars: {} + +## extraSelectorLabels [object] - use to specify own additional selector labels for deployment +extraSelectorLabels: {} +## extraLabels [object] - use to specify own additional labels for deployment +extraLabels: {} + +debug: + enabled: true + remoteDebugPort: 5005 diff --git a/helm/airbyte-v1/charts/cron/Chart.lock b/helm/airbyte-v1/charts/cron/Chart.lock new file mode 100644 index 00000000..435017d8 --- /dev/null +++ b/helm/airbyte-v1/charts/cron/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + version: 1.17.1 +digest: sha256:dacc73770a5640c011e067ff8840ddf89631fc19016c8d0a9e5ea160e7da8690 +generated: "2022-10-25T16:47:38.253767-04:00" diff --git a/helm/airbyte-v1/charts/cron/Chart.yaml b/helm/airbyte-v1/charts/cron/Chart.yaml new file mode 100644 index 00000000..09559008 --- /dev/null +++ b/helm/airbyte-v1/charts/cron/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +appVersion: 1.3.0 +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x +description: Helm chart to deploy airbyte-cron +name: cron +type: application +version: 1.3.0 diff --git a/helm/airbyte-v1/charts/cron/README.md b/helm/airbyte-v1/charts/cron/README.md new file mode 100644 index 00000000..9f628a60 --- /dev/null +++ b/helm/airbyte-v1/charts/cron/README.md @@ -0,0 +1,62 @@ +# cron + +![Version: 0.67.17](https://img.shields.io/badge/Version-0.67.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) + +Helm chart to deploy airbyte-cron + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://charts.bitnami.com/bitnami | common | 1.x.x | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| containerSecurityContext | object | `{}` | | +| enabled | bool | `true` | | +| env_vars | object | `{}` | | +| extraContainers | list | `[]` | | +| extraEnv | list | `[]` | | +| extraInitContainers | list | `[]` | | +| extraLabels | object | `{}` | | +| extraSelectorLabels | object | `{}` | | +| extraVolumeMounts | list | `[]` | | +| extraVolumes | list | `[]` | | +| global.configMapName | string | `""` | | +| global.database.secretName | string | `""` | | +| global.database.secretValue | string | `""` | | +| global.deploymentMode | string | `"oss"` | | +| global.extraContainers | list | `[]` | | +| global.extraLabels | object | `{}` | | +| global.extraSelectorLabels | object | `{}` | | +| global.secretName | string | `""` | | +| global.secrets | object | `{}` | | +| global.serviceAccountName | string | `"placeholderServiceAccount"` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"airbyte/cron"` | | +| livenessProbe.enabled | bool | `true` | | +| livenessProbe.failureThreshold | int | `3` | | +| livenessProbe.initialDelaySeconds | int | `30` | | +| livenessProbe.periodSeconds | int | `10` | | +| livenessProbe.successThreshold | int | `1` | | +| livenessProbe.timeoutSeconds | int | `1` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| podLabels | object | `{}` | | +| readinessProbe.containerSecurityContext | object | `{}` | | +| readinessProbe.enabled | bool | `true` | | +| readinessProbe.failureThreshold | int | `3` | | +| readinessProbe.initialDelaySeconds | int | `10` | | +| readinessProbe.periodSeconds | int | `10` | | +| readinessProbe.successThreshold | int | `1` | | +| readinessProbe.timeoutSeconds | int | `1` | | +| replicaCount | int | `1` | | +| resources.limits | object | `{}` | | +| resources.requests | object | `{}` | | +| secrets | object | `{}` | | +| tolerations | list | `[]` | | +| workloadApi | object | `{}` | | + diff --git a/helm/airbyte-v1/charts/cron/charts/common/.helmignore b/helm/airbyte-v1/charts/cron/charts/common/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/helm/airbyte-v1/charts/cron/charts/common/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/airbyte-v1/charts/cron/charts/common/Chart.yaml b/helm/airbyte-v1/charts/cron/charts/common/Chart.yaml new file mode 100644 index 00000000..39edbb3c --- /dev/null +++ b/helm/airbyte-v1/charts/cron/charts/common/Chart.yaml @@ -0,0 +1,23 @@ +annotations: + category: Infrastructure +apiVersion: v2 +appVersion: 1.17.1 +description: A Library Helm Chart for grouping common logic between bitnami charts. + This chart is not deployable by itself. +home: https://github.com/bitnami/charts/tree/master/bitnami/common +icon: https://bitnami.com/downloads/logos/bitnami-mark.png +keywords: +- common +- helper +- template +- function +- bitnami +maintainers: +- name: Bitnami + url: https://github.com/bitnami/charts +name: common +sources: +- https://github.com/bitnami/charts +- https://www.bitnami.com/ +type: library +version: 1.17.1 diff --git a/helm/airbyte-v1/charts/cron/charts/common/README.md b/helm/airbyte-v1/charts/cron/charts/common/README.md new file mode 100644 index 00000000..a2ecd604 --- /dev/null +++ b/helm/airbyte-v1/charts/cron/charts/common/README.md @@ -0,0 +1,350 @@ +# Bitnami Common Library Chart + +A [Helm Library Chart](https://helm.sh/docs/topics/library_charts/#helm) for grouping common logic between bitnami charts. + +## TL;DR + +```yaml +dependencies: + - name: common + version: 1.x.x + repository: https://charts.bitnami.com/bitnami +``` + +```bash +$ helm dependency update +``` + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.names.fullname" . }} +data: + myvalue: "Hello World" +``` + +## Introduction + +This chart provides a common template helpers which can be used to develop new charts using [Helm](https://helm.sh) package manager. + +Bitnami charts can be used with [Kubeapps](https://kubeapps.dev/) for deployment and management of Helm Charts in clusters. + +## Prerequisites + +- Kubernetes 1.19+ +- Helm 3.2.0+ + +## Parameters + +The following table lists the helpers available in the library which are scoped in different sections. + +### Affinities + +| Helper identifier | Description | Expected Input | +|-------------------------------|------------------------------------------------------|------------------------------------------------| +| `common.affinities.nodes.soft` | Return a soft nodeAffinity definition | `dict "key" "FOO" "values" (list "BAR" "BAZ")` | +| `common.affinities.nodes.hard` | Return a hard nodeAffinity definition | `dict "key" "FOO" "values" (list "BAR" "BAZ")` | +| `common.affinities.pods.soft` | Return a soft podAffinity/podAntiAffinity definition | `dict "component" "FOO" "context" $` | +| `common.affinities.pods.hard` | Return a hard podAffinity/podAntiAffinity definition | `dict "component" "FOO" "context" $` | + +### Capabilities + +| Helper identifier | Description | Expected Input | +|------------------------------------------------|------------------------------------------------------------------------------------------------|-------------------| +| `common.capabilities.kubeVersion` | Return the target Kubernetes version (using client default if .Values.kubeVersion is not set). | `.` Chart context | +| `common.capabilities.cronjob.apiVersion` | Return the appropriate apiVersion for cronjob. | `.` Chart context | +| `common.capabilities.deployment.apiVersion` | Return the appropriate apiVersion for deployment. | `.` Chart context | +| `common.capabilities.statefulset.apiVersion` | Return the appropriate apiVersion for statefulset. | `.` Chart context | +| `common.capabilities.ingress.apiVersion` | Return the appropriate apiVersion for ingress. | `.` Chart context | +| `common.capabilities.rbac.apiVersion` | Return the appropriate apiVersion for RBAC resources. | `.` Chart context | +| `common.capabilities.crd.apiVersion` | Return the appropriate apiVersion for CRDs. | `.` Chart context | +| `common.capabilities.policy.apiVersion` | Return the appropriate apiVersion for podsecuritypolicy. | `.` Chart context | +| `common.capabilities.networkPolicy.apiVersion` | Return the appropriate apiVersion for networkpolicy. | `.` Chart context | +| `common.capabilities.apiService.apiVersion` | Return the appropriate apiVersion for APIService. | `.` Chart context | +| `common.capabilities.hpa.apiVersion` | Return the appropriate apiVersion for Horizontal Pod Autoscaler | `.` Chart context | +| `common.capabilities.supportsHelmVersion` | Returns true if the used Helm version is 3.3+ | `.` Chart context | + +### Errors + +| Helper identifier | Description | Expected Input | +|-----------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------| +| `common.errors.upgrade.passwords.empty` | It will ensure required passwords are given when we are upgrading a chart. If `validationErrors` is not empty it will throw an error and will stop the upgrade action. | `dict "validationErrors" (list $validationError00 $validationError01) "context" $` | + +### Images + +| Helper identifier | Description | Expected Input | +|-----------------------------|------------------------------------------------------|---------------------------------------------------------------------------------------------------------| +| `common.images.image` | Return the proper and full image name | `dict "imageRoot" .Values.path.to.the.image "global" $`, see [ImageRoot](#imageroot) for the structure. | +| `common.images.pullSecrets` | Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) | `dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global` | +| `common.images.renderPullSecrets` | Return the proper Docker Image Registry Secret Names (evaluates values as templates) | `dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "context" $` | + +### Ingress + +| Helper identifier | Description | Expected Input | +|-------------------------------------------|-------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.ingress.backend` | Generate a proper Ingress backend entry depending on the API version | `dict "serviceName" "foo" "servicePort" "bar"`, see the [Ingress deprecation notice](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/) for the syntax differences | +| `common.ingress.supportsPathType` | Prints "true" if the pathType field is supported | `.` Chart context | +| `common.ingress.supportsIngressClassname` | Prints "true" if the ingressClassname field is supported | `.` Chart context | +| `common.ingress.certManagerRequest` | Prints "true" if required cert-manager annotations for TLS signed certificates are set in the Ingress annotations | `dict "annotations" .Values.path.to.the.ingress.annotations` | + +### Labels + +| Helper identifier | Description | Expected Input | +|-----------------------------|-----------------------------------------------------------------------------|-------------------| +| `common.labels.standard` | Return Kubernetes standard labels | `.` Chart context | +| `common.labels.matchLabels` | Labels to use on `deploy.spec.selector.matchLabels` and `svc.spec.selector` | `.` Chart context | + +### Names + +| Helper identifier | Description | Expected Input | +|-----------------------------------|-----------------------------------------------------------------------|-------------------| +| `common.names.name` | Expand the name of the chart or use `.Values.nameOverride` | `.` Chart context | +| `common.names.fullname` | Create a default fully qualified app name. | `.` Chart context | +| `common.names.namespace` | Allow the release namespace to be overridden | `.` Chart context | +| `common.names.fullname.namespace` | Create a fully qualified app name adding the installation's namespace | `.` Chart context | +| `common.names.chart` | Chart name plus version | `.` Chart context | + +### Secrets + +| Helper identifier | Description | Expected Input | +|---------------------------|--------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.secrets.name` | Generate the name of the secret. | `dict "existingSecret" .Values.path.to.the.existingSecret "defaultNameSuffix" "mySuffix" "context" $` see [ExistingSecret](#existingsecret) for the structure. | +| `common.secrets.key` | Generate secret key. | `dict "existingSecret" .Values.path.to.the.existingSecret "key" "keyName"` see [ExistingSecret](#existingsecret) for the structure. | +| `common.passwords.manage` | Generate secret password or retrieve one if already created. | `dict "secret" "secret-name" "key" "keyName" "providedValues" (list "path.to.password1" "path.to.password2") "length" 10 "strong" false "chartName" "chartName" "context" $`, length, strong and chartNAme fields are optional. | +| `common.secrets.exists` | Returns whether a previous generated secret already exists. | `dict "secret" "secret-name" "context" $` | + +### Storage + +| Helper identifier | Description | Expected Input | +|-------------------------------|---------------------------------------|---------------------------------------------------------------------------------------------------------------------| +| `common.storage.class` | Return the proper Storage Class | `dict "persistence" .Values.path.to.the.persistence "global" $`, see [Persistence](#persistence) for the structure. | + +### TplValues + +| Helper identifier | Description | Expected Input | +|---------------------------|----------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.tplvalues.render` | Renders a value that contains template | `dict "value" .Values.path.to.the.Value "context" $`, value is the value should rendered as template, context frequently is the chart context `$` or `.` | + +### Utils + +| Helper identifier | Description | Expected Input | +|--------------------------------|------------------------------------------------------------------------------------------|------------------------------------------------------------------------| +| `common.utils.fieldToEnvVar` | Build environment variable name given a field. | `dict "field" "my-password"` | +| `common.utils.secret.getvalue` | Print instructions to get a secret value. | `dict "secret" "secret-name" "field" "secret-value-field" "context" $` | +| `common.utils.getValueFromKey` | Gets a value from `.Values` object given its key path | `dict "key" "path.to.key" "context" $` | +| `common.utils.getKeyFromList` | Returns first `.Values` key with a defined value or first of the list if all non-defined | `dict "keys" (list "path.to.key1" "path.to.key2") "context" $` | + +### Validations + +| Helper identifier | Description | Expected Input | +|--------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.validations.values.single.empty` | Validate a value must not be empty. | `dict "valueKey" "path.to.value" "secret" "secret.name" "field" "my-password" "subchart" "subchart" "context" $` secret, field and subchart are optional. In case they are given, the helper will generate a how to get instruction. See [ValidateValue](#validatevalue) | +| `common.validations.values.multiple.empty` | Validate a multiple values must not be empty. It returns a shared error for all the values. | `dict "required" (list $validateValueConf00 $validateValueConf01) "context" $`. See [ValidateValue](#validatevalue) | +| `common.validations.values.mariadb.passwords` | This helper will ensure required password for MariaDB are not empty. It returns a shared error for all the values. | `dict "secret" "mariadb-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use mariadb chart and the helper. | +| `common.validations.values.mysql.passwords` | This helper will ensure required password for MySQL are not empty. It returns a shared error for all the values. | `dict "secret" "mysql-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use mysql chart and the helper. | +| `common.validations.values.postgresql.passwords` | This helper will ensure required password for PostgreSQL are not empty. It returns a shared error for all the values. | `dict "secret" "postgresql-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use postgresql chart and the helper. | +| `common.validations.values.redis.passwords` | This helper will ensure required password for Redis® are not empty. It returns a shared error for all the values. | `dict "secret" "redis-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use redis chart and the helper. | +| `common.validations.values.cassandra.passwords` | This helper will ensure required password for Cassandra are not empty. It returns a shared error for all the values. | `dict "secret" "cassandra-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use cassandra chart and the helper. | +| `common.validations.values.mongodb.passwords` | This helper will ensure required password for MongoDB® are not empty. It returns a shared error for all the values. | `dict "secret" "mongodb-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use mongodb chart and the helper. | + +### Warnings + +| Helper identifier | Description | Expected Input | +|------------------------------|----------------------------------|------------------------------------------------------------| +| `common.warnings.rollingTag` | Warning about using rolling tag. | `ImageRoot` see [ImageRoot](#imageroot) for the structure. | + +## Special input schemas + +### ImageRoot + +```yaml +registry: + type: string + description: Docker registry where the image is located + example: docker.io + +repository: + type: string + description: Repository and image name + example: bitnami/nginx + +tag: + type: string + description: image tag + example: 1.16.1-debian-10-r63 + +pullPolicy: + type: string + description: Specify a imagePullPolicy. Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + +pullSecrets: + type: array + items: + type: string + description: Optionally specify an array of imagePullSecrets (evaluated as templates). + +debug: + type: boolean + description: Set to true if you would like to see extra information on logs + example: false + +## An instance would be: +# registry: docker.io +# repository: bitnami/nginx +# tag: 1.16.1-debian-10-r63 +# pullPolicy: IfNotPresent +# debug: false +``` + +### Persistence + +```yaml +enabled: + type: boolean + description: Whether enable persistence. + example: true + +storageClass: + type: string + description: Ghost data Persistent Volume Storage Class, If set to "-", storageClassName: "" which disables dynamic provisioning. + example: "-" + +accessMode: + type: string + description: Access mode for the Persistent Volume Storage. + example: ReadWriteOnce + +size: + type: string + description: Size the Persistent Volume Storage. + example: 8Gi + +path: + type: string + description: Path to be persisted. + example: /bitnami + +## An instance would be: +# enabled: true +# storageClass: "-" +# accessMode: ReadWriteOnce +# size: 8Gi +# path: /bitnami +``` + +### ExistingSecret + +```yaml +name: + type: string + description: Name of the existing secret. + example: mySecret +keyMapping: + description: Mapping between the expected key name and the name of the key in the existing secret. + type: object + +## An instance would be: +# name: mySecret +# keyMapping: +# password: myPasswordKey +``` + +#### Example of use + +When we store sensitive data for a deployment in a secret, some times we want to give to users the possibility of using theirs existing secrets. + +```yaml +# templates/secret.yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "common.names.fullname" . }} + labels: + app: {{ include "common.names.fullname" . }} +type: Opaque +data: + password: {{ .Values.password | b64enc | quote }} + +# templates/dpl.yaml +--- +... + env: + - name: PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "common.secrets.name" (dict "existingSecret" .Values.existingSecret "context" $) }} + key: {{ include "common.secrets.key" (dict "existingSecret" .Values.existingSecret "key" "password") }} +... + +# values.yaml +--- +name: mySecret +keyMapping: + password: myPasswordKey +``` + +### ValidateValue + +#### NOTES.txt + +```console +{{- $validateValueConf00 := (dict "valueKey" "path.to.value00" "secret" "secretName" "field" "password-00") -}} +{{- $validateValueConf01 := (dict "valueKey" "path.to.value01" "secret" "secretName" "field" "password-01") -}} + +{{ include "common.validations.values.multiple.empty" (dict "required" (list $validateValueConf00 $validateValueConf01) "context" $) }} +``` + +If we force those values to be empty we will see some alerts + +```console +$ helm install test mychart --set path.to.value00="",path.to.value01="" + 'path.to.value00' must not be empty, please add '--set path.to.value00=$PASSWORD_00' to the command. To get the current value: + + export PASSWORD_00=$(kubectl get secret --namespace default secretName -o jsonpath="{.data.password-00}" | base64 -d) + + 'path.to.value01' must not be empty, please add '--set path.to.value01=$PASSWORD_01' to the command. To get the current value: + + export PASSWORD_01=$(kubectl get secret --namespace default secretName -o jsonpath="{.data.password-01}" | base64 -d) +``` + +## Upgrading + +### To 1.0.0 + +[On November 13, 2020, Helm v2 support was formally finished](https://github.com/helm/charts#status-of-the-project), this major version is the result of the required changes applied to the Helm Chart to be able to incorporate the different features added in Helm v3 and to be consistent with the Helm project itself regarding the Helm v2 EOL. + +**What changes were introduced in this major version?** + +- Previous versions of this Helm Chart use `apiVersion: v1` (installable by both Helm 2 and 3), this Helm Chart was updated to `apiVersion: v2` (installable by Helm 3 only). [Here](https://helm.sh/docs/topics/charts/#the-apiversion-field) you can find more information about the `apiVersion` field. +- Use `type: library`. [Here](https://v3.helm.sh/docs/faq/#library-chart-support) you can find more information. +- The different fields present in the *Chart.yaml* file has been ordered alphabetically in a homogeneous way for all the Bitnami Helm Charts + +**Considerations when upgrading to this version** + +- If you want to upgrade to this version from a previous one installed with Helm v3, you shouldn't face any issues +- If you want to upgrade to this version using Helm v2, this scenario is not supported as this version doesn't support Helm v2 anymore +- If you installed the previous version with Helm v2 and wants to upgrade to this version with Helm v3, please refer to the [official Helm documentation](https://helm.sh/docs/topics/v2_v3_migration/#migration-use-cases) about migrating from Helm v2 to v3 + +**Useful links** + +- https://docs.bitnami.com/tutorials/resolve-helm2-helm3-post-migration-issues/ +- https://helm.sh/docs/topics/v2_v3_migration/ +- https://helm.sh/blog/migrate-from-helm-v2-to-helm-v3/ + +## License + +Copyright © 2022 Bitnami + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/helm/airbyte-v1/charts/cron/charts/common/templates/_affinities.tpl b/helm/airbyte-v1/charts/cron/charts/common/templates/_affinities.tpl new file mode 100644 index 00000000..2387be26 --- /dev/null +++ b/helm/airbyte-v1/charts/cron/charts/common/templates/_affinities.tpl @@ -0,0 +1,102 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Return a soft nodeAffinity definition +{{ include "common.affinities.nodes.soft" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.soft" -}} +preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . | quote }} + {{- end }} + weight: 1 +{{- end -}} + +{{/* +Return a hard nodeAffinity definition +{{ include "common.affinities.nodes.hard" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.hard" -}} +requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . | quote }} + {{- end }} +{{- end -}} + +{{/* +Return a nodeAffinity definition +{{ include "common.affinities.nodes" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.nodes.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.nodes.hard" . -}} + {{- end -}} +{{- end -}} + +{{/* +Return a soft podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.soft" (dict "component" "FOO" "extraMatchLabels" .Values.extraMatchLabels "context" $) -}} +*/}} +{{- define "common.affinities.pods.soft" -}} +{{- $component := default "" .component -}} +{{- $extraMatchLabels := default (dict) .extraMatchLabels -}} +preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" .context) | nindent 10 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := $extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + namespaces: + - {{ include "common.names.namespace" .context | quote }} + topologyKey: kubernetes.io/hostname + weight: 1 +{{- end -}} + +{{/* +Return a hard podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.hard" (dict "component" "FOO" "extraMatchLabels" .Values.extraMatchLabels "context" $) -}} +*/}} +{{- define "common.affinities.pods.hard" -}} +{{- $component := default "" .component -}} +{{- $extraMatchLabels := default (dict) .extraMatchLabels -}} +requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" .context) | nindent 8 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := $extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + namespaces: + - {{ include "common.names.namespace" .context | quote }} + topologyKey: kubernetes.io/hostname +{{- end -}} + +{{/* +Return a podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.pods" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.pods.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.pods.hard" . -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/cron/charts/common/templates/_capabilities.tpl b/helm/airbyte-v1/charts/cron/charts/common/templates/_capabilities.tpl new file mode 100644 index 00000000..9d9b7600 --- /dev/null +++ b/helm/airbyte-v1/charts/cron/charts/common/templates/_capabilities.tpl @@ -0,0 +1,154 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Return the target Kubernetes version +*/}} +{{- define "common.capabilities.kubeVersion" -}} +{{- if .Values.global }} + {{- if .Values.global.kubeVersion }} + {{- .Values.global.kubeVersion -}} + {{- else }} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} + {{- end -}} +{{- else }} +{{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for poddisruptionbudget. +*/}} +{{- define "common.capabilities.policy.apiVersion" -}} +{{- if semverCompare "<1.21-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "policy/v1beta1" -}} +{{- else -}} +{{- print "policy/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for networkpolicy. +*/}} +{{- define "common.capabilities.networkPolicy.apiVersion" -}} +{{- if semverCompare "<1.7-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for cronjob. +*/}} +{{- define "common.capabilities.cronjob.apiVersion" -}} +{{- if semverCompare "<1.21-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "batch/v1beta1" -}} +{{- else -}} +{{- print "batch/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for deployment. +*/}} +{{- define "common.capabilities.deployment.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for statefulset. +*/}} +{{- define "common.capabilities.statefulset.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apps/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "common.capabilities.ingress.apiVersion" -}} +{{- if .Values.ingress -}} +{{- if .Values.ingress.apiVersion -}} +{{- .Values.ingress.apiVersion -}} +{{- else if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end }} +{{- else if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for RBAC resources. +*/}} +{{- define "common.capabilities.rbac.apiVersion" -}} +{{- if semverCompare "<1.17-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "rbac.authorization.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "rbac.authorization.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for CRDs. +*/}} +{{- define "common.capabilities.crd.apiVersion" -}} +{{- if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiextensions.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiextensions.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for APIService. +*/}} +{{- define "common.capabilities.apiService.apiVersion" -}} +{{- if semverCompare "<1.10-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiregistration.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiregistration.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for Horizontal Pod Autoscaler. +*/}} +{{- define "common.capabilities.hpa.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .context) -}} +{{- if .beta2 -}} +{{- print "autoscaling/v2beta2" -}} +{{- else -}} +{{- print "autoscaling/v2beta1" -}} +{{- end -}} +{{- else -}} +{{- print "autoscaling/v2" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if the used Helm version is 3.3+. +A way to check the used Helm version was not introduced until version 3.3.0 with .Capabilities.HelmVersion, which contains an additional "{}}" structure. +This check is introduced as a regexMatch instead of {{ if .Capabilities.HelmVersion }} because checking for the key HelmVersion in <3.3 results in a "interface not found" error. +**To be removed when the catalog's minimun Helm version is 3.3** +*/}} +{{- define "common.capabilities.supportsHelmVersion" -}} +{{- if regexMatch "{(v[0-9])*[^}]*}}$" (.Capabilities | toString ) }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/cron/charts/common/templates/_errors.tpl b/helm/airbyte-v1/charts/cron/charts/common/templates/_errors.tpl new file mode 100644 index 00000000..a79cc2e3 --- /dev/null +++ b/helm/airbyte-v1/charts/cron/charts/common/templates/_errors.tpl @@ -0,0 +1,23 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Through error when upgrading using empty passwords values that must not be empty. + +Usage: +{{- $validationError00 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password00" "secret" "secretName" "field" "password-00") -}} +{{- $validationError01 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password01" "secret" "secretName" "field" "password-01") -}} +{{ include "common.errors.upgrade.passwords.empty" (dict "validationErrors" (list $validationError00 $validationError01) "context" $) }} + +Required password params: + - validationErrors - String - Required. List of validation strings to be return, if it is empty it won't throw error. + - context - Context - Required. Parent context. +*/}} +{{- define "common.errors.upgrade.passwords.empty" -}} + {{- $validationErrors := join "" .validationErrors -}} + {{- if and $validationErrors .context.Release.IsUpgrade -}} + {{- $errorString := "\nPASSWORDS ERROR: You must provide your current passwords when upgrading the release." -}} + {{- $errorString = print $errorString "\n Note that even after reinstallation, old credentials may be needed as they may be kept in persistent volume claims." -}} + {{- $errorString = print $errorString "\n Further information can be obtained at https://docs.bitnami.com/general/how-to/troubleshoot-helm-chart-issues/#credential-errors-while-upgrading-chart-releases" -}} + {{- $errorString = print $errorString "\n%s" -}} + {{- printf $errorString $validationErrors | fail -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/cron/charts/common/templates/_images.tpl b/helm/airbyte-v1/charts/cron/charts/common/templates/_images.tpl new file mode 100644 index 00000000..42ffbc72 --- /dev/null +++ b/helm/airbyte-v1/charts/cron/charts/common/templates/_images.tpl @@ -0,0 +1,75 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper image name +{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }} +*/}} +{{- define "common.images.image" -}} +{{- $registryName := .imageRoot.registry -}} +{{- $repositoryName := .imageRoot.repository -}} +{{- $tag := .imageRoot.tag | toString -}} +{{- if .global }} + {{- if .global.imageRegistry }} + {{- $registryName = .global.imageRegistry -}} + {{- end -}} +{{- end -}} +{{- if $registryName }} +{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- else -}} +{{- printf "%s:%s" $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) +{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} +*/}} +{{- define "common.images.pullSecrets" -}} + {{- $pullSecrets := list }} + + {{- if .global }} + {{- range .global.imagePullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names evaluating values as templates +{{ include "common.images.renderPullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "context" $) }} +*/}} +{{- define "common.images.renderPullSecrets" -}} + {{- $pullSecrets := list }} + {{- $context := .context }} + + {{- if $context.Values.global }} + {{- range $context.Values.global.imagePullSecrets -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" $context)) -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" $context)) -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/helm/airbyte-v1/charts/cron/charts/common/templates/_ingress.tpl b/helm/airbyte-v1/charts/cron/charts/common/templates/_ingress.tpl new file mode 100644 index 00000000..8caf73a6 --- /dev/null +++ b/helm/airbyte-v1/charts/cron/charts/common/templates/_ingress.tpl @@ -0,0 +1,68 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Generate backend entry that is compatible with all Kubernetes API versions. + +Usage: +{{ include "common.ingress.backend" (dict "serviceName" "backendName" "servicePort" "backendPort" "context" $) }} + +Params: + - serviceName - String. Name of an existing service backend + - servicePort - String/Int. Port name (or number) of the service. It will be translated to different yaml depending if it is a string or an integer. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.ingress.backend" -}} +{{- $apiVersion := (include "common.capabilities.ingress.apiVersion" .context) -}} +{{- if or (eq $apiVersion "extensions/v1beta1") (eq $apiVersion "networking.k8s.io/v1beta1") -}} +serviceName: {{ .serviceName }} +servicePort: {{ .servicePort }} +{{- else -}} +service: + name: {{ .serviceName }} + port: + {{- if typeIs "string" .servicePort }} + name: {{ .servicePort }} + {{- else if or (typeIs "int" .servicePort) (typeIs "float64" .servicePort) }} + number: {{ .servicePort | int }} + {{- end }} +{{- end -}} +{{- end -}} + +{{/* +Print "true" if the API pathType field is supported +Usage: +{{ include "common.ingress.supportsPathType" . }} +*/}} +{{- define "common.ingress.supportsPathType" -}} +{{- if (semverCompare "<1.18-0" (include "common.capabilities.kubeVersion" .)) -}} +{{- print "false" -}} +{{- else -}} +{{- print "true" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if the ingressClassname field is supported +Usage: +{{ include "common.ingress.supportsIngressClassname" . }} +*/}} +{{- define "common.ingress.supportsIngressClassname" -}} +{{- if semverCompare "<1.18-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "false" -}} +{{- else -}} +{{- print "true" -}} +{{- end -}} +{{- end -}} + +{{/* +Return true if cert-manager required annotations for TLS signed +certificates are set in the Ingress annotations +Ref: https://cert-manager.io/docs/usage/ingress/#supported-annotations +Usage: +{{ include "common.ingress.certManagerRequest" ( dict "annotations" .Values.path.to.the.ingress.annotations ) }} +*/}} +{{- define "common.ingress.certManagerRequest" -}} +{{ if or (hasKey .annotations "cert-manager.io/cluster-issuer") (hasKey .annotations "cert-manager.io/issuer") }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/cron/charts/common/templates/_labels.tpl b/helm/airbyte-v1/charts/cron/charts/common/templates/_labels.tpl new file mode 100644 index 00000000..252066c7 --- /dev/null +++ b/helm/airbyte-v1/charts/cron/charts/common/templates/_labels.tpl @@ -0,0 +1,18 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Kubernetes standard labels +*/}} +{{- define "common.labels.standard" -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +helm.sh/chart: {{ include "common.names.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Labels to use on deploy.spec.selector.matchLabels and svc.spec.selector +*/}} +{{- define "common.labels.matchLabels" -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} diff --git a/helm/airbyte-v1/charts/cron/charts/common/templates/_names.tpl b/helm/airbyte-v1/charts/cron/charts/common/templates/_names.tpl new file mode 100644 index 00000000..1bdac8b7 --- /dev/null +++ b/helm/airbyte-v1/charts/cron/charts/common/templates/_names.tpl @@ -0,0 +1,70 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "common.names.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "common.names.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "common.names.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a default fully qualified dependency name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +Usage: +{{ include "common.names.dependency.fullname" (dict "chartName" "dependency-chart-name" "chartValues" .Values.dependency-chart "context" $) }} +*/}} +{{- define "common.names.dependency.fullname" -}} +{{- if .chartValues.fullnameOverride -}} +{{- .chartValues.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .chartName .chartValues.nameOverride -}} +{{- if contains $name .context.Release.Name -}} +{{- .context.Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .context.Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts. +*/}} +{{- define "common.names.namespace" -}} +{{- if .Values.namespaceOverride -}} +{{- .Values.namespaceOverride -}} +{{- else -}} +{{- .Release.Namespace -}} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified app name adding the installation's namespace. +*/}} +{{- define "common.names.fullname.namespace" -}} +{{- printf "%s-%s" (include "common.names.fullname" .) (include "common.names.namespace" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/cron/charts/common/templates/_secrets.tpl b/helm/airbyte-v1/charts/cron/charts/common/templates/_secrets.tpl new file mode 100644 index 00000000..a53fb44f --- /dev/null +++ b/helm/airbyte-v1/charts/cron/charts/common/templates/_secrets.tpl @@ -0,0 +1,140 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Generate secret name. + +Usage: +{{ include "common.secrets.name" (dict "existingSecret" .Values.path.to.the.existingSecret "defaultNameSuffix" "mySuffix" "context" $) }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/master/bitnami/common#existingsecret + - defaultNameSuffix - String - Optional. It is used only if we have several secrets in the same deployment. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.secrets.name" -}} +{{- $name := (include "common.names.fullname" .context) -}} + +{{- if .defaultNameSuffix -}} +{{- $name = printf "%s-%s" $name .defaultNameSuffix | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- with .existingSecret -}} +{{- if not (typeIs "string" .) -}} +{{- with .name -}} +{{- $name = . -}} +{{- end -}} +{{- else -}} +{{- $name = . -}} +{{- end -}} +{{- end -}} + +{{- printf "%s" $name -}} +{{- end -}} + +{{/* +Generate secret key. + +Usage: +{{ include "common.secrets.key" (dict "existingSecret" .Values.path.to.the.existingSecret "key" "keyName") }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/master/bitnami/common#existingsecret + - key - String - Required. Name of the key in the secret. +*/}} +{{- define "common.secrets.key" -}} +{{- $key := .key -}} + +{{- if .existingSecret -}} + {{- if not (typeIs "string" .existingSecret) -}} + {{- if .existingSecret.keyMapping -}} + {{- $key = index .existingSecret.keyMapping $.key -}} + {{- end -}} + {{- end }} +{{- end -}} + +{{- printf "%s" $key -}} +{{- end -}} + +{{/* +Generate secret password or retrieve one if already created. + +Usage: +{{ include "common.secrets.passwords.manage" (dict "secret" "secret-name" "key" "keyName" "providedValues" (list "path.to.password1" "path.to.password2") "length" 10 "strong" false "chartName" "chartName" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - key - String - Required - Name of the key in the secret. + - providedValues - List - Required - The path to the validating value in the values.yaml, e.g: "mysql.password". Will pick first parameter with a defined value. + - length - int - Optional - Length of the generated random password. + - strong - Boolean - Optional - Whether to add symbols to the generated random password. + - chartName - String - Optional - Name of the chart used when said chart is deployed as a subchart. + - context - Context - Required - Parent context. + +The order in which this function returns a secret password: + 1. Already existing 'Secret' resource + (If a 'Secret' resource is found under the name provided to the 'secret' parameter to this function and that 'Secret' resource contains a key with the name passed as the 'key' parameter to this function then the value of this existing secret password will be returned) + 2. Password provided via the values.yaml + (If one of the keys passed to the 'providedValues' parameter to this function is a valid path to a key in the values.yaml and has a value, the value of the first key with a value will be returned) + 3. Randomly generated secret password + (A new random secret password with the length specified in the 'length' parameter will be generated and returned) + +*/}} +{{- define "common.secrets.passwords.manage" -}} + +{{- $password := "" }} +{{- $subchart := "" }} +{{- $chartName := default "" .chartName }} +{{- $passwordLength := default 10 .length }} +{{- $providedPasswordKey := include "common.utils.getKeyFromList" (dict "keys" .providedValues "context" $.context) }} +{{- $providedPasswordValue := include "common.utils.getValueFromKey" (dict "key" $providedPasswordKey "context" $.context) }} +{{- $secretData := (lookup "v1" "Secret" $.context.Release.Namespace .secret).data }} +{{- if $secretData }} + {{- if hasKey $secretData .key }} + {{- $password = index $secretData .key }} + {{- else }} + {{- printf "\nPASSWORDS ERROR: The secret \"%s\" does not contain the key \"%s\"\n" .secret .key | fail -}} + {{- end -}} +{{- else if $providedPasswordValue }} + {{- $password = $providedPasswordValue | toString | b64enc | quote }} +{{- else }} + + {{- if .context.Values.enabled }} + {{- $subchart = $chartName }} + {{- end -}} + + {{- $requiredPassword := dict "valueKey" $providedPasswordKey "secret" .secret "field" .key "subchart" $subchart "context" $.context -}} + {{- $requiredPasswordError := include "common.validations.values.single.empty" $requiredPassword -}} + {{- $passwordValidationErrors := list $requiredPasswordError -}} + {{- include "common.errors.upgrade.passwords.empty" (dict "validationErrors" $passwordValidationErrors "context" $.context) -}} + + {{- if .strong }} + {{- $subStr := list (lower (randAlpha 1)) (randNumeric 1) (upper (randAlpha 1)) | join "_" }} + {{- $password = randAscii $passwordLength }} + {{- $password = regexReplaceAllLiteral "\\W" $password "@" | substr 5 $passwordLength }} + {{- $password = printf "%s%s" $subStr $password | toString | shuffle | b64enc | quote }} + {{- else }} + {{- $password = randAlphaNum $passwordLength | b64enc | quote }} + {{- end }} +{{- end -}} +{{- printf "%s" $password -}} +{{- end -}} + +{{/* +Returns whether a previous generated secret already exists + +Usage: +{{ include "common.secrets.exists" (dict "secret" "secret-name" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - context - Context - Required - Parent context. +*/}} +{{- define "common.secrets.exists" -}} +{{- $secret := (lookup "v1" "Secret" $.context.Release.Namespace .secret) }} +{{- if $secret }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/cron/charts/common/templates/_storage.tpl b/helm/airbyte-v1/charts/cron/charts/common/templates/_storage.tpl new file mode 100644 index 00000000..60e2a844 --- /dev/null +++ b/helm/airbyte-v1/charts/cron/charts/common/templates/_storage.tpl @@ -0,0 +1,23 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper Storage Class +{{ include "common.storage.class" ( dict "persistence" .Values.path.to.the.persistence "global" $) }} +*/}} +{{- define "common.storage.class" -}} + +{{- $storageClass := .persistence.storageClass -}} +{{- if .global -}} + {{- if .global.storageClass -}} + {{- $storageClass = .global.storageClass -}} + {{- end -}} +{{- end -}} + +{{- if $storageClass -}} + {{- if (eq "-" $storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" $storageClass -}} + {{- end -}} +{{- end -}} + +{{- end -}} diff --git a/helm/airbyte-v1/charts/cron/charts/common/templates/_tplvalues.tpl b/helm/airbyte-v1/charts/cron/charts/common/templates/_tplvalues.tpl new file mode 100644 index 00000000..2db16685 --- /dev/null +++ b/helm/airbyte-v1/charts/cron/charts/common/templates/_tplvalues.tpl @@ -0,0 +1,13 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Renders a value that contains template. +Usage: +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $) }} +*/}} +{{- define "common.tplvalues.render" -}} + {{- if typeIs "string" .value }} + {{- tpl .value .context }} + {{- else }} + {{- tpl (.value | toYaml) .context }} + {{- end }} +{{- end -}} diff --git a/helm/airbyte-v1/charts/cron/charts/common/templates/_utils.tpl b/helm/airbyte-v1/charts/cron/charts/common/templates/_utils.tpl new file mode 100644 index 00000000..8c22b2a3 --- /dev/null +++ b/helm/airbyte-v1/charts/cron/charts/common/templates/_utils.tpl @@ -0,0 +1,62 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Print instructions to get a secret value. +Usage: +{{ include "common.utils.secret.getvalue" (dict "secret" "secret-name" "field" "secret-value-field" "context" $) }} +*/}} +{{- define "common.utils.secret.getvalue" -}} +{{- $varname := include "common.utils.fieldToEnvVar" . -}} +export {{ $varname }}=$(kubectl get secret --namespace {{ .context.Release.Namespace | quote }} {{ .secret }} -o jsonpath="{.data.{{ .field }}}" | base64 -d) +{{- end -}} + +{{/* +Build env var name given a field +Usage: +{{ include "common.utils.fieldToEnvVar" dict "field" "my-password" }} +*/}} +{{- define "common.utils.fieldToEnvVar" -}} + {{- $fieldNameSplit := splitList "-" .field -}} + {{- $upperCaseFieldNameSplit := list -}} + + {{- range $fieldNameSplit -}} + {{- $upperCaseFieldNameSplit = append $upperCaseFieldNameSplit ( upper . ) -}} + {{- end -}} + + {{ join "_" $upperCaseFieldNameSplit }} +{{- end -}} + +{{/* +Gets a value from .Values given +Usage: +{{ include "common.utils.getValueFromKey" (dict "key" "path.to.key" "context" $) }} +*/}} +{{- define "common.utils.getValueFromKey" -}} +{{- $splitKey := splitList "." .key -}} +{{- $value := "" -}} +{{- $latestObj := $.context.Values -}} +{{- range $splitKey -}} + {{- if not $latestObj -}} + {{- printf "please review the entire path of '%s' exists in values" $.key | fail -}} + {{- end -}} + {{- $value = ( index $latestObj . ) -}} + {{- $latestObj = $value -}} +{{- end -}} +{{- printf "%v" (default "" $value) -}} +{{- end -}} + +{{/* +Returns first .Values key with a defined value or first of the list if all non-defined +Usage: +{{ include "common.utils.getKeyFromList" (dict "keys" (list "path.to.key1" "path.to.key2") "context" $) }} +*/}} +{{- define "common.utils.getKeyFromList" -}} +{{- $key := first .keys -}} +{{- $reverseKeys := reverse .keys }} +{{- range $reverseKeys }} + {{- $value := include "common.utils.getValueFromKey" (dict "key" . "context" $.context ) }} + {{- if $value -}} + {{- $key = . }} + {{- end -}} +{{- end -}} +{{- printf "%s" $key -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/cron/charts/common/templates/_warnings.tpl b/helm/airbyte-v1/charts/cron/charts/common/templates/_warnings.tpl new file mode 100644 index 00000000..ae10fa41 --- /dev/null +++ b/helm/airbyte-v1/charts/cron/charts/common/templates/_warnings.tpl @@ -0,0 +1,14 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Warning about using rolling tag. +Usage: +{{ include "common.warnings.rollingTag" .Values.path.to.the.imageRoot }} +*/}} +{{- define "common.warnings.rollingTag" -}} + +{{- if and (contains "bitnami/" .repository) (not (.tag | toString | regexFind "-r\\d+$|sha256:")) }} +WARNING: Rolling tag detected ({{ .repository }}:{{ .tag }}), please note that it is strongly recommended to avoid using rolling tags in a production environment. ++info https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/ +{{- end }} + +{{- end -}} diff --git a/helm/airbyte-v1/charts/cron/charts/common/templates/validations/_cassandra.tpl b/helm/airbyte-v1/charts/cron/charts/common/templates/validations/_cassandra.tpl new file mode 100644 index 00000000..ded1ae3b --- /dev/null +++ b/helm/airbyte-v1/charts/cron/charts/common/templates/validations/_cassandra.tpl @@ -0,0 +1,72 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate Cassandra required passwords are not empty. + +Usage: +{{ include "common.validations.values.cassandra.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where Cassandra values are stored, e.g: "cassandra-passwords-secret" + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.cassandra.passwords" -}} + {{- $existingSecret := include "common.cassandra.values.existingSecret" . -}} + {{- $enabled := include "common.cassandra.values.enabled" . -}} + {{- $dbUserPrefix := include "common.cassandra.values.key.dbUser" . -}} + {{- $valueKeyPassword := printf "%s.password" $dbUserPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "cassandra-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.cassandra.values.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.cassandra.values.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.cassandra.dbUser.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.dbUser.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled cassandra. + +Usage: +{{ include "common.cassandra.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.cassandra.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.cassandra.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key dbUser + +Usage: +{{ include "common.cassandra.values.key.dbUser" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.cassandra.values.key.dbUser" -}} + {{- if .subchart -}} + cassandra.dbUser + {{- else -}} + dbUser + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/cron/charts/common/templates/validations/_mariadb.tpl b/helm/airbyte-v1/charts/cron/charts/common/templates/validations/_mariadb.tpl new file mode 100644 index 00000000..b6906ff7 --- /dev/null +++ b/helm/airbyte-v1/charts/cron/charts/common/templates/validations/_mariadb.tpl @@ -0,0 +1,103 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MariaDB required passwords are not empty. + +Usage: +{{ include "common.validations.values.mariadb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MariaDB values are stored, e.g: "mysql-passwords-secret" + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mariadb.passwords" -}} + {{- $existingSecret := include "common.mariadb.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mariadb.values.enabled" . -}} + {{- $architecture := include "common.mariadb.values.architecture" . -}} + {{- $authPrefix := include "common.mariadb.values.key.auth" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mariadb-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- if not (empty $valueUsername) -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mariadb-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replication") -}} + {{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mariadb-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mariadb.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mariadb. + +Usage: +{{ include "common.mariadb.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mariadb.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mariadb.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mariadb.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mariadb.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.key.auth" -}} + {{- if .subchart -}} + mariadb.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/cron/charts/common/templates/validations/_mongodb.tpl b/helm/airbyte-v1/charts/cron/charts/common/templates/validations/_mongodb.tpl new file mode 100644 index 00000000..f820ec10 --- /dev/null +++ b/helm/airbyte-v1/charts/cron/charts/common/templates/validations/_mongodb.tpl @@ -0,0 +1,108 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MongoDB® required passwords are not empty. + +Usage: +{{ include "common.validations.values.mongodb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MongoDB® values are stored, e.g: "mongodb-passwords-secret" + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mongodb.passwords" -}} + {{- $existingSecret := include "common.mongodb.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mongodb.values.enabled" . -}} + {{- $authPrefix := include "common.mongodb.values.key.auth" . -}} + {{- $architecture := include "common.mongodb.values.architecture" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyDatabase := printf "%s.database" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicaSetKey := printf "%s.replicaSetKey" $authPrefix -}} + {{- $valueKeyAuthEnabled := printf "%s.enabled" $authPrefix -}} + + {{- $authEnabled := include "common.utils.getValueFromKey" (dict "key" $valueKeyAuthEnabled "context" .context) -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") (eq $authEnabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mongodb-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- $valueDatabase := include "common.utils.getValueFromKey" (dict "key" $valueKeyDatabase "context" .context) }} + {{- if and $valueUsername $valueDatabase -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mongodb-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replicaset") -}} + {{- $requiredReplicaSetKey := dict "valueKey" $valueKeyReplicaSetKey "secret" .secret "field" "mongodb-replica-set-key" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicaSetKey -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mongodb.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDb is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mongodb.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mongodb. + +Usage: +{{ include "common.mongodb.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mongodb.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mongodb.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mongodb.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.key.auth" -}} + {{- if .subchart -}} + mongodb.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mongodb.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mongodb.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/cron/charts/common/templates/validations/_mysql.tpl b/helm/airbyte-v1/charts/cron/charts/common/templates/validations/_mysql.tpl new file mode 100644 index 00000000..74472a06 --- /dev/null +++ b/helm/airbyte-v1/charts/cron/charts/common/templates/validations/_mysql.tpl @@ -0,0 +1,103 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MySQL required passwords are not empty. + +Usage: +{{ include "common.validations.values.mysql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MySQL values are stored, e.g: "mysql-passwords-secret" + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mysql.passwords" -}} + {{- $existingSecret := include "common.mysql.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mysql.values.enabled" . -}} + {{- $architecture := include "common.mysql.values.architecture" . -}} + {{- $authPrefix := include "common.mysql.values.key.auth" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mysql-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- if not (empty $valueUsername) -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mysql-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replication") -}} + {{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mysql-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mysql.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mysql.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mysql. + +Usage: +{{ include "common.mysql.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mysql.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mysql.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mysql.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mysql.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mysql.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.key.auth" -}} + {{- if .subchart -}} + mysql.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/cron/charts/common/templates/validations/_postgresql.tpl b/helm/airbyte-v1/charts/cron/charts/common/templates/validations/_postgresql.tpl new file mode 100644 index 00000000..164ec0d0 --- /dev/null +++ b/helm/airbyte-v1/charts/cron/charts/common/templates/validations/_postgresql.tpl @@ -0,0 +1,129 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate PostgreSQL required passwords are not empty. + +Usage: +{{ include "common.validations.values.postgresql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where postgresql values are stored, e.g: "postgresql-passwords-secret" + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.postgresql.passwords" -}} + {{- $existingSecret := include "common.postgresql.values.existingSecret" . -}} + {{- $enabled := include "common.postgresql.values.enabled" . -}} + {{- $valueKeyPostgresqlPassword := include "common.postgresql.values.key.postgressPassword" . -}} + {{- $valueKeyPostgresqlReplicationEnabled := include "common.postgresql.values.key.replicationPassword" . -}} + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + {{- $requiredPostgresqlPassword := dict "valueKey" $valueKeyPostgresqlPassword "secret" .secret "field" "postgresql-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlPassword -}} + + {{- $enabledReplication := include "common.postgresql.values.enabled.replication" . -}} + {{- if (eq $enabledReplication "true") -}} + {{- $requiredPostgresqlReplicationPassword := dict "valueKey" $valueKeyPostgresqlReplicationEnabled "secret" .secret "field" "postgresql-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to decide whether evaluate global values. + +Usage: +{{ include "common.postgresql.values.use.global" (dict "key" "key-of-global" "context" $) }} +Params: + - key - String - Required. Field to be evaluated within global, e.g: "existingSecret" +*/}} +{{- define "common.postgresql.values.use.global" -}} + {{- if .context.Values.global -}} + {{- if .context.Values.global.postgresql -}} + {{- index .context.Values.global.postgresql .key | quote -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.postgresql.values.existingSecret" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.existingSecret" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "existingSecret" "context" .context) -}} + + {{- if .subchart -}} + {{- default (.context.Values.postgresql.existingSecret | quote) $globalValue -}} + {{- else -}} + {{- default (.context.Values.existingSecret | quote) $globalValue -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled postgresql. + +Usage: +{{ include "common.postgresql.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key postgressPassword. + +Usage: +{{ include "common.postgresql.values.key.postgressPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.postgressPassword" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "postgresqlUsername" "context" .context) -}} + + {{- if not $globalValue -}} + {{- if .subchart -}} + postgresql.postgresqlPassword + {{- else -}} + postgresqlPassword + {{- end -}} + {{- else -}} + global.postgresql.postgresqlPassword + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled.replication. + +Usage: +{{ include "common.postgresql.values.enabled.replication" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.enabled.replication" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.replication.enabled -}} + {{- else -}} + {{- printf "%v" .context.Values.replication.enabled -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key replication.password. + +Usage: +{{ include "common.postgresql.values.key.replicationPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.replicationPassword" -}} + {{- if .subchart -}} + postgresql.replication.password + {{- else -}} + replication.password + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/cron/charts/common/templates/validations/_redis.tpl b/helm/airbyte-v1/charts/cron/charts/common/templates/validations/_redis.tpl new file mode 100644 index 00000000..dcccfc1a --- /dev/null +++ b/helm/airbyte-v1/charts/cron/charts/common/templates/validations/_redis.tpl @@ -0,0 +1,76 @@ + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate Redis® required passwords are not empty. + +Usage: +{{ include "common.validations.values.redis.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where redis values are stored, e.g: "redis-passwords-secret" + - subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.redis.passwords" -}} + {{- $enabled := include "common.redis.values.enabled" . -}} + {{- $valueKeyPrefix := include "common.redis.values.keys.prefix" . -}} + {{- $standarizedVersion := include "common.redis.values.standarized.version" . }} + + {{- $existingSecret := ternary (printf "%s%s" $valueKeyPrefix "auth.existingSecret") (printf "%s%s" $valueKeyPrefix "existingSecret") (eq $standarizedVersion "true") }} + {{- $existingSecretValue := include "common.utils.getValueFromKey" (dict "key" $existingSecret "context" .context) }} + + {{- $valueKeyRedisPassword := ternary (printf "%s%s" $valueKeyPrefix "auth.password") (printf "%s%s" $valueKeyPrefix "password") (eq $standarizedVersion "true") }} + {{- $valueKeyRedisUseAuth := ternary (printf "%s%s" $valueKeyPrefix "auth.enabled") (printf "%s%s" $valueKeyPrefix "usePassword") (eq $standarizedVersion "true") }} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $useAuth := include "common.utils.getValueFromKey" (dict "key" $valueKeyRedisUseAuth "context" .context) -}} + {{- if eq $useAuth "true" -}} + {{- $requiredRedisPassword := dict "valueKey" $valueKeyRedisPassword "secret" .secret "field" "redis-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRedisPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled redis. + +Usage: +{{ include "common.redis.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.redis.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.redis.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right prefix path for the values + +Usage: +{{ include "common.redis.values.key.prefix" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false +*/}} +{{- define "common.redis.values.keys.prefix" -}} + {{- if .subchart -}}redis.{{- else -}}{{- end -}} +{{- end -}} + +{{/* +Checks whether the redis chart's includes the standarizations (version >= 14) + +Usage: +{{ include "common.redis.values.standarized.version" (dict "context" $) }} +*/}} +{{- define "common.redis.values.standarized.version" -}} + + {{- $standarizedAuth := printf "%s%s" (include "common.redis.values.keys.prefix" .) "auth" -}} + {{- $standarizedAuthValues := include "common.utils.getValueFromKey" (dict "key" $standarizedAuth "context" .context) }} + + {{- if $standarizedAuthValues -}} + {{- true -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/cron/charts/common/templates/validations/_validations.tpl b/helm/airbyte-v1/charts/cron/charts/common/templates/validations/_validations.tpl new file mode 100644 index 00000000..9a814cf4 --- /dev/null +++ b/helm/airbyte-v1/charts/cron/charts/common/templates/validations/_validations.tpl @@ -0,0 +1,46 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate values must not be empty. + +Usage: +{{- $validateValueConf00 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-00") -}} +{{- $validateValueConf01 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-01") -}} +{{ include "common.validations.values.empty" (dict "required" (list $validateValueConf00 $validateValueConf01) "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" +*/}} +{{- define "common.validations.values.multiple.empty" -}} + {{- range .required -}} + {{- include "common.validations.values.single.empty" (dict "valueKey" .valueKey "secret" .secret "field" .field "context" $.context) -}} + {{- end -}} +{{- end -}} + +{{/* +Validate a value must not be empty. + +Usage: +{{ include "common.validations.value.empty" (dict "valueKey" "mariadb.password" "secret" "secretName" "field" "my-password" "subchart" "subchart" "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" + - subchart - String - Optional - Name of the subchart that the validated password is part of. +*/}} +{{- define "common.validations.values.single.empty" -}} + {{- $value := include "common.utils.getValueFromKey" (dict "key" .valueKey "context" .context) }} + {{- $subchart := ternary "" (printf "%s." .subchart) (empty .subchart) }} + + {{- if not $value -}} + {{- $varname := "my-value" -}} + {{- $getCurrentValue := "" -}} + {{- if and .secret .field -}} + {{- $varname = include "common.utils.fieldToEnvVar" . -}} + {{- $getCurrentValue = printf " To get the current value:\n\n %s\n" (include "common.utils.secret.getvalue" .) -}} + {{- end -}} + {{- printf "\n '%s' must not be empty, please add '--set %s%s=$%s' to the command.%s" .valueKey $subchart .valueKey $varname $getCurrentValue -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/cron/charts/common/values.yaml b/helm/airbyte-v1/charts/cron/charts/common/values.yaml new file mode 100644 index 00000000..f2df68e5 --- /dev/null +++ b/helm/airbyte-v1/charts/cron/charts/common/values.yaml @@ -0,0 +1,5 @@ +## bitnami/common +## It is required by CI/CD tools and processes. +## @skip exampleValue +## +exampleValue: common-chart diff --git a/helm/airbyte-v1/charts/cron/templates/_helpers.tpl b/helm/airbyte-v1/charts/cron/templates/_helpers.tpl new file mode 100644 index 00000000..174aebfe --- /dev/null +++ b/helm/airbyte-v1/charts/cron/templates/_helpers.tpl @@ -0,0 +1,59 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "airbyte.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airbyte.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airbyte.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "airbyte.labels" -}} +helm.sh/chart: {{ include "airbyte.chart" . }} +{{ include "airbyte.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "airbyte.selectorLabels" -}} +app.kubernetes.io/name: {{ include "airbyte.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Define db secret +*/}} + +{{- define "database.secret.name" -}} +{{- printf "%s-postgresql" .Release.Name }} +{{- end }} \ No newline at end of file diff --git a/helm/airbyte-v1/charts/cron/templates/_images.tpl b/helm/airbyte-v1/charts/cron/templates/_images.tpl new file mode 100644 index 00000000..6c32ec44 --- /dev/null +++ b/helm/airbyte-v1/charts/cron/templates/_images.tpl @@ -0,0 +1,17 @@ +{{- define "imageUrl" -}} +{{- $img := index . 0 -}} +{{- $root := index . 1 -}} + +{{- $reg := $root.Values.global.image.registry -}} +{{/* ensure the registry has a trailing slash, if set */}} +{{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} + +{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{- if (eq (typeOf $img) "string") -}} +{{- printf "%s%s" $reg (tpl $img $root) | quote -}} +{{- else -}} +{{- $tag := coalesce $img.tag $root.Values.global.image.tag $root.Chart.AppVersion -}} +{{- printf "%s%s:%s" $reg $img.repository $tag | quote -}} +{{- end -}} + +{{- end -}} \ No newline at end of file diff --git a/helm/airbyte-v1/charts/cron/templates/cron-secrets.yaml b/helm/airbyte-v1/charts/cron/templates/cron-secrets.yaml new file mode 100644 index 00000000..1d22b17e --- /dev/null +++ b/helm/airbyte-v1/charts/cron/templates/cron-secrets.yaml @@ -0,0 +1,18 @@ +# Create secrets only for the local deployment +{{- if .Values.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: cron-secrets + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" + +type: Opaque +data: + {{- range $k, $v := mergeOverwrite .Values.secrets .Values.global.secrets }} + {{ $k }}: {{ if $v }}{{ $v | b64enc }} {{else}}""{{end}} + {{- end }} + {{- end }} diff --git a/helm/airbyte-v1/charts/cron/templates/deployment.yaml b/helm/airbyte-v1/charts/cron/templates/deployment.yaml new file mode 100644 index 00000000..f64fb6a6 --- /dev/null +++ b/helm/airbyte-v1/charts/cron/templates/deployment.yaml @@ -0,0 +1,187 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.names.fullname" . }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- if .Values.extraLabels }} + {{ toYaml (mergeOverwrite .Values.extraLabels .Values.global.extraLabels) | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "airbyte.selectorLabels" . | nindent 6 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 6 }} + {{- end }} + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 8 }} + {{- end }} + {{- if .Values.podLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ .Values.global.serviceAccountName }} + automountServiceAccountToken: true + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.extraInitContainers }} + initContainers: + {{- toYaml .Values.extraInitContainers | nindent 6 }} + {{- end }} + containers: + - name: airbyte-cron + image: {{ include "imageUrl" (list .Values.image $) }} + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + env: + {{- if eq .Values.global.deploymentMode "oss" }} + - name: AIRBYTE_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_VERSION + - name: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + - name: MICROMETER_METRICS_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MICROMETER_METRICS_ENABLED + - name: MICROMETER_METRICS_STATSD_FLAVOR + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MICROMETER_METRICS_STATSD_FLAVOR + - name: MICRONAUT_ENVIRONMENTS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CRON_MICRONAUT_ENVIRONMENTS + - name: TEMPORAL_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TEMPORAL_HOST + - name: SEGMENT_WRITE_KEY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: SEGMENT_WRITE_KEY + - name: STATSD_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: STATSD_HOST + - name: STATSD_PORT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: STATSD_PORT + - name: TRACKING_STRATEGY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TRACKING_STRATEGY + - name: WORKFLOW_FAILURE_RESTART_DELAY_SECONDS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKFLOW_FAILURE_RESTART_DELAY_SECONDS + - name: WORKLOAD_API_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: {{ index .Values "workloadApi" "bearerTokenSecretName" | default (printf "%s-airbyte-secrets" .Release.Name ) }} + key: {{ index .Values "workloadApi" "bearerTokenSecretKey" | default "WORKLOAD_API_BEARER_TOKEN" }} + - name: WORKLOAD_API_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKLOAD_API_HOST + - name: WORKSPACE_DOCKER_MOUNT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKSPACE_DOCKER_MOUNT + - name: WORKSPACE_ROOT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKSPACE_ROOT + + # Database + {{- include "airbyte.database.envs" . | nindent 10 }} + + {{- end }} + + + # Values from secret + {{- if .Values.secrets }} + {{- range $k, $v := .Values.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: cron-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if or .Values.env_vars .Values.global.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + # Values from extraEnv for more compability(if you want to use external secret source or other stuff) + {{- if .Values.extraEnv }} + {{- toYaml .Values.extraEnv | nindent 10 }} + {{- end }} + + {{- if .Values.containerSecurityContext }} + securityContext: {{- toYaml .Values.containerSecurityContext | nindent 12 }} + {{- end }} + + {{- if .Values.resources }} + resources: {{- toYaml .Values.resources | nindent 12 }} + {{- end }} + + {{- if .Values.extraVolumeMounts }} + volumeMounts: {{ toYaml .Values.extraVolumeMounts | nindent 10 }} + {{- end }} + + {{- if .Values.extraContainers }} + {{ toYaml .Values.extraContainers | nindent 8 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 8 }} + {{- end }} + securityContext: {{- toYaml .Values.podSecurityContext | nindent 10 }} + {{- if .Values.extraVolumes }} + volumes: {{ toYaml .Values.extraVolumes | nindent 6 }} + {{- end }} diff --git a/helm/airbyte-v1/charts/cron/values.yaml b/helm/airbyte-v1/charts/cron/values.yaml new file mode 100644 index 00000000..1bbace43 --- /dev/null +++ b/helm/airbyte-v1/charts/cron/values.yaml @@ -0,0 +1,233 @@ +global: + serviceAccountName: placeholderServiceAccount + deploymentMode: oss + configMapName: "" + secretName: "" + extraContainers: [] + ## extraSelectorLabels [object] - use to specify own additional selector labels for deployment + extraSelectorLabels: {} + ## extraLabels [object] - use to specify own additional labels for deployment + extraLabels: {} + database: + secretName: "" + secretValue: "" + secrets: {} + + # Docker image config that will apply to all images. + image: + # Docker registry to pull platform images from, e.g. http://my-registry:8000/ + registry: "" + # Image tag to use for airbyte images. + # Does not include non-airbyte images such as busybox, temporal, minio, etc. + tag: "" + +enabled: true +## cron.replicaCount Number of cron replicas +replicaCount: 1 + +## cron.image.repository The repository to use for the airbyte cron image. +## cron.image.pullPolicy the pull policy to use for the airbyte cron image +## cron.image.tag The airbyte cron image tag. Defaults to the chart's AppVersion +image: + repository: airbyte/cron + pullPolicy: IfNotPresent + +## cron.podAnnotations [object] Add extra annotations to the cron pod(s) +## +podAnnotations: {} + +## cron.podLabels [object] Add extra labels to the cron pod(s) +## +podLabels: {} + +## cron.containerSecurityContext Security context for the container +## Examples: +## containerSecurityContext: +## runAsNonRoot: true +## runAsUser: 1000 +## readOnlyRootFilesystem: true +containerSecurityContext: {} + +## Configure extra options for the cron containers' liveness and readiness probes +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes +## cron.livenessProbe.enabled Enable livenessProbe on the cron +## cron.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe +## cron.livenessProbe.periodSeconds Period seconds for livenessProbe +## cron.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe +## cron.livenessProbe.failureThreshold Failure threshold for livenessProbe +## cron.livenessProbe.successThreshold Success threshold for livenessProbe +## +livenessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + +## cron.readinessProbe.enabled Enable readinessProbe on the cron +## cron.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe +## cron.readinessProbe.periodSeconds Period seconds for readinessProbe +## cron.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe +## cron.readinessProbe.failureThreshold Failure threshold for readinessProbe +## cron.readinessProbe.successThreshold Success threshold for readinessProbe +## +readinessProbe: + enabled: true + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + + ## server.containerSecurityContext Security context for the container + ## Examples: + ## containerSecurityContext: + ## runAsNonRoot: true + ## runAsUser: 1000 + ## readOnlyRootFilesystem: true + containerSecurityContext: {} + +## Web app resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## We usually recommend not to specify default resources and to leave this as a conscious +## choice for the user. This also increases chances charts run on environments with little +## resources, such as Minikube. If you do want to specify resources, uncomment the following +## lines, adjust them as necessary, and remove the curly braces after 'resources:'. +## cron.resources.limits [object] The resources limits for the Web container +## cron.resources.requests [object] The requested resources for the Web container +resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + +## cron.nodeSelector [object] Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## cron.tolerations [array] Tolerations for cron pod assignment. +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## cron.affinity [object] Affinity and anti-affinity for cron pod assignment. +## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## cron.extraVolumeMounts [array] Additional volumeMounts for cron container(s). +## Examples (when using `cron.containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumeMounts: +## - name: var-run +## mountPath: /var/run/ +## - name: var-cache-nginx +## mountPath: /var/cache/nginx +## - mountPath: /etc/nginx/conf.d +## name: nginx-conf-d +## +extraVolumeMounts: [] + +## cron.extraVolumes [array] Additional volumes for cron pod(s). +## Examples (when using `cron.containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumes: +## - name: var-run +## emptyDir: {} +## - name: var-cache-nginx +## emptyDir: {} +## - name: nginx-conf-d +## emptyDir: {} +## +extraVolumes: [] + +## extraContainer [array] Additional container for server pod(s) +## Example: +# extraContainers: +# - name: otel_collector +# image: somerepo/someimage:sometag +# args: [ +# "--important-args" +# ] +# ports: +# - containerPort: 443 +# volumeMounts: +# - name: volumeMountCool +# mountPath: /some/path +# readOnly: true +extraContainers: [] + +## extraInitContainers [array] Additional init containers for server pod(s) +## Example: +# extraInitContainers: +# - name: sleepy +# image: alpine +# command: ['sleep', '60'] + +extraInitContainers: [] + +## extraEnv [array] Supply extra env variables to main container using full notation +## Example: (With default env vars and values taken from generated config map) +# extraEnv: +# - name: POSTGRES_USER +# valueFrom: +# secretKeyRef: +# name: airbyte-secrets +# key: DATABASE_USER +# - name: POSTGRES_PWD +# valueFrom: +# secretKeyRef: +# name: airbyte-secrets +# key: DATABASE_PASSWORD +# - name: DYNAMIC_CONFIG_FILE_PATH +# value: "config/dynamicconfig/development.yaml" +# - name: DB +# value: "postgresql" +# - name: DB_PORT +# valueFrom: +# configMapKeyRef: +# name: airbyte-env +# key: DATABASE_PORT +# - name: POSTGRES_SEEDS +# valueFrom: +# configMapKeyRef: +# name: airbyte-env +# key: DATABASE_HOST +## +extraEnv: [] +## secrets [object] Supply additional secrets to container +## Example: +## secrets: +## DATABASE_PASSWORD: strong-password +## DATABASE_USER: my-db-user +secrets: {} + +## env_vars [object] Supply extra env variables to main container using simplified notation +## Example with default vars for cron: +# env_vars: +# AIRBYTE_VERSION: 0.40.4 + +# # Miscellaneous +# TRACKING_STRATEGY: segment +# API_URL: /api/v1/ +# INTERNAL_API_HOST: http://airbyte-server-svc:8001 + +env_vars: {} + + +## workloadApi: +## bearerTokenSecretName: workload-api-bearer-token-name kubernetes secret name where bearer token is stored +## bearerTokenSecretKey: workload-api-bearer-token-key kubernetes secret key where bearer token is stored +workloadApi: {} + + +## extraSelectorLabels [object] - use to specify own additional selector labels for deployment +extraSelectorLabels: {} +## extraLabels [object] - use to specify own additional labels for deployment +extraLabels: {} diff --git a/helm/airbyte-v1/charts/keycloak-setup/.gitignore b/helm/airbyte-v1/charts/keycloak-setup/.gitignore new file mode 100644 index 00000000..88e91e8a --- /dev/null +++ b/helm/airbyte-v1/charts/keycloak-setup/.gitignore @@ -0,0 +1,2 @@ +# Charts are downloaded at install time with `helm dep build`. +charts diff --git a/helm/airbyte-v1/charts/keycloak-setup/Chart.lock b/helm/airbyte-v1/charts/keycloak-setup/Chart.lock new file mode 100644 index 00000000..991eab27 --- /dev/null +++ b/helm/airbyte-v1/charts/keycloak-setup/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + version: 1.17.1 +digest: sha256:dacc73770a5640c011e067ff8840ddf89631fc19016c8d0a9e5ea160e7da8690 +generated: "2024-12-09T21:57:21.272173303Z" diff --git a/helm/airbyte-v1/charts/keycloak-setup/Chart.yaml b/helm/airbyte-v1/charts/keycloak-setup/Chart.yaml new file mode 100644 index 00000000..cab564c4 --- /dev/null +++ b/helm/airbyte-v1/charts/keycloak-setup/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +appVersion: 1.3.0 +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x +description: Helm chart to deploy airbyte-keycloak-setup +name: keycloak-setup +type: application +version: 1.3.0 diff --git a/helm/airbyte-v1/charts/keycloak-setup/README.md b/helm/airbyte-v1/charts/keycloak-setup/README.md new file mode 100644 index 00000000..724dc7c8 --- /dev/null +++ b/helm/airbyte-v1/charts/keycloak-setup/README.md @@ -0,0 +1,48 @@ +# keycloak-setup + +![Version: 0.67.17](https://img.shields.io/badge/Version-0.67.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) + +Helm chart to deploy airbyte-keycloak-setup + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://charts.bitnami.com/bitnami | common | 1.x.x | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| containerSecurityContext | object | `{}` | | +| enabled | bool | `true` | | +| env_vars | object | `{}` | | +| extraContainers | list | `[]` | | +| extraEnv | list | `[]` | | +| extraInitContainers | list | `[]` | | +| extraLabels | object | `{}` | | +| extraSelectorLabels | object | `{}` | | +| extraVolumeMounts | list | `[]` | | +| extraVolumes | list | `[]` | | +| global.database.secretName | string | `""` | | +| global.database.secretValue | string | `""` | | +| global.deploymentMode | string | `"oss"` | | +| global.edition | string | `"community"` | | +| global.env_vars | object | `{}` | | +| global.extraContainers | list | `[]` | | +| global.extraLabels | object | `{}` | | +| global.extraSelectorLabels | object | `{}` | | +| global.secretName | string | `""` | | +| global.secrets | object | `{}` | | +| global.serviceAccountName | string | `"placeholderServiceAccount"` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"airbyte/keycloak-setup"` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| podLabels | object | `{}` | | +| resources.limits | object | `{}` | | +| resources.requests | object | `{}` | | +| secrets | object | `{}` | | +| tolerations | list | `[]` | | + diff --git a/helm/airbyte-v1/charts/keycloak-setup/templates/_helpers.tpl b/helm/airbyte-v1/charts/keycloak-setup/templates/_helpers.tpl new file mode 100644 index 00000000..7be7bc1a --- /dev/null +++ b/helm/airbyte-v1/charts/keycloak-setup/templates/_helpers.tpl @@ -0,0 +1,51 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "airbyte.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airbyte.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airbyte.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "airbyte.labels" -}} +helm.sh/chart: {{ include "airbyte.chart" . }} +{{ include "airbyte.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "airbyte.selectorLabels" -}} +app.kubernetes.io/name: {{ include "airbyte.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} \ No newline at end of file diff --git a/helm/airbyte-v1/charts/keycloak-setup/templates/_images.tpl b/helm/airbyte-v1/charts/keycloak-setup/templates/_images.tpl new file mode 100644 index 00000000..6c32ec44 --- /dev/null +++ b/helm/airbyte-v1/charts/keycloak-setup/templates/_images.tpl @@ -0,0 +1,17 @@ +{{- define "imageUrl" -}} +{{- $img := index . 0 -}} +{{- $root := index . 1 -}} + +{{- $reg := $root.Values.global.image.registry -}} +{{/* ensure the registry has a trailing slash, if set */}} +{{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} + +{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{- if (eq (typeOf $img) "string") -}} +{{- printf "%s%s" $reg (tpl $img $root) | quote -}} +{{- else -}} +{{- $tag := coalesce $img.tag $root.Values.global.image.tag $root.Chart.AppVersion -}} +{{- printf "%s%s:%s" $reg $img.repository $tag | quote -}} +{{- end -}} + +{{- end -}} \ No newline at end of file diff --git a/helm/airbyte-v1/charts/keycloak-setup/templates/job.yaml b/helm/airbyte-v1/charts/keycloak-setup/templates/job.yaml new file mode 100644 index 00000000..41e534df --- /dev/null +++ b/helm/airbyte-v1/charts/keycloak-setup/templates/job.yaml @@ -0,0 +1,143 @@ +{{- if or (eq .Values.global.edition "pro") (eq .Values.global.edition "enterprise") }} +{{- if eq .Values.global.deploymentMode "oss" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "common.names.fullname" . }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- if .Values.extraLabels }} + {{ toYaml (mergeOverwrite .Values.extraLabels .Values.global.extraLabels) | nindent 4 }} + {{- end }} + {{- if .Values.podLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 4 }} + {{- end }} + annotations: + "helm.sh/hook": "post-install, post-upgrade" + "helm.sh/hook-delete-policy": "before-hook-creation" +spec: + completions: 1 + template: + spec: + serviceAccountName: {{ .Values.global.serviceAccountName }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + restartPolicy: Never + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 12 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 12 }} + {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 12 }} + {{- end }} + initContainers: + - name: keycloak-readiness-check + image: {{ include "imageUrl" (list .Values.initContainers.keycloakReadinessCheck.image $) }} + command: [ "sh", "-c", "until curl --output /dev/null --head --fail http://${KEYCLOAK_INTERNAL_HOST}/auth/health/ready; do sleep 1; done;" ] + env: + - name: KEYCLOAK_INTERNAL_HOST + valueFrom: + configMapKeyRef: + name: {{ .Values.global.configMapName | default (printf "%s-airbyte-env" .Release.Name) }} + key: KEYCLOAK_INTERNAL_HOST + securityContext: + {{- toYaml .Values.initContainerSecurityContext | nindent 14 }} + {{- if .Values.extraInitContainers }} + {{- toYaml .Values.extraInitContainers | nindent 8 }} + {{- end }} + containers: + - name: airbyte-keycloak-setup-container + image: {{ include "imageUrl" (list .Values.image $) }} + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + env: + - name: AIRBYTE_URL + valueFrom: + configMapKeyRef: + name: {{ .Values.global.configMapName | default (printf "%s-airbyte-env" .Release.Name) }} + key: AIRBYTE_URL + - name: KEYCLOAK_ADMIN_USER + valueFrom: + secretKeyRef: + name: {{ .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} + key: KEYCLOAK_ADMIN_USER + - name: KEYCLOAK_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} + key: KEYCLOAK_ADMIN_PASSWORD + - name: KEYCLOAK_INTERNAL_HOST + valueFrom: + configMapKeyRef: + name: {{ .Values.global.configMapName | default (printf "%s-airbyte-env" .Release.Name) }} + key: KEYCLOAK_INTERNAL_HOST + + {{- include "airbyte.database.envs" . | nindent 12 }} + {{- include "airbyte.enterprise.instanceAdmin" . | nindent 12 }} + {{- include "airbyte.enterprise.identityProvider" . | nindent 12 }} + {{- include "airbyte.keycloak.database.envs" . | nindent 12 }} + + # Values from secret + {{- if .Values.secrets }} + {{- range $k, $v := .Values.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: keycloak-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if .Values.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + # Values from extraEnv for more compability(if you want to use external secret source or other stuff) + {{- if .Values.extraEnv }} + {{- toYaml .Values.extraEnv | nindent 12 }} + {{- end }} + + {{- if .Values.resources }} + resources: {{- toYaml .Values.resources | nindent 12 }} + {{- end }} + volumeMounts: + {{- if .Values.global.airbyteYml }} + - name: airbyte-yml-volume + mountPath: /app/configs/airbyte.yml + subPath: fileContents + readOnly: true + {{- end }} + {{- if .Values.extraVolumeMounts }} + {{- toYaml .Values.extraVolumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.containerSecurityContext }} + securityContext: {{- toYaml .Values.containerSecurityContext | nindent 12 }} + {{- end }} + {{- if .Values.extraContainers }} + {{ toYaml .Values.extraContainers | nindent 8 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 8 }} + {{- end }} + securityContext: {{- toYaml .Values.podSecurityContext | nindent 10 }} + volumes: + {{- if .Values.global.airbyteYml }} + - name: airbyte-yml-volume + secret: + secretName: {{ .Release.Name }}-airbyte-yml + {{- end }} + {{- if .Values.extraVolumes }} + {{- toYaml .Values.extraVolumes | nindent 8 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/helm/airbyte-v1/charts/keycloak-setup/values.yaml b/helm/airbyte-v1/charts/keycloak-setup/values.yaml new file mode 100644 index 00000000..f134b437 --- /dev/null +++ b/helm/airbyte-v1/charts/keycloak-setup/values.yaml @@ -0,0 +1,190 @@ +## global.serviceAccountName Name of service account to be associated with service +## deploymentMode Determines deployment mode of airbyte +## secretName Overrides the secrate name with with credentials to S3 Bucket and logging +## database.secretName Name of database secret +## database.secretValue Value of database password key stored in secret +global: + serviceAccountName: placeholderServiceAccount + edition: community + deploymentMode: oss + secretName: "" + database: + secretName: "" + secretValue: "" + secrets: {} + env_vars: {} + extraContainers: [] + ## extraSelectorLabels [object] - use to specify own additional selector labels for deployment + extraSelectorLabels: {} + ## extraLabels [object] - use to specify own additional labels for deployment + extraLabels: {} + + # Docker image config that will apply to all images. + image: + # Docker registry to pull platform images from, e.g. http://my-registry:8000/ + registry: "" + # Image tag to use for airbyte images. + # Does not include non-airbyte images such as busybox, temporal, minio, etc. + tag: "" + + +enabled: true +## image.repository The repository to use for the airbyte keycloak-setup image. +## image.pullPolicy the pull policy to use for the airbyte keycloak-setup image +## image.tag The airbyte keycloak-setup image tag. Defaults to the chart's AppVersion +image: + repository: airbyte/keycloak-setup + pullPolicy: IfNotPresent + +## podAnnotations [object] Add extra annotations to the keycloak-setup pod +## +podAnnotations: {} + +## podLabels [object] Add extra labels to the keycloak-setup pod +## +podLabels: {} + +## containerSecurityContext Security context for the container +## Examples: +## containerSecurityContext: +## runAsNonRoot: true +## runAsUser: 1000 +## readOnlyRootFilesystem: true +containerSecurityContext: {} + +## nodeSelector [object] Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## tolerations [array] Tolerations for worker pod assignment. +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## Keycloak Setup resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## We usually recommend not to specify default resources and to leave this as a conscious +## choice for the user. This also increases chances charts run on environments with little +## resources, such as Minikube. If you do want to specify resources, uncomment the following +## lines, adjust them as necessary, and remove the curly braces after 'resources:'. +## keycloak-setup.resources.limits [object] The resources limits for the airbyte keycloak-setup image +## keycloak-setup.resources.requests [object] The requested resources for the airbyte keycloak-setup image +resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + +## affinity [object] Affinity and anti-affinity for keycloak-setup pod assignment. +## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## extraEnv [array] Supply extra env variables to main container using full notation +## Example: (With default env vars and values taken from generated config map) +## extraEnv: +## - name: AIRBYTE_VERSION +## valueFrom: +## configMapKeyRef: +## name: airbyte-env +## key: AIRBYTE_VERSION +## - name: DATABASE_HOST +## valueFrom: +## configMapKeyRef: +## name: airbyte-env +## key: DATABASE_HOST +## - name: DATABASE_PORT +## valueFrom: +## configMapKeyRef: +## name: airbyte-env +## key: DATABASE_PORT +## - name: DATABASE_PASSWORD +## valueFrom: +## secretKeyRef: +## name: airbyte-secrets +## key: DATABASE_PASSWORD +## - name: DATABASE_URL +## valueFrom: +## configMapKeyRef: +## name: airbyte-env +## key: DATABASE_URL +## - name: DATABASE_USER +## valueFrom: +## secretKeyRef: +## name: airbyte-secrets +## key: DATABASE_USER +## +extraEnv: [] + +## secrets [object] Supply additional secrets to container +## Example: +## secrets: +## DATABASE_PASSWORD: strong-password +## DATABASE_USER: my-db-user +secrets: {} + +## extraContainer [array] Additional container for keycloak pod(s) +## Example: +# extraContainers: +# - name: otel_collector +# image: somerepo/someimage:sometag +# args: [ +# "--important-args" +# ] +# ports: +# - containerPort: 443 +# volumeMounts: +# - name: volumeMountCool +# mountPath: /some/path +# readOnly: true +extraContainers: [] + +## extraInitContainers [array] Additional init containers for server pod(s) +## Example: +# extraInitContainers: +# - name: sleepy +# image: alpine +# command: ['sleep', '60'] + +extraInitContainers: [] + +## extraVolumeMounts [array] Additional volumeMounts for server container(s). +## Examples (when using `containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumeMounts: +## - name: tmpdir +## mountPath: /tmp +## +extraVolumeMounts: [] + +## extraVolumes [array] Additional volumes for server pod(s). +## Examples (when using `containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumes: +## - name: tmpdir +## emptyDir: {} +## +extraVolumes: [] + +## env_vars [object] Supply extra env variables to main container using simplified notation +## Example: +# env_vars: +# AIRBYTE_VERSION: 0.40.4 + +# # Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db +# DATABASE_HOST: airbyte-db-svc +# DATABASE_PORT: 5432 +# # translate manually DATABASE_URL: jdbc:postgresql://${DATABASE_HOST}:${DATABASE_PORT/${DATABASE_DB} +# DATABASE_URL: jdbc:postgresql://airbyte-db-svc:5432/airbyte +# DATABASE_PASSWORD: 12345 +# DATABASE_USER: airbyte +env_vars: {} + +## extraSelectorLabels [object] - use to specify own additional selector labels for deployment +extraSelectorLabels: {} +## extraLabels [object] - use to specify own additional labels for deployment +extraLabels: {} diff --git a/helm/airbyte-v1/charts/keycloak/.gitignore b/helm/airbyte-v1/charts/keycloak/.gitignore new file mode 100644 index 00000000..88e91e8a --- /dev/null +++ b/helm/airbyte-v1/charts/keycloak/.gitignore @@ -0,0 +1,2 @@ +# Charts are downloaded at install time with `helm dep build`. +charts diff --git a/helm/airbyte-v1/charts/keycloak/Chart.lock b/helm/airbyte-v1/charts/keycloak/Chart.lock new file mode 100644 index 00000000..cdb6f491 --- /dev/null +++ b/helm/airbyte-v1/charts/keycloak/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + version: 1.17.1 +digest: sha256:dacc73770a5640c011e067ff8840ddf89631fc19016c8d0a9e5ea160e7da8690 +generated: "2024-12-09T21:57:13.706413758Z" diff --git a/helm/airbyte-v1/charts/keycloak/Chart.yaml b/helm/airbyte-v1/charts/keycloak/Chart.yaml new file mode 100644 index 00000000..6be65627 --- /dev/null +++ b/helm/airbyte-v1/charts/keycloak/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +appVersion: 1.3.0 +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x +description: Helm chart to deploy airbyte-keycloak +name: keycloak +type: application +version: 1.3.0 diff --git a/helm/airbyte-v1/charts/keycloak/README.md b/helm/airbyte-v1/charts/keycloak/README.md new file mode 100644 index 00000000..5b86476d --- /dev/null +++ b/helm/airbyte-v1/charts/keycloak/README.md @@ -0,0 +1,71 @@ +# keycloak + +![Version: 0.67.17](https://img.shields.io/badge/Version-0.67.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) + +Helm chart to deploy airbyte-keycloak + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://charts.bitnami.com/bitnami | common | 1.x.x | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| bypassInit | bool | `false` | | +| containerSecurityContext | object | `{}` | | +| enabled | bool | `true` | | +| env_vars | object | `{}` | | +| extraContainers | list | `[]` | | +| extraEnv | list | `[]` | | +| extraInitContainers | list | `[]` | | +| extraLabels | object | `{}` | | +| extraSelectorLabels | object | `{}` | | +| extraVolumeMounts | list | `[]` | | +| extraVolumes | list | `[]` | | +| global.database.secretName | string | `""` | | +| global.database.secretValue | string | `""` | | +| global.deploymentMode | string | `"oss"` | airbyte deployment mode | +| global.edition | string | `"community"` | | +| global.env_vars | object | `{}` | | +| global.extraContainers | list | `[]` | | +| global.extraLabels | object | `{}` | | +| global.extraSelectorLabels | object | `{}` | | +| global.secretName | string | `""` | | +| global.secrets | object | `{}` | | +| global.serviceAccountName | string | `"placeholderServiceAccount"` | | +| headlessService.annotations | object | `{}` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"airbyte/keycloak"` | | +| livenessProbe.enabled | bool | `true` | | +| livenessProbe.failureThreshold | int | `3` | | +| livenessProbe.initialDelaySeconds | int | `30` | | +| livenessProbe.periodSeconds | int | `10` | | +| livenessProbe.successThreshold | int | `1` | | +| livenessProbe.timeoutSeconds | int | `1` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| podLabels | object | `{}` | | +| readinessProbe.enabled | bool | `true` | | +| readinessProbe.failureThreshold | int | `3` | | +| readinessProbe.initialDelaySeconds | int | `30` | | +| readinessProbe.periodSeconds | int | `10` | | +| readinessProbe.successThreshold | int | `1` | | +| readinessProbe.timeoutSeconds | int | `1` | | +| resources.limits | object | `{}` | | +| resources.requests | object | `{}` | | +| secrets | object | `{}` | | +| service.annotations | object | `{}` | | +| service.port | int | `8180` | | +| service.type | string | `"ClusterIP"` | | +| startupProbe.enabled | bool | `true` | | +| startupProbe.failureThreshold | int | `60` | | +| startupProbe.initialDelaySeconds | int | `30` | | +| startupProbe.periodSeconds | int | `5` | | +| startupProbe.successThreshold | int | `1` | | +| startupProbe.timeoutSeconds | int | `1` | | +| tolerations | list | `[]` | | + diff --git a/helm/airbyte-v1/charts/keycloak/templates/_helpers.tpl b/helm/airbyte-v1/charts/keycloak/templates/_helpers.tpl new file mode 100644 index 00000000..7be7bc1a --- /dev/null +++ b/helm/airbyte-v1/charts/keycloak/templates/_helpers.tpl @@ -0,0 +1,51 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "airbyte.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airbyte.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airbyte.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "airbyte.labels" -}} +helm.sh/chart: {{ include "airbyte.chart" . }} +{{ include "airbyte.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "airbyte.selectorLabels" -}} +app.kubernetes.io/name: {{ include "airbyte.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} \ No newline at end of file diff --git a/helm/airbyte-v1/charts/keycloak/templates/_images.tpl b/helm/airbyte-v1/charts/keycloak/templates/_images.tpl new file mode 100644 index 00000000..6c32ec44 --- /dev/null +++ b/helm/airbyte-v1/charts/keycloak/templates/_images.tpl @@ -0,0 +1,17 @@ +{{- define "imageUrl" -}} +{{- $img := index . 0 -}} +{{- $root := index . 1 -}} + +{{- $reg := $root.Values.global.image.registry -}} +{{/* ensure the registry has a trailing slash, if set */}} +{{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} + +{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{- if (eq (typeOf $img) "string") -}} +{{- printf "%s%s" $reg (tpl $img $root) | quote -}} +{{- else -}} +{{- $tag := coalesce $img.tag $root.Values.global.image.tag $root.Chart.AppVersion -}} +{{- printf "%s%s:%s" $reg $img.repository $tag | quote -}} +{{- end -}} + +{{- end -}} \ No newline at end of file diff --git a/helm/airbyte-v1/charts/keycloak/templates/service.yaml b/helm/airbyte-v1/charts/keycloak/templates/service.yaml new file mode 100644 index 00000000..3e7b5fea --- /dev/null +++ b/helm/airbyte-v1/charts/keycloak/templates/service.yaml @@ -0,0 +1,60 @@ +# If Cloud or Pro/Enterprise, render the keycloak service template. +{{- if or (eq .Values.global.deploymentMode "cloud") (eq .Values.global.edition "pro") (eq .Values.global.edition "enterprise") }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{.Release.Name }}-airbyte-keycloak-svc + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + {{ if and (eq .Values.service.type "NodePort") (.Values.service.nodePort) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + selector: + {{- include "airbyte.selectorLabels" . | nindent 4 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 4 }} + {{- end }} +--- +# This headless service sets ClusterIp: None so that it can be +# used for internal DNS resolution of all individual keycloak +# pods. Port 7800 is the default jgroups port that Keycloak +# expects to be open to enable Infinispan cache synchronization. +apiVersion: v1 +kind: Service +metadata: + name: {{.Release.Name }}-airbyte-keycloak-headless-svc + {{- with .Values.headlessService.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: jgroups + port: 7800 + targetPort: 7800 + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "airbyte.selectorLabels" . | nindent 4 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v1/charts/keycloak/templates/statefulset.yaml b/helm/airbyte-v1/charts/keycloak/templates/statefulset.yaml new file mode 100644 index 00000000..89769cff --- /dev/null +++ b/helm/airbyte-v1/charts/keycloak/templates/statefulset.yaml @@ -0,0 +1,193 @@ +# if Cloud or Pro/Enterprise, render the keycloak StatefulSet template. +{{- if or (eq .Values.global.deploymentMode "cloud") (eq .Values.global.edition "pro") (eq .Values.global.edition "enterprise") }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "common.names.fullname" . }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- if .Values.extraLabels }} + {{ toYaml (mergeOverwrite .Values.extraLabels .Values.global.extraLabels) | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicaCount }} + serviceName: {{.Release.Name }}-airbyte-keycloak-headless-svc + podManagementPolicy: OrderedReady # embedded infinispan does not handle parallel joins well, src https://github.com/keycloak/keycloak/issues/21108#issuecomment-1599010602 + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + {{- include "airbyte.selectorLabels" . | nindent 6 }} + {{- if .Values.extraLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 8 }} + {{- end }} + {{- if .Values.podLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ .Values.global.serviceAccountName }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + initContainers: + {{- if and (eq .Values.global.deploymentMode "oss") (ne .Values.bypassInit true)}} + # This init container will only executed if the deployment mode is "oss" and the bypassInit is not true. + - name: init-db + image: {{ include "imageUrl" (list .Values.initContainers.initDb.image $) }} + command: [ "sh", "-c" ] + args: + - > + PGPASSWORD=$DATABASE_PASSWORD psql -h $DATABASE_HOST -p $DATABASE_PORT -U $DATABASE_USER -d $DATABASE_DB -c "CREATE SCHEMA IF NOT EXISTS keycloak"; + env: + {{- include "airbyte.database.envs" . | nindent 12 }} + + securityContext: + {{- toYaml .Values.initContainerSecurityContext | nindent 14 }} + {{- end }} + {{- if .Values.extraInitContainers }} + {{- toYaml .Values.extraInitContainers | nindent 8 }} + {{- end }} + containers: + - name: airbyte-keycloak + image: {{ include "imageUrl" (list .Values.image $)}} + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + {{- if eq .Values.global.deploymentMode "oss" }} + - name: KEYCLOAK_ADMIN_USER + valueFrom: + secretKeyRef: + name: {{ .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} + key: KEYCLOAK_ADMIN_USER + - name: KEYCLOAK_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} + key: KEYCLOAK_ADMIN_PASSWORD + - name: KEYCLOAK_PORT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: KEYCLOAK_PORT + - name: JAVA_OPTS_APPEND + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: KEYCLOAK_JAVA_OPTS_APPEND + {{- include "airbyte.keycloak.database.envs" . | nindent 12 }} + {{- end }} # end if oss + + {{- if .Values.extraEnv }} + {{ .Values.extraEnv | toYaml | nindent 12 }} + {{- end }} + + # Values from secret + {{- if .Values.secrets }} + {{- range $k, $v := .Values.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: keycloak-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if .Values.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + - containerPort: 7800 + {{- if .Values.containerSecurityContext }} + securityContext: {{- toYaml .Values.containerSecurityContext | nindent 12 }} + {{- end }} + volumeMounts: + - name: keycloak-storage + mountPath: /opt/keycloak/data/infinispan + {{- if .Values.extraVolumeMounts }} + {{ toYaml .Values.extraVolumeMounts | nindent 10 }} + {{- end }} + {{- if .Values.resources }} + resources: {{- toYaml .Values.resources | nindent 12 }} + {{- end }} + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /auth/health/live + port: {{ .Values.service.port }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /auth/health/ready + port: {{ .Values.service.port }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + {{- if .Values.startupProbe.enabled }} + startupProbe: + httpGet: + path: /auth/health/started + port: {{ .Values.service.port }} + initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.startupProbe.periodSeconds }} + timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }} + successThreshold: {{ .Values.startupProbe.successThreshold }} + failureThreshold: {{ .Values.startupProbe.failureThreshold }} + {{- end }} + {{- if .Values.extraContainers }} + {{ toYaml .Values.extraContainers | nindent 8 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 8 }} + {{- end }} + securityContext: {{- toYaml .Values.podSecurityContext | nindent 10 }} + volumes: + {{- if .Values.extraVolumes }} + {{ toYaml .Values.extraVolumes | nindent 8 }} + {{- end }} + volumeClaimTemplates: + - metadata: + name: keycloak-storage + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi +{{- end }} diff --git a/helm/airbyte-v1/charts/keycloak/values.yaml b/helm/airbyte-v1/charts/keycloak/values.yaml new file mode 100644 index 00000000..30de3c4b --- /dev/null +++ b/helm/airbyte-v1/charts/keycloak/values.yaml @@ -0,0 +1,245 @@ +## global.serviceAccountName Name of service account to be associated with service +## deploymentMode Determines deployment mode of airbyte +## secretName Overrides the secrate name with with credentials to S3 Bucket and logging +## database.secretName Name of database secret +## database.secretValue Value of database password key stored in secret +global: + # serviceAccountName -- Service Account name override + serviceAccountName: placeholderServiceAccount + edition: community + # -- airbyte deployment mode + deploymentMode: oss + secretName: "" + database: + secretName: "" + secretValue: "" + secrets: {} + env_vars: {} + extraContainers: [] + ## extraSelectorLabels [object] - use to specify own additional selector labels for deployment + extraSelectorLabels: {} + ## extraLabels [object] - use to specify own additional labels for deployment + extraLabels: {} + + # Docker image config that will apply to all images. + image: + # Docker registry to pull platform images from, e.g. http://my-registry:8000/ + registry: "" + # Image tag to use for airbyte images. + # Does not include non-airbyte images such as busybox, temporal, minio, etc. + tag: "" + + +enabled: true + +bypassInit: false +## image.repository The repository to use for the airbyte keycloak image. +## image.pullPolicy the pull policy to use for the airbyte keycloak image +## image.tag The airbyte keycloak image tag. Defaults to the chart's AppVersion +image: + repository: airbyte/keycloak + pullPolicy: IfNotPresent + +## podAnnotations [object] Add extra annotations to the keycloak pod +## +podAnnotations: {} + +## podLabels [object] Add extra labels to the keycloak pod +## +podLabels: {} + +## Configure extra options for the keycloak containers' liveness and readiness probes +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes +## livenessProbe.enabled Enable livenessProbe on keycloak +## livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe +## livenessProbe.periodSeconds Period seconds for livenessProbe +## livenessProbe.timeoutSeconds Timeout seconds for livenessProbe +## livenessProbe.failureThreshold Failure threshold for livenessProbe +## livenessProbe.successThreshold Success threshold for livenessProbe +## +livenessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + +## readinessProbe.enabled Enable readinessProbe on keycloak +## readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe +## readinessProbe.periodSeconds Period seconds for readinessProbe +## readinessProbe.timeoutSeconds Timeout seconds for readinessProbe +## readinessProbe.failureThreshold Failure threshold for readinessProbe +## readinessProbe.successThreshold Success threshold for readinessProbe +## +readinessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + +startupProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 5 + timeoutSeconds: 1 + failureThreshold: 60 + successThreshold: 1 + +## containerSecurityContext Security context for the container +## Examples: +## containerSecurityContext: +## runAsNonRoot: true +## runAsUser: 1000 +## readOnlyRootFilesystem: true +containerSecurityContext: {} + +## nodeSelector [object] Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## tolerations [array] Tolerations for worker pod assignment. +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## service.type The service type to use for the API server +## service.port The service port to expose the API server on +service: + type: ClusterIP + port: 8180 + annotations: {} + +headlessService: + annotations: {} + +## Keycloak resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## We usually recommend not to specify default resources and to leave this as a conscious +## choice for the user. This also increases chances charts run on environments with little +## resources, such as Minikube. If you do want to specify resources, uncomment the following +## lines, adjust them as necessary, and remove the curly braces after 'resources:'. +## keycloak.resources.limits [object] The resources limits for the keycloak image +## keycloak.resources.requests [object] The requested resources for the keycloak image +resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + +## affinity [object] Affinity and anti-affinity for keycloak pod assignment. +## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## extraEnv [array] Supply extra env variables to main container using full notation +## Example: (With default env vars and values taken from generated config map) +## extraEnv: +## - name: AIRBYTE_VERSION +## valueFrom: +## configMapKeyRef: +## name: airbyte-env +## key: AIRBYTE_VERSION +## - name: DATABASE_HOST +## valueFrom: +## configMapKeyRef: +## name: airbyte-env +## key: DATABASE_HOST +## - name: DATABASE_PORT +## valueFrom: +## configMapKeyRef: +## name: airbyte-env +## key: DATABASE_PORT +## - name: DATABASE_PASSWORD +## valueFrom: +## secretKeyRef: +## name: airbyte-secrets +## key: DATABASE_PASSWORD +## - name: DATABASE_URL +## valueFrom: +## configMapKeyRef: +## name: airbyte-env +## key: DATABASE_URL +## - name: DATABASE_USER +## valueFrom: +## secretKeyRef: +## name: airbyte-secrets +## key: DATABASE_USER +## +extraEnv: [] + +## secrets [object] Supply additional secrets to container +## Example: +## secrets: +## DATABASE_PASSWORD: strong-password +## DATABASE_USER: my-db-user +secrets: {} + +## extraContainer [array] Additional container for keycloak pod(s) +## Example: +# extraContainers: +# - name: otel_collector +# image: somerepo/someimage:sometag +# args: [ +# "--important-args" +# ] +# ports: +# - containerPort: 443 +# volumeMounts: +# - name: volumeMountCool +# mountPath: /some/path +# readOnly: true +extraContainers: [] + +## extraInitContainers [array] Additional init containers for server pod(s) +## Example: +# extraInitContainers: +# - name: sleepy +# image: alpine +# command: ['sleep', '60'] + +extraInitContainers: [] + +## extraVolumeMounts [array] Additional volumeMounts for server container(s). +## Examples (when using `containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumeMounts: +## - name: tmpdir +## mountPath: /tmp +## +extraVolumeMounts: [] + +## extraVolumes [array] Additional volumes for server pod(s). +## Examples (when using `containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumes: +## - name: tmpdir +## emptyDir: {} +## +extraVolumes: [] + +## env_vars [object] Supply extra env variables to main container using simplified notation +## Example: +# env_vars: +# AIRBYTE_VERSION: 0.40.4 + +# # Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db +# DATABASE_HOST: airbyte-db-svc +# DATABASE_PORT: 5432 +# # translate manually DATABASE_URL: jdbc:postgresql://${DATABASE_HOST}:${DATABASE_PORT/${DATABASE_DB} +# DATABASE_URL: jdbc:postgresql://airbyte-db-svc:5432/airbyte +# DATABASE_PASSWORD: 12345 +# DATABASE_USER: airbyte +env_vars: {} + +## extraSelectorLabels [object] - use to specify own additional selector labels for deployment +extraSelectorLabels: {} +## extraLabels [object] - use to specify own additional labels for deployment +extraLabels: {} + diff --git a/helm/airbyte-v1/charts/metrics/.helmignore b/helm/airbyte-v1/charts/metrics/.helmignore new file mode 100644 index 00000000..f885da3f --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/.helmignore @@ -0,0 +1,25 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ + +ci.sh diff --git a/helm/airbyte-v1/charts/metrics/Chart.lock b/helm/airbyte-v1/charts/metrics/Chart.lock new file mode 100644 index 00000000..017faffa --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + version: 1.17.1 +digest: sha256:dacc73770a5640c011e067ff8840ddf89631fc19016c8d0a9e5ea160e7da8690 +generated: "2022-08-31T12:09:18.473209+01:00" diff --git a/helm/airbyte-v1/charts/metrics/Chart.yaml b/helm/airbyte-v1/charts/metrics/Chart.yaml new file mode 100644 index 00000000..a5127075 --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +appVersion: 1.3.0 +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x +description: Helm chart to deploy airbyte-metrics +name: metrics +type: application +version: 1.3.0 diff --git a/helm/airbyte-v1/charts/metrics/README.md b/helm/airbyte-v1/charts/metrics/README.md new file mode 100644 index 00000000..51fd055b --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/README.md @@ -0,0 +1,40 @@ +# metrics + +![Version: 0.67.17](https://img.shields.io/badge/Version-0.67.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) + +Helm chart to deploy airbyte-metrics + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://charts.bitnami.com/bitnami | common | 1.x.x | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| containerSecurityContext | object | `{}` | | +| enabled | bool | `true` | | +| env_vars | object | `{}` | | +| extraContainers | list | `[]` | | +| extraEnv | list | `[]` | | +| extraVolumeMounts | list | `[]` | | +| extraVolumes | list | `[]` | | +| global.database.secretName | string | `""` | | +| global.database.secretValue | string | `""` | | +| global.deploymentMode | string | `"oss"` | | +| global.extraContainers | list | `[]` | | +| global.serviceAccountName | string | `"placeholderServiceAccount"` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"airbyte/metrics-reporter"` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| podLabels | object | `{}` | | +| replicaCount | int | `1` | | +| resources.limits | object | `{}` | | +| resources.requests | object | `{}` | | +| secrets | object | `{}` | | +| tolerations | list | `[]` | | + diff --git a/helm/airbyte-v1/charts/metrics/charts/common/.helmignore b/helm/airbyte-v1/charts/metrics/charts/common/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/charts/common/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/airbyte-v1/charts/metrics/charts/common/Chart.yaml b/helm/airbyte-v1/charts/metrics/charts/common/Chart.yaml new file mode 100644 index 00000000..39edbb3c --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/charts/common/Chart.yaml @@ -0,0 +1,23 @@ +annotations: + category: Infrastructure +apiVersion: v2 +appVersion: 1.17.1 +description: A Library Helm Chart for grouping common logic between bitnami charts. + This chart is not deployable by itself. +home: https://github.com/bitnami/charts/tree/master/bitnami/common +icon: https://bitnami.com/downloads/logos/bitnami-mark.png +keywords: +- common +- helper +- template +- function +- bitnami +maintainers: +- name: Bitnami + url: https://github.com/bitnami/charts +name: common +sources: +- https://github.com/bitnami/charts +- https://www.bitnami.com/ +type: library +version: 1.17.1 diff --git a/helm/airbyte-v1/charts/metrics/charts/common/README.md b/helm/airbyte-v1/charts/metrics/charts/common/README.md new file mode 100644 index 00000000..a2ecd604 --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/charts/common/README.md @@ -0,0 +1,350 @@ +# Bitnami Common Library Chart + +A [Helm Library Chart](https://helm.sh/docs/topics/library_charts/#helm) for grouping common logic between bitnami charts. + +## TL;DR + +```yaml +dependencies: + - name: common + version: 1.x.x + repository: https://charts.bitnami.com/bitnami +``` + +```bash +$ helm dependency update +``` + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.names.fullname" . }} +data: + myvalue: "Hello World" +``` + +## Introduction + +This chart provides a common template helpers which can be used to develop new charts using [Helm](https://helm.sh) package manager. + +Bitnami charts can be used with [Kubeapps](https://kubeapps.dev/) for deployment and management of Helm Charts in clusters. + +## Prerequisites + +- Kubernetes 1.19+ +- Helm 3.2.0+ + +## Parameters + +The following table lists the helpers available in the library which are scoped in different sections. + +### Affinities + +| Helper identifier | Description | Expected Input | +|-------------------------------|------------------------------------------------------|------------------------------------------------| +| `common.affinities.nodes.soft` | Return a soft nodeAffinity definition | `dict "key" "FOO" "values" (list "BAR" "BAZ")` | +| `common.affinities.nodes.hard` | Return a hard nodeAffinity definition | `dict "key" "FOO" "values" (list "BAR" "BAZ")` | +| `common.affinities.pods.soft` | Return a soft podAffinity/podAntiAffinity definition | `dict "component" "FOO" "context" $` | +| `common.affinities.pods.hard` | Return a hard podAffinity/podAntiAffinity definition | `dict "component" "FOO" "context" $` | + +### Capabilities + +| Helper identifier | Description | Expected Input | +|------------------------------------------------|------------------------------------------------------------------------------------------------|-------------------| +| `common.capabilities.kubeVersion` | Return the target Kubernetes version (using client default if .Values.kubeVersion is not set). | `.` Chart context | +| `common.capabilities.cronjob.apiVersion` | Return the appropriate apiVersion for cronjob. | `.` Chart context | +| `common.capabilities.deployment.apiVersion` | Return the appropriate apiVersion for deployment. | `.` Chart context | +| `common.capabilities.statefulset.apiVersion` | Return the appropriate apiVersion for statefulset. | `.` Chart context | +| `common.capabilities.ingress.apiVersion` | Return the appropriate apiVersion for ingress. | `.` Chart context | +| `common.capabilities.rbac.apiVersion` | Return the appropriate apiVersion for RBAC resources. | `.` Chart context | +| `common.capabilities.crd.apiVersion` | Return the appropriate apiVersion for CRDs. | `.` Chart context | +| `common.capabilities.policy.apiVersion` | Return the appropriate apiVersion for podsecuritypolicy. | `.` Chart context | +| `common.capabilities.networkPolicy.apiVersion` | Return the appropriate apiVersion for networkpolicy. | `.` Chart context | +| `common.capabilities.apiService.apiVersion` | Return the appropriate apiVersion for APIService. | `.` Chart context | +| `common.capabilities.hpa.apiVersion` | Return the appropriate apiVersion for Horizontal Pod Autoscaler | `.` Chart context | +| `common.capabilities.supportsHelmVersion` | Returns true if the used Helm version is 3.3+ | `.` Chart context | + +### Errors + +| Helper identifier | Description | Expected Input | +|-----------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------| +| `common.errors.upgrade.passwords.empty` | It will ensure required passwords are given when we are upgrading a chart. If `validationErrors` is not empty it will throw an error and will stop the upgrade action. | `dict "validationErrors" (list $validationError00 $validationError01) "context" $` | + +### Images + +| Helper identifier | Description | Expected Input | +|-----------------------------|------------------------------------------------------|---------------------------------------------------------------------------------------------------------| +| `common.images.image` | Return the proper and full image name | `dict "imageRoot" .Values.path.to.the.image "global" $`, see [ImageRoot](#imageroot) for the structure. | +| `common.images.pullSecrets` | Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) | `dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global` | +| `common.images.renderPullSecrets` | Return the proper Docker Image Registry Secret Names (evaluates values as templates) | `dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "context" $` | + +### Ingress + +| Helper identifier | Description | Expected Input | +|-------------------------------------------|-------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.ingress.backend` | Generate a proper Ingress backend entry depending on the API version | `dict "serviceName" "foo" "servicePort" "bar"`, see the [Ingress deprecation notice](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/) for the syntax differences | +| `common.ingress.supportsPathType` | Prints "true" if the pathType field is supported | `.` Chart context | +| `common.ingress.supportsIngressClassname` | Prints "true" if the ingressClassname field is supported | `.` Chart context | +| `common.ingress.certManagerRequest` | Prints "true" if required cert-manager annotations for TLS signed certificates are set in the Ingress annotations | `dict "annotations" .Values.path.to.the.ingress.annotations` | + +### Labels + +| Helper identifier | Description | Expected Input | +|-----------------------------|-----------------------------------------------------------------------------|-------------------| +| `common.labels.standard` | Return Kubernetes standard labels | `.` Chart context | +| `common.labels.matchLabels` | Labels to use on `deploy.spec.selector.matchLabels` and `svc.spec.selector` | `.` Chart context | + +### Names + +| Helper identifier | Description | Expected Input | +|-----------------------------------|-----------------------------------------------------------------------|-------------------| +| `common.names.name` | Expand the name of the chart or use `.Values.nameOverride` | `.` Chart context | +| `common.names.fullname` | Create a default fully qualified app name. | `.` Chart context | +| `common.names.namespace` | Allow the release namespace to be overridden | `.` Chart context | +| `common.names.fullname.namespace` | Create a fully qualified app name adding the installation's namespace | `.` Chart context | +| `common.names.chart` | Chart name plus version | `.` Chart context | + +### Secrets + +| Helper identifier | Description | Expected Input | +|---------------------------|--------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.secrets.name` | Generate the name of the secret. | `dict "existingSecret" .Values.path.to.the.existingSecret "defaultNameSuffix" "mySuffix" "context" $` see [ExistingSecret](#existingsecret) for the structure. | +| `common.secrets.key` | Generate secret key. | `dict "existingSecret" .Values.path.to.the.existingSecret "key" "keyName"` see [ExistingSecret](#existingsecret) for the structure. | +| `common.passwords.manage` | Generate secret password or retrieve one if already created. | `dict "secret" "secret-name" "key" "keyName" "providedValues" (list "path.to.password1" "path.to.password2") "length" 10 "strong" false "chartName" "chartName" "context" $`, length, strong and chartNAme fields are optional. | +| `common.secrets.exists` | Returns whether a previous generated secret already exists. | `dict "secret" "secret-name" "context" $` | + +### Storage + +| Helper identifier | Description | Expected Input | +|-------------------------------|---------------------------------------|---------------------------------------------------------------------------------------------------------------------| +| `common.storage.class` | Return the proper Storage Class | `dict "persistence" .Values.path.to.the.persistence "global" $`, see [Persistence](#persistence) for the structure. | + +### TplValues + +| Helper identifier | Description | Expected Input | +|---------------------------|----------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.tplvalues.render` | Renders a value that contains template | `dict "value" .Values.path.to.the.Value "context" $`, value is the value should rendered as template, context frequently is the chart context `$` or `.` | + +### Utils + +| Helper identifier | Description | Expected Input | +|--------------------------------|------------------------------------------------------------------------------------------|------------------------------------------------------------------------| +| `common.utils.fieldToEnvVar` | Build environment variable name given a field. | `dict "field" "my-password"` | +| `common.utils.secret.getvalue` | Print instructions to get a secret value. | `dict "secret" "secret-name" "field" "secret-value-field" "context" $` | +| `common.utils.getValueFromKey` | Gets a value from `.Values` object given its key path | `dict "key" "path.to.key" "context" $` | +| `common.utils.getKeyFromList` | Returns first `.Values` key with a defined value or first of the list if all non-defined | `dict "keys" (list "path.to.key1" "path.to.key2") "context" $` | + +### Validations + +| Helper identifier | Description | Expected Input | +|--------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.validations.values.single.empty` | Validate a value must not be empty. | `dict "valueKey" "path.to.value" "secret" "secret.name" "field" "my-password" "subchart" "subchart" "context" $` secret, field and subchart are optional. In case they are given, the helper will generate a how to get instruction. See [ValidateValue](#validatevalue) | +| `common.validations.values.multiple.empty` | Validate a multiple values must not be empty. It returns a shared error for all the values. | `dict "required" (list $validateValueConf00 $validateValueConf01) "context" $`. See [ValidateValue](#validatevalue) | +| `common.validations.values.mariadb.passwords` | This helper will ensure required password for MariaDB are not empty. It returns a shared error for all the values. | `dict "secret" "mariadb-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use mariadb chart and the helper. | +| `common.validations.values.mysql.passwords` | This helper will ensure required password for MySQL are not empty. It returns a shared error for all the values. | `dict "secret" "mysql-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use mysql chart and the helper. | +| `common.validations.values.postgresql.passwords` | This helper will ensure required password for PostgreSQL are not empty. It returns a shared error for all the values. | `dict "secret" "postgresql-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use postgresql chart and the helper. | +| `common.validations.values.redis.passwords` | This helper will ensure required password for Redis® are not empty. It returns a shared error for all the values. | `dict "secret" "redis-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use redis chart and the helper. | +| `common.validations.values.cassandra.passwords` | This helper will ensure required password for Cassandra are not empty. It returns a shared error for all the values. | `dict "secret" "cassandra-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use cassandra chart and the helper. | +| `common.validations.values.mongodb.passwords` | This helper will ensure required password for MongoDB® are not empty. It returns a shared error for all the values. | `dict "secret" "mongodb-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use mongodb chart and the helper. | + +### Warnings + +| Helper identifier | Description | Expected Input | +|------------------------------|----------------------------------|------------------------------------------------------------| +| `common.warnings.rollingTag` | Warning about using rolling tag. | `ImageRoot` see [ImageRoot](#imageroot) for the structure. | + +## Special input schemas + +### ImageRoot + +```yaml +registry: + type: string + description: Docker registry where the image is located + example: docker.io + +repository: + type: string + description: Repository and image name + example: bitnami/nginx + +tag: + type: string + description: image tag + example: 1.16.1-debian-10-r63 + +pullPolicy: + type: string + description: Specify a imagePullPolicy. Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + +pullSecrets: + type: array + items: + type: string + description: Optionally specify an array of imagePullSecrets (evaluated as templates). + +debug: + type: boolean + description: Set to true if you would like to see extra information on logs + example: false + +## An instance would be: +# registry: docker.io +# repository: bitnami/nginx +# tag: 1.16.1-debian-10-r63 +# pullPolicy: IfNotPresent +# debug: false +``` + +### Persistence + +```yaml +enabled: + type: boolean + description: Whether enable persistence. + example: true + +storageClass: + type: string + description: Ghost data Persistent Volume Storage Class, If set to "-", storageClassName: "" which disables dynamic provisioning. + example: "-" + +accessMode: + type: string + description: Access mode for the Persistent Volume Storage. + example: ReadWriteOnce + +size: + type: string + description: Size the Persistent Volume Storage. + example: 8Gi + +path: + type: string + description: Path to be persisted. + example: /bitnami + +## An instance would be: +# enabled: true +# storageClass: "-" +# accessMode: ReadWriteOnce +# size: 8Gi +# path: /bitnami +``` + +### ExistingSecret + +```yaml +name: + type: string + description: Name of the existing secret. + example: mySecret +keyMapping: + description: Mapping between the expected key name and the name of the key in the existing secret. + type: object + +## An instance would be: +# name: mySecret +# keyMapping: +# password: myPasswordKey +``` + +#### Example of use + +When we store sensitive data for a deployment in a secret, some times we want to give to users the possibility of using theirs existing secrets. + +```yaml +# templates/secret.yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "common.names.fullname" . }} + labels: + app: {{ include "common.names.fullname" . }} +type: Opaque +data: + password: {{ .Values.password | b64enc | quote }} + +# templates/dpl.yaml +--- +... + env: + - name: PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "common.secrets.name" (dict "existingSecret" .Values.existingSecret "context" $) }} + key: {{ include "common.secrets.key" (dict "existingSecret" .Values.existingSecret "key" "password") }} +... + +# values.yaml +--- +name: mySecret +keyMapping: + password: myPasswordKey +``` + +### ValidateValue + +#### NOTES.txt + +```console +{{- $validateValueConf00 := (dict "valueKey" "path.to.value00" "secret" "secretName" "field" "password-00") -}} +{{- $validateValueConf01 := (dict "valueKey" "path.to.value01" "secret" "secretName" "field" "password-01") -}} + +{{ include "common.validations.values.multiple.empty" (dict "required" (list $validateValueConf00 $validateValueConf01) "context" $) }} +``` + +If we force those values to be empty we will see some alerts + +```console +$ helm install test mychart --set path.to.value00="",path.to.value01="" + 'path.to.value00' must not be empty, please add '--set path.to.value00=$PASSWORD_00' to the command. To get the current value: + + export PASSWORD_00=$(kubectl get secret --namespace default secretName -o jsonpath="{.data.password-00}" | base64 -d) + + 'path.to.value01' must not be empty, please add '--set path.to.value01=$PASSWORD_01' to the command. To get the current value: + + export PASSWORD_01=$(kubectl get secret --namespace default secretName -o jsonpath="{.data.password-01}" | base64 -d) +``` + +## Upgrading + +### To 1.0.0 + +[On November 13, 2020, Helm v2 support was formally finished](https://github.com/helm/charts#status-of-the-project), this major version is the result of the required changes applied to the Helm Chart to be able to incorporate the different features added in Helm v3 and to be consistent with the Helm project itself regarding the Helm v2 EOL. + +**What changes were introduced in this major version?** + +- Previous versions of this Helm Chart use `apiVersion: v1` (installable by both Helm 2 and 3), this Helm Chart was updated to `apiVersion: v2` (installable by Helm 3 only). [Here](https://helm.sh/docs/topics/charts/#the-apiversion-field) you can find more information about the `apiVersion` field. +- Use `type: library`. [Here](https://v3.helm.sh/docs/faq/#library-chart-support) you can find more information. +- The different fields present in the *Chart.yaml* file has been ordered alphabetically in a homogeneous way for all the Bitnami Helm Charts + +**Considerations when upgrading to this version** + +- If you want to upgrade to this version from a previous one installed with Helm v3, you shouldn't face any issues +- If you want to upgrade to this version using Helm v2, this scenario is not supported as this version doesn't support Helm v2 anymore +- If you installed the previous version with Helm v2 and wants to upgrade to this version with Helm v3, please refer to the [official Helm documentation](https://helm.sh/docs/topics/v2_v3_migration/#migration-use-cases) about migrating from Helm v2 to v3 + +**Useful links** + +- https://docs.bitnami.com/tutorials/resolve-helm2-helm3-post-migration-issues/ +- https://helm.sh/docs/topics/v2_v3_migration/ +- https://helm.sh/blog/migrate-from-helm-v2-to-helm-v3/ + +## License + +Copyright © 2022 Bitnami + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/helm/airbyte-v1/charts/metrics/charts/common/templates/_affinities.tpl b/helm/airbyte-v1/charts/metrics/charts/common/templates/_affinities.tpl new file mode 100644 index 00000000..2387be26 --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/charts/common/templates/_affinities.tpl @@ -0,0 +1,102 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Return a soft nodeAffinity definition +{{ include "common.affinities.nodes.soft" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.soft" -}} +preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . | quote }} + {{- end }} + weight: 1 +{{- end -}} + +{{/* +Return a hard nodeAffinity definition +{{ include "common.affinities.nodes.hard" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.hard" -}} +requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . | quote }} + {{- end }} +{{- end -}} + +{{/* +Return a nodeAffinity definition +{{ include "common.affinities.nodes" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.nodes.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.nodes.hard" . -}} + {{- end -}} +{{- end -}} + +{{/* +Return a soft podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.soft" (dict "component" "FOO" "extraMatchLabels" .Values.extraMatchLabels "context" $) -}} +*/}} +{{- define "common.affinities.pods.soft" -}} +{{- $component := default "" .component -}} +{{- $extraMatchLabels := default (dict) .extraMatchLabels -}} +preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" .context) | nindent 10 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := $extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + namespaces: + - {{ include "common.names.namespace" .context | quote }} + topologyKey: kubernetes.io/hostname + weight: 1 +{{- end -}} + +{{/* +Return a hard podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.hard" (dict "component" "FOO" "extraMatchLabels" .Values.extraMatchLabels "context" $) -}} +*/}} +{{- define "common.affinities.pods.hard" -}} +{{- $component := default "" .component -}} +{{- $extraMatchLabels := default (dict) .extraMatchLabels -}} +requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" .context) | nindent 8 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := $extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + namespaces: + - {{ include "common.names.namespace" .context | quote }} + topologyKey: kubernetes.io/hostname +{{- end -}} + +{{/* +Return a podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.pods" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.pods.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.pods.hard" . -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/metrics/charts/common/templates/_capabilities.tpl b/helm/airbyte-v1/charts/metrics/charts/common/templates/_capabilities.tpl new file mode 100644 index 00000000..9d9b7600 --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/charts/common/templates/_capabilities.tpl @@ -0,0 +1,154 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Return the target Kubernetes version +*/}} +{{- define "common.capabilities.kubeVersion" -}} +{{- if .Values.global }} + {{- if .Values.global.kubeVersion }} + {{- .Values.global.kubeVersion -}} + {{- else }} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} + {{- end -}} +{{- else }} +{{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for poddisruptionbudget. +*/}} +{{- define "common.capabilities.policy.apiVersion" -}} +{{- if semverCompare "<1.21-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "policy/v1beta1" -}} +{{- else -}} +{{- print "policy/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for networkpolicy. +*/}} +{{- define "common.capabilities.networkPolicy.apiVersion" -}} +{{- if semverCompare "<1.7-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for cronjob. +*/}} +{{- define "common.capabilities.cronjob.apiVersion" -}} +{{- if semverCompare "<1.21-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "batch/v1beta1" -}} +{{- else -}} +{{- print "batch/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for deployment. +*/}} +{{- define "common.capabilities.deployment.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for statefulset. +*/}} +{{- define "common.capabilities.statefulset.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apps/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "common.capabilities.ingress.apiVersion" -}} +{{- if .Values.ingress -}} +{{- if .Values.ingress.apiVersion -}} +{{- .Values.ingress.apiVersion -}} +{{- else if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end }} +{{- else if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for RBAC resources. +*/}} +{{- define "common.capabilities.rbac.apiVersion" -}} +{{- if semverCompare "<1.17-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "rbac.authorization.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "rbac.authorization.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for CRDs. +*/}} +{{- define "common.capabilities.crd.apiVersion" -}} +{{- if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiextensions.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiextensions.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for APIService. +*/}} +{{- define "common.capabilities.apiService.apiVersion" -}} +{{- if semverCompare "<1.10-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiregistration.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiregistration.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for Horizontal Pod Autoscaler. +*/}} +{{- define "common.capabilities.hpa.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .context) -}} +{{- if .beta2 -}} +{{- print "autoscaling/v2beta2" -}} +{{- else -}} +{{- print "autoscaling/v2beta1" -}} +{{- end -}} +{{- else -}} +{{- print "autoscaling/v2" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if the used Helm version is 3.3+. +A way to check the used Helm version was not introduced until version 3.3.0 with .Capabilities.HelmVersion, which contains an additional "{}}" structure. +This check is introduced as a regexMatch instead of {{ if .Capabilities.HelmVersion }} because checking for the key HelmVersion in <3.3 results in a "interface not found" error. +**To be removed when the catalog's minimun Helm version is 3.3** +*/}} +{{- define "common.capabilities.supportsHelmVersion" -}} +{{- if regexMatch "{(v[0-9])*[^}]*}}$" (.Capabilities | toString ) }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/metrics/charts/common/templates/_errors.tpl b/helm/airbyte-v1/charts/metrics/charts/common/templates/_errors.tpl new file mode 100644 index 00000000..a79cc2e3 --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/charts/common/templates/_errors.tpl @@ -0,0 +1,23 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Through error when upgrading using empty passwords values that must not be empty. + +Usage: +{{- $validationError00 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password00" "secret" "secretName" "field" "password-00") -}} +{{- $validationError01 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password01" "secret" "secretName" "field" "password-01") -}} +{{ include "common.errors.upgrade.passwords.empty" (dict "validationErrors" (list $validationError00 $validationError01) "context" $) }} + +Required password params: + - validationErrors - String - Required. List of validation strings to be return, if it is empty it won't throw error. + - context - Context - Required. Parent context. +*/}} +{{- define "common.errors.upgrade.passwords.empty" -}} + {{- $validationErrors := join "" .validationErrors -}} + {{- if and $validationErrors .context.Release.IsUpgrade -}} + {{- $errorString := "\nPASSWORDS ERROR: You must provide your current passwords when upgrading the release." -}} + {{- $errorString = print $errorString "\n Note that even after reinstallation, old credentials may be needed as they may be kept in persistent volume claims." -}} + {{- $errorString = print $errorString "\n Further information can be obtained at https://docs.bitnami.com/general/how-to/troubleshoot-helm-chart-issues/#credential-errors-while-upgrading-chart-releases" -}} + {{- $errorString = print $errorString "\n%s" -}} + {{- printf $errorString $validationErrors | fail -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/metrics/charts/common/templates/_images.tpl b/helm/airbyte-v1/charts/metrics/charts/common/templates/_images.tpl new file mode 100644 index 00000000..42ffbc72 --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/charts/common/templates/_images.tpl @@ -0,0 +1,75 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper image name +{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }} +*/}} +{{- define "common.images.image" -}} +{{- $registryName := .imageRoot.registry -}} +{{- $repositoryName := .imageRoot.repository -}} +{{- $tag := .imageRoot.tag | toString -}} +{{- if .global }} + {{- if .global.imageRegistry }} + {{- $registryName = .global.imageRegistry -}} + {{- end -}} +{{- end -}} +{{- if $registryName }} +{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- else -}} +{{- printf "%s:%s" $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) +{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} +*/}} +{{- define "common.images.pullSecrets" -}} + {{- $pullSecrets := list }} + + {{- if .global }} + {{- range .global.imagePullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names evaluating values as templates +{{ include "common.images.renderPullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "context" $) }} +*/}} +{{- define "common.images.renderPullSecrets" -}} + {{- $pullSecrets := list }} + {{- $context := .context }} + + {{- if $context.Values.global }} + {{- range $context.Values.global.imagePullSecrets -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" $context)) -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" $context)) -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/helm/airbyte-v1/charts/metrics/charts/common/templates/_ingress.tpl b/helm/airbyte-v1/charts/metrics/charts/common/templates/_ingress.tpl new file mode 100644 index 00000000..8caf73a6 --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/charts/common/templates/_ingress.tpl @@ -0,0 +1,68 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Generate backend entry that is compatible with all Kubernetes API versions. + +Usage: +{{ include "common.ingress.backend" (dict "serviceName" "backendName" "servicePort" "backendPort" "context" $) }} + +Params: + - serviceName - String. Name of an existing service backend + - servicePort - String/Int. Port name (or number) of the service. It will be translated to different yaml depending if it is a string or an integer. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.ingress.backend" -}} +{{- $apiVersion := (include "common.capabilities.ingress.apiVersion" .context) -}} +{{- if or (eq $apiVersion "extensions/v1beta1") (eq $apiVersion "networking.k8s.io/v1beta1") -}} +serviceName: {{ .serviceName }} +servicePort: {{ .servicePort }} +{{- else -}} +service: + name: {{ .serviceName }} + port: + {{- if typeIs "string" .servicePort }} + name: {{ .servicePort }} + {{- else if or (typeIs "int" .servicePort) (typeIs "float64" .servicePort) }} + number: {{ .servicePort | int }} + {{- end }} +{{- end -}} +{{- end -}} + +{{/* +Print "true" if the API pathType field is supported +Usage: +{{ include "common.ingress.supportsPathType" . }} +*/}} +{{- define "common.ingress.supportsPathType" -}} +{{- if (semverCompare "<1.18-0" (include "common.capabilities.kubeVersion" .)) -}} +{{- print "false" -}} +{{- else -}} +{{- print "true" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if the ingressClassname field is supported +Usage: +{{ include "common.ingress.supportsIngressClassname" . }} +*/}} +{{- define "common.ingress.supportsIngressClassname" -}} +{{- if semverCompare "<1.18-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "false" -}} +{{- else -}} +{{- print "true" -}} +{{- end -}} +{{- end -}} + +{{/* +Return true if cert-manager required annotations for TLS signed +certificates are set in the Ingress annotations +Ref: https://cert-manager.io/docs/usage/ingress/#supported-annotations +Usage: +{{ include "common.ingress.certManagerRequest" ( dict "annotations" .Values.path.to.the.ingress.annotations ) }} +*/}} +{{- define "common.ingress.certManagerRequest" -}} +{{ if or (hasKey .annotations "cert-manager.io/cluster-issuer") (hasKey .annotations "cert-manager.io/issuer") }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/metrics/charts/common/templates/_labels.tpl b/helm/airbyte-v1/charts/metrics/charts/common/templates/_labels.tpl new file mode 100644 index 00000000..252066c7 --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/charts/common/templates/_labels.tpl @@ -0,0 +1,18 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Kubernetes standard labels +*/}} +{{- define "common.labels.standard" -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +helm.sh/chart: {{ include "common.names.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Labels to use on deploy.spec.selector.matchLabels and svc.spec.selector +*/}} +{{- define "common.labels.matchLabels" -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} diff --git a/helm/airbyte-v1/charts/metrics/charts/common/templates/_names.tpl b/helm/airbyte-v1/charts/metrics/charts/common/templates/_names.tpl new file mode 100644 index 00000000..1bdac8b7 --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/charts/common/templates/_names.tpl @@ -0,0 +1,70 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "common.names.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "common.names.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "common.names.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a default fully qualified dependency name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +Usage: +{{ include "common.names.dependency.fullname" (dict "chartName" "dependency-chart-name" "chartValues" .Values.dependency-chart "context" $) }} +*/}} +{{- define "common.names.dependency.fullname" -}} +{{- if .chartValues.fullnameOverride -}} +{{- .chartValues.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .chartName .chartValues.nameOverride -}} +{{- if contains $name .context.Release.Name -}} +{{- .context.Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .context.Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts. +*/}} +{{- define "common.names.namespace" -}} +{{- if .Values.namespaceOverride -}} +{{- .Values.namespaceOverride -}} +{{- else -}} +{{- .Release.Namespace -}} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified app name adding the installation's namespace. +*/}} +{{- define "common.names.fullname.namespace" -}} +{{- printf "%s-%s" (include "common.names.fullname" .) (include "common.names.namespace" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/metrics/charts/common/templates/_secrets.tpl b/helm/airbyte-v1/charts/metrics/charts/common/templates/_secrets.tpl new file mode 100644 index 00000000..a53fb44f --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/charts/common/templates/_secrets.tpl @@ -0,0 +1,140 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Generate secret name. + +Usage: +{{ include "common.secrets.name" (dict "existingSecret" .Values.path.to.the.existingSecret "defaultNameSuffix" "mySuffix" "context" $) }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/master/bitnami/common#existingsecret + - defaultNameSuffix - String - Optional. It is used only if we have several secrets in the same deployment. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.secrets.name" -}} +{{- $name := (include "common.names.fullname" .context) -}} + +{{- if .defaultNameSuffix -}} +{{- $name = printf "%s-%s" $name .defaultNameSuffix | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- with .existingSecret -}} +{{- if not (typeIs "string" .) -}} +{{- with .name -}} +{{- $name = . -}} +{{- end -}} +{{- else -}} +{{- $name = . -}} +{{- end -}} +{{- end -}} + +{{- printf "%s" $name -}} +{{- end -}} + +{{/* +Generate secret key. + +Usage: +{{ include "common.secrets.key" (dict "existingSecret" .Values.path.to.the.existingSecret "key" "keyName") }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/master/bitnami/common#existingsecret + - key - String - Required. Name of the key in the secret. +*/}} +{{- define "common.secrets.key" -}} +{{- $key := .key -}} + +{{- if .existingSecret -}} + {{- if not (typeIs "string" .existingSecret) -}} + {{- if .existingSecret.keyMapping -}} + {{- $key = index .existingSecret.keyMapping $.key -}} + {{- end -}} + {{- end }} +{{- end -}} + +{{- printf "%s" $key -}} +{{- end -}} + +{{/* +Generate secret password or retrieve one if already created. + +Usage: +{{ include "common.secrets.passwords.manage" (dict "secret" "secret-name" "key" "keyName" "providedValues" (list "path.to.password1" "path.to.password2") "length" 10 "strong" false "chartName" "chartName" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - key - String - Required - Name of the key in the secret. + - providedValues - List - Required - The path to the validating value in the values.yaml, e.g: "mysql.password". Will pick first parameter with a defined value. + - length - int - Optional - Length of the generated random password. + - strong - Boolean - Optional - Whether to add symbols to the generated random password. + - chartName - String - Optional - Name of the chart used when said chart is deployed as a subchart. + - context - Context - Required - Parent context. + +The order in which this function returns a secret password: + 1. Already existing 'Secret' resource + (If a 'Secret' resource is found under the name provided to the 'secret' parameter to this function and that 'Secret' resource contains a key with the name passed as the 'key' parameter to this function then the value of this existing secret password will be returned) + 2. Password provided via the values.yaml + (If one of the keys passed to the 'providedValues' parameter to this function is a valid path to a key in the values.yaml and has a value, the value of the first key with a value will be returned) + 3. Randomly generated secret password + (A new random secret password with the length specified in the 'length' parameter will be generated and returned) + +*/}} +{{- define "common.secrets.passwords.manage" -}} + +{{- $password := "" }} +{{- $subchart := "" }} +{{- $chartName := default "" .chartName }} +{{- $passwordLength := default 10 .length }} +{{- $providedPasswordKey := include "common.utils.getKeyFromList" (dict "keys" .providedValues "context" $.context) }} +{{- $providedPasswordValue := include "common.utils.getValueFromKey" (dict "key" $providedPasswordKey "context" $.context) }} +{{- $secretData := (lookup "v1" "Secret" $.context.Release.Namespace .secret).data }} +{{- if $secretData }} + {{- if hasKey $secretData .key }} + {{- $password = index $secretData .key }} + {{- else }} + {{- printf "\nPASSWORDS ERROR: The secret \"%s\" does not contain the key \"%s\"\n" .secret .key | fail -}} + {{- end -}} +{{- else if $providedPasswordValue }} + {{- $password = $providedPasswordValue | toString | b64enc | quote }} +{{- else }} + + {{- if .context.Values.enabled }} + {{- $subchart = $chartName }} + {{- end -}} + + {{- $requiredPassword := dict "valueKey" $providedPasswordKey "secret" .secret "field" .key "subchart" $subchart "context" $.context -}} + {{- $requiredPasswordError := include "common.validations.values.single.empty" $requiredPassword -}} + {{- $passwordValidationErrors := list $requiredPasswordError -}} + {{- include "common.errors.upgrade.passwords.empty" (dict "validationErrors" $passwordValidationErrors "context" $.context) -}} + + {{- if .strong }} + {{- $subStr := list (lower (randAlpha 1)) (randNumeric 1) (upper (randAlpha 1)) | join "_" }} + {{- $password = randAscii $passwordLength }} + {{- $password = regexReplaceAllLiteral "\\W" $password "@" | substr 5 $passwordLength }} + {{- $password = printf "%s%s" $subStr $password | toString | shuffle | b64enc | quote }} + {{- else }} + {{- $password = randAlphaNum $passwordLength | b64enc | quote }} + {{- end }} +{{- end -}} +{{- printf "%s" $password -}} +{{- end -}} + +{{/* +Returns whether a previous generated secret already exists + +Usage: +{{ include "common.secrets.exists" (dict "secret" "secret-name" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - context - Context - Required - Parent context. +*/}} +{{- define "common.secrets.exists" -}} +{{- $secret := (lookup "v1" "Secret" $.context.Release.Namespace .secret) }} +{{- if $secret }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/metrics/charts/common/templates/_storage.tpl b/helm/airbyte-v1/charts/metrics/charts/common/templates/_storage.tpl new file mode 100644 index 00000000..60e2a844 --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/charts/common/templates/_storage.tpl @@ -0,0 +1,23 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper Storage Class +{{ include "common.storage.class" ( dict "persistence" .Values.path.to.the.persistence "global" $) }} +*/}} +{{- define "common.storage.class" -}} + +{{- $storageClass := .persistence.storageClass -}} +{{- if .global -}} + {{- if .global.storageClass -}} + {{- $storageClass = .global.storageClass -}} + {{- end -}} +{{- end -}} + +{{- if $storageClass -}} + {{- if (eq "-" $storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" $storageClass -}} + {{- end -}} +{{- end -}} + +{{- end -}} diff --git a/helm/airbyte-v1/charts/metrics/charts/common/templates/_tplvalues.tpl b/helm/airbyte-v1/charts/metrics/charts/common/templates/_tplvalues.tpl new file mode 100644 index 00000000..2db16685 --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/charts/common/templates/_tplvalues.tpl @@ -0,0 +1,13 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Renders a value that contains template. +Usage: +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $) }} +*/}} +{{- define "common.tplvalues.render" -}} + {{- if typeIs "string" .value }} + {{- tpl .value .context }} + {{- else }} + {{- tpl (.value | toYaml) .context }} + {{- end }} +{{- end -}} diff --git a/helm/airbyte-v1/charts/metrics/charts/common/templates/_utils.tpl b/helm/airbyte-v1/charts/metrics/charts/common/templates/_utils.tpl new file mode 100644 index 00000000..8c22b2a3 --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/charts/common/templates/_utils.tpl @@ -0,0 +1,62 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Print instructions to get a secret value. +Usage: +{{ include "common.utils.secret.getvalue" (dict "secret" "secret-name" "field" "secret-value-field" "context" $) }} +*/}} +{{- define "common.utils.secret.getvalue" -}} +{{- $varname := include "common.utils.fieldToEnvVar" . -}} +export {{ $varname }}=$(kubectl get secret --namespace {{ .context.Release.Namespace | quote }} {{ .secret }} -o jsonpath="{.data.{{ .field }}}" | base64 -d) +{{- end -}} + +{{/* +Build env var name given a field +Usage: +{{ include "common.utils.fieldToEnvVar" dict "field" "my-password" }} +*/}} +{{- define "common.utils.fieldToEnvVar" -}} + {{- $fieldNameSplit := splitList "-" .field -}} + {{- $upperCaseFieldNameSplit := list -}} + + {{- range $fieldNameSplit -}} + {{- $upperCaseFieldNameSplit = append $upperCaseFieldNameSplit ( upper . ) -}} + {{- end -}} + + {{ join "_" $upperCaseFieldNameSplit }} +{{- end -}} + +{{/* +Gets a value from .Values given +Usage: +{{ include "common.utils.getValueFromKey" (dict "key" "path.to.key" "context" $) }} +*/}} +{{- define "common.utils.getValueFromKey" -}} +{{- $splitKey := splitList "." .key -}} +{{- $value := "" -}} +{{- $latestObj := $.context.Values -}} +{{- range $splitKey -}} + {{- if not $latestObj -}} + {{- printf "please review the entire path of '%s' exists in values" $.key | fail -}} + {{- end -}} + {{- $value = ( index $latestObj . ) -}} + {{- $latestObj = $value -}} +{{- end -}} +{{- printf "%v" (default "" $value) -}} +{{- end -}} + +{{/* +Returns first .Values key with a defined value or first of the list if all non-defined +Usage: +{{ include "common.utils.getKeyFromList" (dict "keys" (list "path.to.key1" "path.to.key2") "context" $) }} +*/}} +{{- define "common.utils.getKeyFromList" -}} +{{- $key := first .keys -}} +{{- $reverseKeys := reverse .keys }} +{{- range $reverseKeys }} + {{- $value := include "common.utils.getValueFromKey" (dict "key" . "context" $.context ) }} + {{- if $value -}} + {{- $key = . }} + {{- end -}} +{{- end -}} +{{- printf "%s" $key -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/metrics/charts/common/templates/_warnings.tpl b/helm/airbyte-v1/charts/metrics/charts/common/templates/_warnings.tpl new file mode 100644 index 00000000..ae10fa41 --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/charts/common/templates/_warnings.tpl @@ -0,0 +1,14 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Warning about using rolling tag. +Usage: +{{ include "common.warnings.rollingTag" .Values.path.to.the.imageRoot }} +*/}} +{{- define "common.warnings.rollingTag" -}} + +{{- if and (contains "bitnami/" .repository) (not (.tag | toString | regexFind "-r\\d+$|sha256:")) }} +WARNING: Rolling tag detected ({{ .repository }}:{{ .tag }}), please note that it is strongly recommended to avoid using rolling tags in a production environment. ++info https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/ +{{- end }} + +{{- end -}} diff --git a/helm/airbyte-v1/charts/metrics/charts/common/templates/validations/_cassandra.tpl b/helm/airbyte-v1/charts/metrics/charts/common/templates/validations/_cassandra.tpl new file mode 100644 index 00000000..ded1ae3b --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/charts/common/templates/validations/_cassandra.tpl @@ -0,0 +1,72 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate Cassandra required passwords are not empty. + +Usage: +{{ include "common.validations.values.cassandra.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where Cassandra values are stored, e.g: "cassandra-passwords-secret" + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.cassandra.passwords" -}} + {{- $existingSecret := include "common.cassandra.values.existingSecret" . -}} + {{- $enabled := include "common.cassandra.values.enabled" . -}} + {{- $dbUserPrefix := include "common.cassandra.values.key.dbUser" . -}} + {{- $valueKeyPassword := printf "%s.password" $dbUserPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "cassandra-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.cassandra.values.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.cassandra.values.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.cassandra.dbUser.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.dbUser.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled cassandra. + +Usage: +{{ include "common.cassandra.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.cassandra.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.cassandra.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key dbUser + +Usage: +{{ include "common.cassandra.values.key.dbUser" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.cassandra.values.key.dbUser" -}} + {{- if .subchart -}} + cassandra.dbUser + {{- else -}} + dbUser + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/metrics/charts/common/templates/validations/_mariadb.tpl b/helm/airbyte-v1/charts/metrics/charts/common/templates/validations/_mariadb.tpl new file mode 100644 index 00000000..b6906ff7 --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/charts/common/templates/validations/_mariadb.tpl @@ -0,0 +1,103 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MariaDB required passwords are not empty. + +Usage: +{{ include "common.validations.values.mariadb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MariaDB values are stored, e.g: "mysql-passwords-secret" + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mariadb.passwords" -}} + {{- $existingSecret := include "common.mariadb.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mariadb.values.enabled" . -}} + {{- $architecture := include "common.mariadb.values.architecture" . -}} + {{- $authPrefix := include "common.mariadb.values.key.auth" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mariadb-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- if not (empty $valueUsername) -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mariadb-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replication") -}} + {{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mariadb-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mariadb.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mariadb. + +Usage: +{{ include "common.mariadb.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mariadb.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mariadb.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mariadb.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mariadb.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.key.auth" -}} + {{- if .subchart -}} + mariadb.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/metrics/charts/common/templates/validations/_mongodb.tpl b/helm/airbyte-v1/charts/metrics/charts/common/templates/validations/_mongodb.tpl new file mode 100644 index 00000000..f820ec10 --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/charts/common/templates/validations/_mongodb.tpl @@ -0,0 +1,108 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MongoDB® required passwords are not empty. + +Usage: +{{ include "common.validations.values.mongodb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MongoDB® values are stored, e.g: "mongodb-passwords-secret" + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mongodb.passwords" -}} + {{- $existingSecret := include "common.mongodb.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mongodb.values.enabled" . -}} + {{- $authPrefix := include "common.mongodb.values.key.auth" . -}} + {{- $architecture := include "common.mongodb.values.architecture" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyDatabase := printf "%s.database" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicaSetKey := printf "%s.replicaSetKey" $authPrefix -}} + {{- $valueKeyAuthEnabled := printf "%s.enabled" $authPrefix -}} + + {{- $authEnabled := include "common.utils.getValueFromKey" (dict "key" $valueKeyAuthEnabled "context" .context) -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") (eq $authEnabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mongodb-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- $valueDatabase := include "common.utils.getValueFromKey" (dict "key" $valueKeyDatabase "context" .context) }} + {{- if and $valueUsername $valueDatabase -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mongodb-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replicaset") -}} + {{- $requiredReplicaSetKey := dict "valueKey" $valueKeyReplicaSetKey "secret" .secret "field" "mongodb-replica-set-key" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicaSetKey -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mongodb.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDb is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mongodb.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mongodb. + +Usage: +{{ include "common.mongodb.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mongodb.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mongodb.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mongodb.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.key.auth" -}} + {{- if .subchart -}} + mongodb.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mongodb.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mongodb.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/metrics/charts/common/templates/validations/_mysql.tpl b/helm/airbyte-v1/charts/metrics/charts/common/templates/validations/_mysql.tpl new file mode 100644 index 00000000..74472a06 --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/charts/common/templates/validations/_mysql.tpl @@ -0,0 +1,103 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MySQL required passwords are not empty. + +Usage: +{{ include "common.validations.values.mysql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MySQL values are stored, e.g: "mysql-passwords-secret" + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mysql.passwords" -}} + {{- $existingSecret := include "common.mysql.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mysql.values.enabled" . -}} + {{- $architecture := include "common.mysql.values.architecture" . -}} + {{- $authPrefix := include "common.mysql.values.key.auth" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mysql-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- if not (empty $valueUsername) -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mysql-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replication") -}} + {{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mysql-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mysql.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mysql.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mysql. + +Usage: +{{ include "common.mysql.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mysql.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mysql.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mysql.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mysql.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mysql.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.key.auth" -}} + {{- if .subchart -}} + mysql.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/metrics/charts/common/templates/validations/_postgresql.tpl b/helm/airbyte-v1/charts/metrics/charts/common/templates/validations/_postgresql.tpl new file mode 100644 index 00000000..164ec0d0 --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/charts/common/templates/validations/_postgresql.tpl @@ -0,0 +1,129 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate PostgreSQL required passwords are not empty. + +Usage: +{{ include "common.validations.values.postgresql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where postgresql values are stored, e.g: "postgresql-passwords-secret" + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.postgresql.passwords" -}} + {{- $existingSecret := include "common.postgresql.values.existingSecret" . -}} + {{- $enabled := include "common.postgresql.values.enabled" . -}} + {{- $valueKeyPostgresqlPassword := include "common.postgresql.values.key.postgressPassword" . -}} + {{- $valueKeyPostgresqlReplicationEnabled := include "common.postgresql.values.key.replicationPassword" . -}} + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + {{- $requiredPostgresqlPassword := dict "valueKey" $valueKeyPostgresqlPassword "secret" .secret "field" "postgresql-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlPassword -}} + + {{- $enabledReplication := include "common.postgresql.values.enabled.replication" . -}} + {{- if (eq $enabledReplication "true") -}} + {{- $requiredPostgresqlReplicationPassword := dict "valueKey" $valueKeyPostgresqlReplicationEnabled "secret" .secret "field" "postgresql-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to decide whether evaluate global values. + +Usage: +{{ include "common.postgresql.values.use.global" (dict "key" "key-of-global" "context" $) }} +Params: + - key - String - Required. Field to be evaluated within global, e.g: "existingSecret" +*/}} +{{- define "common.postgresql.values.use.global" -}} + {{- if .context.Values.global -}} + {{- if .context.Values.global.postgresql -}} + {{- index .context.Values.global.postgresql .key | quote -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.postgresql.values.existingSecret" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.existingSecret" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "existingSecret" "context" .context) -}} + + {{- if .subchart -}} + {{- default (.context.Values.postgresql.existingSecret | quote) $globalValue -}} + {{- else -}} + {{- default (.context.Values.existingSecret | quote) $globalValue -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled postgresql. + +Usage: +{{ include "common.postgresql.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key postgressPassword. + +Usage: +{{ include "common.postgresql.values.key.postgressPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.postgressPassword" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "postgresqlUsername" "context" .context) -}} + + {{- if not $globalValue -}} + {{- if .subchart -}} + postgresql.postgresqlPassword + {{- else -}} + postgresqlPassword + {{- end -}} + {{- else -}} + global.postgresql.postgresqlPassword + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled.replication. + +Usage: +{{ include "common.postgresql.values.enabled.replication" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.enabled.replication" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.replication.enabled -}} + {{- else -}} + {{- printf "%v" .context.Values.replication.enabled -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key replication.password. + +Usage: +{{ include "common.postgresql.values.key.replicationPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.replicationPassword" -}} + {{- if .subchart -}} + postgresql.replication.password + {{- else -}} + replication.password + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/metrics/charts/common/templates/validations/_redis.tpl b/helm/airbyte-v1/charts/metrics/charts/common/templates/validations/_redis.tpl new file mode 100644 index 00000000..dcccfc1a --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/charts/common/templates/validations/_redis.tpl @@ -0,0 +1,76 @@ + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate Redis® required passwords are not empty. + +Usage: +{{ include "common.validations.values.redis.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where redis values are stored, e.g: "redis-passwords-secret" + - subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.redis.passwords" -}} + {{- $enabled := include "common.redis.values.enabled" . -}} + {{- $valueKeyPrefix := include "common.redis.values.keys.prefix" . -}} + {{- $standarizedVersion := include "common.redis.values.standarized.version" . }} + + {{- $existingSecret := ternary (printf "%s%s" $valueKeyPrefix "auth.existingSecret") (printf "%s%s" $valueKeyPrefix "existingSecret") (eq $standarizedVersion "true") }} + {{- $existingSecretValue := include "common.utils.getValueFromKey" (dict "key" $existingSecret "context" .context) }} + + {{- $valueKeyRedisPassword := ternary (printf "%s%s" $valueKeyPrefix "auth.password") (printf "%s%s" $valueKeyPrefix "password") (eq $standarizedVersion "true") }} + {{- $valueKeyRedisUseAuth := ternary (printf "%s%s" $valueKeyPrefix "auth.enabled") (printf "%s%s" $valueKeyPrefix "usePassword") (eq $standarizedVersion "true") }} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $useAuth := include "common.utils.getValueFromKey" (dict "key" $valueKeyRedisUseAuth "context" .context) -}} + {{- if eq $useAuth "true" -}} + {{- $requiredRedisPassword := dict "valueKey" $valueKeyRedisPassword "secret" .secret "field" "redis-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRedisPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled redis. + +Usage: +{{ include "common.redis.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.redis.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.redis.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right prefix path for the values + +Usage: +{{ include "common.redis.values.key.prefix" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false +*/}} +{{- define "common.redis.values.keys.prefix" -}} + {{- if .subchart -}}redis.{{- else -}}{{- end -}} +{{- end -}} + +{{/* +Checks whether the redis chart's includes the standarizations (version >= 14) + +Usage: +{{ include "common.redis.values.standarized.version" (dict "context" $) }} +*/}} +{{- define "common.redis.values.standarized.version" -}} + + {{- $standarizedAuth := printf "%s%s" (include "common.redis.values.keys.prefix" .) "auth" -}} + {{- $standarizedAuthValues := include "common.utils.getValueFromKey" (dict "key" $standarizedAuth "context" .context) }} + + {{- if $standarizedAuthValues -}} + {{- true -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/metrics/charts/common/templates/validations/_validations.tpl b/helm/airbyte-v1/charts/metrics/charts/common/templates/validations/_validations.tpl new file mode 100644 index 00000000..9a814cf4 --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/charts/common/templates/validations/_validations.tpl @@ -0,0 +1,46 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate values must not be empty. + +Usage: +{{- $validateValueConf00 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-00") -}} +{{- $validateValueConf01 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-01") -}} +{{ include "common.validations.values.empty" (dict "required" (list $validateValueConf00 $validateValueConf01) "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" +*/}} +{{- define "common.validations.values.multiple.empty" -}} + {{- range .required -}} + {{- include "common.validations.values.single.empty" (dict "valueKey" .valueKey "secret" .secret "field" .field "context" $.context) -}} + {{- end -}} +{{- end -}} + +{{/* +Validate a value must not be empty. + +Usage: +{{ include "common.validations.value.empty" (dict "valueKey" "mariadb.password" "secret" "secretName" "field" "my-password" "subchart" "subchart" "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" + - subchart - String - Optional - Name of the subchart that the validated password is part of. +*/}} +{{- define "common.validations.values.single.empty" -}} + {{- $value := include "common.utils.getValueFromKey" (dict "key" .valueKey "context" .context) }} + {{- $subchart := ternary "" (printf "%s." .subchart) (empty .subchart) }} + + {{- if not $value -}} + {{- $varname := "my-value" -}} + {{- $getCurrentValue := "" -}} + {{- if and .secret .field -}} + {{- $varname = include "common.utils.fieldToEnvVar" . -}} + {{- $getCurrentValue = printf " To get the current value:\n\n %s\n" (include "common.utils.secret.getvalue" .) -}} + {{- end -}} + {{- printf "\n '%s' must not be empty, please add '--set %s%s=$%s' to the command.%s" .valueKey $subchart .valueKey $varname $getCurrentValue -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/metrics/charts/common/values.yaml b/helm/airbyte-v1/charts/metrics/charts/common/values.yaml new file mode 100644 index 00000000..f2df68e5 --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/charts/common/values.yaml @@ -0,0 +1,5 @@ +## bitnami/common +## It is required by CI/CD tools and processes. +## @skip exampleValue +## +exampleValue: common-chart diff --git a/helm/airbyte-v1/charts/metrics/templates/_helpers.tpl b/helm/airbyte-v1/charts/metrics/templates/_helpers.tpl new file mode 100644 index 00000000..9f3ccdc6 --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/templates/_helpers.tpl @@ -0,0 +1,39 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "airbyte.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airbyte.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airbyte.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Define db secret +*/}} + +{{- define "database.secret.name" -}} +{{- printf "%s-postgresql" .Release.Name }} +{{- end }} \ No newline at end of file diff --git a/helm/airbyte-v1/charts/metrics/templates/_images.tpl b/helm/airbyte-v1/charts/metrics/templates/_images.tpl new file mode 100644 index 00000000..6c32ec44 --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/templates/_images.tpl @@ -0,0 +1,17 @@ +{{- define "imageUrl" -}} +{{- $img := index . 0 -}} +{{- $root := index . 1 -}} + +{{- $reg := $root.Values.global.image.registry -}} +{{/* ensure the registry has a trailing slash, if set */}} +{{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} + +{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{- if (eq (typeOf $img) "string") -}} +{{- printf "%s%s" $reg (tpl $img $root) | quote -}} +{{- else -}} +{{- $tag := coalesce $img.tag $root.Values.global.image.tag $root.Chart.AppVersion -}} +{{- printf "%s%s:%s" $reg $img.repository $tag | quote -}} +{{- end -}} + +{{- end -}} \ No newline at end of file diff --git a/helm/airbyte-v1/charts/metrics/templates/deployment.yaml b/helm/airbyte-v1/charts/metrics/templates/deployment.yaml new file mode 100644 index 00000000..d9bf336c --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/templates/deployment.yaml @@ -0,0 +1,124 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.names.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "airbyte.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + helm.sh/chart: {{ include "airbyte.chart" . }} + app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.airbyte.io/fullname: {{ include "airbyte.fullname" . }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + airbyte: metrics + strategy: + type: Recreate # Needed due to volume claims + template: + metadata: + labels: + airbyte: metrics + {{- if .Values.podLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ .Values.global.serviceAccountName }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- end }} + containers: + - name: airbyte-metrics-container + image: {{ include "imageUrl" (list .Values.image $) }} + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + env: + {{- if eq .Values.global.deploymentMode "oss" }} + - name: AIRBYTE_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_VERSION + - name: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + - name: METRIC_CLIENT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: METRIC_CLIENT + - name: OTEL_COLLECTOR_ENDPOINT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: OTEL_COLLECTOR_ENDPOINT + + {{- include "airbyte.database.envs" . | nindent 8 }} + + {{- end }} + # Values from secret + {{- if .Values.secrets }} + {{- range $k, $v := .Values.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: metrics-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if or .Values.env_vars .Values.global.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + # Values from extraEnv for more compability(if you want to use external secret source or other stuff) + {{- if .Values.extraEnv }} + {{- toYaml .Values.extraEnv | nindent 8 }} + {{- end }} + + {{- if .Values.resources }} + resources: + {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + {{- if .Values.containerSecurityContext }} + securityContext: + {{- toYaml .Values.containerSecurityContext | nindent 10 }} + {{- end }} + {{- if .Values.extraVolumeMounts }} + volumeMounts: + {{ toYaml .Values.extraVolumeMounts | nindent 10 }} + {{- end }} + {{- if .Values.extraContainers }} + {{ toYaml .Values.extraContainers | nindent 6 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 6 }} + {{- end }} + securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + volumes: + {{- if .Values.extraVolumes }} + {{ toYaml .Values.extraVolumes | nindent 6 }} + {{- end }} diff --git a/helm/airbyte-v1/charts/metrics/values.yaml b/helm/airbyte-v1/charts/metrics/values.yaml new file mode 100644 index 00000000..6ccaa691 --- /dev/null +++ b/helm/airbyte-v1/charts/metrics/values.yaml @@ -0,0 +1,110 @@ + +global: + # imagePullSecrets: + # - name: "" + serviceAccountName: placeholderServiceAccount + deploymentMode: oss + extraContainers: [] + database: + secretName: "" + secretValue: "" + + # Docker image config that will apply to all images. + image: + # Docker registry to pull platform images from, e.g. http://my-registry:8000/ + registry: "" + # Image tag to use for airbyte images. + # Does not include non-airbyte images such as busybox, temporal, minio, etc. + tag: "" + +enabled: true +## metrics.replicaCount Number of metrics-reporter replicas +replicaCount: 1 + +## metrics.image.repository The repository to use for the airbyte metrics-reporter image. +## metrics.image.pullPolicy the pull policy to use for the airbyte metrics-reporter image +## metrics.image.tag The airbyte metrics-reporter image tag. Defaults to the chart's AppVersion +image: + repository: airbyte/metrics-reporter + pullPolicy: IfNotPresent + +## metrics.podAnnotations [object] Add extra annotations to the metrics-reporter pod +## +podAnnotations: {} + +## metrics.podLabels [object] Add extra labels to the metrics-reporter pod +## +podLabels: {} + +## metrics.containerSecurityContext Security context for the container +## Examples: +## containerSecurityContext: +## runAsNonRoot: true +## runAsUser: 1000 +## readOnlyRootFilesystem: true +containerSecurityContext: {} + +## metrics-reporter app resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## We usually recommend not to specify default resources and to leave this as a conscious +## choice for the user. This also increases chances charts run on environments with little +## resources, such as Minikube. If you do want to specify resources, uncomment the following +## lines, adjust them as necessary, and remove the curly braces after 'resources:'. +## metrics.resources.limits [object] The resources limits for the metrics-reporter container +## metrics.resources.requests [object] The requested resources for the metrics-reporter container +resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + +## metrics.nodeSelector [object] Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## metrics.tolerations [array] Tolerations for metrics-reporter pod assignment. +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## metrics.affinity [object] Affinity and anti-affinity for metrics-reporter pod assignment. +## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## metrics.extraEnv [array] Additional env vars for metrics-reporter pod(s). +## Example: +## +## extraEnv: +## - name: SAMPLE_ENV_VAR +## value: "key=sample-value" +extraEnv: [] + +## metrics.extraVolumeMounts [array] Additional volumeMounts for metrics-reporter container(s). +## Examples (when using `metrics.containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumeMounts: +## - name: tmpdir +## mountPath: /tmp +## +extraVolumeMounts: [] + +## metrics.extraVolumes [array] Additional volumes for metrics-reporter pod(s). +## Examples (when using `metrics.containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumes: +## - name: tmpdir +## emptyDir: {} +## +extraVolumes: [] + +extraContainers: [] + +secrets: {} + +env_vars: {} diff --git a/helm/airbyte-v1/charts/pod-sweeper/.gitignore b/helm/airbyte-v1/charts/pod-sweeper/.gitignore new file mode 100644 index 00000000..88e91e8a --- /dev/null +++ b/helm/airbyte-v1/charts/pod-sweeper/.gitignore @@ -0,0 +1,2 @@ +# Charts are downloaded at install time with `helm dep build`. +charts diff --git a/helm/airbyte-v1/charts/pod-sweeper/Chart.lock b/helm/airbyte-v1/charts/pod-sweeper/Chart.lock new file mode 100644 index 00000000..5fdb23f1 --- /dev/null +++ b/helm/airbyte-v1/charts/pod-sweeper/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + version: 1.17.1 +digest: sha256:dacc73770a5640c011e067ff8840ddf89631fc19016c8d0a9e5ea160e7da8690 +generated: "2022-10-17T18:35:27.057672332Z" diff --git a/helm/airbyte-v1/charts/pod-sweeper/Chart.yaml b/helm/airbyte-v1/charts/pod-sweeper/Chart.yaml new file mode 100644 index 00000000..bc63323e --- /dev/null +++ b/helm/airbyte-v1/charts/pod-sweeper/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +appVersion: 1.3.0 +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x +description: Helm chart to deploy airbyte-pod-sweeper +name: pod-sweeper +type: application +version: 1.3.0 diff --git a/helm/airbyte-v1/charts/pod-sweeper/README.md b/helm/airbyte-v1/charts/pod-sweeper/README.md new file mode 100644 index 00000000..cc34eedd --- /dev/null +++ b/helm/airbyte-v1/charts/pod-sweeper/README.md @@ -0,0 +1,71 @@ +# pod-sweeper + +![Version: 0.67.17](https://img.shields.io/badge/Version-0.67.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) + +Helm chart to deploy airbyte-pod-sweeper + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://charts.bitnami.com/bitnami | common | 1.x.x | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| containerSecurityContext | object | `{}` | | +| enabled | bool | `true` | | +| extraVolumeMounts | list | `[]` | | +| extraVolumes | list | `[]` | | +| global.database.secretName | string | `""` | | +| global.database.secretValue | string | `""` | | +| global.jobs.kube.annotations | object | `{}` | | +| global.jobs.kube.main_container_image_pull_secret | string | `""` | | +| global.jobs.kube.nodeSelector | object | `{}` | | +| global.jobs.kube.tolerations | list | `[]` | | +| global.jobs.resources.limits | object | `{}` | | +| global.jobs.resources.requests | object | `{}` | | +| global.logs.accessKey.existingSecret | string | `""` | | +| global.logs.accessKey.existingSecretKey | string | `""` | | +| global.logs.accessKey.password | string | `"minio"` | | +| global.logs.gcs.bucket | string | `""` | | +| global.logs.gcs.credentials | string | `""` | | +| global.logs.gcs.credentialsJson | string | `""` | | +| global.logs.minio.enabled | bool | `true` | | +| global.logs.s3.bucket | string | `"airbyte-dev-logs"` | | +| global.logs.s3.bucketRegion | string | `""` | | +| global.logs.s3.enabled | bool | `false` | | +| global.logs.secretKey.existingSecret | string | `""` | | +| global.logs.secretKey.existingSecretKey | string | `""` | | +| global.logs.secretKey.password | string | `"minio123"` | | +| global.secretName | string | `""` | | +| global.serviceAccountName | string | `"airbyte-admin"` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"bitnami/kubectl"` | | +| image.tag | string | `"latest"` | | +| livenessProbe.enabled | bool | `true` | | +| livenessProbe.failureThreshold | int | `3` | | +| livenessProbe.initialDelaySeconds | int | `5` | | +| livenessProbe.periodSeconds | int | `30` | | +| livenessProbe.successThreshold | int | `1` | | +| livenessProbe.timeoutSeconds | int | `1` | | +| namespace | string | `""` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| podLabels | object | `{}` | | +| readinessProbe.enabled | bool | `true` | | +| readinessProbe.failureThreshold | int | `3` | | +| readinessProbe.initialDelaySeconds | int | `5` | | +| readinessProbe.periodSeconds | int | `30` | | +| readinessProbe.successThreshold | int | `1` | | +| readinessProbe.timeoutSeconds | int | `1` | | +| replicaCount | int | `1` | | +| resources.limits | object | `{}` | | +| resources.requests | object | `{}` | | +| timeToDeletePods.running | string | `""` | | +| timeToDeletePods.succeeded | int | `120` | | +| timeToDeletePods.unsuccessful | int | `1440` | | +| tolerations | list | `[]` | | + diff --git a/helm/airbyte-v1/charts/pod-sweeper/templates/_helpers.tpl b/helm/airbyte-v1/charts/pod-sweeper/templates/_helpers.tpl new file mode 100644 index 00000000..847ec7b4 --- /dev/null +++ b/helm/airbyte-v1/charts/pod-sweeper/templates/_helpers.tpl @@ -0,0 +1,59 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "airbyte.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airbyte.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airbyte.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "airbyte.labels" -}} +helm.sh/chart: {{ include "airbyte.chart" . }} +{{ include "airbyte.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "airbyte.selectorLabels" -}} +app.kubernetes.io/name: {{ include "airbyte.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Define db secret +*/}} + +{{- define "database.secret.name" -}} +{{- printf "%s-postgresql" .Release.Name }} +{{- end }} diff --git a/helm/airbyte-v1/charts/pod-sweeper/templates/_images.tpl b/helm/airbyte-v1/charts/pod-sweeper/templates/_images.tpl new file mode 100644 index 00000000..6c32ec44 --- /dev/null +++ b/helm/airbyte-v1/charts/pod-sweeper/templates/_images.tpl @@ -0,0 +1,17 @@ +{{- define "imageUrl" -}} +{{- $img := index . 0 -}} +{{- $root := index . 1 -}} + +{{- $reg := $root.Values.global.image.registry -}} +{{/* ensure the registry has a trailing slash, if set */}} +{{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} + +{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{- if (eq (typeOf $img) "string") -}} +{{- printf "%s%s" $reg (tpl $img $root) | quote -}} +{{- else -}} +{{- $tag := coalesce $img.tag $root.Values.global.image.tag $root.Chart.AppVersion -}} +{{- printf "%s%s:%s" $reg $img.repository $tag | quote -}} +{{- end -}} + +{{- end -}} \ No newline at end of file diff --git a/helm/airbyte-v1/charts/pod-sweeper/templates/configmap.yaml b/helm/airbyte-v1/charts/pod-sweeper/templates/configmap.yaml new file mode 100644 index 00000000..96ce9350 --- /dev/null +++ b/helm/airbyte-v1/charts/pod-sweeper/templates/configmap.yaml @@ -0,0 +1,73 @@ +--- + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.names.fullname" . }}-sweep-pod-script + namespace: {{ .Values.namespace | default .Release.Namespace }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} + +data: + sweep-pod.sh: | + #!/bin/bash + get_job_pods () { + kubectl -n ${KUBE_NAMESPACE} -L airbyte -l airbyte=job-pod \ + get pods \ + -o=jsonpath='{range .items[*]} {.metadata.name} {.status.phase} {.status.conditions[0].lastTransitionTime} {.status.startTime}{"\n"}{end}' + } + delete_pod() { + printf "From status '%s' since '%s', " $2 $3 + echo "$1" | grep -v "STATUS" | awk '{print $1}' | xargs --no-run-if-empty kubectl -n ${KUBE_NAMESPACE} delete pod + } + while : + do + echo "Starting pod sweeper cycle:" + + if [ -n "${RUNNING_TTL_MINUTES}" ]; then + # Time window for running pods + RUNNING_DATE_STR=`date -d "now - ${RUNNING_TTL_MINUTES} minutes" --utc -Ins` + RUNNING_DATE=`date -d $RUNNING_DATE_STR +%s` + echo "Will sweep running pods from before ${RUNNING_DATE_STR}" + fi + + if [ -n "${SUCCEEDED_TTL_MINUTES}" ]; then + # Shorter time window for succeeded pods + SUCCESS_DATE_STR=`date -d "now - ${SUCCEEDED_TTL_MINUTES} minutes" --utc -Ins` + SUCCESS_DATE=`date -d $SUCCESS_DATE_STR +%s` + echo "Will sweep succeeded pods from before ${SUCCESS_DATE_STR}" + fi + + if [ -n "${UNSUCCESSFUL_TTL_MINUTES}" ]; then + # Longer time window for unsuccessful pods (to debug) + NON_SUCCESS_DATE_STR=`date -d "now - ${UNSUCCESSFUL_TTL_MINUTES} minutes" --utc -Ins` + NON_SUCCESS_DATE=`date -d $NON_SUCCESS_DATE_STR +%s` + echo "Will sweep unsuccessful pods from before ${NON_SUCCESS_DATE_STR}" + fi + ( + IFS=$'\n' + for POD in `get_job_pods`; do + IFS=' ' + POD_NAME=`echo $POD | cut -d " " -f 1` + POD_STATUS=`echo $POD | cut -d " " -f 2` + POD_DATE_STR=`echo $POD | cut -d " " -f 3` + POD_START_DATE_STR=`echo $POD | cut -d " " -f 4` + POD_DATE=`date -d ${POD_DATE_STR:-$POD_START_DATE_STR} '+%s'` + if [ -n "${RUNNING_TTL_MINUTES}" ] && [ "$POD_STATUS" = "Running" ]; then + if [ "$POD_DATE" -lt "$RUNNING_DATE" ]; then + delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" + fi + elif [ -n "${SUCCEEDED_TTL_MINUTES}" ] && [ "$POD_STATUS" = "Succeeded" ]; then + if [ "$POD_DATE" -lt "$SUCCESS_DATE" ]; then + delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" + fi + elif [ -n "${UNSUCCESSFUL_TTL_MINUTES}" ] && [ "$POD_STATUS" != "Running" ] && [ "$POD_STATUS" != "Succeeded" ]; then + if [ "$POD_DATE" -lt "$NON_SUCCESS_DATE" ]; then + delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" + fi + fi + done + ) + echo "Completed pod sweeper cycle. Sleeping for 60 seconds..." + sleep 60 + done diff --git a/helm/airbyte-v1/charts/pod-sweeper/templates/deployment.yaml b/helm/airbyte-v1/charts/pod-sweeper/templates/deployment.yaml new file mode 100644 index 00000000..374b9a4f --- /dev/null +++ b/helm/airbyte-v1/charts/pod-sweeper/templates/deployment.yaml @@ -0,0 +1,105 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.names.fullname" . }} + namespace: {{ .Values.namespace | default .Release.Namespace }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} + namespace: {{ .Values.namespace | default .Release.Namespace }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "airbyte.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + {{- if .Values.podLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} + {{- end }} + annotations: + checksum/sweep-pod-script: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- if .Values.podAnnotations }} + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ .Values.global.serviceAccountName }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + containers: + - name: airbyte-pod-sweeper + image: {{ include "imageUrl" (list .Values.image $) }} + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + env: + - name: KUBE_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: RUNNING_TTL_MINUTES + value: "{{ .Values.timeToDeletePods.running }}" + - name: SUCCEEDED_TTL_MINUTES + value: "{{ .Values.timeToDeletePods.succeeded }}" + - name: UNSUCCESSFUL_TTL_MINUTES + value: "{{ .Values.timeToDeletePods.unsuccessful }}" + {{- if .Values.containerSecurityContext }} + securityContext: {{- toYaml .Values.containerSecurityContext | nindent 10 }} + {{- end }} + volumeMounts: + - mountPath: /script/sweep-pod.sh + subPath: sweep-pod.sh + name: sweep-pod-script + - mountPath: /.kube + name: kube-config + command: ["/bin/bash", "-c", /script/sweep-pod.sh] + {{- if .Values.resources }} + resources: {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + exec: + command: + - /bin/sh + - -ec + - grep -aq sweep-pod.sh /proc/1/cmdline + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + exec: + command: + - /bin/sh + - -ec + - grep -aq sweep-pod.sh /proc/1/cmdline + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + volumes: + - name: kube-config + emptyDir: {} + - name: sweep-pod-script + configMap: + name: {{ include "common.names.fullname" . }}-sweep-pod-script + defaultMode: 0755 diff --git a/helm/airbyte-v1/charts/pod-sweeper/values.yaml b/helm/airbyte-v1/charts/pod-sweeper/values.yaml new file mode 100644 index 00000000..fa4bb8a6 --- /dev/null +++ b/helm/airbyte-v1/charts/pod-sweeper/values.yaml @@ -0,0 +1,120 @@ +## global.serviceAccountName Name of service account to be associated with service +global: + serviceAccountName: &service-account-name "airbyte-admin" + # Docker image config that will apply to all images. + image: + # Docker registry to pull platform images from, e.g. http://my-registry:8000/ + registry: "" + # Image tag to use for airbyte images. + # Does not include non-airbyte images such as busybox, temporal, minio, etc. + tag: "" + +enabled: true +## podSweeper.image.repository The image repository to use for the pod sweeper +## podSweeper.image.pullPolicy The pull policy for the pod sweeper image +## podSweeper.image.tag The pod sweeper image tag to use +image: + repository: bitnami/kubectl + pullPolicy: IfNotPresent + tag: latest + +## podSweeper.podAnnotations [object] Add extra annotations to the podSweeper pod +## +podAnnotations: {} + +## podSweeper.podLabels [object] Add extra labels to the podSweeper pod +## +podLabels: {} + +## podSweeper.containerSecurityContext Security context for the container +## Examples: +## containerSecurityContext: +## runAsNonRoot: true +## runAsUser: 1000 +## readOnlyRootFilesystem: true +containerSecurityContext: {} + +## Configure extra options for the podSweeper containers' liveness and readiness probes +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes +## podSweeper.livenessProbe.enabled Enable livenessProbe on the podSweeper +## podSweeper.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe +## podSweeper.livenessProbe.periodSeconds Period seconds for livenessProbe +## podSweeper.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe +## podSweeper.livenessProbe.failureThreshold Failure threshold for livenessProbe +## podSweeper.livenessProbe.successThreshold Success threshold for livenessProbe +## +livenessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 30 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + +## podSweeper.readinessProbe.enabled Enable readinessProbe on the podSweeper +## podSweeper.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe +## podSweeper.readinessProbe.periodSeconds Period seconds for readinessProbe +## podSweeper.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe +## podSweeper.readinessProbe.failureThreshold Failure threshold for readinessProbe +## podSweeper.readinessProbe.successThreshold Success threshold for readinessProbe +## +readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 30 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + +## Pod Sweeper app resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## We usually recommend not to specify default resources and to leave this as a conscious +## choice for the user. This also increases chances charts run on environments with little +## resources, such as Minikube. If you do want to specify resources, uncomment the following +## lines, adjust them as necessary, and remove the curly braces after 'resources:'. +## podSweeper.resources.limits [object] The resources limits for the podSweeper container +## podSweeper.resources.requests [object] The requested resources for the podSweeper container +resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + +## podSweeper.nodeSelector [object] Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## podSweeper.tolerations [array] Tolerations for podSweeper pod assignment. +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## podSweeper.affinity [object] Affinity and anti-affinity for podSweeper pod assignment. +## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Pod TTLs in minutes +## pod-sweeper will skip over pods with any status types with no TTL set (empty string) +## Pod Status docs: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase +## +## podSweeper.timeToDeletePods.running Time to remove pods on running status (minutes). +## podSweeper.timeToDeletePods.succeeded Time to remove pods on succeeded status (minutes). +## podSweeper.timeToDeletePods.unsuccessful Time to remove pods on neither running nor succeeded status (minutes). +timeToDeletePods: + running: "" + succeeded: 10 + ## 2 hours - generally long enough for someone trying to reproduce an error to debug. + unsuccessful: 120 + +replicaCount: 1 + +## namespace [string] namespace to deploy pod-sweeper at +namespace: "" diff --git a/helm/airbyte-v1/charts/server/.gitignore b/helm/airbyte-v1/charts/server/.gitignore new file mode 100644 index 00000000..88e91e8a --- /dev/null +++ b/helm/airbyte-v1/charts/server/.gitignore @@ -0,0 +1,2 @@ +# Charts are downloaded at install time with `helm dep build`. +charts diff --git a/helm/airbyte-v1/charts/server/Chart.lock b/helm/airbyte-v1/charts/server/Chart.lock new file mode 100644 index 00000000..746fb41b --- /dev/null +++ b/helm/airbyte-v1/charts/server/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + version: 1.17.1 +digest: sha256:dacc73770a5640c011e067ff8840ddf89631fc19016c8d0a9e5ea160e7da8690 +generated: "2022-10-17T18:35:15.123937677Z" diff --git a/helm/airbyte-v1/charts/server/Chart.yaml b/helm/airbyte-v1/charts/server/Chart.yaml new file mode 100644 index 00000000..f3dbb6e0 --- /dev/null +++ b/helm/airbyte-v1/charts/server/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +appVersion: 1.3.0 +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x +description: Helm chart to deploy airbyte-server +name: server +type: application +version: 1.3.0 diff --git a/helm/airbyte-v1/charts/server/README.md b/helm/airbyte-v1/charts/server/README.md new file mode 100644 index 00000000..b31933fb --- /dev/null +++ b/helm/airbyte-v1/charts/server/README.md @@ -0,0 +1,68 @@ +# server + +![Version: 0.67.17](https://img.shields.io/badge/Version-0.67.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) + +Helm chart to deploy airbyte-server + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://charts.bitnami.com/bitnami | common | 1.x.x | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| containerSecurityContext | object | `{}` | | +| debug.enabled | bool | `false` | | +| debug.remoteDebugPort | int | `5005` | | +| deploymentStrategyType | string | `"Recreate"` | | +| enabled | bool | `true` | | +| env_vars | object | `{}` | | +| extraContainers | list | `[]` | | +| extraEnv | list | `[]` | | +| extraInitContainers | list | `[]` | | +| extraLabels | object | `{}` | | +| extraSelectorLabels | object | `{}` | | +| extraVolumeMounts | list | `[]` | | +| extraVolumes | list | `[]` | | +| global.configMapName | string | `""` | | +| global.credVolumeOverride | string | `""` | | +| global.database.secretName | string | `""` | | +| global.database.secretValue | string | `""` | | +| global.deploymentMode | string | `"oss"` | | +| global.edition | string | `"community"` | | +| global.extraContainers | list | `[]` | | +| global.extraLabels | object | `{}` | | +| global.extraSelectorLabels | object | `{}` | | +| global.secretName | string | `""` | | +| global.serviceAccountName | string | `"placeholderServiceAccount"` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"airbyte/server"` | | +| livenessProbe.enabled | bool | `true` | | +| livenessProbe.failureThreshold | int | `3` | | +| livenessProbe.initialDelaySeconds | int | `60` | | +| livenessProbe.periodSeconds | int | `10` | | +| livenessProbe.successThreshold | int | `1` | | +| livenessProbe.timeoutSeconds | int | `1` | | +| log.level | string | `"INFO"` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| podLabels | object | `{}` | | +| readinessProbe.enabled | bool | `true` | | +| readinessProbe.failureThreshold | int | `3` | | +| readinessProbe.initialDelaySeconds | int | `30` | | +| readinessProbe.periodSeconds | int | `10` | | +| readinessProbe.successThreshold | int | `1` | | +| readinessProbe.timeoutSeconds | int | `1` | | +| replicaCount | int | `1` | | +| resources.limits | object | `{}` | | +| resources.requests | object | `{}` | | +| secrets | object | `{}` | | +| service.annotations | object | `{}` | | +| service.port | int | `8001` | | +| service.type | string | `"ClusterIP"` | | +| tolerations | list | `[]` | | + diff --git a/helm/airbyte-v1/charts/server/templates/_helpers.tpl b/helm/airbyte-v1/charts/server/templates/_helpers.tpl new file mode 100644 index 00000000..b8ac92f8 --- /dev/null +++ b/helm/airbyte-v1/charts/server/templates/_helpers.tpl @@ -0,0 +1,242 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "airbyte.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airbyte.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airbyte.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "airbyte.labels" -}} +helm.sh/chart: {{ include "airbyte.chart" . }} +{{ include "airbyte.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "airbyte.selectorLabels" -}} +app.kubernetes.io/name: {{ include "airbyte.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Define db secret +*/}} + +{{- define "database.secret.name" -}} +{{- printf "%s-postgresql" .Release.Name }} +{{- end }} + +## DEFAULT HELM VALUES +# Secret Manager Defaults +{{/* +Define secret persistence +*/}} +{{- define "airbyte.secretPersistence" -}} +{{- if (((.Values.global).secretsManager).type) }} + {{- printf "%s" (snakecase .Values.global.secretsManager.type) }} +{{- else }} + {{- printf "" }} +{{- end }} +{{- end }} + +{{/* +Get secret store name or default +*/}} +{{- define "airbyte.secretStoreName" -}} +{{- $secretStoreName := . -}} +{{- if $secretStoreName -}} + {{- printf "%s" $secretStoreName -}} +{{- else -}} + {{- printf "airbyte-config-secrets" -}} +{{- end -}} +{{- end -}} + +{{/* +Get awsSecretManager access key id secret key or default +*/}} +{{- define "airbyte.awsSecretManagerAccessKeyIdSecretKey" -}} +{{- $awsSecretManagerAccessKeyIdSecretKey := . -}} +{{- if $awsSecretManagerAccessKeyIdSecretKey -}} + {{- printf "%s" $awsSecretManagerAccessKeyIdSecretKey -}} +{{- else -}} + {{- printf "aws-secret-manager-access-key-id" -}} +{{- end -}} +{{- end -}} + +{{/* +Get awsSecretManager secret access key secret key or default +*/}} +{{- define "airbyte.awsSecretManagerSecretAccessKeySecretKey" -}} +{{- $awsSecretManagerSecretAccessKeySecretKey := . -}} +{{- if $awsSecretManagerSecretAccessKeySecretKey -}} + {{- printf "%s" $awsSecretManagerSecretAccessKeySecretKey -}} +{{- else -}} + {{- printf "aws-secret-manager-secret-access-key" -}} +{{- end -}} +{{- end -}} + +{{/* +Get googleSecretManager credentials secret key or default +*/}} +{{- define "airbyte.googleSecretManagerCredentialsSecretKey" -}} +{{- $googleSecretManagerCredentialsSecretKey := . -}} +{{- if $googleSecretManagerCredentialsSecretKey -}} + {{- printf "%s" $googleSecretManagerCredentialsSecretKey -}} +{{- else -}} + {{- printf "google-secret-manager-credentials" -}} +{{- end -}} +{{- end -}} + +{{/* +Get vault auth token secret key or default +*/}} +{{- define "airbyte.vaultAuthTokenSecretKey" -}} +{{- $vaultAuthTokenSecretKey := . -}} +{{- if $vaultAuthTokenSecretKey -}} + {{- printf "%s" $vaultAuthTokenSecretKey -}} +{{- else -}} + {{- printf "vault-auth-token" -}} +{{- end -}} +{{- end -}} + + +# Storage Defaults +{{/* +Get storage type or default +*/}} +{{- define "airbyte.storageType" -}} +{{- $storageType := . -}} +{{- if $storageType -}} + {{- printf "%s" $storageType -}} +{{- else -}} + {{- printf "local" -}} +{{- end -}} +{{- end -}} + +{{/* +Get storage bucket log or default +*/}} +{{- define "airbyte.storageBucketLog" -}} +{{- $storageBucketLog := . -}} +{{- if $storageBucketLog -}} + {{- printf "%s" $storageBucketLog -}} +{{- else -}} + {{- printf "airbyte-storage" -}} +{{- end -}} +{{- end -}} + +{{/* +Get storage bucket state or default +*/}} +{{- define "airbyte.storageBucketState" -}} +{{- $storageBucketState := . -}} +{{- if $storageBucketState -}} + {{- printf "%s" $storageBucketState -}} +{{- else -}} + {{- printf "airbyte-storage" -}} +{{- end -}} +{{- end -}} + +{{/* +Get storage bucket workload output or default +*/}} +{{- define "airbyte.storageBucketWorkloadOutput" -}} +{{- $storageBucketWorkloadOutput := . -}} +{{- if $storageBucketWorkloadOutput -}} + {{- printf "%s" $storageBucketWorkloadOutput -}} +{{- else -}} + {{- printf "airbyte-storage" -}} +{{- end -}} +{{- end -}} + +{{/* +Get s3 access key id secret key or default +*/}} +{{- define "airbyte.s3AccessKeyIdSecretKey" -}} +{{- $s3AccessKeyIdSecretKey := . -}} +{{- if $s3AccessKeyIdSecretKey -}} + {{- printf "%s" $s3AccessKeyIdSecretKey -}} +{{- else -}} + {{- printf "s3-access-key-id" -}} +{{- end -}} +{{- end -}} + +{{/* +Get s3 secret access key secret key or default +*/}} +{{- define "airbyte.s3SecretAccessKeySecretKey" -}} +{{- $s3SecretAccessKeySecretKey := . -}} +{{- if $s3SecretAccessKeySecretKey -}} + {{- printf "%s" $s3SecretAccessKeySecretKey -}} +{{- else -}} + {{- printf "s3-secret-access-key" -}} +{{- end -}} +{{- end -}} + +{{/* +Get minio access key id secret key or default +*/}} +{{- define "airbyte.minioAccessKeyIdSecretKey" -}} +{{- $minioAccessKeyIdSecretKey := . -}} +{{- if $minioAccessKeyIdSecretKey -}} + {{- printf "%s" $minioAccessKeyIdSecretKey -}} +{{- else -}} + {{- printf "minio-access-key-id" -}} +{{- end -}} +{{- end -}} + +{{/* +Get minio secret access key secret key or default +*/}} +{{- define "airbyte.minioSecretAccessKeySecretKey" -}} +{{- $minioSecretAccessKeySecretKey := . -}} +{{- if $minioSecretAccessKeySecretKey -}} + {{- printf "%s" $minioSecretAccessKeySecretKey -}} +{{- else -}} + {{- printf "minio-secret-access-key" -}} +{{- end -}} +{{- end -}} + +{{/* +Get gcs credentials secret key or default +*/}} +{{- define "airbyte.gcsCredentialsSecretKey" -}} +{{- $gcsCredentialsSecretKey := . -}} +{{- if $gcsCredentialsSecretKey -}} + {{- printf "%s" $gcsCredentialsSecretKey -}} +{{- else -}} + {{- printf "gcs-credentials" -}} +{{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/server/templates/_images.tpl b/helm/airbyte-v1/charts/server/templates/_images.tpl new file mode 100644 index 00000000..6c32ec44 --- /dev/null +++ b/helm/airbyte-v1/charts/server/templates/_images.tpl @@ -0,0 +1,17 @@ +{{- define "imageUrl" -}} +{{- $img := index . 0 -}} +{{- $root := index . 1 -}} + +{{- $reg := $root.Values.global.image.registry -}} +{{/* ensure the registry has a trailing slash, if set */}} +{{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} + +{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{- if (eq (typeOf $img) "string") -}} +{{- printf "%s%s" $reg (tpl $img $root) | quote -}} +{{- else -}} +{{- $tag := coalesce $img.tag $root.Values.global.image.tag $root.Chart.AppVersion -}} +{{- printf "%s%s:%s" $reg $img.repository $tag | quote -}} +{{- end -}} + +{{- end -}} \ No newline at end of file diff --git a/helm/airbyte-v1/charts/server/templates/deployment.yaml b/helm/airbyte-v1/charts/server/templates/deployment.yaml new file mode 100644 index 00000000..f9a1b72e --- /dev/null +++ b/helm/airbyte-v1/charts/server/templates/deployment.yaml @@ -0,0 +1,463 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.names.fullname" . }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- if .Values.extraLabels }} + {{ toYaml (mergeOverwrite .Values.extraLabels .Values.global.extraLabels) | nindent 4 }} + {{- end }} +spec: + minReadySeconds: 30 + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "airbyte.selectorLabels" . | nindent 6 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 6 }} + {{- end }} + strategy: + type: {{ .Values.deploymentStrategyType }} # Needed due to volume claims + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 8 }} + {{- end }} + {{- if .Values.podLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ .Values.global.serviceAccountName }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.extraInitContainers }} + initContainers: + {{- toYaml .Values.extraInitContainers | nindent 6 }} + {{- end }} + containers: + - name: airbyte-server-container + image: {{ include "imageUrl" (list .Values.image $) }} + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + env: + {{- if .Values.debug.enabled }} + - name: JAVA_TOOL_OPTIONS + value: "-Xdebug -agentlib:jdwp=transport=dt_socket,address=0.0.0.0:{{ .Values.debug.remoteDebugPort }},server=y,suspend=n" + {{- end}} + {{- if or (eq .Values.global.edition "pro") (eq .Values.global.edition "enterprise") }} + - name: API_AUTHORIZATION_ENABLED + value: "true" + {{- else if and (eq .Values.global.deploymentMode "oss") .Values.global.auth.enabled }} + # Self-Managed Enterprise should always have API_AUTHORIZATION_ENABLED set to true, even + # if global.auth.enabled is not set to true. This can be simplified in the future, once + # globa.auth.enabled is changed to always default to true across all editions of Airbyte. + - name: API_AUTHORIZATION_ENABLED + value: "true" + {{- end }} + {{- if eq .Values.global.deploymentMode "oss" }} + {{- include "airbyte.logging.envs" . | nindent 8 }} + - name: AIRBYTE_API_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_API_HOST + - name: AIRBYTE_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_VERSION + - name: AIRBYTE_EDITION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_EDITION + - name: AIRBYTE_URL + valueFrom: + configMapKeyRef: + name: {{ .Values.global.configMapName | default (printf "%s-airbyte-env" .Release.Name) }} + key: AIRBYTE_URL + - name: CONFIG_ROOT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONFIG_ROOT + # - name: BLOTOUT_AUTH_ENDPOINT + # valueFrom: + # configMapKeyRef: + # name: {{ .Release.Name }}-airbyte-env + # key: BLOTOUT_AUTH_ENDPOINT + # - name: BLOTOUT_BASE_URL + # valueFrom: + # configMapKeyRef: + # name: {{ .Release.Name }}-airbyte-env + # key: BLOTOUT_BASE_URL + - name: MICROMETER_METRICS_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MICROMETER_METRICS_ENABLED + - name: MICROMETER_METRICS_STATSD_FLAVOR + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MICROMETER_METRICS_STATSD_FLAVOR + - name: MICRONAUT_ENVIRONMENTS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: SERVER_MICRONAUT_ENVIRONMENTS + - name: SEGMENT_WRITE_KEY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: SEGMENT_WRITE_KEY + - name: STATSD_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: STATSD_HOST + - name: STATSD_PORT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: STATSD_PORT + - name: TRACKING_STRATEGY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TRACKING_STRATEGY + - name: WORKER_ENVIRONMENT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKER_ENVIRONMENT + - name: WORKSPACE_ROOT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKSPACE_ROOT + - name: WEBAPP_URL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WEBAPP_URL + - name: TEMPORAL_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TEMPORAL_HOST + - name: JOB_MAIN_CONTAINER_CPU_REQUEST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_MAIN_CONTAINER_CPU_REQUEST + - name: JOB_MAIN_CONTAINER_CPU_LIMIT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_MAIN_CONTAINER_CPU_LIMIT + - name: JOB_MAIN_CONTAINER_MEMORY_REQUEST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_MAIN_CONTAINER_MEMORY_REQUEST + - name: JOB_MAIN_CONTAINER_MEMORY_LIMIT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_MAIN_CONTAINER_MEMORY_LIMIT + - name: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + - name: JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + - name: KEYCLOAK_INTERNAL_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: KEYCLOAK_INTERNAL_HOST + {{- if or (eq .Values.global.edition "pro") (eq .Values.global.edition "enterprise") }} + - name: KEYCLOAK_ADMIN_USER + valueFrom: + secretKeyRef: + name: {{ .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} + key: KEYCLOAK_ADMIN_USER + - name: KEYCLOAK_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} + key: KEYCLOAK_ADMIN_PASSWORD + {{- end }} + - name: CONNECTOR_BUILDER_SERVER_API_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONNECTOR_BUILDER_SERVER_API_HOST + - name: AIRBYTE_API_AUTH_HEADER_NAME + value: "X-Airbyte-Auth" + - name: AIRBYTE_API_AUTH_HEADER_VALUE + value: "Internal server" + - name: ENTERPRISE_SOURCE_STUBS_URL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: ENTERPRISE_SOURCE_STUBS_URL + + {{- if and (eq .Values.global.edition "community") .Values.global.auth.enabled }} + # Values for Airbyte auth configurations, secrets, and credentials in Community edition + - name: AB_INSTANCE_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.auth.secretName | default "airbyte-auth-secrets" | quote }} + key: {{ .Values.global.auth.instanceAdmin.passwordSecretKey | default "instance-admin-password" | quote }} + - name: AB_INSTANCE_ADMIN_CLIENT_ID + valueFrom: + secretKeyRef: + name: {{ .Values.global.auth.secretName | default "airbyte-auth-secrets" | quote }} + key: {{ .Values.global.auth.instanceAdmin.clientIdSecretKey | default "instance-admin-client-id" | quote }} + - name: AB_INSTANCE_ADMIN_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ .Values.global.auth.secretName | default "airbyte-auth-secrets" | quote }} + key: {{ .Values.global.auth.instanceAdmin.clientSecretSecretKey | default "instance-admin-client-secret" | quote }} + - name: AB_JWT_SIGNATURE_SECRET + valueFrom: + secretKeyRef: + name: {{ .Values.global.auth.secretName | default "airbyte-auth-secrets" | quote }} + key: {{ .Values.global.auth.jwtSignatureSecretKey | default "jwt-signature-secret" | quote }} + - name: AB_COOKIE_SECURE + value: {{ .Values.global.auth.cookieSecureSetting | default "true" | quote }} + - name: AB_COOKIE_SAME_SITE + value: {{ .Values.global.auth.cookieSameSiteSetting | default "Strict" | quote }} + {{- end }} + + # Secrets Manager + - name: SECRET_PERSISTENCE + value: {{ include "airbyte.secretPersistence" . }} + # Values for AwsSecretsManager + {{- if eq ((((.Values.global).secretsManager).awsSecretManager).authenticationType) "credentials" }} + - name: AWS_SECRET_MANAGER_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }} + key: {{ include "airbyte.awsSecretManagerAccessKeyIdSecretKey" .Values.global.secretsManager.awsSecretManager.accessKeyIdSecretKey }} + - name: AWS_SECRET_MANAGER_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }} + key: {{ include "airbyte.awsSecretManagerSecretAccessKeySecretKey" .Values.global.secretsManager.awsSecretManager.secretAccessKeySecretKey }} + {{- end }} + {{- if ((((.Values.global).secretsManager).awsSecretManager).region) }} + - name: AWS_SECRET_MANAGER_REGION + value: {{ (((.Values.global).secretsManager).awsSecretManager).region }} + {{- end }} + {{- if ((((.Values.global).secretsManager).awsSecretManager).tags) }} + - name: AWS_SECRET_MANAGER_SECRET_TAGS + value: {{ include "airbyte.tagsToString" .Values.global.secretsManager.awsSecretManager.tags }} + {{- end }} + {{- if ((((.Values.global).secretsManager).awsSecretManager).kms) }} + - name: AWS_KMS_KEY_ARN + value: {{ ((((.Values.global).secretsManager).awsSecretManager).kms) | default "" }} + {{- end }} + + # Values for Azure Key Vault + {{- if (((.Values.global).secretsManager).azureKeyVault) }} + + - name: AB_AZURE_KEY_VAULT_VAULT_URL + value: {{ (((.Values.global).secretsManager).azureKeyVault).vaultUrl }} + + - name: AB_AZURE_KEY_VAULT_TENANT_ID + value: {{ (((.Values.global).secretsManager).azureKeyVault).tenantId }} + + - name: AB_AZURE_KEY_VAULT_CLIENT_ID + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }} + key: {{ include "airbyte.azureKeyVaultClientIdSecretKey" .Values.global.secretsManager.azureKeyVault.clientIdSecretKey }} + - name: AB_AZURE_KEY_VAULT_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }} + key: {{ include "airbyte.azureKeyVaultClientSecretSecretKey" .Values.global.secretsManager.azureKeyVault.clientSecretSecretKey }} + + {{- end }} + + {{- if ((((.Values.global).secretsManager).azureKeyVault).tags) }} + - name: AB_AZURE_KEY_VAULT_TAGS + value: {{ include "airbyte.tagsToString" .Values.global.secretsManager.azureKeyVault.tags }} + {{- end }} + + # Values for googleSecretManager secrets + {{- if (((.Values.global).secretsManager).googleSecretManager) }} + - name: SECRET_STORE_GCP_PROJECT_ID + value: {{ .Values.global.secretsManager.googleSecretManager.projectId }} + - name: SECRET_STORE_GCP_CREDENTIALS + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }} + key: {{ include "airbyte.googleSecretManagerCredentialsSecretKey" .Values.global.secretsManager.googleSecretManager.credentialsSecretKey }} + {{- end }} + + # Values for vault secrets + {{- if (((.Values.global).secretsManager).vault) }} + - name: VAULT_ADDRESS + value: {{ (((.Values.global).secretsManager).vault).address }} + - name: VAULT_PREFIX + value: {{ (((.Values.global).secretsManager).vault).prefix }} + - name: VAULT_AUTH_TOKEN + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }} + key: {{ include "airbyte.vaultAuthTokenSecretKey" .Values.global.secretsManager.vault.authTokenSecretKey }} + {{- end }} + + # Storage + {{- include "airbyte.storage.envs" . | nindent 8 }} + + {{- include "airbyte.database.envs" . | nindent 8 }} + {{- include "airbyte.enterprise.instanceAdmin" . | nindent 8 }} + {{- include "airbyte.enterprise.identityProvider" . | nindent 8 }} + {{- include "airbyte.enterprise.license" . | nindent 8 }} + + {{- end }} + + # Values from secret + {{- if .Values.secrets }} + {{- range $k, $v := .Values.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: server-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if or .Values.env_vars .Values.global.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + # Values from extraEnv for more compability(if you want to use external secret source or other stuff) + {{- if .Values.extraEnv }} + {{- toYaml .Values.extraEnv | nindent 8 }} + {{- end }} + + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /api/v1/health + port: http + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /api/v1/health + port: http + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + ports: + - name: http + containerPort: 8001 + protocol: TCP + {{- if .Values.debug.enabled }} + - name: debug + containerPort: {{ .Values.debug.remoteDebugPort }} + protocol: TCP + {{- end}} + {{- if .Values.resources }} + resources: {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + {{- if .Values.containerSecurityContext }} + securityContext: {{- toYaml .Values.containerSecurityContext | nindent 10 }} + {{- end }} + volumeMounts: + {{- if eq (lower .Values.global.storage.type) "local" }} + - name: airbyte-storage + mountPath: /storage + {{- end }} + {{- if eq .Values.global.deploymentMode "oss" }} + {{- if eq (lower (default "" .Values.global.storage.type)) "gcs" }} + - name: gcs-log-creds-volume + mountPath: /secrets/gcs-log-creds + readOnly: true + {{- end }} + {{- if .Values.global.airbyteYml }} + - name: airbyte-yml-volume + mountPath: /app/configs/airbyte.yml + subPath: fileContents + readOnly: true + {{- end }} + {{- end }} + + {{- if .Values.extraVolumeMounts }} + {{ toYaml .Values.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.global.extraVolumeMounts }} + {{ toYaml .Values.global.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.extraContainers }} + {{ toYaml .Values.extraContainers | nindent 6 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 6 }} + {{- end }} + securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + volumes: + {{- if eq (lower .Values.global.storage.type) "local" }} + - name: airbyte-storage + persistentVolumeClaim: + claimName: airbyte-storage-pvc + {{- end }} + + {{- if eq .Values.global.deploymentMode "oss" }} + {{- if eq (lower (default "" .Values.global.storage.type)) "gcs" }} + - name: gcs-log-creds-volume + secret: + secretName: {{ ternary (printf "%s-gcs-log-creds" ( .Release.Name )) .Values.global.storage.storageSecretName (not ((.Values.global.storage).storageSecretName)) }} + {{- end }} + + {{- if .Values.global.airbyteYml }} + - name: airbyte-yml-volume + secret: + secretName: {{ .Release.Name }}-airbyte-yml + {{- end }} + {{- end }} + {{- if .Values.extraVolumes }} +{{ toYaml .Values.extraVolumes | nindent 6 }} + {{- end }} diff --git a/helm/airbyte-v1/charts/server/templates/secrets.yaml b/helm/airbyte-v1/charts/server/templates/secrets.yaml new file mode 100644 index 00000000..f0101d7a --- /dev/null +++ b/helm/airbyte-v1/charts/server/templates/secrets.yaml @@ -0,0 +1,17 @@ +# Create secrets only for the local deployment +{{- if .Values.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: server-secrets + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" +type: Opaque +data: + {{- range $k, $v := mergeOverwrite .Values.secrets .Values.global.secrets }} + {{ $k }}: {{ if $v }}{{ $v | b64enc }} {{else}}""{{end}} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v1/charts/server/templates/service.yaml b/helm/airbyte-v1/charts/server/templates/service.yaml new file mode 100644 index 00000000..d2d8d9b9 --- /dev/null +++ b/helm/airbyte-v1/charts/server/templates/service.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{.Release.Name }}-airbyte-server-svc + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "airbyte.selectorLabels" . | nindent 4 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 4 }} + {{- end }} diff --git a/helm/airbyte-v1/charts/server/values.yaml b/helm/airbyte-v1/charts/server/values.yaml new file mode 100644 index 00000000..4798c10b --- /dev/null +++ b/helm/airbyte-v1/charts/server/values.yaml @@ -0,0 +1,220 @@ + +global: + serviceAccountName: placeholderServiceAccount + edition: community + deploymentMode: oss + configMapName: "" + secretName: "" + credVolumeOverride: "" + extraContainers: [] + ## extraSelectorLabels [object] - use to specify own additional selector labels for deployment + extraSelectorLabels: {} + ## extraLabels [object] - use to specify own additional labels for deployment + extraLabels: {} + database: + secretName: "" + secretValue: "" + + storage: + type: minio + + # Docker image config that will apply to all images. + image: + # Docker registry to pull platform images from, e.g. http://my-registry:8000/ + registry: "" + # Image tag to use for airbyte images. + # Does not include non-airbyte images such as busybox, temporal, minio, etc. + tag: "" + +enabled: true +## replicaCount Number of server replicas +replicaCount: 1 + +## image.repository The repository to use for the airbyte server image. +## image.pullPolicy the pull policy to use for the airbyte server image +## image.tag The airbyte server image tag. Defaults to the chart's AppVersion +image: + repository: airbyte/server + pullPolicy: IfNotPresent + +## podAnnotations [object] Add extra annotations to the server pod +## +podAnnotations: {} + +## podLabels [object] Add extra labels to the server pod +## +podLabels: {} + +## containerSecurityContext Security context for the container +## Examples: +## containerSecurityContext: +## runAsNonRoot: true +## runAsUser: 1000 +## readOnlyRootFilesystem: true +containerSecurityContext: {} + +## Configure extra options for the server containers' liveness and readiness probes +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes +## livenessProbe.enabled Enable livenessProbe on the server +## livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe +## livenessProbe.periodSeconds Period seconds for livenessProbe +## livenessProbe.timeoutSeconds Timeout seconds for livenessProbe +## livenessProbe.failureThreshold Failure threshold for livenessProbe +## livenessProbe.successThreshold Success threshold for livenessProbe +## +livenessProbe: + enabled: true + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + +## readinessProbe.enabled Enable readinessProbe on the server +## readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe +## readinessProbe.periodSeconds Period seconds for readinessProbe +## readinessProbe.timeoutSeconds Timeout seconds for readinessProbe +## readinessProbe.failureThreshold Failure threshold for readinessProbe +## readinessProbe.successThreshold Success threshold for readinessProbe +## +readinessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + +## Server app resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## We usually recommend not to specify default resources and to leave this as a conscious +## choice for the user. This also increases chances charts run on environments with little +## resources, such as Minikube. If you do want to specify resources, uncomment the following +## lines, adjust them as necessary, and remove the curly braces after 'resources:'. +## resources.limits [object] The resources limits for the server container +## resources.requests [object] The requested resources for the server container +resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + +## service.type The service type to use for the API server +## service.port The service port to expose the API server on +service: + type: ClusterIP + port: 8001 + annotations: {} + +## nodeSelector [object] Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## tolerations [array] Tolerations for server pod assignment. +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## affinity [object] Affinity and anti-affinity for server pod assignment. +## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## log.level The log level to log at +log: + level: "INFO" + +## extraVolumeMounts [array] Additional volumeMounts for server container(s). +## Examples (when using `containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumeMounts: +## - name: tmpdir +## mountPath: /tmp +## +extraVolumeMounts: [] + +## extraVolumes [array] Additional volumes for server pod(s). +## Examples (when using `containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumes: +## - name: tmpdir +## emptyDir: {} +## +extraVolumes: [] + +## extraContainer [array] Additional container for server pod(s) +## Example: +# extraContainers: +# - name: otel_collector +# image: somerepo/someimage:sometag +# args: [ +# "--important-args" +# ] +# ports: +# - containerPort: 443 +# volumeMounts: +# - name: volumeMountCool +# mountPath: /some/path +# readOnly: true +extraContainers: [] + +## extraInitContainers [array] Additional init containers for server pod(s) +## Example: +# extraInitContainers: +# - name: sleepy +# image: alpine +# command: ['sleep', '60'] + +extraInitContainers: [] + +## extraEnv [array] Supply extra env variables to main container using full notation +## Example: (With default env vars and values taken from generated config map) +# extraEnv: +# - name: AIRBYTE_VERSION +# valueFrom: +# configMapKeyRef: +# name: airbyte-env +# key: AIRBYTE_VERSION +## +extraEnv: [] +## secrets [object] Supply additional secrets to container +## Example: +## secrets: +## DATABASE_PASSWORD: strong-password +## DATABASE_USER: my-db-user +secrets: {} + +## env_vars [object] Supply extra env variables to main container using simplified notation +## Example: +# env_vars: +# AIRBYTE_VERSION: 0.40.4 + +# # Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db +# DATABASE_HOST: airbyte-db-svc +# DATABASE_PORT: 5432 +# DATABASE_DB: airbyte +# # translate manually DATABASE_URL:jdbc:postgresql://${DATABASE_HOST}:${DATABASE_PORT/${DATABASE_DB} +# DATABASE_URL: jdbc:postgresql://airbyte-db-svc:5432/airbyte +# JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION: 0.29.15.001 +# CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION: 0.35.15.001 +env_vars: {} + + +## extraSelectorLabels [object] - use to specify own additional selector labels for deployment +extraSelectorLabels: {} +## extraLabels [object] - use to specify own additional labels for deployment +extraLabels: {} + + +## deploymentStrategyType [string] - deployment strategy type for airbyte-server deployment. +## Defaults to Recreate since the pod is using pvc +deploymentStrategyType: RollingUpdate + +debug: + enabled: false + remoteDebugPort: 5005 diff --git a/helm/airbyte-v1/charts/temporal-ui/Chart.lock b/helm/airbyte-v1/charts/temporal-ui/Chart.lock new file mode 100644 index 00000000..19b1aca9 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + version: 1.17.1 +digest: sha256:dacc73770a5640c011e067ff8840ddf89631fc19016c8d0a9e5ea160e7da8690 +generated: "2024-12-09T21:57:43.35443455Z" diff --git a/helm/airbyte-v1/charts/temporal-ui/Chart.yaml b/helm/airbyte-v1/charts/temporal-ui/Chart.yaml new file mode 100644 index 00000000..9e6cdc0b --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +appVersion: 1.3.0 +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x +description: Helm chart to deploy airbyte-temporal-ui +name: temporal-ui +type: application +version: 1.3.0 diff --git a/helm/airbyte-v1/charts/temporal-ui/charts/common/.helmignore b/helm/airbyte-v1/charts/temporal-ui/charts/common/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/charts/common/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/airbyte-v1/charts/temporal-ui/charts/common/Chart.yaml b/helm/airbyte-v1/charts/temporal-ui/charts/common/Chart.yaml new file mode 100644 index 00000000..39edbb3c --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/charts/common/Chart.yaml @@ -0,0 +1,23 @@ +annotations: + category: Infrastructure +apiVersion: v2 +appVersion: 1.17.1 +description: A Library Helm Chart for grouping common logic between bitnami charts. + This chart is not deployable by itself. +home: https://github.com/bitnami/charts/tree/master/bitnami/common +icon: https://bitnami.com/downloads/logos/bitnami-mark.png +keywords: +- common +- helper +- template +- function +- bitnami +maintainers: +- name: Bitnami + url: https://github.com/bitnami/charts +name: common +sources: +- https://github.com/bitnami/charts +- https://www.bitnami.com/ +type: library +version: 1.17.1 diff --git a/helm/airbyte-v1/charts/temporal-ui/charts/common/README.md b/helm/airbyte-v1/charts/temporal-ui/charts/common/README.md new file mode 100644 index 00000000..a2ecd604 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/charts/common/README.md @@ -0,0 +1,350 @@ +# Bitnami Common Library Chart + +A [Helm Library Chart](https://helm.sh/docs/topics/library_charts/#helm) for grouping common logic between bitnami charts. + +## TL;DR + +```yaml +dependencies: + - name: common + version: 1.x.x + repository: https://charts.bitnami.com/bitnami +``` + +```bash +$ helm dependency update +``` + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.names.fullname" . }} +data: + myvalue: "Hello World" +``` + +## Introduction + +This chart provides a common template helpers which can be used to develop new charts using [Helm](https://helm.sh) package manager. + +Bitnami charts can be used with [Kubeapps](https://kubeapps.dev/) for deployment and management of Helm Charts in clusters. + +## Prerequisites + +- Kubernetes 1.19+ +- Helm 3.2.0+ + +## Parameters + +The following table lists the helpers available in the library which are scoped in different sections. + +### Affinities + +| Helper identifier | Description | Expected Input | +|-------------------------------|------------------------------------------------------|------------------------------------------------| +| `common.affinities.nodes.soft` | Return a soft nodeAffinity definition | `dict "key" "FOO" "values" (list "BAR" "BAZ")` | +| `common.affinities.nodes.hard` | Return a hard nodeAffinity definition | `dict "key" "FOO" "values" (list "BAR" "BAZ")` | +| `common.affinities.pods.soft` | Return a soft podAffinity/podAntiAffinity definition | `dict "component" "FOO" "context" $` | +| `common.affinities.pods.hard` | Return a hard podAffinity/podAntiAffinity definition | `dict "component" "FOO" "context" $` | + +### Capabilities + +| Helper identifier | Description | Expected Input | +|------------------------------------------------|------------------------------------------------------------------------------------------------|-------------------| +| `common.capabilities.kubeVersion` | Return the target Kubernetes version (using client default if .Values.kubeVersion is not set). | `.` Chart context | +| `common.capabilities.cronjob.apiVersion` | Return the appropriate apiVersion for cronjob. | `.` Chart context | +| `common.capabilities.deployment.apiVersion` | Return the appropriate apiVersion for deployment. | `.` Chart context | +| `common.capabilities.statefulset.apiVersion` | Return the appropriate apiVersion for statefulset. | `.` Chart context | +| `common.capabilities.ingress.apiVersion` | Return the appropriate apiVersion for ingress. | `.` Chart context | +| `common.capabilities.rbac.apiVersion` | Return the appropriate apiVersion for RBAC resources. | `.` Chart context | +| `common.capabilities.crd.apiVersion` | Return the appropriate apiVersion for CRDs. | `.` Chart context | +| `common.capabilities.policy.apiVersion` | Return the appropriate apiVersion for podsecuritypolicy. | `.` Chart context | +| `common.capabilities.networkPolicy.apiVersion` | Return the appropriate apiVersion for networkpolicy. | `.` Chart context | +| `common.capabilities.apiService.apiVersion` | Return the appropriate apiVersion for APIService. | `.` Chart context | +| `common.capabilities.hpa.apiVersion` | Return the appropriate apiVersion for Horizontal Pod Autoscaler | `.` Chart context | +| `common.capabilities.supportsHelmVersion` | Returns true if the used Helm version is 3.3+ | `.` Chart context | + +### Errors + +| Helper identifier | Description | Expected Input | +|-----------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------| +| `common.errors.upgrade.passwords.empty` | It will ensure required passwords are given when we are upgrading a chart. If `validationErrors` is not empty it will throw an error and will stop the upgrade action. | `dict "validationErrors" (list $validationError00 $validationError01) "context" $` | + +### Images + +| Helper identifier | Description | Expected Input | +|-----------------------------|------------------------------------------------------|---------------------------------------------------------------------------------------------------------| +| `common.images.image` | Return the proper and full image name | `dict "imageRoot" .Values.path.to.the.image "global" $`, see [ImageRoot](#imageroot) for the structure. | +| `common.images.pullSecrets` | Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) | `dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global` | +| `common.images.renderPullSecrets` | Return the proper Docker Image Registry Secret Names (evaluates values as templates) | `dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "context" $` | + +### Ingress + +| Helper identifier | Description | Expected Input | +|-------------------------------------------|-------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.ingress.backend` | Generate a proper Ingress backend entry depending on the API version | `dict "serviceName" "foo" "servicePort" "bar"`, see the [Ingress deprecation notice](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/) for the syntax differences | +| `common.ingress.supportsPathType` | Prints "true" if the pathType field is supported | `.` Chart context | +| `common.ingress.supportsIngressClassname` | Prints "true" if the ingressClassname field is supported | `.` Chart context | +| `common.ingress.certManagerRequest` | Prints "true" if required cert-manager annotations for TLS signed certificates are set in the Ingress annotations | `dict "annotations" .Values.path.to.the.ingress.annotations` | + +### Labels + +| Helper identifier | Description | Expected Input | +|-----------------------------|-----------------------------------------------------------------------------|-------------------| +| `common.labels.standard` | Return Kubernetes standard labels | `.` Chart context | +| `common.labels.matchLabels` | Labels to use on `deploy.spec.selector.matchLabels` and `svc.spec.selector` | `.` Chart context | + +### Names + +| Helper identifier | Description | Expected Input | +|-----------------------------------|-----------------------------------------------------------------------|-------------------| +| `common.names.name` | Expand the name of the chart or use `.Values.nameOverride` | `.` Chart context | +| `common.names.fullname` | Create a default fully qualified app name. | `.` Chart context | +| `common.names.namespace` | Allow the release namespace to be overridden | `.` Chart context | +| `common.names.fullname.namespace` | Create a fully qualified app name adding the installation's namespace | `.` Chart context | +| `common.names.chart` | Chart name plus version | `.` Chart context | + +### Secrets + +| Helper identifier | Description | Expected Input | +|---------------------------|--------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.secrets.name` | Generate the name of the secret. | `dict "existingSecret" .Values.path.to.the.existingSecret "defaultNameSuffix" "mySuffix" "context" $` see [ExistingSecret](#existingsecret) for the structure. | +| `common.secrets.key` | Generate secret key. | `dict "existingSecret" .Values.path.to.the.existingSecret "key" "keyName"` see [ExistingSecret](#existingsecret) for the structure. | +| `common.passwords.manage` | Generate secret password or retrieve one if already created. | `dict "secret" "secret-name" "key" "keyName" "providedValues" (list "path.to.password1" "path.to.password2") "length" 10 "strong" false "chartName" "chartName" "context" $`, length, strong and chartNAme fields are optional. | +| `common.secrets.exists` | Returns whether a previous generated secret already exists. | `dict "secret" "secret-name" "context" $` | + +### Storage + +| Helper identifier | Description | Expected Input | +|-------------------------------|---------------------------------------|---------------------------------------------------------------------------------------------------------------------| +| `common.storage.class` | Return the proper Storage Class | `dict "persistence" .Values.path.to.the.persistence "global" $`, see [Persistence](#persistence) for the structure. | + +### TplValues + +| Helper identifier | Description | Expected Input | +|---------------------------|----------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.tplvalues.render` | Renders a value that contains template | `dict "value" .Values.path.to.the.Value "context" $`, value is the value should rendered as template, context frequently is the chart context `$` or `.` | + +### Utils + +| Helper identifier | Description | Expected Input | +|--------------------------------|------------------------------------------------------------------------------------------|------------------------------------------------------------------------| +| `common.utils.fieldToEnvVar` | Build environment variable name given a field. | `dict "field" "my-password"` | +| `common.utils.secret.getvalue` | Print instructions to get a secret value. | `dict "secret" "secret-name" "field" "secret-value-field" "context" $` | +| `common.utils.getValueFromKey` | Gets a value from `.Values` object given its key path | `dict "key" "path.to.key" "context" $` | +| `common.utils.getKeyFromList` | Returns first `.Values` key with a defined value or first of the list if all non-defined | `dict "keys" (list "path.to.key1" "path.to.key2") "context" $` | + +### Validations + +| Helper identifier | Description | Expected Input | +|--------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.validations.values.single.empty` | Validate a value must not be empty. | `dict "valueKey" "path.to.value" "secret" "secret.name" "field" "my-password" "subchart" "subchart" "context" $` secret, field and subchart are optional. In case they are given, the helper will generate a how to get instruction. See [ValidateValue](#validatevalue) | +| `common.validations.values.multiple.empty` | Validate a multiple values must not be empty. It returns a shared error for all the values. | `dict "required" (list $validateValueConf00 $validateValueConf01) "context" $`. See [ValidateValue](#validatevalue) | +| `common.validations.values.mariadb.passwords` | This helper will ensure required password for MariaDB are not empty. It returns a shared error for all the values. | `dict "secret" "mariadb-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use mariadb chart and the helper. | +| `common.validations.values.mysql.passwords` | This helper will ensure required password for MySQL are not empty. It returns a shared error for all the values. | `dict "secret" "mysql-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use mysql chart and the helper. | +| `common.validations.values.postgresql.passwords` | This helper will ensure required password for PostgreSQL are not empty. It returns a shared error for all the values. | `dict "secret" "postgresql-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use postgresql chart and the helper. | +| `common.validations.values.redis.passwords` | This helper will ensure required password for Redis® are not empty. It returns a shared error for all the values. | `dict "secret" "redis-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use redis chart and the helper. | +| `common.validations.values.cassandra.passwords` | This helper will ensure required password for Cassandra are not empty. It returns a shared error for all the values. | `dict "secret" "cassandra-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use cassandra chart and the helper. | +| `common.validations.values.mongodb.passwords` | This helper will ensure required password for MongoDB® are not empty. It returns a shared error for all the values. | `dict "secret" "mongodb-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use mongodb chart and the helper. | + +### Warnings + +| Helper identifier | Description | Expected Input | +|------------------------------|----------------------------------|------------------------------------------------------------| +| `common.warnings.rollingTag` | Warning about using rolling tag. | `ImageRoot` see [ImageRoot](#imageroot) for the structure. | + +## Special input schemas + +### ImageRoot + +```yaml +registry: + type: string + description: Docker registry where the image is located + example: docker.io + +repository: + type: string + description: Repository and image name + example: bitnami/nginx + +tag: + type: string + description: image tag + example: 1.16.1-debian-10-r63 + +pullPolicy: + type: string + description: Specify a imagePullPolicy. Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + +pullSecrets: + type: array + items: + type: string + description: Optionally specify an array of imagePullSecrets (evaluated as templates). + +debug: + type: boolean + description: Set to true if you would like to see extra information on logs + example: false + +## An instance would be: +# registry: docker.io +# repository: bitnami/nginx +# tag: 1.16.1-debian-10-r63 +# pullPolicy: IfNotPresent +# debug: false +``` + +### Persistence + +```yaml +enabled: + type: boolean + description: Whether enable persistence. + example: true + +storageClass: + type: string + description: Ghost data Persistent Volume Storage Class, If set to "-", storageClassName: "" which disables dynamic provisioning. + example: "-" + +accessMode: + type: string + description: Access mode for the Persistent Volume Storage. + example: ReadWriteOnce + +size: + type: string + description: Size the Persistent Volume Storage. + example: 8Gi + +path: + type: string + description: Path to be persisted. + example: /bitnami + +## An instance would be: +# enabled: true +# storageClass: "-" +# accessMode: ReadWriteOnce +# size: 8Gi +# path: /bitnami +``` + +### ExistingSecret + +```yaml +name: + type: string + description: Name of the existing secret. + example: mySecret +keyMapping: + description: Mapping between the expected key name and the name of the key in the existing secret. + type: object + +## An instance would be: +# name: mySecret +# keyMapping: +# password: myPasswordKey +``` + +#### Example of use + +When we store sensitive data for a deployment in a secret, some times we want to give to users the possibility of using theirs existing secrets. + +```yaml +# templates/secret.yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "common.names.fullname" . }} + labels: + app: {{ include "common.names.fullname" . }} +type: Opaque +data: + password: {{ .Values.password | b64enc | quote }} + +# templates/dpl.yaml +--- +... + env: + - name: PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "common.secrets.name" (dict "existingSecret" .Values.existingSecret "context" $) }} + key: {{ include "common.secrets.key" (dict "existingSecret" .Values.existingSecret "key" "password") }} +... + +# values.yaml +--- +name: mySecret +keyMapping: + password: myPasswordKey +``` + +### ValidateValue + +#### NOTES.txt + +```console +{{- $validateValueConf00 := (dict "valueKey" "path.to.value00" "secret" "secretName" "field" "password-00") -}} +{{- $validateValueConf01 := (dict "valueKey" "path.to.value01" "secret" "secretName" "field" "password-01") -}} + +{{ include "common.validations.values.multiple.empty" (dict "required" (list $validateValueConf00 $validateValueConf01) "context" $) }} +``` + +If we force those values to be empty we will see some alerts + +```console +$ helm install test mychart --set path.to.value00="",path.to.value01="" + 'path.to.value00' must not be empty, please add '--set path.to.value00=$PASSWORD_00' to the command. To get the current value: + + export PASSWORD_00=$(kubectl get secret --namespace default secretName -o jsonpath="{.data.password-00}" | base64 -d) + + 'path.to.value01' must not be empty, please add '--set path.to.value01=$PASSWORD_01' to the command. To get the current value: + + export PASSWORD_01=$(kubectl get secret --namespace default secretName -o jsonpath="{.data.password-01}" | base64 -d) +``` + +## Upgrading + +### To 1.0.0 + +[On November 13, 2020, Helm v2 support was formally finished](https://github.com/helm/charts#status-of-the-project), this major version is the result of the required changes applied to the Helm Chart to be able to incorporate the different features added in Helm v3 and to be consistent with the Helm project itself regarding the Helm v2 EOL. + +**What changes were introduced in this major version?** + +- Previous versions of this Helm Chart use `apiVersion: v1` (installable by both Helm 2 and 3), this Helm Chart was updated to `apiVersion: v2` (installable by Helm 3 only). [Here](https://helm.sh/docs/topics/charts/#the-apiversion-field) you can find more information about the `apiVersion` field. +- Use `type: library`. [Here](https://v3.helm.sh/docs/faq/#library-chart-support) you can find more information. +- The different fields present in the *Chart.yaml* file has been ordered alphabetically in a homogeneous way for all the Bitnami Helm Charts + +**Considerations when upgrading to this version** + +- If you want to upgrade to this version from a previous one installed with Helm v3, you shouldn't face any issues +- If you want to upgrade to this version using Helm v2, this scenario is not supported as this version doesn't support Helm v2 anymore +- If you installed the previous version with Helm v2 and wants to upgrade to this version with Helm v3, please refer to the [official Helm documentation](https://helm.sh/docs/topics/v2_v3_migration/#migration-use-cases) about migrating from Helm v2 to v3 + +**Useful links** + +- https://docs.bitnami.com/tutorials/resolve-helm2-helm3-post-migration-issues/ +- https://helm.sh/docs/topics/v2_v3_migration/ +- https://helm.sh/blog/migrate-from-helm-v2-to-helm-v3/ + +## License + +Copyright © 2022 Bitnami + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_affinities.tpl b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_affinities.tpl new file mode 100644 index 00000000..2387be26 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_affinities.tpl @@ -0,0 +1,102 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Return a soft nodeAffinity definition +{{ include "common.affinities.nodes.soft" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.soft" -}} +preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . | quote }} + {{- end }} + weight: 1 +{{- end -}} + +{{/* +Return a hard nodeAffinity definition +{{ include "common.affinities.nodes.hard" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.hard" -}} +requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . | quote }} + {{- end }} +{{- end -}} + +{{/* +Return a nodeAffinity definition +{{ include "common.affinities.nodes" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.nodes.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.nodes.hard" . -}} + {{- end -}} +{{- end -}} + +{{/* +Return a soft podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.soft" (dict "component" "FOO" "extraMatchLabels" .Values.extraMatchLabels "context" $) -}} +*/}} +{{- define "common.affinities.pods.soft" -}} +{{- $component := default "" .component -}} +{{- $extraMatchLabels := default (dict) .extraMatchLabels -}} +preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" .context) | nindent 10 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := $extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + namespaces: + - {{ include "common.names.namespace" .context | quote }} + topologyKey: kubernetes.io/hostname + weight: 1 +{{- end -}} + +{{/* +Return a hard podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.hard" (dict "component" "FOO" "extraMatchLabels" .Values.extraMatchLabels "context" $) -}} +*/}} +{{- define "common.affinities.pods.hard" -}} +{{- $component := default "" .component -}} +{{- $extraMatchLabels := default (dict) .extraMatchLabels -}} +requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" .context) | nindent 8 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := $extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + namespaces: + - {{ include "common.names.namespace" .context | quote }} + topologyKey: kubernetes.io/hostname +{{- end -}} + +{{/* +Return a podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.pods" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.pods.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.pods.hard" . -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_capabilities.tpl b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_capabilities.tpl new file mode 100644 index 00000000..9d9b7600 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_capabilities.tpl @@ -0,0 +1,154 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Return the target Kubernetes version +*/}} +{{- define "common.capabilities.kubeVersion" -}} +{{- if .Values.global }} + {{- if .Values.global.kubeVersion }} + {{- .Values.global.kubeVersion -}} + {{- else }} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} + {{- end -}} +{{- else }} +{{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for poddisruptionbudget. +*/}} +{{- define "common.capabilities.policy.apiVersion" -}} +{{- if semverCompare "<1.21-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "policy/v1beta1" -}} +{{- else -}} +{{- print "policy/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for networkpolicy. +*/}} +{{- define "common.capabilities.networkPolicy.apiVersion" -}} +{{- if semverCompare "<1.7-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for cronjob. +*/}} +{{- define "common.capabilities.cronjob.apiVersion" -}} +{{- if semverCompare "<1.21-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "batch/v1beta1" -}} +{{- else -}} +{{- print "batch/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for deployment. +*/}} +{{- define "common.capabilities.deployment.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for statefulset. +*/}} +{{- define "common.capabilities.statefulset.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apps/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "common.capabilities.ingress.apiVersion" -}} +{{- if .Values.ingress -}} +{{- if .Values.ingress.apiVersion -}} +{{- .Values.ingress.apiVersion -}} +{{- else if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end }} +{{- else if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for RBAC resources. +*/}} +{{- define "common.capabilities.rbac.apiVersion" -}} +{{- if semverCompare "<1.17-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "rbac.authorization.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "rbac.authorization.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for CRDs. +*/}} +{{- define "common.capabilities.crd.apiVersion" -}} +{{- if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiextensions.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiextensions.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for APIService. +*/}} +{{- define "common.capabilities.apiService.apiVersion" -}} +{{- if semverCompare "<1.10-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiregistration.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiregistration.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for Horizontal Pod Autoscaler. +*/}} +{{- define "common.capabilities.hpa.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .context) -}} +{{- if .beta2 -}} +{{- print "autoscaling/v2beta2" -}} +{{- else -}} +{{- print "autoscaling/v2beta1" -}} +{{- end -}} +{{- else -}} +{{- print "autoscaling/v2" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if the used Helm version is 3.3+. +A way to check the used Helm version was not introduced until version 3.3.0 with .Capabilities.HelmVersion, which contains an additional "{}}" structure. +This check is introduced as a regexMatch instead of {{ if .Capabilities.HelmVersion }} because checking for the key HelmVersion in <3.3 results in a "interface not found" error. +**To be removed when the catalog's minimun Helm version is 3.3** +*/}} +{{- define "common.capabilities.supportsHelmVersion" -}} +{{- if regexMatch "{(v[0-9])*[^}]*}}$" (.Capabilities | toString ) }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_errors.tpl b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_errors.tpl new file mode 100644 index 00000000..a79cc2e3 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_errors.tpl @@ -0,0 +1,23 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Through error when upgrading using empty passwords values that must not be empty. + +Usage: +{{- $validationError00 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password00" "secret" "secretName" "field" "password-00") -}} +{{- $validationError01 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password01" "secret" "secretName" "field" "password-01") -}} +{{ include "common.errors.upgrade.passwords.empty" (dict "validationErrors" (list $validationError00 $validationError01) "context" $) }} + +Required password params: + - validationErrors - String - Required. List of validation strings to be return, if it is empty it won't throw error. + - context - Context - Required. Parent context. +*/}} +{{- define "common.errors.upgrade.passwords.empty" -}} + {{- $validationErrors := join "" .validationErrors -}} + {{- if and $validationErrors .context.Release.IsUpgrade -}} + {{- $errorString := "\nPASSWORDS ERROR: You must provide your current passwords when upgrading the release." -}} + {{- $errorString = print $errorString "\n Note that even after reinstallation, old credentials may be needed as they may be kept in persistent volume claims." -}} + {{- $errorString = print $errorString "\n Further information can be obtained at https://docs.bitnami.com/general/how-to/troubleshoot-helm-chart-issues/#credential-errors-while-upgrading-chart-releases" -}} + {{- $errorString = print $errorString "\n%s" -}} + {{- printf $errorString $validationErrors | fail -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_images.tpl b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_images.tpl new file mode 100644 index 00000000..42ffbc72 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_images.tpl @@ -0,0 +1,75 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper image name +{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }} +*/}} +{{- define "common.images.image" -}} +{{- $registryName := .imageRoot.registry -}} +{{- $repositoryName := .imageRoot.repository -}} +{{- $tag := .imageRoot.tag | toString -}} +{{- if .global }} + {{- if .global.imageRegistry }} + {{- $registryName = .global.imageRegistry -}} + {{- end -}} +{{- end -}} +{{- if $registryName }} +{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- else -}} +{{- printf "%s:%s" $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) +{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} +*/}} +{{- define "common.images.pullSecrets" -}} + {{- $pullSecrets := list }} + + {{- if .global }} + {{- range .global.imagePullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names evaluating values as templates +{{ include "common.images.renderPullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "context" $) }} +*/}} +{{- define "common.images.renderPullSecrets" -}} + {{- $pullSecrets := list }} + {{- $context := .context }} + + {{- if $context.Values.global }} + {{- range $context.Values.global.imagePullSecrets -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" $context)) -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" $context)) -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_ingress.tpl b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_ingress.tpl new file mode 100644 index 00000000..8caf73a6 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_ingress.tpl @@ -0,0 +1,68 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Generate backend entry that is compatible with all Kubernetes API versions. + +Usage: +{{ include "common.ingress.backend" (dict "serviceName" "backendName" "servicePort" "backendPort" "context" $) }} + +Params: + - serviceName - String. Name of an existing service backend + - servicePort - String/Int. Port name (or number) of the service. It will be translated to different yaml depending if it is a string or an integer. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.ingress.backend" -}} +{{- $apiVersion := (include "common.capabilities.ingress.apiVersion" .context) -}} +{{- if or (eq $apiVersion "extensions/v1beta1") (eq $apiVersion "networking.k8s.io/v1beta1") -}} +serviceName: {{ .serviceName }} +servicePort: {{ .servicePort }} +{{- else -}} +service: + name: {{ .serviceName }} + port: + {{- if typeIs "string" .servicePort }} + name: {{ .servicePort }} + {{- else if or (typeIs "int" .servicePort) (typeIs "float64" .servicePort) }} + number: {{ .servicePort | int }} + {{- end }} +{{- end -}} +{{- end -}} + +{{/* +Print "true" if the API pathType field is supported +Usage: +{{ include "common.ingress.supportsPathType" . }} +*/}} +{{- define "common.ingress.supportsPathType" -}} +{{- if (semverCompare "<1.18-0" (include "common.capabilities.kubeVersion" .)) -}} +{{- print "false" -}} +{{- else -}} +{{- print "true" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if the ingressClassname field is supported +Usage: +{{ include "common.ingress.supportsIngressClassname" . }} +*/}} +{{- define "common.ingress.supportsIngressClassname" -}} +{{- if semverCompare "<1.18-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "false" -}} +{{- else -}} +{{- print "true" -}} +{{- end -}} +{{- end -}} + +{{/* +Return true if cert-manager required annotations for TLS signed +certificates are set in the Ingress annotations +Ref: https://cert-manager.io/docs/usage/ingress/#supported-annotations +Usage: +{{ include "common.ingress.certManagerRequest" ( dict "annotations" .Values.path.to.the.ingress.annotations ) }} +*/}} +{{- define "common.ingress.certManagerRequest" -}} +{{ if or (hasKey .annotations "cert-manager.io/cluster-issuer") (hasKey .annotations "cert-manager.io/issuer") }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_labels.tpl b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_labels.tpl new file mode 100644 index 00000000..252066c7 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_labels.tpl @@ -0,0 +1,18 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Kubernetes standard labels +*/}} +{{- define "common.labels.standard" -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +helm.sh/chart: {{ include "common.names.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Labels to use on deploy.spec.selector.matchLabels and svc.spec.selector +*/}} +{{- define "common.labels.matchLabels" -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} diff --git a/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_names.tpl b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_names.tpl new file mode 100644 index 00000000..1bdac8b7 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_names.tpl @@ -0,0 +1,70 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "common.names.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "common.names.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "common.names.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a default fully qualified dependency name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +Usage: +{{ include "common.names.dependency.fullname" (dict "chartName" "dependency-chart-name" "chartValues" .Values.dependency-chart "context" $) }} +*/}} +{{- define "common.names.dependency.fullname" -}} +{{- if .chartValues.fullnameOverride -}} +{{- .chartValues.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .chartName .chartValues.nameOverride -}} +{{- if contains $name .context.Release.Name -}} +{{- .context.Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .context.Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts. +*/}} +{{- define "common.names.namespace" -}} +{{- if .Values.namespaceOverride -}} +{{- .Values.namespaceOverride -}} +{{- else -}} +{{- .Release.Namespace -}} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified app name adding the installation's namespace. +*/}} +{{- define "common.names.fullname.namespace" -}} +{{- printf "%s-%s" (include "common.names.fullname" .) (include "common.names.namespace" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_secrets.tpl b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_secrets.tpl new file mode 100644 index 00000000..a53fb44f --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_secrets.tpl @@ -0,0 +1,140 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Generate secret name. + +Usage: +{{ include "common.secrets.name" (dict "existingSecret" .Values.path.to.the.existingSecret "defaultNameSuffix" "mySuffix" "context" $) }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/master/bitnami/common#existingsecret + - defaultNameSuffix - String - Optional. It is used only if we have several secrets in the same deployment. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.secrets.name" -}} +{{- $name := (include "common.names.fullname" .context) -}} + +{{- if .defaultNameSuffix -}} +{{- $name = printf "%s-%s" $name .defaultNameSuffix | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- with .existingSecret -}} +{{- if not (typeIs "string" .) -}} +{{- with .name -}} +{{- $name = . -}} +{{- end -}} +{{- else -}} +{{- $name = . -}} +{{- end -}} +{{- end -}} + +{{- printf "%s" $name -}} +{{- end -}} + +{{/* +Generate secret key. + +Usage: +{{ include "common.secrets.key" (dict "existingSecret" .Values.path.to.the.existingSecret "key" "keyName") }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/master/bitnami/common#existingsecret + - key - String - Required. Name of the key in the secret. +*/}} +{{- define "common.secrets.key" -}} +{{- $key := .key -}} + +{{- if .existingSecret -}} + {{- if not (typeIs "string" .existingSecret) -}} + {{- if .existingSecret.keyMapping -}} + {{- $key = index .existingSecret.keyMapping $.key -}} + {{- end -}} + {{- end }} +{{- end -}} + +{{- printf "%s" $key -}} +{{- end -}} + +{{/* +Generate secret password or retrieve one if already created. + +Usage: +{{ include "common.secrets.passwords.manage" (dict "secret" "secret-name" "key" "keyName" "providedValues" (list "path.to.password1" "path.to.password2") "length" 10 "strong" false "chartName" "chartName" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - key - String - Required - Name of the key in the secret. + - providedValues - List - Required - The path to the validating value in the values.yaml, e.g: "mysql.password". Will pick first parameter with a defined value. + - length - int - Optional - Length of the generated random password. + - strong - Boolean - Optional - Whether to add symbols to the generated random password. + - chartName - String - Optional - Name of the chart used when said chart is deployed as a subchart. + - context - Context - Required - Parent context. + +The order in which this function returns a secret password: + 1. Already existing 'Secret' resource + (If a 'Secret' resource is found under the name provided to the 'secret' parameter to this function and that 'Secret' resource contains a key with the name passed as the 'key' parameter to this function then the value of this existing secret password will be returned) + 2. Password provided via the values.yaml + (If one of the keys passed to the 'providedValues' parameter to this function is a valid path to a key in the values.yaml and has a value, the value of the first key with a value will be returned) + 3. Randomly generated secret password + (A new random secret password with the length specified in the 'length' parameter will be generated and returned) + +*/}} +{{- define "common.secrets.passwords.manage" -}} + +{{- $password := "" }} +{{- $subchart := "" }} +{{- $chartName := default "" .chartName }} +{{- $passwordLength := default 10 .length }} +{{- $providedPasswordKey := include "common.utils.getKeyFromList" (dict "keys" .providedValues "context" $.context) }} +{{- $providedPasswordValue := include "common.utils.getValueFromKey" (dict "key" $providedPasswordKey "context" $.context) }} +{{- $secretData := (lookup "v1" "Secret" $.context.Release.Namespace .secret).data }} +{{- if $secretData }} + {{- if hasKey $secretData .key }} + {{- $password = index $secretData .key }} + {{- else }} + {{- printf "\nPASSWORDS ERROR: The secret \"%s\" does not contain the key \"%s\"\n" .secret .key | fail -}} + {{- end -}} +{{- else if $providedPasswordValue }} + {{- $password = $providedPasswordValue | toString | b64enc | quote }} +{{- else }} + + {{- if .context.Values.enabled }} + {{- $subchart = $chartName }} + {{- end -}} + + {{- $requiredPassword := dict "valueKey" $providedPasswordKey "secret" .secret "field" .key "subchart" $subchart "context" $.context -}} + {{- $requiredPasswordError := include "common.validations.values.single.empty" $requiredPassword -}} + {{- $passwordValidationErrors := list $requiredPasswordError -}} + {{- include "common.errors.upgrade.passwords.empty" (dict "validationErrors" $passwordValidationErrors "context" $.context) -}} + + {{- if .strong }} + {{- $subStr := list (lower (randAlpha 1)) (randNumeric 1) (upper (randAlpha 1)) | join "_" }} + {{- $password = randAscii $passwordLength }} + {{- $password = regexReplaceAllLiteral "\\W" $password "@" | substr 5 $passwordLength }} + {{- $password = printf "%s%s" $subStr $password | toString | shuffle | b64enc | quote }} + {{- else }} + {{- $password = randAlphaNum $passwordLength | b64enc | quote }} + {{- end }} +{{- end -}} +{{- printf "%s" $password -}} +{{- end -}} + +{{/* +Returns whether a previous generated secret already exists + +Usage: +{{ include "common.secrets.exists" (dict "secret" "secret-name" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - context - Context - Required - Parent context. +*/}} +{{- define "common.secrets.exists" -}} +{{- $secret := (lookup "v1" "Secret" $.context.Release.Namespace .secret) }} +{{- if $secret }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_storage.tpl b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_storage.tpl new file mode 100644 index 00000000..60e2a844 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_storage.tpl @@ -0,0 +1,23 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper Storage Class +{{ include "common.storage.class" ( dict "persistence" .Values.path.to.the.persistence "global" $) }} +*/}} +{{- define "common.storage.class" -}} + +{{- $storageClass := .persistence.storageClass -}} +{{- if .global -}} + {{- if .global.storageClass -}} + {{- $storageClass = .global.storageClass -}} + {{- end -}} +{{- end -}} + +{{- if $storageClass -}} + {{- if (eq "-" $storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" $storageClass -}} + {{- end -}} +{{- end -}} + +{{- end -}} diff --git a/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_tplvalues.tpl b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_tplvalues.tpl new file mode 100644 index 00000000..2db16685 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_tplvalues.tpl @@ -0,0 +1,13 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Renders a value that contains template. +Usage: +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $) }} +*/}} +{{- define "common.tplvalues.render" -}} + {{- if typeIs "string" .value }} + {{- tpl .value .context }} + {{- else }} + {{- tpl (.value | toYaml) .context }} + {{- end }} +{{- end -}} diff --git a/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_utils.tpl b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_utils.tpl new file mode 100644 index 00000000..8c22b2a3 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_utils.tpl @@ -0,0 +1,62 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Print instructions to get a secret value. +Usage: +{{ include "common.utils.secret.getvalue" (dict "secret" "secret-name" "field" "secret-value-field" "context" $) }} +*/}} +{{- define "common.utils.secret.getvalue" -}} +{{- $varname := include "common.utils.fieldToEnvVar" . -}} +export {{ $varname }}=$(kubectl get secret --namespace {{ .context.Release.Namespace | quote }} {{ .secret }} -o jsonpath="{.data.{{ .field }}}" | base64 -d) +{{- end -}} + +{{/* +Build env var name given a field +Usage: +{{ include "common.utils.fieldToEnvVar" dict "field" "my-password" }} +*/}} +{{- define "common.utils.fieldToEnvVar" -}} + {{- $fieldNameSplit := splitList "-" .field -}} + {{- $upperCaseFieldNameSplit := list -}} + + {{- range $fieldNameSplit -}} + {{- $upperCaseFieldNameSplit = append $upperCaseFieldNameSplit ( upper . ) -}} + {{- end -}} + + {{ join "_" $upperCaseFieldNameSplit }} +{{- end -}} + +{{/* +Gets a value from .Values given +Usage: +{{ include "common.utils.getValueFromKey" (dict "key" "path.to.key" "context" $) }} +*/}} +{{- define "common.utils.getValueFromKey" -}} +{{- $splitKey := splitList "." .key -}} +{{- $value := "" -}} +{{- $latestObj := $.context.Values -}} +{{- range $splitKey -}} + {{- if not $latestObj -}} + {{- printf "please review the entire path of '%s' exists in values" $.key | fail -}} + {{- end -}} + {{- $value = ( index $latestObj . ) -}} + {{- $latestObj = $value -}} +{{- end -}} +{{- printf "%v" (default "" $value) -}} +{{- end -}} + +{{/* +Returns first .Values key with a defined value or first of the list if all non-defined +Usage: +{{ include "common.utils.getKeyFromList" (dict "keys" (list "path.to.key1" "path.to.key2") "context" $) }} +*/}} +{{- define "common.utils.getKeyFromList" -}} +{{- $key := first .keys -}} +{{- $reverseKeys := reverse .keys }} +{{- range $reverseKeys }} + {{- $value := include "common.utils.getValueFromKey" (dict "key" . "context" $.context ) }} + {{- if $value -}} + {{- $key = . }} + {{- end -}} +{{- end -}} +{{- printf "%s" $key -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_warnings.tpl b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_warnings.tpl new file mode 100644 index 00000000..ae10fa41 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/_warnings.tpl @@ -0,0 +1,14 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Warning about using rolling tag. +Usage: +{{ include "common.warnings.rollingTag" .Values.path.to.the.imageRoot }} +*/}} +{{- define "common.warnings.rollingTag" -}} + +{{- if and (contains "bitnami/" .repository) (not (.tag | toString | regexFind "-r\\d+$|sha256:")) }} +WARNING: Rolling tag detected ({{ .repository }}:{{ .tag }}), please note that it is strongly recommended to avoid using rolling tags in a production environment. ++info https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/ +{{- end }} + +{{- end -}} diff --git a/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/validations/_cassandra.tpl b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/validations/_cassandra.tpl new file mode 100644 index 00000000..ded1ae3b --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/validations/_cassandra.tpl @@ -0,0 +1,72 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate Cassandra required passwords are not empty. + +Usage: +{{ include "common.validations.values.cassandra.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where Cassandra values are stored, e.g: "cassandra-passwords-secret" + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.cassandra.passwords" -}} + {{- $existingSecret := include "common.cassandra.values.existingSecret" . -}} + {{- $enabled := include "common.cassandra.values.enabled" . -}} + {{- $dbUserPrefix := include "common.cassandra.values.key.dbUser" . -}} + {{- $valueKeyPassword := printf "%s.password" $dbUserPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "cassandra-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.cassandra.values.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.cassandra.values.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.cassandra.dbUser.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.dbUser.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled cassandra. + +Usage: +{{ include "common.cassandra.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.cassandra.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.cassandra.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key dbUser + +Usage: +{{ include "common.cassandra.values.key.dbUser" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.cassandra.values.key.dbUser" -}} + {{- if .subchart -}} + cassandra.dbUser + {{- else -}} + dbUser + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/validations/_mariadb.tpl b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/validations/_mariadb.tpl new file mode 100644 index 00000000..b6906ff7 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/validations/_mariadb.tpl @@ -0,0 +1,103 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MariaDB required passwords are not empty. + +Usage: +{{ include "common.validations.values.mariadb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MariaDB values are stored, e.g: "mysql-passwords-secret" + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mariadb.passwords" -}} + {{- $existingSecret := include "common.mariadb.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mariadb.values.enabled" . -}} + {{- $architecture := include "common.mariadb.values.architecture" . -}} + {{- $authPrefix := include "common.mariadb.values.key.auth" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mariadb-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- if not (empty $valueUsername) -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mariadb-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replication") -}} + {{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mariadb-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mariadb.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mariadb. + +Usage: +{{ include "common.mariadb.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mariadb.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mariadb.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mariadb.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mariadb.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.key.auth" -}} + {{- if .subchart -}} + mariadb.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/validations/_mongodb.tpl b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/validations/_mongodb.tpl new file mode 100644 index 00000000..f820ec10 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/validations/_mongodb.tpl @@ -0,0 +1,108 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MongoDB® required passwords are not empty. + +Usage: +{{ include "common.validations.values.mongodb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MongoDB® values are stored, e.g: "mongodb-passwords-secret" + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mongodb.passwords" -}} + {{- $existingSecret := include "common.mongodb.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mongodb.values.enabled" . -}} + {{- $authPrefix := include "common.mongodb.values.key.auth" . -}} + {{- $architecture := include "common.mongodb.values.architecture" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyDatabase := printf "%s.database" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicaSetKey := printf "%s.replicaSetKey" $authPrefix -}} + {{- $valueKeyAuthEnabled := printf "%s.enabled" $authPrefix -}} + + {{- $authEnabled := include "common.utils.getValueFromKey" (dict "key" $valueKeyAuthEnabled "context" .context) -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") (eq $authEnabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mongodb-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- $valueDatabase := include "common.utils.getValueFromKey" (dict "key" $valueKeyDatabase "context" .context) }} + {{- if and $valueUsername $valueDatabase -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mongodb-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replicaset") -}} + {{- $requiredReplicaSetKey := dict "valueKey" $valueKeyReplicaSetKey "secret" .secret "field" "mongodb-replica-set-key" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicaSetKey -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mongodb.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDb is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mongodb.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mongodb. + +Usage: +{{ include "common.mongodb.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mongodb.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mongodb.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mongodb.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.key.auth" -}} + {{- if .subchart -}} + mongodb.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mongodb.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mongodb.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/validations/_mysql.tpl b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/validations/_mysql.tpl new file mode 100644 index 00000000..74472a06 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/validations/_mysql.tpl @@ -0,0 +1,103 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MySQL required passwords are not empty. + +Usage: +{{ include "common.validations.values.mysql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MySQL values are stored, e.g: "mysql-passwords-secret" + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mysql.passwords" -}} + {{- $existingSecret := include "common.mysql.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mysql.values.enabled" . -}} + {{- $architecture := include "common.mysql.values.architecture" . -}} + {{- $authPrefix := include "common.mysql.values.key.auth" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mysql-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- if not (empty $valueUsername) -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mysql-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replication") -}} + {{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mysql-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mysql.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mysql.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mysql. + +Usage: +{{ include "common.mysql.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mysql.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mysql.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mysql.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mysql.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mysql.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.key.auth" -}} + {{- if .subchart -}} + mysql.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/validations/_postgresql.tpl b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/validations/_postgresql.tpl new file mode 100644 index 00000000..164ec0d0 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/validations/_postgresql.tpl @@ -0,0 +1,129 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate PostgreSQL required passwords are not empty. + +Usage: +{{ include "common.validations.values.postgresql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where postgresql values are stored, e.g: "postgresql-passwords-secret" + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.postgresql.passwords" -}} + {{- $existingSecret := include "common.postgresql.values.existingSecret" . -}} + {{- $enabled := include "common.postgresql.values.enabled" . -}} + {{- $valueKeyPostgresqlPassword := include "common.postgresql.values.key.postgressPassword" . -}} + {{- $valueKeyPostgresqlReplicationEnabled := include "common.postgresql.values.key.replicationPassword" . -}} + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + {{- $requiredPostgresqlPassword := dict "valueKey" $valueKeyPostgresqlPassword "secret" .secret "field" "postgresql-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlPassword -}} + + {{- $enabledReplication := include "common.postgresql.values.enabled.replication" . -}} + {{- if (eq $enabledReplication "true") -}} + {{- $requiredPostgresqlReplicationPassword := dict "valueKey" $valueKeyPostgresqlReplicationEnabled "secret" .secret "field" "postgresql-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to decide whether evaluate global values. + +Usage: +{{ include "common.postgresql.values.use.global" (dict "key" "key-of-global" "context" $) }} +Params: + - key - String - Required. Field to be evaluated within global, e.g: "existingSecret" +*/}} +{{- define "common.postgresql.values.use.global" -}} + {{- if .context.Values.global -}} + {{- if .context.Values.global.postgresql -}} + {{- index .context.Values.global.postgresql .key | quote -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.postgresql.values.existingSecret" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.existingSecret" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "existingSecret" "context" .context) -}} + + {{- if .subchart -}} + {{- default (.context.Values.postgresql.existingSecret | quote) $globalValue -}} + {{- else -}} + {{- default (.context.Values.existingSecret | quote) $globalValue -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled postgresql. + +Usage: +{{ include "common.postgresql.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key postgressPassword. + +Usage: +{{ include "common.postgresql.values.key.postgressPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.postgressPassword" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "postgresqlUsername" "context" .context) -}} + + {{- if not $globalValue -}} + {{- if .subchart -}} + postgresql.postgresqlPassword + {{- else -}} + postgresqlPassword + {{- end -}} + {{- else -}} + global.postgresql.postgresqlPassword + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled.replication. + +Usage: +{{ include "common.postgresql.values.enabled.replication" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.enabled.replication" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.replication.enabled -}} + {{- else -}} + {{- printf "%v" .context.Values.replication.enabled -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key replication.password. + +Usage: +{{ include "common.postgresql.values.key.replicationPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.replicationPassword" -}} + {{- if .subchart -}} + postgresql.replication.password + {{- else -}} + replication.password + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/validations/_redis.tpl b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/validations/_redis.tpl new file mode 100644 index 00000000..dcccfc1a --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/validations/_redis.tpl @@ -0,0 +1,76 @@ + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate Redis® required passwords are not empty. + +Usage: +{{ include "common.validations.values.redis.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where redis values are stored, e.g: "redis-passwords-secret" + - subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.redis.passwords" -}} + {{- $enabled := include "common.redis.values.enabled" . -}} + {{- $valueKeyPrefix := include "common.redis.values.keys.prefix" . -}} + {{- $standarizedVersion := include "common.redis.values.standarized.version" . }} + + {{- $existingSecret := ternary (printf "%s%s" $valueKeyPrefix "auth.existingSecret") (printf "%s%s" $valueKeyPrefix "existingSecret") (eq $standarizedVersion "true") }} + {{- $existingSecretValue := include "common.utils.getValueFromKey" (dict "key" $existingSecret "context" .context) }} + + {{- $valueKeyRedisPassword := ternary (printf "%s%s" $valueKeyPrefix "auth.password") (printf "%s%s" $valueKeyPrefix "password") (eq $standarizedVersion "true") }} + {{- $valueKeyRedisUseAuth := ternary (printf "%s%s" $valueKeyPrefix "auth.enabled") (printf "%s%s" $valueKeyPrefix "usePassword") (eq $standarizedVersion "true") }} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $useAuth := include "common.utils.getValueFromKey" (dict "key" $valueKeyRedisUseAuth "context" .context) -}} + {{- if eq $useAuth "true" -}} + {{- $requiredRedisPassword := dict "valueKey" $valueKeyRedisPassword "secret" .secret "field" "redis-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRedisPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled redis. + +Usage: +{{ include "common.redis.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.redis.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.redis.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right prefix path for the values + +Usage: +{{ include "common.redis.values.key.prefix" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false +*/}} +{{- define "common.redis.values.keys.prefix" -}} + {{- if .subchart -}}redis.{{- else -}}{{- end -}} +{{- end -}} + +{{/* +Checks whether the redis chart's includes the standarizations (version >= 14) + +Usage: +{{ include "common.redis.values.standarized.version" (dict "context" $) }} +*/}} +{{- define "common.redis.values.standarized.version" -}} + + {{- $standarizedAuth := printf "%s%s" (include "common.redis.values.keys.prefix" .) "auth" -}} + {{- $standarizedAuthValues := include "common.utils.getValueFromKey" (dict "key" $standarizedAuth "context" .context) }} + + {{- if $standarizedAuthValues -}} + {{- true -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/validations/_validations.tpl b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/validations/_validations.tpl new file mode 100644 index 00000000..9a814cf4 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/charts/common/templates/validations/_validations.tpl @@ -0,0 +1,46 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate values must not be empty. + +Usage: +{{- $validateValueConf00 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-00") -}} +{{- $validateValueConf01 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-01") -}} +{{ include "common.validations.values.empty" (dict "required" (list $validateValueConf00 $validateValueConf01) "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" +*/}} +{{- define "common.validations.values.multiple.empty" -}} + {{- range .required -}} + {{- include "common.validations.values.single.empty" (dict "valueKey" .valueKey "secret" .secret "field" .field "context" $.context) -}} + {{- end -}} +{{- end -}} + +{{/* +Validate a value must not be empty. + +Usage: +{{ include "common.validations.value.empty" (dict "valueKey" "mariadb.password" "secret" "secretName" "field" "my-password" "subchart" "subchart" "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" + - subchart - String - Optional - Name of the subchart that the validated password is part of. +*/}} +{{- define "common.validations.values.single.empty" -}} + {{- $value := include "common.utils.getValueFromKey" (dict "key" .valueKey "context" .context) }} + {{- $subchart := ternary "" (printf "%s." .subchart) (empty .subchart) }} + + {{- if not $value -}} + {{- $varname := "my-value" -}} + {{- $getCurrentValue := "" -}} + {{- if and .secret .field -}} + {{- $varname = include "common.utils.fieldToEnvVar" . -}} + {{- $getCurrentValue = printf " To get the current value:\n\n %s\n" (include "common.utils.secret.getvalue" .) -}} + {{- end -}} + {{- printf "\n '%s' must not be empty, please add '--set %s%s=$%s' to the command.%s" .valueKey $subchart .valueKey $varname $getCurrentValue -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/temporal-ui/charts/common/values.yaml b/helm/airbyte-v1/charts/temporal-ui/charts/common/values.yaml new file mode 100644 index 00000000..f2df68e5 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/charts/common/values.yaml @@ -0,0 +1,5 @@ +## bitnami/common +## It is required by CI/CD tools and processes. +## @skip exampleValue +## +exampleValue: common-chart diff --git a/helm/airbyte-v1/charts/temporal-ui/templates/_helpers.tpl b/helm/airbyte-v1/charts/temporal-ui/templates/_helpers.tpl new file mode 100644 index 00000000..7be7bc1a --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/templates/_helpers.tpl @@ -0,0 +1,51 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "airbyte.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airbyte.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airbyte.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "airbyte.labels" -}} +helm.sh/chart: {{ include "airbyte.chart" . }} +{{ include "airbyte.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "airbyte.selectorLabels" -}} +app.kubernetes.io/name: {{ include "airbyte.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} \ No newline at end of file diff --git a/helm/airbyte-v1/charts/temporal-ui/templates/_images.tpl b/helm/airbyte-v1/charts/temporal-ui/templates/_images.tpl new file mode 100644 index 00000000..6c32ec44 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/templates/_images.tpl @@ -0,0 +1,17 @@ +{{- define "imageUrl" -}} +{{- $img := index . 0 -}} +{{- $root := index . 1 -}} + +{{- $reg := $root.Values.global.image.registry -}} +{{/* ensure the registry has a trailing slash, if set */}} +{{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} + +{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{- if (eq (typeOf $img) "string") -}} +{{- printf "%s%s" $reg (tpl $img $root) | quote -}} +{{- else -}} +{{- $tag := coalesce $img.tag $root.Values.global.image.tag $root.Chart.AppVersion -}} +{{- printf "%s%s:%s" $reg $img.repository $tag | quote -}} +{{- end -}} + +{{- end -}} \ No newline at end of file diff --git a/helm/airbyte-v1/charts/temporal-ui/templates/deployment.yaml b/helm/airbyte-v1/charts/temporal-ui/templates/deployment.yaml new file mode 100644 index 00000000..06a07ea5 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/templates/deployment.yaml @@ -0,0 +1,124 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.names.fullname" . }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "airbyte.selectorLabels" . | nindent 6 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 6 }} + {{- end }} + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 8 }} + {{- end }} + {{- if .Values.podLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ .Values.global.serviceAccountName }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.extraInitContainers }} + initContainers: + {{- toYaml .Values.extraInitContainers | nindent 6 }} + {{- end }} + containers: + - name: airbyte-temporal-ui + image: {{ include "imageUrl" (list .Values.image $) }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + # Values from env + {{- if or .Values.env_vars .Values.global.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + # Values from extraEnv for more compability(if you want to use external secret source or other stuff) + {{- if .Values.extraEnv }} + {{- toYaml .Values.extraEnv | nindent 8 }} + {{- end }} + + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /v1/health + port: http + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /v1/health + port: http + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + + ports: + - name: http + containerPort: 8080 + protocol: TCP + {{- if .Values.debug.enabled }} + - name: debug + containerPort: {{ .Values.debug.remoteDebugPort }} + protocol: TCP + {{- end}} + {{- if .Values.resources }} + resources: {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + {{- if .Values.containerSecurityContext }} + securityContext: {{- toYaml .Values.containerSecurityContext | nindent 10 }} + {{- end }} + volumeMounts: + {{- if .Values.extraVolumeMounts }} + {{ toYaml .Values.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.global.extraVolumeMounts }} + {{ toYaml .Values.global.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.extraContainers }} + {{ toYaml .Values.extraContainers | nindent 6 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 6 }} + {{- end }} + securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + volumes: + {{- if .Values.extraVolumes }} + {{ toYaml .Values.extraVolumes | nindent 6 }} + {{- end }} \ No newline at end of file diff --git a/helm/airbyte-v1/charts/temporal-ui/templates/service.yaml b/helm/airbyte-v1/charts/temporal-ui/templates/service.yaml new file mode 100644 index 00000000..93d2c3eb --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/templates/service.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{.Release.Name }}-airbyte-temporal-ui-svc + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "airbyte.selectorLabels" . | nindent 4 }} diff --git a/helm/airbyte-v1/charts/temporal-ui/values.yaml b/helm/airbyte-v1/charts/temporal-ui/values.yaml new file mode 100644 index 00000000..9430941d --- /dev/null +++ b/helm/airbyte-v1/charts/temporal-ui/values.yaml @@ -0,0 +1,194 @@ +global: + serviceAccountName: placeholderServiceAccount + deploymentMode: oss + deploymentStrategyType: RollingUpdate + configMapName: "" + secretName: "" + credVolumeOverride: "" + extraContainers: [] + ## extraSelectorLabels [object] - use to specify own additional selector labels for deployment + extraSelectorLabels: {} + ## extraLabels [object] - use to specify own additional labels for deployment + extraLabels: {} + database: + secretName: "" + secretValue: "" + + # Docker image config that will apply to all images. + image: + # Docker registry to pull platform images from, e.g. http://my-registry:8000/ + registry: "" + # Image tag to use for airbyte images. + # Does not include non-airbyte images such as busybox, temporal, minio, etc. + tag: "" + +enabled: false +## replicaCount Number of server replicas +replicaCount: 1 + +## image.repository The repository to use for the airbyte server image. +## image.pullPolicy the pull policy to use for the airbyte server image +## image.tag The airbyte server image tag. Defaults to the chart's AppVersion +image: + repository: temporalio/ui + pullPolicy: IfNotPresent + tag: 2.30.1 + +## podAnnotations [object] Add extra annotations to the server pod +## +podAnnotations: {} + +## containerSecurityContext Security context for the container +## Examples: +## containerSecurityContext: +## runAsNonRoot: true +## runAsUser: 1000 +## readOnlyRootFilesystem: true +containerSecurityContext: {} + +## Configure extra options for the server containers' liveness and readiness probes +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes +## livenessProbe.enabled Enable livenessProbe on the server +## livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe +## livenessProbe.periodSeconds Period seconds for livenessProbe +## livenessProbe.timeoutSeconds Timeout seconds for livenessProbe +## livenessProbe.failureThreshold Failure threshold for livenessProbe +## livenessProbe.successThreshold Success threshold for livenessProbe +## +livenessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + +## readinessProbe.enabled Enable readinessProbe on the server +## readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe +## readinessProbe.periodSeconds Period seconds for readinessProbe +## readinessProbe.timeoutSeconds Timeout seconds for readinessProbe +## readinessProbe.failureThreshold Failure threshold for readinessProbe +## readinessProbe.successThreshold Success threshold for readinessProbe +## +readinessProbe: + enabled: true + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + +## Server app resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## We usually recommend not to specify default resources and to leave this as a conscious +## choice for the user. This also increases chances charts run on environments with little +## resources, such as Minikube. If you do want to specify resources, uncomment the following +## lines, adjust them as necessary, and remove the curly braces after 'resources:'. +## resources.limits [object] The resources limits for the server container +## resources.requests [object] The requested resources for the server container +resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + +## service.type The service type to use for the API server +## service.port The service port to expose the API server on +service: + type: NodePort + port: 80 + annotations: {} + +## nodeSelector [object] Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## tolerations [array] Tolerations for server pod assignment. +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## affinity [object] Affinity and anti-affinity for server pod assignment. +## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## log.level The log level to log at +log: + level: "INFO" + +## extraVolumeMounts [array] Additional volumeMounts for server container(s). +## Examples (when using `containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumeMounts: +## - name: tmpdir +## mountPath: /tmp +## +extraVolumeMounts: [] + +## extraVolumes [array] Additional volumes for server pod(s). +## Examples (when using `containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumes: +## - name: tmpdir +## emptyDir: {} +## +extraVolumes: [] + +## extraContainer [array] Additional container for server pod(s) +## Example: +# extraContainers: +# - name: otel_collector +# image: somerepo/someimage:sometag +# args: [ +# "--important-args" +# ] +# ports: +# - containerPort: 443 +# volumeMounts: +# - name: volumeMountCool +# mountPath: /some/path +# readOnly: true +extraContainers: [] + +## extraInitContainers [array] Additional init containers for server pod(s) +## Example: +# extraInitContainers: +# - name: sleepy +# image: alpine +# command: ['sleep', '60'] + +extraInitContainers: [] + +## extraEnv [array] Supply extra env variables to main container using full notation +## Example: (With default env vars and values taken from generated config map) +# extraEnv: +# - name: AIRBYTE_VERSION +# valueFrom: +# configMapKeyRef: +# name: airbyte-env +# key: AIRBYTE_VERSION +## +extraEnv: [] +## secrets [object] Supply additional secrets to container +## Example: +## secrets: +## DATABASE_PASSWORD: strong-password +## DATABASE_USER: my-db-user +secrets: {} + +## env_vars [object] Supply extra env variables to main container using simplified notation +## Example: +# env_vars: +# AIRBYTE_VERSION: 0.40.4 +env_vars: + TEMPORAL_ADDRESS: ab-temporal:7233 + +debug: + enabled: false + remoteDebugPort: 5005 diff --git a/helm/airbyte-v1/charts/temporal/.gitignore b/helm/airbyte-v1/charts/temporal/.gitignore new file mode 100644 index 00000000..88e91e8a --- /dev/null +++ b/helm/airbyte-v1/charts/temporal/.gitignore @@ -0,0 +1,2 @@ +# Charts are downloaded at install time with `helm dep build`. +charts diff --git a/helm/airbyte-v1/charts/temporal/Chart.lock b/helm/airbyte-v1/charts/temporal/Chart.lock new file mode 100644 index 00000000..4deb970e --- /dev/null +++ b/helm/airbyte-v1/charts/temporal/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + version: 1.17.1 +digest: sha256:dacc73770a5640c011e067ff8840ddf89631fc19016c8d0a9e5ea160e7da8690 +generated: "2022-10-17T18:35:19.234598793Z" diff --git a/helm/airbyte-v1/charts/temporal/Chart.yaml b/helm/airbyte-v1/charts/temporal/Chart.yaml new file mode 100644 index 00000000..f0d84631 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +appVersion: 1.3.0 +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x +description: Helm chart to deploy airbyte-temporal +name: temporal +type: application +version: 1.3.0 diff --git a/helm/airbyte-v1/charts/temporal/README.md b/helm/airbyte-v1/charts/temporal/README.md new file mode 100644 index 00000000..036ddee1 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal/README.md @@ -0,0 +1,62 @@ +# temporal + +![Version: 0.67.17](https://img.shields.io/badge/Version-0.67.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) + +Helm chart to deploy airbyte-temporal + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://charts.bitnami.com/bitnami | common | 1.x.x | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| containerSecurityContext | object | `{}` | | +| enabled | bool | `true` | | +| env_vars | object | `{}` | | +| extraContainers | list | `[]` | | +| extraEnv | list | `[]` | | +| extraInitContainers | list | `[]` | | +| extraLabels | object | `{}` | | +| extraSelectorLabels | object | `{}` | | +| extraVolumeMounts | list | `[]` | | +| extraVolumes | list | `[]` | | +| global.configMapName | string | `""` | | +| global.database.secretName | string | `""` | | +| global.database.secretValue | string | `""` | | +| global.deploymentMode | string | `"oss"` | | +| global.extraContainers | list | `[]` | | +| global.extraLabels | object | `{}` | | +| global.extraSelectorLabels | object | `{}` | | +| global.secretName | string | `""` | | +| global.serviceAccountName | string | `"placeholderServiceAccount"` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"temporalio/auto-setup"` | | +| image.tag | string | `"1.23.0"` | | +| livenessProbe.enabled | bool | `true` | | +| livenessProbe.failureThreshold | int | `3` | | +| livenessProbe.initialDelaySeconds | int | `5` | | +| livenessProbe.periodSeconds | int | `30` | | +| livenessProbe.successThreshold | int | `1` | | +| livenessProbe.timeoutSeconds | int | `1` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| podLabels | object | `{}` | | +| readinessProbe.enabled | bool | `true` | | +| readinessProbe.failureThreshold | int | `3` | | +| readinessProbe.initialDelaySeconds | int | `5` | | +| readinessProbe.periodSeconds | int | `30` | | +| readinessProbe.successThreshold | int | `1` | | +| readinessProbe.timeoutSeconds | int | `1` | | +| replicaCount | int | `1` | | +| resources.limits | object | `{}` | | +| resources.requests | object | `{}` | | +| secrets | object | `{}` | | +| service.port | int | `7233` | | +| service.type | string | `"ClusterIP"` | | +| tolerations | list | `[]` | | + diff --git a/helm/airbyte-v1/charts/temporal/templates/_helpers.tpl b/helm/airbyte-v1/charts/temporal/templates/_helpers.tpl new file mode 100644 index 00000000..847ec7b4 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal/templates/_helpers.tpl @@ -0,0 +1,59 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "airbyte.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airbyte.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airbyte.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "airbyte.labels" -}} +helm.sh/chart: {{ include "airbyte.chart" . }} +{{ include "airbyte.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "airbyte.selectorLabels" -}} +app.kubernetes.io/name: {{ include "airbyte.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Define db secret +*/}} + +{{- define "database.secret.name" -}} +{{- printf "%s-postgresql" .Release.Name }} +{{- end }} diff --git a/helm/airbyte-v1/charts/temporal/templates/_images.tpl b/helm/airbyte-v1/charts/temporal/templates/_images.tpl new file mode 100644 index 00000000..6c32ec44 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal/templates/_images.tpl @@ -0,0 +1,17 @@ +{{- define "imageUrl" -}} +{{- $img := index . 0 -}} +{{- $root := index . 1 -}} + +{{- $reg := $root.Values.global.image.registry -}} +{{/* ensure the registry has a trailing slash, if set */}} +{{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} + +{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{- if (eq (typeOf $img) "string") -}} +{{- printf "%s%s" $reg (tpl $img $root) | quote -}} +{{- else -}} +{{- $tag := coalesce $img.tag $root.Values.global.image.tag $root.Chart.AppVersion -}} +{{- printf "%s%s:%s" $reg $img.repository $tag | quote -}} +{{- end -}} + +{{- end -}} \ No newline at end of file diff --git a/helm/airbyte-v1/charts/temporal/templates/configmap.yaml b/helm/airbyte-v1/charts/temporal/templates/configmap.yaml new file mode 100644 index 00000000..5f4b89fe --- /dev/null +++ b/helm/airbyte-v1/charts/temporal/templates/configmap.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.names.fullname" . }}-dynamicconfig + labels: + {{- include "airbyte.labels" . | nindent 4 }} +data: + "development.yaml": | + # when modifying, remember to update the docker-compose version of this file in temporal/dynamicconfig/development.yaml + frontend.enableClientVersionCheck: + - value: true + constraints: {} + history.persistenceMaxQPS: + - value: 3000 + constraints: {} + frontend.persistenceMaxQPS: + - value: 3000 + constraints: {} + frontend.historyMgrNumConns: + - value: 30 + constraints: {} + frontend.throttledLogRPS: + - value: 20 + constraints: {} + frontend.enableUpdateWorkflowExecution: + - value: true + frontend.enableUpdateWorkflowExecutionAsyncAccepted: + - value: true + history.historyMgrNumConns: + - value: 50 + constraints: {} + system.advancedVisibilityWritingMode: + - value: "off" + constraints: {} + history.defaultActivityRetryPolicy: + - value: + InitialIntervalInSeconds: 1 + MaximumIntervalCoefficient: 100.0 + BackoffCoefficient: 2.0 + MaximumAttempts: 0 + history.defaultWorkflowRetryPolicy: + - value: + InitialIntervalInSeconds: 1 + MaximumIntervalCoefficient: 100.0 + BackoffCoefficient: 2.0 + MaximumAttempts: 0 + # Limit for responses. This mostly impacts discovery jobs since they have the largest responses. + limit.blobSize.error: + - value: 15728640 # 15MB + constraints: {} + limit.blobSize.warn: + - value: 10485760 # 10MB + constraints: {} diff --git a/helm/airbyte-v1/charts/temporal/templates/deployment.yaml b/helm/airbyte-v1/charts/temporal/templates/deployment.yaml new file mode 100644 index 00000000..8db977aa --- /dev/null +++ b/helm/airbyte-v1/charts/temporal/templates/deployment.yaml @@ -0,0 +1,160 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.names.fullname" . }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- if .Values.extraLabels }} + {{ toYaml (mergeOverwrite .Values.extraLabels .Values.global.extraLabels) | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "airbyte.selectorLabels" . | nindent 6 }} + {{- if .Values.extraLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 8 }} + {{- end }} + {{- if .Values.podLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ .Values.global.serviceAccountName }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- end }} +{{- if .Values.extraInitContainers }} + initContainers: +{{- toYaml .Values.extraInitContainers | nindent 6 }} +{{- end }} + containers: + - name: airbyte-temporal + image: {{ include "imageUrl" (list .Values.image $) }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + {{- if eq .Values.global.deploymentMode "oss"}} + - name: AUTO_SETUP + value: "true" + - name: DB # The DB engine to use + value: "postgresql" + - name: DYNAMIC_CONFIG_FILE_PATH + value: "config/dynamicconfig/development.yaml" + + {{- include "airbyte.temporal.database.envs" . | nindent 10 }} + + {{- end }} + + {{- if eq .Values.global.database.type "external" }} + # Assume an external database requires SSL. + - name: POSTGRES_TLS_ENABLED + value: "true" + - name: POSTGRES_TLS_DISABLE_HOST_VERIFICATION + value: "true" + - name: SQL_TLS_ENABLED + value: "true" + - name: SQL_TLS_DISABLE_HOST_VERIFICATION + value: "true" + {{- end }} + + {{- if .Values.extraEnv }} + {{ .Values.extraEnv | toYaml | nindent 10 }} + {{- end }} + # Values from secret + {{- if .Values.secrets }} + {{- range $k, $v := .Values.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: temporal-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if or .Values.env_vars .Values.global.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + ports: + - containerPort: 7233 + {{- if .Values.containerSecurityContext }} + securityContext: {{- toYaml .Values.containerSecurityContext | nindent 10 }} + {{- end }} + volumeMounts: + - name: airbyte-temporal-dynamicconfig + mountPath: "/etc/temporal/config/dynamicconfig/" + {{- if .Values.extraVolumeMounts }} + {{ toYaml .Values.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.resources }} + resources: {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + exec: + command: + - /bin/sh + - -ec + - 'test $(ps -ef | grep -v grep | grep temporal-server | wc -l) -eq 1' + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + exec: + command: + - /bin/sh + - -ec + - 'test $(ps -ef | grep -v grep | grep temporal-server | wc -l) -eq 1' + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + {{- if .Values.extraContainers }} + {{ toYaml .Values.extraContainers | nindent 6 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 6 }} + {{- end }} + securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + volumes: + - name: airbyte-temporal-dynamicconfig + configMap: + name: {{ include "common.names.fullname" . }}-dynamicconfig + items: + - key: development.yaml + path: development.yaml + {{- if .Values.extraVolumes }} + {{ toYaml .Values.extraVolumes | nindent 6 }} + {{- end }} diff --git a/helm/airbyte-v1/charts/temporal/templates/secrets.yaml b/helm/airbyte-v1/charts/temporal/templates/secrets.yaml new file mode 100644 index 00000000..f77d1ff1 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal/templates/secrets.yaml @@ -0,0 +1,17 @@ +# Create secrets only for the local deployment +{{- if .Values.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: temporal-secrets + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" +type: Opaque +data: + {{- range $k, $v := mergeOverwrite .Values.secrets .Values.global.secrets }} + {{ $k }}: {{ if $v }}{{ $v | b64enc }} {{else}}""{{end}} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v1/charts/temporal/templates/service.yaml b/helm/airbyte-v1/charts/temporal/templates/service.yaml new file mode 100644 index 00000000..39f13632 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal/templates/service.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-temporal + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + protocol: TCP + targetPort: 7233 + selector: + {{- include "airbyte.selectorLabels" . | nindent 4 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 4 }} + {{- end }} diff --git a/helm/airbyte-v1/charts/temporal/values.yaml b/helm/airbyte-v1/charts/temporal/values.yaml new file mode 100644 index 00000000..64f90af2 --- /dev/null +++ b/helm/airbyte-v1/charts/temporal/values.yaml @@ -0,0 +1,193 @@ +global: + serviceAccountName: placeholderServiceAccount + deploymentMode: oss + configMapName: "" + secretName: "" + extraContainers: [] + ## extraSelectorLabels [object] - use to specify own additional selector labels for deployment + extraSelectorLabels: {} + ## extraLabels [object] - use to specify own additional labels for deployment + extraLabels: {} + database: + type: internal + secretName: "" + secretValue: "" + + # Docker image config that will apply to all images. + image: + # Docker registry to pull platform images from, e.g. http://my-registry:8000/ + registry: "" + # Image tag to use for airbyte images. + # Does not include non-airbyte images such as busybox, temporal, minio, etc. + tag: "" + +enabled: true +## temporal.replicaCount The number of temporal replicas to deploy +replicaCount: 1 + +## temporal.image.repository The temporal image repository to use +## temporal.image.pullPolicy The pull policy for the temporal image +## temporal.image.tag The temporal image tag to use +image: + repository: temporalio/auto-setup + pullPolicy: IfNotPresent + tag: "1.23.0" + +## temporal.service.type The Kubernetes Service Type +## temporal.service.port The temporal port and exposed kubernetes port +service: + type: ClusterIP + port: 7233 + +## temporal.podAnnotations [object] Add extra annotations to the temporal pod +## +podAnnotations: {} + +## temporal.podLabels [object] Add extra labels to the temporal pod +## +podLabels: {} + +## temporal.containerSecurityContext Security context for the container +## Examples: +## containerSecurityContext: +## runAsNonRoot: true +## runAsUser: 1000 +## readOnlyRootFilesystem: true +containerSecurityContext: {} + +## temporal.extraInitContainers Additional InitContainers to initialize the pod +## Examples (when using `temporal.containerSecurityContext.readOnlyRootFilesystem=true`): +## extraInitContainers: +## - name: config-loader +## image: temporalio/auto-setup:1.23 +## command: +## - /bin/sh +## - -c +## - >- +## find /etc/temporal/config/ -maxdepth 1 -mindepth 1 -exec cp -ar {} /config/ \; +## volumeMounts: +## - name: config +## mountPath: /config +extraInitContainers: [] + +## Configure extra options for the temporal containers' liveness and readiness probes +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes +## temporal.livenessProbe.enabled Enable livenessProbe on the temporal +## temporal.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe +## temporal.livenessProbe.periodSeconds Period seconds for livenessProbe +## temporal.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe +## temporal.livenessProbe.failureThreshold Failure threshold for livenessProbe +## temporal.livenessProbe.successThreshold Success threshold for livenessProbe +## +livenessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 30 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + +## temporal.readinessProbe.enabled Enable readinessProbe on the temporal +## temporal.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe +## temporal.readinessProbe.periodSeconds Period seconds for readinessProbe +## temporal.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe +## temporal.readinessProbe.failureThreshold Failure threshold for readinessProbe +## temporal.readinessProbe.successThreshold Success threshold for readinessProbe +## +readinessProbe: + enabled: true + initialDelaySeconds: 5 + periodSeconds: 30 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + +## temporal.nodeSelector [object] Node labels for temporal pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## temporal.tolerations [array] Tolerations for temporal pod assignment. +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## temporal.affinity [object] Affinity and anti-affinity for temporal pod assignment. +## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## temporal.extraEnv [array] Additional env vars for temporal pod(s). +## Example: +## +## extraEnv: +## - name: SAMPLE_ENV_VAR +## value: "key=sample-value" +extraEnv: [] + +## temporal.extraVolumeMounts [array] Additional volumeMounts for temporal container(s). +## Examples (when using `temporal.containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumeMounts: +## - name: tmpdir +## mountPath: /tmp +## - name: config +## mountPath: /etc/temporal/config +## +extraVolumeMounts: [] + +## temporal.extraVolumes [array] Additional volumes for temporal pod(s). +## Examples (when using `temporal.containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumes: +## - name: tmpdir +## emptyDir: {} +## - name: config +## emptyDir: {} +## +extraVolumes: [] + +## Temporal resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## We usually recommend not to specify default resources and to leave this as a conscious +## choice for the user. This also increases chances charts run on environments with little +## resources, such as Minikube. If you do want to specify resources, uncomment the following +## lines, adjust them as necessary, and remove the curly braces after 'resources:'. +## temporal.resources.limits [object] The resources limits for temporal pod(s) +## temporal.resources.requests [object] The requested resources for temporal pod(s) +resources: + ## Example: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + +extraContainers: [] + +## secrets [object] Supply additional secrets to container +## Example: +## secrets: +## DATABASE_PASSWORD: strong-password +## DATABASE_USER: my-db-user +secrets: {} + +## env_vars [object] Supply extra env variables to main container using simplified notation +## Example with default vars for webapp: +# env_vars: + +# # Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db +# DB: airbyte-db-svc +# DB_PORT: 5432 +# DATABASE_DB: airbyte +# POSTGRES_USER: airbyte +# POSTGRES_PWD: 12345 + +env_vars: {} + +## extraSelectorLabels [object] - use to specify own additional selector labels for deployment +extraSelectorLabels: {} +## extraLabels [object] - use to specify own additional labels for deployment +extraLabels: {} diff --git a/helm/airbyte-v1/charts/webapp/.gitignore b/helm/airbyte-v1/charts/webapp/.gitignore new file mode 100644 index 00000000..88e91e8a --- /dev/null +++ b/helm/airbyte-v1/charts/webapp/.gitignore @@ -0,0 +1,2 @@ +# Charts are downloaded at install time with `helm dep build`. +charts diff --git a/helm/airbyte-v1/charts/webapp/Chart.lock b/helm/airbyte-v1/charts/webapp/Chart.lock new file mode 100644 index 00000000..6d8b135d --- /dev/null +++ b/helm/airbyte-v1/charts/webapp/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + version: 1.17.1 +digest: sha256:dacc73770a5640c011e067ff8840ddf89631fc19016c8d0a9e5ea160e7da8690 +generated: "2022-10-17T18:35:23.164183749Z" diff --git a/helm/airbyte-v1/charts/webapp/Chart.yaml b/helm/airbyte-v1/charts/webapp/Chart.yaml new file mode 100644 index 00000000..7cd62e56 --- /dev/null +++ b/helm/airbyte-v1/charts/webapp/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +appVersion: 1.3.0 +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x +description: Helm chart to deploy airbyte-webapp +name: webapp +type: application +version: 1.3.0 diff --git a/helm/airbyte-v1/charts/webapp/README.md b/helm/airbyte-v1/charts/webapp/README.md new file mode 100644 index 00000000..8becc7c3 --- /dev/null +++ b/helm/airbyte-v1/charts/webapp/README.md @@ -0,0 +1,69 @@ +# webapp + +![Version: 0.67.17](https://img.shields.io/badge/Version-0.67.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) + +Helm chart to deploy airbyte-webapp + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://charts.bitnami.com/bitnami | common | 1.x.x | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| api.url | string | `"/api/v1/"` | | +| containerSecurityContext | object | `{}` | | +| enabled | bool | `true` | | +| env_vars | object | `{}` | | +| extraContainers | list | `[]` | | +| extraEnv | list | `[]` | | +| extraInitContainers | list | `[]` | | +| extraLabels | object | `{}` | | +| extraSelectorLabels | object | `{}` | | +| extraVolumeMounts | list | `[]` | | +| extraVolumes | list | `[]` | | +| fullstory.enabled | bool | `false` | | +| global.configMapName | string | `""` | | +| global.database.secretName | string | `""` | | +| global.database.secretValue | string | `""` | | +| global.deploymentMode | string | `"oss"` | | +| global.extraContainers | list | `[]` | | +| global.extraLabels | object | `{}` | | +| global.extraSelectorLabels | object | `{}` | | +| global.secretName | string | `""` | | +| global.serviceAccountName | string | `"placeholderServiceAccount"` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"airbyte/webapp"` | | +| ingress.annotations | object | `{}` | | +| ingress.className | string | `""` | | +| ingress.enabled | bool | `false` | | +| ingress.hosts | list | `[]` | | +| ingress.tls | list | `[]` | | +| livenessProbe.enabled | bool | `true` | | +| livenessProbe.failureThreshold | int | `3` | | +| livenessProbe.initialDelaySeconds | int | `30` | | +| livenessProbe.periodSeconds | int | `10` | | +| livenessProbe.successThreshold | int | `1` | | +| livenessProbe.timeoutSeconds | int | `1` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| podLabels | object | `{}` | | +| readinessProbe.enabled | bool | `true` | | +| readinessProbe.failureThreshold | int | `3` | | +| readinessProbe.initialDelaySeconds | int | `10` | | +| readinessProbe.periodSeconds | int | `10` | | +| readinessProbe.successThreshold | int | `1` | | +| readinessProbe.timeoutSeconds | int | `1` | | +| replicaCount | int | `1` | | +| resources.limits | object | `{}` | | +| resources.requests | object | `{}` | | +| secrets | object | `{}` | | +| service.annotations | object | `{}` | | +| service.port | int | `80` | | +| service.type | string | `"ClusterIP"` | | +| tolerations | list | `[]` | | + diff --git a/helm/airbyte-v1/charts/webapp/templates/_helpers.tpl b/helm/airbyte-v1/charts/webapp/templates/_helpers.tpl new file mode 100644 index 00000000..e3f0d38f --- /dev/null +++ b/helm/airbyte-v1/charts/webapp/templates/_helpers.tpl @@ -0,0 +1,60 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "airbyte.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airbyte.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airbyte.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "airbyte.labels" -}} +helm.sh/chart: {{ include "airbyte.chart" . }} +{{ include "airbyte.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "airbyte.selectorLabels" -}} +app.kubernetes.io/name: {{ include "airbyte.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* +Define db secret +*/}} + +{{- define "database.secret.name" -}} +{{- printf "%s-postgresql" .Release.Name }} +{{- end }} \ No newline at end of file diff --git a/helm/airbyte-v1/charts/webapp/templates/_images.tpl b/helm/airbyte-v1/charts/webapp/templates/_images.tpl new file mode 100644 index 00000000..6c32ec44 --- /dev/null +++ b/helm/airbyte-v1/charts/webapp/templates/_images.tpl @@ -0,0 +1,17 @@ +{{- define "imageUrl" -}} +{{- $img := index . 0 -}} +{{- $root := index . 1 -}} + +{{- $reg := $root.Values.global.image.registry -}} +{{/* ensure the registry has a trailing slash, if set */}} +{{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} + +{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{- if (eq (typeOf $img) "string") -}} +{{- printf "%s%s" $reg (tpl $img $root) | quote -}} +{{- else -}} +{{- $tag := coalesce $img.tag $root.Values.global.image.tag $root.Chart.AppVersion -}} +{{- printf "%s%s:%s" $reg $img.repository $tag | quote -}} +{{- end -}} + +{{- end -}} \ No newline at end of file diff --git a/helm/airbyte-v1/charts/webapp/templates/deployment.yaml b/helm/airbyte-v1/charts/webapp/templates/deployment.yaml new file mode 100644 index 00000000..9fc93770 --- /dev/null +++ b/helm/airbyte-v1/charts/webapp/templates/deployment.yaml @@ -0,0 +1,172 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.names.fullname" . }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- if .Values.extraLabels }} + {{ toYaml (mergeOverwrite .Values.extraLabels .Values.global.extraLabels) | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "airbyte.selectorLabels" . | nindent 6 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 6 }} + {{- end }} + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 8 }} + {{- end }} + {{- if .Values.podLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ .Values.global.serviceAccountName }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.extraInitContainers }} + initContainers: + {{- toYaml .Values.extraInitContainers | nindent 6 }} + {{- end }} + containers: + - name: airbyte-webapp-container + image: {{ include "imageUrl" (list .Values.image $) }} + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + env: + {{- if eq .Values.global.deploymentMode "oss" }} + - name: TRACKING_STRATEGY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TRACKING_STRATEGY + - name: AIRBYTE_SERVER_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_SERVER_HOST + - name: KEYCLOAK_INTERNAL_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: KEYCLOAK_INTERNAL_HOST + - name: CONNECTOR_BUILDER_API_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONNECTOR_BUILDER_API_HOST + - name: AIRBYTE_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_VERSION + - name: API_URL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: API_URL + - name: CONNECTOR_BUILDER_API_URL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONNECTOR_BUILDER_API_URL + {{- end }} + # Values from secret + {{- if .Values.secrets }} + {{- range $k, $v := .Values.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: webapp-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if or .Values.env_vars .Values.global.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + # Values from extraEnv for more compability(if you want to use external secret source or other stuff) + {{- if .Values.extraEnv }} + {{- toYaml .Values.extraEnv | nindent 8 }} + {{- end }} + + + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + tcpSocket: + port: http + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /index.html + port: http + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + ports: + - name: http + containerPort: 8080 + protocol: TCP + {{- if .Values.resources }} + resources: {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + {{- if .Values.containerSecurityContext }} + securityContext: {{- toYaml .Values.containerSecurityContext | nindent 10 }} + {{- end }} + volumeMounts: + {{- if .Values.extraVolumeMounts }} + {{ toYaml .Values.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.global.extraVolumeMounts }} + {{ toYaml .Values.global.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.extraContainers }} + {{ toYaml .Values.extraContainers | nindent 6 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 6 }} + {{- end }} + securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + volumes: + {{- if .Values.extraVolumes }} +{{ toYaml .Values.extraVolumes | nindent 6 }} + {{- end }} + {{- if .Values.global.extraVolumes }} +{{ toYaml .Values.global.extraVolumes | nindent 6 }} + {{- end }} diff --git a/helm/airbyte-v1/charts/webapp/templates/ingress.yaml b/helm/airbyte-v1/charts/webapp/templates/ingress.yaml new file mode 100644 index 00000000..5fef4e02 --- /dev/null +++ b/helm/airbyte-v1/charts/webapp/templates/ingress.yaml @@ -0,0 +1,62 @@ +--- +{{ if .Values.ingress.enabled -}} +{{- $fullName := include "airbyte.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $.Release.Name }}-airbyte-webapp-svc + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }}-svc + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v1/charts/webapp/templates/secrets.yaml b/helm/airbyte-v1/charts/webapp/templates/secrets.yaml new file mode 100644 index 00000000..8ac24955 --- /dev/null +++ b/helm/airbyte-v1/charts/webapp/templates/secrets.yaml @@ -0,0 +1,17 @@ +# Create secrets only for the local deployment +{{- if .Values.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: webapp-secrets + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" +type: Opaque +data: + {{- range $k, $v := mergeOverwrite .Values.secrets .Values.global.secrets }} + {{ $k }}: {{ if $v }}{{ $v | b64enc }} {{else}}""{{end}} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v1/charts/webapp/templates/service.yaml b/helm/airbyte-v1/charts/webapp/templates/service.yaml new file mode 100644 index 00000000..a4ccded3 --- /dev/null +++ b/helm/airbyte-v1/charts/webapp/templates/service.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-airbyte-webapp-svc + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + {{ if and (eq .Values.service.type "NodePort") (.Values.service.nodePort) }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + selector: + {{- include "airbyte.selectorLabels" . | nindent 4 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 4 }} + {{- end }} + {{- if .Values.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} diff --git a/helm/airbyte-v1/charts/webapp/values.yaml b/helm/airbyte-v1/charts/webapp/values.yaml new file mode 100644 index 00000000..aedb5076 --- /dev/null +++ b/helm/airbyte-v1/charts/webapp/values.yaml @@ -0,0 +1,261 @@ +global: + serviceAccountName: placeholderServiceAccount + deploymentMode: oss + configMapName: "" + secretName: "" + extraContainers: [] + ## extraSelectorLabels [object] - use to specify own additional selector labels for deployment + extraSelectorLabels: {} + ## extraLabels [object] - use to specify own additional labels for deployment + extraLabels: {} + database: + secretName: "" + secretValue: "" + + # Docker image config that will apply to all images. + image: + # Docker registry to pull platform images from, e.g. http://my-registry:8000/ + registry: "" + # Image tag to use for airbyte images. + # Does not include non-airbyte images such as busybox, temporal, minio, etc. + tag: "" + +enabled: true +## webapp.replicaCount Number of webapp replicas +replicaCount: 1 + +## webapp.image.repository The repository to use for the airbyte webapp image. +## webapp.image.pullPolicy the pull policy to use for the airbyte webapp image +## webapp.image.tag The airbyte webapp image tag. Defaults to the chart's AppVersion +image: + repository: airbyte/webapp + pullPolicy: IfNotPresent + +## webapp.podAnnotations [object] Add extra annotations to the webapp pod(s) +## +podAnnotations: {} + +## webapp.podLabels [object] Add extra labels to the webapp pod(s) +## +podLabels: {} + +## webapp.containerSecurityContext Security context for the container +## Examples: +## containerSecurityContext: +## runAsNonRoot: true +## runAsUser: 1000 +## readOnlyRootFilesystem: true +containerSecurityContext: {} + +## Configure extra options for the webapp containers' liveness and readiness probes +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes +## webapp.livenessProbe.enabled Enable livenessProbe on the webapp +## webapp.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe +## webapp.livenessProbe.periodSeconds Period seconds for livenessProbe +## webapp.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe +## webapp.livenessProbe.failureThreshold Failure threshold for livenessProbe +## webapp.livenessProbe.successThreshold Success threshold for livenessProbe +## +livenessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + +## webapp.readinessProbe.enabled Enable readinessProbe on the webapp +## webapp.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe +## webapp.readinessProbe.periodSeconds Period seconds for readinessProbe +## webapp.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe +## webapp.readinessProbe.failureThreshold Failure threshold for readinessProbe +## webapp.readinessProbe.successThreshold Success threshold for readinessProbe +## +readinessProbe: + enabled: true + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + +## webapp.service.type The service type to use for the webapp service +## webapp.service.port The service port to expose the webapp on +## webapp.service.annotations Annotations for the webapp service resource +service: + type: ClusterIP + port: 80 + annotations: {} + +## Web app resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## We usually recommend not to specify default resources and to leave this as a conscious +## choice for the user. This also increases chances charts run on environments with little +## resources, such as Minikube. If you do want to specify resources, uncomment the following +## lines, adjust them as necessary, and remove the curly braces after 'resources:'. +## webapp.resources.limits [object] The resources limits for the Web container +## webapp.resources.requests [object] The requested resources for the Web container +resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + +## webapp.nodeSelector [object] Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## webapp.tolerations [array] Tolerations for webapp pod assignment. +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## webapp.affinity [object] Affinity and anti-affinity for webapp pod assignment. +## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Configure the ingress resource that allows you to access the Airbyte installation. +## ref: http://kubernetes.io/docs/user-guide/ingress/ +## webapp.ingress.enabled Set to true to enable ingress record generation +## webapp.ingress.className Specifies ingressClassName for clusters >= 1.18+ +## webapp.ingress.annotations [object] Ingress annotations done as key:value pairs +## webapp.ingress.hosts The list of hostnames to be covered with this ingress record. +## webapp.ingress.tls [array] Custom ingress TLS configuration +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: [] + # - host: chart-example.local + # paths: + # - path: / + # pathType: ImplementationSpecific + + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +## webapp.api.url The webapp API url. +api: + url: /api/v1/ + +## webapp.fullstory.enabled Whether or not to enable fullstory +fullstory: + enabled: false + + +## webapp.extraVolumeMounts [array] Additional volumeMounts for webapp container(s). +## Examples (when using `webapp.containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumeMounts: +## - name: var-run +## mountPath: /var/run/ +## - name: var-cache-nginx +## mountPath: /var/cache/nginx +## - mountPath: /etc/nginx/conf.d +## name: nginx-conf-d +## +extraVolumeMounts: [] + +## webapp.extraVolumes [array] Additional volumes for webapp pod(s). +## Examples (when using `webapp.containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumes: +## - name: var-run +## emptyDir: {} +## - name: var-cache-nginx +## emptyDir: {} +## - name: nginx-conf-d +## emptyDir: {} +## +extraVolumes: [] + +## extraContainer [array] Additional container for server pod(s) +## Example: +# extraContainers: +# - name: otel_collector +# image: somerepo/someimage:sometag +# args: [ +# "--important-args" +# ] +# ports: +# - containerPort: 443 +# volumeMounts: +# - name: volumeMountCool +# mountPath: /some/path +# readOnly: true +extraContainers: [] + +## extraInitContainers [array] Additional init containers for server pod(s) +## Example: +# extraInitContainers: +# - name: sleepy +# image: alpine +# command: ['sleep', '60'] + +extraInitContainers: [] + +## extraEnv [array] Supply extra env variables to main container using full notation +## Example: (With default env vars and values taken from generated config map) +# extraEnv: +# - name: POSTGRES_USER +# valueFrom: +# secretKeyRef: +# name: airbyte-secrets +# key: DATABASE_USER +# - name: POSTGRES_PWD +# valueFrom: +# secretKeyRef: +# name: airbyte-secrets +# key: DATABASE_PASSWORD +# - name: DYNAMIC_CONFIG_FILE_PATH +# value: "config/dynamicconfig/development.yaml" +# - name: DB +# value: "postgresql" +# - name: DB_PORT +# valueFrom: +# configMapKeyRef: +# name: airbyte-env +# key: DATABASE_PORT +# - name: POSTGRES_SEEDS +# valueFrom: +# configMapKeyRef: +# name: airbyte-env +# key: DATABASE_HOST +## +extraEnv: [] +## secrets [object] Supply additional secrets to container +## Example: +## secrets: +## DATABASE_PASSWORD: strong-password +## DATABASE_USER: my-db-user +secrets: {} + +## env_vars [object] Supply extra env variables to main container using simplified notation +## Example with default vars for webapp: +# env_vars: +# AIRBYTE_VERSION: 0.40.4 + +# # Miscellaneous +# TRACKING_STRATEGY: segment +# API_URL: /api/v1/ +# AIRBYTE_SERVER_HOST: airbyte-server-svc:8001 + +env_vars: {} + + + + +## extraSelectorLabels [object] - use to specify own additional selector labels for deployment +extraSelectorLabels: {} +## extraLabels [object] - use to specify own additional labels for deployment +extraLabels: {} diff --git a/helm/airbyte-v1/charts/worker/.gitignore b/helm/airbyte-v1/charts/worker/.gitignore new file mode 100644 index 00000000..88e91e8a --- /dev/null +++ b/helm/airbyte-v1/charts/worker/.gitignore @@ -0,0 +1,2 @@ +# Charts are downloaded at install time with `helm dep build`. +charts diff --git a/helm/airbyte-v1/charts/worker/Chart.lock b/helm/airbyte-v1/charts/worker/Chart.lock new file mode 100644 index 00000000..7000cfe2 --- /dev/null +++ b/helm/airbyte-v1/charts/worker/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + version: 1.17.1 +digest: sha256:dacc73770a5640c011e067ff8840ddf89631fc19016c8d0a9e5ea160e7da8690 +generated: "2022-10-17T18:35:31.130386898Z" diff --git a/helm/airbyte-v1/charts/worker/Chart.yaml b/helm/airbyte-v1/charts/worker/Chart.yaml new file mode 100644 index 00000000..1625f982 --- /dev/null +++ b/helm/airbyte-v1/charts/worker/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +appVersion: 1.3.0 +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x +description: Helm chart to deploy airbyte-worker +name: worker +type: application +version: 1.3.0 diff --git a/helm/airbyte-v1/charts/worker/README.md b/helm/airbyte-v1/charts/worker/README.md new file mode 100644 index 00000000..a9d1b210 --- /dev/null +++ b/helm/airbyte-v1/charts/worker/README.md @@ -0,0 +1,72 @@ +# worker + +![Version: 0.67.17](https://img.shields.io/badge/Version-0.67.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) + +Helm chart to deploy airbyte-worker + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://charts.bitnami.com/bitnami | common | 1.x.x | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| containerSecurityContext | object | `{}` | | +| debug.enabled | bool | `false` | | +| debug.remoteDebugPort | int | `5005` | | +| enabled | bool | `true` | | +| env_vars.CONTAINER_ORCHESTRATOR_ENABLED | bool | `true` | | +| extraContainers | list | `[]` | | +| extraEnv | list | `[]` | | +| extraInitContainers | list | `[]` | | +| extraLabels | object | `{}` | | +| extraSelectorLabels | object | `{}` | | +| extraVolumeMounts | list | `[]` | | +| extraVolumes | list | `[]` | | +| global.credVolumeOverride | string | `""` | | +| global.database.secretName | string | `""` | | +| global.database.secretValue | string | `""` | | +| global.edition | string | `"community"` | | +| global.extraContainers | list | `[]` | | +| global.extraLabels | object | `{}` | | +| global.extraSelectorLabels | object | `{}` | | +| global.jobs.kube.annotations | object | `{}` | | +| global.jobs.kube.images.busybox | string | `""` | | +| global.jobs.kube.images.curl | string | `""` | | +| global.jobs.kube.labels | object | `{}` | | +| global.jobs.kube.main_container_image_pull_secret | string | `""` | | +| global.jobs.kube.nodeSelector | object | `{}` | | +| global.jobs.kube.tolerations | list | `[]` | | +| global.jobs.resources.limits | object | `{}` | | +| global.jobs.resources.requests | object | `{}` | | +| global.secretName | string | `""` | | +| global.serviceAccountName | string | `"airbyte-admin"` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"airbyte/worker"` | | +| livenessProbe.enabled | bool | `true` | | +| livenessProbe.failureThreshold | int | `3` | | +| livenessProbe.initialDelaySeconds | int | `50` | | +| livenessProbe.periodSeconds | int | `10` | | +| livenessProbe.successThreshold | int | `1` | | +| livenessProbe.timeoutSeconds | int | `1` | | +| log.level | string | `"INFO"` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| podLabels | object | `{}` | | +| readinessProbe.enabled | bool | `true` | | +| readinessProbe.failureThreshold | int | `3` | | +| readinessProbe.initialDelaySeconds | int | `10` | | +| readinessProbe.periodSeconds | int | `10` | | +| readinessProbe.successThreshold | int | `1` | | +| readinessProbe.timeoutSeconds | int | `1` | | +| replicaCount | int | `1` | | +| resources.limits | object | `{}` | | +| resources.requests | object | `{}` | | +| secrets | object | `{}` | | +| tolerations | list | `[]` | | +| workloadApi | object | `{}` | | + diff --git a/helm/airbyte-v1/charts/worker/templates/_helpers.tpl b/helm/airbyte-v1/charts/worker/templates/_helpers.tpl new file mode 100644 index 00000000..b8ac92f8 --- /dev/null +++ b/helm/airbyte-v1/charts/worker/templates/_helpers.tpl @@ -0,0 +1,242 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "airbyte.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airbyte.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airbyte.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "airbyte.labels" -}} +helm.sh/chart: {{ include "airbyte.chart" . }} +{{ include "airbyte.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "airbyte.selectorLabels" -}} +app.kubernetes.io/name: {{ include "airbyte.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Define db secret +*/}} + +{{- define "database.secret.name" -}} +{{- printf "%s-postgresql" .Release.Name }} +{{- end }} + +## DEFAULT HELM VALUES +# Secret Manager Defaults +{{/* +Define secret persistence +*/}} +{{- define "airbyte.secretPersistence" -}} +{{- if (((.Values.global).secretsManager).type) }} + {{- printf "%s" (snakecase .Values.global.secretsManager.type) }} +{{- else }} + {{- printf "" }} +{{- end }} +{{- end }} + +{{/* +Get secret store name or default +*/}} +{{- define "airbyte.secretStoreName" -}} +{{- $secretStoreName := . -}} +{{- if $secretStoreName -}} + {{- printf "%s" $secretStoreName -}} +{{- else -}} + {{- printf "airbyte-config-secrets" -}} +{{- end -}} +{{- end -}} + +{{/* +Get awsSecretManager access key id secret key or default +*/}} +{{- define "airbyte.awsSecretManagerAccessKeyIdSecretKey" -}} +{{- $awsSecretManagerAccessKeyIdSecretKey := . -}} +{{- if $awsSecretManagerAccessKeyIdSecretKey -}} + {{- printf "%s" $awsSecretManagerAccessKeyIdSecretKey -}} +{{- else -}} + {{- printf "aws-secret-manager-access-key-id" -}} +{{- end -}} +{{- end -}} + +{{/* +Get awsSecretManager secret access key secret key or default +*/}} +{{- define "airbyte.awsSecretManagerSecretAccessKeySecretKey" -}} +{{- $awsSecretManagerSecretAccessKeySecretKey := . -}} +{{- if $awsSecretManagerSecretAccessKeySecretKey -}} + {{- printf "%s" $awsSecretManagerSecretAccessKeySecretKey -}} +{{- else -}} + {{- printf "aws-secret-manager-secret-access-key" -}} +{{- end -}} +{{- end -}} + +{{/* +Get googleSecretManager credentials secret key or default +*/}} +{{- define "airbyte.googleSecretManagerCredentialsSecretKey" -}} +{{- $googleSecretManagerCredentialsSecretKey := . -}} +{{- if $googleSecretManagerCredentialsSecretKey -}} + {{- printf "%s" $googleSecretManagerCredentialsSecretKey -}} +{{- else -}} + {{- printf "google-secret-manager-credentials" -}} +{{- end -}} +{{- end -}} + +{{/* +Get vault auth token secret key or default +*/}} +{{- define "airbyte.vaultAuthTokenSecretKey" -}} +{{- $vaultAuthTokenSecretKey := . -}} +{{- if $vaultAuthTokenSecretKey -}} + {{- printf "%s" $vaultAuthTokenSecretKey -}} +{{- else -}} + {{- printf "vault-auth-token" -}} +{{- end -}} +{{- end -}} + + +# Storage Defaults +{{/* +Get storage type or default +*/}} +{{- define "airbyte.storageType" -}} +{{- $storageType := . -}} +{{- if $storageType -}} + {{- printf "%s" $storageType -}} +{{- else -}} + {{- printf "local" -}} +{{- end -}} +{{- end -}} + +{{/* +Get storage bucket log or default +*/}} +{{- define "airbyte.storageBucketLog" -}} +{{- $storageBucketLog := . -}} +{{- if $storageBucketLog -}} + {{- printf "%s" $storageBucketLog -}} +{{- else -}} + {{- printf "airbyte-storage" -}} +{{- end -}} +{{- end -}} + +{{/* +Get storage bucket state or default +*/}} +{{- define "airbyte.storageBucketState" -}} +{{- $storageBucketState := . -}} +{{- if $storageBucketState -}} + {{- printf "%s" $storageBucketState -}} +{{- else -}} + {{- printf "airbyte-storage" -}} +{{- end -}} +{{- end -}} + +{{/* +Get storage bucket workload output or default +*/}} +{{- define "airbyte.storageBucketWorkloadOutput" -}} +{{- $storageBucketWorkloadOutput := . -}} +{{- if $storageBucketWorkloadOutput -}} + {{- printf "%s" $storageBucketWorkloadOutput -}} +{{- else -}} + {{- printf "airbyte-storage" -}} +{{- end -}} +{{- end -}} + +{{/* +Get s3 access key id secret key or default +*/}} +{{- define "airbyte.s3AccessKeyIdSecretKey" -}} +{{- $s3AccessKeyIdSecretKey := . -}} +{{- if $s3AccessKeyIdSecretKey -}} + {{- printf "%s" $s3AccessKeyIdSecretKey -}} +{{- else -}} + {{- printf "s3-access-key-id" -}} +{{- end -}} +{{- end -}} + +{{/* +Get s3 secret access key secret key or default +*/}} +{{- define "airbyte.s3SecretAccessKeySecretKey" -}} +{{- $s3SecretAccessKeySecretKey := . -}} +{{- if $s3SecretAccessKeySecretKey -}} + {{- printf "%s" $s3SecretAccessKeySecretKey -}} +{{- else -}} + {{- printf "s3-secret-access-key" -}} +{{- end -}} +{{- end -}} + +{{/* +Get minio access key id secret key or default +*/}} +{{- define "airbyte.minioAccessKeyIdSecretKey" -}} +{{- $minioAccessKeyIdSecretKey := . -}} +{{- if $minioAccessKeyIdSecretKey -}} + {{- printf "%s" $minioAccessKeyIdSecretKey -}} +{{- else -}} + {{- printf "minio-access-key-id" -}} +{{- end -}} +{{- end -}} + +{{/* +Get minio secret access key secret key or default +*/}} +{{- define "airbyte.minioSecretAccessKeySecretKey" -}} +{{- $minioSecretAccessKeySecretKey := . -}} +{{- if $minioSecretAccessKeySecretKey -}} + {{- printf "%s" $minioSecretAccessKeySecretKey -}} +{{- else -}} + {{- printf "minio-secret-access-key" -}} +{{- end -}} +{{- end -}} + +{{/* +Get gcs credentials secret key or default +*/}} +{{- define "airbyte.gcsCredentialsSecretKey" -}} +{{- $gcsCredentialsSecretKey := . -}} +{{- if $gcsCredentialsSecretKey -}} + {{- printf "%s" $gcsCredentialsSecretKey -}} +{{- else -}} + {{- printf "gcs-credentials" -}} +{{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/worker/templates/_images.tpl b/helm/airbyte-v1/charts/worker/templates/_images.tpl new file mode 100644 index 00000000..6c32ec44 --- /dev/null +++ b/helm/airbyte-v1/charts/worker/templates/_images.tpl @@ -0,0 +1,17 @@ +{{- define "imageUrl" -}} +{{- $img := index . 0 -}} +{{- $root := index . 1 -}} + +{{- $reg := $root.Values.global.image.registry -}} +{{/* ensure the registry has a trailing slash, if set */}} +{{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} + +{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{- if (eq (typeOf $img) "string") -}} +{{- printf "%s%s" $reg (tpl $img $root) | quote -}} +{{- else -}} +{{- $tag := coalesce $img.tag $root.Values.global.image.tag $root.Chart.AppVersion -}} +{{- printf "%s%s:%s" $reg $img.repository $tag | quote -}} +{{- end -}} + +{{- end -}} \ No newline at end of file diff --git a/helm/airbyte-v1/charts/worker/templates/deployment.yaml b/helm/airbyte-v1/charts/worker/templates/deployment.yaml new file mode 100644 index 00000000..ad46daeb --- /dev/null +++ b/helm/airbyte-v1/charts/worker/templates/deployment.yaml @@ -0,0 +1,500 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.names.fullname" . }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- if .Values.extraLabels }} + {{ toYaml (mergeOverwrite .Values.extraLabels .Values.global.extraLabels) | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "airbyte.selectorLabels" . | nindent 6 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 6 }} + {{- end }} + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 8 }} + {{- end }} + {{- if .Values.podLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ .Values.global.serviceAccountName }} + automountServiceAccountToken: true + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.extraInitContainers }} + initContainers: + {{- toYaml .Values.extraInitContainers | nindent 6 }} + {{- end }} + containers: + - name: airbyte-worker-container + image: {{ include "imageUrl" (list .Values.image $) }} + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + env: + {{- if .Values.debug.enabled }} + - name: JAVA_TOOL_OPTIONS + value: "-Xdebug -agentlib:jdwp=transport=dt_socket,address=0.0.0.0:{{ .Values.debug.remoteDebugPort }},server=y,suspend=n" + {{- end}} + {{- if eq .Values.global.deploymentMode "oss" }} + - name: AIRBYTE_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_VERSION + - name: CONFIG_ROOT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONFIG_ROOT + {{- if eq (lower (default "" .Values.global.storage.type)) "gcs" }} + - name: CONTAINER_ORCHESTRATOR_SECRET_MOUNT_PATH + value: "/secrets/gcs-log-creds" + - name: CONTAINER_ORCHESTRATOR_SECRET_NAME + value: {{ include "airbyte.secretStoreName" .Values.global.storage.storageSecretName }} + {{- end }} + {{- include "airbyte.logging.envs" . | nindent 8 }} + - name: MICROMETER_METRICS_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MICROMETER_METRICS_ENABLED + - name: MICROMETER_METRICS_STATSD_FLAVOR + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MICROMETER_METRICS_STATSD_FLAVOR + - name: SEGMENT_WRITE_KEY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: SEGMENT_WRITE_KEY + - name: STATSD_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: STATSD_HOST + - name: STATSD_PORT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: STATSD_PORT + - name: TRACKING_STRATEGY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TRACKING_STRATEGY + - name: WORKSPACE_DOCKER_MOUNT + value: workspace + - name: WORKSPACE_ROOT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKSPACE_ROOT + - name: WEBAPP_URL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WEBAPP_URL + - name: TEMPORAL_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TEMPORAL_HOST + - name: TEMPORAL_WORKER_PORTS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TEMPORAL_WORKER_PORTS + - name: JOB_KUBE_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: JOB_KUBE_SERVICEACCOUNT + value: {{ .Values.global.serviceAccountName }} + {{- if $.Values.global.jobs.kube.annotations }} + - name: JOB_KUBE_ANNOTATIONS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_KUBE_ANNOTATIONS + {{- end }} + {{- if $.Values.global.jobs.kube.labels }} + - name: JOB_KUBE_LABELS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_KUBE_LABELS + {{- end }} + {{- if $.Values.global.jobs.kube.nodeSelector }} + - name: JOB_KUBE_NODE_SELECTORS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_KUBE_NODE_SELECTORS + {{- end }} + {{- if $.Values.global.jobs.kube.tolerations }} + - name: JOB_KUBE_TOLERATIONS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_KUBE_TOLERATIONS + {{- end }} + - name: JOB_KUBE_BUSYBOX_IMAGE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_KUBE_BUSYBOX_IMAGE + - name: JOB_KUBE_CURL_IMAGE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_KUBE_CURL_IMAGE + {{- if $.Values.global.jobs.kube.main_container_image_pull_secret }} + - name: JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET + {{- end }} + - name: JOB_MAIN_CONTAINER_CPU_REQUEST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_MAIN_CONTAINER_CPU_REQUEST + - name: JOB_MAIN_CONTAINER_CPU_LIMIT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_MAIN_CONTAINER_CPU_LIMIT + - name: JOB_MAIN_CONTAINER_MEMORY_REQUEST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_MAIN_CONTAINER_MEMORY_REQUEST + - name: JOB_MAIN_CONTAINER_MEMORY_LIMIT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_MAIN_CONTAINER_MEMORY_LIMIT + - name: INTERNAL_API_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: INTERNAL_API_HOST + - name: WORKLOAD_API_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKLOAD_API_HOST + - name: WORKLOAD_API_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: {{ index .Values "workloadApi" "bearerTokenSecretName" | default (printf "%s-airbyte-secrets" .Release.Name ) }} + key: {{ index .Values "workloadApi" "bearerTokenSecretKey" | default "WORKLOAD_API_BEARER_TOKEN" }} + - name: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + - name: JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + - name: METRIC_CLIENT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: METRIC_CLIENT + - name: OTEL_COLLECTOR_ENDPOINT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: OTEL_COLLECTOR_ENDPOINT + - name: ACTIVITY_MAX_ATTEMPT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: ACTIVITY_MAX_ATTEMPT + - name: ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS + - name: ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS + - name: WORKFLOW_FAILURE_RESTART_DELAY_SECONDS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKFLOW_FAILURE_RESTART_DELAY_SECONDS + - name: SHOULD_RUN_NOTIFY_WORKFLOWS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: SHOULD_RUN_NOTIFY_WORKFLOWS + - name: MICRONAUT_ENVIRONMENTS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKERS_MICRONAUT_ENVIRONMENTS + {{- if or (eq .Values.global.edition "pro") (eq .Values.global.edition "enterprise") }} + - name: AIRBYTE_API_AUTH_HEADER_NAME + value: "X-Airbyte-Auth" + - name: AIRBYTE_API_AUTH_HEADER_VALUE + value: "Internal worker" + {{- else if and (eq .Values.global.deploymentMode "oss") .Values.global.auth.enabled }} + # Self-Managed Enterprise and Community w/ auth enabled use the same auth header, just + # splitting into two separate blocks for readability. + - name: AIRBYTE_API_AUTH_HEADER_NAME + value: "X-Airbyte-Auth" + - name: AIRBYTE_API_AUTH_HEADER_VALUE + value: "Internal worker" + {{- end }} + # SECRETS MANAGER + - name: SECRET_PERSISTENCE + value: {{ include "airbyte.secretPersistence" . }} + # Values for awsSecretManager + {{- if eq ((((.Values.global).secretsManager).awsSecretManager).authenticationType) "credentials" }} + - name: AWS_SECRET_MANAGER_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }} + key: {{ include "airbyte.awsSecretManagerAccessKeyIdSecretKey" .Values.global.secretsManager.awsSecretManager.accessKeyIdSecretKey }} + - name: AWS_SECRET_MANAGER_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }} + key: {{ include "airbyte.awsSecretManagerSecretAccessKeySecretKey" .Values.global.secretsManager.awsSecretManager.secretAccessKeySecretKey }} + {{- end }} + {{- if ((((.Values.global).secretsManager).awsSecretManager).region) }} + - name: AWS_SECRET_MANAGER_REGION + value: {{ (((.Values.global).secretsManager).awsSecretManager).region }} + {{- end }} + + {{- if ((((.Values.global).secretsManager).awsSecretManager).tags) }} + - name: AWS_SECRET_MANAGER_SECRET_TAGS + value: {{ include "airbyte.tagsToString" .Values.global.secretsManager.awsSecretManager.tags }} + {{- end }} + {{- if ((((.Values.global).secretsManager).awsSecretManager).kms) }} + - name: AWS_KMS_KEY_ARN + value: {{ ((((.Values.global).secretsManager).awsSecretManager).kms) | default "" }} + {{- end }} + + # Values for Azure Key Vault + {{- if (((.Values.global).secretsManager).azureKeyVault) }} + + - name: AB_AZURE_KEY_VAULT_VAULT_URL + value: {{ (((.Values.global).secretsManager).azureKeyVault).vaultUrl }} + + - name: AB_AZURE_KEY_VAULT_TENANT_ID + value: {{ (((.Values.global).secretsManager).azureKeyVault).tenantId }} + + - name: AB_AZURE_KEY_VAULT_CLIENT_ID + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }} + key: {{ include "airbyte.azureKeyVaultClientIdSecretKey" .Values.global.secretsManager.azureKeyVault.clientIdSecretKey }} + - name: AB_AZURE_KEY_VAULT_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }} + key: {{ include "airbyte.azureKeyVaultClientSecretSecretKey" .Values.global.secretsManager.azureKeyVault.clientSecretSecretKey }} + + {{- end }} + + {{- if ((((.Values.global).secretsManager).azureKeyVault).tags) }} + - name: AB_AZURE_KEY_VAULT_TAGS + value: {{ include "airbyte.tagsToString" .Values.global.secretsManager.azureKeyVault.tags }} + {{- end }} + + # Values for googleSecretManager secrets + {{- if (((.Values.global).secretsManager).googleSecretManager) }} + - name: SECRET_STORE_GCP_PROJECT_ID + value: {{ .Values.global.secretsManager.googleSecretManager.projectId }} + - name: SECRET_STORE_GCP_CREDENTIALS + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }} + key: {{ include "airbyte.googleSecretManagerCredentialsSecretKey" .Values.global.secretsManager.googleSecretManager.credentialsSecretKey }} + {{- end }} + + # Values for vault secrets + {{- if (((.Values.global).secretsManager).vault) }} + - name: VAULT_ADDRESS + value: {{ (((.Values.global).secretsManager).vault).address }} + - name: VAULT_PREFIX + value: {{ (((.Values.global).secretsManager).vault).prefix }} + - name: VAULT_AUTH_TOKEN + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }} + key: {{ include "airbyte.vaultAuthTokenSecretKey" .Values.global.secretsManager.vault.authTokenSecretKey }} + {{- end }} + + # Storage + {{- include "airbyte.storage.envs" . | nindent 8 }} + + # Database + {{- include "airbyte.database.envs" . | nindent 8 }} + + {{- end }} + + # Values from secret + {{- if .Values.secrets }} + {{- range $k, $v := .Values.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: worker-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if or .Values.env_vars .Values.global.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + # Values from extraEnv for more compatibility (if you want to use external secret source or other stuff) + {{- if .Values.extraEnv }} + {{- toYaml .Values.extraEnv | nindent 8 }} + {{- end }} + + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: / + port: heartbeat + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: / + port: heartbeat + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + ports: + {{- if .Values.debug.enabled }} + - name: debug + containerPort: {{ .Values.debug.remoteDebugPort }} + protocol: TCP + {{- end}} + - name: heartbeat + containerPort: 9000 # for heartbeat server + - containerPort: 9001 # start temporal worker port pool + - containerPort: 9002 + - containerPort: 9003 + - containerPort: 9004 + - containerPort: 9005 + - containerPort: 9006 + - containerPort: 9007 + - containerPort: 9008 + - containerPort: 9009 + - containerPort: 9010 + - containerPort: 9011 + - containerPort: 9012 + - containerPort: 9013 + - containerPort: 9014 + - containerPort: 9015 + - containerPort: 9016 + - containerPort: 9017 + - containerPort: 9018 + - containerPort: 9019 + - containerPort: 9020 + - containerPort: 9021 + - containerPort: 9022 + - containerPort: 9023 + - containerPort: 9024 + - containerPort: 9025 + - containerPort: 9026 + - containerPort: 9027 + - containerPort: 9028 + - containerPort: 9029 + - containerPort: 9030 # end temporal worker port pool + {{- if .Values.resources }} + resources: {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + {{- if .Values.containerSecurityContext }} + securityContext: {{- toYaml .Values.containerSecurityContext | nindent 10 }} + {{- end }} + volumeMounts: + {{- if eq (lower .Values.global.storage.type) "local" }} + - name: airbyte-storage + mountPath: /storage + {{- end }} + {{- if and (eq .Values.global.deploymentMode "oss") (eq (lower (default "" .Values.global.storage.type)) "gcs") }} + - name: gcs-log-creds-volume + mountPath: /secrets/gcs-log-creds + readOnly: true + {{- end }} + {{- if .Values.extraVolumeMounts }} +{{- toYaml .Values.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.global.extraVolumeMounts }} +{{- toYaml .Values.global.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.extraContainers }} + {{ toYaml .Values.extraContainers | nindent 6 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 6 }} + {{- end }} + securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + volumes: + {{- if eq (lower .Values.global.storage.type) "local" }} + - name: airbyte-storage + persistentVolumeClaim: + claimName: airbyte-storage-pvc + {{- end }} + {{- if and (eq .Values.global.deploymentMode "oss") (eq (lower (default "" .Values.global.storage.type)) "gcs") }} + - name: gcs-log-creds-volume + secret: + secretName: {{ ternary (printf "%s-gcs-log-creds" ( .Release.Name )) .Values.global.storage.storageSecretName (not ((.Values.global.storage).storageSecretName)) }} + {{- end }} + {{- if .Values.extraVolumes }} +{{- toYaml .Values.extraVolumes | nindent 6 }} + {{- end }} + {{- if .Values.global.extraVolumes }} +{{- toYaml .Values.global.extraVolumes | nindent 6 }} + {{- end }} diff --git a/helm/airbyte-v1/charts/worker/templates/secrets.yaml b/helm/airbyte-v1/charts/worker/templates/secrets.yaml new file mode 100644 index 00000000..93bb0c63 --- /dev/null +++ b/helm/airbyte-v1/charts/worker/templates/secrets.yaml @@ -0,0 +1,17 @@ +# Create secrets only for the local deployment +{{- if .Values.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: worker-secrets + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" +type: Opaque +data: + {{- range $k, $v := mergeOverwrite .Values.secrets .Values.global.secrets }} + {{ $k }}: {{ if $v }}{{ $v | b64enc }} {{else}}""{{end}} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v1/charts/worker/values.yaml b/helm/airbyte-v1/charts/worker/values.yaml new file mode 100644 index 00000000..b05d279e --- /dev/null +++ b/helm/airbyte-v1/charts/worker/values.yaml @@ -0,0 +1,294 @@ +global: + serviceAccountName: &service-account-name "airbyte-admin" + edition: community + credVolumeOverride: "" + secretName: "" + extraContainers: [] + ## extraSelectorLabels [object] - use to specify own additional selector labels for deployment + extraSelectorLabels: {} + ## extraLabels [object] - use to specify own additional labels for deployment + extraLabels: {} + database: + secretName: "" + secretValue: "" + + storage: + type: minio + + # Docker image config that will apply to all images. + image: + # Docker registry to pull platform images from, e.g. http://my-registry:8000/ + registry: "" + # Image tag to use for airbyte images. + # Does not include non-airbyte images such as busybox, temporal, minio, etc. + tag: "" + + jobs: + ## Jobs resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + ## jobs.resources.limits [object] The resources limits for jobs + ## jobs.resources.requests [object] The requested resources for jobs + resources: + ## Example: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + + kube: + ## JOB_KUBE_ANNOTATIONS + ## pod annotations of the sync job and the default pod annotations fallback for others jobs + ## jobs.kube.annotations [object] key/value annotations applied to kube jobs + annotations: {} + + ## JOB_KUBE_LABELS + ## pod labels of the sync job and the default pod labels fallback for others jobs + ## jobs.kube.labels [object] key/value labels applied to kube jobs + labels: {} + + ## JOB_KUBE_NODE_SELECTORS + ## pod node selector of the sync job and the default pod node selector fallback for others jobs + ## jobs.kube.nodeSelector [object] key/value node selector applied to kube jobs + nodeSelector: {} + + ## JOB_KUBE_TOLERATIONS + ## jobs.kube.tolerations [array] Tolerations for jobs.kube pod assignment. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## any boolean values should be quoted to ensure the value is passed through as a string, eg: + ## - key: airbyte-server + ## operator: Equal + ## value: "true" + ## effect: NoSchedule + tolerations: [] + + ## JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET + ## image pull secret to use for job pod + ## jobs.kube.main_container_image_pull_secret [string] image pull secret to use for job pod + main_container_image_pull_secret: "" + + images: + ## JOB_KUBE_BUSYBOX_IMAGE + ## busybox image used by the job pod + ## jobs.kube.images.busybox [string] busybox image used by the job pod + busybox: "" + ## JOB_KUBE_CURL_IMAGE + ## curl image used by the job pod + ## jobs.kube.images.curl [string] curl image used by the job pod + curl: "" + +enabled: true +## worker.replicaCount Number of worker replicas +replicaCount: 1 + +## worker.image.repository The repository to use for the airbyte worker image. +## worker.image.pullPolicy the pull policy to use for the airbyte worker image +## worker.image.tag The airbyte worker image tag. Defaults to the chart's AppVersion +image: + repository: airbyte/worker + pullPolicy: IfNotPresent + +## worker.podAnnotations [object] Add extra annotations to the worker pod(s) +## +podAnnotations: {} + +## worker.podLabels [object] Add extra labels to the worker pod(s) +## +podLabels: {} + +## worker.containerSecurityContext Security context for the container +## Examples: +## containerSecurityContext: +## runAsNonRoot: true +## runAsUser: 1000 +## readOnlyRootFilesystem: true +containerSecurityContext: {} + +## Configure extra options for the worker containers' liveness and readiness probes +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes +## worker.livenessProbe.enabled Enable livenessProbe on the worker +## worker.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe +## worker.livenessProbe.periodSeconds Period seconds for livenessProbe +## worker.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe +## worker.livenessProbe.failureThreshold Failure threshold for livenessProbe +## worker.livenessProbe.successThreshold Success threshold for livenessProbe +## +livenessProbe: + enabled: true + initialDelaySeconds: 50 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + +## worker.readinessProbe.enabled Enable readinessProbe on the worker +## worker.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe +## worker.readinessProbe.periodSeconds Period seconds for readinessProbe +## worker.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe +## worker.readinessProbe.failureThreshold Failure threshold for readinessProbe +## worker.readinessProbe.successThreshold Success threshold for readinessProbe +## +readinessProbe: + enabled: true + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + +## worker resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## We usually recommend not to specify default resources and to leave this as a conscious +## choice for the user. This also increases chances charts run on environments with little +## resources, such as Minikube. If you do want to specify resources, uncomment the following +## lines, adjust them as necessary, and remove the curly braces after 'resources:'. +## worker.resources.limits [object] The resources limits for the worker container +## worker.resources.requests [object] The requested resources for the worker container +resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + +## worker.nodeSelector [object] Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## worker.tolerations [array] Tolerations for worker pod assignment. +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## worker.affinity [object] Affinity and anti-affinity for worker pod assignment. +## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## worker.log.level The log level to log at. +log: + level: "INFO" + + + +## worker.extraVolumeMounts [array] Additional volumeMounts for worker container(s). +## Examples (when using `worker.containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumeMounts: +## - name: tmpdir +## mountPath: /tmp +## +extraVolumeMounts: [] + +## worker.extraVolumes [array] Additional volumes for worker pod(s). +## Examples (when using `worker.containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumes: +## - name: tmpdir +## emptyDir: {} +## +extraVolumes: [] + +## extraContainer [array] Additional container for server pod(s) +## Example: +# extraContainers: +# - name: otel_collector +# image: somerepo/someimage:sometag +# args: [ +# "--important-args" +# ] +# ports: +# - containerPort: 443 +# volumeMounts: +# - name: volumeMountCool +# mountPath: /some/path +# readOnly: true +extraContainers: [] + +## extraInitContainers [array] Additional init containers for server pod(s) +## Example: +# extraInitContainers: +# - name: sleepy +# image: alpine +# command: ['sleep', '60'] + +extraInitContainers: [] + +## extraEnv [array] Supply extra env variables to main container using full notation +## Example: (With default env vars and values taken from generated config map) +# extraEnv: +# - name: AIRBYTE_VERSION +# valueFrom: +# configMapKeyRef: +# name: airbyte-env +# key: AIRBYTE_VERSION +# - name: API_URL +# valueFrom: +# configMapKeyRef: +# name: airbyte-env +# key: API_URL +# - name: TRACKING_STRATEGY +# valueFrom: +# configMapKeyRef: +# name: airbyte-env +# key: TRACKING_STRATEGY +# - name: FULLSTORY +# valueFrom: +# configMapKeyRef: +# name: airbyte-env +# key: FULLSTORY +# - name: INTERNAL_API_HOST +# valueFrom: +# configMapKeyRef: +# name: airbyte-env +# key: INTERNAL_API_HOST +## +extraEnv: [] +## secrets [object] Supply additional secrets to container +## Example: +## secrets: +## DATABASE_PASSWORD: strong-password +## DATABASE_USER: my-db-user +secrets: {} + +## env_vars [object] Supply extra env variables to main container using simplified notation +## Example with default values for worker: +# env_vars: +# AIRBYTE_VERSION: 0.40.4 + +# # Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db +# DATABASE_HOST: airbyte-db-svc +# DATABASE_PORT: 5432 +# DATABASE_DB: airbyte +# # translate manually DATABASE_URL:jdbc:postgresql://${DATABASE_HOST}:${DATABASE_PORT/${DATABASE_DB} +# DATABASE_URL: jdbc:postgresql://airbyte-db-svc:5432/airbyte +# JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION: 0.29.15.001 +# CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION: 0.35.15.001 +env_vars: + CONTAINER_ORCHESTRATOR_ENABLED: true + +## extraSelectorLabels [object] - use to specify own additional selector labels for deployment +extraSelectorLabels: {} +## extraLabels [object] - use to specify own additional labels for deployment +extraLabels: {} + +## workloadApi: +## bearerTokenSecretName: workload-api-bearer-token-name kubernetes secret name where bearer token is stored +## bearerTokenSecretKey: workload-api-bearer-token-key kubernetes secret key where bearer token is stored +workloadApi: {} + +debug: + enabled: false + remoteDebugPort: 5005 diff --git a/helm/airbyte-v1/charts/workload-api-server/.gitignore b/helm/airbyte-v1/charts/workload-api-server/.gitignore new file mode 100644 index 00000000..88e91e8a --- /dev/null +++ b/helm/airbyte-v1/charts/workload-api-server/.gitignore @@ -0,0 +1,2 @@ +# Charts are downloaded at install time with `helm dep build`. +charts diff --git a/helm/airbyte-v1/charts/workload-api-server/Chart.lock b/helm/airbyte-v1/charts/workload-api-server/Chart.lock new file mode 100644 index 00000000..d174ec61 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-api-server/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + version: 1.17.1 +digest: sha256:dacc73770a5640c011e067ff8840ddf89631fc19016c8d0a9e5ea160e7da8690 +generated: "2023-11-02T09:27:52.217541-07:00" diff --git a/helm/airbyte-v1/charts/workload-api-server/Chart.yaml b/helm/airbyte-v1/charts/workload-api-server/Chart.yaml new file mode 100644 index 00000000..b7dc7d1f --- /dev/null +++ b/helm/airbyte-v1/charts/workload-api-server/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +appVersion: 1.3.0 +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x +description: Helm chart to deploy the workload-api service +name: workload-api-server +type: application +version: 1.3.0 diff --git a/helm/airbyte-v1/charts/workload-api-server/README.md b/helm/airbyte-v1/charts/workload-api-server/README.md new file mode 100644 index 00000000..7c1e72e6 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-api-server/README.md @@ -0,0 +1,85 @@ +# workload-api-server + +![Version: 0.67.17](https://img.shields.io/badge/Version-0.67.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) + +Helm chart to deploy the workload-api service + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://charts.bitnami.com/bitnami | common | 1.x.x | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| containerSecurityContext | object | `{}` | | +| debug.enabled | bool | `false` | | +| debug.remoteDebugPort | int | `5005` | | +| enabled | bool | `false` | | +| env_vars | object | `{}` | | +| extraContainers | list | `[]` | | +| extraEnv | list | `[]` | | +| extraInitContainers | list | `[]` | | +| extraLabels | object | `{}` | | +| extraSelectorLabels | object | `{}` | | +| extraVolumeMounts | list | `[]` | | +| extraVolumes | list | `[]` | | +| global.configMapName | string | `""` | | +| global.credVolumeOverride | string | `""` | | +| global.database.secretName | string | `""` | | +| global.database.secretValue | string | `""` | | +| global.deploymentMode | string | `"oss"` | | +| global.extraContainers | list | `[]` | | +| global.extraLabels | object | `{}` | | +| global.extraSelectorLabels | object | `{}` | | +| global.logs.accessKey.existingSecret | string | `""` | | +| global.logs.accessKey.existingSecretKey | string | `""` | | +| global.logs.accessKey.password | string | `"minio"` | | +| global.logs.gcs.bucket | string | `""` | | +| global.logs.gcs.credentials | string | `""` | | +| global.logs.gcs.credentialsJson | string | `""` | | +| global.logs.minio.enabled | bool | `true` | | +| global.logs.s3.bucket | string | `"airbyte-dev-logs"` | | +| global.logs.s3.bucketRegion | string | `""` | | +| global.logs.s3.enabled | bool | `false` | | +| global.logs.secretKey.existingSecret | string | `""` | | +| global.logs.secretKey.existingSecretKey | string | `""` | | +| global.logs.secretKey.password | string | `"minio123"` | | +| global.secretName | string | `""` | | +| global.serviceAccountName | string | `"placeholderServiceAccount"` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"airbyte/workload-api-server"` | | +| ingress.annotations | object | `{}` | | +| ingress.className | string | `""` | | +| ingress.enabled | bool | `false` | | +| ingress.hosts | list | `[]` | | +| ingress.tls | list | `[]` | | +| livenessProbe.enabled | bool | `true` | | +| livenessProbe.failureThreshold | int | `3` | | +| livenessProbe.initialDelaySeconds | int | `60` | | +| livenessProbe.periodSeconds | int | `10` | | +| livenessProbe.successThreshold | int | `1` | | +| livenessProbe.timeoutSeconds | int | `1` | | +| log.level | string | `"INFO"` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| podLabels | object | `{}` | | +| readinessProbe.enabled | bool | `true` | | +| readinessProbe.failureThreshold | int | `3` | | +| readinessProbe.initialDelaySeconds | int | `30` | | +| readinessProbe.periodSeconds | int | `10` | | +| readinessProbe.successThreshold | int | `1` | | +| readinessProbe.timeoutSeconds | int | `1` | | +| replicaCount | int | `1` | | +| resources.limits | object | `{}` | | +| resources.requests | object | `{}` | | +| secrets | object | `{}` | | +| service.annotations | object | `{}` | | +| service.port | int | `8007` | | +| service.type | string | `"ClusterIP"` | | +| tolerations | list | `[]` | | +| workloadApi | object | `{}` | | + diff --git a/helm/airbyte-v1/charts/workload-api-server/templates/_helpers.tpl b/helm/airbyte-v1/charts/workload-api-server/templates/_helpers.tpl new file mode 100644 index 00000000..847ec7b4 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-api-server/templates/_helpers.tpl @@ -0,0 +1,59 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "airbyte.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airbyte.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airbyte.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "airbyte.labels" -}} +helm.sh/chart: {{ include "airbyte.chart" . }} +{{ include "airbyte.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "airbyte.selectorLabels" -}} +app.kubernetes.io/name: {{ include "airbyte.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Define db secret +*/}} + +{{- define "database.secret.name" -}} +{{- printf "%s-postgresql" .Release.Name }} +{{- end }} diff --git a/helm/airbyte-v1/charts/workload-api-server/templates/_images.tpl b/helm/airbyte-v1/charts/workload-api-server/templates/_images.tpl new file mode 100644 index 00000000..6c32ec44 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-api-server/templates/_images.tpl @@ -0,0 +1,17 @@ +{{- define "imageUrl" -}} +{{- $img := index . 0 -}} +{{- $root := index . 1 -}} + +{{- $reg := $root.Values.global.image.registry -}} +{{/* ensure the registry has a trailing slash, if set */}} +{{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} + +{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{- if (eq (typeOf $img) "string") -}} +{{- printf "%s%s" $reg (tpl $img $root) | quote -}} +{{- else -}} +{{- $tag := coalesce $img.tag $root.Values.global.image.tag $root.Chart.AppVersion -}} +{{- printf "%s%s:%s" $reg $img.repository $tag | quote -}} +{{- end -}} + +{{- end -}} \ No newline at end of file diff --git a/helm/airbyte-v1/charts/workload-api-server/templates/deployment.yaml b/helm/airbyte-v1/charts/workload-api-server/templates/deployment.yaml new file mode 100644 index 00000000..9eb4fcdd --- /dev/null +++ b/helm/airbyte-v1/charts/workload-api-server/templates/deployment.yaml @@ -0,0 +1,208 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.names.fullname" . }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + minReadySeconds: 30 + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "airbyte.selectorLabels" . | nindent 6 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 6 }} + {{- end }} + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 100% + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 8 }} + {{- end }} + {{- if .Values.podLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ .Values.global.serviceAccountName }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.extraInitContainers }} + initContainers: + {{- toYaml .Values.extraInitContainers | nindent 6 }} + {{- end }} + containers: + - name: airbyte-workload-api-server-container + image: {{ include "imageUrl" (list .Values.image $) }} + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + env: + {{- if .Values.debug.enabled }} + - name: JAVA_TOOL_OPTIONS + value: "-Xdebug -agentlib:jdwp=transport=dt_socket,address=0.0.0.0:{{ .Values.debug.remoteDebugPort }},server=y,suspend=n" + {{- end }} + {{- if eq .Values.global.deploymentMode "oss" }} + - name: AIRBYTE_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_VERSION + - name: INTERNAL_API_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: INTERNAL_API_HOST + - name: AIRBYTE_API_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_API_HOST + - name: MICROMETER_METRICS_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MICROMETER_METRICS_ENABLED + - name: MICROMETER_METRICS_STATSD_FLAVOR + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MICROMETER_METRICS_STATSD_FLAVOR + - name: STATSD_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: STATSD_HOST + - name: STATSD_PORT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: STATSD_PORT + - name: TEMPORAL_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TEMPORAL_HOST + - name: WORKLOAD_API_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: {{ index .Values "workloadApi" "bearerTokenSecretName" | default (printf "%s-airbyte-secrets" .Release.Name ) }} + key: {{ index .Values "workloadApi" "bearerTokenSecretKey" | default "WORKLOAD_API_BEARER_TOKEN" }} + # Database + {{- include "airbyte.database.envs" . | nindent 8 }} + + {{- end }} + + # Values from secret + {{- if .Values.secrets }} + {{- range $k, $v := .Values.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: workload-api-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if or .Values.env_vars .Values.global.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + # Values from extraEnv for more compability(if you want to use external secret source or other stuff) + {{- if .Values.extraEnv }} + {{- toYaml .Values.extraEnv | nindent 8 }} + {{- end }} + + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /health/liveness + port: http + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /health/liveness + port: http + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + + ports: + - name: http + containerPort: 8007 + protocol: TCP + {{- if .Values.debug.enabled }} + - name: debug + containerPort: {{ .Values.debug.remoteDebugPort }} + protocol: TCP + {{- end }} + {{- if .Values.resources }} + resources: {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + {{- if .Values.containerSecurityContext }} + securityContext: {{- toYaml .Values.containerSecurityContext | nindent 10 }} + {{- end }} + volumeMounts: + {{- if and (eq .Values.global.deploymentMode "oss") (eq (lower (default "" .Values.global.storage.type)) "gcs") }} + - name: gcs-log-creds-volume + mountPath: /secrets/gcs-log-creds + readOnly: true + {{- end }} + {{- if .Values.extraVolumeMounts }} +{{- toYaml .Values.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.global.extraVolumeMounts }} +{{- toYaml .Values.global.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.extraContainers }} + {{ toYaml .Values.extraContainers | nindent 8 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 8 }} + {{- end }} + securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + volumes: + {{- if and (eq .Values.global.deploymentMode "oss") (eq (lower (default "" .Values.global.storage.type)) "gcs") }} + - name: gcs-log-creds-volume + secret: + secretName: {{ ternary (printf "%s-gcs-log-creds" ( .Release.Name )) .Values.global.storage.storageSecretName (not ((.Values.global.storage).storageSecretName)) }} + {{- end }} + {{- if .Values.extraVolumes }} + {{- toYaml .Values.extraVolumes | nindent 6 }} + {{- end }} + {{- if .Values.global.extraVolumes }} + {{- toYaml .Values.global.extraVolumes | nindent 6 }} + {{- end }} diff --git a/helm/airbyte-v1/charts/workload-api-server/templates/secrets.yaml b/helm/airbyte-v1/charts/workload-api-server/templates/secrets.yaml new file mode 100644 index 00000000..03e6f309 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-api-server/templates/secrets.yaml @@ -0,0 +1,17 @@ +# Create secrets only for the local deployment + {{- if .Values.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: workload-api-secrets + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" +type: Opaque +data: + {{- range $k, $v := mergeOverwrite .Values.secrets .Values.global.secrets }} + {{ $k }}: {{ if $v }}{{ $v | b64enc }} {{else}}""{{end}} + {{- end }} + {{- end }} diff --git a/helm/airbyte-v1/charts/workload-api-server/templates/service.yaml b/helm/airbyte-v1/charts/workload-api-server/templates/service.yaml new file mode 100644 index 00000000..7319b2e2 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-api-server/templates/service.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{.Release.Name }}-workload-api-server-svc + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "airbyte.selectorLabels" . | nindent 4 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 4 }} + {{- end }} diff --git a/helm/airbyte-v1/charts/workload-api-server/values.yaml b/helm/airbyte-v1/charts/workload-api-server/values.yaml new file mode 100644 index 00000000..2c378abc --- /dev/null +++ b/helm/airbyte-v1/charts/workload-api-server/values.yaml @@ -0,0 +1,281 @@ +global: + serviceAccountName: placeholderServiceAccount + deploymentMode: oss + edition: community + configMapName: "" + secretName: "" + credVolumeOverride: "" + extraContainers: [] + ## extraSelectorLabels [object] - use to specify own additional selector labels for deployment + extraSelectorLabels: {} + ## extraLabels [object] - use to specify own additional labels for deployment + extraLabels: {} + database: + secretName: "" + secretValue: "" + + # Docker image config that will apply to all images. + image: + # Docker registry to pull platform images from, e.g. http://my-registry:8000/ + registry: "" + # Image tag to use for airbyte images. + # Does not include non-airbyte images such as busybox, temporal, minio, etc. + tag: "" + + logs: + ## logs.accessKey.password Logs Access Key + ## logs.accessKey.existingSecret + ## logs.accessKey.existingSecretKey + accessKey: + password: minio + existingSecret: "" + existingSecretKey: "" + ## logs.secretKey.password Logs Secret Key + ## logs.secretKey.existingSecret + ## logs.secretKey.existingSecretKey + secretKey: + password: minio123 + existingSecret: "" + existingSecretKey: "" + + ## logs.minio.enabled Switch to enable or disable the Minio helm chart + minio: + enabled: true + + ## logs.s3.enabled Switch to enable or disable custom S3 Log location + ## logs.s3.bucket Bucket name where logs should be stored + ## logs.s3.bucketRegion Region of the bucket (must be empty if using minio) + s3: + enabled: false + bucket: airbyte-dev-logs + bucketRegion: "" + + ## Google Cloud Storage (GCS) Log Location Configuration + ## logs.gcs.bucket GCS bucket name + ## logs.gcs.credentials The path the GCS creds are written to + ## logs.gcs.credentialsJson Base64 encoded json GCP credentials file contents + gcs: + bucket: "" + # If you are mounting an existing secret to extraVolumes on scheduler, server and worker + # deployments, then set credentials to the path of the mounted JSON file + credentials: "" + # If credentialsJson is set then credentials auto resolves (to /secrets/gcs-log-creds/gcp.json) + credentialsJson: "" + +enabled: true +## replicaCount Number of server replicas +replicaCount: 1 + +## image.repository The repository to use for the airbyte workload api server image. +## image.pullPolicy the pull policy to use for the airbyte workload api server image +## image.tag The airbyte workload api server image tag. Defaults to the chart's AppVersion +image: + repository: airbyte/workload-api-server + pullPolicy: IfNotPresent + +## podAnnotations [object] Add extra annotations to the server pod +## +podAnnotations: {} + +## podLabels [object] Add extra labels to the server pod +## +podLabels: {} + +## containerSecurityContext Security context for the container +## Examples: +## containerSecurityContext: +## runAsNonRoot: true +## runAsUser: 1000 +## readOnlyRootFilesystem: true +containerSecurityContext: {} + +## Configure extra options for the server containers' liveness and readiness probes +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes +## livenessProbe.enabled Enable livenessProbe on the server +## livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe +## livenessProbe.periodSeconds Period seconds for livenessProbe +## livenessProbe.timeoutSeconds Timeout seconds for livenessProbe +## livenessProbe.failureThreshold Failure threshold for livenessProbe +## livenessProbe.successThreshold Success threshold for livenessProbe +## +livenessProbe: + enabled: true + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + +## readinessProbe.enabled Enable readinessProbe on the server +## readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe +## readinessProbe.periodSeconds Period seconds for readinessProbe +## readinessProbe.timeoutSeconds Timeout seconds for readinessProbe +## readinessProbe.failureThreshold Failure threshold for readinessProbe +## readinessProbe.successThreshold Success threshold for readinessProbe +## +readinessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + +## Server app resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## We usually recommend not to specify default resources and to leave this as a conscious +## choice for the user. This also increases chances charts run on environments with little +## resources, such as Minikube. If you do want to specify resources, uncomment the following +## lines, adjust them as necessary, and remove the curly braces after 'resources:'. +## resources.limits [object] The resources limits for the server container +## resources.requests [object] The requested resources for the server container +resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + +## service.type The service type to use for the API server +## service.port The service port to expose the API server on +service: + type: ClusterIP + port: 8007 + annotations: {} + +## nodeSelector [object] Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## tolerations [array] Tolerations for server pod assignment. +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## affinity [object] Affinity and anti-affinity for server pod assignment. +## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## Configure the ingress resource that allows you to access the Airbyte API. +## ref: http://kubernetes.io/docs/user-guide/ingress/ +## webapp.ingress.enabled Set to true to enable ingress record generation +## webapp.ingress.className Specifies ingressClassName for clusters >= 1.18+ +## webapp.ingress.annotations [object] Ingress annotations done as key:value pairs +## webapp.ingress.hosts The list of hostnames to be covered with this ingress record. +## webapp.ingress.tls [array] Custom ingress TLS configuration +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: [] + # - host: chart-example.local + # paths: + # - path: / + # pathType: ImplementationSpecific + + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +## log.level The log level to log at +log: + level: "INFO" + +## extraVolumeMounts [array] Additional volumeMounts for server container(s). +## Examples (when using `containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumeMounts: +## - name: tmpdir +## mountPath: /tmp +## +extraVolumeMounts: [] + +## extraVolumes [array] Additional volumes for server pod(s). +## Examples (when using `containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumes: +## - name: tmpdir +## emptyDir: {} +## +extraVolumes: [] + +## extraContainer [array] Additional container for server pod(s) +## Example: +# extraContainers: +# - name: otel_collector +# image: somerepo/someimage:sometag +# args: [ +# "--important-args" +# ] +# ports: +# - containerPort: 443 +# volumeMounts: +# - name: volumeMountCool +# mountPath: /some/path +# readOnly: true +extraContainers: [] + +## extraInitContainers [array] Additional init containers for server pod(s) +## Example: +# extraInitContainers: +# - name: sleepy +# image: alpine +# command: ['sleep', '60'] + +extraInitContainers: [] + +## extraEnv [array] Supply extra env variables to main container using full notation +## Example: (With default env vars and values taken from generated config map) +# extraEnv: +# - name: AIRBYTE_VERSION +# valueFrom: +# configMapKeyRef: +# name: airbyte-env +# key: AIRBYTE_VERSION +## +extraEnv: [] +## secrets [object] Supply additional secrets to container +## Example: +## secrets: +## DATABASE_PASSWORD: strong-password +## DATABASE_USER: my-db-user +secrets: {} + +## env_vars [object] Supply extra env variables to main container using simplified notation +## Example: +# env_vars: +# AIRBYTE_VERSION: 0.40.4 + +# # Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db +# DATABASE_HOST: airbyte-db-svc +# DATABASE_PORT: 5432 +# DATABASE_DB: airbyte +# # translate manually DATABASE_URL:jdbc:postgresql://${DATABASE_HOST}:${DATABASE_PORT/${DATABASE_DB} +# DATABASE_URL: jdbc:postgresql://airbyte-db-svc:5432/airbyte +# JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION: 0.29.15.001 +# CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION: 0.35.15.001 +env_vars: {} + + +## extraSelectorLabels [object] - use to specify own additional selector labels for deployment +extraSelectorLabels: {} +## extraLabels [object] - use to specify own additional labels for deployment +extraLabels: {} + + +## workloadApi: +## bearerTokenSecretName: workload-api-bearer-token-name kubernetes secret name where bearer token is stored +## bearerTokenSecretKey: workload-api-bearer-token-key kubernetes secret key where bearer token is stored +workloadApi: {} + +debug: + enabled: false + remoteDebugPort: 5005 diff --git a/helm/airbyte-v1/charts/workload-launcher/.helmignore b/helm/airbyte-v1/charts/workload-launcher/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/airbyte-v1/charts/workload-launcher/Chart.lock b/helm/airbyte-v1/charts/workload-launcher/Chart.lock new file mode 100644 index 00000000..13ffbcf4 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + version: 1.17.1 +digest: sha256:dacc73770a5640c011e067ff8840ddf89631fc19016c8d0a9e5ea160e7da8690 +generated: "2024-12-09T21:57:35.922522213Z" diff --git a/helm/airbyte-v1/charts/workload-launcher/Chart.yaml b/helm/airbyte-v1/charts/workload-launcher/Chart.yaml new file mode 100644 index 00000000..83cd8391 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +appVersion: 1.3.0 +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x +description: Helm chart to deploy airbyte-workload-launcher +name: workload-launcher +type: application +version: 1.3.0 diff --git a/helm/airbyte-v1/charts/workload-launcher/README.md b/helm/airbyte-v1/charts/workload-launcher/README.md new file mode 100644 index 00000000..d636166f --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/README.md @@ -0,0 +1,71 @@ +# workload-launcher + +![Version: 0.67.17](https://img.shields.io/badge/Version-0.67.17-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) + +Helm chart to deploy airbyte-workload-launcher + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://charts.bitnami.com/bitnami | common | 1.x.x | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| containerSecurityContext | object | `{}` | | +| debug.enabled | bool | `false` | | +| debug.remoteDebugPort | int | `5005` | | +| enabled | bool | `false` | | +| env_vars | object | `{}` | | +| extraContainers | list | `[]` | | +| extraEnv | list | `[]` | | +| extraInitContainers | list | `[]` | | +| extraLabels | object | `{}` | | +| extraSelectorLabels | object | `{}` | | +| extraVolumeMounts | list | `[]` | | +| extraVolumes | list | `[]` | | +| global.credVolumeOverride | string | `""` | | +| global.database.secretName | string | `""` | | +| global.database.secretValue | string | `""` | | +| global.extraContainers | list | `[]` | | +| global.extraLabels | object | `{}` | | +| global.extraSelectorLabels | object | `{}` | | +| global.jobs.kube.annotations | object | `{}` | | +| global.jobs.kube.images.busybox | string | `""` | | +| global.jobs.kube.images.curl | string | `""` | | +| global.jobs.kube.labels | object | `{}` | | +| global.jobs.kube.main_container_image_pull_secret | string | `""` | | +| global.jobs.kube.nodeSelector | object | `{}` | | +| global.jobs.kube.tolerations | list | `[]` | | +| global.jobs.resources.limits | object | `{}` | | +| global.jobs.resources.requests | object | `{}` | | +| global.secretName | string | `""` | | +| global.serviceAccountName | string | `"airbyte-admin"` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"airbyte/workload-launcher"` | | +| livenessProbe.enabled | bool | `true` | | +| livenessProbe.failureThreshold | int | `3` | | +| livenessProbe.initialDelaySeconds | int | `50` | | +| livenessProbe.periodSeconds | int | `10` | | +| livenessProbe.successThreshold | int | `1` | | +| livenessProbe.timeoutSeconds | int | `1` | | +| log.level | string | `"INFO"` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| podLabels | object | `{}` | | +| readinessProbe.enabled | bool | `true` | | +| readinessProbe.failureThreshold | int | `3` | | +| readinessProbe.initialDelaySeconds | int | `10` | | +| readinessProbe.periodSeconds | int | `10` | | +| readinessProbe.successThreshold | int | `1` | | +| readinessProbe.timeoutSeconds | int | `1` | | +| replicaCount | int | `1` | | +| resources.limits | object | `{}` | | +| resources.requests | object | `{}` | | +| secrets | object | `{}` | | +| tolerations | list | `[]` | | +| workloadApi | object | `{}` | | + diff --git a/helm/airbyte-v1/charts/workload-launcher/charts/common/.helmignore b/helm/airbyte-v1/charts/workload-launcher/charts/common/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/charts/common/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/airbyte-v1/charts/workload-launcher/charts/common/Chart.yaml b/helm/airbyte-v1/charts/workload-launcher/charts/common/Chart.yaml new file mode 100644 index 00000000..39edbb3c --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/charts/common/Chart.yaml @@ -0,0 +1,23 @@ +annotations: + category: Infrastructure +apiVersion: v2 +appVersion: 1.17.1 +description: A Library Helm Chart for grouping common logic between bitnami charts. + This chart is not deployable by itself. +home: https://github.com/bitnami/charts/tree/master/bitnami/common +icon: https://bitnami.com/downloads/logos/bitnami-mark.png +keywords: +- common +- helper +- template +- function +- bitnami +maintainers: +- name: Bitnami + url: https://github.com/bitnami/charts +name: common +sources: +- https://github.com/bitnami/charts +- https://www.bitnami.com/ +type: library +version: 1.17.1 diff --git a/helm/airbyte-v1/charts/workload-launcher/charts/common/README.md b/helm/airbyte-v1/charts/workload-launcher/charts/common/README.md new file mode 100644 index 00000000..a2ecd604 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/charts/common/README.md @@ -0,0 +1,350 @@ +# Bitnami Common Library Chart + +A [Helm Library Chart](https://helm.sh/docs/topics/library_charts/#helm) for grouping common logic between bitnami charts. + +## TL;DR + +```yaml +dependencies: + - name: common + version: 1.x.x + repository: https://charts.bitnami.com/bitnami +``` + +```bash +$ helm dependency update +``` + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.names.fullname" . }} +data: + myvalue: "Hello World" +``` + +## Introduction + +This chart provides a common template helpers which can be used to develop new charts using [Helm](https://helm.sh) package manager. + +Bitnami charts can be used with [Kubeapps](https://kubeapps.dev/) for deployment and management of Helm Charts in clusters. + +## Prerequisites + +- Kubernetes 1.19+ +- Helm 3.2.0+ + +## Parameters + +The following table lists the helpers available in the library which are scoped in different sections. + +### Affinities + +| Helper identifier | Description | Expected Input | +|-------------------------------|------------------------------------------------------|------------------------------------------------| +| `common.affinities.nodes.soft` | Return a soft nodeAffinity definition | `dict "key" "FOO" "values" (list "BAR" "BAZ")` | +| `common.affinities.nodes.hard` | Return a hard nodeAffinity definition | `dict "key" "FOO" "values" (list "BAR" "BAZ")` | +| `common.affinities.pods.soft` | Return a soft podAffinity/podAntiAffinity definition | `dict "component" "FOO" "context" $` | +| `common.affinities.pods.hard` | Return a hard podAffinity/podAntiAffinity definition | `dict "component" "FOO" "context" $` | + +### Capabilities + +| Helper identifier | Description | Expected Input | +|------------------------------------------------|------------------------------------------------------------------------------------------------|-------------------| +| `common.capabilities.kubeVersion` | Return the target Kubernetes version (using client default if .Values.kubeVersion is not set). | `.` Chart context | +| `common.capabilities.cronjob.apiVersion` | Return the appropriate apiVersion for cronjob. | `.` Chart context | +| `common.capabilities.deployment.apiVersion` | Return the appropriate apiVersion for deployment. | `.` Chart context | +| `common.capabilities.statefulset.apiVersion` | Return the appropriate apiVersion for statefulset. | `.` Chart context | +| `common.capabilities.ingress.apiVersion` | Return the appropriate apiVersion for ingress. | `.` Chart context | +| `common.capabilities.rbac.apiVersion` | Return the appropriate apiVersion for RBAC resources. | `.` Chart context | +| `common.capabilities.crd.apiVersion` | Return the appropriate apiVersion for CRDs. | `.` Chart context | +| `common.capabilities.policy.apiVersion` | Return the appropriate apiVersion for podsecuritypolicy. | `.` Chart context | +| `common.capabilities.networkPolicy.apiVersion` | Return the appropriate apiVersion for networkpolicy. | `.` Chart context | +| `common.capabilities.apiService.apiVersion` | Return the appropriate apiVersion for APIService. | `.` Chart context | +| `common.capabilities.hpa.apiVersion` | Return the appropriate apiVersion for Horizontal Pod Autoscaler | `.` Chart context | +| `common.capabilities.supportsHelmVersion` | Returns true if the used Helm version is 3.3+ | `.` Chart context | + +### Errors + +| Helper identifier | Description | Expected Input | +|-----------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------| +| `common.errors.upgrade.passwords.empty` | It will ensure required passwords are given when we are upgrading a chart. If `validationErrors` is not empty it will throw an error and will stop the upgrade action. | `dict "validationErrors" (list $validationError00 $validationError01) "context" $` | + +### Images + +| Helper identifier | Description | Expected Input | +|-----------------------------|------------------------------------------------------|---------------------------------------------------------------------------------------------------------| +| `common.images.image` | Return the proper and full image name | `dict "imageRoot" .Values.path.to.the.image "global" $`, see [ImageRoot](#imageroot) for the structure. | +| `common.images.pullSecrets` | Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) | `dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global` | +| `common.images.renderPullSecrets` | Return the proper Docker Image Registry Secret Names (evaluates values as templates) | `dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "context" $` | + +### Ingress + +| Helper identifier | Description | Expected Input | +|-------------------------------------------|-------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.ingress.backend` | Generate a proper Ingress backend entry depending on the API version | `dict "serviceName" "foo" "servicePort" "bar"`, see the [Ingress deprecation notice](https://kubernetes.io/blog/2019/07/18/api-deprecations-in-1-16/) for the syntax differences | +| `common.ingress.supportsPathType` | Prints "true" if the pathType field is supported | `.` Chart context | +| `common.ingress.supportsIngressClassname` | Prints "true" if the ingressClassname field is supported | `.` Chart context | +| `common.ingress.certManagerRequest` | Prints "true" if required cert-manager annotations for TLS signed certificates are set in the Ingress annotations | `dict "annotations" .Values.path.to.the.ingress.annotations` | + +### Labels + +| Helper identifier | Description | Expected Input | +|-----------------------------|-----------------------------------------------------------------------------|-------------------| +| `common.labels.standard` | Return Kubernetes standard labels | `.` Chart context | +| `common.labels.matchLabels` | Labels to use on `deploy.spec.selector.matchLabels` and `svc.spec.selector` | `.` Chart context | + +### Names + +| Helper identifier | Description | Expected Input | +|-----------------------------------|-----------------------------------------------------------------------|-------------------| +| `common.names.name` | Expand the name of the chart or use `.Values.nameOverride` | `.` Chart context | +| `common.names.fullname` | Create a default fully qualified app name. | `.` Chart context | +| `common.names.namespace` | Allow the release namespace to be overridden | `.` Chart context | +| `common.names.fullname.namespace` | Create a fully qualified app name adding the installation's namespace | `.` Chart context | +| `common.names.chart` | Chart name plus version | `.` Chart context | + +### Secrets + +| Helper identifier | Description | Expected Input | +|---------------------------|--------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.secrets.name` | Generate the name of the secret. | `dict "existingSecret" .Values.path.to.the.existingSecret "defaultNameSuffix" "mySuffix" "context" $` see [ExistingSecret](#existingsecret) for the structure. | +| `common.secrets.key` | Generate secret key. | `dict "existingSecret" .Values.path.to.the.existingSecret "key" "keyName"` see [ExistingSecret](#existingsecret) for the structure. | +| `common.passwords.manage` | Generate secret password or retrieve one if already created. | `dict "secret" "secret-name" "key" "keyName" "providedValues" (list "path.to.password1" "path.to.password2") "length" 10 "strong" false "chartName" "chartName" "context" $`, length, strong and chartNAme fields are optional. | +| `common.secrets.exists` | Returns whether a previous generated secret already exists. | `dict "secret" "secret-name" "context" $` | + +### Storage + +| Helper identifier | Description | Expected Input | +|-------------------------------|---------------------------------------|---------------------------------------------------------------------------------------------------------------------| +| `common.storage.class` | Return the proper Storage Class | `dict "persistence" .Values.path.to.the.persistence "global" $`, see [Persistence](#persistence) for the structure. | + +### TplValues + +| Helper identifier | Description | Expected Input | +|---------------------------|----------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.tplvalues.render` | Renders a value that contains template | `dict "value" .Values.path.to.the.Value "context" $`, value is the value should rendered as template, context frequently is the chart context `$` or `.` | + +### Utils + +| Helper identifier | Description | Expected Input | +|--------------------------------|------------------------------------------------------------------------------------------|------------------------------------------------------------------------| +| `common.utils.fieldToEnvVar` | Build environment variable name given a field. | `dict "field" "my-password"` | +| `common.utils.secret.getvalue` | Print instructions to get a secret value. | `dict "secret" "secret-name" "field" "secret-value-field" "context" $` | +| `common.utils.getValueFromKey` | Gets a value from `.Values` object given its key path | `dict "key" "path.to.key" "context" $` | +| `common.utils.getKeyFromList` | Returns first `.Values` key with a defined value or first of the list if all non-defined | `dict "keys" (list "path.to.key1" "path.to.key2") "context" $` | + +### Validations + +| Helper identifier | Description | Expected Input | +|--------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `common.validations.values.single.empty` | Validate a value must not be empty. | `dict "valueKey" "path.to.value" "secret" "secret.name" "field" "my-password" "subchart" "subchart" "context" $` secret, field and subchart are optional. In case they are given, the helper will generate a how to get instruction. See [ValidateValue](#validatevalue) | +| `common.validations.values.multiple.empty` | Validate a multiple values must not be empty. It returns a shared error for all the values. | `dict "required" (list $validateValueConf00 $validateValueConf01) "context" $`. See [ValidateValue](#validatevalue) | +| `common.validations.values.mariadb.passwords` | This helper will ensure required password for MariaDB are not empty. It returns a shared error for all the values. | `dict "secret" "mariadb-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use mariadb chart and the helper. | +| `common.validations.values.mysql.passwords` | This helper will ensure required password for MySQL are not empty. It returns a shared error for all the values. | `dict "secret" "mysql-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use mysql chart and the helper. | +| `common.validations.values.postgresql.passwords` | This helper will ensure required password for PostgreSQL are not empty. It returns a shared error for all the values. | `dict "secret" "postgresql-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use postgresql chart and the helper. | +| `common.validations.values.redis.passwords` | This helper will ensure required password for Redis® are not empty. It returns a shared error for all the values. | `dict "secret" "redis-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use redis chart and the helper. | +| `common.validations.values.cassandra.passwords` | This helper will ensure required password for Cassandra are not empty. It returns a shared error for all the values. | `dict "secret" "cassandra-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use cassandra chart and the helper. | +| `common.validations.values.mongodb.passwords` | This helper will ensure required password for MongoDB® are not empty. It returns a shared error for all the values. | `dict "secret" "mongodb-secret" "subchart" "true" "context" $` subchart field is optional and could be true or false it depends on where you will use mongodb chart and the helper. | + +### Warnings + +| Helper identifier | Description | Expected Input | +|------------------------------|----------------------------------|------------------------------------------------------------| +| `common.warnings.rollingTag` | Warning about using rolling tag. | `ImageRoot` see [ImageRoot](#imageroot) for the structure. | + +## Special input schemas + +### ImageRoot + +```yaml +registry: + type: string + description: Docker registry where the image is located + example: docker.io + +repository: + type: string + description: Repository and image name + example: bitnami/nginx + +tag: + type: string + description: image tag + example: 1.16.1-debian-10-r63 + +pullPolicy: + type: string + description: Specify a imagePullPolicy. Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent' + +pullSecrets: + type: array + items: + type: string + description: Optionally specify an array of imagePullSecrets (evaluated as templates). + +debug: + type: boolean + description: Set to true if you would like to see extra information on logs + example: false + +## An instance would be: +# registry: docker.io +# repository: bitnami/nginx +# tag: 1.16.1-debian-10-r63 +# pullPolicy: IfNotPresent +# debug: false +``` + +### Persistence + +```yaml +enabled: + type: boolean + description: Whether enable persistence. + example: true + +storageClass: + type: string + description: Ghost data Persistent Volume Storage Class, If set to "-", storageClassName: "" which disables dynamic provisioning. + example: "-" + +accessMode: + type: string + description: Access mode for the Persistent Volume Storage. + example: ReadWriteOnce + +size: + type: string + description: Size the Persistent Volume Storage. + example: 8Gi + +path: + type: string + description: Path to be persisted. + example: /bitnami + +## An instance would be: +# enabled: true +# storageClass: "-" +# accessMode: ReadWriteOnce +# size: 8Gi +# path: /bitnami +``` + +### ExistingSecret + +```yaml +name: + type: string + description: Name of the existing secret. + example: mySecret +keyMapping: + description: Mapping between the expected key name and the name of the key in the existing secret. + type: object + +## An instance would be: +# name: mySecret +# keyMapping: +# password: myPasswordKey +``` + +#### Example of use + +When we store sensitive data for a deployment in a secret, some times we want to give to users the possibility of using theirs existing secrets. + +```yaml +# templates/secret.yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "common.names.fullname" . }} + labels: + app: {{ include "common.names.fullname" . }} +type: Opaque +data: + password: {{ .Values.password | b64enc | quote }} + +# templates/dpl.yaml +--- +... + env: + - name: PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "common.secrets.name" (dict "existingSecret" .Values.existingSecret "context" $) }} + key: {{ include "common.secrets.key" (dict "existingSecret" .Values.existingSecret "key" "password") }} +... + +# values.yaml +--- +name: mySecret +keyMapping: + password: myPasswordKey +``` + +### ValidateValue + +#### NOTES.txt + +```console +{{- $validateValueConf00 := (dict "valueKey" "path.to.value00" "secret" "secretName" "field" "password-00") -}} +{{- $validateValueConf01 := (dict "valueKey" "path.to.value01" "secret" "secretName" "field" "password-01") -}} + +{{ include "common.validations.values.multiple.empty" (dict "required" (list $validateValueConf00 $validateValueConf01) "context" $) }} +``` + +If we force those values to be empty we will see some alerts + +```console +$ helm install test mychart --set path.to.value00="",path.to.value01="" + 'path.to.value00' must not be empty, please add '--set path.to.value00=$PASSWORD_00' to the command. To get the current value: + + export PASSWORD_00=$(kubectl get secret --namespace default secretName -o jsonpath="{.data.password-00}" | base64 -d) + + 'path.to.value01' must not be empty, please add '--set path.to.value01=$PASSWORD_01' to the command. To get the current value: + + export PASSWORD_01=$(kubectl get secret --namespace default secretName -o jsonpath="{.data.password-01}" | base64 -d) +``` + +## Upgrading + +### To 1.0.0 + +[On November 13, 2020, Helm v2 support was formally finished](https://github.com/helm/charts#status-of-the-project), this major version is the result of the required changes applied to the Helm Chart to be able to incorporate the different features added in Helm v3 and to be consistent with the Helm project itself regarding the Helm v2 EOL. + +**What changes were introduced in this major version?** + +- Previous versions of this Helm Chart use `apiVersion: v1` (installable by both Helm 2 and 3), this Helm Chart was updated to `apiVersion: v2` (installable by Helm 3 only). [Here](https://helm.sh/docs/topics/charts/#the-apiversion-field) you can find more information about the `apiVersion` field. +- Use `type: library`. [Here](https://v3.helm.sh/docs/faq/#library-chart-support) you can find more information. +- The different fields present in the *Chart.yaml* file has been ordered alphabetically in a homogeneous way for all the Bitnami Helm Charts + +**Considerations when upgrading to this version** + +- If you want to upgrade to this version from a previous one installed with Helm v3, you shouldn't face any issues +- If you want to upgrade to this version using Helm v2, this scenario is not supported as this version doesn't support Helm v2 anymore +- If you installed the previous version with Helm v2 and wants to upgrade to this version with Helm v3, please refer to the [official Helm documentation](https://helm.sh/docs/topics/v2_v3_migration/#migration-use-cases) about migrating from Helm v2 to v3 + +**Useful links** + +- https://docs.bitnami.com/tutorials/resolve-helm2-helm3-post-migration-issues/ +- https://helm.sh/docs/topics/v2_v3_migration/ +- https://helm.sh/blog/migrate-from-helm-v2-to-helm-v3/ + +## License + +Copyright © 2022 Bitnami + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_affinities.tpl b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_affinities.tpl new file mode 100644 index 00000000..2387be26 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_affinities.tpl @@ -0,0 +1,102 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Return a soft nodeAffinity definition +{{ include "common.affinities.nodes.soft" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.soft" -}} +preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . | quote }} + {{- end }} + weight: 1 +{{- end -}} + +{{/* +Return a hard nodeAffinity definition +{{ include "common.affinities.nodes.hard" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.hard" -}} +requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . | quote }} + {{- end }} +{{- end -}} + +{{/* +Return a nodeAffinity definition +{{ include "common.affinities.nodes" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.nodes.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.nodes.hard" . -}} + {{- end -}} +{{- end -}} + +{{/* +Return a soft podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.soft" (dict "component" "FOO" "extraMatchLabels" .Values.extraMatchLabels "context" $) -}} +*/}} +{{- define "common.affinities.pods.soft" -}} +{{- $component := default "" .component -}} +{{- $extraMatchLabels := default (dict) .extraMatchLabels -}} +preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" .context) | nindent 10 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := $extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + namespaces: + - {{ include "common.names.namespace" .context | quote }} + topologyKey: kubernetes.io/hostname + weight: 1 +{{- end -}} + +{{/* +Return a hard podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.hard" (dict "component" "FOO" "extraMatchLabels" .Values.extraMatchLabels "context" $) -}} +*/}} +{{- define "common.affinities.pods.hard" -}} +{{- $component := default "" .component -}} +{{- $extraMatchLabels := default (dict) .extraMatchLabels -}} +requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" .context) | nindent 8 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := $extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + namespaces: + - {{ include "common.names.namespace" .context | quote }} + topologyKey: kubernetes.io/hostname +{{- end -}} + +{{/* +Return a podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.pods" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.pods.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.pods.hard" . -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_capabilities.tpl b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_capabilities.tpl new file mode 100644 index 00000000..9d9b7600 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_capabilities.tpl @@ -0,0 +1,154 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Return the target Kubernetes version +*/}} +{{- define "common.capabilities.kubeVersion" -}} +{{- if .Values.global }} + {{- if .Values.global.kubeVersion }} + {{- .Values.global.kubeVersion -}} + {{- else }} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} + {{- end -}} +{{- else }} +{{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for poddisruptionbudget. +*/}} +{{- define "common.capabilities.policy.apiVersion" -}} +{{- if semverCompare "<1.21-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "policy/v1beta1" -}} +{{- else -}} +{{- print "policy/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for networkpolicy. +*/}} +{{- define "common.capabilities.networkPolicy.apiVersion" -}} +{{- if semverCompare "<1.7-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for cronjob. +*/}} +{{- define "common.capabilities.cronjob.apiVersion" -}} +{{- if semverCompare "<1.21-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "batch/v1beta1" -}} +{{- else -}} +{{- print "batch/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for deployment. +*/}} +{{- define "common.capabilities.deployment.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for statefulset. +*/}} +{{- define "common.capabilities.statefulset.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apps/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "common.capabilities.ingress.apiVersion" -}} +{{- if .Values.ingress -}} +{{- if .Values.ingress.apiVersion -}} +{{- .Values.ingress.apiVersion -}} +{{- else if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end }} +{{- else if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for RBAC resources. +*/}} +{{- define "common.capabilities.rbac.apiVersion" -}} +{{- if semverCompare "<1.17-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "rbac.authorization.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "rbac.authorization.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for CRDs. +*/}} +{{- define "common.capabilities.crd.apiVersion" -}} +{{- if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiextensions.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiextensions.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for APIService. +*/}} +{{- define "common.capabilities.apiService.apiVersion" -}} +{{- if semverCompare "<1.10-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiregistration.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiregistration.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for Horizontal Pod Autoscaler. +*/}} +{{- define "common.capabilities.hpa.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .context) -}} +{{- if .beta2 -}} +{{- print "autoscaling/v2beta2" -}} +{{- else -}} +{{- print "autoscaling/v2beta1" -}} +{{- end -}} +{{- else -}} +{{- print "autoscaling/v2" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if the used Helm version is 3.3+. +A way to check the used Helm version was not introduced until version 3.3.0 with .Capabilities.HelmVersion, which contains an additional "{}}" structure. +This check is introduced as a regexMatch instead of {{ if .Capabilities.HelmVersion }} because checking for the key HelmVersion in <3.3 results in a "interface not found" error. +**To be removed when the catalog's minimun Helm version is 3.3** +*/}} +{{- define "common.capabilities.supportsHelmVersion" -}} +{{- if regexMatch "{(v[0-9])*[^}]*}}$" (.Capabilities | toString ) }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_errors.tpl b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_errors.tpl new file mode 100644 index 00000000..a79cc2e3 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_errors.tpl @@ -0,0 +1,23 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Through error when upgrading using empty passwords values that must not be empty. + +Usage: +{{- $validationError00 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password00" "secret" "secretName" "field" "password-00") -}} +{{- $validationError01 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password01" "secret" "secretName" "field" "password-01") -}} +{{ include "common.errors.upgrade.passwords.empty" (dict "validationErrors" (list $validationError00 $validationError01) "context" $) }} + +Required password params: + - validationErrors - String - Required. List of validation strings to be return, if it is empty it won't throw error. + - context - Context - Required. Parent context. +*/}} +{{- define "common.errors.upgrade.passwords.empty" -}} + {{- $validationErrors := join "" .validationErrors -}} + {{- if and $validationErrors .context.Release.IsUpgrade -}} + {{- $errorString := "\nPASSWORDS ERROR: You must provide your current passwords when upgrading the release." -}} + {{- $errorString = print $errorString "\n Note that even after reinstallation, old credentials may be needed as they may be kept in persistent volume claims." -}} + {{- $errorString = print $errorString "\n Further information can be obtained at https://docs.bitnami.com/general/how-to/troubleshoot-helm-chart-issues/#credential-errors-while-upgrading-chart-releases" -}} + {{- $errorString = print $errorString "\n%s" -}} + {{- printf $errorString $validationErrors | fail -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_images.tpl b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_images.tpl new file mode 100644 index 00000000..42ffbc72 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_images.tpl @@ -0,0 +1,75 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper image name +{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" $) }} +*/}} +{{- define "common.images.image" -}} +{{- $registryName := .imageRoot.registry -}} +{{- $repositoryName := .imageRoot.repository -}} +{{- $tag := .imageRoot.tag | toString -}} +{{- if .global }} + {{- if .global.imageRegistry }} + {{- $registryName = .global.imageRegistry -}} + {{- end -}} +{{- end -}} +{{- if $registryName }} +{{- printf "%s/%s:%s" $registryName $repositoryName $tag -}} +{{- else -}} +{{- printf "%s:%s" $repositoryName $tag -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) +{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} +*/}} +{{- define "common.images.pullSecrets" -}} + {{- $pullSecrets := list }} + + {{- if .global }} + {{- range .global.imagePullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names evaluating values as templates +{{ include "common.images.renderPullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "context" $) }} +*/}} +{{- define "common.images.renderPullSecrets" -}} + {{- $pullSecrets := list }} + {{- $context := .context }} + + {{- if $context.Values.global }} + {{- range $context.Values.global.imagePullSecrets -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" $context)) -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" $context)) -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_ingress.tpl b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_ingress.tpl new file mode 100644 index 00000000..8caf73a6 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_ingress.tpl @@ -0,0 +1,68 @@ +{{/* vim: set filetype=mustache: */}} + +{{/* +Generate backend entry that is compatible with all Kubernetes API versions. + +Usage: +{{ include "common.ingress.backend" (dict "serviceName" "backendName" "servicePort" "backendPort" "context" $) }} + +Params: + - serviceName - String. Name of an existing service backend + - servicePort - String/Int. Port name (or number) of the service. It will be translated to different yaml depending if it is a string or an integer. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.ingress.backend" -}} +{{- $apiVersion := (include "common.capabilities.ingress.apiVersion" .context) -}} +{{- if or (eq $apiVersion "extensions/v1beta1") (eq $apiVersion "networking.k8s.io/v1beta1") -}} +serviceName: {{ .serviceName }} +servicePort: {{ .servicePort }} +{{- else -}} +service: + name: {{ .serviceName }} + port: + {{- if typeIs "string" .servicePort }} + name: {{ .servicePort }} + {{- else if or (typeIs "int" .servicePort) (typeIs "float64" .servicePort) }} + number: {{ .servicePort | int }} + {{- end }} +{{- end -}} +{{- end -}} + +{{/* +Print "true" if the API pathType field is supported +Usage: +{{ include "common.ingress.supportsPathType" . }} +*/}} +{{- define "common.ingress.supportsPathType" -}} +{{- if (semverCompare "<1.18-0" (include "common.capabilities.kubeVersion" .)) -}} +{{- print "false" -}} +{{- else -}} +{{- print "true" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if the ingressClassname field is supported +Usage: +{{ include "common.ingress.supportsIngressClassname" . }} +*/}} +{{- define "common.ingress.supportsIngressClassname" -}} +{{- if semverCompare "<1.18-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "false" -}} +{{- else -}} +{{- print "true" -}} +{{- end -}} +{{- end -}} + +{{/* +Return true if cert-manager required annotations for TLS signed +certificates are set in the Ingress annotations +Ref: https://cert-manager.io/docs/usage/ingress/#supported-annotations +Usage: +{{ include "common.ingress.certManagerRequest" ( dict "annotations" .Values.path.to.the.ingress.annotations ) }} +*/}} +{{- define "common.ingress.certManagerRequest" -}} +{{ if or (hasKey .annotations "cert-manager.io/cluster-issuer") (hasKey .annotations "cert-manager.io/issuer") }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_labels.tpl b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_labels.tpl new file mode 100644 index 00000000..252066c7 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_labels.tpl @@ -0,0 +1,18 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Kubernetes standard labels +*/}} +{{- define "common.labels.standard" -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +helm.sh/chart: {{ include "common.names.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Labels to use on deploy.spec.selector.matchLabels and svc.spec.selector +*/}} +{{- define "common.labels.matchLabels" -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} diff --git a/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_names.tpl b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_names.tpl new file mode 100644 index 00000000..1bdac8b7 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_names.tpl @@ -0,0 +1,70 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "common.names.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "common.names.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "common.names.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a default fully qualified dependency name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +Usage: +{{ include "common.names.dependency.fullname" (dict "chartName" "dependency-chart-name" "chartValues" .Values.dependency-chart "context" $) }} +*/}} +{{- define "common.names.dependency.fullname" -}} +{{- if .chartValues.fullnameOverride -}} +{{- .chartValues.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .chartName .chartValues.nameOverride -}} +{{- if contains $name .context.Release.Name -}} +{{- .context.Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .context.Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts. +*/}} +{{- define "common.names.namespace" -}} +{{- if .Values.namespaceOverride -}} +{{- .Values.namespaceOverride -}} +{{- else -}} +{{- .Release.Namespace -}} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified app name adding the installation's namespace. +*/}} +{{- define "common.names.fullname.namespace" -}} +{{- printf "%s-%s" (include "common.names.fullname" .) (include "common.names.namespace" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_secrets.tpl b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_secrets.tpl new file mode 100644 index 00000000..a53fb44f --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_secrets.tpl @@ -0,0 +1,140 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Generate secret name. + +Usage: +{{ include "common.secrets.name" (dict "existingSecret" .Values.path.to.the.existingSecret "defaultNameSuffix" "mySuffix" "context" $) }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/master/bitnami/common#existingsecret + - defaultNameSuffix - String - Optional. It is used only if we have several secrets in the same deployment. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.secrets.name" -}} +{{- $name := (include "common.names.fullname" .context) -}} + +{{- if .defaultNameSuffix -}} +{{- $name = printf "%s-%s" $name .defaultNameSuffix | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- with .existingSecret -}} +{{- if not (typeIs "string" .) -}} +{{- with .name -}} +{{- $name = . -}} +{{- end -}} +{{- else -}} +{{- $name = . -}} +{{- end -}} +{{- end -}} + +{{- printf "%s" $name -}} +{{- end -}} + +{{/* +Generate secret key. + +Usage: +{{ include "common.secrets.key" (dict "existingSecret" .Values.path.to.the.existingSecret "key" "keyName") }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/master/bitnami/common#existingsecret + - key - String - Required. Name of the key in the secret. +*/}} +{{- define "common.secrets.key" -}} +{{- $key := .key -}} + +{{- if .existingSecret -}} + {{- if not (typeIs "string" .existingSecret) -}} + {{- if .existingSecret.keyMapping -}} + {{- $key = index .existingSecret.keyMapping $.key -}} + {{- end -}} + {{- end }} +{{- end -}} + +{{- printf "%s" $key -}} +{{- end -}} + +{{/* +Generate secret password or retrieve one if already created. + +Usage: +{{ include "common.secrets.passwords.manage" (dict "secret" "secret-name" "key" "keyName" "providedValues" (list "path.to.password1" "path.to.password2") "length" 10 "strong" false "chartName" "chartName" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - key - String - Required - Name of the key in the secret. + - providedValues - List - Required - The path to the validating value in the values.yaml, e.g: "mysql.password". Will pick first parameter with a defined value. + - length - int - Optional - Length of the generated random password. + - strong - Boolean - Optional - Whether to add symbols to the generated random password. + - chartName - String - Optional - Name of the chart used when said chart is deployed as a subchart. + - context - Context - Required - Parent context. + +The order in which this function returns a secret password: + 1. Already existing 'Secret' resource + (If a 'Secret' resource is found under the name provided to the 'secret' parameter to this function and that 'Secret' resource contains a key with the name passed as the 'key' parameter to this function then the value of this existing secret password will be returned) + 2. Password provided via the values.yaml + (If one of the keys passed to the 'providedValues' parameter to this function is a valid path to a key in the values.yaml and has a value, the value of the first key with a value will be returned) + 3. Randomly generated secret password + (A new random secret password with the length specified in the 'length' parameter will be generated and returned) + +*/}} +{{- define "common.secrets.passwords.manage" -}} + +{{- $password := "" }} +{{- $subchart := "" }} +{{- $chartName := default "" .chartName }} +{{- $passwordLength := default 10 .length }} +{{- $providedPasswordKey := include "common.utils.getKeyFromList" (dict "keys" .providedValues "context" $.context) }} +{{- $providedPasswordValue := include "common.utils.getValueFromKey" (dict "key" $providedPasswordKey "context" $.context) }} +{{- $secretData := (lookup "v1" "Secret" $.context.Release.Namespace .secret).data }} +{{- if $secretData }} + {{- if hasKey $secretData .key }} + {{- $password = index $secretData .key }} + {{- else }} + {{- printf "\nPASSWORDS ERROR: The secret \"%s\" does not contain the key \"%s\"\n" .secret .key | fail -}} + {{- end -}} +{{- else if $providedPasswordValue }} + {{- $password = $providedPasswordValue | toString | b64enc | quote }} +{{- else }} + + {{- if .context.Values.enabled }} + {{- $subchart = $chartName }} + {{- end -}} + + {{- $requiredPassword := dict "valueKey" $providedPasswordKey "secret" .secret "field" .key "subchart" $subchart "context" $.context -}} + {{- $requiredPasswordError := include "common.validations.values.single.empty" $requiredPassword -}} + {{- $passwordValidationErrors := list $requiredPasswordError -}} + {{- include "common.errors.upgrade.passwords.empty" (dict "validationErrors" $passwordValidationErrors "context" $.context) -}} + + {{- if .strong }} + {{- $subStr := list (lower (randAlpha 1)) (randNumeric 1) (upper (randAlpha 1)) | join "_" }} + {{- $password = randAscii $passwordLength }} + {{- $password = regexReplaceAllLiteral "\\W" $password "@" | substr 5 $passwordLength }} + {{- $password = printf "%s%s" $subStr $password | toString | shuffle | b64enc | quote }} + {{- else }} + {{- $password = randAlphaNum $passwordLength | b64enc | quote }} + {{- end }} +{{- end -}} +{{- printf "%s" $password -}} +{{- end -}} + +{{/* +Returns whether a previous generated secret already exists + +Usage: +{{ include "common.secrets.exists" (dict "secret" "secret-name" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - context - Context - Required - Parent context. +*/}} +{{- define "common.secrets.exists" -}} +{{- $secret := (lookup "v1" "Secret" $.context.Release.Namespace .secret) }} +{{- if $secret }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_storage.tpl b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_storage.tpl new file mode 100644 index 00000000..60e2a844 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_storage.tpl @@ -0,0 +1,23 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper Storage Class +{{ include "common.storage.class" ( dict "persistence" .Values.path.to.the.persistence "global" $) }} +*/}} +{{- define "common.storage.class" -}} + +{{- $storageClass := .persistence.storageClass -}} +{{- if .global -}} + {{- if .global.storageClass -}} + {{- $storageClass = .global.storageClass -}} + {{- end -}} +{{- end -}} + +{{- if $storageClass -}} + {{- if (eq "-" $storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" $storageClass -}} + {{- end -}} +{{- end -}} + +{{- end -}} diff --git a/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_tplvalues.tpl b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_tplvalues.tpl new file mode 100644 index 00000000..2db16685 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_tplvalues.tpl @@ -0,0 +1,13 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Renders a value that contains template. +Usage: +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $) }} +*/}} +{{- define "common.tplvalues.render" -}} + {{- if typeIs "string" .value }} + {{- tpl .value .context }} + {{- else }} + {{- tpl (.value | toYaml) .context }} + {{- end }} +{{- end -}} diff --git a/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_utils.tpl b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_utils.tpl new file mode 100644 index 00000000..8c22b2a3 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_utils.tpl @@ -0,0 +1,62 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Print instructions to get a secret value. +Usage: +{{ include "common.utils.secret.getvalue" (dict "secret" "secret-name" "field" "secret-value-field" "context" $) }} +*/}} +{{- define "common.utils.secret.getvalue" -}} +{{- $varname := include "common.utils.fieldToEnvVar" . -}} +export {{ $varname }}=$(kubectl get secret --namespace {{ .context.Release.Namespace | quote }} {{ .secret }} -o jsonpath="{.data.{{ .field }}}" | base64 -d) +{{- end -}} + +{{/* +Build env var name given a field +Usage: +{{ include "common.utils.fieldToEnvVar" dict "field" "my-password" }} +*/}} +{{- define "common.utils.fieldToEnvVar" -}} + {{- $fieldNameSplit := splitList "-" .field -}} + {{- $upperCaseFieldNameSplit := list -}} + + {{- range $fieldNameSplit -}} + {{- $upperCaseFieldNameSplit = append $upperCaseFieldNameSplit ( upper . ) -}} + {{- end -}} + + {{ join "_" $upperCaseFieldNameSplit }} +{{- end -}} + +{{/* +Gets a value from .Values given +Usage: +{{ include "common.utils.getValueFromKey" (dict "key" "path.to.key" "context" $) }} +*/}} +{{- define "common.utils.getValueFromKey" -}} +{{- $splitKey := splitList "." .key -}} +{{- $value := "" -}} +{{- $latestObj := $.context.Values -}} +{{- range $splitKey -}} + {{- if not $latestObj -}} + {{- printf "please review the entire path of '%s' exists in values" $.key | fail -}} + {{- end -}} + {{- $value = ( index $latestObj . ) -}} + {{- $latestObj = $value -}} +{{- end -}} +{{- printf "%v" (default "" $value) -}} +{{- end -}} + +{{/* +Returns first .Values key with a defined value or first of the list if all non-defined +Usage: +{{ include "common.utils.getKeyFromList" (dict "keys" (list "path.to.key1" "path.to.key2") "context" $) }} +*/}} +{{- define "common.utils.getKeyFromList" -}} +{{- $key := first .keys -}} +{{- $reverseKeys := reverse .keys }} +{{- range $reverseKeys }} + {{- $value := include "common.utils.getValueFromKey" (dict "key" . "context" $.context ) }} + {{- if $value -}} + {{- $key = . }} + {{- end -}} +{{- end -}} +{{- printf "%s" $key -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_warnings.tpl b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_warnings.tpl new file mode 100644 index 00000000..ae10fa41 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/_warnings.tpl @@ -0,0 +1,14 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Warning about using rolling tag. +Usage: +{{ include "common.warnings.rollingTag" .Values.path.to.the.imageRoot }} +*/}} +{{- define "common.warnings.rollingTag" -}} + +{{- if and (contains "bitnami/" .repository) (not (.tag | toString | regexFind "-r\\d+$|sha256:")) }} +WARNING: Rolling tag detected ({{ .repository }}:{{ .tag }}), please note that it is strongly recommended to avoid using rolling tags in a production environment. ++info https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/ +{{- end }} + +{{- end -}} diff --git a/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/validations/_cassandra.tpl b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/validations/_cassandra.tpl new file mode 100644 index 00000000..ded1ae3b --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/validations/_cassandra.tpl @@ -0,0 +1,72 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate Cassandra required passwords are not empty. + +Usage: +{{ include "common.validations.values.cassandra.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where Cassandra values are stored, e.g: "cassandra-passwords-secret" + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.cassandra.passwords" -}} + {{- $existingSecret := include "common.cassandra.values.existingSecret" . -}} + {{- $enabled := include "common.cassandra.values.enabled" . -}} + {{- $dbUserPrefix := include "common.cassandra.values.key.dbUser" . -}} + {{- $valueKeyPassword := printf "%s.password" $dbUserPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "cassandra-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.cassandra.values.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.cassandra.values.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.cassandra.dbUser.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.dbUser.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled cassandra. + +Usage: +{{ include "common.cassandra.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.cassandra.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.cassandra.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key dbUser + +Usage: +{{ include "common.cassandra.values.key.dbUser" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.cassandra.values.key.dbUser" -}} + {{- if .subchart -}} + cassandra.dbUser + {{- else -}} + dbUser + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/validations/_mariadb.tpl b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/validations/_mariadb.tpl new file mode 100644 index 00000000..b6906ff7 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/validations/_mariadb.tpl @@ -0,0 +1,103 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MariaDB required passwords are not empty. + +Usage: +{{ include "common.validations.values.mariadb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MariaDB values are stored, e.g: "mysql-passwords-secret" + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mariadb.passwords" -}} + {{- $existingSecret := include "common.mariadb.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mariadb.values.enabled" . -}} + {{- $architecture := include "common.mariadb.values.architecture" . -}} + {{- $authPrefix := include "common.mariadb.values.key.auth" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mariadb-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- if not (empty $valueUsername) -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mariadb-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replication") -}} + {{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mariadb-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mariadb.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mariadb. + +Usage: +{{ include "common.mariadb.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mariadb.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mariadb.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mariadb.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mariadb.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.key.auth" -}} + {{- if .subchart -}} + mariadb.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/validations/_mongodb.tpl b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/validations/_mongodb.tpl new file mode 100644 index 00000000..f820ec10 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/validations/_mongodb.tpl @@ -0,0 +1,108 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MongoDB® required passwords are not empty. + +Usage: +{{ include "common.validations.values.mongodb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MongoDB® values are stored, e.g: "mongodb-passwords-secret" + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mongodb.passwords" -}} + {{- $existingSecret := include "common.mongodb.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mongodb.values.enabled" . -}} + {{- $authPrefix := include "common.mongodb.values.key.auth" . -}} + {{- $architecture := include "common.mongodb.values.architecture" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyDatabase := printf "%s.database" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicaSetKey := printf "%s.replicaSetKey" $authPrefix -}} + {{- $valueKeyAuthEnabled := printf "%s.enabled" $authPrefix -}} + + {{- $authEnabled := include "common.utils.getValueFromKey" (dict "key" $valueKeyAuthEnabled "context" .context) -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") (eq $authEnabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mongodb-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- $valueDatabase := include "common.utils.getValueFromKey" (dict "key" $valueKeyDatabase "context" .context) }} + {{- if and $valueUsername $valueDatabase -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mongodb-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replicaset") -}} + {{- $requiredReplicaSetKey := dict "valueKey" $valueKeyReplicaSetKey "secret" .secret "field" "mongodb-replica-set-key" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicaSetKey -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mongodb.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDb is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mongodb.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mongodb. + +Usage: +{{ include "common.mongodb.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mongodb.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mongodb.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mongodb.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.key.auth" -}} + {{- if .subchart -}} + mongodb.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mongodb.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mongodb.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/validations/_mysql.tpl b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/validations/_mysql.tpl new file mode 100644 index 00000000..74472a06 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/validations/_mysql.tpl @@ -0,0 +1,103 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MySQL required passwords are not empty. + +Usage: +{{ include "common.validations.values.mysql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MySQL values are stored, e.g: "mysql-passwords-secret" + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mysql.passwords" -}} + {{- $existingSecret := include "common.mysql.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mysql.values.enabled" . -}} + {{- $architecture := include "common.mysql.values.architecture" . -}} + {{- $authPrefix := include "common.mysql.values.key.auth" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mysql-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- if not (empty $valueUsername) -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mysql-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replication") -}} + {{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mysql-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mysql.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mysql.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mysql. + +Usage: +{{ include "common.mysql.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mysql.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mysql.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mysql.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mysql.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mysql.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.key.auth" -}} + {{- if .subchart -}} + mysql.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/validations/_postgresql.tpl b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/validations/_postgresql.tpl new file mode 100644 index 00000000..164ec0d0 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/validations/_postgresql.tpl @@ -0,0 +1,129 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate PostgreSQL required passwords are not empty. + +Usage: +{{ include "common.validations.values.postgresql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where postgresql values are stored, e.g: "postgresql-passwords-secret" + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.postgresql.passwords" -}} + {{- $existingSecret := include "common.postgresql.values.existingSecret" . -}} + {{- $enabled := include "common.postgresql.values.enabled" . -}} + {{- $valueKeyPostgresqlPassword := include "common.postgresql.values.key.postgressPassword" . -}} + {{- $valueKeyPostgresqlReplicationEnabled := include "common.postgresql.values.key.replicationPassword" . -}} + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + {{- $requiredPostgresqlPassword := dict "valueKey" $valueKeyPostgresqlPassword "secret" .secret "field" "postgresql-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlPassword -}} + + {{- $enabledReplication := include "common.postgresql.values.enabled.replication" . -}} + {{- if (eq $enabledReplication "true") -}} + {{- $requiredPostgresqlReplicationPassword := dict "valueKey" $valueKeyPostgresqlReplicationEnabled "secret" .secret "field" "postgresql-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to decide whether evaluate global values. + +Usage: +{{ include "common.postgresql.values.use.global" (dict "key" "key-of-global" "context" $) }} +Params: + - key - String - Required. Field to be evaluated within global, e.g: "existingSecret" +*/}} +{{- define "common.postgresql.values.use.global" -}} + {{- if .context.Values.global -}} + {{- if .context.Values.global.postgresql -}} + {{- index .context.Values.global.postgresql .key | quote -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.postgresql.values.existingSecret" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.existingSecret" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "existingSecret" "context" .context) -}} + + {{- if .subchart -}} + {{- default (.context.Values.postgresql.existingSecret | quote) $globalValue -}} + {{- else -}} + {{- default (.context.Values.existingSecret | quote) $globalValue -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled postgresql. + +Usage: +{{ include "common.postgresql.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key postgressPassword. + +Usage: +{{ include "common.postgresql.values.key.postgressPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.postgressPassword" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "postgresqlUsername" "context" .context) -}} + + {{- if not $globalValue -}} + {{- if .subchart -}} + postgresql.postgresqlPassword + {{- else -}} + postgresqlPassword + {{- end -}} + {{- else -}} + global.postgresql.postgresqlPassword + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled.replication. + +Usage: +{{ include "common.postgresql.values.enabled.replication" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.enabled.replication" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.replication.enabled -}} + {{- else -}} + {{- printf "%v" .context.Values.replication.enabled -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key replication.password. + +Usage: +{{ include "common.postgresql.values.key.replicationPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.replicationPassword" -}} + {{- if .subchart -}} + postgresql.replication.password + {{- else -}} + replication.password + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/validations/_redis.tpl b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/validations/_redis.tpl new file mode 100644 index 00000000..dcccfc1a --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/validations/_redis.tpl @@ -0,0 +1,76 @@ + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate Redis® required passwords are not empty. + +Usage: +{{ include "common.validations.values.redis.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where redis values are stored, e.g: "redis-passwords-secret" + - subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.redis.passwords" -}} + {{- $enabled := include "common.redis.values.enabled" . -}} + {{- $valueKeyPrefix := include "common.redis.values.keys.prefix" . -}} + {{- $standarizedVersion := include "common.redis.values.standarized.version" . }} + + {{- $existingSecret := ternary (printf "%s%s" $valueKeyPrefix "auth.existingSecret") (printf "%s%s" $valueKeyPrefix "existingSecret") (eq $standarizedVersion "true") }} + {{- $existingSecretValue := include "common.utils.getValueFromKey" (dict "key" $existingSecret "context" .context) }} + + {{- $valueKeyRedisPassword := ternary (printf "%s%s" $valueKeyPrefix "auth.password") (printf "%s%s" $valueKeyPrefix "password") (eq $standarizedVersion "true") }} + {{- $valueKeyRedisUseAuth := ternary (printf "%s%s" $valueKeyPrefix "auth.enabled") (printf "%s%s" $valueKeyPrefix "usePassword") (eq $standarizedVersion "true") }} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $useAuth := include "common.utils.getValueFromKey" (dict "key" $valueKeyRedisUseAuth "context" .context) -}} + {{- if eq $useAuth "true" -}} + {{- $requiredRedisPassword := dict "valueKey" $valueKeyRedisPassword "secret" .secret "field" "redis-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRedisPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled redis. + +Usage: +{{ include "common.redis.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.redis.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.redis.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right prefix path for the values + +Usage: +{{ include "common.redis.values.key.prefix" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false +*/}} +{{- define "common.redis.values.keys.prefix" -}} + {{- if .subchart -}}redis.{{- else -}}{{- end -}} +{{- end -}} + +{{/* +Checks whether the redis chart's includes the standarizations (version >= 14) + +Usage: +{{ include "common.redis.values.standarized.version" (dict "context" $) }} +*/}} +{{- define "common.redis.values.standarized.version" -}} + + {{- $standarizedAuth := printf "%s%s" (include "common.redis.values.keys.prefix" .) "auth" -}} + {{- $standarizedAuthValues := include "common.utils.getValueFromKey" (dict "key" $standarizedAuth "context" .context) }} + + {{- if $standarizedAuthValues -}} + {{- true -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/validations/_validations.tpl b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/validations/_validations.tpl new file mode 100644 index 00000000..9a814cf4 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/charts/common/templates/validations/_validations.tpl @@ -0,0 +1,46 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Validate values must not be empty. + +Usage: +{{- $validateValueConf00 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-00") -}} +{{- $validateValueConf01 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-01") -}} +{{ include "common.validations.values.empty" (dict "required" (list $validateValueConf00 $validateValueConf01) "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" +*/}} +{{- define "common.validations.values.multiple.empty" -}} + {{- range .required -}} + {{- include "common.validations.values.single.empty" (dict "valueKey" .valueKey "secret" .secret "field" .field "context" $.context) -}} + {{- end -}} +{{- end -}} + +{{/* +Validate a value must not be empty. + +Usage: +{{ include "common.validations.value.empty" (dict "valueKey" "mariadb.password" "secret" "secretName" "field" "my-password" "subchart" "subchart" "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" + - subchart - String - Optional - Name of the subchart that the validated password is part of. +*/}} +{{- define "common.validations.values.single.empty" -}} + {{- $value := include "common.utils.getValueFromKey" (dict "key" .valueKey "context" .context) }} + {{- $subchart := ternary "" (printf "%s." .subchart) (empty .subchart) }} + + {{- if not $value -}} + {{- $varname := "my-value" -}} + {{- $getCurrentValue := "" -}} + {{- if and .secret .field -}} + {{- $varname = include "common.utils.fieldToEnvVar" . -}} + {{- $getCurrentValue = printf " To get the current value:\n\n %s\n" (include "common.utils.secret.getvalue" .) -}} + {{- end -}} + {{- printf "\n '%s' must not be empty, please add '--set %s%s=$%s' to the command.%s" .valueKey $subchart .valueKey $varname $getCurrentValue -}} + {{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/workload-launcher/charts/common/values.yaml b/helm/airbyte-v1/charts/workload-launcher/charts/common/values.yaml new file mode 100644 index 00000000..f2df68e5 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/charts/common/values.yaml @@ -0,0 +1,5 @@ +## bitnami/common +## It is required by CI/CD tools and processes. +## @skip exampleValue +## +exampleValue: common-chart diff --git a/helm/airbyte-v1/charts/workload-launcher/templates/_helpers.tpl b/helm/airbyte-v1/charts/workload-launcher/templates/_helpers.tpl new file mode 100644 index 00000000..933c125b --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/templates/_helpers.tpl @@ -0,0 +1,258 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "airbyte.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airbyte.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airbyte.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "airbyte.labels" -}} +helm.sh/chart: {{ include "airbyte.chart" . }} +{{ include "airbyte.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "airbyte.selectorLabels" -}} +app.kubernetes.io/name: {{ include "airbyte.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +## DEFAULT HELM VALUES +# Secret Manager Defaults +{{/* +Define secret persistence +*/}} +{{- define "airbyte.secretPersistence" -}} +{{- if (((.Values.global).secretsManager).type) }} + {{- printf "%s" (snakecase .Values.global.secretsManager.type) }} +{{- else }} + {{- printf "" }} +{{- end }} +{{- end }} + +{{/* +Get secret store name or default +*/}} +{{- define "airbyte.secretStoreName" -}} +{{- $secretStoreName := . -}} +{{- if $secretStoreName -}} + {{- printf "%s" $secretStoreName -}} +{{- else -}} + {{- printf "airbyte-config-secrets" -}} +{{- end -}} +{{- end -}} + +{{/* +Get awsSecretManager access key id secret key or default +*/}} +{{- define "airbyte.awsSecretManagerAccessKeyIdSecretKey" -}} +{{- $awsSecretManagerAccessKeyIdSecretKey := . -}} +{{- if $awsSecretManagerAccessKeyIdSecretKey -}} + {{- printf "%s" $awsSecretManagerAccessKeyIdSecretKey -}} +{{- else -}} + {{- printf "aws-secret-manager-access-key-id" -}} +{{- end -}} +{{- end -}} + +{{/* +Get awsSecretManager secret access key secret key or default +*/}} +{{- define "airbyte.awsSecretManagerSecretAccessKeySecretKey" -}} +{{- $awsSecretManagerSecretAccessKeySecretKey := . -}} +{{- if $awsSecretManagerSecretAccessKeySecretKey -}} + {{- printf "%s" $awsSecretManagerSecretAccessKeySecretKey -}} +{{- else -}} + {{- printf "aws-secret-manager-secret-access-key" -}} +{{- end -}} +{{- end -}} + +{{/* +Get azureKeyVault clientId or default +*/}} +{{- define "airbyte.azureKeyVaultClientIdSecretKey" -}} +{{- $azureKeyVaultClientIdSecretKey := . -}} +{{- if $azureKeyVaultClientIdSecretKey -}} + {{- printf "%s" $azureKeyVaultClientIdSecretKey -}} +{{- else -}} + {{- printf "azure-key-vault-client-id" -}} +{{- end -}} +{{- end -}} + +{{/* +Get azureKeyVault clientSecret or default +*/}} +{{- define "airbyte.azureKeyVaultClientSecretSecretKey" -}} +{{- $azureKeyVaultClientSecretSecretKey := . -}} +{{- if $azureKeyVaultClientSecretSecretKey -}} + {{- printf "%s" $azureKeyVaultClientSecretSecretKey -}} +{{- else -}} + {{- printf "azure-key-vault-client-secret" -}} +{{- end -}} +{{- end -}} + +{{/* +Get googleSecretManager credentials secret key or default +*/}} +{{- define "airbyte.googleSecretManagerCredentialsSecretKey" -}} +{{- $googleSecretManagerCredentialsSecretKey := . -}} +{{- if $googleSecretManagerCredentialsSecretKey -}} + {{- printf "%s" $googleSecretManagerCredentialsSecretKey -}} +{{- else -}} + {{- printf "google-secret-manager-credentials" -}} +{{- end -}} +{{- end -}} + +{{/* +Get vault auth token secret key or default +*/}} +{{- define "airbyte.vaultAuthTokenSecretKey" -}} +{{- $vaultAuthTokenSecretKey := . -}} +{{- if $vaultAuthTokenSecretKey -}} + {{- printf "%s" $vaultAuthTokenSecretKey -}} +{{- else -}} + {{- printf "vault-auth-token" -}} +{{- end -}} +{{- end -}} + + +# Storage Defaults +{{/* +Get storage type or default +*/}} +{{- define "airbyte.storageType" -}} +{{- $storageType := . -}} +{{- if $storageType -}} + {{- printf "%s" $storageType -}} +{{- else -}} + {{- printf "local" -}} +{{- end -}} +{{- end -}} + +{{/* +Get storage bucket log or default +*/}} +{{- define "airbyte.storageBucketLog" -}} +{{- $storageBucketLog := . -}} +{{- if $storageBucketLog -}} + {{- printf "%s" $storageBucketLog -}} +{{- else -}} + {{- printf "airbyte-storage" -}} +{{- end -}} +{{- end -}} + +{{/* +Get storage bucket state or default +*/}} +{{- define "airbyte.storageBucketState" -}} +{{- $storageBucketState := . -}} +{{- if $storageBucketState -}} + {{- printf "%s" $storageBucketState -}} +{{- else -}} + {{- printf "airbyte-storage" -}} +{{- end -}} +{{- end -}} + +{{/* +Get storage bucket workload output or default +*/}} +{{- define "airbyte.storageBucketWorkloadOutput" -}} +{{- $storageBucketWorkloadOutput := . -}} +{{- if $storageBucketWorkloadOutput -}} + {{- printf "%s" $storageBucketWorkloadOutput -}} +{{- else -}} + {{- printf "airbyte-storage" -}} +{{- end -}} +{{- end -}} + +{{/* +Get s3 access key id secret key or default +*/}} +{{- define "airbyte.s3AccessKeyIdSecretKey" -}} +{{- $s3AccessKeyIdSecretKey := . -}} +{{- if $s3AccessKeyIdSecretKey -}} + {{- printf "%s" $s3AccessKeyIdSecretKey -}} +{{- else -}} + {{- printf "s3-access-key-id" -}} +{{- end -}} +{{- end -}} + +{{/* +Get s3 secret access key secret key or default +*/}} +{{- define "airbyte.s3SecretAccessKeySecretKey" -}} +{{- $s3SecretAccessKeySecretKey := . -}} +{{- if $s3SecretAccessKeySecretKey -}} + {{- printf "%s" $s3SecretAccessKeySecretKey -}} +{{- else -}} + {{- printf "s3-secret-access-key" -}} +{{- end -}} +{{- end -}} + +{{/* +Get minio access key id secret key or default +*/}} +{{- define "airbyte.minioAccessKeyIdSecretKey" -}} +{{- $minioAccessKeyIdSecretKey := . -}} +{{- if $minioAccessKeyIdSecretKey -}} + {{- printf "%s" $minioAccessKeyIdSecretKey -}} +{{- else -}} + {{- printf "minio-access-key-id" -}} +{{- end -}} +{{- end -}} + +{{/* +Get minio secret access key secret key or default +*/}} +{{- define "airbyte.minioSecretAccessKeySecretKey" -}} +{{- $minioSecretAccessKeySecretKey := . -}} +{{- if $minioSecretAccessKeySecretKey -}} + {{- printf "%s" $minioSecretAccessKeySecretKey -}} +{{- else -}} + {{- printf "minio-secret-access-key" -}} +{{- end -}} +{{- end -}} + +{{/* +Get gcs credentials secret key or default +*/}} +{{- define "airbyte.gcsCredentialsSecretKey" -}} +{{- $gcsCredentialsSecretKey := . -}} +{{- if $gcsCredentialsSecretKey -}} + {{- printf "%s" $gcsCredentialsSecretKey -}} +{{- else -}} + {{- printf "gcs-credentials" -}} +{{- end -}} +{{- end -}} diff --git a/helm/airbyte-v1/charts/workload-launcher/templates/_images.tpl b/helm/airbyte-v1/charts/workload-launcher/templates/_images.tpl new file mode 100644 index 00000000..6c32ec44 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/templates/_images.tpl @@ -0,0 +1,17 @@ +{{- define "imageUrl" -}} +{{- $img := index . 0 -}} +{{- $root := index . 1 -}} + +{{- $reg := $root.Values.global.image.registry -}} +{{/* ensure the registry has a trailing slash, if set */}} +{{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} + +{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{- if (eq (typeOf $img) "string") -}} +{{- printf "%s%s" $reg (tpl $img $root) | quote -}} +{{- else -}} +{{- $tag := coalesce $img.tag $root.Values.global.image.tag $root.Chart.AppVersion -}} +{{- printf "%s%s:%s" $reg $img.repository $tag | quote -}} +{{- end -}} + +{{- end -}} \ No newline at end of file diff --git a/helm/airbyte-v1/charts/workload-launcher/templates/deployment.yaml b/helm/airbyte-v1/charts/workload-launcher/templates/deployment.yaml new file mode 100644 index 00000000..ce0dfa93 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/templates/deployment.yaml @@ -0,0 +1,557 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.names.fullname" . }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- if .Values.extraLabels }} + {{ toYaml (mergeOverwrite .Values.extraLabels .Values.global.extraLabels) | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "airbyte.selectorLabels" . | nindent 6 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 6 }} + {{- end }} + strategy: + type: Recreate + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + {{- if .Values.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 8 }} + {{- end }} + {{- if .Values.podLabels }} + {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ .Values.global.serviceAccountName }} + automountServiceAccountToken: true + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.extraInitContainers }} + initContainers: + {{- toYaml .Values.extraInitContainers | nindent 6 }} + {{- end }} + containers: + - name: airbyte-workload-launcher-container + image: {{ include "imageUrl" (list .Values.image $) }} + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + env: + {{- if .Values.debug.enabled }} + - name: JAVA_TOOL_OPTIONS + value: "-Xdebug -agentlib:jdwp=transport=dt_socket,address=0.0.0.0:{{ .Values.debug.remoteDebugPort }},server=y,suspend=n" + {{- end}} + {{- if eq .Values.global.deploymentMode "oss" }} + - name: AIRBYTE_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_VERSION + - name: DATA_PLANE_ID + value: "local" + - name: PUB_SUB_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: PUB_SUB_ENABLED + - name: PUB_SUB_TOPIC_NAME + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: PUB_SUB_TOPIC_NAME + - name: CONFIG_ROOT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONFIG_ROOT + {{- if eq (lower (default "" .Values.global.storage.type)) "gcs" }} + - name: CONTAINER_ORCHESTRATOR_SECRET_MOUNT_PATH + value: "/secrets/gcs-log-creds" + - name: CONTAINER_ORCHESTRATOR_SECRET_NAME + value: {{ include "airbyte.secretStoreName" .Values.global.storage.storageSecretName }} + {{- end }} + + {{- include "airbyte.database.envs" . | nindent 8 }} + {{- include "airbyte.logging.envs" . | nindent 8 }} + - name: MICROMETER_METRICS_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MICROMETER_METRICS_ENABLED + - name: MICROMETER_METRICS_STATSD_FLAVOR + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MICROMETER_METRICS_STATSD_FLAVOR + - name: SEGMENT_WRITE_KEY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: SEGMENT_WRITE_KEY + - name: STATSD_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: STATSD_HOST + - name: STATSD_PORT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: STATSD_PORT + - name: TRACKING_STRATEGY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TRACKING_STRATEGY + - name: WORKSPACE_DOCKER_MOUNT + value: workspace + - name: WORKSPACE_ROOT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKSPACE_ROOT + - name: WEBAPP_URL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WEBAPP_URL + - name: WORKLOAD_API_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKLOAD_API_HOST + - name: TEMPORAL_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TEMPORAL_HOST + - name: TEMPORAL_WORKER_PORTS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TEMPORAL_WORKER_PORTS + - name: JOB_KUBE_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: JOB_KUBE_SERVICEACCOUNT + value: {{ .Values.global.serviceAccountName }} + {{- if $.Values.global.jobs.kube.annotations }} + - name: JOB_KUBE_ANNOTATIONS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_KUBE_ANNOTATIONS + {{- end }} + {{- if $.Values.global.jobs.kube.labels }} + - name: JOB_KUBE_LABELS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_KUBE_LABELS + {{- end }} + {{- if $.Values.global.jobs.kube.nodeSelector }} + - name: JOB_KUBE_NODE_SELECTORS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_KUBE_NODE_SELECTORS + {{- end }} + {{- if $.Values.global.jobs.kube.tolerations }} + - name: JOB_KUBE_TOLERATIONS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_KUBE_TOLERATIONS + {{- end }} + - name: CONNECTOR_SIDECAR_IMAGE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONNECTOR_SIDECAR_IMAGE + - name: WORKLOAD_INIT_IMAGE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKLOAD_INIT_IMAGE + - name: CONTAINER_ORCHESTRATOR_IMAGE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONTAINER_ORCHESTRATOR_IMAGE + - name: JOB_KUBE_BUSYBOX_IMAGE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_KUBE_BUSYBOX_IMAGE + - name: JOB_KUBE_CURL_IMAGE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_KUBE_CURL_IMAGE + {{- if $.Values.global.jobs.kube.main_container_image_pull_secret }} + - name: JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET + {{- end }} + - name: JOB_MAIN_CONTAINER_CPU_REQUEST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_MAIN_CONTAINER_CPU_REQUEST + - name: JOB_MAIN_CONTAINER_CPU_LIMIT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_MAIN_CONTAINER_CPU_LIMIT + - name: JOB_MAIN_CONTAINER_MEMORY_REQUEST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_MAIN_CONTAINER_MEMORY_REQUEST + - name: JOB_MAIN_CONTAINER_MEMORY_LIMIT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_MAIN_CONTAINER_MEMORY_LIMIT + - name: JOB_KUBE_LOCAL_VOLUME_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_KUBE_LOCAL_VOLUME_ENABLED + - name: KUBERNETES_CLIENT_MAX_IDLE_CONNECTIONS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: KUBERNETES_CLIENT_MAX_IDLE_CONNECTIONS + - name: WORKLOAD_API_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: {{ index .Values "workloadApi" "bearerTokenSecretName" | default (printf "%s-airbyte-secrets" .Release.Name ) }} + key: {{ index .Values "workloadApi" "bearerTokenSecretKey" | default "WORKLOAD_API_BEARER_TOKEN" }} + - name: WORKLOAD_API_BEARER_TOKEN_SECRET_NAME + value: {{ index .Values "workloadApi" "bearerTokenSecretName" | default (printf "%s-airbyte-secrets" .Release.Name ) }} + - name: WORKLOAD_API_BEARER_TOKEN_SECRET_KEY + value: {{ index .Values "workloadApi" "bearerTokenSecretKey" | default "WORKLOAD_API_BEARER_TOKEN" }} + - name: INTERNAL_API_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: INTERNAL_API_HOST + - name: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + - name: JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + - name: METRIC_CLIENT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: METRIC_CLIENT + - name: OTEL_COLLECTOR_ENDPOINT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: OTEL_COLLECTOR_ENDPOINT + - name: ACTIVITY_MAX_ATTEMPT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: ACTIVITY_MAX_ATTEMPT + - name: ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS + - name: ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS + - name: WORKFLOW_FAILURE_RESTART_DELAY_SECONDS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKFLOW_FAILURE_RESTART_DELAY_SECONDS + - name: SHOULD_RUN_NOTIFY_WORKFLOWS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: SHOULD_RUN_NOTIFY_WORKFLOWS + - name: MICRONAUT_ENVIRONMENTS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: LAUNCHER_MICRONAUT_ENVIRONMENTS + - name: WORKLOAD_LAUNCHER_PARALLELISM + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKLOAD_LAUNCHER_PARALLELISM + - name: FILE_TRANSFER_EPHEMERAL_STORAGE_LIMIT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: FILE_TRANSFER_EPHEMERAL_STORAGE_LIMIT + - name: FILE_TRANSFER_EPHEMERAL_STORAGE_REQUEST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: FILE_TRANSFER_EPHEMERAL_STORAGE_REQUEST + {{- if or (eq .Values.global.edition "pro") (eq .Values.global.edition "enterprise") }} + - name: AIRBYTE_API_AUTH_HEADER_NAME + value: "X-Airbyte-Auth" + - name: AIRBYTE_API_AUTH_HEADER_VALUE + value: "Internal worker" + {{- else if and (eq .Values.global.deploymentMode "oss") .Values.global.auth.enabled }} + # Self-Managed Enterprise and Community w/ auth enabled use the same auth header, just + # splitting into two separate blocks for readability. + - name: AIRBYTE_API_AUTH_HEADER_NAME + value: "X-Airbyte-Auth" + - name: AIRBYTE_API_AUTH_HEADER_VALUE + value: "Internal worker" + {{- end }} + + # SECRETS MANAGER + - name: SECRET_PERSISTENCE + value: {{ include "airbyte.secretPersistence" . }} + # Values for AwsSecretsManager + {{- if eq ((((.Values.global).secretsManager).awsSecretManager).authenticationType) "credentials" }} + - name: AWS_SECRET_MANAGER_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }} + key: {{ include "airbyte.awsSecretManagerAccessKeyIdSecretKey" .Values.global.secretsManager.awsSecretManager.accessKeyIdSecretKey }} + - name: AWS_SECRET_MANAGER_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }} + key: {{ include "airbyte.awsSecretManagerSecretAccessKeySecretKey" .Values.global.secretsManager.awsSecretManager.secretAccessKeySecretKey }} + + - name: AWS_SECRET_MANAGER_ACCESS_KEY_ID_REF_NAME + value: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }} + - name: AWS_SECRET_MANAGER_ACCESS_KEY_ID_REF_KEY + value: {{ include "airbyte.awsSecretManagerAccessKeyIdSecretKey" .Values.global.secretsManager.awsSecretManager.accessKeyIdSecretKey }} + + - name: AWS_SECRET_MANAGER_SECRET_ACCESS_KEY_REF_NAME + value: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }} + - name: AWS_SECRET_MANAGER_SECRET_ACCESS_KEY_REF_KEY + value: {{ include "airbyte.awsSecretManagerSecretAccessKeySecretKey" .Values.global.secretsManager.awsSecretManager.secretAccessKeySecretKey }} + + {{- end }} + {{- if ((((.Values.global).secretsManager).awsSecretManager).region) }} + - name: AWS_SECRET_MANAGER_REGION + value: {{ (((.Values.global).secretsManager).awsSecretManager).region }} + {{- end }} + + {{- if ((((.Values.global).secretsManager).awsSecretManager).tags) }} + - name: AWS_SECRET_MANAGER_SECRET_TAGS + value: {{ include "airbyte.tagsToString" .Values.global.secretsManager.awsSecretManager.tags }} + {{- end }} + {{- if ((((.Values.global).secretsManager).awsSecretManager).kms) }} + - name: AWS_KMS_KEY_ARN + value: {{ ((((.Values.global).secretsManager).awsSecretManager).kms) | default "" }} + {{- end }} + + # Values for Azure Key Vault + {{- if (((.Values.global).secretsManager).azureKeyVault) }} + + - name: AB_AZURE_KEY_VAULT_VAULT_URL + value: {{ (((.Values.global).secretsManager).azureKeyVault).vaultUrl }} + + - name: AB_AZURE_KEY_VAULT_TENANT_ID + value: {{ (((.Values.global).secretsManager).azureKeyVault).tenantId }} + + - name: AB_AZURE_KEY_VAULT_CLIENT_ID + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }} + key: {{ include "airbyte.azureKeyVaultClientIdSecretKey" .Values.global.secretsManager.azureKeyVault.clientIdSecretKey }} + - name: AB_AZURE_KEY_VAULT_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }} + key: {{ include "airbyte.azureKeyVaultClientSecretSecretKey" .Values.global.secretsManager.azureKeyVault.clientSecretSecretKey }} + + - name: AB_AZURE_KEY_VAULT_CLIENT_ID_REF_NAME + value: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }} + - name: AB_AZURE_KEY_VAULT_CLIENT_ID_REF_REF_KEY + value: {{ include "airbyte.azureKeyVaultClientIdSecretKey" .Values.global.secretsManager.azureKeyVault.clientIdSecretKey }} + + - name: AB_AZURE_KEY_VAULT_CLIENT_SECRET_REF_REF_NAME + value: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }} + - name: AB_AZURE_KEY_VAULT_CLIENT_SECRET_REF_REF_KEY + value: {{ include "airbyte.azureKeyVaultClientSecretSecretKey" .Values.global.secretsManager.azureKeyVault.clientSecretSecretKey }} + + {{- end }} + + {{- if ((((.Values.global).secretsManager).azureKeyVault).tags) }} + - name: AB_AZURE_KEY_VAULT_TAGS + value: {{ include "airbyte.tagsToString" .Values.global.secretsManager.azureKeyVault.tags }} + {{- end }} + + # Values for googleSecretManager secrets + {{- if (((.Values.global).secretsManager).googleSecretManager) }} + - name: SECRET_STORE_GCP_PROJECT_ID + value: {{ .Values.global.secretsManager.googleSecretManager.projectId }} + - name: SECRET_STORE_GCP_CREDENTIALS + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }} + key: {{ include "airbyte.googleSecretManagerCredentialsSecretKey" .Values.global.secretsManager.googleSecretManager.credentialsSecretKey }} + - name: SECRET_STORE_GCP_SECRET_NAME + value: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }} + - name: SECRET_STORE_GCP_SECRET_KEY + value: {{ include "airbyte.googleSecretManagerCredentialsSecretKey" .Values.global.secretsManager.googleSecretManager.credentialsSecretKey }} + {{- end }} + + # Values for vault secrets + {{- if (((.Values.global).secretsManager).vault) }} + - name: VAULT_ADDRESS + value: {{ (((.Values.global).secretsManager).vault).address }} + - name: VAULT_PREFIX + value: {{ (((.Values.global).secretsManager).vault).prefix }} + - name: VAULT_AUTH_TOKEN + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }} + key: {{ include "airbyte.vaultAuthTokenSecretKey" .Values.global.secretsManager.vault.authTokenSecretKey }} + - name: VAULT_AUTH_TOKEN_REF_NAME + value: {{ include "airbyte.secretStoreName" .Values.global.secretsManager.secretsManagerSecretName }} + - name: VAULT_AUTH_TOKEN_REF_KEY + value: {{ include "airbyte.vaultAuthTokenSecretKey" .Values.global.secretsManager.vault.authTokenSecretKey }} + {{- end }} + + # Storage + {{- include "airbyte.storage.envs" . | nindent 8 }} + + {{- end }} + + # Values from secret + {{- if .Values.secrets }} + {{- range $k, $v := .Values.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: workload-launcher-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if or .Values.env_vars .Values.global.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + # Values from extraEnv for more compability(if you want to use external secret source or other stuff) + {{- if .Values.extraEnv }} + {{- toYaml .Values.extraEnv | nindent 8 }} + {{- end }} + + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /health/liveness + port: heartbeat + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /health/readiness + port: heartbeat + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + ports: + {{- if .Values.debug.enabled }} + - name: debug + containerPort: {{ .Values.debug.remoteDebugPort }} + protocol: TCP + {{- end}} + - name: heartbeat + containerPort: 8016 # for heartbeat server + {{- if .Values.resources }} + resources: {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + {{- if .Values.containerSecurityContext }} + securityContext: {{- toYaml .Values.containerSecurityContext | nindent 10 }} + {{- end }} + volumeMounts: + {{- if eq (lower .Values.global.storage.type) "local" }} + - name: airbyte-storage + mountPath: /storage + {{- end }} + {{- if and (eq .Values.global.deploymentMode "oss") (eq (lower (default "" .Values.global.storage.type)) "gcs") }} + - name: gcs-log-creds-volume + mountPath: /secrets/gcs-log-creds + readOnly: true + {{- end }} + {{- if .Values.extraVolumeMounts }} +{{- toYaml .Values.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.global.extraVolumeMounts }} +{{- toYaml .Values.global.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.extraContainers }} + {{ toYaml .Values.extraContainers | nindent 6 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 6 }} + {{- end }} + securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + volumes: + {{- if eq (lower .Values.global.storage.type) "local" }} + - name: airbyte-storage + persistentVolumeClaim: + claimName: airbyte-storage-pvc + {{- end }} + {{- if and (eq .Values.global.deploymentMode "oss") (eq (lower (default "" .Values.global.storage.type)) "gcs") }} + - name: gcs-log-creds-volume + secret: + secretName: {{ ternary (printf "%s-gcs-log-creds" ( .Release.Name )) .Values.global.storage.storageSecretName (not ((.Values.global.storage).storageSecretName)) }} + {{- end }} + {{- if .Values.extraVolumes }} +{{- toYaml .Values.extraVolumes | nindent 6 }} + {{- end }} + {{- if .Values.global.extraVolumes }} +{{- toYaml .Values.global.extraVolumes | nindent 6 }} + {{- end }} diff --git a/helm/airbyte-v1/charts/workload-launcher/templates/jobs-secrets.yaml b/helm/airbyte-v1/charts/workload-launcher/templates/jobs-secrets.yaml new file mode 100644 index 00000000..c49cec64 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/templates/jobs-secrets.yaml @@ -0,0 +1,18 @@ +# Create dataplane secrets only for the local deployment + {{- if .Values.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: workload-launcher-secrets + namespace: {{ .Values.jobs_namespace }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" +type: Opaque +data: + {{- range $k, $v := mergeOverwrite .Values.secrets .Values.global.secrets }} + {{ $k }}: {{ if $v }}{{ $v | b64enc }} {{else}}""{{end}} + {{- end }} + {{- end }} diff --git a/helm/airbyte-v1/charts/workload-launcher/templates/secrets.yaml b/helm/airbyte-v1/charts/workload-launcher/templates/secrets.yaml new file mode 100644 index 00000000..c72f6bfe --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/templates/secrets.yaml @@ -0,0 +1,17 @@ +# Create secrets only for the local deployment +{{- if .Values.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: workload-launcher-secrets + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" +type: Opaque +data: + {{- range $k, $v := mergeOverwrite .Values.secrets .Values.global.secrets }} + {{ $k }}: {{ if $v }}{{ $v | b64enc }} {{else}}""{{end}} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v1/charts/workload-launcher/values.yaml b/helm/airbyte-v1/charts/workload-launcher/values.yaml new file mode 100644 index 00000000..486760a5 --- /dev/null +++ b/helm/airbyte-v1/charts/workload-launcher/values.yaml @@ -0,0 +1,294 @@ +global: + + serviceAccountName: &service-account-name "airbyte-admin" + edition: community + credVolumeOverride: "" + secretName: "" + extraContainers: [] + ## extraSelectorLabels [object] - use to specify own additional selector labels for deployment + extraSelectorLabels: {} + ## extraLabels [object] - use to specify own additional labels for deployment + extraLabels: {} + database: + secretName: "" + secretValue: "" + + storage: + type: minio + + # Docker image config that will apply to all images. + image: + # Docker registry to pull platform images from, e.g. http://my-registry:8000/ + registry: "" + # Image tag to use for airbyte images. + # Does not include non-airbyte images such as busybox, temporal, minio, etc. + tag: "" + + jobs: + ## Jobs resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + ## jobs.resources.limits [object] The resources limits for jobs + ## jobs.resources.requests [object] The requested resources for jobs + resources: + ## Example: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + + kube: + ## JOB_KUBE_ANNOTATIONS + ## pod annotations of the sync job and the default pod annotations fallback for others jobs + ## jobs.kube.annotations [object] key/value annotations applied to kube jobs + annotations: {} + + ## JOB_KUBE_LABELS + ## pod labels of the sync job and the default pod labels fallback for others jobs + ## jobs.kube.labels [object] key/value labels applied to kube jobs + labels: {} + + ## JOB_KUBE_NODE_SELECTORS + ## pod node selector of the sync job and the default pod node selector fallback for others jobs + ## jobs.kube.nodeSelector [object] key/value node selector applied to kube jobs + nodeSelector: {} + + ## JOB_KUBE_TOLERATIONS + ## jobs.kube.tolerations [array] Tolerations for jobs.kube pod assignment. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## any boolean values should be quoted to ensure the value is passed through as a string, eg: + ## - key: airbyte-server + ## operator: Equal + ## value: "true" + ## effect: NoSchedule + tolerations: [] + + ## JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET + ## image pull secret to use for job pod + ## jobs.kube.main_container_image_pull_secret [string] image pull secret to use for job pod + main_container_image_pull_secret: "" + + images: + ## JOB_KUBE_BUSYBOX_IMAGE + ## busybox image used by the job pod + ## jobs.kube.images.busybox [string] busybox image used by the job pod + busybox: "" + ## JOB_KUBE_CURL_IMAGE + ## curl image used by the job pod + ## jobs.kube.images.curl [string] curl image used by the job pod + curl: "" + +enabled: true +## workload-launcher.replicaCount Number of workload launcher replicas +replicaCount: 1 + +## workload-launcher.image.repository The repository to use for the airbyte workload launcher image. +## workload-launcher.image.pullPolicy the pull policy to use for the airbyte workload launcher image +## workload-launcher.image.tag The airbyte workload launcher image tag. Defaults to the chart's AppVersion +image: + repository: airbyte/workload-launcher + pullPolicy: IfNotPresent + +## workload-launcher.podAnnotations [object] Add extra annotations to the workload launcher pod(s) +## +podAnnotations: {} + +## workload-launcher.podLabels [object] Add extra labels to the workload launcher pod(s) +## +podLabels: {} + +## workload-launcher.containerSecurityContext Security context for the container +## Examples: +## containerSecurityContext: +## runAsNonRoot: true +## runAsUser: 1000 +## readOnlyRootFilesystem: true +containerSecurityContext: {} + +## Configure extra options for the workload launcher containers' liveness and readiness probes +## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes +## workload-launcher.livenessProbe.enabled Enable livenessProbe on the workload launcher +## workload-launcher.livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe +## workload-launcher.livenessProbe.periodSeconds Period seconds for livenessProbe +## workload-launcher.livenessProbe.timeoutSeconds Timeout seconds for livenessProbe +## workload-launcher.livenessProbe.failureThreshold Failure threshold for livenessProbe +## workload-launcher.livenessProbe.successThreshold Success threshold for livenessProbe +## +livenessProbe: + enabled: true + initialDelaySeconds: 50 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + +## workload-launcher.readinessProbe.enabled Enable readinessProbe on the workload launcher +## workload-launcher.readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe +## workload-launcher.readinessProbe.periodSeconds Period seconds for readinessProbe +## workload-launcher.readinessProbe.timeoutSeconds Timeout seconds for readinessProbe +## workload-launcher.readinessProbe.failureThreshold Failure threshold for readinessProbe +## workload-launcher.readinessProbe.successThreshold Success threshold for readinessProbe +## +readinessProbe: + enabled: true + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + +## workload launcher resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## We usually recommend not to specify default resources and to leave this as a conscious +## choice for the user. This also increases chances charts run on environments with little +## resources, such as Minikube. If you do want to specify resources, uncomment the following +## lines, adjust them as necessary, and remove the curly braces after 'resources:'. +## workload-launcher.resources.limits [object] The resources limits for the workload launcher container +## workload-launcher.resources.requests [object] The requested resources for the workload launcher container +resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + +## workload-launcher.nodeSelector [object] Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## workload-launcher.tolerations [array] Tolerations for workload launcher pod assignment. +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## workload-launcher.affinity [object] Affinity and anti-affinity for workload launcher pod assignment. +## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## workload-launcher.log.level The log level to log at. +log: + level: "INFO" + + + +## workload-launcher.extraVolumeMounts [array] Additional volumeMounts for workload launcher container(s). +## Examples (when using `workload-launcher.containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumeMounts: +## - name: tmpdir +## mountPath: /tmp +## +extraVolumeMounts: [] + +## workload-launcher.extraVolumes [array] Additional volumes for workload launcher pod(s). +## Examples (when using `workload-launcher.containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumes: +## - name: tmpdir +## emptyDir: {} +## +extraVolumes: [] + +## extraContainer [array] Additional container for server pod(s) +## Example: +# extraContainers: +# - name: otel_collector +# image: somerepo/someimage:sometag +# args: [ +# "--important-args" +# ] +# ports: +# - containerPort: 443 +# volumeMounts: +# - name: volumeMountCool +# mountPath: /some/path +# readOnly: true +extraContainers: [] + +## extraInitContainers [array] Additional init containers for server pod(s) +## Example: +# extraInitContainers: +# - name: sleepy +# image: alpine +# command: ['sleep', '60'] + +extraInitContainers: [] + +## extraEnv [array] Supply extra env variables to main container using full notation +## Example: (With default env vars and values taken from generated config map) +# extraEnv: +# - name: AIRBYTE_VERSION +# valueFrom: +# configMapKeyRef: +# name: airbyte-env +# key: AIRBYTE_VERSION +# - name: API_URL +# valueFrom: +# configMapKeyRef: +# name: airbyte-env +# key: API_URL +# - name: TRACKING_STRATEGY +# valueFrom: +# configMapKeyRef: +# name: airbyte-env +# key: TRACKING_STRATEGY +# - name: FULLSTORY +# valueFrom: +# configMapKeyRef: +# name: airbyte-env +# key: FULLSTORY +# - name: INTERNAL_API_HOST +# valueFrom: +# configMapKeyRef: +# name: airbyte-env +# key: INTERNAL_API_HOST +## +extraEnv: [] +## secrets [object] Supply additional secrets to container +## Example: +## secrets: +## DATABASE_PASSWORD: strong-password +## DATABASE_USER: my-db-user +secrets: {} + +## env_vars [object] Supply extra env variables to main container using simplified notation +## Example with default values for workload launcher: +# env_vars: +# AIRBYTE_VERSION: 0.40.4 + +# # Workspace storage for running jobs (logs, etc) +# WORKSPACE_ROOT: /workspace +# WORKSPACE_DOCKER_MOUNT: airbyte_workspace + +# # Miscellaneous +# TRACKING_STRATEGY: segment +# WEBAPP_URL: airbyte-webapp-svc:80 +# INTERNAL_API_HOST: http://airbyte-server-svc:8001 +env_vars: {} + +## extraSelectorLabels [object] - use to specify own additional selector labels for deployment +extraSelectorLabels: {} +## extraLabels [object] - use to specify own additional labels for deployment +extraLabels: {} + +## workloadApi: +## bearerTokenSecretName: workload-api-bearer-token-name kubernetes secret name where bearer token is stored +## bearerTokenSecretKey: workload-api-bearer-token-key kubernetes secret key where bearer token is stored +workloadApi: {} + +debug: + enabled: false + remoteDebugPort: 5005 diff --git a/helm/airbyte-v1/templates/NOTES.txt b/helm/airbyte-v1/templates/NOTES.txt new file mode 100644 index 00000000..6487ae65 --- /dev/null +++ b/helm/airbyte-v1/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.webapp.ingress.enabled }} +{{- range $host := .Values.webapp.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.webapp.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.webapp.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "airbyte.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.webapp.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "airbyte.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "airbyte.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.webapp.service.port }} +{{- else if contains "ClusterIP" .Values.webapp.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name=webapp" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/helm/airbyte-v1/templates/_database.tpl b/helm/airbyte-v1/templates/_database.tpl new file mode 100644 index 00000000..3e8d567d --- /dev/null +++ b/helm/airbyte-v1/templates/_database.tpl @@ -0,0 +1,234 @@ +{{/* +Database Configuration +*/}} + +{{/* +Renders the database host +*/}} +{{- define "airbyte.database.host" }} + {{- if .Values.postgresql.enabled }} + {{- printf "%s" "airbyte-db-svc" }} + {{- else if .Values.global.database.host }} + {{- .Values.global.database.host }} + {{- else }} + {{ $host := .Values.global.database.host | required "You must set `global.database.host` when using an external database" }} + {{- end }} +{{- end }} + +{{/* +Renders an environment variable definition that provides the database host +*/}} +{{- define "airbyte.database.host.env" }} +- name: DATABASE_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATABASE_HOST +{{- end }} + +{{/* +Renders the database port +*/}} +{{- define "airbyte.database.port" }} + {{- if .Values.postgresql.enabled }} + {{- printf "%s" "5432" }} + {{- else if .Values.global.database.port }} + {{- .Values.global.database.port }} + {{- else }} + {{ $port := .Values.global.database.port | required "You must set `global.database.port` when using an external database" }} + {{- end }} +{{- end }} + +{{/* +Renders an environment variable definition that provides the database port +*/}} +{{- define "airbyte.database.port.env" }} +- name: DATABASE_PORT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATABASE_PORT +{{- end }} + +{{/* +Renders the database name +*/}} +{{- define "airbyte.database.name" }} + {{- if .Values.postgresql.enabled }} + {{- .Values.postgresql.postgresqlDatabase }} + {{- else if .Values.global.database.database }} + {{- .Values.global.database.database }} + {{- else }} + {{ $database := .Values.global.database.database | required "You must set `global.database.database` when using an external database" }} + {{- end }} +{{- end }} + +{{/* +Renders an environment variable definition that provides the database name +*/}} +{{- define "airbyte.database.name.env" }} +- name: DATABASE_DB + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATABASE_DB +{{- end }} + +{{/* +Renders the database user +*/}} +{{- define "airbyte.database.user" }} + {{- if .Values.postgresql.enabled }} + {{- .Values.postgresql.postgresqlUsername }} + {{- else if .Values.global.database.user }} + {{- .Values.global.database.user }} + {{- else }} + {{- if .Values.global.database.userSecretKey }} + {{ $secretName := .Values.global.database.secretName | required "You must set `global.database.secretName` when using an external database" }} + {{- else }} + {{ $user := .Values.global.database.user | required "You must set `global.database.user` when using an external database" }} + {{- end }} + {{- end }} +{{- end }} + +{{/* +Renders the name of the secret where the database user will be referenced +*/}} +{{- define "airbyte.database.userSecretKey" }} + {{- if .Values.global.database.userSecretKey }} + {{ $secretName := .Values.global.database.secretName | required "You must set `global.database.secretName` when using an external database" }} + {{- .Values.global.database.userSecretKey }} + {{- else }} + {{- printf "%s" "DATABASE_USER" }} + {{- end }} +{{- end }} + +{{/* +Renders an environment variable definition that provides the database user +*/}} +{{- define "airbyte.database.user.env" }} +- name: DATABASE_USER + valueFrom: + secretKeyRef: + {{- if .Values.global.database.userSecretKey }} + name: {{ .Values.global.database.secretName }} + {{- else }} + name: {{ .Release.Name }}-airbyte-secrets + {{- end }} + key: {{ include "airbyte.database.userSecretKey" .}} +{{- end }} + +{{/* +Renders the database password +*/}} +{{- define "airbyte.database.password" }} + {{- if .Values.postgresql.enabled }} + {{- .Values.postgresql.postgresqlPassword }} + {{- else if .Values.global.database.password }} + {{- .Values.global.database.password }} + {{- else }} + {{- if .Values.global.database.passwordSecretKey }} + {{ $secretName := .Values.global.database.secretName | required "You must set `global.database.secretName` when using an external database" }} + {{- else }} + {{ $password := .Values.global.database.password | required "You must set `global.database.password` when using an external database" }} + {{- end }} + {{- end }} +{{- end }} + +{{/* +Renders the name of the secret where the database password will be referenced +*/}} +{{- define "airbyte.database.passwordSecretKey" }} + {{- if .Values.global.database.passwordSecretKey }} + {{ $secretName := .Values.global.database.secretName | required "You must set `global.database.secretName` when using an external database" }} + {{- .Values.global.database.passwordSecretKey }} + {{- else }} + {{- printf "%s" "DATABASE_PASSWORD" }} + {{- end }} +{{- end }} + +{{/* +Renders an environment variable definition that provides the database password +*/}} +{{- define "airbyte.database.password.env" }} +- name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "airbyte.database.secretName" . }} + key: {{ include "airbyte.database.passwordSecretKey" . }} +{{- end }} + + +{{/* +Renders the database url (JDBC) +*/}} +{{- define "airbyte.database.url" }} +{{- $host := (include "airbyte.database.host" .) }} +{{- $dbName := (include "airbyte.database.name" .) }} +{{- $port := (include "airbyte.database.port" . ) }} +{{- printf "jdbc:postgresql://%s:%s/%s" $host $port $dbName }} +{{- end }} + +{{/* +Renders an environment variable definition that provides the database url (JDBC) +*/}} +{{- define "airbyte.database.url.env" }} +- name: DATABASE_URL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATABASE_URL +{{- end }} + +{{/* +Renders the name of the secret containing database credentials +*/}} +{{- define "airbyte.database.secretName" }} + {{- if .Values.global.database.secretName }} + {{- .Values.global.database.secretName | quote }} + {{- else }} + {{ .Release.Name }}-airbyte-secrets + {{- end }} +{{- end }} + +{{/* +Renders all of the common environment variables which provide database credentials +*/}} +{{- define "airbyte.database.envs" }} +{{ include "airbyte.database.host.env" . }} +{{ include "airbyte.database.port.env" . }} +{{ include "airbyte.database.name.env" . }} +{{ include "airbyte.database.user.env" . }} +{{ include "airbyte.database.password.env" . }} +{{ include "airbyte.database.url.env" . }} +{{- end }} + +{{/* +Renders a set of database secrets to be included in the shared Airbyte secret +*/}} +{{- define "airbyte.database.secrets" }} +{{ $user := (include "airbyte.database.user" .)}} +{{- if not (empty $user) }} +DATABASE_USER: {{ $user }} +{{- end }} +{{ $password := (include "airbyte.database.password" .)}} +{{- if not (empty $password) }} +DATABASE_PASSWORD: {{ $password }} +{{- end}} +{{- end }} + +{{/* +Renders a set of database configuration variables to be included in the shared Airbyte config map +*/}} +{{- define "airbyte.database.configVars" }} +DATABASE_HOST: {{ include "airbyte.database.host" . }} +DATABASE_PORT: {{ include "airbyte.database.port" . | quote }} +DATABASE_DB: {{ include "airbyte.database.name" . }} +DATABASE_URL: {{ include "airbyte.database.url" . }} +{{- if .Values.global.database.user }} +DATABASE_USER: {{ include "airbyte.database.user" . }} +{{- end}} +{{- if .Values.global.database.password }} +DATABASE_PASSWORD: {{ include "airbyte.database.password" . }} +{{- end}} +{{- end }} diff --git a/helm/airbyte-v1/templates/_enterprise.tpl b/helm/airbyte-v1/templates/_enterprise.tpl new file mode 100644 index 00000000..8e74d5aa --- /dev/null +++ b/helm/airbyte-v1/templates/_enterprise.tpl @@ -0,0 +1,83 @@ +{{/* +Enteprise Configuration +*/}} + +{{- define "airbyte.enterprise.license" -}} +{{- if and (or (eq .Values.global.edition "pro") (eq .Values.global.edition "enterprise")) (not .Values.global.airbyteYml) }} +{{- $secretName := .Values.global.enterprise.secretName | required "You must set `global.enterprise.secretName` when `global.edition` is 'enterprise'" }} +{{- $secretKey := .Values.global.enterprise.licenseKeySecretKey | required "You must set `global.enterprise.licenseKeySecretKey` when `global.edition` is 'enterprise'" }} +- name: AIRBYTE_LICENSE_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.global.enterprise.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} + key: {{ .Values.global.enterprise.licenseKeySecretKey }} +{{- end }} +{{- end }} + +{{- define "airbyte.enterprise.instanceAdmin" -}} +{{- if and (or (eq .Values.global.edition "pro") (eq .Values.global.edition "enterprise")) (not .Values.global.airbyteYml) }} +{{- $auth := .Values.global.auth | required "You must set `global.auth` when `global.edition` is 'enterprise'"}} +{{- $authInstanceAdminSecretName := .Values.global.auth.instanceAdmin.secretName | required "You must set `global.auth.instanceAdmin.secretName` when `global.edition` is 'enterprise'" }} +{{- $authInstanceAdminFirstName := .Values.global.auth.instanceAdmin.firstName | required "You must set `global.auth.instanceAdmin.firstName` when `global.edition` is 'enterprise'" }} +{{- $authInstanceAdminLastName := .Values.global.auth.instanceAdmin.lastName | required "You must set `global.auth.instanceAdmin.lastName` when `global.edition` is 'enterprise'" }} +{{- $authInstanceAdminEmailSecretKey := .Values.global.auth.instanceAdmin.emailSecretKey | required "You must set `global.auth.instanceAdmin.emailSecretKey` when `global.edition` is 'enterprise'" }} +{{- $authInstanceAdminPasswordSecretKey := .Values.global.auth.instanceAdmin.passwordSecretKey | required "You must set `global.auth.instanceAdmin.passwordSecretKey` when `global.edition` is 'enterprise'" }} +- name: INITIAL_USER_FIRST_NAME + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: INITIAL_USER_FIRST_NAME +- name: INITIAL_USER_LAST_NAME + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: INITIAL_USER_LAST_NAME +- name: INITIAL_USER_EMAIL + valueFrom: + secretKeyRef: + name: {{ .Values.global.auth.instanceAdmin.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} + key: {{ .Values.global.auth.instanceAdmin.emailSecretKey }} +- name: INITIAL_USER_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.auth.instanceAdmin.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} + key: {{ .Values.global.auth.instanceAdmin.passwordSecretKey }} +{{- end }} +{{- end }} + +{{- define "airbyte.enterprise.identityProvider" -}} +{{- if and (or (eq .Values.global.edition "pro") (eq .Values.global.edition "enterprise")) (not .Values.global.airbyteYml) .Values.global.auth.identityProvider }} +{{- $authIdentityProviderSecretName := .Values.global.auth.identityProvider.secretName | required "You must set `global.auth.identityProvider.secretName` when enabling SSO" }} +{{- $authIdentityProviderType := .Values.global.auth.identityProvider.type | required "You must set `global.auth.identityProvider.type` when enabling SSO "}} +{{- $authIdentityProviderOIDC := .Values.global.auth.identityProvider.oidc | required "You must set `global.auth.identityProvider.oidc` when enabling SSO" }} +{{- $authIdentityProviderOIDCDomain := .Values.global.auth.identityProvider.oidc.domain | required "You must set `global.auth.identityProvider.oidc.domain` when enabling SSO" }} +{{- $authIdentityProviderOIDCAppName := .Values.global.auth.identityProvider.oidc.appName | required "You must set `global.auth.identityProvider.oidc.appName` when enabling SSO" }} +{{- $authIdentityProviderOIDCClientIdSecretKey := .Values.global.auth.identityProvider.oidc.clientIdSecretKey | required "You must set `global.auth.identityProvider.oidc.clientIdSecretKey` when enabling SSO" }} +{{- $authIdentityProviderOIDCClientSecretSecretKey := .Values.global.auth.identityProvider.oidc.clientSecretSecretKey | required "You must set `global.auth.identityProvider.oidc.clientSecretSecretKey` when enabling SSO" }} +- name: IDENTITY_PROVIDER_TYPE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: IDENTITY_PROVIDER_TYPE +- name: OIDC_DOMAIN + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: OIDC_DOMAIN +- name: OIDC_APP_NAME + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: OIDC_APP_NAME +- name: OIDC_CLIENT_ID + valueFrom: + secretKeyRef: + name: {{ .Values.global.auth.identityProvider.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} + key: {{ .Values.global.auth.identityProvider.oidc.clientIdSecretKey }} +- name: OIDC_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ .Values.global.auth.identityProvider.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} + key: {{ .Values.global.auth.identityProvider.oidc.clientSecretSecretKey }} +{{- end }} +{{- end }} diff --git a/helm/airbyte-v1/templates/_helpers.tpl b/helm/airbyte-v1/templates/_helpers.tpl new file mode 100644 index 00000000..d99db7f4 --- /dev/null +++ b/helm/airbyte-v1/templates/_helpers.tpl @@ -0,0 +1,350 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "airbyte.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airbyte.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airbyte.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "airbyte.labels" -}} +helm.sh/chart: {{ include "airbyte.chart" . }} +{{ include "airbyte.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "airbyte.selectorLabels" -}} +app.kubernetes.io/name: {{ include "airbyte.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* +Common DB labels +*/}} +{{- define "airbyte.databaseLabels" -}} +helm.sh/chart: {{ include "airbyte.chart" . }} +{{ include "airbyte.databaseSelectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector DB labels +*/}} +{{- define "airbyte.databaseSelectorLabels" -}} +app.kubernetes.io/name: {{ printf "%s-db" .Release.Name }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Common DB labels +*/}} +{{- define "airbyte.minioLabels" -}} +helm.sh/chart: {{ include "airbyte.chart" . }} +{{ include "airbyte.minioSelectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector DB labels +*/}} +{{- define "airbyte.minioSelectorLabels" -}} +app.kubernetes.io/name: {{ printf "%s-minio" .Release.Name }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "airbyte.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "airbyte.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Add environment variables to configure minio +*/}} +{{- define "airbyte.storage.minio.endpoint" -}} +{{- if ((((.Values.global).storage).minio).endpoint) }} + {{- .Values.global.storage.minio.endpoint -}} +{{- else -}} + {{- printf "http://airbyte-minio-svc:9000" -}} +{{- end -}} +{{- end -}} + +{{- define "airbyte.s3PathStyleAccess" -}} +{{- ternary "true" "" (eq (lower (default "" .Values.global.storage.type)) "minio") -}} +{{- end -}} + +{{/* +Returns the GCP credentials path +*/}} +{{- define "airbyte.gcpLogCredentialsPath" -}} +{{- if ((((.Values.global).storage).gcs).credentialsPath) }} + {{- printf "%s" .Values.global.storage.gcs.credentialsPath -}} +{{- else -}} + {{- printf "%s" "/secrets/gcs-log-creds/gcp.json" -}} +{{- end -}} +{{- end -}} + +{{/* +Construct comma separated list of key/value pairs from object (useful for ENV var values) +*/}} +{{- define "airbyte.flattenMap" -}} +{{- $kvList := list -}} +{{- range $key, $value := . -}} +{{- $kvList = printf "%s=%s" $key $value | mustAppend $kvList -}} +{{- end -}} +{{ join "," $kvList }} +{{- end -}} + +{{/* +Construct semi-colon delimited list of comma separated key/value pairs from array of objects (useful for ENV var values) +*/}} +{{- define "airbyte.flattenArrayMap" -}} +{{- $mapList := list -}} +{{- range $element := . -}} +{{- $mapList = include "airbyte.flattenMap" $element | mustAppend $mapList -}} +{{- end -}} +{{ join ";" $mapList }} +{{- end -}} + +## DEFAULT HELM VALUES +# Secret Manager Defaults +{{/* +Define secret persistence +*/}} +{{- define "airbyte.secretPersistence" -}} +{{- if (((.Values.global).secretsManager).type) }} + {{- printf "%s" (snakecase .Values.global.secretsManager.type) }} +{{- else }} + {{- printf "" }} +{{- end }} +{{- end }} + +{{/* +Get secret store name or default +*/}} +{{- define "airbyte.secretStoreName" -}} +{{- $secretStoreName := . -}} +{{- if $secretStoreName -}} + {{- printf "%s" $secretStoreName -}} +{{- else -}} + {{- printf "airbyte-config-secrets" -}} +{{- end -}} +{{- end -}} + +{{/* +Get awsSecretManager access key id secret key or default +*/}} +{{- define "airbyte.awsSecretManagerAccessKeyIdSecretKey" -}} +{{- $awsSecretManagerAccessKeyIdSecretKey := . -}} +{{- if $awsSecretManagerAccessKeyIdSecretKey -}} + {{- printf "%s" $awsSecretManagerAccessKeyIdSecretKey -}} +{{- else -}} + {{- printf "aws-secret-manager-access-key-id" -}} +{{- end -}} +{{- end -}} + +{{/* +Get awsSecretManager secret access key secret key or default +*/}} +{{- define "airbyte.awsSecretManagerSecretAccessKeySecretKey" -}} +{{- $awsSecretManagerSecretAccessKeySecretKey := . -}} +{{- if $awsSecretManagerSecretAccessKeySecretKey -}} + {{- printf "%s" $awsSecretManagerSecretAccessKeySecretKey -}} +{{- else -}} + {{- printf "aws-secret-manager-secret-access-key" -}} +{{- end -}} +{{- end -}} + +{{/* +Get googleSecretManager credentials secret key or default +*/}} +{{- define "airbyte.googleSecretManagerCredentialsSecretKey" -}} +{{- $googleSecretManagerCredentialsSecretKey := . -}} +{{- if $googleSecretManagerCredentialsSecretKey -}} + {{- printf "%s" $googleSecretManagerCredentialsSecretKey -}} +{{- else -}} + {{- printf "google-secret-manager-credentials" -}} +{{- end -}} +{{- end -}} + +{{/* +Get vault auth token secret key or default +*/}} +{{- define "airbyte.vaultAuthTokenSecretKey" -}} +{{- $vaultAuthTokenSecretKey := . -}} +{{- if $vaultAuthTokenSecretKey -}} + {{- printf "%s" $vaultAuthTokenSecretKey -}} +{{- else -}} + {{- printf "vault-auth-token" -}} +{{- end -}} +{{- end -}} + + +# Storage Defaults +{{/* +Get storage type or default +*/}} +{{- define "airbyte.storageType" -}} +{{- $storageType := . -}} +{{- if $storageType -}} + {{- printf "%s" $storageType -}} +{{- else -}} + {{- printf "local" -}} +{{- end -}} +{{- end -}} + +{{/* +Get storage bucket log or default +*/}} +{{- define "airbyte.storageBucketLog" -}} +{{- $storageBucketLog := . -}} +{{- if $storageBucketLog -}} + {{- printf "%s" $storageBucketLog -}} +{{- else -}} + {{- printf "airbyte-storage" -}} +{{- end -}} +{{- end -}} + +{{/* +Get storage bucket state or default +*/}} +{{- define "airbyte.storageBucketState" -}} +{{- $storageBucketState := . -}} +{{- if $storageBucketState -}} + {{- printf "%s" $storageBucketState -}} +{{- else -}} + {{- printf "airbyte-storage" -}} +{{- end -}} +{{- end -}} + +{{/* +Get storage bucket workload output or default +*/}} +{{- define "airbyte.storageBucketWorkloadOutput" -}} +{{- $storageBucketWorkloadOutput := . -}} +{{- if $storageBucketWorkloadOutput -}} + {{- printf "%s" $storageBucketWorkloadOutput -}} +{{- else -}} + {{- printf "airbyte-storage" -}} +{{- end -}} +{{- end -}} + +{{/* +Get s3 access key id secret key or default +*/}} +{{- define "airbyte.s3AccessKeyIdSecretKey" -}} +{{- $s3AccessKeyIdSecretKey := . -}} +{{- if $s3AccessKeyIdSecretKey -}} + {{- printf "%s" $s3AccessKeyIdSecretKey -}} +{{- else -}} + {{- printf "s3-access-key-id" -}} +{{- end -}} +{{- end -}} + +{{/* +Get s3 secret access key secret key or default +*/}} +{{- define "airbyte.s3SecretAccessKeySecretKey" -}} +{{- $s3SecretAccessKeySecretKey := . -}} +{{- if $s3SecretAccessKeySecretKey -}} + {{- printf "%s" $s3SecretAccessKeySecretKey -}} +{{- else -}} + {{- printf "s3-secret-access-key" -}} +{{- end -}} +{{- end -}} + +{{/* +Get minio access key id secret key or default +*/}} +{{- define "airbyte.minioAccessKeyIdSecretKey" -}} +{{- $minioAccessKeyIdSecretKey := . -}} +{{- if $minioAccessKeyIdSecretKey -}} + {{- printf "%s" $minioAccessKeyIdSecretKey -}} +{{- else -}} + {{- printf "minio-access-key-id" -}} +{{- end -}} +{{- end -}} + +{{/* +Get minio secret access key secret key or default +*/}} +{{- define "airbyte.minioSecretAccessKeySecretKey" -}} +{{- $minioSecretAccessKeySecretKey := . -}} +{{- if $minioSecretAccessKeySecretKey -}} + {{- printf "%s" $minioSecretAccessKeySecretKey -}} +{{- else -}} + {{- printf "minio-secret-access-key" -}} +{{- end -}} +{{- end -}} + +{{/* +Get gcs credentials secret key or default +*/}} +{{- define "airbyte.gcsCredentialsSecretKey" -}} +{{- $gcsCredentialsSecretKey := . -}} +{{- if $gcsCredentialsSecretKey -}} + {{- printf "%s" $gcsCredentialsSecretKey -}} +{{- else -}} + {{- printf "gcs-credentials" -}} +{{- end -}} +{{- end -}} + +{{/* +Convert tags to a comma-separated list of key=value pairs. +*/}} +{{- define "airbyte.tagsToString" -}} +{{- $result := list -}} +{{- range . -}} + {{- $key := .key -}} + {{- $value := .value -}} + {{- if eq (typeOf $value) "bool" -}} + {{- $value = ternary "true" "false" $value -}} + {{- end -}} + {{- $result = append $result (printf "%s=%s" $key $value) -}} +{{- end -}} +{{- join "," $result -}} +{{- end -}} diff --git a/helm/airbyte-v1/templates/_images.tpl b/helm/airbyte-v1/templates/_images.tpl new file mode 100644 index 00000000..6c32ec44 --- /dev/null +++ b/helm/airbyte-v1/templates/_images.tpl @@ -0,0 +1,17 @@ +{{- define "imageUrl" -}} +{{- $img := index . 0 -}} +{{- $root := index . 1 -}} + +{{- $reg := $root.Values.global.image.registry -}} +{{/* ensure the registry has a trailing slash, if set */}} +{{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} + +{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{- if (eq (typeOf $img) "string") -}} +{{- printf "%s%s" $reg (tpl $img $root) | quote -}} +{{- else -}} +{{- $tag := coalesce $img.tag $root.Values.global.image.tag $root.Chart.AppVersion -}} +{{- printf "%s%s:%s" $reg $img.repository $tag | quote -}} +{{- end -}} + +{{- end -}} \ No newline at end of file diff --git a/helm/airbyte-v1/templates/_keycloak.tpl b/helm/airbyte-v1/templates/_keycloak.tpl new file mode 100644 index 00000000..ac0545db --- /dev/null +++ b/helm/airbyte-v1/templates/_keycloak.tpl @@ -0,0 +1,44 @@ +{{/* +Keycloak Configuration +*/}} + +{{- define "airbyte.keycloak.database.user.env" }} +- name: KEYCLOAK_DATABASE_USERNAME + valueFrom: + secretKeyRef: + {{- if .Values.global.database.userSecretKey }} + name: {{ .Values.global.database.secretName }} + {{- else }} + name: {{ .Release.Name }}-airbyte-secrets + {{- end }} + key: {{ include "airbyte.database.userSecretKey" .}} +{{- end }} + +{{- define "airbyte.keycloak.database.password.env" }} +- name: KEYCLOAK_DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "airbyte.database.secretName" . }} + key: {{ include "airbyte.database.passwordSecretKey" . }} +{{- end }} + +{{- define "airbyte.keycloak.database.url" }} +{{ $host := (include "airbyte.database.host" .) }} +{{ $dbName := (include "airbyte.database.name" .) }} +{{ $port := (include "airbyte.database.port" . ) }} +{{- printf "jdbc:postgresql://%s:%s/%s?currentSchema=keycloak" $host $port $dbName }} +{{- end }} + +{{- define "airbyte.keycloak.database.url.env" }} +- name: KEYCLOAK_DATABASE_URL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: KEYCLOAK_DATABASE_URL +{{- end }} + +{{- define "airbyte.keycloak.database.envs" }} +{{ include "airbyte.keycloak.database.user.env" . }} +{{ include "airbyte.keycloak.database.password.env" . }} +{{ include "airbyte.keycloak.database.url.env" . }} +{{- end }} diff --git a/helm/airbyte-v1/templates/_logging.tpl b/helm/airbyte-v1/templates/_logging.tpl new file mode 100644 index 00000000..f1382739 --- /dev/null +++ b/helm/airbyte-v1/templates/_logging.tpl @@ -0,0 +1,10 @@ +{{/* +Common logging configs +*/}} +{{- define "airbyte.logging.envs" }} +- name: LOG_LEVEL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: LOG_LEVEL +{{- end }} diff --git a/helm/airbyte-v1/templates/_storage.tpl b/helm/airbyte-v1/templates/_storage.tpl new file mode 100644 index 00000000..70819063 --- /dev/null +++ b/helm/airbyte-v1/templates/_storage.tpl @@ -0,0 +1,243 @@ +{{/* +Storage configs +*/}} + +{{/* +Returns the provider name +*/}} +{{- define "airbyte.storage.provider" }} +{{- if .Values.global.storage.type }} + {{- printf "%s" (lower .Values.global.storage.type) }} +{{- else }} + {{- printf "minio" }} +{{- end }} +{{- end }} + +{{/* +Returns the storage provider secret name +*/}} +{{- define "airbyte.storage.secretName" }} +{{- if .Values.global.storage.secretName }} + {{- printf "%s" .Values.global.storage.secretName }} +{{- else if .Values.global.storage.storageSecretName }} + {{- printf "%s" .Values.global.storage.storageSecretName }} +{{- else -}} + {{/* GCS has its own default secret we create */}} + {{- if eq (include "airbyte.storage.provider" .) "gcs" }} + {{- printf "%s-gcs-log-creds" .Release.Name }} + {{- else }} + {{- printf "%s-airbyte-secrets" .Release.Name }} + {{- end }} +{{- end }} +{{- end }} + +{{/* +Returns azure environment variables. +*/}} +{{- define "airbyte.storage.azure.envs" }} +{{- if .Values.global.storage.azure.connectionString }} +- name: AZURE_STORAGE_CONNECTION_STRING + value: {{ .Values.global.storage.azure.connectionString }} +{{- end }} +{{- if .Values.global.storage.azure.connectionStringSecretKey }} +- name: AZURE_STORAGE_CONNECTION_STRING + valueFrom: + secretKeyRef: + name: {{ include "airbyte.storage.secretName" . }} + key: {{ .Values.global.storage.azure.connectionStringSecretKey }} +{{- end }} +{{- end }} + +{{/* +Returns azure secrets +*/}} +{{- define "airbyte.storage.azure.secrets" }} +{{- if .Values.global.storage.azure }} +AZURE_STORAGE_CONNECTION_STRING: {{ .Values.global.storage.azure.connectionString | default "" }} +{{- end }} +{{- end }} + +{{/* +Returns S3 environment variables. +*/}} +{{- define "airbyte.storage.s3.envs" }} +{{- if eq .Values.global.storage.s3.authenticationType "credentials" }} +- name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ include "airbyte.storage.secretName" . }} + key: {{ .Values.global.storage.s3.accessKeyIdSecretKey | default "s3-access-key-id" }} +- name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ include "airbyte.storage.secretName" . }} + key: {{ .Values.global.storage.s3.secretAccessKeySecretKey | default "s3-secret-access-key" }} +{{- end }} +{{- if .Values.global.storage.s3.region }} +- name: AWS_DEFAULT_REGION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AWS_DEFAULT_REGION +{{- end }} +{{- end}} + +{{/* +Returns S3 secrets +*/}} +{{- define "airbyte.storage.s3.secrets" }} +{{- if and (.Values.global.storage.s3) (eq .Values.global.storage.s3.authenticationType "credentials") }} +AWS_ACCESS_KEY_ID: {{ .Values.global.storage.s3.accessKeyId | default "" }} +AWS_SECRET_ACCESS_KEY: {{ .Values.global.storage.s3.secretAccessKey | default "" }} +{{- end }} +{{- end }} + +{{/* +Returns GCS environment variables. +*/}} +{{- define "airbyte.storage.gcs.envs" }} +- name: GOOGLE_APPLICATION_CREDENTIALS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: GOOGLE_APPLICATION_CREDENTIALS +{{- end}} + +{{/* +Returns GCS secrets. +*/}} +{{- define "airbyte.storage.gcs.secrets" }} +{{- if .Values.global.storage.gcs }} +gcp.json: {{ .Values.global.storage.gcs.credentialsJson }} +{{- end }} +{{- end}} + +{{/* +Returns Minio environment variables. +*/}} +{{- define "airbyte.storage.minio.envs" }} +{{- if .Values.global.storage.minio }} +- name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ include "airbyte.storage.secretName" . }} + key: {{ .Values.global.storage.minio.accessKeyIdSecretKey | default "MINIO_ACCESS_KEY_ID" }} +- name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ include "airbyte.storage.secretName" . }} + key: {{ .Values.global.storage.minio.secretAccessKeySecretKey | default "MINIO_SECRET_ACCESS_KEY" }} +{{- else }} +- name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ include "airbyte.storage.secretName" . }} + key: "MINIO_ACCESS_KEY_ID" +- name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ include "airbyte.storage.secretName" . }} + key: "MINIO_SECRET_ACCESS_KEY" +{{- end }} +- name: MINIO_ENDPOINT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MINIO_ENDPOINT +{{- end }} + +{{- define "airbyte.storage.minio.secrets" }} +{{- if .Values.global.storage.minio }} +MINIO_ACCESS_KEY_ID: {{ .Values.global.storage.minio.accessKeyId | default "minio" | quote }} +MINIO_SECRET_ACCESS_KEY: {{ .Values.global.storage.minio.secretAccessKey | default "minio123" | quote }} +{{- else }} +MINIO_ACCESS_KEY_ID: "minio" +MINIO_SECRET_ACCESS_KEY: "minio123" +{{- end }} +{{- end }} + +{{/* +Returns storage config environment variables. +*/}} +{{- define "airbyte.storage.envs" }} +{{- $storageProvider := (include "airbyte.storage.provider" .) }} +- name: S3_PATH_STYLE_ACCESS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: S3_PATH_STYLE_ACCESS +- name: STORAGE_TYPE + value: {{ upper $storageProvider }} +- name: STORAGE_BUCKET_ACTIVITY_PAYLOAD + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: STORAGE_BUCKET_ACTIVITY_PAYLOAD +- name: STORAGE_BUCKET_LOG + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: STORAGE_BUCKET_LOG +- name: STORAGE_BUCKET_STATE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: STORAGE_BUCKET_STATE +- name: STORAGE_BUCKET_WORKLOAD_OUTPUT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: STORAGE_BUCKET_WORKLOAD_OUTPUT +{{/* S3 */}} +{{- if eq $storageProvider "s3" }} +{{- include "airbyte.storage.s3.envs" . }} +{{- end }} + +{{/* GCS */}} +{{- if eq $storageProvider "gcs" }} +{{- include "airbyte.storage.gcs.envs" . }} +{{- end }} + +{{/* MINIO */}} +{{- if eq $storageProvider "minio" }} +{{- include "airbyte.storage.minio.envs" . }} +{{- end }} + +{{/* AZURE */}} +{{- if eq $storageProvider "azure" }} +{{- include "airbyte.storage.azure.envs" . }} +{{- end }} + +{{- end }} + +{{/* +Returns storage config variables. +*/}} +{{- define "airbyte.storage.configVars" }} +{{- end }} + +{{/* +Returns storage config secrets. +*/}} +{{- define "airbyte.storage.secrets" }} +{{- $storageProvider := (include "airbyte.storage.provider" .) }} +{{/* S3 */}} +{{- if eq $storageProvider "s3" }} +{{- include "airbyte.storage.s3.secrets" . }} +{{- end }} + +{{/* GCS */}} +{{- if eq $storageProvider "gcs" }} +{{- include "airbyte.storage.gcs.secrets" . }} +{{- end }} + +{{/* MINIO */}} +{{- if eq $storageProvider "minio" }} +{{- include "airbyte.storage.minio.secrets" . }} +{{- end }} + +{{/* AZURE */}} +{{- if eq $storageProvider "azure" }} +{{- include "airbyte.storage.azure.secrets" . }} +{{- end }} +{{- end }} diff --git a/helm/airbyte-v1/templates/_temporal.tpl b/helm/airbyte-v1/templates/_temporal.tpl new file mode 100644 index 00000000..c4e213c9 --- /dev/null +++ b/helm/airbyte-v1/templates/_temporal.tpl @@ -0,0 +1,58 @@ +{{/* +Temporal Configuration +*/}} + +{{- define "airbyte.temporal.database.host.env" }} +- name: POSTGRES_SEEDS + valueFrom: + {{- if .Values.global.database.hostSecretKey }} + secretKeyRef: + name: {{ include "airbyte.database.secretName" . }} + key: {{ .Values.global.database.hostSecretKey }} + {{- else }} + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATABASE_HOST + {{- end }} +{{- end }} + +{{- define "airbyte.temporal.database.port.env" }} +- name: DB_PORT + valueFrom: + {{- if .Values.global.database.portSecretKey }} + secretKeyRef: + name: {{ include "airbyte.database.secretName" . }} + key: {{ .Values.global.database.portSecretKey }} + {{- else }} + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATABASE_PORT + {{- end }} +{{- end }} + +{{- define "airbyte.temporal.database.user.env" }} +- name: POSTGRES_USER + valueFrom: + secretKeyRef: + {{- if .Values.global.database.userSecretKey }} + name: {{ .Values.global.database.secretName }} + {{- else }} + name: {{ .Release.Name }}-airbyte-secrets + {{- end }} + key: {{ include "airbyte.database.userSecretKey" .}} +{{- end }} + +{{- define "airbyte.temporal.database.password.env" }} +- name: POSTGRES_PWD + valueFrom: + secretKeyRef: + name: {{ include "airbyte.database.secretName" . }} + key: {{ include "airbyte.database.passwordSecretKey" . }} +{{- end }} + +{{- define "airbyte.temporal.database.envs" }} +{{ include "airbyte.temporal.database.host.env" . }} +{{ include "airbyte.temporal.database.port.env" . }} +{{ include "airbyte.temporal.database.user.env" . }} +{{ include "airbyte.temporal.database.password.env" . }} +{{- end }} diff --git a/helm/airbyte-v1/templates/airbyte-db.yaml b/helm/airbyte-v1/templates/airbyte-db.yaml new file mode 100644 index 00000000..91cbd658 --- /dev/null +++ b/helm/airbyte-v1/templates/airbyte-db.yaml @@ -0,0 +1,85 @@ +{{- if .Values.postgresql.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: airbyte-db-svc + annotations: + helm.sh/hook: pre-install + helm.sh/hook-weight: "-1" + labels: + {{ include "airbyte.databaseLabels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: 5432 + protocol: TCP + selector: + {{ include "airbyte.databaseSelectorLabels" . | nindent 4 }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airbyte-db + annotations: + helm.sh/hook: pre-install + helm.sh/hook-weight: "-1" + labels: + {{ include "airbyte.databaseLabels" . | nindent 4 }} +spec: + replicas: 1 + serviceName: airbyte-db-svc + selector: + matchLabels: + {{ include "airbyte.databaseSelectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{ include "airbyte.databaseSelectorLabels" . | nindent 8 }} + spec: + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + containers: + - name: airbyte-db-container + image: {{ include "imageUrl" (list .Values.postgresql.image $) }} + env: + - name: POSTGRES_DB + value: {{ .Values.postgresql.postgresqlDatabase }} + - name: POSTGRES_PASSWORD + value: {{ .Values.postgresql.postgresqlPassword }} + - name: POSTGRES_USER + value: {{ .Values.postgresql.postgresqlUsername }} + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + ports: + - containerPort: 5432 + securityContext: {{- toYaml .Values.postgresql.containerSecurityContext | nindent 12 }} + volumeMounts: + - name: airbyte-volume-db + mountPath: /var/lib/postgresql/data + {{- with .Values.postgresql.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.postgresql.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.postgresql.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: {{- toYaml .Values.postgresql.podSecurityContext | nindent 8 }} + + volumeClaimTemplates: + - metadata: + name: airbyte-volume-db + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 500Mi +{{- end }} diff --git a/helm/airbyte-v1/templates/airbyte-yml-secret.yaml b/helm/airbyte-v1/templates/airbyte-yml-secret.yaml new file mode 100644 index 00000000..eb6574e6 --- /dev/null +++ b/helm/airbyte-v1/templates/airbyte-yml-secret.yaml @@ -0,0 +1,19 @@ +{{- if .Values.global.airbyteYml }} +# The Secret that contains the airbyte.yml content. For this to work helm must be started with +# --set-file global.airbyteYml=/airbyte.yml +# For Airbyte Enterprise our installation script automatically specifies this, for Community +# users will need to specify it, if they want to use the airbyte.yml config file. +# If nothing is specified we'll assume an empty config +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-airbyte-yml + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" + labels: + {{- include "airbyte.labels" . | nindent 4 }} +type: Opaque +data: + fileContents: "{{ .Values.global.airbyteYml | default "" | nindent 4 | b64enc }}" +{{- end }} diff --git a/helm/airbyte-v1/templates/env-configmap.yaml b/helm/airbyte-v1/templates/env-configmap.yaml new file mode 100644 index 00000000..12c489d8 --- /dev/null +++ b/helm/airbyte-v1/templates/env-configmap.yaml @@ -0,0 +1,175 @@ +{{- $airbyteYmlDict := (default "" .Values.global.airbyteYml | fromYaml) }} +# default to empty dict if airbyteYml is undefined +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-airbyte-env + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" + labels: + {{- include "airbyte.labels" . | nindent 4 }} +data: + AIRBYTE_VERSION: {{ .Values.version | default .Chart.AppVersion }} + {{- if or (eq .Values.global.edition "pro") (eq .Values.global.edition "enterprise") }} + AIRBYTE_EDITION: "pro" + {{- else }} + AIRBYTE_EDITION: "community" + {{- end }} + + {{- if $airbyteYmlDict }} + AIRBYTE_URL: {{ (index $airbyteYmlDict "webapp-url") | quote }} + {{- else}} + AIRBYTE_URL: {{ .Values.global.airbyteUrl | quote }} + {{- end }} + +### Env vars requested by the Blotout team + # BLOTOUT_AUTH_ENDPOINT: "" + # BLOTOUT_BASE_URL: "" + AWS_SECRET_ACCESS_KEY: {{ .Values.global.aws_secret_key | default "" }} + AWS_ACCESS_KEY_ID: {{ .Values.global.aws_access_key | default "" | quote }} + JOB_MAIN_CONTAINER_CPU_REQUEST: {{ .Values.global.resource_cpu_request | default "0.5" }} + JOB_MAIN_CONTAINER_CPU_LIMIT: {{ .Values.global.resource_cpu_limit | default "2" }} + JOB_MAIN_CONTAINER_MEMORY_REQUEST: {{ .Values.global.resource_memory_request | default "1Gi" }} + JOB_MAIN_CONTAINER_MEMORY_LIMIT: {{ .Values.global.resource_memory_limit | default "2Gi" }} + SECRET_PERSISTENCE: {{ .Values.global.secret_persistence | default "" }} + S3_PATH_STYLE_ACCESS: {{ .Values.global.s3_path_style_access | default "" }} + +##################################################### + + + AIRBYTE_SERVER_HOST: {{ .Release.Name }}-airbyte-server-svc:{{ .Values.server.service.port }} + API_URL: {{ .Values.webapp.api.url }} + CONNECTOR_BUILDER_API_URL: {{ index .Values.webapp "connector-builder-server" "url" | quote }} + CONFIG_API_HOST: http://{{ .Release.Name }}-airbyte-server-svc:{{ .Values.server.service.port }} # temporary solution for oss kube deploys for airbyte api server until that server is wrapped into the config server + CONFIG_ROOT: /configs + CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION: "0.35.15.001" + DATA_DOCKER_MOUNT: airbyte_data + DB_DOCKER_MOUNT: airbyte_db + {{- include "airbyte.database.configVars" . | nindent 2 }} + KEYCLOAK_DATABASE_URL: {{ include "airbyte.keycloak.database.url" . | quote }} + GOOGLE_APPLICATION_CREDENTIALS: {{ include "airbyte.gcpLogCredentialsPath" . | quote }} + INTERNAL_API_HOST: http://{{ .Release.Name }}-airbyte-server-svc:{{ .Values.server.service.port }} + WORKLOAD_API_HOST: http://{{ .Release.Name }}-workload-api-server-svc:{{ index .Values "workload-api-server" "service" "port" }} + + + {{- if or (eq .Values.global.edition "pro") (eq .Values.global.edition "enterprise") }} + INITIAL_USER_FIRST_NAME: {{ .Values.global.auth.instanceAdmin.firstName }} + INITIAL_USER_LAST_NAME: {{ .Values.global.auth.instanceAdmin.lastName }} + {{- if .Values.global.auth.identityProvider }} + IDENTITY_PROVIDER_TYPE: {{ .Values.global.auth.identityProvider.type }} + OIDC_DOMAIN: {{ .Values.global.auth.identityProvider.oidc.domain }} + OIDC_APP_NAME: {{ .Values.global.auth.identityProvider.oidc.appName }} + {{- end }} + KEYCLOAK_INTERNAL_HOST: {{ .Release.Name }}-airbyte-keycloak-svc:{{ .Values.keycloak.service.port }} + KEYCLOAK_PORT: {{ .Values.keycloak.service.port | quote }} + + {{- if $airbyteYmlDict }} + KEYCLOAK_HOSTNAME_URL: {{ printf "%s/auth" (index $airbyteYmlDict "webapp-url") | quote }} + {{- else }} + KEYCLOAK_HOSTNAME_URL: {{ printf "%s/auth" .Values.global.airbyteUrl | quote }} + {{- end }} + + KEYCLOAK_JAVA_OPTS_APPEND: -Djgroups.dns.query={{ .Release.Name }}-airbyte-keycloak-headless-svc + {{- else }} + KEYCLOAK_INTERNAL_HOST: localhost # just a placeholder so that nginx template is valid - shouldn't be used when edition isn't "pro" + {{- end }} + + CONNECTOR_BUILDER_API_HOST: {{ .Release.Name }}-airbyte-connector-builder-server-svc:{{ index .Values "connector-builder-server" "service" "port" }} + + {{- if or (eq .Values.global.edition "pro") (eq .Values.global.edition "enterprise") }} + {{- if $airbyteYmlDict }} + AIRBYTE_API_HOST: {{ printf "%s/api/public" (index $airbyteYmlDict "webapp-url") | quote }} + {{- else }} + AIRBYTE_API_HOST: {{ printf "%s/api/public" .Values.global.airbyteUrl | quote }} + {{- end }} + {{- else if (eq .Values.global.edition "community")}} + AIRBYTE_API_HOST: http://localhost:{{ .Values.server.service.port }}/api/public + {{- else }} + AIRBYTE_API_HOST: http://{{ .Release.Name }}-airbyte-server-svc:{{ .Values.server.service.port }}/api/public + {{- end }} + + {{- if $.Values.global.jobs.kube.annotations }} + JOB_KUBE_ANNOTATIONS: {{ $.Values.global.jobs.kube.annotations | include "airbyte.flattenMap" | quote }} + {{- end }} + + {{- if $.Values.global.jobs.kube.labels }} + JOB_KUBE_LABELS: {{ $.Values.global.jobs.kube.labels | include "airbyte.flattenMap" | quote }} + {{- end }} + + {{- if $.Values.global.jobs.kube.nodeSelector }} + JOB_KUBE_NODE_SELECTORS: {{ $.Values.global.jobs.kube.nodeSelector | include "airbyte.flattenMap" | quote }} + {{- end }} + + {{- if $.Values.global.jobs.kube.tolerations }} + JOB_KUBE_TOLERATIONS: {{ $.Values.global.jobs.kube.tolerations | include "airbyte.flattenArrayMap" | quote }} + {{- end }} + + JOB_KUBE_BUSYBOX_IMAGE: {{ include "imageUrl" (list $.Values.global.jobs.kube.images.busybox $) }} + JOB_KUBE_CURL_IMAGE: {{ include "imageUrl" (list $.Values.global.jobs.kube.images.curl $) }} + {{ $workloadLauncher := (index .Values "workload-launcher") }} + CONTAINER_ORCHESTRATOR_IMAGE: {{ include "imageUrl" (list $workloadLauncher.containerOrchestrator.image $) }} + WORKLOAD_INIT_IMAGE: {{ include "imageUrl" (list $workloadLauncher.workloadInit.image $) }} + CONNECTOR_SIDECAR_IMAGE: {{ include "imageUrl" (list $workloadLauncher.connectorSidecar.image $) }} + + JOB_MAIN_CONTAINER_CPU_LIMIT: {{ ((.Values.global.jobs.resources | default dict).limits | default dict).cpu | default "" | quote }} + JOB_MAIN_CONTAINER_CPU_REQUEST: {{ ((.Values.global.jobs.resources | default dict).requests | default dict).cpu | default "" | quote }} + JOB_MAIN_CONTAINER_MEMORY_LIMIT: {{ ((.Values.global.jobs.resources | default dict).limits | default dict).memory | default "" | quote }} + JOB_MAIN_CONTAINER_MEMORY_REQUEST: {{ ((.Values.global.jobs.resources | default dict).requests | default dict).memory | default "" | quote }} + + {{- if $.Values.global.jobs.kube.main_container_image_pull_secret }} + JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET: {{ $.Values.global.jobs.kube.main_container_image_pull_secret }} + {{- end }} + JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION: "0.29.15.001" + + JOB_KUBE_LOCAL_VOLUME_ENABLED: {{ .Values.global.jobs.local.volume.enabled | quote }} + + LOG_LEVEL: {{ default "INFO" (((.Values.global).logging).level) }} + MICROMETER_METRICS_ENABLED: "false" + MICROMETER_METRICS_STATSD_FLAVOR: "datadog" + RUN_DATABASE_MIGRATION_ON_STARTUP: {{ index .Values "airbyte-bootloader" "runDatabaseMigrationsOnStartup" | default "true" | quote }} + SEGMENT_WRITE_KEY: 7UDdp5K55CyiGgsauOr2pNNujGvmhaeu + # Storage start + STORAGE_TYPE: {{ (((.Values.global).storage).type) | default "minio" | quote }} + STORAGE_BUCKET_ACTIVITY_PAYLOAD: {{ ((((.Values.global).storage).bucket).activityPayload) | default "airbyte-storage" | quote }} + STORAGE_BUCKET_LOG: {{ ((((.Values.global).storage).bucket).log) | default "airbyte-storage" | quote }} + STORAGE_BUCKET_STATE: {{ ((((.Values.global).storage).bucket).state) | default "airbyte-storage" | quote }} + STORAGE_BUCKET_WORKLOAD_OUTPUT: {{ ((((.Values.global).storage).bucket).workloadOutput) | default "airbyte-storage" | quote }} + + {{- with .Values.global.storage.s3 }} + AWS_DEFAULT_REGION: {{ .region | default "" | quote }} + {{- end}} + + MINIO_ENDPOINT: {{ include "airbyte.storage.minio.endpoint" . | quote }} + S3_PATH_STYLE_ACCESS: {{ include "airbyte.s3PathStyleAccess" . | quote }} + # Storage end + STATSD_HOST: "localhost" + STATSD_PORT: "8125" + TEMPORAL_HOST: {{ .Release.Name }}-temporal:{{ .Values.temporal.service.port }} + TEMPORAL_WORKER_PORTS: 9001,9002,9003,9004,9005,9006,9007,9008,9009,9010,9011,9012,9013,9014,9015,9016,9017,9018,9019,9020,9021,9022,9023,9024,9025,9026,9027,9028,9029,9030,9031,9032,9033,9034,9035,9036,9037,9038,9039,9040 + TRACKING_STRATEGY: segment + WEBAPP_URL: {{ .Values.webapp.url | default (printf "http://%s-airbyte-webapp-svc:%d" .Release.Name (.Values.webapp.service.port | int)) }} + WORKER_ENVIRONMENT: kubernetes + WORKSPACE_DOCKER_MOUNT: airbyte_workspace + WORKSPACE_ROOT: /workspace + METRIC_CLIENT: {{ .Values.global.metrics.metricClient | default "" | quote }} + OTEL_COLLECTOR_ENDPOINT: {{ .Values.global.metrics.otelCollectorEndpoint | default "" | quote }} + ACTIVITY_MAX_ATTEMPT: {{ .Values.worker.activityMaxAttempt | default "" | quote }} + ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS: {{ .Values.worker.activityInitialDelayBetweenAttemptsSeconds | default "" | quote }} + ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS: {{ .Values.worker.activityMaxDelayBetweenAttemptsSeconds | default "" | quote }} + WORKFLOW_FAILURE_RESTART_DELAY_SECONDS: "" + FILE_TRANSFER_EPHEMERAL_STORAGE_LIMIT: 5G + FILE_TRANSFER_EPHEMERAL_STORAGE_REQUEST: 5G + + LAUNCHER_MICRONAUT_ENVIRONMENTS: "control-plane,oss" + WORKERS_MICRONAUT_ENVIRONMENTS: "control-plane" + CRON_MICRONAUT_ENVIRONMENTS: "control-plane" + SERVER_MICRONAUT_ENVIRONMENTS: "control-plane" + SHOULD_RUN_NOTIFY_WORKFLOWS: "true" + MAX_NOTIFY_WORKERS: {{ .Values.worker.maxNotifyWorkers | default "5" | quote }} + KUBERNETES_CLIENT_MAX_IDLE_CONNECTIONS: "" + WORKLOAD_LAUNCHER_PARALLELISM: "10" + CONNECTOR_BUILDER_SERVER_API_HOST: http://{{ .Release.Name }}-airbyte-connector-builder-server-svc:{{ index .Values "connector-builder-server" "service" "port" }} + PUB_SUB_ENABLED: "false" + PUB_SUB_TOPIC_NAME: "" + ENTERPRISE_SOURCE_STUBS_URL: {{ .Values.server.env_vars.ENTERPRISE_SOURCE_STUBS_URL | default "https://connectors.airbyte.com/files/resources/connector_stubs/v0/connector_stubs.json" }} diff --git a/helm/airbyte-v1/templates/gcs-log-creds-secret.yaml b/helm/airbyte-v1/templates/gcs-log-creds-secret.yaml new file mode 100644 index 00000000..acdc1f93 --- /dev/null +++ b/helm/airbyte-v1/templates/gcs-log-creds-secret.yaml @@ -0,0 +1,18 @@ +{{- if and + (eq .Values.global.deploymentMode "oss") + (eq (lower (default "" .Values.global.storage.type)) "gcs") + (not .Values.global.storage.secretName) + (not .Values.global.storage.storageSecretName) +}} +{{ $gcsConfig := required "You must set 'global.storage.gcs' when setting 'global.storage.type' to 'gcs'" .Values.global.storage.gcs }} +{{ $credentialsJson := required "You must set 'global.storage.gcs.credentialsJson' when 'global.storage.secretName' is not set" .Values.global.storage.gcs.credentialsJson }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-gcs-log-creds + labels: + {{- include "airbyte.labels" . | nindent 4 }} +type: Opaque +data: + gcp.json: {{ $credentialsJson | default "" | quote }} +{{- end }} diff --git a/helm/airbyte-v1/templates/minio.yaml b/helm/airbyte-v1/templates/minio.yaml new file mode 100644 index 00000000..4c6bd8cb --- /dev/null +++ b/helm/airbyte-v1/templates/minio.yaml @@ -0,0 +1,185 @@ +{{- if (eq (lower (default "" .Values.global.storage.type)) "minio")}} +apiVersion: apps/v1 # for k8s versions before 1.9.0 use apps/v1beta2 and before 1.8.0 use extensions/v1beta1 +kind: StatefulSet +metadata: + # This name uniquely identifies the Deployment + name: airbyte-minio + annotations: + helm.sh/hook: pre-install + helm.sh/hook-weight: "-1" + labels: + {{ include "airbyte.minioLabels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{ include "airbyte.minioSelectorLabels" . | nindent 6 }} + serviceName: airbyte-minio-svc + volumeClaimTemplates: + - metadata: + name: airbyte-minio-pv-claim + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{ .Values.minio.storage.volumeClaimValue }} + template: + metadata: + labels: + # Label is used as selector in the service. + {{ include "airbyte.minioSelectorLabels" . | nindent 8 }} + spec: + # Refer to the PVC created earlier + securityContext: + fsGroup: 1000 + containers: + - name: airbyte-minio + image: {{ include "imageUrl" (list .Values.minio.image $) }} + args: + - server + - /storage + env: + # Minio access key and secret key. This must match the S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY declared in /dev/.env. + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-airbyte-secrets + key: MINIO_ACCESS_KEY_ID + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-airbyte-secrets + key: MINIO_SECRET_ACCESS_KEY + ports: + - containerPort: 9000 + resources: + requests: + memory: {{ .Values.minio.resources.requests.memory }} + cpu: {{ .Values.minio.resources.requests.cpu }} + limits: + memory: {{ .Values.minio.resources.limits.memory }} + cpu: {{ .Values.minio.resources.limits.cpu }} + # Mount the volume into the pod + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + volumeMounts: + - name: airbyte-minio-pv-claim # must match the volume name, above + mountPath: "/storage" + {{- with .Values.minio.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.minio.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.minio.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + +--- +apiVersion: v1 +kind: Service +metadata: + name: airbyte-minio-svc + labels: + {{ include "airbyte.minioLabels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install + helm.sh/hook-weight: "-1" +spec: + ports: + - port: 9000 + targetPort: 9000 + protocol: TCP + selector: + {{ include "airbyte.minioSelectorLabels" . | nindent 4 }} +--- +# This pod creates the state-storage bucket in the minio server, +# which the local cloud deployment requires to store its state. +apiVersion: v1 +kind: Pod +metadata: + name: airbyte-minio-create-bucket + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +spec: + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + restartPolicy: OnFailure + securityContext: + fsGroup: 1000 + containers: + - name: minio-mc + image: {{ include "imageUrl" (list .Values.minio.mcImage $) }} + command: ["/bin/sh", "-c", + "until (/usr/bin/mc config host add myminio $MINIO_ENDPOINT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY) do echo '...waiting...' && sleep 1; done; + /usr/bin/mc mb --ignore-existing myminio/state-storage; + /usr/bin/mc policy set public myminio/state-storage; + /usr/bin/mc mb --ignore-existing myminio/airbyte-dev-logs; + /usr/bin/mc policy set public myminio/airbyte-dev-logs;"] + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + env: + {{- if and (eq (lower (default "" .Values.global.storage.type)) "minio") (((.Values.global).storage).minio) }} + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretStoreName" .Values.global.storage.storageSecretName }} + key: {{ include "airbyte.minioAccessKeyIdSecretKey" .Values.global.storage.minio.accessKeyIdSecretKey }} + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretStoreName" .Values.global.storage.storageSecretName }} + key: {{ include "airbyte.minioSecretAccessKeySecretKey" .Values.global.storage.minio.secretAccessKeySecretKey }} + {{- else }} + # this is for the internally deployed minio + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-airbyte-secrets + key: MINIO_ACCESS_KEY_ID + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-airbyte-secrets + key: MINIO_SECRET_ACCESS_KEY + {{- end }} + - name: MINIO_ENDPOINT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MINIO_ENDPOINT +{{- end }} diff --git a/helm/airbyte-v1/templates/secret.yaml b/helm/airbyte-v1/templates/secret.yaml new file mode 100644 index 00000000..7d4230d0 --- /dev/null +++ b/helm/airbyte-v1/templates/secret.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-airbyte-secrets + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" + labels: + {{- include "airbyte.labels" . | nindent 4 }} +type: Opaque +stringData: + {{- include "airbyte.database.secrets" . | nindent 2 }} + {{- include "airbyte.storage.secrets" . | nindent 2 }} + {{- if or (eq .Values.global.edition "pro") (eq .Values.global.edition "enterprise") }} + KEYCLOAK_ADMIN_USER: {{ .Values.keycloak.auth.adminUsername | quote }} + KEYCLOAK_ADMIN_PASSWORD: {{ .Values.keycloak.auth.adminPassword | quote }} + {{- end }} + WORKLOAD_API_BEARER_TOKEN: {{ index ".Values.workload-api.bearerToken" | quote }} diff --git a/helm/airbyte-v1/templates/serviceaccount.yaml b/helm/airbyte-v1/templates/serviceaccount.yaml new file mode 100644 index 00000000..7f7acd85 --- /dev/null +++ b/helm/airbyte-v1/templates/serviceaccount.yaml @@ -0,0 +1,77 @@ +{{- if eq .Values.global.deploymentMode "oss" }} +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "airbyte.serviceAccountName" . }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install + helm.sh/hook-weight: "-10" + {{- with .Values.serviceAccount.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "airbyte.serviceAccountName" . }}-role + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install + helm.sh/hook-weight: "-5" +rules: + - apiGroups: ["*"] +{{- if eq .Values.global.edition "community" }} + resources: ["jobs", "pods", "pods/log", "pods/exec", "pods/attach", "secrets"] +{{- else }} + resources: ["jobs", "pods", "pods/log", "pods/exec", "pods/attach"] +{{- end }} + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] # over-permission for now +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "airbyte.serviceAccountName" . }}-binding + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install + helm.sh/hook-weight: "-3" +roleRef: + apiGroup: "" + kind: Role + name: {{ include "airbyte.serviceAccountName" . }}-role +subjects: + - kind: ServiceAccount + name: {{ include "airbyte.serviceAccountName" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + {{- include "airbyte.labels" . | nindent 4 }} + name: node-viewer +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + {{- include "airbyte.labels" . | nindent 4 }} + name: {{ include "airbyte.serviceAccountName" . }}-admin-node-viewer +subjects: + - kind: ServiceAccount + name: {{ include "airbyte.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: node-viewer + apiGroup: rbac.authorization.k8s.io +{{- end }} +{{- end }} diff --git a/helm/airbyte-v1/templates/tests/test-webapp.yaml b/helm/airbyte-v1/templates/tests/test-webapp.yaml new file mode 100644 index 00000000..8b33ae24 --- /dev/null +++ b/helm/airbyte-v1/templates/tests/test-webapp.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "common.names.fullname" . }}-test-connection" + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + containers: + - name: wget + image: {{ include "imageUrl" (list .Values.testWebapp.image $) }} + command: ['wget'] + args: ['{{ .Release.Name }}-airbyte-webapp-svc:{{ .Values.webapp.service.port }}'] + resources: + requests: + memory: "64Mi" + cpu: "100m" + limits: + memory: "128Mi" + cpu: "200m" + restartPolicy: Never diff --git a/helm/airbyte-v1/values.yaml b/helm/airbyte-v1/values.yaml new file mode 100644 index 00000000..d8341768 --- /dev/null +++ b/helm/airbyte-v1/values.yaml @@ -0,0 +1,2417 @@ +### Airbyte V1 Helm Chart + + +# Global params that are overwritten with umbrella chart +global: + # -- Service Account name override + serviceAccountName: &service-account-name "airbyte-admin" + # -- Deployment mode, whether or not render the default env vars and volumes in deployment spec + deploymentMode: "oss" + # -- Edition; "community" or "pro" + edition: "community" + + # --AWS keys + aws_access_key: "" + aws_secret_key: "" + # -- Blotout required configuration + resource_cpu_limit: "6" + resource_cpu_request: "0.5" + resource_memory_limit: "9Gi" + resource_memory_request: "1Gi" + + # -- Airbyte configuration + airbyteYml: "" + + enterprise: + # -- Secret name where an Airbyte license key is stored + secretName: "airbyte-v1-airbyte-secrets" + # -- The key within `licenseKeySecretName` where the Airbyte license key is stored + licenseKeySecretKey: "license-key" + + # -- The URL where Airbyte will be reached; This should match your Ingress host + airbyteUrl: "" + + # Docker image config that will apply to all images. + image: + # Docker registry to pull platform images from, e.g. http://my-registry:8000/ + registry: "" + # Image tag to use for airbyte images. + # Does not include non-airbyte images such as busybox, temporal, minio, etc. + tag: "1.3.0" + + # Docker image pull secret + imagePullSecrets: + - name: regcred + + # -- Auth configuration + auth: + # -- Whether auth is enabled + enabled: true + # -- Admin user configuration + instanceAdmin: + # -- Secret name where the instanceAdmin configuration is stored + secretName: "airbyte-v1-airbyte-secrets" + # -- The first name of the initial user + firstName: "" + # -- The last name of the initial user + lastName: "" + # -- The key within `emailSecretName` where the initial user's email is stored + emailSecretKey: "instance-admin-email" + # -- The key within `passwordSecretName` where the initial user's password is stored + passwordSecretKey: "instance-admin-password" + + # -- SSO Identify Provider configuration; (requires Enterprise) + #identityProvider: + # # -- Secret name where the OIDC configuration is stored + # secretName: "airbyte-v1-airbyte-secrets" + # # -- The identity provider type (e.g. oidc) + # type: "" + # # -- OIDC configuration (required if `auth.identityProvider.type` is "oidc") + # oidc: + # # -- OIDC application domain + # domain: "" + # # -- OIDC application name + # appName: "" + # # -- The key within `clientIdSecretName` where the OIDC client id is stored + # clientIdSecretKey: "client-id" + # # -- The key within `clientSecretSecretName` where the OIDC client secret is stored + # clientSecretSecretKey: "client-secret" + + # -- Environment variables + env_vars: {} + + # -- Database configuration + database: + type: "external" # "external" + + # -- Secret name where database credentials are stored + secretName: "airbyte-v1-airbyte-secrets" + + # -- The database host + host: "airbyte-host" + + # -- The database port + port: "3306" + + # -- The database name + database: "airbyte-v1" + + # -- The database user + user: "rds_admin" + # -- The key within `secretName` where the user is stored + #userSecretKey: "" # e.g. "database-user" + + # -- The database password + password: "pwd" + # -- The key within `secretName` where the password is stored + #passwordSecretKey: "" # e.g."database-password" + + + + storage: + # -- The storage backend type. Supports s3, gcs, azure, minio (default) + type: s3 # default storage used + # -- Secret name where storage provider credentials are stored + storageSecretName: "airbyte-v1-airbyte-secrets" + + # S3 + bucket: ## S3 bucket names that you've created. We recommend storing the following all in one bucket. + log: b-multitenantdev-dev-airbyte-v1-logs + state: b-multitenantdev-dev-airbyte-v1-logs + workloadOutput: b-multitenantdev-dev-airbyte-v1-logs + s3: + region: "us-east-1" ## e.g. us-east-1 + authenticationType: credentials ## Use "credentials" or "instanceProfile" + accessKeyId: "" + secretAccessKey: "" + + + # GCS + #bucket: ## GCS bucket names that you've created. We recommend storing the following all in one bucket. + # log: airbyte-bucket + # state: airbyte-bucket + # workloadOutput: airbyte-bucket + #gcs: + # projectId: + # credentialsJson: /secrets/gcs-log-creds/gcp.json + + # Azure + #bucket: ## Azure Blob Storage container names that you've created. We recommend storing the following all in one bucket. + # log: airbyte-bucket + # state: airbyte-bucket + # workloadOutput: airbyte-bucket + #azure: + # # one of the following: connectionString, connectionStringSecretKey + # connectionString: + # connectionStringSecretKey: + + metrics: + # -- The metric client to configure globally. Supports "otel" + metricClient: "" + # -- The open-telemetry-collector endpoint that metrics will be sent to + otelCollectorEndpoint: "" + + # Jobs resource requests and limits, see http://kubernetes.io/docs/user-guide/compute-resources/ + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. + jobs: + resources: + ## Example: + ## requests: + ## memory: 256Mi + ## cpu: 250m + # -- Job resource requests + requests: {} + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- Job resource limits + limits: {} + + local: + volume: + enabled: false + + kube: + ## JOB_KUBE_ANNOTATIONS + # pod annotations of the sync job and the default pod annotations fallback for others jobs + # -- key/value annotations applied to kube jobs + annotations: {} + + ## JOB_KUBE_LABELS + ## pod labels of the sync job and the default pod labels fallback for others jobs + # -- key/value labels applied to kube jobs + labels: {} + + ## JOB_KUBE_NODE_SELECTORS + ## pod node selector of the sync job and the default pod node selector fallback for others jobs + # -- Node labels for pod assignment + nodeSelector: {} + + ## JOB_KUBE_TOLERATIONS + # -- Node tolerations for pod assignment + # Any boolean values should be quoted to ensure the value is passed through as a string. + tolerations: [] + + ## JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET + # -- image pull secret to use for job pod + main_container_image_pull_secret: "" + + images: + ## JOB_KUBE_BUSYBOX_IMAGE + # -- busybox image used by the job pod + busybox: "busybox:1.35" + ## JOB_KUBE_CURL_IMAGE + # -- curl image used by the job pod + curl: "curlimages/curl:8.1.1" + +## @section Common Parameters + +# -- String to partially override airbyte.fullname template with a string (will prepend the release name) +nameOverride: "" +# -- String to fully override airbyte.fullname template with a string +fullnameOverride: "" + +# Pods Service Account, see https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +serviceAccount: + # -- Specifies whether a ServiceAccount should be created + create: true + # -- Annotations for service account. Evaluated as a template. Only used if `create` is `true`. + annotations: {} + # -- Name of the service account to use. If not set and create is true, a name is generated using the fullname template. + name: *service-account-name + +# -- Sets the AIRBYTE_VERSION environment variable. Defaults to Chart.AppVersion. +## If changing the image tags below, you should probably also update this. +version: "" + +## @section Webapp Parameters + +webapp: + enabled: true + # -- Number of webapp replicas + replicaCount: 1 + + ## webapp.image.repository + ## webapp.image.pullPolicy + ## webapp.image.tag The airbyte webapp image tag. Defaults to the chart's AppVersion + image: + # -- The repository to use for the airbyte webapp image + repository: airbyte/webapp + # -- The pull policy to use for the airbyte webapp image + pullPolicy: IfNotPresent + + # -- Add extra annotations to the webapp pods + podAnnotations: {} + + # -- webapp.podLabels [object] Add extra labels to the webapp pods + podLabels: {} + + # -- Security context for the container + podSecurityContext: + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + ## Configure extra options for the webapp containers' liveness and readiness probes, + ## see https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes + livenessProbe: + # -- Enable livenessProbe on the webapp + enabled: true + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 30 + # -- Period seconds for livenessProbe + periodSeconds: 10 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 1 + # -- Failure threshold for livenessProbe + failureThreshold: 3 + # -- Success threshold for livenessProbe + successThreshold: 1 + + readinessProbe: + # -- Enable readinessProbe on the webapp + enabled: true + # -- Initial delay seconds for readinessProbe + initialDelaySeconds: 10 + # -- Period seconds for readinessProbe + periodSeconds: 10 + # -- Timeout seconds for readinessProbe + timeoutSeconds: 1 + # -- Failure threshold for readinessProbe + failureThreshold: 3 + # -- Success threshold for readinessProbe + successThreshold: 1 + + service: + # -- The service type to use for the webapp service + type: ClusterIP + # -- The service port to expose the webapp on + port: 80 + # -- Annotations for the webapp service resource + annotations: {} + + ## Web app resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- The resources limits for the Web container + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + # -- The requested resources for the Web container + requests: {} + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for webapp pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for webapp pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + ## Configure the ingress resource that allows you to access the Airbyte installation. + ## ref: http://kubernetes.io/docs/user-guide/ingress/ + ingress: + # -- Set to true to enable ingress record generation + enabled: true + # -- Specifies ingressClassName for clusters >= 1.18+ + className: "nginx" + # -- Ingress annotations done as key:value pairs + annotations: + nginx.ingress.kubernetes.io/proxy-read-timeout: "600" + nginx.ingress.kubernetes.io/proxy-send-timeout: "600" + nginx.ingress.kubernetes.io/proxy-connect-timeout: "360" + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + # -- The list of hostnames to be covered with this ingress record. + hosts: + - host: + paths: + - path: / + pathType: ImplementationSpecific + # -- Custom ingress TLS configuration + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + + api: + # -- The webapp API url + url: /api/v1/ + + connector-builder-server: + url: /connector-builder-api + + fullstory: + # -- Whether or not to enable fullstory + enabled: false + + ## Examples (when using `webapp.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: var-run + ## mountPath: /var/run/ + ## - name: var-cache-nginx + ## mountPath: /var/cache/nginx + ## - mountPath: /etc/nginx/conf.d + ## name: nginx-conf-d + ## + # -- Additional volumeMounts for webapp containers + extraVolumeMounts: [] + + ## Examples (when using `webapp.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: var-run + ## emptyDir: {} + ## - name: var-cache-nginx + ## emptyDir: {} + ## - name: nginx-conf-d + ## emptyDir: {} + # -- Additional volumes for webapp pods + extraVolumes: [] + + ## Example: + # extraContainers: + # - name: otel_collector + # image: somerepo/someimage:sometag + # args: [ + # "--important-args" + # ] + # ports: + # - containerPort: 443 + # volumeMounts: + # - name: volumeMountCool + # mountPath: /some/path + # readOnly: true + # -- Additional container for server pods + extraContainers: [] + + ## Example: + # extraInitContainers: + # - name: sleepy + # image: alpine + # command: ['sleep', '60'] + # -- Additional init containers for server pods + extraInitContainers: [] + + ## Example: (With default env vars and values taken from generated config map) + # extraEnv: + # - name: POSTGRES_USER + # valueFrom: + # secretKeyRef: + # name: airbyte-secrets + # key: DATABASE_USER + # - name: POSTGRES_PWD + # valueFrom: + # secretKeyRef: + # name: airbyte-secrets + # key: DATABASE_PASSWORD + # - name: DYNAMIC_CONFIG_FILE_PATH + # value: "config/dynamicconfig/development.yaml" + # - name: DB + # value: "postgresql" + # - name: DB_PORT + # valueFrom: + # configMapKeyRef: + # name: airbyte-env + # key: DATABASE_PORT + # - name: POSTGRES_SEEDS + # valueFrom: + # configMapKeyRef: + # name: airbyte-env + # key: DATABASE_HOST + # -- Supply extra env variables to main container using full notation + extraEnv: [] + ## Example: + ## secrets: + ## DATABASE_PASSWORD: strong-password + ## DATABASE_USER: my-db-user + # -- Supply additional secrets to container + secrets: {} + + ## Example: + ## env_vars: + ## DATABASE_HOST: airbyte-db + ## DATABASE_PORT: 5432 + # -- Supply extra env variables to main container using simplified notation + env_vars: {} + +## @section Pod Sweeper parameters + +pod-sweeper: + enabled: true + image: + # -- The image repository to use for the pod sweeper + repository: bitnami/kubectl + # -- The pull policy for the pod sweeper image + pullPolicy: IfNotPresent + # -- The pod sweeper image tag to use + tag: 1.28.9 + + # -- Add extra annotations to the podSweeper pod + podAnnotations: {} + + # -- Add extra labels to the podSweeper pod + podLabels: {} + + # -- Security context for the container + podSecurityContext: + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + livenessProbe: + # -- Enable livenessProbe on the podSweeper + enabled: true + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 5 + # -- Period seconds for livenessProbe + periodSeconds: 30 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 1 + # -- Failure threshold for livenessProbe + failureThreshold: 3 + # -- Success threshold for livenessProbe + successThreshold: 1 + + readinessProbe: + # -- Enable readinessProbe on the podSweeper + enabled: true + # -- Initial delay seconds for readinessProbe + initialDelaySeconds: 5 + # -- Period seconds for readinessProbe + periodSeconds: 30 + # -- Timeout seconds for readinessProbe + timeoutSeconds: 1 + # -- Failure threshold for readinessProbe + failureThreshold: 3 + # -- Success threshold for readinessProbe + successThreshold: 1 + + ## Pod Sweeper app resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- The resources limits for the podSweeper container + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + # -- The requested resources for the podSweeper container + requests: {} + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for pod assignment, see https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + ## Examples: + ## extraVolumeMounts: + ## - name: tmpdir + ## mountPath: /tmp + # -- Additional volumeMounts for podSweeper container(s). + extraVolumeMounts: [] + + ## Examples: + ## extraVolumes: + ## - name: tmpdir + ## emptyDir: {} + # -- Additional volumes for podSweeper pod(s). + extraVolumes: [] + +## @section Server parameters + +server: + enabled: true + # -- Number of server replicas + replicaCount: 1 + + image: + # -- The repository to use for the airbyte server image. + repository: airbyte/server + # -- the pull policy to use for the airbyte server image + pullPolicy: IfNotPresent + + # -- Add extra annotations to the server pods + podAnnotations: {} + + # -- Add extra labels to the server pods + podLabels: {} + + # -- Security context for the container + podSecurityContext: + # gid=1000(airbyte) + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + ## Configure extra options for the server containers' liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes + livenessProbe: + # -- Enable livenessProbe on the server + enabled: true + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 30 + # -- Period seconds for livenessProbe + periodSeconds: 10 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 10 + # -- Failure threshold for livenessProbe + failureThreshold: 3 + # -- Success threshold for livenessProbe + successThreshold: 1 + + readinessProbe: + # -- Enable readinessProbe on the server + enabled: true + # -- Initial delay seconds for readinessProbe + initialDelaySeconds: 10 + # -- Period seconds for readinessProbe + periodSeconds: 10 + # -- Timeout seconds for readinessProbe + timeoutSeconds: 10 + # -- Failure threshold for readinessProbe + failureThreshold: 3 + # -- Success threshold for readinessProbe + successThreshold: 1 + + ## server resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- The resources limits for the server container + limits: + cpu: 2 + memory: 6Gi + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + # -- The requested resources for the workload launcher container + requests: + cpu: 400m + memory: 1Gi + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for server pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for server pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + log: + # -- The log level to log at + level: "INFO" + + ## Examples (when using `server.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: tmpdir + ## mountPath: /tmp + # -- Additional volumeMounts for server containers + extraVolumeMounts: [] + + ## Examples (when using `server.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: tmpdir + ## emptyDir: {} + # -- Additional volumes for server pods + extraVolumes: [] + + ## Example: + # extraContainers: + # - name: otel_collector + # image: somerepo/someimage:sometag + # args: [ + # "--important-args" + # ] + # ports: + # - containerPort: 443 + # volumeMounts: + # - name: volumeMountCool + # mountPath: /some/path + # readOnly: true + # -- Additional container for server pods + extraContainers: [] + + ## Example: + # extraInitContainers: + # - name: sleepy + # image: alpine + # command: ['sleep', '60'] + # -- Additional init containers for server pods + extraInitContainers: [] + + ## Example: (With default env vars and values taken from generated config map) + # extraEnv: + # - name: AIRBYTE_VERSION + # valueFrom: + # configMapKeyRef: + # name: airbyte-env + # key: AIRBYTE_VERSION + # - name: API_URL + # valueFrom: + # configMapKeyRef: + # name: airbyte-env + # key: API_URL + # - name: TRACKING_STRATEGY + # valueFrom: + # configMapKeyRef: + # name: airbyte-env + # key: TRACKING_STRATEGY + # - name: FULLSTORY + # valueFrom: + # configMapKeyRef: + # name: airbyte-env + # key: FULLSTORY + # - name: INTERNAL_API_HOST + # valueFrom: + # configMapKeyRef: + # name: airbyte-env + # key: INTERNAL_API_HOST + ## + # -- Supply extra env variables to main container using full notation + extraEnv: [] + ## Example: + ## secrets: + ## DATABASE_PASSWORD: strong-password + ## DATABASE_USER: my-db-user + # -- Supply additional secrets to container + secrets: {} + ## Example: + ## env_vars: + ## DATABASE_HOST: airbyte-db + ## DATABASE_PORT: 5432 + # -- Supply extra env variables to main container using simplified notation + env_vars: + BLOTOUT_AUTH_ENDPOINT: "" + BLOTOUT_BASE_URL: "" + +## @section Worker Parameters + +worker: + enabled: true + # -- Number of worker replicas + replicaCount: 1 + + image: + # -- The repository to use for the airbyte worker image. + repository: airbyte/worker + # -- the pull policy to use for the airbyte worker image + pullPolicy: IfNotPresent + + # -- Add extra annotations to the worker pods + podAnnotations: {} + + # -- Add extra labels to the worker pods + podLabels: {} + + # -- Security context for the container + podSecurityContext: + # gid=1000(airbyte) + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + livenessProbe: + # -- Enable livenessProbe on the worker + enabled: true + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 30 + # -- Period seconds for livenessProbe + periodSeconds: 10 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 1 + # -- Failure threshold for livenessProbe + failureThreshold: 3 + # -- Success threshold for livenessProbe + successThreshold: 1 + + readinessProbe: + # -- Enable readinessProbe on the worker + enabled: true + # -- Initial delay seconds for readinessProbe + initialDelaySeconds: 10 + # -- Period seconds for readinessProbe + periodSeconds: 10 + # -- Timeout seconds for readinessProbe + timeoutSeconds: 1 + # -- Failure threshold for readinessProbe + failureThreshold: 3 + # -- Success threshold for readinessProbe + successThreshold: 1 + + ## worker resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + #! -- The resources limits for the worker container + limits: + cpu: 2 + memory: 6Gi + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + # -- The requested resources for the workload launcher container + requests: + cpu: 400m + memory: 1Gi + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for worker pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for worker pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + log: + #! -- The log level to log at. + level: "INFO" + + ## Example: + ## + ## extraEnv: + ## - name: JOB_KUBE_TOLERATIONS + ## value: "key=airbyte-server,operator=Equals,value=true,effect=NoSchedule" + # -- Additional env vars for worker pods + extraEnv: [] + + ## Examples (when using `worker.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: tmpdir + ## mountPath: /tmp + # -- Additional volumeMounts for worker containers + extraVolumeMounts: [] + + ## Examples (when using `worker.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: tmpdir + ## emptyDir: {} + # -- Additional volumes for worker pods + extraVolumes: [] + # -- Additional container for worker pods + extraContainers: [] + + hpa: + enabled: false + + debug: + enabled: false + + ## current no exist documentations + activityMaxAttempt: "" + activityInitialDelayBetweenAttemptsSeconds: "" + activityMaxDelayBetweenAttemptsSeconds: "" + maxNotifyWorkers: 5 + +## @section Workload Launcher Parameters + +workload-launcher: + enabled: true + # -- Number of workload launcher replicas + replicaCount: 1 + + image: + # -- The repository to use for the airbyte workload launcher image. + repository: airbyte/workload-launcher + # -- The pull policy to use for the airbyte workload launcher image + pullPolicy: IfNotPresent + + # -- Add extra annotations to the workload launcher pods + podAnnotations: {} + + # -- Add extra labels to the workload launcher pods + podLabels: {} + + # -- Security context for the container + podSecurityContext: + # gid=1000(airbyte) + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + livenessProbe: + # -- Enable livenessProbe on the workload launcher + enabled: true + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 30 + # -- Period seconds for livenessProbe + periodSeconds: 10 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 1 + # -- Failure threshold for livenessProbe + failureThreshold: 3 + # -- Success threshold for livenessProbe + successThreshold: 1 + + readinessProbe: + # -- Enable readinessProbe on the workload launcher + enabled: true + # -- Initial delay seconds for readinessProbe + initialDelaySeconds: 10 + # -- Period seconds for readinessProbe + periodSeconds: 10 + # -- Timeout seconds for readinessProbe + timeoutSeconds: 1 + # -- Failure threshold for readinessProbe + failureThreshold: 3 + # -- Success threshold for readinessProbe + successThreshold: 1 + + ## workload launcher resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- The resources limits for the workload launcher container + limits: + cpu: 2 + memory: 6Gi + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + # -- The requested resources for the workload launcher container + requests: + cpu: 400m + memory: 1Gi + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for workload launcher pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # Affinity and anti-affinity for workload launcher pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + log: + # -- The log level to log at + level: "INFO" + + ## Example: + ## + ## extraEnv: + ## - name: JOB_KUBE_TOLERATIONS + ## value: "key=airbyte-server,operator=Equals,value=true,effect=NoSchedule" + # -- Additional env vars for workload launcher pods + extraEnv: [] + + ## Examples (when using `workload launcher.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: tmpdir + ## mountPath: /tmp + # -- Additional volumeMounts for workload launcher containers + extraVolumeMounts: [] + + ## Examples (when using `workload-launcher.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: tmpdir + ## emptyDir: {} + # -- Additional volumes for workload launcher pods + extraVolumes: [] + + extraContainers: [] + + hpa: + enabled: false + + debug: + enabled: false + + containerOrchestrator: + # -- Enable or disable Orchestrator + enabled: true + # -- Orchestrator image + # This is a template string that will be passed to the "tpl" helper. + image: "airbyte/container-orchestrator:{{ default .Chart.AppVersion .Values.global.image.tag }}" + + connectorSidecar: + # -- Connector Sidecar image + # This is a template string that will be passed to the "tpl" helper. + image: "airbyte/connector-sidecar:{{ default .Chart.AppVersion .Values.global.image.tag }}" + + workloadInit: + # -- Workload init image + # This is a template string that will be passed to the "tpl" helper. + image: "airbyte/workload-init-container:{{ default .Chart.AppVersion .Values.global.image.tag }}" + + ## current no exist documentations + activityMaxAttempt: "" + activityInitialDelayBetweenAttemptsSeconds: "" + activityMaxDelayBetweenAttemptsSeconds: "" + + maxNotifyWorkers: 5 + +## @section Rollout Worker Parameters + +connector-rollout-worker: + enabled: false + # -- Number of connector rollout worker replicas + replicaCount: 1 + + image: + # -- The repository to use for the airbyte connector rollout worker image. + repository: airbyte/connector-rollout-worker + # -- The pull policy to use for the airbyte connector rollout worker image + pullPolicy: IfNotPresent + + # -- Add extra annotations to the connector rollout worker pods + podAnnotations: {} + + # -- Add extra labels to the connector rollout worker pods + podLabels: {} + + # -- Security context for the container + podSecurityContext: + # gid=1000(airbyte) + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + livenessProbe: + # -- Enable livenessProbe on the connector rollout worker + enabled: true + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 30 + # -- Period seconds for livenessProbe + periodSeconds: 10 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 1 + # -- Failure threshold for livenessProbe + failureThreshold: 3 + # -- Success threshold for livenessProbe + successThreshold: 1 + + readinessProbe: + # -- Enable readinessProbe on the connector rollout worker + enabled: true + # -- Initial delay seconds for readinessProbe + initialDelaySeconds: 10 + # -- Period seconds for readinessProbe + periodSeconds: 10 + # -- Timeout seconds for readinessProbe + timeoutSeconds: 1 + # -- Failure threshold for readinessProbe + failureThreshold: 3 + # -- Success threshold for readinessProbe + successThreshold: 1 + + ## connector rollout worker resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + requests: + memory: 1Gi + cpu: 350m + limits: + cpu: 2 + memory: 6Gi + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- The resources limits for the connector rollout worker container + # limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + # -- The requested resources for the connector rollout worker container + # requests: {} + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for connector rollout worker pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # Affinity and anti-affinity for connector rollout worker pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + log: + # -- The log level to log at + level: "INFO" + + ## Example: + ## + ## extraEnv: + ## - name: JOB_KUBE_TOLERATIONS + ## value: "key=airbyte-server,operator=Equals,value=true,effect=NoSchedule" + # -- Additional env vars for connector rollout worker pods + extraEnv: [] + + ## Examples (when using `connector rollout worker.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: tmpdir + ## mountPath: /tmp + # -- Additional volumeMounts for connector rollout worker containers + extraVolumeMounts: [] + + ## Examples (when using `connector-rollout-worker.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: tmpdir + ## emptyDir: {} + # -- Additional volumes for connector rollout worker pods + extraVolumes: [] + + extraContainers: [] + + hpa: + enabled: false + + debug: + enabled: false + + containerOrchestrator: + # -- Enable or disable Orchestrator + enabled: true + # -- Orchestrator image + image: "" + + ## current no exist documentations + activityMaxAttempt: "" + activityInitialDelayBetweenAttemptsSeconds: "" + activityMaxDelayBetweenAttemptsSeconds: "" + + maxNotifyWorkers: 5 + +## @section Metrics parameters +metrics: + enabled: false + + # -- Number of metrics-reporter replicas + replicaCount: 1 + + image: + # -- The repository to use for the airbyte metrics-reporter image. + repository: airbyte/metrics-reporter + # -- The pull policy to use for the airbyte metrics-reporter image + pullPolicy: IfNotPresent + + # -- Add extra annotations to the metrics-reporter pod + podAnnotations: {} + + # -- Add extra labels to the metrics-reporter pod + podLabels: {} + + # -- Security context for the container + podSecurityContext: + # gid=1000(airbyte) + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + ## metrics-reporter app resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- The resources limits for the metrics-reporter container + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + # -- The requested resources for the metrics-reporter container + requests: {} + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for metrics-reporter pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for metrics-reporter pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + ## Example: + ## + ## extraEnv: + ## - name: SAMPLE_ENV_VAR + ## value: "key=sample-value" + # -- Additional env vars for metrics-reporter pods + extraEnv: [] + + ## Examples (when using `metrics.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: tmpdir + ## mountPath: /tmp + # -- Additional volumeMounts for metrics-reporter containers + extraVolumeMounts: [] + + ## Examples (when using `metrics.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: tmpdir + ## emptyDir: {} + # -- Additional volumes for metrics-reporter pods + extraVolumes: [] + + extraContainers: [] + + secrets: {} + + env_vars: {} + +## @section Bootloader Parameters + +airbyte-bootloader: + enabled: true + image: + # -- The repository to use for the airbyte bootloader image. + repository: airbyte/bootloader + # -- The pull policy to use for the airbyte bootloader image + pullPolicy: IfNotPresent + + # -- Add extra annotations to the bootloader pod + podAnnotations: {} + + # -- Add extra labels to the bootloader pod + podLabels: {} + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for worker pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + ## Bootloader resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## requests: + ## memory: 256Mi + ## cpu: 250m + # -- The requested resources for the airbyte bootloader image + requests: {} + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- The resources limits for the airbyte bootloader image + limits: {} + + # -- Affinity and anti-affinity for bootloader pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + # -- Security context for the container + podSecurityContext: + # gid=1000(airbyte) + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + ## Example: (With default env vars and values taken from generated config map) + ## extraEnv: + ## - name: AIRBYTE_VERSION + ## valueFrom: + ## configMapKeyRef: + ## name: airbyte-env + ## key: AIRBYTE_VERSION + ## - name: DATABASE_HOST + ## valueFrom: + ## configMapKeyRef: + ## name: airbyte-env + ## key: DATABASE_HOST + ## - name: DATABASE_PORT + ## valueFrom: + ## configMapKeyRef: + ## name: airbyte-env + ## key: DATABASE_PORT + ## - name: DATABASE_PASSWORD + ## valueFrom: + ## secretKeyRef: + ## name: airbyte-secrets + ## key: DATABASE_PASSWORD + ## - name: DATABASE_URL + ## valueFrom: + ## configMapKeyRef: + ## name: airbyte-env + ## key: DATABASE_URL + ## - name: DATABASE_USER + ## valueFrom: + ## secretKeyRef: + ## name: airbyte-secrets + ## key: DATABASE_USER + # -- Supply extra env variables to main container using full notation + extraEnv: [] + ## Example: + ## secrets: + ## DATABASE_PASSWORD: strong-password + ## DATABASE_USER: my-db-user + # -- Supply additional secrets to container + secrets: {} + + ## Example: + ## env_vars: + ## DATABASE_HOST: airbyte-db + ## DATABASE_PORT: 5432 + # -- Supply extra env variables to main container using simplified notation + env_vars: {} + ## Example: + # extraContainers: + # - name: otel_collector + # image: somerepo/someimage:sometag + # args: [ + # "--important-args" + # ] + # ports: + # - containerPort: 443 + # volumeMounts: + # - name: volumeMountCool + # mountPath: /some/path + # readOnly: true + # -- Additional container for server pod(s) + extraContainers: [] + + ## Example: + # extraInitContainers: + # - name: sleepy + # image: alpine + # command: ['sleep', '60'] + # -- Additional init containers for server pods + extraInitContainers: [] + + ## Examples (when using `containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: tmpdir + ## mountPath: /tmp + # -- Additional volumeMounts for server containers + extraVolumeMounts: [] + + ## Examples (when using `containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: tmpdir + ## emptyDir: {} + # -- Additional volumes for server pods + extraVolumes: [] + +## @section Temporal parameters +## TODO: Move to consuming temporal from a dedicated helm chart + +temporal: + enabled: true + # -- The number of temporal replicas to deploy + replicaCount: 1 + + image: + # -- The temporal image repository to use + repository: temporalio/auto-setup + # -- The pull policy for the temporal image + pullPolicy: IfNotPresent + # -- The temporal image tag to use + tag: "1.23.0" + + service: + # -- The Kubernetes Service Type + type: ClusterIP + # -- The temporal port and exposed kubernetes port + port: 7233 + + # -- Add extra annotations to the temporal pod + podAnnotations: {} + + # -- Add extra labels to the temporal pod + podLabels: {} + + # -- Security context for the container + podSecurityContext: + # gid=1000(temporal) + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(temporal) + runAsUser: 1000 + # gid=1000(temporal) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + ## Examples (when using `temporal.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraInitContainers: + ## - name: config-loader + ## image: temporalio/auto-setup:1.23 + ## command: + ## - /bin/sh + ## - -c + ## - >- + ## find /etc/temporal/config/ -maxdepth 1 -mindepth 1 -exec cp -ar {} /config/ \; + ## volumeMounts: + ## - name: config + ## mountPath: /config + # -- Additional InitContainers to initialize the pod + extraInitContainers: [] + + livenessProbe: + # -- Enable livenessProbe on the temporal + enabled: false + + readinessProbe: + # -- Enable readinessProbe on the temporal + enabled: false + + # -- Node labels for temporal pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for temporal pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for temporal pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + ## Example: + ## + ## extraEnv: + ## - name: SAMPLE_ENV_VAR + ## value: "key=sample-value" + # -- Additional env vars for temporal pod(s). + extraEnv: [] + + ## Examples (when using `temporal.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: tmpdir + ## mountPath: /tmp + ## - name: config + ## mountPath: /etc/temporal/config + # -- Additional volumeMounts for temporal containers + extraVolumeMounts: [] + + ## Examples (when using `temporal.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: tmpdir + ## emptyDir: {} + ## - name: config + ## emptyDir: {} + # -- Additional volumes for temporal pods + extraVolumes: [] + + ## Temporal resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## requests: + ## memory: 256Mi + ## cpu: 250m + # -- The requested resources for temporal pods + requests: {} + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- The resources limits for temporal pods + limits: {} + + extraContainers: [] + +## @section Temporal parameters +## TODO: Move to consuming temporal from a dedicated helm chart + +temporal-ui: + enabled: false + # -- The number of temporal-ui replicas to deploy + replicaCount: 1 + + image: + # -- The temporal-ui image repository to use + repository: temporalio/ui + # -- The pull policy for the temporal-ui image + pullPolicy: IfNotPresent + tag: "2.30.1" + + service: + # -- The Kubernetes Service Type + type: ClusterIP + # -- The temporal-ui port and exposed kubernetes port + port: 8088 + + # -- Add extra annotations to the temporal-ui pod + podAnnotations: {} + + # -- Add extra labels to the temporal-ui pod + podLabels: {} + + # -- Security context for the container + podSecurityContext: + # gid=1000(temporal-ui) + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: false + # uid=1000(temporal-ui) + runAsUser: 0 + # gid=1000(temporal-ui) + runAsGroup: 0 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + ## Examples (when using `temporal.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraInitContainers: + ## - name: config-loader + ## image: temporalio/ui:2.30.1 + ## command: + ## - /bin/sh + ## - -c + ## - >- + ## find /etc/temporal/config/ -maxdepth 1 -mindepth 1 -exec cp -ar {} /config/ \; + ## volumeMounts: + ## - name: config + ## mountPath: /config + # -- Additional InitContainers to initialize the pod + extraInitContainers: [] + + livenessProbe: + # -- Enable livenessProbe on the temporal-ui + enabled: false + + readinessProbe: + # -- Enable readinessProbe on the temporal-ui + enabled: false + + # -- Node labels for temporal-ui pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for temporal-ui pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for temporal-ui pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + ## Example: + ## + ## extraEnv: + ## - name: SAMPLE_ENV_VAR + ## value: "key=sample-value" + # -- Additional env vars for temporal-ui pod(s). + extraEnv: [] + + ## Examples (when using `temporal.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: tmpdir + ## mountPath: /tmp + ## - name: config + ## mountPath: /etc/temporal/config + # -- Additional volumeMounts for temporal containers + extraVolumeMounts: [] + + ## Examples (when using `temporal.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: tmpdir + ## emptyDir: {} + ## - name: config + ## emptyDir: {} + # -- Additional volumes for temporal-ui pods + extraVolumes: [] + + ## Temporal UI resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## requests: + ## memory: 256Mi + ## cpu: 250m + # -- The requested resources for temporal-ui pods + requests: {} + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- The resources limits for temporal-ui pods + limits: {} + + extraContainers: [] + +## @section Airbyte Database parameters + +# PostgreSQL chart configuration, see https://github.com/bitnami/charts/blob/master/bitnami/postgresql/values.yaml +postgresql: + # -- Switch to enable or disable the PostgreSQL helm chart + enabled: false + + ## image.repository Repository for airbyte-db statefulset + image: + repository: airbyte/db + # -- Airbyte Postgresql username + postgresqlUsername: airbyte + # -- Airbyte Postgresql password + postgresqlPassword: airbyte + # -- Airbyte Postgresql database + postgresqlDatabase: db-airbyte + # fullnameOverride: *db-hostname + ## This secret is used in case of postgresql.enabled=true and we would like to specify password for newly created postgresql instance + # -- Name of an existing secret containing the PostgreSQL password ('postgresql-password' key) + existingSecret: "" + podSecurityContext: + # gid=70(postgres) + fsGroup: 70 + containerSecurityContext: + # -- Ensures the container will run with a non-root user + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=70(postgres) + runAsUser: 70 + # gid=70(postgres) + runAsGroup: 70 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + commonAnnotations: + # -- It will determine when the hook should be rendered + helm.sh/hook: pre-install + # -- The order in which the hooks are executed. If weight is lower, it has higher priority + helm.sh/hook-weight: "-1" + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for postgresql pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for postgresql pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + +# External PostgreSQL configuration, All of these values are only used when postgresql.enabled is set to false +externalDatabase: + # -- Database host + host: "" + # -- non-root Username for Airbyte Database + user: "" + # -- Database password + password: "" + # -- Name of an existing secret resource containing the DB password + existingSecret: "" + # -- Name of an existing secret key containing the DB password + existingSecretPasswordKey: "" + # -- Database name + database: "" + # -- Database port number + port: "" + # -- Database full JDBL URL (ex: jdbc:postgresql://host:port/db?parameters) + jdbcUrl: "" + +minio: + image: + # -- Minio image used by Minio helm chart + repository: minio/minio + # -- Minio tag image + tag: RELEASE.2023-11-20T22-40-07Z + + mcImage: + repository: airbyte/mc + tag: latest + + storage: + volumeClaimValue: 500Mi + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + # -- Tolerations for minio pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + + # -- Affinity and anti-affinity for minio pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + resources: + requests: + memory: 1Gi + cpu: 250m + limits: + cpu: 1 + memory: 4Gi +## @section cron parameters +cron: + enabled: true + # -- Number of cron replicas + replicaCount: 1 + image: + # -- The repository to use for the airbyte cron image. + repository: airbyte/cron + # -- The pull policy to use for the airbyte cron image + pullPolicy: IfNotPresent + + # -- Add extra annotations to the cron pods + podAnnotations: {} + + # -- Add extra labels to the cron pods + podLabels: {} + + # -- Security context for the container + podSecurityContext: + # uid=1000(airbyte) + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + livenessProbe: + # -- Enable livenessProbe on the cron + enabled: true + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 30 + # -- Period seconds for livenessProbe + periodSeconds: 10 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 1 + # -- Failure threshold for livenessProbe + failureThreshold: 3 + # -- Success threshold for livenessProbe + successThreshold: 1 + + readinessProbe: + # -- Enable readinessProbe on the cron + enabled: true + # -- Initial delay seconds for readinessProbe + initialDelaySeconds: 10 + # -- Period seconds for readinessProbe + periodSeconds: 10 + # -- Timeout seconds for readinessProbe + timeoutSeconds: 1 + # -- Failure threshold for readinessProbe + failureThreshold: 3 + # -- Success threshold for readinessProbe + successThreshold: 1 + + ## cron resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- The resources limits for the cron container + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + # -- The requested resources for the cron container + requests: {} + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for cron pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for cron pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + log: + # -- The log level to log at. + level: "INFO" + + ## Examples (when using `cron.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: tmpdir + ## mountPath: /tmp + # -- Additional volumeMounts for cron containers + extraVolumeMounts: [] + + ## Examples (when using `cron.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: tmpdir + ## emptyDir: {} + # -- Additional volumes for cron pods + extraVolumes: [] + + ## Example: + # extraContainers: + # - name: otel_collector + # image: somerepo/someimage:sometag + # args: [ + # "--important-args" + # ] + # ports: + # - containerPort: 443 + # volumeMounts: + # - name: volumeMountCool + # mountPath: /some/path + # readOnly: true + # -- Additional container for cron pods + extraContainers: [] + + ## Example: + # extraInitContainers: + # - name: sleepy + # image: alpine + # command: ['sleep', '60'] + # -- Additional init containers for cron pods + extraInitContainers: [] + + ## Example: (With default env vars and values taken from generated config map) + # extraEnv: + # - name: AIRBYTE_VERSION + # valueFrom: + # configMapKeyRef: + # name: airbyte-env + # key: AIRBYTE_VERSION + # - name: API_URL + # valueFrom: + # configMapKeyRef: + # name: airbyte-env + # key: API_URL + # - name: TRACKING_STRATEGY + # valueFrom: + # configMapKeyRef: + # name: airbyte-env + # key: TRACKING_STRATEGY + # - name: FULLSTORY + # valueFrom: + # configMapKeyRef: + # name: airbyte-env + # key: FULLSTORY + # - name: INTERNAL_API_HOST + # valueFrom: + # configMapKeyRef: + # name: airbyte-env + # key: INTERNAL_API_HOST + # -- Supply extra env variables to main container using full notation + extraEnv: [] + ## Example: + ## secrets: + ## DATABASE_PASSWORD: strong-password + ## DATABASE_USER: my-db-user + # -- Supply additional secrets to container + secrets: {} + ## Example: + ## env_vars: + ## DATABASE_HOST: airbyte-db + ## DATABASE_PORT: 5432 + # -- Supply extra env variables to main container using simplified notation + env_vars: {} + +connector-builder-server: + enabled: true + # -- Number of connector-builder-server replicas + replicaCount: 1 + + image: + # -- The repository to use for the airbyte connector-builder-server image. + repository: airbyte/connector-builder-server + # -- The pull policy to use for the airbyte connector-builder-server image + pullPolicy: IfNotPresent + + # -- Security context for the container + podSecurityContext: + # gid=1000(airbyte) + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + livenessProbe: + # -- Enable livenessProbe on the server + enabled: true + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 30 + # -- Period seconds for livenessProbe + periodSeconds: 10 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 10 + # -- Failure threshold for livenessProbe + failureThreshold: 3 + # -- Success threshold for livenessProbe + successThreshold: 1 + + readinessProbe: + # -- Enable readinessProbe on the server + enabled: true + # -- Initial delay seconds for readinessProbe + initialDelaySeconds: 10 + # -- Period seconds for readinessProbe + periodSeconds: 10 + # -- Timeout seconds for readinessProbe + timeoutSeconds: 10 + # -- Failure threshold for readinessProbe + failureThreshold: 3 + # -- Success threshold for readinessProbe + successThreshold: 1 + + ## connector-builder-server resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- The resources limits for the connector-builder-server container + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + # -- The requested resources for the connector-builder-server container + requests: {} + + log: + # -- The log level to log at. + level: "INFO" + + env_vars: {} + service: + port: 80 + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for webapp pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for webapp pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + +keycloak: + enabled: false + env_vars: {} + + auth: + adminUsername: airbyteAdmin + adminPassword: keycloak123 + + image: + repository: airbyte/keycloak + pullPolicy: IfNotPresent + + # -- Security context for the container + podSecurityContext: + fsGroup: 1000 + + initContainers: + initDb: + image: "postgres:13-alpine" + + initContainerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for webapp pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for webapp pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + +keycloak-setup: + enabled: false + env_vars: {} + + image: + repository: airbyte/keycloak-setup + pullPolicy: IfNotPresent + + # -- Security context for the container + podSecurityContext: + # gid=1000(airbyte) + fsGroup: 1000 + + initContainerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + initContainers: + keycloakReadinessCheck: + image: "curlimages/curl:8.1.1" + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for webapp pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for webapp pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + +workload-api-server: + enabled: true + + bearerToken: token + + # -- workload-api-server replicas + replicaCount: 1 + + image: + # -- The repository to use for the airbyte-workload-api-server image. + repository: airbyte/workload-api-server + # -- The pull policy to use for the airbyte-workload-api-server image + pullPolicy: IfNotPresent + + # -- Security context for the container + podSecurityContext: + # gid=1000(airbyte) + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + livenessProbe: + # -- Enable livenessProbe on the server + enabled: true + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 30 + # -- Period seconds for livenessProbe + periodSeconds: 10 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 10 + # -- Failure threshold for livenessProbe + failureThreshold: 3 + # -- Success threshold for livenessProbe + successThreshold: 1 + + readinessProbe: + # -- Enable readinessProbe on the server + enabled: true + # -- Initial delay seconds for readinessProbe + initialDelaySeconds: 10 + # -- Period seconds for readinessProbe + periodSeconds: 10 + # -- Timeout seconds for readinessProbe + timeoutSeconds: 10 + # -- Failure threshold for readinessProbe + failureThreshold: 3 + # -- Success threshold for readinessProbe + successThreshold: 1 + + ## airbyte-workload-api-server resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- The resources limits for the airbyte-workload-api-server container + limits: + cpu: 2 + memory: 6Gi + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + # -- The requested resources for the workload launcher container + requests: + cpu: 400m + memory: 1Gi + + log: + # -- The log level at which to log + level: "INFO" + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for webapp pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for webapp pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + env_vars: {} + service: + port: 8007 + + # Configure the ingress resource that allows you to access the Airbyte Workload API, see http://kubernetes.io/docs/user-guide/ingress/ + ingress: + # -- Set to true to enable ingress record generation + enabled: false + # -- Specifies ingressClassName for clusters >= 1.18+ + className: "" + # -- Ingress annotations done as key:value pairs + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + # -- The list of hostnames to be covered with this ingress record + hosts: [] + # - host: chart-example.local + # paths: + # - path: / + # pathType: ImplementationSpecific + # -- Custom ingress TLS configuration + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +featureflag-server: + enabled: false + + # -- workload-api-server replicas + replicaCount: 1 + + image: + # -- The repository to use for the airbyte-workload-api-server image. + repository: airbyte/featureflag-server + # -- The pull policy to use for the airbyte-workload-api-server image + pullPolicy: IfNotPresent + + # -- Security context for the container + podSecurityContext: + # gid=1000(airbyte) + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + livenessProbe: + # -- Enable livenessProbe on the server + enabled: true + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 30 + # -- Period seconds for livenessProbe + periodSeconds: 10 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 10 + # -- Failure threshold for livenessProbe + failureThreshold: 3 + # -- Success threshold for livenessProbe + successThreshold: 1 + + readinessProbe: + # -- Enable readinessProbe on the server + enabled: true + # -- Initial delay seconds for readinessProbe + initialDelaySeconds: 10 + # -- Period seconds for readinessProbe + periodSeconds: 10 + # -- Timeout seconds for readinessProbe + timeoutSeconds: 10 + # -- Failure threshold for readinessProbe + failureThreshold: 3 + # -- Success threshold for readinessProbe + successThreshold: 1 + + ## airbyte-workload-api-server resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- The resources limits for the airbyte-workload-api-server container + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + # -- The requested resources for the airbyte-workload-api-server container + requests: {} + + log: + # -- The log level at which to log + level: "INFO" + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for webapp pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for webapp pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + env_vars: {} + service: + port: 8007 + + # Configure the ingress resource that allows you to access the Airbyte Workload API, see http://kubernetes.io/docs/user-guide/ingress/ + ingress: + # -- Set to true to enable ingress record generation + enabled: false + # -- Specifies ingressClassName for clusters >= 1.18+ + className: "" + # -- Ingress annotations done as key:value pairs + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + # -- The list of hostnames to be covered with this ingress record + hosts: [] + # - host: chart-example.local + # paths: + # - path: / + # pathType: ImplementationSpecific + # -- Custom ingress TLS configuration + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +testWebapp: + image: + repository: busybox + tag: latest diff --git a/helm/airbyte-v2/.helmignore b/helm/airbyte-v2/.helmignore new file mode 100644 index 00000000..c638f485 --- /dev/null +++ b/helm/airbyte-v2/.helmignore @@ -0,0 +1,26 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ + +Makefile +config.yaml diff --git a/helm/airbyte-v2/Chart.lock b/helm/airbyte-v2/Chart.lock new file mode 100644 index 00000000..c4fa89d6 --- /dev/null +++ b/helm/airbyte-v2/Chart.lock @@ -0,0 +1,3 @@ +dependencies: [] +digest: sha256:643d5437104296e21d906ecb15b2c96ad278f20cfc4af53b12bb6069bd853726 +generated: "2025-05-08T13:08:53.01932485Z" diff --git a/helm/airbyte-v2/Chart.yaml b/helm/airbyte-v2/Chart.yaml new file mode 100644 index 00000000..eb66b6b5 --- /dev/null +++ b/helm/airbyte-v2/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +appVersion: 2.0.0 +description: Helm chart to deploy airbyte +name: airbyte-v2 +type: application +version: "v2.0.69" diff --git a/helm/airbyte-v2/templates/NOTES.txt b/helm/airbyte-v2/templates/NOTES.txt new file mode 100644 index 00000000..9caef835 --- /dev/null +++ b/helm/airbyte-v2/templates/NOTES.txt @@ -0,0 +1,49 @@ +{{- if .Values.webapp.ingress.enabled }} +Airbyte is available at: +{{- range $host := .Values.webapp.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.webapp.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.webapp.service.type }} +Get the application URL by running these commands: + + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "airbyte.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.webapp.service.type }} +Get the application URL by running these commands: + + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "airbyte.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.webapp.service.port }} + +NOTE: It may take a few minutes for the LoadBalancer IP to be available. +You can watch the status of by running 'kubectl get -n {{ .Release.Namespace }} svc -w {{ include "airbyte.fullname" . }}' +{{- else if contains "ClusterIP" .Values.webapp.service.type }} +Get the application URL by running these commands: + + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl -n {{ .Release.Namespace }} port-forward deployment/{{ .Release.Name }}-webapp 8080:8080 +{{- end }} + +Auth: This chart uses Airbyte built-in auth only. Log in via the Airbyte instance admin (or your configured identity provider) to access the API and UI. + +Troubleshooting: +- Worker: "Premature end of stream, expectedLastEventID=22 but no more events after eventID=20" / workflow stuck (discover_, check_): Temporal workflow history is incomplete or mismatched. Check Temporal server and DB health; ensure worker and Temporal server versions match. Restart worker after Temporal is healthy: kubectl rollout restart deployment/{{ .Release.Name }}-worker -n {{ .Release.Namespace }}. Stuck workflows may need to be terminated in Temporal UI or via tctl. +- Using a separate Temporal cluster for v2: Set temporal.host to your v2 Temporal address (e.g. "temporal-v2.other-namespace.svc.cluster.local:7233" for a Temporal deployed in another namespace, or an external host:port). Set temporal.enabled: false so this chart does not deploy Temporal; all components will connect to the address in temporal.host. Ensure that Temporal is reachable (same cluster: use FQDN; external: use ingress/LB and TLS if required). +- Inspecting Temporal pod (correct commands): To see TEMPORAL_HOST inside the pod use "echo $TEMPORAL_HOST" (print the variable). Do not run "TEMPORAL_HOST" (shell tries to run a command with that name) or "$TEMPORAL_HOST" (shell runs the value as a command, hence "not found"). Example: kubectl exec deployment/{{ .Release.Name }}-temporal -n {{ .Release.Namespace }} -- sh -c 'echo TEMPORAL_HOST=$TEMPORAL_HOST; env | grep -E "TEMPORAL|PORT"'. To list env from outside: kubectl exec deployment/{{ .Release.Name }}-temporal -n {{ .Release.Namespace }} -- env | grep TEMPORAL. +- v1 and v2 sharing one Temporal (logs show both): If {{ .Release.Name }}-temporal logs show "Current reachable members" with two IPs (e.g. 10.0.26.21 and 10.0.26.28) and task queues for both airbyte-v1-* and airbyte-v2-*, then (1) this Temporal has multiple replicas forming one cluster, and (2) Airbyte v1 is connecting to this same cluster (v1's TEMPORAL_HOST resolves to this cluster). ForceLoadTaskQueuePartition occurs when one replica calls another that doesn't implement that gRPC method (version mismatch). Fix: give v2 a dedicated Temporal (temporal.replicaCount: 1 and ensure v1 does not point at v2's Temporal; or temporal.enabled: false and temporal.host to a separate Temporal), and ensure all Temporal server pods use the same image/tag. +- TEMPORAL_HOST airbyte-v1-temporal:7233 vs airbyte-v2-temporal:7233 but still same cluster: Short names resolve only in the pod's namespace. So v1 pods in namespace "airbyte" resolve airbyte-v1-temporal to airbyte/airbyte-v1-temporal; v2 pods in "airbyte-v2" resolve airbyte-v2-temporal to airbyte-v2/airbyte-v2-temporal—those are different clusters. If both appear in one Temporal's logs, one release is likely pointing at the other's Temporal (e.g. temporal.host override to the other's address, or both in the same namespace with only one Temporal deployed). Check each release's values for temporal.host and that each namespace has its own Temporal deployment. +- Temporal server: "unknown method ForceLoadTaskQueuePartition" / "Failed to force load non-root partition": Version mismatch between Temporal server replicas—one pod expects a gRPC method another does not implement. With temporalio/auto-setup:1.28.2, ensure every Temporal pod uses the same image tag (temporal.replicaCount: 1 or all replicas same tag). If you see task queue names like "airbyte-v1-worker", "airbyte-v1-temporal", "1@airbyte-v1-worker" in {{ .Release.Name }}-temporal or {{ .Release.Name }}-worker logs, then Airbyte v1 and v2 are sharing one Temporal cluster (v1's TEMPORAL_HOST points at v2's Temporal). That causes ForceLoadTaskQueuePartition, "Premature end of stream", and v1 workers running activities for v2 workflows (or 404/NPE). Fix: give v2 its own Temporal and ensure v1 does not point at it—in v1's Helm values set TEMPORAL_HOST to v1's Temporal only (e.g. airbyte-v1-temporal.airbyte-v1:7233); in v2 use temporal.enabled: true (default) so v2 deploys its own Temporal, or temporal.enabled: false and temporal.host to a dedicated v2-only Temporal. Use one Temporal server image version for all replicas of that cluster. +- Temporal pod shows "airbyte-workers" and "Connection refused" to {{ .Release.Name }}-temporal:7233: The temporal deployment must run the Temporal server (temporalio/auto-setup), not the Airbyte workers image. If you overrode temporal.image to airbyte/worker, revert it so temporal runs temporalio/auto-setup; run workers in the worker deployment ({{ .Release.Name }}-worker) only. The workers app is a client of Temporal and needs a separate pod listening on 7233. +- Temporal pod: "/etc/temporal/auto-setup.sh: No such file or directory": The chart no longer overrides temporal.command by default so temporalio/auto-setup uses its image entrypoint. If you had set temporal.command for a different image, remove it (set to [] or delete) when using temporalio/auto-setup. For airbyte/temporal image use temporal.command: ["/bin/sh", "-c", "exec ./entrypoint.sh autosetup"]. +- "Could not resolve placeholder ${STORAGE_TYPE}" (Micronaut/Java in temporal or other pod): The chart injects STORAGE_TYPE into the Temporal deployment. Ensure global.storage.type is set (default "s3"). +- "Error resolving property value [${airbyte.workspace.root}]. Property doesn't exist" in temporal pod: The chart injects WORKSPACE_ROOT into the Temporal deployment (default "/workspace"). +- "Error resolving property value [${airbyte.internal-api.base-path}]. Property doesn't exist" in temporal pod: The chart injects AIRBYTE_INTERNAL_API_BASE_PATH (internal server URL) into the Temporal deployment so the Airbyte workers app can reach the API. +- "Could not get the current Airbyte user due to an internal error" / NoSuchElementException in SecurityAwareCurrentUserService: The server received a request without a valid session (cookie missing or not sent). Use the app from the same origin as login (e.g. https://airbytev2.blotout.io), ensure global.airbyteUrl is set to that URL, and use cookieSameSiteSetting: Lax. Clear site data and log in again; retry the action. If this appears in server logs when the workload-launcher calls the API, the dataplane token endpoint may be subject to the same session check and return 500; upgrade the Airbyte server image to a version that exempts /api/v1/dataplanes/token from requiring a current user, or check Airbyte docs/issues for security configuration. +- 504 (or intermittent 504) when creating a source: Ingress proxy default is 60s; discovery can be slow. Add to webapp.ingress.annotations: proxy-read-timeout: "600", proxy-send-timeout: "600", proxy-connect-timeout: "300" (nginx.ingress.kubernetes.io/ prefix; values in seconds). Retry after increasing timeouts. +- 401 on /api/v1: Expected when there is no valid session. Open the app from the root URL, log in with Airbyte, then use the UI. +- Sources or connections not loading / "Could not find configuration for STANDARD_SYNC: ": Clear site data for this host: DevTools → Application → Storage → Clear site data for the Airbyte origin, then reload and open the app from the root URL. +- Workload-launcher: "Failed to connect to {{ .Release.Name }}-airbyte-server-svc... Connection refused" / "Failed to initialize data-plane": The launcher cannot reach the Airbyte server. Ensure the server pod is running and ready: kubectl get pods -n {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }} | grep server; kubectl logs deployment/{{ .Release.Name }}-server -n {{ .Release.Namespace }}. Fix any server startup errors first, then restart the workload-launcher: kubectl rollout restart deployment/{{ .Release.Name }}-workload-launcher -n {{ .Release.Namespace }}. +- Job pod init container: "Configuration requires entries under the prefix: [datasources.local-secrets]" / LocalTestingSecretPersistence / NoSuchBeanException @Named('local-secrets'): The workload-init-container needs a DB connection for SECRET_PERSISTENCE=testing_config_db_table. The chart adds DATASOURCES_LOCALSECRETS_URL to the configmap and DATASOURCES_LOCALSECRETS_USERNAME/PASSWORD to the secret, but the workload launcher (which builds the job pod) may not inject these env vars into the pod. (1) Verify they exist: kubectl get configmap {{ .Release.Name }}-airbyte-env -n {{ .Release.Namespace }} -o jsonpath='{.data.DATASOURCES_LOCALSECRETS_URL}'; kubectl get secret {{ .Release.Name }}-airbyte-secrets -n {{ .Release.Namespace }} -o jsonpath='{.data}' | grep -o 'DATASOURCES_LOCALSECRETS_[^"]*'. (2) If the keys exist but the job pod still fails, the launcher is not passing them. Workaround: set global.secretsManager.type: "no_op" so the init container does not require the local-secrets DB (connector secrets will not be stored in the config DB; use only if acceptable). Then helm upgrade and restart workload-launcher; retry the check/sync. +- Workload-launcher: "Token request failed with HTTP code 500" / "401 Unauthorized" on dataplane init or heartbeat: (1) Confirm you ran helm upgrade and restarted so the launcher gets the new config: kubectl get configmap {{ .Release.Name }}-airbyte-env -n {{ .Release.Namespace }} -o jsonpath='{.data.CONTROL_PLANE_TOKEN_ENDPOINT}' should show the internal URL (e.g. http://{{ .Release.Name }}-airbyte-server-svc.{{ .Release.Namespace }}:8001/api/v1/dataplanes/token). If it still shows your public airbyteUrl, set workloadLauncher.controlPlane.useInternalUrl: true and upgrade again, then kubectl rollout restart deployment/{{ .Release.Name }}-workload-launcher -n {{ .Release.Namespace }}. (2) The 500 is returned by the Airbyte server application; the chart cannot fix it. Check server logs: kubectl logs deployment/{{ .Release.Name }}-server -n {{ .Release.Namespace }} --tail=300. If you see "Could not get the current Airbyte user" / SecurityAwareCurrentUserService / NoSuchElementException when the launcher calls the token endpoint, the server is requiring a session—the launcher has no session and uses only the credentials from secret keys dataplane-client-id and dataplane-client-secret. (3) Temporary test to confirm: set global.auth.enabled: false in values, helm upgrade, then restart server and workload-launcher. If the launcher then starts, the issue is the server's session requirement on /api/v1/dataplanes/token. Re-enable auth (global.auth.enabled: true) and upgrade the Airbyte server image to a version that exempts the dataplane token path from session auth, or see Airbyte GitHub issues. (4) Credentials: ensure the secret has dataplane-client-id and dataplane-client-secret (kubectl get secret {{ .Release.Name }}-airbyte-secrets -n {{ .Release.Namespace }} -o jsonpath='{.data}' | jq 'keys'), run the bootloader (airbyteBootloader.enabled: true) then restart server and launcher. Custom internal auth header: set global.api.internalAuthHeaderValue (same for server and launcher). diff --git a/helm/airbyte-v2/templates/_helpers.tpl b/helm/airbyte-v2/templates/_helpers.tpl new file mode 100644 index 00000000..a1fb331a --- /dev/null +++ b/helm/airbyte-v2/templates/_helpers.tpl @@ -0,0 +1,218 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "airbyte.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Returns the name of a given component +*/}} +{{- define "airbyte.componentName" -}} +{{- $tplPathParts := split "/" $.Template.Name }} +{{- $indexLast := printf "_%d" (sub (len $tplPathParts) 2) }} +{{- $componentName := trimPrefix "airbyte-" (index $tplPathParts $indexLast) }} +{{- printf "%s" $componentName }} +{{- end }} + +{{/* +Returns the name of a given component with the `airbyte-` prefix +*/}} +{{- define "airbyte.componentNameWithAirbytePrefix" -}} +{{- printf "airbyte-%s" (include "airbyte.componentName" .) }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airbyte.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airbyte.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Base string used for generating default secrets (passwords, client ids, etc.). +When global.secretSeed is set (e.g. cluster name), include it so the same release+namespace +on different infrastructures gets different dynamic secret values (v1-like behavior). +*/}} +{{- define "airbyte.secretSeed.hashBase" -}} +{{- if .Values.global.secretSeed -}} +{{- printf "%s-%s-%s" .Release.Name .Release.Namespace .Values.global.secretSeed -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name .Release.Namespace -}} +{{- end -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "airbyte.labels" -}} +helm.sh/chart: {{ include "airbyte.chart" . }} +{{ include "airbyte.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "airbyte.selectorLabels" -}} +{{ $tplPathParts := split "/" $.Template.Name }} +{{ $indexLast := printf "_%d" (sub (len $tplPathParts) 2) }} +{{ $componentName := trimPrefix "airbyte-" (index $tplPathParts $indexLast) }} +airbyte: {{ $componentName }} +app.kubernetes.io/name: {{ $componentName }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + + +{{/* +Common DB labels +*/}} +{{- define "airbyte.databaseLabels" -}} +helm.sh/chart: {{ include "airbyte.chart" . }} +{{ include "airbyte.databaseSelectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector DB labels +*/}} +{{- define "airbyte.databaseSelectorLabels" -}} +app.kubernetes.io/name: {{ printf "%s-db" .Release.Name }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Common DB labels +*/}} +{{- define "airbyte.minioLabels" -}} +helm.sh/chart: {{ include "airbyte.chart" . }} +{{ include "airbyte.minioSelectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector DB labels +*/}} +{{- define "airbyte.minioSelectorLabels" -}} +app.kubernetes.io/name: {{ printf "%s-minio" .Release.Name }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Default chart-wide service account name (release-namespaced). Use so worker and job pods use the same SA that has RBAC; override with global.serviceAccountName or component .serviceAccountName. +*/}} +{{- define "airbyte.effectiveServiceAccountName" -}} +{{- default .Values.global.serviceAccountName (printf "%s-worker-sa" .Release.Name) }} +{{- end }} + +{{/* +Create the name of the service account to use. When create is true, uses serviceAccount.name or effectiveServiceAccountName so the created SA matches what deployments and job pods use (e.g. airbyte-v2-worker-sa). +*/}} +{{- define "airbyte.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default .Values.serviceAccount.name (include "airbyte.effectiveServiceAccountName" .) }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Service account name used for job pods (JOB_KUBE_SERVICEACCOUNT). Must match the created SA and have the pod-launcher Role. Defaults to effectiveServiceAccountName (e.g. airbyte-v2-worker-sa). +*/}} +{{- define "airbyte.jobPodServiceAccountName" -}} +{{- default .Values.workloadLauncher.serviceAccountName (include "airbyte.effectiveServiceAccountName" .) }} +{{- end }} + +{{/* +Construct comma separated list of key/value pairs from object (useful for ENV var values) +*/}} +{{- define "airbyte.flattenMap" -}} +{{- $kvList := list -}} +{{- range $key, $value := . -}} +{{- $kvList = printf "%s=%s" $key $value | mustAppend $kvList -}} +{{- end -}} +{{ join "," $kvList }} +{{- end -}} + +{{/* +Construct semi-colon delimited list of comma separated key/value pairs from array of objects (useful for ENV var values) +*/}} +{{- define "airbyte.flattenArrayMap" -}} +{{- $mapList := list -}} +{{- range $element := . -}} +{{- $mapList = include "airbyte.flattenMap" $element | mustAppend $mapList -}} +{{- end -}} +{{ join ";" $mapList }} +{{- end -}} + +{{/* +Convert tags to a comma-separated list of key=value pairs. +*/}} +{{- define "airbyte.tagsToString" -}} +{{- $result := list -}} +{{- range . -}} + {{- $key := .key -}} + {{- $value := .value -}} + {{- if eq (typeOf $value) "bool" -}} + {{- $value = ternary "true" "false" $value -}} + {{- end -}} + {{- $result = append $result (printf "%s=%s" $key $value) -}} +{{- end -}} +{{- join "," $result -}} +{{- end -}} + +{{/* +Hook for passing in extra config map vars +*/}} +{{- define "airbyte.extra.configVars" }} +{{- end }} + +{{/* +Hook for passing in extra secrets +*/}} +{{- define "airbyte.extra.secrets" }} +{{- end }} + + +{{/* +Returns a comma-delimited string of imagePullSecret names. +Usage: + {{ include "airbyte.imagePullSecretNames" (dict "secrets" .Values.global.imagePullSecrets "extra" (list "foo" "bar")) }} +*/}} +{{- define "airbyte.imagePullSecretNames" -}} + {{- $secrets := default list .secrets }} + {{- $extra := default list .extra }} + {{- $names := list }} + {{- range $secrets }} + {{- $names = append $names .name }} + {{- end }} + {{- $all := concat $names $extra }} + {{- join "," $all }} +{{- end }} diff --git a/helm/airbyte-v2/templates/_images.tpl b/helm/airbyte-v2/templates/_images.tpl new file mode 100644 index 00000000..fc4fd601 --- /dev/null +++ b/helm/airbyte-v2/templates/_images.tpl @@ -0,0 +1,17 @@ +{{- define "imageUrl" -}} +{{- $img := index . 0 -}} +{{- $root := index . 1 -}} + +{{- $reg := $root.Values.global.image.registry -}} +{{/* ensure the registry has a trailing slash, if set */}} +{{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} + +{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{- if (eq (typeOf $img) "string") -}} +{{- printf "%s%s" $reg (tpl $img $root) -}} +{{- else -}} +{{- $tag := coalesce $img.tag $root.Values.global.image.tag $root.Chart.AppVersion -}} +{{- printf "%s%s:%s" $reg $img.repository $tag -}} +{{- end -}} + +{{- end -}} diff --git a/helm/airbyte-v2/templates/airbyte-bootloader/bootloader-secrets.yaml b/helm/airbyte-v2/templates/airbyte-bootloader/bootloader-secrets.yaml new file mode 100644 index 00000000..c6acd63e --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-bootloader/bootloader-secrets.yaml @@ -0,0 +1,18 @@ +# Create secrets only for the local deployment +{{- if .Values.airbyteBootloader.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: bootloader-secrets + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" + +type: Opaque +data: + {{- range $k, $v := mergeOverwrite .Values.airbyteBootloader.secrets .Values.global.secrets }} + {{ $k }}: {{ if $v }}{{ $v | b64enc }} {{else}}""{{end}} + {{- end }} + {{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-bootloader/pod.yaml b/helm/airbyte-v2/templates/airbyte-bootloader/pod.yaml new file mode 100644 index 00000000..d674fbd7 --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-bootloader/pod.yaml @@ -0,0 +1,121 @@ +{{- if .Values.airbyteBootloader.enabled }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ .Release.Name }}-bootloader + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- if .Values.airbyteBootloader.extraLabels }} + {{ toYaml (mergeOverwrite .Values.airbyteBootloader.extraLabels .Values.global.extraLabels) | nindent 4 }} + {{- end }} + {{- if .Values.airbyteBootloader.podLabels}} + {{- include "airbyte.tplvalues.render" (dict "value" .Values.airbyteBootloader.podLabels "context" $) | nindent 4 }} + {{- end }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "0" + {{- if .Values.airbyteBootloader.podAnnotations }} + {{- include "airbyte.tplvalues.render" (dict "value" .Values.airbyteBootloader.podAnnotations "context" $) | nindent 4 }} + {{- end }} +spec: + serviceAccountName: {{ default .Values.airbyteBootloader.serviceAccountName (include "airbyte.effectiveServiceAccountName" .) }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 2 }} + {{- end }} + {{- end }} + restartPolicy: Never + {{- if .Values.airbyteBootloader.nodeSelector }} + nodeSelector: {{- include "airbyte.tplvalues.render" (dict "value" .Values.airbyteBootloader.nodeSelector "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.airbyteBootloader.tolerations }} + tolerations: {{- include "airbyte.tplvalues.render" (dict "value" .Values.airbyteBootloader.tolerations "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.airbyteBootloader.affinity }} + affinity: {{- include "airbyte.tplvalues.render" (dict "value" .Values.airbyteBootloader.affinity "context" $) | nindent 4 }} + {{- end }} + {{- if .Values.airbyteBootloader.extraInitContainers }} + initContainers: + {{- toYaml .Values.airbyteBootloader.extraInitContainers | nindent 4 }} + {{- end }} + containers: + - name: airbyte-bootloader-container + image: {{ include "imageUrl" (list .Values.airbyteBootloader.image $) }} + imagePullPolicy: "{{ .Values.airbyteBootloader.image.pullPolicy }}" + env: + {{- if (or (eq .Values.global.edition "community") (eq .Values.global.edition "enterprise")) }} + {{- include "airbyte.auth.identityProvider.envs" . | nindent 8 }} + {{- end }} + + {{- if and (or (eq .Values.global.edition "community") (eq .Values.global.edition "enterprise")) .Values.global.auth.enabled }} + {{- include "airbyte.auth.bootstrap.envs.v1style" . | nindent 8 }} + {{- else }} + {{- include "airbyte.auth.bootstrap.envs" . | nindent 8 }} + {{- end }} + {{- include "airbyte.common.version.env" . | nindent 8 }} + {{- include "airbyte.connector.envs" . | nindent 8 }} + {{- include "airbyte.database.envs" . | nindent 8 }} + {{- include "airbyte.database.migrations.runAtStartup.env" . | nindent 8 }} + {{- include "airbyte.featureFlags.envs" . | nindent 8 }} + {{- include "airbyte.logging.envs" . | nindent 8 }} + {{- include "airbyte.micronaut.envs" . | nindent 8 }} + + {{- if .Values.global.datadog.enabled }} + - name: DD_SERVICE + value: {{ include "airbyte.componentNameWithAirbytePrefix" . }} + {{- include "airbyte.datadog.envs" . | nindent 8 }} + {{- include "airbyte.otel.envs" . | nindent 8 }} + {{- end}} + + {{- if (eq .Values.global.edition "cloud") }} + {{- include "airbyte.secretsManager.envs" . | nindent 8 }} + {{- end }} + + # Values from secret + {{- if .Values.airbyteBootloader.secrets }} + {{- range $k, $v := .Values.airbyteBootloader.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: bootloader-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if or .Values.airbyteBootloader.env_vars .Values.global.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.airbyteBootloader.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + # Values from extraEnv for more compability(if you want to use external secret source or other stuff) + {{- if .Values.airbyteBootloader.extraEnv }} + {{- toYaml .Values.airbyteBootloader.extraEnv | nindent 8 }} + {{- end }} + + {{- if .Values.airbyteBootloader.resources }} + resources: {{- toYaml .Values.airbyteBootloader.resources | nindent 8 }} + {{- end }} + {{- if .Values.airbyteBootloader.extraVolumeMount }} + volumeMounts: + {{- toYaml .Values.airbyteBootloader.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.airbyteBootloader.containerSecurityContext }} + securityContext: {{- toYaml .Values.airbyteBootloader.containerSecurityContext | nindent 8 }} + {{- end }} + {{- if .Values.airbyteBootloader.extraContainers }} + {{ toYaml .Values.airbyteBootloader.extraContainers | nindent 4 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 4 }} + {{- end }} + securityContext: {{- toYaml .Values.airbyteBootloader.podSecurityContext | nindent 4 }} + {{- if .Values.airbyteBootloader.extraVolumes }} + volumes: + {{- toYaml .Values.airbyteBootloader.extraVolumes | nindent 4 }} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-connector-builder-server/deployment.yaml b/helm/airbyte-v2/templates/airbyte-connector-builder-server/deployment.yaml new file mode 100644 index 00000000..7295ddf9 --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-connector-builder-server/deployment.yaml @@ -0,0 +1,170 @@ +{{- if .Values.connectorBuilderServer.enabled }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-connector-builder-server + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + {{ toYaml .Values.connectorBuilderServer.annotations | nindent 4 }} +spec: + replicas: {{ .Values.connectorBuilderServer.replicaCount }} + selector: + matchLabels: + {{- include "airbyte.selectorLabels" . | nindent 6 }} + {{- if .Values.connectorBuilderServer.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.connectorBuilderServer.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 6 }} + {{- end }} + strategy: + type: {{ .Values.connectorBuilderServer.deploymentStrategyType }} + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + {{- if .Values.connectorBuilderServer.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.connectorBuilderServer.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 8 }} + {{- end }} + {{- if .Values.connectorBuilderServer.podLabels }} + {{- include "airbyte.tplvalues.render" (dict "value" .Values.connectorBuilderServer.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.connectorBuilderServer.podAnnotations }} + annotations: + {{- include "airbyte.tplvalues.render" (dict "value" .Values.connectorBuilderServer.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ default .Values.connectorBuilderServer.serviceAccountName (include "airbyte.effectiveServiceAccountName" .) }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.connectorBuilderServer.nodeSelector }} + nodeSelector: {{- include "airbyte.tplvalues.render" (dict "value" .Values.connectorBuilderServer.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.connectorBuilderServer.tolerations }} + tolerations: {{- include "airbyte.tplvalues.render" (dict "value" .Values.connectorBuilderServer.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.connectorBuilderServer.affinity }} + affinity: {{- include "airbyte.tplvalues.render" (dict "value" .Values.connectorBuilderServer.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.connectorBuilderServer.extraInitContainers }} + initContainers: + {{- toYaml .Values.connectorBuilderServer.extraInitContainers | nindent 8 }} + {{- end }} + containers: + - name: airbyte-connector-builder-server + image: {{ include "imageUrl" (list .Values.connectorBuilderServer.image $) }} + imagePullPolicy: "{{ .Values.connectorBuilderServer.image.pullPolicy }}" + env: + {{- if .Values.connectorBuilderServer.debug.enabled }} + - name: JAVA_TOOL_OPTIONS + value: "-Xdebug -agentlib:jdwp=transport=dt_socket,address=0.0.0.0:{{ .Values.connectorBuilderServer.debug.remoteDebugPort }},server=y,suspend=n" + {{- end}} + + {{- include "airbyte.common.envs" . | nindent 8 }} + {{- include "airbyte.connectorBuilder.envs" . | nindent 8 }} + {{- include "airbyte.logging.envs" . | nindent 8 }} + {{- include "airbyte.metrics.envs" . | nindent 8 }} + {{- include "airbyte.micronaut.envs" . | nindent 8 }} + {{- include "airbyte.tracking.envs" . | nindent 8 }} + + {{- if .Values.global.datadog.enabled }} + - name: DD_SERVICE + value: {{ include "airbyte.componentNameWithAirbytePrefix" . }} + {{- include "airbyte.datadog.envs" . | nindent 8 }} + {{- include "airbyte.otel.envs" . | nindent 8 }} + {{- end}} + + {{- if (eq .Values.global.edition "enterprise") }} + {{- include "airbyte.enterprise.envs" . | nindent 8 }} + {{- end }} + + {{- if (eq .Values.global.edition "cloud") }} + {{- include "airbyte.featureFlags.envs" . | nindent 8 }} + {{- include "airbyte.keycloak.envs" . | nindent 8 }} + {{- include "airbyte.stripe.envs" . | nindent 8 }} + {{- end }} + + # Values from secret + {{- if .Values.connectorBuilderServer.secrets }} + {{- range $k, $v := .Values.connectorBuilderServer.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: server-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if or .Values.connectorBuilderServer.env_vars .Values.global.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.connectorBuilderServer.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + # Values from extraEnv for more compatibility (if you want to use external secret source or other stuff) + {{- if .Values.connectorBuilderServer.extraEnv }} + {{- toYaml .Values.connectorBuilderServer.extraEnv | nindent 8 }} + {{- end }} + + {{- if .Values.connectorBuilderServer.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /v1/health + port: http + initialDelaySeconds: {{ .Values.connectorBuilderServer.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.connectorBuilderServer.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.connectorBuilderServer.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.connectorBuilderServer.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.connectorBuilderServer.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.connectorBuilderServer.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /v1/health + port: http + initialDelaySeconds: {{ .Values.connectorBuilderServer.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.connectorBuilderServer.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.connectorBuilderServer.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.connectorBuilderServer.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.connectorBuilderServer.readinessProbe.failureThreshold }} + {{- end }} + + ports: + - name: http + containerPort: 8080 + protocol: TCP + {{- if .Values.connectorBuilderServer.debug.enabled }} + - name: debug + containerPort: {{ .Values.connectorBuilderServer.debug.remoteDebugPort }} + protocol: TCP + {{- end}} + {{- if .Values.connectorBuilderServer.resources }} + resources: {{- toYaml .Values.connectorBuilderServer.resources | nindent 10 }} + {{- end }} + {{- if .Values.connectorBuilderServer.containerSecurityContext }} + securityContext: {{- toYaml .Values.connectorBuilderServer.containerSecurityContext | nindent 10 }} + {{- end }} + volumeMounts: + {{- if .Values.connectorBuilderServer.extraVolumeMounts }} + {{ toYaml .Values.connectorBuilderServer.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.global.extraVolumeMounts }} + {{ toYaml .Values.global.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.connectorBuilderServer.extraContainers }} + {{ toYaml .Values.connectorBuilderServer.extraContainers | nindent 6 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 6 }} + {{- end }} + securityContext: {{- toYaml .Values.connectorBuilderServer.podSecurityContext | nindent 8 }} + volumes: + {{- if .Values.connectorBuilderServer.extraVolumes }} +{{ toYaml .Values.connectorBuilderServer.extraVolumes | nindent 6 }} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-connector-builder-server/secrets.yaml b/helm/airbyte-v2/templates/airbyte-connector-builder-server/secrets.yaml new file mode 100644 index 00000000..2e4f8e04 --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-connector-builder-server/secrets.yaml @@ -0,0 +1,17 @@ +# Create secrets only for the local deployment +{{- if .Values.connectorBuilderServer.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: connector-builder-server-secrets + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" +type: Opaque +data: + {{- range $k, $v := mergeOverwrite .Values.connectorBuilderServer.secrets .Values.global.secrets }} + {{ $k }}: {{ if $v }}{{ $v | b64enc }} {{else}}""{{end}} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-connector-builder-server/service.yaml b/helm/airbyte-v2/templates/airbyte-connector-builder-server/service.yaml new file mode 100644 index 00000000..a9cc84d8 --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-connector-builder-server/service.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{.Release.Name }}-airbyte-connector-builder-server-svc + {{- with .Values.connectorBuilderServer.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + type: {{ .Values.connectorBuilderServer.service.type }} + ports: + - port: {{ .Values.connectorBuilderServer.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "airbyte.selectorLabels" . | nindent 4 }} diff --git a/helm/airbyte-v2/templates/airbyte-connector-rollout-worker/deployment.yaml b/helm/airbyte-v2/templates/airbyte-connector-rollout-worker/deployment.yaml new file mode 100644 index 00000000..da336c48 --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-connector-rollout-worker/deployment.yaml @@ -0,0 +1,176 @@ +{{- if .Values.connectorRolloutWorker.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-connector-rollout-worker + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- if .Values.connectorRolloutWorker.extraLabels }} + {{ toYaml (mergeOverwrite .Values.connectorRolloutWorker.extraLabels .Values.global.extraLabels) | nindent 4 }} + {{- end }} + annotations: + {{ toYaml .Values.connectorRolloutWorker.annotations | nindent 4 }} +spec: + replicas: {{ .Values.connectorRolloutWorker.replicaCount }} + selector: + matchLabels: + {{- include "airbyte.selectorLabels" . | nindent 6 }} + {{- if .Values.connectorRolloutWorker.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.connectorRolloutWorker.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 6 }} + {{- end }} + strategy: + type: Recreate + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + {{- if .Values.connectorRolloutWorker.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.connectorRolloutWorker.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 8 }} + {{- end }} + {{- if .Values.connectorRolloutWorker.podLabels }} + {{- include "airbyte.tplvalues.render" (dict "value" .Values.connectorRolloutWorker.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.connectorRolloutWorker.podAnnotations }} + annotations: + {{- include "airbyte.tplvalues.render" (dict "value" .Values.connectorRolloutWorker.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ default .Values.connectorRolloutWorker.serviceAccountName (include "airbyte.effectiveServiceAccountName" .) }} + automountServiceAccountToken: true + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.connectorRolloutWorker.nodeSelector }} + nodeSelector: {{- include "airbyte.tplvalues.render" (dict "value" .Values.connectorRolloutWorker.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.connectorRolloutWorker.tolerations }} + tolerations: {{- include "airbyte.tplvalues.render" (dict "value" .Values.connectorRolloutWorker.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.connectorRolloutWorker.affinity }} + affinity: {{- include "airbyte.tplvalues.render" (dict "value" .Values.connectorRolloutWorker.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.connectorRolloutWorker.extraInitContainers }} + initContainers: + {{- toYaml .Values.connectorRolloutWorker.extraInitContainers | nindent 8 }} + {{- end }} + containers: + - name: airbyte-connector-rollout-worker-container + image: {{ include "imageUrl" (list .Values.connectorRolloutWorker.image $) }} + imagePullPolicy: "{{ .Values.connectorRolloutWorker.image.pullPolicy }}" + env: + {{- if .Values.connectorRolloutWorker.debug.enabled }} + - name: JAVA_TOOL_OPTIONS + value: "-Xdebug -agentlib:jdwp=transport=dt_socket,address=0.0.0.0:{{ .Values.connectorRolloutWorker.debug.remoteDebugPort }},server=y,suspend=n" + {{- end }} + + {{- if or (eq .Values.global.edition "community") (eq .Values.global.edition "enterprise") }} + - name: AIRBYTE_API_AUTH_HEADER_VALUE + value: "Internal worker" + {{- end }} + + {{- include "airbyte.common.envs" . | nindent 8 }} + {{- include "airbyte.keycloak.client.envs" . | nindent 8 }} + {{- include "airbyte.logging.envs" . | nindent 8 }} + {{- include "airbyte.metrics.envs" . | nindent 8 }} + {{- include "airbyte.tracking.envs" . | nindent 8 }} + {{- include "airbyte.temporal.envs" . | nindent 8 }} + {{- include "airbyte.temporal.cloud.enabled.env" . | nindent 8 }} + {{- include "airbyte.temporal.sdk.envs" . | nindent 8 }} + {{- include "airbyte.temporal.worker.envs" . | nindent 8 }} + + {{- if .Values.global.datadog.enabled }} + - name: DD_SERVICE + value: {{ include "airbyte.componentNameWithAirbytePrefix" . }} + {{- include "airbyte.datadog.envs" . | nindent 8 }} + {{- include "airbyte.otel.envs" . | nindent 8 }} + {{- end}} + + {{- if (eq .Values.global.edition "cloud") }} + {{- include "airbyte.connectorRollout.temporal.envs" . | nindent 8 }} + {{- include "airbyte.temporal.cloud.credentials.envs" . | nindent 8 }} + {{- end }} + + # Values from secret + {{- if .Values.connectorRolloutWorker.secrets }} + {{- range $k, $v := .Values.connectorRolloutWorker.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: connector-rollout-worker-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if or .Values.connectorRolloutWorker.env_vars .Values.global.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.connectorRolloutWorker.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + {{- if .Values.connectorRolloutWorker.extraEnv }} + {{- toYaml .Values.connectorRolloutWorker.extraEnv | nindent 8 }} + {{- end }} + + {{- if .Values.connectorRolloutWorker.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /health/liveness + port: heartbeat + initialDelaySeconds: {{ .Values.connectorRolloutWorker.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.connectorRolloutWorker.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.connectorRolloutWorker.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.connectorRolloutWorker.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.connectorRolloutWorker.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.connectorRolloutWorker.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /health/readiness + port: heartbeat + initialDelaySeconds: {{ .Values.connectorRolloutWorker.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.connectorRolloutWorker.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.connectorRolloutWorker.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.connectorRolloutWorker.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.connectorRolloutWorker.readinessProbe.failureThreshold }} + {{- end }} + ports: + {{- if .Values.connectorRolloutWorker.debug.enabled }} + - name: debug + containerPort: {{ .Values.connectorRolloutWorker.debug.remoteDebugPort }} + protocol: TCP + {{- end}} + - name: heartbeat + containerPort: 8016 # for heartbeat server + + {{- if .Values.connectorRolloutWorker.resources }} + resources: {{- toYaml .Values.connectorRolloutWorker.resources | nindent 10 }} + {{- end }} + + {{- if .Values.connectorRolloutWorker.containerSecurityContext }} + securityContext: {{- toYaml .Values.connectorRolloutWorker.containerSecurityContext | nindent 10 }} + {{- end }} + + volumeMounts: + {{- if .Values.connectorRolloutWorker.extraVolumeMounts }} + {{ toYaml .Values.connectorRolloutWorker.extraVolumeMounts | nindent 10 }} + {{- end }} + {{- if .Values.global.extraVolumeMounts }} + {{ toYaml .Values.global.extraVolumeMounts | nindent 10 }} + {{- end }} + + {{- if .Values.connectorRolloutWorker.extraContainers }} + {{ toYaml .Values.connectorRolloutWorker.extraContainers | nindent 8 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 8 }} + {{- end }} + securityContext: {{- toYaml .Values.connectorRolloutWorker.podSecurityContext | nindent 8 }} + volumes: + {{- if .Values.connectorRolloutWorker.extraVolumes }} + {{ toYaml .Values.connectorRolloutWorker.extraVolumes | nindent 4 }} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-connector-rollout-worker/secrets.yaml b/helm/airbyte-v2/templates/airbyte-connector-rollout-worker/secrets.yaml new file mode 100644 index 00000000..fc96a26c --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-connector-rollout-worker/secrets.yaml @@ -0,0 +1,17 @@ +# Create secrets only for the local deployment +{{- if .Values.connectorRolloutWorker.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: connector-rollout-worker-secrets + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" +type: Opaque +data: + {{- range $k, $v := mergeOverwrite .Values.connectorRolloutWorker.secrets .Values.global.secrets }} + {{ $k }}: {{ if $v }}{{ $v | b64enc }} {{else}}""{{end}} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-cron/cron-secrets.yaml b/helm/airbyte-v2/templates/airbyte-cron/cron-secrets.yaml new file mode 100644 index 00000000..6071994f --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-cron/cron-secrets.yaml @@ -0,0 +1,18 @@ +# Create secrets only for the local deployment +{{- if .Values.cron.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: cron-secrets + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" + +type: Opaque +data: + {{- range $k, $v := mergeOverwrite .Values.cron.secrets .Values.global.secrets }} + {{ $k }}: {{ if $v }}{{ $v | b64enc }} {{else}}""{{end}} + {{- end }} + {{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-cron/deployment.yaml b/helm/airbyte-v2/templates/airbyte-cron/deployment.yaml new file mode 100644 index 00000000..792497d0 --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-cron/deployment.yaml @@ -0,0 +1,140 @@ +{{- if .Values.cron.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-cron + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- if .Values.cron.extraLabels }} + {{ toYaml (mergeOverwrite .Values.cron.extraLabels .Values.global.extraLabels) | nindent 4 }} + {{- end }} + annotations: + {{ toYaml .Values.cron.annotations | nindent 4 }} +spec: + replicas: {{ .Values.cron.replicaCount }} + selector: + matchLabels: + {{- include "airbyte.selectorLabels" . | nindent 6 }} + {{- if .Values.cron.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.cron.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 6 }} + {{- end }} + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + {{- if .Values.cron.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.cron.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 8 }} + {{- end }} + {{- if .Values.cron.podLabels }} + {{- include "airbyte.tplvalues.render" (dict "value" .Values.cron.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.cron.podAnnotations }} + annotations: + {{- include "airbyte.tplvalues.render" (dict "value" .Values.cron.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ default .Values.cron.serviceAccountName (include "airbyte.effectiveServiceAccountName" .) }} + automountServiceAccountToken: true + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.cron.nodeSelector }} + nodeSelector: {{- include "airbyte.tplvalues.render" (dict "value" .Values.cron.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.cron.tolerations }} + tolerations: {{- include "airbyte.tplvalues.render" (dict "value" .Values.cron.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.cron.affinity }} + affinity: {{- include "airbyte.tplvalues.render" (dict "value" .Values.cron.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.cron.extraInitContainers }} + initContainers: + {{- toYaml .Values.cron.extraInitContainers | nindent 6 }} + {{- end }} + containers: + - name: airbyte-cron + image: {{ include "imageUrl" (list .Values.cron.image $) }} + imagePullPolicy: "{{ .Values.cron.image.pullPolicy }}" + env: + {{- include "airbyte.common.envs" . | nindent 12 }} + {{- include "airbyte.connector.envs" . | nindent 12 }} + {{- include "airbyte.cron.envs" . | nindent 12 }} + {{- include "airbyte.database.envs" . | nindent 12 }} + {{- include "airbyte.database.migrations.envs" . | nindent 12 }} + {{- include "airbyte.featureFlags.envs" . | nindent 12 }} + {{- include "airbyte.metrics.envs" . | nindent 12 }} + {{- include "airbyte.micronaut.envs" . | nindent 12 }} + {{- include "airbyte.temporal.envs" . | nindent 12 }} + {{- include "airbyte.tracking.envs" . | nindent 12 }} + {{- include "airbyte.worker.envs" . | nindent 12 }} + {{- /* AB_JWT_SIGNATURE_SECRET required for WorkloadApiClient internal auth (signatureSecret) when calling workload API */}} + {{- include "airbyte.auth.jwt.envs" . | nindent 12 }} + {{- include "airbyte.workloadApiServer.envs" . | nindent 12 }} + + {{- if .Values.global.datadog.enabled }} + - name: DD_SERVICE + value: {{ include "airbyte.componentNameWithAirbytePrefix" . }} + {{- include "airbyte.datadog.envs" . | nindent 12 }} + {{- include "airbyte.otel.envs" . | nindent 12 }} + {{- end}} + + {{- if (eq .Values.global.edition "enterprise") }} + {{- include "airbyte.enterprise.envs" . | nindent 12 }} + {{- end }} + + {{- if (eq .Values.global.edition "cloud") }} + {{- include "airbyte.temporal.cloud.envs" . | nindent 12 }} + {{- include "airbyte.temporal.cloud.credentials.envs" . | nindent 12 }} + {{- end }} + + # Values from secret + {{- if .Values.cron.secrets }} + {{- range $k, $v := .Values.cron.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: cron-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if or .Values.cron.env_vars .Values.global.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.cron.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + # Values from extraEnv for more compability(if you want to use external secret source or other stuff) + {{- if .Values.cron.extraEnv }} + {{- toYaml .Values.cron.extraEnv | nindent 12 }} + {{- end }} + + {{- if .Values.cron.containerSecurityContext }} + securityContext: {{- toYaml .Values.cron.containerSecurityContext | nindent 12 }} + {{- end }} + + {{- if .Values.cron.resources }} + resources: {{- toYaml .Values.cron.resources | nindent 12 }} + {{- end }} + + {{- if .Values.cron.extraVolumeMounts }} + volumeMounts: {{ toYaml .Values.cron.extraVolumeMounts | nindent 12 }} + {{- end }} + + {{- if .Values.cron.extraContainers }} + {{ toYaml .Values.cron.extraContainers | nindent 8 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 8 }} + {{- end }} + + securityContext: {{- toYaml .Values.cron.podSecurityContext | nindent 8 }} + {{- if .Values.cron.extraVolumes }} + volumes: {{ toYaml .Values.cron.extraVolumes | nindent 8 }} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-db.yaml b/helm/airbyte-v2/templates/airbyte-db.yaml new file mode 100644 index 00000000..91cbd658 --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-db.yaml @@ -0,0 +1,85 @@ +{{- if .Values.postgresql.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: airbyte-db-svc + annotations: + helm.sh/hook: pre-install + helm.sh/hook-weight: "-1" + labels: + {{ include "airbyte.databaseLabels" . | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: 5432 + protocol: TCP + selector: + {{ include "airbyte.databaseSelectorLabels" . | nindent 4 }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airbyte-db + annotations: + helm.sh/hook: pre-install + helm.sh/hook-weight: "-1" + labels: + {{ include "airbyte.databaseLabels" . | nindent 4 }} +spec: + replicas: 1 + serviceName: airbyte-db-svc + selector: + matchLabels: + {{ include "airbyte.databaseSelectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{ include "airbyte.databaseSelectorLabels" . | nindent 8 }} + spec: + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + containers: + - name: airbyte-db-container + image: {{ include "imageUrl" (list .Values.postgresql.image $) }} + env: + - name: POSTGRES_DB + value: {{ .Values.postgresql.postgresqlDatabase }} + - name: POSTGRES_PASSWORD + value: {{ .Values.postgresql.postgresqlPassword }} + - name: POSTGRES_USER + value: {{ .Values.postgresql.postgresqlUsername }} + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + ports: + - containerPort: 5432 + securityContext: {{- toYaml .Values.postgresql.containerSecurityContext | nindent 12 }} + volumeMounts: + - name: airbyte-volume-db + mountPath: /var/lib/postgresql/data + {{- with .Values.postgresql.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.postgresql.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.postgresql.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: {{- toYaml .Values.postgresql.podSecurityContext | nindent 8 }} + + volumeClaimTemplates: + - metadata: + name: airbyte-volume-db + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 500Mi +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-featureflag-server/deployment.yaml b/helm/airbyte-v2/templates/airbyte-featureflag-server/deployment.yaml new file mode 100644 index 00000000..953d854c --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-featureflag-server/deployment.yaml @@ -0,0 +1,157 @@ +{{- if and (.Values.featureflagServer.enabled) (eq .Values.global.edition "community") }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-featureflag-server + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + {{ toYaml .Values.featureflagServer.annotations | nindent 4 }} +spec: + minReadySeconds: 30 + replicas: {{ .Values.featureflagServer.replicaCount }} + selector: + matchLabels: + {{- include "airbyte.selectorLabels" . | nindent 6 }} + {{- if .Values.featureflagServer.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.featureflagServer.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 6 }} + {{- end }} + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 100% + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + {{- if .Values.featureflagServer.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.featureflagServer.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 8 }} + {{- end }} + {{- if .Values.featureflagServer.podLabels}} + {{- include "airbyte.tplvalues.render" (dict "value" .Values.featureflagServer.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.featureflagServer.podAnnotations }} + annotations: + {{- include "airbyte.tplvalues.render" (dict "value" .Values.featureflagServer.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ default .Values.featureflagServer.serviceAccountName (include "airbyte.effectiveServiceAccountName" .) }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.featureflagServer.nodeSelector }} + nodeSelector: {{- include "airbyte.tplvalues.render" (dict "value" .Values.featureflagServer.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.featureflagServer.tolerations }} + tolerations: {{- include "airbyte.tplvalues.render" (dict "value" .Values.featureflagServer.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.featureflagServer.affinity }} + affinity: {{- include "airbyte.tplvalues.render" (dict "value" .Values.featureflagServer.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.featureflagServer.extraInitContainers }} + initContainers: + {{- toYaml .Values.featureflagServer.extraInitContainers | nindent 8 }} + {{- end }} + containers: + - name: airbyte-featureflag-server-container + image: {{ include "imageUrl" (list .Values.featureflagServer.image $) }} + imagePullPolicy: "{{ .Values.featureflagServer.image.pullPolic }}" + env: + {{- if .Values.featureflagServer.debug.enabled }} + - name: JAVA_TOOL_OPTIONS + value: "-Xdebug -agentlib:jdwp=transport=dt_socket,address=0.0.0.0:{{ .Values.featureflagServer.debug.remoteDebugPort }},server=y,suspend=n" + {{- end }} + + {{- include "airbyte.common.envs" . | nindent 8 }} + {{- include "airbyte.metrics.envs" . | nindent 8 }} + + # Values from env + {{- if or .Values.featureflagServer.env_vars .Values.global.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.featureflagServer.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + # Values from extraEnv for more compability(if you want to use external secret source or other stuff) + {{- if .Values.featureflagServer.extraEnv }} + {{- toYaml .Values.featureflagServer.extraEnv | nindent 8 }} + {{- end }} + + {{- if .Values.featureflagServer.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /health/liveness + port: http + initialDelaySeconds: {{ .Values.featureflagServer.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.featureflagServer.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.featureflagServer.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.featureflagServer.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.featureflagServer.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.featureflagServer.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /health/liveness + port: http + initialDelaySeconds: {{ .Values.featureflagServer.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.featureflagServer.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.featureflagServer.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.featureflagServer.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.featureflagServer.readinessProbe.failureThreshold }} + {{- end }} + + ports: + - name: http + containerPort: 8007 + protocol: TCP + {{- if .Values.featureflagServer.debug.enabled }} + - name: debug + containerPort: {{ .Values.featureflagServer.debug.remoteDebugPort }} + protocol: TCP + {{- end }} + {{- if .Values.featureflagServer.resources }} + resources: {{- toYaml .Values.featureflagServer.resources | nindent 8 }} + {{- end }} + {{- if .Values.featureflagServer.containerSecurityContext }} + securityContext: {{- toYaml .Values.featureflagServer.containerSecurityContext | nindent 8 }} + {{- end }} + volumeMounts: + {{- if eq (include "airbyte.storage.type" .) "gcs" }} + - name: gcs-log-creds-volume + mountPath: {{ include "airbyte.storage.gcs.credentialsJsonPath" . }} + readOnly: true + {{- end }} + {{- if .Values.featureflagServer.extraVolumeMount }} +{{- toYaml .Values.featureflagServer.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.global.extraVolumeMounts }} +{{- toYaml .Values.global.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.featureflagServer.extraContainers }} + {{ toYaml .Values.featureflagServer.extraContainers | nindent 6 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 6 }} + {{- end }} + securityContext: {{- toYaml .Values.featureflagServer.podSecurityContext | nindent 6 }} + volumes: + {{- if eq (include "airbyte.storage.type" .) "gcs" }} + - name: gcs-log-creds-volume + secret: + secretName: {{ include "airbyte.storage.secretName" }} + items: + - key: {{ include "airbyte.storage.gcs.credentialsJson.secretKey" . }} + path: "gcp.json" + {{- end }} + {{- if .Values.featureflagServer.extraVolumes }} + {{- toYaml .Values.featureflagServer.extraVolumes | nindent 6 }} + {{- end }} + {{- if .Values.global.extraVolumes }} + {{- toYaml .Values.global.extraVolumes | nindent 6 }} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-featureflag-server/secrets.yaml b/helm/airbyte-v2/templates/airbyte-featureflag-server/secrets.yaml new file mode 100644 index 00000000..0828178b --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-featureflag-server/secrets.yaml @@ -0,0 +1,17 @@ +# Create secrets only for the local deployment + {{- if .Values.featureflagServer.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: featureflag-secrets + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" +type: Opaque +data: + {{- range $k, $v := mergeOverwrite .Values.featureflagServer.secrets .Values.global.secrets }} + {{ $k }}: {{ if $v }}{{ $v | b64enc }} {{else}}""{{end}} + {{- end }} + {{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-featureflag-server/service.yaml b/helm/airbyte-v2/templates/airbyte-featureflag-server/service.yaml new file mode 100644 index 00000000..7161873a --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-featureflag-server/service.yaml @@ -0,0 +1,24 @@ +{{- if and (.Values.featureflagServer.enabled) (eq .Values.global.edition "community") }} +apiVersion: v1 +kind: Service +metadata: + name: {{.Release.Name }}-featureflag-server-svc + {{- with .Values.featureflagServer.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + type: {{ .Values.featureflagServer.service.type }} + ports: + - port: {{ .Values.featureflagServer.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "airbyte.selectorLabels" . | nindent 4 }} + {{- if .Values.featureflagServer.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.featureflagServer.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-keycloak-setup/job.yaml b/helm/airbyte-v2/templates/airbyte-keycloak-setup/job.yaml new file mode 100644 index 00000000..c6c4df76 --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-keycloak-setup/job.yaml @@ -0,0 +1,133 @@ +{{- if (eq .Values.global.edition "enterprise") }} +{{- if .Values.keycloakSetup.enabled }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-keycloak-setup + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- if .Values.keycloakSetup.extraLabels }} + {{ toYaml (mergeOverwrite .Values.keycloakSetup.extraLabels .Values.global.extraLabels) | nindent 4 }} + {{- end }} + {{- if .Values.keycloakSetup.podLabels}} + {{- include "airbyte.tplvalues.render" (dict "value" .Values.keycloakSetup.podLabels "context" $) | nindent 4 }} + {{- end }} + annotations: + "helm.sh/hook": "post-install, post-upgrade" + "helm.sh/hook-delete-policy": "before-hook-creation" + {{- if .Values.keycloakSetup.annotations -}} + {{ toYaml .Values.keycloakSetup.annotations | nindent 4 }} + {{- end }} +spec: + completions: 1 + template: + spec: + serviceAccountName: {{ default .Values.keycloakSetup.serviceAccountName (include "airbyte.effectiveServiceAccountName" .) }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + restartPolicy: Never + {{- if .Values.keycloakSetup.nodeSelector }} + nodeSelector: {{- include "airbyte.tplvalues.render" (dict "value" .Values.keycloakSetup.nodeSelector "context" $) | nindent 6 }} + {{- end }} + {{- if .Values.keycloakSetup.tolerations }} + tolerations: {{- include "airbyte.tplvalues.render" (dict "value" .Values.keycloakSetup.tolerations "context" $) | nindent 6 }} + {{- end }} + {{- if .Values.keycloakSetup.affinity }} + affinity: {{- include "airbyte.tplvalues.render" (dict "value" .Values.keycloakSetup.affinity "context" $) | nindent 6 }} + {{- end }} + initContainers: + {{- if .Values.keycloakSetup.extraInitContainers }} + {{- toYaml .Values.keycloakSetup.extraInitContainers | nindent 8 }} + {{- end }} + containers: + - name: airbyte-keycloak-setup-container + image: {{ include "imageUrl" (list .Values.keycloakSetup.image $) }} + imagePullPolicy: "{{ .Values.keycloakSetup.image.pullPolicy }}" + env: + {{- if .Values.keycloakSetup.debug.enabled }} + - name: JAVA_TOOL_OPTIONS + value: "-Xdebug -agentlib:jdwp=transport=dt_socket,address=0.0.0.0:{{ .Values.keycloakSetup.debug.remoteDebugPort }},server=y,suspend=y" + {{- end}} + + {{- include "airbyte.common.envs" . | nindent 12 }} + {{- include "airbyte.database.envs" . | nindent 12 }} + {{- include "airbyte.keycloak.envs" . | nindent 12 }} + {{- include "airbyte.keycloak.admin.client.envs" . | nindent 12 }} + {{- include "airbyte.keycloak.admin.user.envs" . | nindent 12 }} + {{- include "airbyte.keycloak.client.envs" . | nindent 12 }} + + {{- if (eq .Values.global.edition "enterprise") }} + {{- include "airbyte.enterprise.envs" . | nindent 12 }} + {{- include "airbyte.auth.instanceAdmin.enterprise.envs" . | nindent 12 }} + {{- end}} + + # Values from secret + {{- if .Values.keycloakSetup.secrets }} + {{- range $k, $v := .Values.keycloakSetup.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: keycloak-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if .Values.keycloakSetup.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.keycloakSetup.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + # Values from extraEnv for more compability(if you want to use external secret source or other stuff) + {{- if .Values.keycloakSetup.extraEnv }} + {{- toYaml .Values.keycloakSetup.extraEnv | nindent 12 }} + {{- end }} + + {{- if .Values.keycloakSetup.debug.enabled }} + ports: + - name: debug + containerPort: {{ .Values.keycloakSetup.debug.remoteDebugPort }} + protocol: TCP + {{- end}} + + {{- if .Values.keycloakSetup.resources }} + resources: {{- toYaml .Values.keycloakSetup.resources | nindent 10 }} + {{- end }} + volumeMounts: + {{- if .Values.global.airbyteYml }} + - name: airbyte-yml-volume + mountPath: /app/configs/airbyte.yml + subPath: fileContents + readOnly: true + {{- end }} + {{- if .Values.keycloakSetup.extraVolumeMount }} + {{- toYaml .Values.keycloakSetup.extraVolumeMounts | nindent 12 }} + {{- end }} + {{- if .Values.keycloakSetup.containerSecurityContext }} + securityContext: {{- toYaml .Values.keycloakSetup.containerSecurityContext | nindent 12 }} + {{- end }} + {{- if .Values.keycloakSetup.extraContainers }} + {{ toYaml .Values.keycloakSetup.extraContainers | nindent 8 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 8 }} + {{- end }} + securityContext: {{- toYaml .Values.keycloakSetup.podSecurityContext | nindent 8 }} + volumes: + {{- if .Values.global.airbyteYml }} + - name: airbyte-yml-volume + secret: + secretName: {{ .Release.Name }}-airbyte-yml + {{- end }} + {{- if .Values.keycloakSetup.extraVolumes }} + {{- toYaml .Values.keycloakSetup.extraVolumes | nindent 8 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-keycloak/service.yaml b/helm/airbyte-v2/templates/airbyte-keycloak/service.yaml new file mode 100644 index 00000000..1d189e9b --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-keycloak/service.yaml @@ -0,0 +1,60 @@ +# If Cloud or Pro/Enterprise, render the keycloak service template. +{{- if or (eq .Values.global.edition "cloud") (eq .Values.global.edition "enterprise") }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{.Release.Name }}-airbyte-keycloak-svc + {{- with .Values.keycloak.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + type: {{ .Values.keycloak.service.type }} + ports: + - port: {{ .Values.keycloak.service.port }} + targetPort: http + protocol: TCP + name: http + {{ if and (eq .Values.keycloak.service.type "NodePort") (.Values.keycloak.service.nodePort) }} + nodePort: {{ .Values.keycloak.service.nodePort }} + {{- end }} + selector: + {{- include "airbyte.selectorLabels" . | nindent 4 }} + {{- if .Values.keycloak.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.keycloak.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 4 }} + {{- end }} +--- +# This headless service sets ClusterIp: None so that it can be +# used for internal DNS resolution of all individual keycloak +# pods. Port 7800 is the default jgroups port that Keycloak +# expects to be open to enable Infinispan cache synchronization. +apiVersion: v1 +kind: Service +metadata: + name: {{.Release.Name }}-airbyte-keycloak-headless-svc + {{- with .Values.keycloak.headlessService.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: jgroups + port: 7800 + targetPort: 7800 + - port: {{ .Values.keycloak.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "airbyte.selectorLabels" . | nindent 4 }} + {{- if .Values.keycloak.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.keycloak.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-keycloak/statefulset.yaml b/helm/airbyte-v2/templates/airbyte-keycloak/statefulset.yaml new file mode 100644 index 00000000..443be01c --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-keycloak/statefulset.yaml @@ -0,0 +1,181 @@ +# if Cloud or Pro/Enterprise, render the keycloak StatefulSet template. +{{- if or (eq .Values.global.edition "cloud") (eq .Values.global.edition "enterprise") }} +{{- if .Values.keycloak.enabled }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ .Release.Name }}-keycloak + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- if .Values.keycloak.extraLabels }} + {{ toYaml (mergeOverwrite .Values.keycloak.extraLabels .Values.global.extraLabels) | nindent 4 }} + {{- end }} + annotations: + {{ toYaml .Values.keycloak.annotations | nindent 4 }} +spec: + replicas: {{ .Values.keycloak.replicaCount }} + serviceName: {{.Release.Name }}-airbyte-keycloak-headless-svc + podManagementPolicy: OrderedReady # embedded infinispan does not handle parallel joins well, src https://github.com/keycloak/keycloak/issues/21108#issuecomment-1599010602 + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + {{- include "airbyte.selectorLabels" . | nindent 6 }} + {{- if .Values.keycloak.extraLabels }} + {{ toYaml (mergeOverwrite .Values.keycloak.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + {{- if .Values.keycloak.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.keycloak.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 8 }} + {{- end }} + {{- if .Values.keycloak.podLabels }} + {{- include "airbyte.tplvalues.render" (dict "value" .Values.keycloak.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.keycloak.podAnnotations }} + annotations: + {{- include "airbyte.tplvalues.render" (dict "value" .Values.keycloak.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ default .Values.keycloak.serviceAccountName (include "airbyte.effectiveServiceAccountName" .) }} + {{- if .Values.keycloak.nodeSelector }} + nodeSelector: {{- include "airbyte.tplvalues.render" (dict "value" .Values.keycloak.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.keycloak.tolerations }} + tolerations: {{- include "airbyte.tplvalues.render" (dict "value" .Values.keycloak.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.keycloak.affinity }} + affinity: {{- include "airbyte.tplvalues.render" (dict "value" .Values.keycloak.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + initContainers: + {{- if (ne .Values.keycloak.bypassInit true) }} + # This init container will only executed if the deployment mode is "oss" and the bypassInit is not true. + - name: init-db + image: {{ include "imageUrl" (list .Values.keycloak.initContainers.initDb.image $) }} + command: [ "sh", "-c" ] + args: + - > + PGPASSWORD=$DATABASE_PASSWORD psql -h $DATABASE_HOST -p $DATABASE_PORT -U $DATABASE_USER -d $DATABASE_DB -c "CREATE SCHEMA IF NOT EXISTS keycloak"; + env: + {{- include "airbyte.database.envs" . | nindent 12 }} + + {{- if .Values.keycloak.extraEnv }} + {{ .Values.keycloak.extraEnv | toYaml | nindent 12 }} + {{- end }} + + securityContext: + {{- toYaml .Values.keycloak.initContainerSecurityContext | nindent 14 }} + {{- end }} + {{- if .Values.keycloak.extraInitContainers }} + {{- toYaml .Values.keycloak.extraInitContainers | nindent 8 }} + {{- end }} + containers: + - name: airbyte-keycloak + image: {{ include "imageUrl" (list .Values.keycloak.image $)}} + imagePullPolicy: {{ .Values.keycloak.image.pullPolicy }} + env: + {{- include "airbyte.keycloak.envs" . | nindent 12 }} + {{- include "airbyte.keycloak.admin.user.envs" . | nindent 12 }} + {{- include "airbyte.keycloak.database.envs" . | nindent 12 }} + + {{- if .Values.keycloak.extraEnv }} + {{ .Values.keycloak.extraEnv | toYaml | nindent 12 }} + {{- end }} + + # Values from secret + {{- if .Values.keycloak.secrets }} + {{- range $k, $v := .Values.keycloak.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: keycloak-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if .Values.keycloak.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.keycloak.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.keycloak.service.port }} + protocol: TCP + - containerPort: 7800 + {{- if .Values.keycloak.containerSecurityContext }} + securityContext: {{- toYaml .Values.keycloak.containerSecurityContext | nindent 12 }} + {{- end }} + volumeMounts: + - name: keycloak-storage + mountPath: /opt/keycloak/data/infinispan + {{- if .Values.keycloak.extraVolumeMounts }} + {{ toYaml .Values.keycloak.extraVolumeMounts | nindent 10 }} + {{- end }} + {{- if .Values.keycloak.resources }} + resources: {{- toYaml .Values.keycloak.resources | nindent 12 }} + {{- end }} + {{- if .Values.keycloak.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /auth/health/live + port: {{ .Values.keycloak.service.port }} + initialDelaySeconds: {{ .Values.keycloak.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.keycloak.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.keycloak.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.keycloak.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.keycloak.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.keycloak.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /auth/health/ready + port: {{ .Values.keycloak.service.port }} + initialDelaySeconds: {{ .Values.keycloak.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.keycloak.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.keycloak.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.keycloak.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.keycloak.readinessProbe.failureThreshold }} + {{- end }} + {{- if .Values.keycloak.startupProbe.enabled }} + startupProbe: + httpGet: + path: /auth/health/started + port: {{ .Values.keycloak.service.port }} + initialDelaySeconds: {{ .Values.keycloak.startupProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.keycloak.startupProbe.periodSeconds }} + timeoutSeconds: {{ .Values.keycloak.startupProbe.timeoutSeconds }} + successThreshold: {{ .Values.keycloak.startupProbe.successThreshold }} + failureThreshold: {{ .Values.keycloak.startupProbe.failureThreshold }} + {{- end }} + {{- if .Values.keycloak.extraContainers }} + {{ toYaml .Values.keycloak.extraContainers | nindent 8 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 8 }} + {{- end }} + securityContext: {{- toYaml .Values.keycloak.podSecurityContext | nindent 10 }} + volumes: + {{- if .Values.keycloak.extraVolumes }} + {{ toYaml .Values.keycloak.extraVolumes | nindent 8 }} + {{- end }} + volumeClaimTemplates: + - metadata: + name: keycloak-storage + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi +{{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-metrics/deployment.yaml b/helm/airbyte-v2/templates/airbyte-metrics/deployment.yaml new file mode 100644 index 00000000..dcab78f2 --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-metrics/deployment.yaml @@ -0,0 +1,108 @@ +{{- if .Values.metrics.enabled }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-metrics + labels: + app.kubernetes.io/name: {{ include "airbyte.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + helm.sh/chart: {{ include "airbyte.chart" . }} + app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.airbyte.io/fullname: {{ include "airbyte.fullname" . }} + annotations: + {{ toYaml .Values.metrics.annotations | nindent 4 }} +spec: + replicas: {{ .Values.metrics.replicaCount }} + selector: + matchLabels: + airbyte: metrics + strategy: + type: Recreate # Needed due to volume claims + template: + metadata: + labels: + airbyte: metrics + {{- if .Values.metrics.podLabels }} + {{- include "airbyte.tplvalues.render" (dict "value" .Values.metrics.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.metrics.podAnnotations }} + annotations: + {{- include "airbyte.tplvalues.render" (dict "value" .Values.metrics.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ default .Values.metrics.serviceAccountName (include "airbyte.effectiveServiceAccountName" .) }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.metrics.nodeSelector }} + nodeSelector: {{- include "airbyte.tplvalues.render" (dict "value" .Values.metrics.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.metrics.tolerations }} + tolerations: {{- include "airbyte.tplvalues.render" (dict "value" .Values.metrics.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.metrics.affinity }} + affinity: {{- include "airbyte.tplvalues.render" (dict "value" .Values.metrics.affinity "context" $) | nindent 8 }} + {{- end }} + containers: + - name: airbyte-metrics-container + image: {{ include "imageUrl" (list .Values.metrics.image $) }} + imagePullPolicy: "{{ .Values.metrics.image.pullPolicy }}" + env: + {{- include "airbyte.common.envs" . | nindent 8 }} + {{- include "airbyte.database.envs" . | nindent 8 }} + {{- include "airbyte.database.migrations.envs" . | nindent 8 }} + {{- include "airbyte.metrics.envs" . | nindent 8 }} + + # Values from secret + {{- if .Values.metrics.secrets }} + {{- range $k, $v := .Values.metrics.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: metrics-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if or .Values.metrics.env_vars .Values.global.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.metrics.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + # Values from extraEnv for more compability(if you want to use external secret source or other stuff) + {{- if .Values.metrics.extraEnv }} + {{- toYaml .Values.metrics.extraEnv | nindent 8 }} + {{- end }} + + {{- if .Values.metrics.resources }} + resources: + {{- toYaml .Values.metrics.resources | nindent 10 }} + {{- end }} + {{- if .Values.metrics.containerSecurityContext }} + securityContext: + {{- toYaml .Values.metrics.containerSecurityContext | nindent 10 }} + {{- end }} + {{- if .Values.metrics.extraVolumeMounts }} + volumeMounts: + {{ toYaml .Values.metrics.extraVolumeMounts | nindent 10 }} + {{- end }} + {{- if .Values.metrics.extraContainers }} + {{ toYaml .Values.metrics.extraContainers | nindent 6 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 6 }} + {{- end }} + securityContext: {{- toYaml .Values.metrics.podSecurityContext | nindent 8 }} + volumes: + {{- if .Values.metrics.extraVolumes }} + {{ toYaml .Values.metrics.extraVolumes | nindent 6 }} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-server/deployment.yaml b/helm/airbyte-v2/templates/airbyte-server/deployment.yaml new file mode 100644 index 00000000..f5961dbc --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-server/deployment.yaml @@ -0,0 +1,260 @@ +{{- if .Values.server.enabled }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-server + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- if .Values.server.extraLabels }} + {{ toYaml (mergeOverwrite .Values.server.extraLabels .Values.global.extraLabels) | nindent 4 }} + {{- end }} + annotations: + {{ toYaml .Values.server.annotations | nindent 4 }} +spec: + minReadySeconds: 30 + replicas: {{ .Values.server.replicaCount }} + selector: + matchLabels: + {{- include "airbyte.selectorLabels" . | nindent 6 }} + {{- if .Values.server.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.server.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 6 }} + {{- end }} + strategy: + type: {{ .Values.server.deploymentStrategyType }} # Needed due to volume claims + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + {{- if .Values.server.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.server.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 8 }} + {{- end }} + {{- if .Values.server.podLabels }} + {{- include "airbyte.tplvalues.render" (dict "value" .Values.server.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.server.podAnnotations }} + annotations: + {{- include "airbyte.tplvalues.render" (dict "value" .Values.server.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ default .Values.server.serviceAccountName (include "airbyte.effectiveServiceAccountName" .) }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.server.nodeSelector }} + nodeSelector: {{- include "airbyte.tplvalues.render" (dict "value" .Values.server.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.server.tolerations }} + tolerations: {{- include "airbyte.tplvalues.render" (dict "value" .Values.server.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.server.affinity }} + affinity: {{- include "airbyte.tplvalues.render" (dict "value" .Values.server.affinity "context" $) | nindent 8 }} + {{- end }} + {{- $useTemporalWait := and .Values.temporal.enabled (ne (include "airbyte.temporal.cloud.enabled" .) "true") }} + {{- if or $useTemporalWait .Values.server.extraInitContainers }} + initContainers: + {{- if $useTemporalWait }} + - name: wait-for-temporal + image: {{ quote (default "busybox:1.36" (index (index .Values.server "temporalWaitInitContainer" | default dict) "image")) }} + command: + - sh + - -c + - | + echo "Waiting for Temporal at {{ .Release.Name }}-temporal:{{ .Values.temporal.service.port }}..." + until nc -z {{ .Release.Name }}-temporal {{ .Values.temporal.service.port }}; do + sleep 2 + done + echo "Temporal is ready." + {{- end }} + {{- if .Values.server.extraInitContainers }} + {{- toYaml .Values.server.extraInitContainers | nindent 6 }} + {{- end }} + {{- end }} + containers: + - name: airbyte-server-container + image: {{ include "imageUrl" (list .Values.server.image $) }} + imagePullPolicy: "{{ .Values.server.image.pullPolicy }}" + env: + {{- if .Values.server.debug.enabled }} + - name: JAVA_TOOL_OPTIONS + value: "-Xdebug -agentlib:jdwp=transport=dt_socket,address=0.0.0.0:{{ .Values.server.debug.remoteDebugPort }},server=y,suspend=n" + {{- end}} + + {{- if (or (eq .Values.global.edition "community") (eq .Values.global.edition "enterprise")) }} + {{- include "airbyte.auth.envs" . | nindent 8 }} + {{- include "airbyte.auth.bootstrap.envs" . | nindent 8 }} + {{- /* DATAPLANE_* from same secret as launcher so token endpoint accepts launcher; include secret name/key so server can resolve creds */}} + {{- include "airbyte.workloadLauncher.dataPlane.clientId.env" . | nindent 8 }} + {{- include "airbyte.workloadLauncher.dataPlane.clientIdSecretName.env" . | nindent 8 }} + {{- include "airbyte.workloadLauncher.dataPlane.clientIdSecretKey.env" . | nindent 8 }} + {{- include "airbyte.workloadLauncher.dataPlane.clientSecret.env" . | nindent 8 }} + {{- include "airbyte.workloadLauncher.dataPlane.clientSecretSecretName.env" . | nindent 8 }} + {{- include "airbyte.workloadLauncher.dataPlane.clientSecretSecretKey.env" . | nindent 8 }} + - name: AIRBYTE_API_AUTH_HEADER_VALUE + value: {{ include "airbyte.common.api.internalAuthHeaderValue" . | quote }} + {{- end}} + + {{- include "airbyte.auth.jwt.envs" . | nindent 8 }} + {{- include "airbyte.common.envs" . | nindent 8 }} + {{- include "airbyte.cluster.envs" . | nindent 8 }} + {{- include "airbyte.connector.envs" . | nindent 8 }} + {{- include "airbyte.customerio.envs" . | nindent 8 }} + {{- include "airbyte.database.envs" . | nindent 8 }} + {{- include "airbyte.database.migrations.envs" . | nindent 8 }} + {{- include "airbyte.jobs.envs" . | nindent 8 }} + {{- include "airbyte.jobs.errors.envs" . | nindent 8 }} + {{- include "airbyte.jobs.scheduling.envs" . | nindent 8 }} + {{- include "airbyte.logging.envs" . | nindent 8 }} + {{- include "airbyte.metrics.envs" . | nindent 8 }} + {{- include "airbyte.micronaut.envs" . | nindent 8 }} + {{- include "airbyte.storage.envs" . | nindent 8 }} + {{- include "airbyte.secretsManager.envs" . | nindent 8 }} + {{- include "airbyte.temporal.envs" . | nindent 8 }} + {{- include "airbyte.tracking.envs" . | nindent 8 }} + {{- include "airbyte.worker.envs" . | nindent 8 }} + {{- include "airbyte.workloads.resources.envs" . | nindent 8 }} + {{- /* AIRBYTE_MANIFEST_SERVER_API_BASE_PATH required for ManifestServerApiClient (ConnectorBuilderProjectApiController) */}} + {{- include "airbyte.connectorBuilder.manifestServerApi.basePath.env" . | nindent 8 }} + + {{- if .Values.global.datadog.enabled }} + - name: DD_SERVICE + value: {{ include "airbyte.componentNameWithAirbytePrefix" . }} + {{- include "airbyte.datadog.envs" . | nindent 8 }} + {{- include "airbyte.otel.envs" . | nindent 8 }} + {{- end}} + + {{- if (eq .Values.global.edition "enterprise") }} + {{- include "airbyte.auth.instanceAdmin.enterprise.envs" . | nindent 8 }} + {{- include "airbyte.auth.identityProvider.envs" . | nindent 8 }} + {{- include "airbyte.enterprise.envs" . | nindent 8 }} + {{- include "airbyte.keycloak.envs" . | nindent 8 }} + {{- include "airbyte.keycloak.admin.client.envs" . | nindent 8 }} + {{- include "airbyte.keycloak.admin.user.envs" . | nindent 8 }} + {{- include "airbyte.keycloak.client.envs" . | nindent 8 }} + {{- else if (eq .Values.global.edition "cloud") }} + {{- include "airbyte.connectorRollout.envs" . | nindent 8 }} + {{- include "airbyte.featureFlags.envs" . | nindent 8 }} + {{- include "airbyte.keycloak.admin.client.envs" . | nindent 8 }} + {{- include "airbyte.keycloak.admin.user.envs" . | nindent 8 }} + {{- include "airbyte.keycloak.client.envs" . | nindent 8 }} + {{- include "airbyte.orb.envs" . | nindent 8 -}} + {{- include "airbyte.stripe.envs" . | nindent 8 }} + {{- include "airbyte.temporal.cloud.envs" . | nindent 8 }} + {{- include "airbyte.temporal.cloud.billing.envs" . | nindent 8 }} + {{- include "airbyte.temporal.cloud.connectorRollout.envs" . | nindent 8 }} + {{- include "airbyte.temporal.cloud.credentials.envs" . | nindent 8 }} + {{- end}} + + # Values from secret + {{- if .Values.server.secrets }} + {{- range $k, $v := .Values.server.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: server-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if or .Values.server.env_vars .Values.global.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.server.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + {{- /* Dataplane credentials last so they are not overwritten by env_vars (token endpoint needs these) */}} + {{- if (or (eq .Values.global.edition "community") (eq .Values.global.edition "enterprise")) }} + {{- include "airbyte.workloadLauncher.dataPlane.clientId.env" . | nindent 8 }} + {{- include "airbyte.workloadLauncher.dataPlane.clientIdSecretName.env" . | nindent 8 }} + {{- include "airbyte.workloadLauncher.dataPlane.clientIdSecretKey.env" . | nindent 8 }} + {{- include "airbyte.workloadLauncher.dataPlane.clientSecret.env" . | nindent 8 }} + {{- include "airbyte.workloadLauncher.dataPlane.clientSecretSecretName.env" . | nindent 8 }} + {{- include "airbyte.workloadLauncher.dataPlane.clientSecretSecretKey.env" . | nindent 8 }} + {{- end }} + + # Values from extraEnv for more compability(if you want to use external secret source or other stuff) + {{- if .Values.server.extraEnv }} + {{- toYaml .Values.server.extraEnv | nindent 8 }} + {{- end }} + + {{- if .Values.server.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /api/v1/health + port: http + initialDelaySeconds: {{ .Values.server.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.server.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.server.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.server.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.server.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.server.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /api/v1/health + port: http + initialDelaySeconds: {{ .Values.server.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.server.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.server.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.server.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.server.readinessProbe.failureThreshold }} + {{- end }} + ports: + - name: http + containerPort: 8001 + protocol: TCP + {{- if .Values.server.debug.enabled }} + - name: debug + containerPort: {{ .Values.server.debug.remoteDebugPort }} + protocol: TCP + {{- end}} + {{- if .Values.server.resources }} + resources: {{- toYaml .Values.server.resources | nindent 10 }} + {{- end }} + {{- if .Values.server.containerSecurityContext }} + securityContext: {{- toYaml .Values.server.containerSecurityContext | nindent 10 }} + {{- end }} + volumeMounts: + {{- if eq (include "airbyte.storage.type" .) "gcs" }} + - name: gcs-log-creds-volume + mountPath: {{ include "airbyte.storage.gcs.credentialsJsonPath" . | dir }} + readOnly: true + {{- end }} + + {{- if .Values.server.extraVolumeMounts }} + {{ toYaml .Values.server.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.global.extraVolumeMounts }} + {{ toYaml .Values.global.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.server.extraContainers }} + {{ toYaml .Values.server.extraContainers | nindent 6 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 6 }} + {{- end }} + securityContext: {{- toYaml .Values.server.podSecurityContext | nindent 8 }} + volumes: + {{- if eq (include "airbyte.storage.type" .) "gcs" }} + - name: gcs-log-creds-volume + secret: + secretName: {{ include "airbyte.storage.secretName" . }} + items: + - key: {{ include "airbyte.storage.gcs.credentialsJson.secretKey" . }} + path: "gcp.json" + {{- end }} + + {{- if .Values.global.airbyteYml }} + - name: airbyte-yml-volume + secret: + secretName: {{ .Release.Name }}-airbyte-yml + {{- end }} + {{- if .Values.server.extraVolumes }} + {{ toYaml .Values.server.extraVolumes | nindent 6 }} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-server/secrets.yaml b/helm/airbyte-v2/templates/airbyte-server/secrets.yaml new file mode 100644 index 00000000..7fd3947b --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-server/secrets.yaml @@ -0,0 +1,17 @@ +# Create secrets only for the local deployment +{{- if .Values.server.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: server-secrets + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" +type: Opaque +data: + {{- range $k, $v := mergeOverwrite .Values.server.secrets .Values.global.secrets }} + {{ $k }}: {{ if $v }}{{ $v | b64enc }} {{else}}""{{end}} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-server/service.yaml b/helm/airbyte-v2/templates/airbyte-server/service.yaml new file mode 100644 index 00000000..6e79b620 --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-server/service.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{.Release.Name }}-airbyte-server-svc + {{- with .Values.server.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + type: {{ .Values.server.service.type }} + ports: + - port: {{ .Values.server.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "airbyte.selectorLabels" . | nindent 4 }} + {{- if .Values.server.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.server.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 4 }} + {{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-temporal-ui/deployment.yaml b/helm/airbyte-v2/templates/airbyte-temporal-ui/deployment.yaml new file mode 100644 index 00000000..be146c4c --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-temporal-ui/deployment.yaml @@ -0,0 +1,126 @@ +{{- if .Values.temporalUi.enabled }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-temporal-ui + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.temporalUi.replicaCount }} + selector: + matchLabels: + {{- include "airbyte.selectorLabels" . | nindent 6 }} + {{- if .Values.temporalUi.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.temporalUi.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 6 }} + {{- end }} + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + {{- if .Values.temporalUi.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.temporalUi.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 8 }} + {{- end }} + {{- if .Values.temporalUi.podLabels}} + {{- include "airbyte.tplvalues.render" (dict "value" .Values.temporalUi.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.temporalUi.podAnnotations }} + annotations: + {{- include "airbyte.tplvalues.render" (dict "value" .Values.temporalUi.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ default .Values.temporalUi.serviceAccountName (include "airbyte.effectiveServiceAccountName" .) }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.temporalUi.nodeSelector }} + nodeSelector: {{- include "airbyte.tplvalues.render" (dict "value" .Values.temporalUi.nodeSelector "context" $) | nindent 6 }} + {{- end }} + {{- if .Values.temporalUi.tolerations }} + tolerations: {{- include "airbyte.tplvalues.render" (dict "value" .Values.temporalUi.tolerations "context" $) | nindent 6 }} + {{- end }} + {{- if .Values.temporalUi.affinity }} + affinity: {{- include "airbyte.tplvalues.render" (dict "value" .Values.temporalUi.affinity "context" $) | nindent 6 }} + {{- end }} + {{- if .Values.temporalUi.extraInitContainers }} + initContainers: + {{- toYaml .Values.temporalUi.extraInitContainers | nindent 6 }} + {{- end }} + containers: + - name: airbyte-temporal-ui + image: {{ include "imageUrl" (list .Values.temporalUi.image $) }} + imagePullPolicy: {{ .Values.temporalUi.image.pullPolic }} + env: + # Values from env + {{- if or .Values.temporalUi.env_vars .Values.global.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.temporalUi.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + # Values from extraEnv for more compability(if you want to use external secret source or other stuff) + {{- if .Values.temporalUi.extraEnv }} + {{- toYaml .Values.temporalUi.extraEnv | nindent 8 }} + {{- end }} + + {{- if .Values.temporalUi.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /v1/health + port: http + initialDelaySeconds: {{ .Values.temporalUi.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.temporalUi.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.temporalUi.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.temporalUi.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.temporalUi.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.temporalUi.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /v1/health + port: http + initialDelaySeconds: {{ .Values.temporalUi.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.temporalUi.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.temporalUi.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.temporalUi.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.temporalUi.readinessProbe.failureThreshold }} + {{- end }} + + ports: + - name: http + containerPort: 8080 + protocol: TCP + {{- if .Values.temporalUi.debug.enabled }} + - name: debug + containerPort: {{ .Values.temporalUi.debug.remoteDebugPort }} + protocol: TCP + {{- end}} + {{- if .Values.temporalUi.resources }} + resources: {{- toYaml .Values.temporalUi.resources | nindent 8 }} + {{- end }} + {{- if .Values.temporalUi.containerSecurityContext }} + securityContext: {{- toYaml .Values.temporalUi.containerSecurityContext | nindent 8 }} + {{- end }} + volumeMounts: + {{- if .Values.temporalUi.extraVolumeMount }} + {{ toYaml .Values.temporalUi.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.global.extraVolumeMounts }} + {{ toYaml .Values.global.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.temporalUi.extraContainers }} + {{ toYaml .Values.temporalUi.extraContainers | nindent 8 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 8 }} + {{- end }} + securityContext: {{- toYaml .Values.temporalUi.podSecurityContext | nindent 8 }} + volumes: + {{- if .Values.temporalUi.extraVolumes }} + {{ toYaml .Values.temporalUi.extraVolumes | nindent 8 }} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-temporal-ui/service.yaml b/helm/airbyte-v2/templates/airbyte-temporal-ui/service.yaml new file mode 100644 index 00000000..bad36c9b --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-temporal-ui/service.yaml @@ -0,0 +1,22 @@ +{{- if .Values.temporalUi.enabled }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{.Release.Name }}-airbyte-temporal-ui-svc + {{- with .Values.temporalUi.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + type: {{ .Values.temporalUi.service.type }} + ports: + - port: {{ .Values.temporalUi.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "airbyte.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-temporal/configmap.yaml b/helm/airbyte-v2/templates/airbyte-temporal/configmap.yaml new file mode 100644 index 00000000..fb2ac701 --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-temporal/configmap.yaml @@ -0,0 +1,58 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-temporal-dynamicconfig + labels: + {{- include "airbyte.labels" . | nindent 4 }} +data: + "development.yaml": | + # when modifying, remember to update the docker-compose version of this file in temporal/dynamicconfig/development.yaml + # Disable Nexus to avoid "unknown method ListNexusEndpoints" when matching service does not implement it (version skew). + system.enableNexus: + - value: false + constraints: {} + frontend.enableClientVersionCheck: + - value: true + constraints: {} + history.persistenceMaxQPS: + - value: 3000 + constraints: {} + frontend.persistenceMaxQPS: + - value: 3000 + constraints: {} + frontend.historyMgrNumConns: + - value: 30 + constraints: {} + frontend.throttledLogRPS: + - value: 20 + constraints: {} + frontend.enableUpdateWorkflowExecution: + - value: true + frontend.enableUpdateWorkflowExecutionAsyncAccepted: + - value: true + history.historyMgrNumConns: + - value: 50 + constraints: {} + system.advancedVisibilityWritingMode: + - value: "off" + constraints: {} + history.defaultActivityRetryPolicy: + - value: + InitialIntervalInSeconds: 1 + MaximumIntervalCoefficient: 100.0 + BackoffCoefficient: 2.0 + MaximumAttempts: 0 + history.defaultWorkflowRetryPolicy: + - value: + InitialIntervalInSeconds: 1 + MaximumIntervalCoefficient: 100.0 + BackoffCoefficient: 2.0 + MaximumAttempts: 0 + # Limit for responses. This mostly impacts discovery jobs since they have the largest responses. + limit.blobSize.error: + - value: 15728640 # 15MB + constraints: {} + limit.blobSize.warn: + - value: 10485760 # 10MB + constraints: {} diff --git a/helm/airbyte-v2/templates/airbyte-temporal/deployment.yaml b/helm/airbyte-v2/templates/airbyte-temporal/deployment.yaml new file mode 100644 index 00000000..0c2602f3 --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-temporal/deployment.yaml @@ -0,0 +1,160 @@ +{{- if .Values.temporal.enabled }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-temporal + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- if .Values.temporal.extraLabels }} + {{ toYaml (mergeOverwrite .Values.temporal.extraLabels .Values.global.extraLabels) | nindent 4 }} + {{- end }} + annotations: + {{ toYaml .Values.temporal.annotations | nindent 4 }} +spec: + replicas: {{ .Values.temporal.replicaCount }} + selector: + matchLabels: + {{- include "airbyte.selectorLabels" . | nindent 6 }} + {{- if .Values.temporal.extraLabels }} + {{ toYaml (mergeOverwrite .Values.temporal.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + {{- if .Values.temporal.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.temporal.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 8 }} + {{- end }} + {{- if .Values.temporal.podLabels }} + {{- include "airbyte.tplvalues.render" (dict "value" .Values.temporal.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.temporal.podAnnotations }} + annotations: + {{- include "airbyte.tplvalues.render" (dict "value" .Values.temporal.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ default .Values.temporal.serviceAccountName (include "airbyte.effectiveServiceAccountName" .) }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.temporal.nodeSelector }} + nodeSelector: {{- include "airbyte.tplvalues.render" (dict "value" .Values.temporal.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.temporal.tolerations }} + tolerations: {{- include "airbyte.tplvalues.render" (dict "value" .Values.temporal.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.temporal.affinity }} + affinity: {{- include "airbyte.tplvalues.render" (dict "value" .Values.temporal.affinity "context" $) | nindent 8 }} + {{- end }} +{{- if .Values.temporal.extraInitContainers }} + initContainers: +{{- toYaml .Values.temporal.extraInitContainers | nindent 6 }} +{{- end }} + containers: + - name: airbyte-temporal + image: {{ include "imageUrl" (list .Values.temporal.image $) }} + imagePullPolicy: {{ .Values.temporal.image.pullPolicy }} + {{- if and .Values.temporal.command (gt (len .Values.temporal.command) 0) }} + command: {{ toYaml .Values.temporal.command | nindent 8 }} + {{- end }} + {{- if .Values.temporal.args }} + args: {{ toYaml .Values.temporal.args | nindent 8 }} + {{- end }} + {{- /* When this pod runs the Airbyte workers app, load full airbyte-env so all Micronaut properties resolve (same source as worker) */}} + envFrom: + - configMapRef: + name: {{ .Release.Name }}-airbyte-env + env: + {{- include "airbyte.temporal.envs" . | nindent 10 }} + {{- include "airbyte.temporal.database.envs" . | nindent 10 }} + {{- /* Literal overrides so these are set even if ConfigMap is missing a key */}} + - name: STORAGE_TYPE + value: {{ include "airbyte.storage.type" . | quote }} + - name: WORKSPACE_ROOT + value: {{ include "airbyte.worker.workspaceRoot" . | quote }} + - name: AIRBYTE_INTERNAL_API_BASE_PATH + value: {{ include "airbyte.common.api.internalHost" . | quote }} + + {{- if .Values.temporal.extraEnv }} + {{ .Values.temporal.extraEnv | toYaml | nindent 10 }} + {{- end }} + # Values from secret + {{- if .Values.temporal.secrets }} + {{- range $k, $v := .Values.temporal.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: temporal-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if or .Values.temporal.env_vars .Values.global.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.temporal.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + ports: + - containerPort: 7233 + {{- if .Values.temporal.containerSecurityContext }} + securityContext: {{- toYaml .Values.temporal.containerSecurityContext | nindent 10 }} + {{- end }} + volumeMounts: + - name: airbyte-temporal-dynamicconfig + mountPath: "/etc/temporal/config/dynamicconfig/" + {{- if .Values.temporal.extraVolumeMounts }} + {{ toYaml .Values.temporal.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.temporal.resources }} + resources: {{- toYaml .Values.temporal.resources | nindent 10 }} + {{- end }} + {{- if .Values.temporal.livenessProbe.enabled }} + livenessProbe: + exec: + command: + - /bin/sh + - -ec + - 'test $(ps -ef | grep -v grep | grep temporal-server | wc -l) -eq 1' + initialDelaySeconds: {{ .Values.temporal.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.temporal.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.temporal.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.temporal.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.temporal.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.temporal.readinessProbe.enabled }} + readinessProbe: + exec: + command: + - /bin/sh + - -ec + - 'test $(ps -ef | grep -v grep | grep temporal-server | wc -l) -eq 1' + initialDelaySeconds: {{ .Values.temporal.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.temporal.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.temporal.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.temporal.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.temporal.readinessProbe.failureThreshold }} + {{- end }} + {{- if .Values.temporal.extraContainers }} + {{ toYaml .Values.temporal.extraContainers | nindent 6 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 6 }} + {{- end }} + securityContext: {{- toYaml .Values.temporal.podSecurityContext | nindent 8 }} + volumes: + - name: airbyte-temporal-dynamicconfig + configMap: + name: {{ .Release.Name }}-temporal-dynamicconfig + items: + - key: development.yaml + path: development.yaml + {{- if .Values.temporal.extraVolumes }} + {{ toYaml .Values.temporal.extraVolumes | nindent 6 }} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-temporal/secrets.yaml b/helm/airbyte-v2/templates/airbyte-temporal/secrets.yaml new file mode 100644 index 00000000..f5276be0 --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-temporal/secrets.yaml @@ -0,0 +1,17 @@ +# Create secrets only for the local deployment +{{- if .Values.temporal.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: temporal-secrets + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" +type: Opaque +data: + {{- range $k, $v := mergeOverwrite .Values.temporal.secrets .Values.global.secrets }} + {{ $k }}: {{ if $v }}{{ $v | b64enc }} {{else}}""{{end}} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-temporal/service.yaml b/helm/airbyte-v2/templates/airbyte-temporal/service.yaml new file mode 100644 index 00000000..d6409d46 --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-temporal/service.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-temporal + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + type: {{ .Values.temporal.service.type }} + ports: + - port: {{ .Values.temporal.service.port }} + protocol: TCP + targetPort: 7233 + selector: + {{- include "airbyte.selectorLabels" . | nindent 4 }} + {{- if .Values.temporal.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.temporal.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 4 }} + {{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-webapp/deployment.yaml b/helm/airbyte-v2/templates/airbyte-webapp/deployment.yaml new file mode 100644 index 00000000..a361919c --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-webapp/deployment.yaml @@ -0,0 +1,152 @@ +{{- if .Values.webapp.enabled }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-webapp + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- if .Values.webapp.extraLabels }} + {{ toYaml (mergeOverwrite .Values.extraLabels .Values.global.extraLabels) | nindent 4 }} + {{- end }} + annotations: + {{ toYaml .Values.webapp.annotations | nindent 4 }} +spec: + replicas: {{ .Values.webapp.replicaCount }} + selector: + matchLabels: + {{- include "airbyte.selectorLabels" . | nindent 6 }} + {{- if .Values.webapp.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.webapp.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 6 }} + {{- end }} + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + {{- if .Values.webapp.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.webapp.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 8 }} + {{- end }} + {{- if .Values.webapp.podLabels }} + {{- include "airbyte.tplvalues.render" (dict "value" .Values.webapp.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.webapp.podAnnotations }} + annotations: + {{- include "airbyte.tplvalues.render" (dict "value" .Values.webapp.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ default .Values.webapp.serviceAccountName (include "airbyte.effectiveServiceAccountName" .) }} + {{- if .Values.webapp.nodeSelector }} + nodeSelector: {{- include "airbyte.tplvalues.render" (dict "value" .Values.webapp.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.webapp.tolerations }} + tolerations: {{- include "airbyte.tplvalues.render" (dict "value" .Values.webapp.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.webapp.affinity }} + affinity: {{- include "airbyte.tplvalues.render" (dict "value" .Values.webapp.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.webapp.extraInitContainers }} + initContainers: + {{- toYaml .Values.webapp.extraInitContainers | nindent 6 }} + {{- end }} + containers: + - name: airbyte-webapp-container + image: {{ include "imageUrl" (list .Values.webapp.image $) }} + imagePullPolicy: "{{ .Values.webapp.image.pullPolicy }}" + env: + + {{ include "airbyte.common.envs" . | nindent 8 }} + {{ include "airbyte.keycloak.admin.client.internalHost.env" . | nindent 8 }} + {{ include "airbyte.tracking.envs" . | nindent 8 }} + {{ include "airbyte.webapp.envs" . | nindent 8 }} + + {{- if .Values.global.datadog.enabled }} + - name: DD_SERVICE + value: {{ include "airbyte.componentNameWithAirbytePrefix" . }} + {{- include "airbyte.datadog.envs" . | nindent 8 }} + {{- include "airbyte.otel.envs" . | nindent 8 }} + {{- end}} + + # Values from secret + {{- if .Values.webapp.secrets }} + {{- range $k, $v := .Values.webapp.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: webapp-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if or .Values.webapp.env_vars .Values.global.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.webapp.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + # Values from extraEnv for more compability(if you want to use external secret source or other stuff) + {{- if .Values.webapp.extraEnv }} + {{- toYaml .Values.webapp.extraEnv | nindent 8 }} + {{- end }} + + + {{- if .Values.webapp.livenessProbe.enabled }} + livenessProbe: + tcpSocket: + port: http + initialDelaySeconds: {{ .Values.webapp.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.webapp.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.webapp.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.webapp.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.webapp.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.webapp.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /index.html + port: http + initialDelaySeconds: {{ .Values.webapp.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.webapp.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.webapp.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.webapp.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.webapp.readinessProbe.failureThreshold }} + {{- end }} + ports: + - name: http + containerPort: 8080 + protocol: TCP + {{- if .Values.webapp.resources }} + resources: {{- toYaml .Values.webapp.resources | nindent 10 }} + {{- end }} + {{- if .Values.webapp.containerSecurityContext }} + securityContext: {{- toYaml .Values.webapp.containerSecurityContext | nindent 10 }} + {{- end }} + volumeMounts: + {{- if .Values.webapp.extraVolumeMounts }} + {{ toYaml .Values.webapp.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.global.extraVolumeMounts }} + {{ toYaml .Values.global.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.webapp.extraContainers }} + {{ toYaml .Values.webapp.extraContainers | nindent 6 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 6 }} + {{- end }} + securityContext: {{- toYaml .Values.webapp.podSecurityContext | nindent 8 }} + volumes: + {{- if .Values.webapp.extraVolumes }} +{{ toYaml .Values.webapp.extraVolumes | nindent 6 }} + {{- end }} + {{- if .Values.global.extraVolumes }} +{{ toYaml .Values.global.extraVolumes | nindent 6 }} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-webapp/ingress.yaml b/helm/airbyte-v2/templates/airbyte-webapp/ingress.yaml new file mode 100644 index 00000000..7ed8f08f --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-webapp/ingress.yaml @@ -0,0 +1,66 @@ +{{- if .Values.webapp.ingress.enabled }} + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "airbyte.fullname" . }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- with .Values.webapp.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + +spec: + {{- if .Values.webapp.ingress.className }} + ingressClassName: {{ .Values.webapp.ingress.className }} + {{- end }} + + {{- if .Values.webapp.ingress.tls }} + tls: + {{- range .Values.webapp.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.webapp.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- $serverPort := int ($.Values.server.service.port | default 8001) }} + - path: /api + pathType: Prefix + backend: + service: + name: {{ $.Release.Name }}-airbyte-server-svc + port: + number: {{ $serverPort }} + - path: /auth + pathType: Prefix + backend: + service: + name: {{ $.Release.Name }}-airbyte-server-svc + port: + number: {{ $serverPort }} + - path: /connector-builder-api + pathType: Prefix + backend: + service: + name: {{ $.Release.Name }}-airbyte-connector-builder-server-svc + port: + number: {{ $.Values.connectorBuilderServer.service.port | default 80 }} + - path: / + pathType: Prefix + backend: + service: + name: {{ $.Release.Name }}-airbyte-server-svc + port: + number: {{ $serverPort }} + + {{- end }} + +{{- end }} \ No newline at end of file diff --git a/helm/airbyte-v2/templates/airbyte-webapp/secrets.yaml b/helm/airbyte-v2/templates/airbyte-webapp/secrets.yaml new file mode 100644 index 00000000..8ac24955 --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-webapp/secrets.yaml @@ -0,0 +1,17 @@ +# Create secrets only for the local deployment +{{- if .Values.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: webapp-secrets + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" +type: Opaque +data: + {{- range $k, $v := mergeOverwrite .Values.secrets .Values.global.secrets }} + {{ $k }}: {{ if $v }}{{ $v | b64enc }} {{else}}""{{end}} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-webapp/service.yaml b/helm/airbyte-v2/templates/airbyte-webapp/service.yaml new file mode 100644 index 00000000..a8d35e0d --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-webapp/service.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-airbyte-webapp-svc + {{- with .Values.webapp.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + type: {{ .Values.webapp.service.type }} + ports: + - port: {{ .Values.webapp.service.port }} + targetPort: http + protocol: TCP + name: http + {{ if and (eq .Values.webapp.service.type "NodePort") (.Values.webapp.service.nodePort) }} + nodePort: {{ .Values.webapp.service.nodePort }} + {{- end }} + selector: + {{- include "airbyte.selectorLabels" . | nindent 4 }} + {{- if .Values.webapp.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.webapp.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 4 }} + {{- end }} + {{- if .Values.webapp.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.webapp.service.loadBalancerIP }} + {{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-worker/deployment.yaml b/helm/airbyte-v2/templates/airbyte-worker/deployment.yaml new file mode 100644 index 00000000..af5ff24c --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-worker/deployment.yaml @@ -0,0 +1,240 @@ +{{- if .Values.worker.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-worker + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- if .Values.worker.extraLabels }} + {{ toYaml (mergeOverwrite .Values.worker.extraLabels .Values.global.extraLabels) | nindent 4 }} + {{- end }} + annotations: + {{ toYaml .Values.worker.annotations | nindent 4 }} +spec: + replicas: {{ .Values.worker.replicaCount }} + selector: + matchLabels: + {{- include "airbyte.selectorLabels" . | nindent 6 }} + {{- if .Values.worker.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.worker.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 6 }} + {{- end }} + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + {{- if .Values.worker.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.worker.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 8 }} + {{- end }} + {{- if .Values.worker.podLabels }} + {{- include "airbyte.tplvalues.render" (dict "value" .Values.worker.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.worker.podAnnotations }} + annotations: + {{- include "airbyte.tplvalues.render" (dict "value" .Values.worker.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ default .Values.worker.serviceAccountName (include "airbyte.effectiveServiceAccountName" .) }} + automountServiceAccountToken: true + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.worker.nodeSelector }} + nodeSelector: {{- include "airbyte.tplvalues.render" (dict "value" .Values.worker.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.worker.tolerations }} + tolerations: {{- include "airbyte.tplvalues.render" (dict "value" .Values.worker.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.worker.affinity }} + affinity: {{- include "airbyte.tplvalues.render" (dict "value" .Values.worker.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.worker.extraInitContainers }} + initContainers: + {{- toYaml .Values.worker.extraInitContainers | nindent 6 }} + {{- end }} + containers: + - name: airbyte-worker-container + image: {{ include "imageUrl" (list .Values.worker.image $) }} + imagePullPolicy: "{{ .Values.worker.image.pullPolicy }}" + env: + {{- if .Values.worker.debug.enabled }} + - name: JAVA_TOOL_OPTIONS + value: "-Xdebug -agentlib:jdwp=transport=dt_socket,address=0.0.0.0:{{ .Values.worker.debug.remoteDebugPort }},server=y,suspend=n" + {{- end}} + + {{- if (or (eq .Values.global.edition "community") (eq .Values.global.edition "enterprise")) }} + - name: AIRBYTE_API_AUTH_HEADER_VALUE + value: "Internal worker" + {{- end }} + + {{- include "airbyte.common.envs" . | nindent 8 }} + {{- include "airbyte.database.envs" . | nindent 8 }} + {{- include "airbyte.database.migrations.envs" . | nindent 8 }} + {{- include "airbyte.jobs.envs" . | nindent 8 }} + {{- include "airbyte.jobs.errors.envs" . | nindent 8 }} + {{- include "airbyte.keycloak.client.envs" . | nindent 8 }} + {{- include "airbyte.logging.envs" . | nindent 8 }} + {{- include "airbyte.metrics.envs" . | nindent 8 }} + {{- include "airbyte.micronaut.envs" . | nindent 8 }} + {{- include "airbyte.secretsManager.envs" . | nindent 8 }} + {{- include "airbyte.storage.envs" . | nindent 8 }} + {{- include "airbyte.temporal.envs" . | nindent 8 }} + {{- include "airbyte.temporal.worker.envs" . | nindent 8 }} + {{- include "airbyte.tracking.envs" . | nindent 8 }} + {{- include "airbyte.worker.envs" . | nindent 8 }} + {{- include "airbyte.workloads.resources.envs" . | nindent 8 }} + {{- include "airbyte.workloadApiServer.envs" . | nindent 8 }} + {{- /* AB_JWT_SIGNATURE_SECRET required for WorkloadClient internal auth (signatureSecret) */}} + {{- if or (eq .Values.global.edition "community") (eq .Values.global.edition "enterprise") }} + {{- include "airbyte.auth.jwt.envs" . | nindent 8 }} + {{- end }} + + {{- if .Values.global.datadog.enabled }} + {{- if (eq .Values.global.edition "cloud") }} + - name: DD_SERVICE + value: airbyte-cloud-workers + {{- else }} + - name: DD_SERVICE + value: {{ include "airbyte.componentNameWithAirbytePrefix" . }} + {{- end }} + {{- include "airbyte.datadog.envs" . | nindent 8 }} + {{- include "airbyte.otel.envs" . | nindent 8 }} + {{- end}} + + {{- if (eq .Values.global.edition "enterprise") }} + {{- include "airbyte.enterprise.envs" . | nindent 8 }} + {{- end }} + + {{- if (eq .Values.global.edition "cloud") }} + {{- include "airbyte.featureFlags.envs" . | nindent 8 }} + {{- include "airbyte.temporal.cloud.envs" . | nindent 8 }} + {{- include "airbyte.temporal.cloud.credentials.envs" . | nindent 8 }} + {{- end }} + + # Values from secret + {{- if .Values.worker.secrets }} + {{- range $k, $v := .Values.worker.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: worker-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if or .Values.worker.env_vars .Values.global.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.worker.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + # Values from extraEnv for more compatibility (if you want to use external secret source or other stuff) + {{- if .Values.worker.extraEnv }} + {{- toYaml .Values.worker.extraEnv | nindent 8 }} + {{- end }} + + {{- if .Values.worker.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: / + port: heartbeat + initialDelaySeconds: {{ .Values.worker.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.worker.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.worker.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.worker.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.worker.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.worker.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: / + port: heartbeat + initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.worker.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.worker.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }} + {{- end }} + ports: + {{- if .Values.worker.debug.enabled }} + - name: debug + containerPort: {{ .Values.worker.debug.remoteDebugPort }} + protocol: TCP + {{- end}} + - name: heartbeat + containerPort: 9000 # for heartbeat server + - containerPort: 9001 # start temporal worker port pool + - containerPort: 9002 + - containerPort: 9003 + - containerPort: 9004 + - containerPort: 9005 + - containerPort: 9006 + - containerPort: 9007 + - containerPort: 9008 + - containerPort: 9009 + - containerPort: 9010 + - containerPort: 9011 + - containerPort: 9012 + - containerPort: 9013 + - containerPort: 9014 + - containerPort: 9015 + - containerPort: 9016 + - containerPort: 9017 + - containerPort: 9018 + - containerPort: 9019 + - containerPort: 9020 + - containerPort: 9021 + - containerPort: 9022 + - containerPort: 9023 + - containerPort: 9024 + - containerPort: 9025 + - containerPort: 9026 + - containerPort: 9027 + - containerPort: 9028 + - containerPort: 9029 + - containerPort: 9030 # end temporal worker port pool + {{- if .Values.worker.resources }} + resources: {{- toYaml .Values.worker.resources | nindent 10 }} + {{- end }} + {{- if .Values.worker.containerSecurityContext }} + securityContext: {{- toYaml .Values.worker.containerSecurityContext | nindent 10 }} + {{- end }} + volumeMounts: + {{- if eq (include "airbyte.storage.type" .) "gcs" }} + - name: gcs-log-creds-volume + mountPath: {{ include "airbyte.storage.gcs.credentialsJsonPath" . | dir }} + readOnly: true + {{- end }} + {{- if .Values.worker.extraVolumeMounts }} +{{- toYaml .Values.worker.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.global.extraVolumeMounts }} +{{- toYaml .Values.global.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.worker.extraContainers }} + {{ toYaml .Values.worker.extraContainers | nindent 6 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 6 }} + {{- end }} + securityContext: {{- toYaml .Values.worker.podSecurityContext | nindent 8 }} + volumes: + {{- if eq (include "airbyte.storage.type" .) "gcs" }} + - name: gcs-log-creds-volume + secret: + secretName: {{ include "airbyte.storage.secretName" . }} + items: + - key: {{ include "airbyte.storage.gcs.credentialsJson.secretKey" . }} + path: "gcp.json" + {{- end }} + {{- if .Values.worker.extraVolumes }} +{{- toYaml .Values.worker.extraVolumes | nindent 6 }} + {{- end }} + {{- if .Values.global.extraVolumes }} +{{- toYaml .Values.global.extraVolumes | nindent 6 }} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-worker/secrets.yaml b/helm/airbyte-v2/templates/airbyte-worker/secrets.yaml new file mode 100644 index 00000000..ffc03ea5 --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-worker/secrets.yaml @@ -0,0 +1,17 @@ +# Create secrets only for the local deployment +{{- if .Values.worker.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: worker-secrets + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" +type: Opaque +data: + {{- range $k, $v := mergeOverwrite .Values.worker.secrets .Values.global.secrets }} + {{ $k }}: {{ if $v }}{{ $v | b64enc }} {{else}}""{{end}} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-workload-api-server/deployment.yaml b/helm/airbyte-v2/templates/airbyte-workload-api-server/deployment.yaml new file mode 100644 index 00000000..ffc9969c --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-workload-api-server/deployment.yaml @@ -0,0 +1,200 @@ +{{- if .Values.workloadApiServer.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-workload-api-server + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + {{ toYaml .Values.workloadApiServer.annotations | nindent 4 }} +spec: + minReadySeconds: 30 + replicas: {{ .Values.workloadApiServer.replicaCount }} + selector: + matchLabels: + {{- include "airbyte.selectorLabels" . | nindent 6 }} + {{- if .Values.workloadApiServer.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.workloadApiServer.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 6 }} + {{- end }} + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 100% + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + {{- if .Values.workloadApiServer.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.workloadApiServer.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 8 }} + {{- end }} + {{- if .Values.workloadApiServer.podLabels }} + {{- include "airbyte.tplvalues.render" (dict "value" .Values.workloadApiServer.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.workloadApiServer.podAnnotations }} + annotations: + {{- include "airbyte.tplvalues.render" (dict "value" .Values.workloadApiServer.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ default .Values.workloadApiServer.serviceAccountName (include "airbyte.effectiveServiceAccountName" .) }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.workloadApiServer.nodeSelector }} + nodeSelector: {{- include "airbyte.tplvalues.render" (dict "value" .Values.workloadApiServer.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.workloadApiServer.tolerations }} + tolerations: {{- include "airbyte.tplvalues.render" (dict "value" .Values.workloadApiServer.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.workloadApiServer.affinity }} + affinity: {{- include "airbyte.tplvalues.render" (dict "value" .Values.workloadApiServer.affinity "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.workloadApiServer.extraInitContainers }} + initContainers: + {{- toYaml .Values.workloadApiServer.extraInitContainers | nindent 8 }} + {{- end }} + containers: + - name: airbyte-workload-api-server-container + image: {{ include "imageUrl" (list .Values.workloadApiServer.image $) }} + imagePullPolicy: "{{ .Values.workloadApiServer.image.pullPolicy }}" + env: + {{- if .Values.workloadApiServer.debug.enabled }} + - name: JAVA_TOOL_OPTIONS + value: "-Xdebug -agentlib:jdwp=transport=dt_socket,address=0.0.0.0:{{ .Values.workloadApiServer.debug.remoteDebugPort }},server=y,suspend=n" + {{- end }} + + {{- if (or (eq .Values.global.edition "community") (eq .Values.global.edition "enterprise")) }} + - name: AIRBYTE_API_AUTH_HEADER_VALUE + value: "Internal worker" + {{- end }} + + {{- include "airbyte.auth.jwt.envs" . | nindent 8 }} + {{- include "airbyte.common.envs" . | nindent 8 }} + {{- include "airbyte.database.envs" . | nindent 8 }} + {{- include "airbyte.keycloak.client.envs" . | nindent 8 }} + {{- include "airbyte.metrics.envs" . | nindent 8 }} + {{- include "airbyte.logging.envs" . | nindent 8 }} + {{- include "airbyte.micronaut.envs" . | nindent 8 }} + {{- include "airbyte.temporal.envs" . | nindent 8 }} + {{- include "airbyte.workloadApiServer.envs" . | nindent 8 }} + {{- /* SECRET_PERSISTENCE, S3_PATH_STYLE_ACCESS from configmap to avoid "Unsetting empty" */}} + {{- include "airbyte.secretsManager.envs" . | nindent 8 }} + {{- include "airbyte.storage.envs" . | nindent 8 }} + {{- /* DATAPLANE_CLIENT_ID/SECRET required for AirbyteInternalApiClientConfig (DataplaneIdentityService) */}} + {{- include "airbyte.workloadLauncher.dataPlane.envs" . | nindent 8 }} + + {{- if .Values.global.datadog.enabled }} + - name: DD_SERVICE + value: airbyte-workload-api + {{- include "airbyte.datadog.envs" . | nindent 8 }} + {{- include "airbyte.otel.envs" . | nindent 8 }} + {{- end}} + + {{- if (eq .Values.global.edition "enterprise") }} + {{- include "airbyte.enterprise.envs" . | nindent 8 }} + {{- end }} + + {{- if (eq .Values.global.edition "cloud") }} + {{- include "airbyte.temporal.cloud.envs" . | nindent 8 }} + {{- include "airbyte.temporal.cloud.credentials.envs" . | nindent 8 }} + {{- end }} + + + # Values from secret + {{- if .Values.workloadApiServer.secrets }} + {{- range $k, $v := .Values.workloadApiServer.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: workload-api-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if or .Values.workloadApiServer.env_vars .Values.global.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.workloadApiServer.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + # Values from extraEnv for more compability(if you want to use external secret source or other stuff) + {{- if .Values.workloadApiServer.extraEnv }} + {{- toYaml .Values.workloadApiServer.extraEnv | nindent 8 }} + {{- end }} + + {{- if .Values.workloadApiServer.livenessProbe.enabled }} + livenessProbe: + tcpSocket: + port: http + initialDelaySeconds: {{ .Values.workloadApiServer.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.workloadApiServer.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.workloadApiServer.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.workloadApiServer.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.workloadApiServer.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.workloadApiServer.readinessProbe.enabled }} + readinessProbe: + tcpSocket: + port: http + initialDelaySeconds: {{ .Values.workloadApiServer.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.workloadApiServer.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.workloadApiServer.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.workloadApiServer.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.workloadApiServer.readinessProbe.failureThreshold }} + {{- end }} + + ports: + - name: http + containerPort: 8007 + protocol: TCP + {{- if .Values.workloadApiServer.debug.enabled }} + - name: debug + containerPort: {{ .Values.workloadApiServer.debug.remoteDebugPort }} + protocol: TCP + {{- end }} + {{- if .Values.workloadApiServer.resources }} + resources: {{- toYaml .Values.workloadApiServer.resources | nindent 10 }} + {{- end }} + {{- if .Values.workloadApiServer.containerSecurityContext }} + securityContext: {{- toYaml .Values.workloadApiServer.containerSecurityContext | nindent 10 }} + {{- end }} + volumeMounts: + {{- if eq (include "airbyte.storage.type" .) "gcs" }} + - name: gcs-log-creds-volume + mountPath: {{ include "airbyte.storage.gcs.credentialsJsonPath" . }} + readOnly: true + {{- end }} + {{- if .Values.workloadApiServer.extraVolumeMounts }} + {{- toYaml .Values.workloadApiServer.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.global.extraVolumeMounts }} + {{- toYaml .Values.global.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.workloadApiServer.extraContainers }} + {{- toYaml .Values.workloadApiServer.extraContainers | nindent 6 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 6 }} + {{- end }} + securityContext: {{- toYaml .Values.workloadApiServer.podSecurityContext | nindent 8 }} + volumes: + {{- if eq (include "airbyte.storage.type" .) "gcs" }} + - name: gcs-log-creds-volume + secret: + secretName: {{ include "airbyte.storage.secretName" . }} + items: + - key: {{ include "airbyte.storage.gcs.credentialsJson.secretKey" . }} + path: "gcp.json" + {{- end }} + {{- if .Values.workloadApiServer.extraVolumes }} + {{- toYaml .Values.workloadApiServer.extraVolumes | nindent 6 }} + {{- end }} + {{- if .Values.global.extraVolumes }} + {{- toYaml .Values.global.extraVolumes | nindent 6 }} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-workload-api-server/secrets.yaml b/helm/airbyte-v2/templates/airbyte-workload-api-server/secrets.yaml new file mode 100644 index 00000000..9af0b77f --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-workload-api-server/secrets.yaml @@ -0,0 +1,17 @@ +# Create secrets only for the local deployment + {{- if .Values.workloadApiServer.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: workload-api-secrets + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" +type: Opaque +data: + {{- range $k, $v := mergeOverwrite .Values.workloadApiServer.secrets .Values.global.secrets }} + {{ $k }}: {{ if $v }}{{ $v | b64enc }} {{else}}""{{end}} + {{- end }} + {{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-workload-api-server/service.yaml b/helm/airbyte-v2/templates/airbyte-workload-api-server/service.yaml new file mode 100644 index 00000000..417ef7e7 --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-workload-api-server/service.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-workload-api-server-svc + {{- with .Values.workloadApiServer.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} +spec: + type: {{ .Values.workloadApiServer.service.type }} + ports: + - port: {{ .Values.workloadApiServer.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "airbyte.selectorLabels" . | nindent 4 }} + {{- if .Values.workloadApiServer.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.workloadApiServer.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 4 }} + {{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-workload-launcher/deployment.yaml b/helm/airbyte-v2/templates/airbyte-workload-launcher/deployment.yaml new file mode 100644 index 00000000..d89b1220 --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-workload-launcher/deployment.yaml @@ -0,0 +1,271 @@ +{{- if .Values.workloadLauncher.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-workload-launcher + labels: + {{- include "airbyte.labels" . | nindent 4 }} + {{- if .Values.workloadLauncher.extraLabels }} + {{ toYaml (mergeOverwrite .Values.workloadLauncher.extraLabels .Values.global.extraLabels) | nindent 4 }} + {{- end }} + annotations: + {{ toYaml .Values.workloadLauncher.annotations | nindent 4 }} +spec: + replicas: {{ .Values.workloadLauncher.replicaCount }} + selector: + matchLabels: + {{- include "airbyte.selectorLabels" . | nindent 6 }} + {{- if .Values.workloadLauncher.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.workloadLauncher.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 6 }} + {{- end }} + strategy: + type: Recreate + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + {{- if .Values.workloadLauncher.extraSelectorLabels }} + {{ toYaml (mergeOverwrite .Values.workloadLauncher.extraSelectorLabels .Values.global.extraSelectorLabels) | nindent 8 }} + {{- end }} + {{- if .Values.workloadLauncher.podLabels}} + {{- include "airbyte.tplvalues.render" (dict "value" .Values.workloadLauncher.podLabels "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.workloadLauncher.podAnnotations }} + annotations: + {{- include "airbyte.tplvalues.render" (dict "value" .Values.workloadLauncher.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ default .Values.workloadLauncher.serviceAccountName (include "airbyte.effectiveServiceAccountName" .) }} + automountServiceAccountToken: true + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.workloadLauncher.nodeSelector }} + nodeSelector: {{- include "airbyte.tplvalues.render" (dict "value" .Values.workloadLauncher.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.workloadLauncher.tolerations }} + tolerations: {{- include "airbyte.tplvalues.render" (dict "value" .Values.workloadLauncher.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.workloadLauncher.affinity }} + affinity: {{- include "airbyte.tplvalues.render" (dict "value" .Values.workloadLauncher.affinity "context" $) | nindent 8 }} + {{- end }} + {{- $useServerWait := and .Values.server.enabled (not (eq (include "airbyte.common.cluster.type" .) "data-plane")) }} + {{- if or $useServerWait .Values.workloadLauncher.extraInitContainers }} + initContainers: + {{- if $useServerWait }} + - name: wait-for-server + image: {{ quote (default "busybox:1.36" (index (index .Values.workloadLauncher "serverWaitInitContainer" | default dict) "image")) }} + command: + - sh + - -c + - | + echo "Waiting for airbyte-server at {{ .Release.Name }}-airbyte-server-svc:{{ .Values.server.service.port }}..." + until nc -z {{ .Release.Name }}-airbyte-server-svc {{ .Values.server.service.port }}; do + sleep 2 + done + echo "Server is ready." + {{- end }} + {{- if .Values.workloadLauncher.extraInitContainers }} + {{- toYaml .Values.workloadLauncher.extraInitContainers | nindent 6 }} + {{- end }} + {{- end }} + containers: + - name: airbyte-workload-launcher-container + image: {{ include "imageUrl" (list .Values.workloadLauncher.image $) }} + imagePullPolicy: "{{ .Values.workloadLauncher.image.pullPolicy }}" + env: + {{- if .Values.workloadLauncher.debug.enabled }} + - name: JAVA_TOOL_OPTIONS + value: "-Xdebug -agentlib:jdwp=transport=dt_socket,address=0.0.0.0:{{ .Values.workloadLauncher.debug.remoteDebugPort }},server=y,suspend=n" + {{- end}} + + {{- /* Match server's AIRBYTE_API_AUTH_HEADER_VALUE so control-plane token endpoint accepts launcher (avoids 500) */}} + {{- if (or (eq .Values.global.edition "community") (eq .Values.global.edition "enterprise")) }} + - name: AIRBYTE_API_AUTH_HEADER_VALUE + value: {{ include "airbyte.common.api.internalAuthHeaderValue" . | quote }} + {{- end }} + + {{- /* v1-style: DATA_PLANE_ID=local for same-cluster/hybrid launcher */}} + - name: DATA_PLANE_ID + value: {{ .Values.workloadLauncher.dataPlaneId | default "local" | quote }} + + {{- include "airbyte.cluster.envs" . | nindent 8 }} + {{- include "airbyte.common.envs" . | nindent 8 }} + {{- include "airbyte.database.envs" . | nindent 8 }} + {{- include "airbyte.database.migrations.envs" . | nindent 8 }} + {{- include "airbyte.jobs.envs" . | nindent 8 }} + {{- include "airbyte.jobs.errors.envs" . | nindent 8 }} + {{- include "airbyte.jobs.scheduling.envs" . | nindent 8 }} + {{- include "airbyte.keycloak.client.envs" . | nindent 8 }} + {{- include "airbyte.logging.envs" . | nindent 8 }} + {{- include "airbyte.metrics.envs" . | nindent 8 }} + {{- include "airbyte.micronaut.envs" . | nindent 8 }} + {{- include "airbyte.storage.envs" . | nindent 8 }} + {{- include "airbyte.secretsManager.envs" . | nindent 8 }} + {{- include "airbyte.temporal.envs" . | nindent 8 }} + {{- include "airbyte.temporal.worker.envs" . | nindent 8 }} + {{- include "airbyte.tracking.envs" . | nindent 8 }} + {{- include "airbyte.worker.envs" . | nindent 8 }} + {{- include "airbyte.workloadApiServer.envs" . | nindent 8 }} + {{- include "airbyte.workloads.envs" . | nindent 8 }} + {{- include "airbyte.workloads.queues.envs" . | nindent 8 }} + {{- include "airbyte.workloads.resources.envs" . | nindent 8 }} + {{- include "airbyte.workloadLauncher.envs" . | nindent 8 }} + {{- include "airbyte.workloadLauncher.dataPlane.envs" . | nindent 8 }} + {{- include "airbyte.workloadLauncher.images.envs" . | nindent 8 }} + + {{- /* v1-style: literal AIRBYTE_API_AUTH_HEADER_NAME (overrides configmap so launcher has same as v1) */}} + {{- if (or (eq .Values.global.edition "community") (eq .Values.global.edition "enterprise")) }} + - name: AIRBYTE_API_AUTH_HEADER_NAME + value: {{ include "airbyte.common.api.authHeaderName" . | quote }} + {{- end }} + + {{- /* v1-style: literal WORKLOAD_API_BEARER_TOKEN_SECRET_NAME and _KEY so launcher never sees null */}} + - name: WORKLOAD_API_BEARER_TOKEN_SECRET_NAME + value: {{ include "airbyte.workloadApiServer.secretName" . | quote }} + - name: WORKLOAD_API_BEARER_TOKEN_SECRET_KEY + value: {{ include "airbyte.workloadApiServer.bearerToken.secretKey" . | quote }} + + {{- /* v1-style: JOB_KUBE_SERVICEACCOUNT and WORKSPACE_DOCKER_MOUNT (v1: airbyte-admin, workspace) */}} + - name: JOB_KUBE_SERVICEACCOUNT + value: {{ include "airbyte.jobPodServiceAccountName" . | quote }} + - name: WORKSPACE_DOCKER_MOUNT + value: {{ .Values.workloadLauncher.workspaceDockerMount | default "workspace" | quote }} + + {{- if .Values.global.datadog.enabled }} + - name: DD_SERVICE + value: {{ include "airbyte.componentNameWithAirbytePrefix" . }} + {{- include "airbyte.datadog.envs" . | nindent 8 }} + {{- include "airbyte.otel.envs" . | nindent 8 }} + {{- end}} + + {{- if (eq .Values.global.edition "enterprise") }} + {{- include "airbyte.enterprise.envs" . | nindent 8 }} + {{- end }} + + {{- if (eq .Values.global.edition "cloud") }} + {{- include "airbyte.featureFlags.envs" . | nindent 8 }} + {{- include "airbyte.temporal.cloud.envs" . | nindent 8 }} + {{- include "airbyte.temporal.cloud.credentials.envs" . | nindent 8 }} + {{- end }} + + # Values from secret + {{- if .Values.workloadLauncher.secrets }} + {{- range $k, $v := .Values.workloadLauncher.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: workload-launcher-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if or .Values.workloadLauncher.env_vars .Values.global.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.workloadLauncher.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + # Values from extraEnv for more compability(if you want to use external secret source or other stuff) + {{- if .Values.workloadLauncher.extraEnv }} + {{- toYaml .Values.workloadLauncher.extraEnv | nindent 8 }} + {{- end }} + + {{- if .Values.workloadLauncher.livenessProbe.enabled }} + livenessProbe: + tcpSocket: + port: heartbeat + initialDelaySeconds: {{ .Values.workloadLauncher.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.workloadLauncher.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.workloadLauncher.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.workloadLauncher.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.workloadLauncher.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.workloadLauncher.readinessProbe.enabled }} + readinessProbe: + tcpSocket: + port: heartbeat + initialDelaySeconds: {{ .Values.workloadLauncher.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.workloadLauncher.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.workloadLauncher.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.workloadLauncher.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.workloadLauncher.readinessProbe.failureThreshold }} + {{- end }} + ports: + {{- if .Values.workloadLauncher.debug.enabled }} + - name: debug + containerPort: {{ .Values.workloadLauncher.debug.remoteDebugPort }} + protocol: TCP + {{- end}} + - name: heartbeat + containerPort: 8016 # for heartbeat server + {{- if .Values.workloadLauncher.resources }} + resources: {{- toYaml .Values.workloadLauncher.resources | nindent 10 }} + {{- end }} + {{- if .Values.workloadLauncher.containerSecurityContext }} + securityContext: {{- toYaml .Values.workloadLauncher.containerSecurityContext | nindent 10 }} + {{- end }} + volumeMounts: + {{- /* Workload API bearer token mounted at /secrets for launcher */}} + - name: workload-token + mountPath: /secrets + readOnly: true + {{- if eq (include "airbyte.storage.type" .) "gcs" }} + - name: gcs-log-creds-volume + mountPath: {{ include "airbyte.storage.gcs.credentialsJsonPath" . | dir }} + readOnly: true + {{- end }} + {{- if eq (include "airbyte.cluster.type" .) "data-plane" }} + - name: dataplane-creds-volume + mountPath: {{ include "airbyte.workloads.containerOrchestrator.dataPlane.secretMountPath" . | dir }} + readOnly: true + {{- end }} + {{- if .Values.workloadLauncher.extraVolumeMounts }} +{{- toYaml .Values.workloadLauncher.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.global.extraVolumeMounts }} +{{- toYaml .Values.global.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.workloadLauncher.extraContainers }} + {{ toYaml .Values.workloadLauncher.extraContainers | nindent 6 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | nindent 6 }} + {{- end }} + securityContext: {{- toYaml .Values.workloadLauncher.podSecurityContext | nindent 8 }} + volumes: + {{- /* Secret with WORKLOAD_API_BEARER_TOKEN at /secrets/workload_api_bearer_token */}} + - name: workload-token + secret: + secretName: {{ include "airbyte.workloadApiServer.secretName" . }} + items: + - key: {{ include "airbyte.workloadApiServer.bearerToken.secretKey" . }} + path: workload_api_bearer_token + {{- if eq (include "airbyte.storage.type" .) "gcs" }} + - name: gcs-log-creds-volume + secret: + secretName: {{ include "airbyte.storage.secretName" . }} + items: + - key: {{ include "airbyte.storage.gcs.credentialsJson.secretKey" . }} + path: "gcp.json" + {{- end }} + {{- if eq (include "airbyte.cluster.type" .) "data-plane" }} + - name: dataplane-creds-volume + secret: + secretName: {{ include "airbyte.workloads.containerOrchestrator.dataPlane.secretName" . }} + items: + - key: {{ include "airbyte.workloads.containerOrchestrator.dataPlane.secretKey" . }} + path: "sa.json" + {{- end }} + {{- if .Values.workloadLauncher.extraVolumes }} +{{- toYaml .Values.workloadLauncher.extraVolumes | nindent 6 }} + {{- end }} + {{- if .Values.global.extraVolumes }} +{{- toYaml .Values.global.extraVolumes | nindent 6 }} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-workload-launcher/jobs-secrets.yaml b/helm/airbyte-v2/templates/airbyte-workload-launcher/jobs-secrets.yaml new file mode 100644 index 00000000..d12f4cd3 --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-workload-launcher/jobs-secrets.yaml @@ -0,0 +1,19 @@ +# Create dataplane secrets only for the local deployment +{{- if .Values.global.local }} +apiVersion: v1 +kind: Secret +metadata: + name: workload-launcher-secrets + namespace: {{ include "airbyte.jobs.kube.namespace" . }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" +type: Opaque +stringData: + {{- include "airbyte.secretsManager.secrets" . | nindent 2 }} + {{- include "airbyte.storage.secrets" . | nindent 2 }} + {{- include "airbyte.workloadApiServer.secrets" . | nindent 2 }} + +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-workload-launcher/secrets.yaml b/helm/airbyte-v2/templates/airbyte-workload-launcher/secrets.yaml new file mode 100644 index 00000000..cae4e5b5 --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-workload-launcher/secrets.yaml @@ -0,0 +1,17 @@ +# Create secrets only for the local deployment +{{- if .Values.workloadLauncher.secrets }} +apiVersion: v1 +kind: Secret +metadata: + name: workload-launcher-secrets + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" +type: Opaque +data: + {{- range $k, $v := mergeOverwrite .Values.workloadLauncher.secrets .Values.global.secrets }} + {{ $k }}: {{ if $v }}{{ $v | b64enc }} {{else}}""{{end}} + {{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/airbyte-yml-secret.yaml b/helm/airbyte-v2/templates/airbyte-yml-secret.yaml new file mode 100644 index 00000000..eb6574e6 --- /dev/null +++ b/helm/airbyte-v2/templates/airbyte-yml-secret.yaml @@ -0,0 +1,19 @@ +{{- if .Values.global.airbyteYml }} +# The Secret that contains the airbyte.yml content. For this to work helm must be started with +# --set-file global.airbyteYml=/airbyte.yml +# For Airbyte Enterprise our installation script automatically specifies this, for Community +# users will need to specify it, if they want to use the airbyte.yml config file. +# If nothing is specified we'll assume an empty config +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-airbyte-yml + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" + labels: + {{- include "airbyte.labels" . | nindent 4 }} +type: Opaque +data: + fileContents: "{{ .Values.global.airbyteYml | default "" | nindent 4 | b64enc }}" +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_auth.tpl b/helm/airbyte-v2/templates/config/_auth.tpl new file mode 100644 index 00000000..a548e127 --- /dev/null +++ b/helm/airbyte-v2/templates/config/_auth.tpl @@ -0,0 +1,1041 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Auth Configuration +*/}} + +{{/* +Renders the auth.instanceAdmin.password secret key +*/}} +{{- define "airbyte.auth.instanceAdmin.password.secretKey" }} + {{- .Values.global.auth.instanceAdmin.passwordSecretKey | default "instance-admin-password" }} +{{- end }} + +{{/* +Renders the auth.instanceAdmin.password environment variable +*/}} +{{- define "airbyte.auth.instanceAdmin.password.env" }} +- name: AB_INSTANCE_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "airbyte.auth.bootstrap.managedSecretName" . }} + key: {{ include "airbyte.auth.instanceAdmin.password.secretKey" . }} +{{- end }} + +{{/* +Renders the global.auth.instanceAdmin.email value (community login username) +*/}} +{{- define "airbyte.auth.instanceAdmin.email" }} + {{- .Values.global.auth.instanceAdmin.email | default "airbyte@example.com" }} +{{- end }} + +{{/* +Renders the auth.instanceAdmin.email secret key +*/}} +{{- define "airbyte.auth.instanceAdmin.email.secretKey" }} + {{- .Values.global.auth.instanceAdmin.emailSecretKey | default "instance-admin-email" }} +{{- end }} + +{{/* +Renders the auth.instanceAdmin.username environment variable +*/}} +{{- define "airbyte.auth.instanceAdmin.username.env" }} +- name: AB_INSTANCE_ADMIN_USERNAME + valueFrom: + secretKeyRef: + name: {{ include "airbyte.auth.bootstrap.managedSecretName" . }} + key: {{ include "airbyte.auth.instanceAdmin.email.secretKey" . }} +{{- end }} + +{{/* +Renders the set of all auth environment variables +*/}} +{{- define "airbyte.auth.envs" }} +{{- include "airbyte.auth.instanceAdmin.password.env" . }} +{{- include "airbyte.auth.instanceAdmin.username.env" . }} +{{- end }} + +{{/* +Renders the auth.bootstrap secret name +*/}} +{{- define "airbyte.auth.bootstrap.secretName" }} +{{- if .Values.global.auth.secretName }} + {{- .Values.global.auth.secretName }} +{{- else }} + {{- .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} +{{- end }} +{{- end }} + +{{/* +Renders the global.auth.secretCreationEnabled value +*/}} +{{- define "airbyte.auth.bootstrap.secretCreationEnabled" }} + {{- if eq .Values.global.auth.secretCreationEnabled nil }} + {{- true }} + {{- else }} + {{- .Values.global.auth.secretCreationEnabled }} + {{- end }} +{{- end }} + +{{/* +Renders the auth.bootstrap.secretCreationEnabled environment variable +*/}} +{{- define "airbyte.auth.bootstrap.secretCreationEnabled.env" }} +- name: AB_AUTH_SECRET_CREATION_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AB_AUTH_SECRET_CREATION_ENABLED +{{- end }} + +{{/* +Renders the global.auth.managedSecretName value (v1: auth.secretName / AB_KUBERNETES_SECRET_NAME). +Defaults to global.secretName so bootloader writes to the same chart-created secret. +*/}} +{{- define "airbyte.auth.bootstrap.managedSecretName" }} + {{- .Values.global.auth.managedSecretName | default .Values.global.auth.secretName | default .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} +{{- end }} + +{{/* +Renders the auth.bootstrap.managedSecretName environment variable +*/}} +{{- define "airbyte.auth.bootstrap.managedSecretName.env" }} +- name: AB_KUBERNETES_SECRET_NAME + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AB_KUBERNETES_SECRET_NAME +{{- end }} + +{{/* +Release-unique default password (deterministic from release+namespace, stable across upgrades). +Used when global.auth.instanceAdmin.password is not set. +*/}} +{{- define "airbyte.auth.bootstrap.instanceAdmin.password.default" }} + {{- include "airbyte.secretSeed.hashBase" . | sha256sum | trunc 24 }} +{{- end }} + +{{/* +Renders the global.auth.instanceAdmin.password value. +When not set, uses a release-unique default (not "changeme"). Set in values for production. +*/}} +{{- define "airbyte.auth.bootstrap.instanceAdmin.password" }} + {{- .Values.global.auth.instanceAdmin.password | default (include "airbyte.auth.bootstrap.instanceAdmin.password.default" .) }} +{{- end }} + +{{/* +Renders the auth.bootstrap.instanceAdmin.password secret key +*/}} +{{- define "airbyte.auth.bootstrap.instanceAdmin.password.secretKey" }} + {{- .Values.global.auth.instanceAdmin.passwordSecretKey | default "AB_INSTANCE_ADMIN_PASSWORD" }} +{{- end }} + +{{/* +Renders the auth.bootstrap.instanceAdmin.password environment variable. +V1-style: when password is set in values, use literal so bootloader can write to secret; else read from secret. +*/}} +{{- define "airbyte.auth.bootstrap.instanceAdmin.password.env" }} +{{- if .Values.global.auth.instanceAdmin.password }} +- name: AB_INSTANCE_ADMIN_PASSWORD + value: {{ .Values.global.auth.instanceAdmin.password | quote }} +{{- else }} +- name: AB_INSTANCE_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "airbyte.auth.bootstrap.secretName" . }} + key: {{ include "airbyte.auth.bootstrap.instanceAdmin.password.secretKey" . }} +{{- end }} +{{- end }} + +{{/* +Renders the global.auth.instanceAdmin.passwordSecretKey value +*/}} +{{- define "airbyte.auth.bootstrap.instanceAdmin.passwordSecretKey" }} + {{- .Values.global.auth.instanceAdmin.passwordSecretKey | default "instance-admin-password" }} +{{- end }} + +{{/* +Renders the auth.bootstrap.instanceAdmin.passwordSecretKey environment variable +*/}} +{{- define "airbyte.auth.bootstrap.instanceAdmin.passwordSecretKey.env" }} +- name: AB_INSTANCE_ADMIN_PASSWORD_SECRET_KEY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AB_INSTANCE_ADMIN_PASSWORD_SECRET_KEY +{{- end }} + +{{/* +Release-unique default instance admin clientId (stable across upgrades). +*/}} +{{- define "airbyte.auth.bootstrap.instanceAdmin.clientId.default" }} + {{- printf "admin-%s" (printf "clientid-%s" (include "airbyte.secretSeed.hashBase" .) | sha256sum | trunc 16) }} +{{- end }} + +{{/* +Renders the global.auth.instanceAdmin.clientId value. +Uses non-empty default so secret has a value; set in values for production. +*/}} +{{- define "airbyte.auth.bootstrap.instanceAdmin.clientId" }} + {{- .Values.global.auth.instanceAdmin.clientId | default (include "airbyte.auth.bootstrap.instanceAdmin.clientId.default" .) }} +{{- end }} + +{{/* +Renders the auth.bootstrap.instanceAdmin.clientId secret key +*/}} +{{- define "airbyte.auth.bootstrap.instanceAdmin.clientId.secretKey" }} + {{- .Values.global.auth.instanceAdmin.clientIdSecretKey | default "AB_INSTANCE_ADMIN_CLIENT_ID" }} +{{- end }} + +{{/* +Renders the auth.bootstrap.instanceAdmin.clientId environment variable +*/}} +{{- define "airbyte.auth.bootstrap.instanceAdmin.clientId.env" }} +- name: AB_INSTANCE_ADMIN_CLIENT_ID + valueFrom: + secretKeyRef: + name: {{ include "airbyte.auth.bootstrap.secretName" . }} + key: {{ include "airbyte.auth.bootstrap.instanceAdmin.clientId.secretKey" . }} +{{- end }} + +{{/* +Renders the global.auth.instanceAdmin.clientIdSecretKey value +*/}} +{{- define "airbyte.auth.bootstrap.instanceAdmin.clientIdSecretKey" }} + {{- .Values.global.auth.instanceAdmin.clientIdSecretKey | default "instance-admin-client-id" }} +{{- end }} + +{{/* +Renders the auth.bootstrap.instanceAdmin.clientIdSecretKey environment variable +*/}} +{{- define "airbyte.auth.bootstrap.instanceAdmin.clientIdSecretKey.env" }} +- name: AB_INSTANCE_ADMIN_CLIENT_ID_SECRET_KEY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AB_INSTANCE_ADMIN_CLIENT_ID_SECRET_KEY +{{- end }} + +{{/* +Release-unique default instance admin clientSecret (stable across upgrades). +Uses secretSeed.hashBase so different clusters get different values when global.secretSeed is set. +*/}} +{{- define "airbyte.auth.bootstrap.instanceAdmin.clientSecret.default" }} + {{- printf "admin-%s" (printf "clientsecret-%s" (include "airbyte.secretSeed.hashBase" .) | sha256sum | trunc 32) }} +{{- end }} + +{{/* +Renders the global.auth.instanceAdmin.clientSecret value. +Uses non-empty default so secret has a value; set in values for production. +*/}} +{{- define "airbyte.auth.bootstrap.instanceAdmin.clientSecret" }} + {{- .Values.global.auth.instanceAdmin.clientSecret | default (include "airbyte.auth.bootstrap.instanceAdmin.clientSecret.default" .) }} +{{- end }} + +{{/* +Renders the auth.bootstrap.instanceAdmin.clientSecret secret key +*/}} +{{- define "airbyte.auth.bootstrap.instanceAdmin.clientSecret.secretKey" }} + {{- .Values.global.auth.instanceAdmin.clientSecretSecretKey | default "AB_INSTANCE_ADMIN_CLIENT_SECRET" }} +{{- end }} + +{{/* +Renders the auth.bootstrap.instanceAdmin.clientSecret environment variable +*/}} +{{- define "airbyte.auth.bootstrap.instanceAdmin.clientSecret.env" }} +- name: AB_INSTANCE_ADMIN_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ include "airbyte.auth.bootstrap.secretName" . }} + key: {{ include "airbyte.auth.bootstrap.instanceAdmin.clientSecret.secretKey" . }} +{{- end }} + +{{/* +Renders the global.auth.instanceAdmin.clientSecretSecretKey value +*/}} +{{- define "airbyte.auth.bootstrap.instanceAdmin.clientSecretSecretKey" }} + {{- .Values.global.auth.instanceAdmin.clientSecretSecretKey | default "instance-admin-client-secret" }} +{{- end }} + +{{/* +Renders the auth.bootstrap.instanceAdmin.clientSecretSecretKey environment variable +*/}} +{{- define "airbyte.auth.bootstrap.instanceAdmin.clientSecretSecretKey.env" }} +- name: AB_INSTANCE_ADMIN_CLIENT_SECRET_SECRET_KEY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AB_INSTANCE_ADMIN_CLIENT_SECRET_SECRET_KEY +{{- end }} + +{{/* +Release-unique default JWT secret (min 32 chars for HS256). Stable across upgrades. +*/}} +{{- define "airbyte.auth.bootstrap.security.jwtSignatureSecret.default" }} + {{- printf "airbyte-jwt-%s" (include "airbyte.secretSeed.hashBase" . | sha256sum) }} +{{- end }} + +{{/* +Renders the global.auth.security.jwtSignatureSecret value. +When not set, uses a release-unique default (32+ chars). Set in values for production. +*/}} +{{- define "airbyte.auth.bootstrap.security.jwtSignatureSecret" }} + {{- .Values.global.auth.security.jwtSignatureSecret | default (include "airbyte.auth.bootstrap.security.jwtSignatureSecret.default" .) }} +{{- end }} + +{{/* +Renders the auth.bootstrap.security.jwtSignatureSecret secret key +*/}} +{{- define "airbyte.auth.bootstrap.security.jwtSignatureSecret.secretKey" }} + {{- .Values.global.auth.security.jwtSignatureSecretSecretKey | default "AB_JWT_SIGNATURE_SECRET" }} +{{- end }} + +{{/* +Renders the auth.bootstrap.security.jwtSignatureSecret environment variable +*/}} +{{- define "airbyte.auth.bootstrap.security.jwtSignatureSecret.env" }} +- name: AB_JWT_SIGNATURE_SECRET + valueFrom: + secretKeyRef: + name: {{ include "airbyte.auth.bootstrap.secretName" . }} + key: {{ include "airbyte.auth.bootstrap.security.jwtSignatureSecret.secretKey" . }} +{{- end }} + +{{/* +Renders the global.auth.security.jwtSignatureSecretKey value +*/}} +{{- define "airbyte.auth.bootstrap.security.jwtSignatureSecretKey" }} + {{- .Values.global.auth.security.jwtSignatureSecretKey | default "jwt-signature-secret" }} +{{- end }} + +{{/* +Renders the auth.bootstrap.security.jwtSignatureSecretKey environment variable +*/}} +{{- define "airbyte.auth.bootstrap.security.jwtSignatureSecretKey.env" }} +- name: AB_JWT_SIGNATURE_SECRET_KEY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AB_JWT_SIGNATURE_SECRET_KEY +{{- end }} + +{{/* +Renders the global.auth.dataPlane.clientIdSecretKey value. +Default "dataplane-client-id" so server reads the key that matches service_accounts in DB (avoids 500 on /api/v1/dataplanes/token). +*/}} +{{- define "airbyte.auth.bootstrap.dataPlane.clientIdSecretKey" }} + {{- .Values.global.auth.dataPlane.clientIdSecretKey | default "dataplane-client-id" }} +{{- end }} + +{{/* +Renders the auth.bootstrap.dataPlane.clientIdSecretKey environment variable +*/}} +{{- define "airbyte.auth.bootstrap.dataPlane.clientIdSecretKey.env" }} +- name: DATAPLANE_CLIENT_ID_SECRET_KEY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATAPLANE_CLIENT_ID_SECRET_KEY +{{- end }} + +{{/* +Renders the global.auth.dataPlane.clientSecretSecretKey value. +Default "dataplane-client-secret" so server reads the key that matches service_accounts in DB (avoids 500 on /api/v1/dataplanes/token). +*/}} +{{- define "airbyte.auth.bootstrap.dataPlane.clientSecretSecretKey" }} + {{- .Values.global.auth.dataPlane.clientSecretSecretKey | default "dataplane-client-secret" }} +{{- end }} + +{{/* +Renders the auth.bootstrap.dataPlane.clientSecretSecretKey environment variable +*/}} +{{- define "airbyte.auth.bootstrap.dataPlane.clientSecretSecretKey.env" }} +- name: DATAPLANE_CLIENT_SECRET_SECRET_KEY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATAPLANE_CLIENT_SECRET_SECRET_KEY +{{- end }} + +{{/* +Renders the set of all auth.bootstrap environment variables +*/}} +{{- define "airbyte.auth.bootstrap.envs" }} +{{- include "airbyte.auth.bootstrap.secretCreationEnabled.env" . }} +{{- include "airbyte.auth.bootstrap.managedSecretName.env" . }} +{{- include "airbyte.auth.bootstrap.instanceAdmin.password.env" . }} +{{- include "airbyte.auth.bootstrap.instanceAdmin.passwordSecretKey.env" . }} +{{- include "airbyte.auth.bootstrap.instanceAdmin.clientId.env" . }} +{{- include "airbyte.auth.bootstrap.instanceAdmin.clientIdSecretKey.env" . }} +{{- include "airbyte.auth.bootstrap.instanceAdmin.clientSecret.env" . }} +{{- include "airbyte.auth.bootstrap.instanceAdmin.clientSecretSecretKey.env" . }} +{{- include "airbyte.auth.bootstrap.security.jwtSignatureSecret.env" . }} +{{- include "airbyte.auth.bootstrap.security.jwtSignatureSecretKey.env" . }} +{{- include "airbyte.auth.bootstrap.dataPlane.clientIdSecretKey.env" . }} +{{- include "airbyte.auth.bootstrap.dataPlane.clientSecretSecretKey.env" . }} +{{- end }} + +{{/* +V1-style: bootstrap env vars as literal values (same as airbyte-v1 bootloader). +Use when global.auth.enabled and edition community/enterprise so bootloader writes to chart-created secret. +*/}} +{{- define "airbyte.auth.bootstrap.envs.v1style" }} +- name: AB_AUTH_SECRET_CREATION_ENABLED + value: "true" +- name: AB_KUBERNETES_SECRET_NAME + value: {{ include "airbyte.auth.bootstrap.managedSecretName" . | quote }} +- name: AB_INSTANCE_ADMIN_PASSWORD_SECRET_KEY + value: {{ include "airbyte.auth.bootstrap.instanceAdmin.passwordSecretKey" . | quote }} +- name: AB_INSTANCE_ADMIN_CLIENT_ID_SECRET_KEY + value: {{ include "airbyte.auth.bootstrap.instanceAdmin.clientIdSecretKey" . | quote }} +- name: AB_INSTANCE_ADMIN_CLIENT_SECRET_SECRET_KEY + value: {{ include "airbyte.auth.bootstrap.instanceAdmin.clientSecretSecretKey" . | quote }} +- name: AB_JWT_SIGNATURE_SECRET_KEY + value: {{ include "airbyte.auth.bootstrap.security.jwtSignatureSecretKey" . | quote }} +- name: AB_INSTANCE_ADMIN_PASSWORD + value: {{ include "airbyte.auth.bootstrap.instanceAdmin.password" . | quote }} +- name: AB_INSTANCE_ADMIN_CLIENT_ID + value: {{ include "airbyte.auth.bootstrap.instanceAdmin.clientId" . | quote }} +- name: AB_INSTANCE_ADMIN_CLIENT_SECRET + value: {{ include "airbyte.auth.bootstrap.instanceAdmin.clientSecret" . | quote }} +- name: AB_JWT_SIGNATURE_SECRET + value: {{ include "airbyte.auth.bootstrap.security.jwtSignatureSecret" . | quote }} +- name: DATAPLANE_CLIENT_ID_SECRET_KEY + value: {{ include "airbyte.auth.bootstrap.dataPlane.clientIdSecretKey" . | quote }} +- name: DATAPLANE_CLIENT_SECRET_SECRET_KEY + value: {{ include "airbyte.auth.bootstrap.dataPlane.clientSecretSecretKey" . | quote }} +- name: DATAPLANE_CLIENT_ID + value: {{ include "airbyte.workloadLauncher.dataPlane.clientId" . | quote }} +- name: DATAPLANE_CLIENT_SECRET + value: {{ include "airbyte.workloadLauncher.dataPlane.clientSecret" . | quote }} +{{- end }} + +{{/* +Renders the set of all auth.bootstrap config map variables +*/}} +{{- define "airbyte.auth.bootstrap.configVars" }} +AB_AUTH_SECRET_CREATION_ENABLED: {{ include "airbyte.auth.bootstrap.secretCreationEnabled" . | quote }} +AB_KUBERNETES_SECRET_NAME: {{ include "airbyte.auth.bootstrap.managedSecretName" . | quote }} +AB_INSTANCE_ADMIN_PASSWORD_SECRET_KEY: {{ include "airbyte.auth.bootstrap.instanceAdmin.passwordSecretKey" . | quote }} +AB_INSTANCE_ADMIN_CLIENT_ID_SECRET_KEY: {{ include "airbyte.auth.bootstrap.instanceAdmin.clientIdSecretKey" . | quote }} +AB_INSTANCE_ADMIN_CLIENT_SECRET_SECRET_KEY: {{ include "airbyte.auth.bootstrap.instanceAdmin.clientSecretSecretKey" . | quote }} +AB_JWT_SIGNATURE_SECRET_KEY: {{ include "airbyte.auth.bootstrap.security.jwtSignatureSecretKey" . | quote }} +DATAPLANE_CLIENT_ID_SECRET_KEY: {{ include "airbyte.auth.bootstrap.dataPlane.clientIdSecretKey" . | quote }} +DATAPLANE_CLIENT_SECRET_SECRET_KEY: {{ include "airbyte.auth.bootstrap.dataPlane.clientSecretSecretKey" . | quote }} +{{- end }} + +{{/* +Renders the set of all auth.bootstrap secret variables +*/}} +{{- define "airbyte.auth.bootstrap.secrets" }} +AB_INSTANCE_ADMIN_PASSWORD: {{ include "airbyte.auth.bootstrap.instanceAdmin.password" . | quote }} +{{ include "airbyte.auth.instanceAdmin.password.secretKey" . }}: {{ include "airbyte.auth.bootstrap.instanceAdmin.password" . | quote }} +AB_INSTANCE_ADMIN_USERNAME: {{ include "airbyte.auth.instanceAdmin.email" . | quote }} +{{ include "airbyte.auth.instanceAdmin.email.secretKey" . }}: {{ include "airbyte.auth.instanceAdmin.email" . | quote }} +AB_INSTANCE_ADMIN_CLIENT_ID: {{ include "airbyte.auth.bootstrap.instanceAdmin.clientId" . | quote }} +{{ include "airbyte.auth.bootstrap.instanceAdmin.clientIdSecretKey" . }}: {{ include "airbyte.auth.bootstrap.instanceAdmin.clientId" . | quote }} +AB_INSTANCE_ADMIN_CLIENT_SECRET: {{ include "airbyte.auth.bootstrap.instanceAdmin.clientSecret" . | quote }} +{{ include "airbyte.auth.bootstrap.instanceAdmin.clientSecretSecretKey" . }}: {{ include "airbyte.auth.bootstrap.instanceAdmin.clientSecret" . | quote }} +AB_JWT_SIGNATURE_SECRET: {{ include "airbyte.auth.bootstrap.security.jwtSignatureSecret" . | quote }} +{{ include "airbyte.auth.bootstrap.security.jwtSignatureSecretKey" . }}: {{ include "airbyte.auth.bootstrap.security.jwtSignatureSecret" . | quote }} +{{ include "airbyte.auth.bootstrap.dataPlane.clientIdSecretKey" . }}: {{ include "airbyte.workloadLauncher.dataPlane.clientId" . | quote }} +{{ include "airbyte.auth.bootstrap.dataPlane.clientSecretSecretKey" . }}: {{ include "airbyte.workloadLauncher.dataPlane.clientSecret" . | quote }} +{{- end }} + +{{/* +Renders the auth.identityProvider secret name +*/}} +{{- define "airbyte.auth.identityProvider.secretName" }} +{{- if .Values.global.auth.identityProvider.secretName }} + {{- .Values.global.auth.identityProvider.secretName }} +{{- else }} + {{- .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} +{{- end }} +{{- end }} + +{{/* +Renders the global.auth.identityProvider.type value +*/}} +{{- define "airbyte.auth.identityProvider.type" }} + {{- .Values.global.auth.identityProvider.type | default "simple" }} +{{- end }} + +{{/* +Renders the auth.identityProvider.type environment variable +*/}} +{{- define "airbyte.auth.identityProvider.type.env" }} +- name: IDENTITY_PROVIDER_TYPE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: IDENTITY_PROVIDER_TYPE +{{- end }} + +{{/* +Renders the global.auth.identityProvider.verifyIssuer value +*/}} +{{- define "airbyte.auth.identityProvider.verifyIssuer" }} + {{- if eq .Values.global.auth.identityProvider.verifyIssuer nil }} + {{- false }} + {{- else }} + {{- .Values.global.auth.identityProvider.verifyIssuer }} + {{- end }} +{{- end }} + +{{/* +Renders the auth.identityProvider.verifyIssuer environment variable +*/}} +{{- define "airbyte.auth.identityProvider.verifyIssuer.env" }} +- name: AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_VERIFY_ISSUER + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_VERIFY_ISSUER +{{- end }} + +{{/* +Renders the global.auth.identityProvider.verifyAudience value +*/}} +{{- define "airbyte.auth.identityProvider.verifyAudience" }} + {{- if eq .Values.global.auth.identityProvider.verifyAudience nil }} + {{- false }} + {{- else }} + {{- .Values.global.auth.identityProvider.verifyAudience }} + {{- end }} +{{- end }} + +{{/* +Renders the auth.identityProvider.verifyAudience environment variable +*/}} +{{- define "airbyte.auth.identityProvider.verifyAudience.env" }} +- name: AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_VERIFY_AUDIENCE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_VERIFY_AUDIENCE +{{- end }} + +{{/* +Renders the global.auth.identityProvider.oidc.domain value +*/}} +{{- define "airbyte.auth.identityProvider.oidc.domain" }} + {{- .Values.global.auth.identityProvider.oidc.domain }} +{{- end }} + +{{/* +Renders the auth.identityProvider.oidc.domain environment variable +*/}} +{{- define "airbyte.auth.identityProvider.oidc.domain.env" }} +- name: OIDC_DOMAIN + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: OIDC_DOMAIN +{{- end }} + +{{/* +Renders the global.auth.identityProvider.oidc.appName value +*/}} +{{- define "airbyte.auth.identityProvider.oidc.appName" }} + {{- .Values.global.auth.identityProvider.oidc.appName }} +{{- end }} + +{{/* +Renders the auth.identityProvider.oidc.appName environment variable +*/}} +{{- define "airbyte.auth.identityProvider.oidc.appName.env" }} +- name: OIDC_APP_NAME + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: OIDC_APP_NAME +{{- end }} + +{{/* +Renders the global.auth.identityProvider.oidc.clientId value +*/}} +{{- define "airbyte.auth.identityProvider.oidc.clientId" }} + {{- .Values.global.auth.identityProvider.oidc.clientId }} +{{- end }} + +{{/* +Renders the auth.identityProvider.oidc.clientId secret key +*/}} +{{- define "airbyte.auth.identityProvider.oidc.clientId.secretKey" }} + {{- .Values.global.auth.identityProvider.oidc.clientIdSecretKey | default "OIDC_CLIENT_ID" }} +{{- end }} + +{{/* +Renders the auth.identityProvider.oidc.clientId environment variable +*/}} +{{- define "airbyte.auth.identityProvider.oidc.clientId.env" }} +- name: OIDC_CLIENT_ID + valueFrom: + secretKeyRef: + name: {{ include "airbyte.auth.identityProvider.secretName" . }} + key: {{ include "airbyte.auth.identityProvider.oidc.clientId.secretKey" . }} +{{- end }} + +{{/* +Renders the global.auth.identityProvider.oidc.clientSecret value +*/}} +{{- define "airbyte.auth.identityProvider.oidc.clientSecret" }} + {{- .Values.global.auth.identityProvider.oidc.clientSecret }} +{{- end }} + +{{/* +Renders the auth.identityProvider.oidc.clientSecret secret key +*/}} +{{- define "airbyte.auth.identityProvider.oidc.clientSecret.secretKey" }} + {{- .Values.global.auth.identityProvider.oidc.clientSecretSecretKey | default "OIDC_CLIENT_SECRET" }} +{{- end }} + +{{/* +Renders the auth.identityProvider.oidc.clientSecret environment variable +*/}} +{{- define "airbyte.auth.identityProvider.oidc.clientSecret.env" }} +- name: OIDC_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ include "airbyte.auth.identityProvider.secretName" . }} + key: {{ include "airbyte.auth.identityProvider.oidc.clientSecret.secretKey" . }} +{{- end }} + +{{/* +Renders the global.auth.identityProvider.genericOidc.clientId value +*/}} +{{- define "airbyte.auth.identityProvider.genericOidc.clientId" }} + {{- .Values.global.auth.identityProvider.genericOidc.clientId }} +{{- end }} + +{{/* +Renders the auth.identityProvider.genericOidc.clientId environment variable +*/}} +{{- define "airbyte.auth.identityProvider.genericOidc.clientId.env" }} +- name: AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_OIDC_CLIENT_ID + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_OIDC_CLIENT_ID +{{- end }} + +{{/* +Renders the global.auth.identityProvider.genericOidc.audience value +*/}} +{{- define "airbyte.auth.identityProvider.genericOidc.audience" }} + {{- .Values.global.auth.identityProvider.genericOidc.audience }} +{{- end }} + +{{/* +Renders the auth.identityProvider.genericOidc.audience environment variable +*/}} +{{- define "airbyte.auth.identityProvider.genericOidc.audience.env" }} +- name: AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_OIDC_AUDIENCE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_OIDC_AUDIENCE +{{- end }} + +{{/* +Renders the global.auth.identityProvider.genericOidc.issuer value +*/}} +{{- define "airbyte.auth.identityProvider.genericOidc.issuer" }} + {{- .Values.global.auth.identityProvider.genericOidc.issuer }} +{{- end }} + +{{/* +Renders the auth.identityProvider.genericOidc.issuer environment variable +*/}} +{{- define "airbyte.auth.identityProvider.genericOidc.issuer.env" }} +- name: DEFAULT_REALM + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DEFAULT_REALM +{{- end }} + +{{/* +Renders the global.auth.identityProvider.genericOidc.endpoints.authorizationServerEndpoint value +*/}} +{{- define "airbyte.auth.identityProvider.genericOidc.endpoints.authorizationServerEndpoint" }} + {{- .Values.global.auth.identityProvider.genericOidc.endpoints.authorizationServerEndpoint }} +{{- end }} + +{{/* +Renders the auth.identityProvider.genericOidc.endpoints.authorizationServerEndpoint environment variable +*/}} +{{- define "airbyte.auth.identityProvider.genericOidc.endpoints.authorizationServerEndpoint.env" }} +- name: AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_OIDC_ENDPOINTS_AUTHORIZATION_SERVER_ENDPOINT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_OIDC_ENDPOINTS_AUTHORIZATION_SERVER_ENDPOINT +{{- end }} + +{{/* +Renders the global.auth.identityProvider.genericOidc.endpoints.jwksEndpoint value +*/}} +{{- define "airbyte.auth.identityProvider.genericOidc.endpoints.jwksEndpoint" }} + {{- .Values.global.auth.identityProvider.genericOidc.endpoints.jwksEndpoint }} +{{- end }} + +{{/* +Renders the auth.identityProvider.genericOidc.endpoints.jwksEndpoint environment variable +*/}} +{{- define "airbyte.auth.identityProvider.genericOidc.endpoints.jwksEndpoint.env" }} +- name: AB_AIRBYTE_AUTH_JWKS_ENDPOINT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AB_AIRBYTE_AUTH_JWKS_ENDPOINT +{{- end }} + +{{/* +Renders the global.auth.identityProvider.genericOidc.fields.subject value +*/}} +{{- define "airbyte.auth.identityProvider.genericOidc.fields.subject" }} + {{- .Values.global.auth.identityProvider.genericOidc.fields.subject | default "sub" }} +{{- end }} + +{{/* +Renders the auth.identityProvider.genericOidc.fields.subject environment variable +*/}} +{{- define "airbyte.auth.identityProvider.genericOidc.fields.subject.env" }} +- name: AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_OIDC_FIELDS_SUB + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_OIDC_FIELDS_SUB +{{- end }} + +{{/* +Renders the global.auth.identityProvider.genericOidc.fields.email value +*/}} +{{- define "airbyte.auth.identityProvider.genericOidc.fields.email" }} + {{- .Values.global.auth.identityProvider.genericOidc.fields.email | default "email" }} +{{- end }} + +{{/* +Renders the auth.identityProvider.genericOidc.fields.email environment variable +*/}} +{{- define "airbyte.auth.identityProvider.genericOidc.fields.email.env" }} +- name: AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_OIDC_FIELDS_EMAIL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_OIDC_FIELDS_EMAIL +{{- end }} + +{{/* +Renders the global.auth.identityProvider.genericOidc.fields.name value +*/}} +{{- define "airbyte.auth.identityProvider.genericOidc.fields.name" }} + {{- .Values.global.auth.identityProvider.genericOidc.fields.name | default "name" }} +{{- end }} + +{{/* +Renders the auth.identityProvider.genericOidc.fields.name environment variable +*/}} +{{- define "airbyte.auth.identityProvider.genericOidc.fields.name.env" }} +- name: AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_OIDC_FIELDS_NAME + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_OIDC_FIELDS_NAME +{{- end }} + +{{/* +Renders the global.auth.identityProvider.genericOidc.fields.issuer value +*/}} +{{- define "airbyte.auth.identityProvider.genericOidc.fields.issuer" }} + {{- .Values.global.auth.identityProvider.genericOidc.fields.issuer | default "iss" }} +{{- end }} + +{{/* +Renders the auth.identityProvider.genericOidc.fields.issuer environment variable +*/}} +{{- define "airbyte.auth.identityProvider.genericOidc.fields.issuer.env" }} +- name: AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_OIDC_FIELDS_ISSUER + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_OIDC_FIELDS_ISSUER +{{- end }} + +{{/* +Renders the set of all auth.identityProvider environment variables +*/}} +{{- define "airbyte.auth.identityProvider.envs" }} +{{- include "airbyte.auth.identityProvider.type.env" . }} +{{- include "airbyte.auth.identityProvider.verifyIssuer.env" . }} +{{- include "airbyte.auth.identityProvider.verifyAudience.env" . }} +{{- $opt := (include "airbyte.auth.identityProvider.type" .) }} + +{{- if eq $opt "oidc" }} +{{- include "airbyte.auth.identityProvider.oidc.domain.env" . }} +{{- include "airbyte.auth.identityProvider.oidc.appName.env" . }} +{{- include "airbyte.auth.identityProvider.oidc.clientId.env" . }} +{{- include "airbyte.auth.identityProvider.oidc.clientSecret.env" . }} +{{- end }} + +{{- if eq $opt "generic-oidc" }} +{{- include "airbyte.auth.identityProvider.genericOidc.clientId.env" . }} +{{- include "airbyte.auth.identityProvider.genericOidc.audience.env" . }} +{{- include "airbyte.auth.identityProvider.genericOidc.issuer.env" . }} +{{- include "airbyte.auth.identityProvider.genericOidc.endpoints.authorizationServerEndpoint.env" . }} +{{- include "airbyte.auth.identityProvider.genericOidc.endpoints.jwksEndpoint.env" . }} +{{- include "airbyte.auth.identityProvider.genericOidc.fields.subject.env" . }} +{{- include "airbyte.auth.identityProvider.genericOidc.fields.email.env" . }} +{{- include "airbyte.auth.identityProvider.genericOidc.fields.name.env" . }} +{{- include "airbyte.auth.identityProvider.genericOidc.fields.issuer.env" . }} +{{- end }} + +{{- end }} + +{{/* +Renders the set of all auth.identityProvider config map variables +*/}} +{{- define "airbyte.auth.identityProvider.configVars" }} +IDENTITY_PROVIDER_TYPE: {{ include "airbyte.auth.identityProvider.type" . | quote }} +AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_VERIFY_ISSUER: {{ include "airbyte.auth.identityProvider.verifyIssuer" . | quote }} +AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_VERIFY_AUDIENCE: {{ include "airbyte.auth.identityProvider.verifyAudience" . | quote }} +{{- $opt := (include "airbyte.auth.identityProvider.type" .) }} + +{{- if eq $opt "oidc" }} +OIDC_DOMAIN: {{ include "airbyte.auth.identityProvider.oidc.domain" . | quote }} +OIDC_APP_NAME: {{ include "airbyte.auth.identityProvider.oidc.appName" . | quote }} +{{- end }} + +{{- if eq $opt "generic-oidc" }} +AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_OIDC_CLIENT_ID: {{ include "airbyte.auth.identityProvider.genericOidc.clientId" . | quote }} +AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_OIDC_AUDIENCE: {{ include "airbyte.auth.identityProvider.genericOidc.audience" . | quote }} +DEFAULT_REALM: {{ include "airbyte.auth.identityProvider.genericOidc.issuer" . | quote }} +AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_OIDC_ENDPOINTS_AUTHORIZATION_SERVER_ENDPOINT: {{ include "airbyte.auth.identityProvider.genericOidc.endpoints.authorizationServerEndpoint" . | quote }} +AB_AIRBYTE_AUTH_JWKS_ENDPOINT: {{ include "airbyte.auth.identityProvider.genericOidc.endpoints.jwksEndpoint" . | quote }} +AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_OIDC_FIELDS_SUB: {{ include "airbyte.auth.identityProvider.genericOidc.fields.subject" . | quote }} +AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_OIDC_FIELDS_EMAIL: {{ include "airbyte.auth.identityProvider.genericOidc.fields.email" . | quote }} +AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_OIDC_FIELDS_NAME: {{ include "airbyte.auth.identityProvider.genericOidc.fields.name" . | quote }} +AB_AIRBYTE_AUTH_IDENTITY_PROVIDER_OIDC_FIELDS_ISSUER: {{ include "airbyte.auth.identityProvider.genericOidc.fields.issuer" . | quote }} +{{- end }} + +{{- end }} + +{{/* +Renders the set of all auth.identityProvider secret variables +*/}} +{{- define "airbyte.auth.identityProvider.secrets" }} +{{- $opt := (include "airbyte.auth.identityProvider.type" .) }} + +{{- if eq $opt "oidc" }} +OIDC_CLIENT_ID: {{ include "airbyte.auth.identityProvider.oidc.clientId" . | quote }} +OIDC_CLIENT_SECRET: {{ include "airbyte.auth.identityProvider.oidc.clientSecret" . | quote }} +{{- end }} + +{{- if eq $opt "generic-oidc" }} +{{- end }} + +{{- end }} + +{{/* +Renders the auth.instanceAdmin.enterprise secret name +*/}} +{{- define "airbyte.auth.instanceAdmin.enterprise.secretName" }} +{{- if .Values.global.auth.instanceAdmin.secretName }} + {{- .Values.global.auth.instanceAdmin.secretName }} +{{- else }} + {{- .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} +{{- end }} +{{- end }} + +{{/* +Renders the global.auth.instanceAdmin.firstName value +*/}} +{{- define "airbyte.auth.instanceAdmin.enterprise.firstName" }} + {{- .Values.global.auth.instanceAdmin.firstName }} +{{- end }} + +{{/* +Renders the auth.instanceAdmin.enterprise.firstName environment variable +*/}} +{{- define "airbyte.auth.instanceAdmin.enterprise.firstName.env" }} +- name: INITIAL_USER_FIRST_NAME + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: INITIAL_USER_FIRST_NAME +{{- end }} + +{{/* +Renders the global.auth.instanceAdmin.lastName value +*/}} +{{- define "airbyte.auth.instanceAdmin.enterprise.lastName" }} + {{- .Values.global.auth.instanceAdmin.lastName }} +{{- end }} + +{{/* +Renders the auth.instanceAdmin.enterprise.lastName environment variable +*/}} +{{- define "airbyte.auth.instanceAdmin.enterprise.lastName.env" }} +- name: INITIAL_USER_LAST_NAME + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: INITIAL_USER_LAST_NAME +{{- end }} + +{{/* +Renders the global.auth.instanceAdmin.password value +*/}} +{{- define "airbyte.auth.instanceAdmin.enterprise.password" }} + {{- .Values.global.auth.instanceAdmin.password | default (ternary (printf "initial-%s" (include "airbyte.secretSeed.hashBase" . | sha256sum | nospace | trunc 24)) "" (ne (toString .Values.global.secretSeed) "")) }} +{{- end }} + +{{/* +Renders the auth.instanceAdmin.enterprise.password secret key +*/}} +{{- define "airbyte.auth.instanceAdmin.enterprise.password.secretKey" }} + {{- .Values.global.auth.instanceAdmin.passwordSecretKey | default "INITIAL_USER_PASSWORD" }} +{{- end }} + +{{/* +Renders the auth.instanceAdmin.enterprise.password environment variable +*/}} +{{- define "airbyte.auth.instanceAdmin.enterprise.password.env" }} +- name: INITIAL_USER_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "airbyte.auth.instanceAdmin.enterprise.secretName" . }} + key: {{ include "airbyte.auth.instanceAdmin.enterprise.password.secretKey" . }} +{{- end }} + +{{/* +Renders the set of all auth.instanceAdmin.enterprise environment variables +*/}} +{{- define "airbyte.auth.instanceAdmin.enterprise.email.env" }} +- name: INITIAL_USER_EMAIL + valueFrom: + secretKeyRef: + name: {{ include "airbyte.auth.instanceAdmin.enterprise.secretName" . }} + key: {{ include "airbyte.auth.instanceAdmin.email.secretKey" . }} +{{- end }} + +{{/* +Renders the set of all auth.instanceAdmin.enterprise environment variables +*/}} +{{- define "airbyte.auth.instanceAdmin.enterprise.envs" }} +{{- include "airbyte.auth.instanceAdmin.enterprise.firstName.env" . }} +{{- include "airbyte.auth.instanceAdmin.enterprise.lastName.env" . }} +{{- include "airbyte.auth.instanceAdmin.enterprise.email.env" . }} +{{- include "airbyte.auth.instanceAdmin.enterprise.password.env" . }} +{{- end }} + +{{/* +Renders the set of all auth.instanceAdmin.enterprise config map variables +*/}} +{{- define "airbyte.auth.instanceAdmin.enterprise.configVars" }} +INITIAL_USER_FIRST_NAME: {{ include "airbyte.auth.instanceAdmin.enterprise.firstName" . | quote }} +INITIAL_USER_LAST_NAME: {{ include "airbyte.auth.instanceAdmin.enterprise.lastName" . | quote }} +{{- end }} + +{{/* +Renders the set of all auth.instanceAdmin.enterprise secret variables +*/}} +{{- define "airbyte.auth.instanceAdmin.enterprise.secrets" }} +INITIAL_USER_PASSWORD: {{ include "airbyte.auth.instanceAdmin.enterprise.password" . | quote }} +INITIAL_USER_EMAIL: {{ include "airbyte.auth.instanceAdmin.email" . | quote }} +{{ include "airbyte.auth.instanceAdmin.email.secretKey" . }}: {{ include "airbyte.auth.instanceAdmin.email" . | quote }} +{{- end }} + +{{/* +Renders the auth.jwt secret name +*/}} +{{- define "airbyte.auth.jwt.secretName" }} +{{- if .Values.global.auth.instanceAdmin.secretName }} + {{- .Values.global.auth.instanceAdmin.secretName }} +{{- else }} + {{- .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} +{{- end }} +{{- end }} + +{{/* +Renders the auth.jwt.jwtSignatureSecret secret key +*/}} +{{- define "airbyte.auth.jwt.jwtSignatureSecret.secretKey" }} + {{- .Values.global.auth.security.jwtSignatureSecretSecretKey | default "jwt-signature-secret" }} +{{- end }} + +{{/* +Renders the auth.jwt.jwtSignatureSecret environment variable +*/}} +{{- define "airbyte.auth.jwt.jwtSignatureSecret.env" }} +- name: AB_JWT_SIGNATURE_SECRET + valueFrom: + secretKeyRef: + name: {{ include "airbyte.auth.bootstrap.managedSecretName" . }} + key: {{ include "airbyte.auth.jwt.jwtSignatureSecret.secretKey" . }} +{{- end }} + +{{/* +Renders the set of all auth.jwt environment variables +*/}} +{{- define "airbyte.auth.jwt.envs" }} +{{- include "airbyte.auth.jwt.jwtSignatureSecret.env" . }} +{{- end }} + +{{/* +Renders the global.auth.security.cookieSecureSetting value +*/}} +{{- define "airbyte.auth.security.cookieSecureSetting" }} + {{- if eq .Values.global.auth.security.cookieSecureSetting nil }} + {{- true }} + {{- else }} + {{- .Values.global.auth.security.cookieSecureSetting }} + {{- end }} +{{- end }} + +{{/* +Renders the auth.security.cookieSecureSetting environment variable +*/}} +{{- define "airbyte.auth.security.cookieSecureSetting.env" }} +- name: AB_COOKIE_SECURE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AB_COOKIE_SECURE +{{- end }} + +{{/* +Renders the global.auth.security.cookieSameSiteSetting value +*/}} +{{- define "airbyte.auth.security.cookieSameSiteSetting" }} + {{- .Values.global.auth.security.cookieSameSiteSetting | default "strict" }} +{{- end }} + +{{/* +Renders the auth.security.cookieSameSiteSetting environment variable +*/}} +{{- define "airbyte.auth.security.cookieSameSiteSetting.env" }} +- name: AB_COOKIE_SAME_SITE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AB_COOKIE_SAME_SITE +{{- end }} + +{{/* +Renders the set of all auth.security environment variables +*/}} +{{- define "airbyte.auth.security.envs" }} +{{- include "airbyte.auth.security.cookieSecureSetting.env" . }} +{{- include "airbyte.auth.security.cookieSameSiteSetting.env" . }} +{{- end }} + +{{/* +Renders the set of all auth.security config map variables +*/}} +{{- define "airbyte.auth.security.configVars" }} +AB_COOKIE_SECURE: {{ include "airbyte.auth.security.cookieSecureSetting" . | quote }} +AB_COOKIE_SAME_SITE: {{ include "airbyte.auth.security.cookieSameSiteSetting" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_cluster.tpl b/helm/airbyte-v2/templates/config/_cluster.tpl new file mode 100644 index 00000000..d8585e7b --- /dev/null +++ b/helm/airbyte-v2/templates/config/_cluster.tpl @@ -0,0 +1,88 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Cluster Configuration +*/}} + +{{/* +Renders the global.cluster.type value +*/}} +{{- define "airbyte.cluster.type" }} + {{- .Values.global.cluster.type | default "hybrid" }} +{{- end }} + +{{/* +Renders the cluster.type environment variable +*/}} +{{- define "airbyte.cluster.type.env" }} +- name: CLUSTER_TYPE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CLUSTER_TYPE +{{- end }} + +{{/* +Renders the global.cluster.dataPlane.controlPlaneAuthEndpoint value. +Defaults to internal server URL (http://RELEASE-airbyte-server-svc.NAMESPACE:8001) when empty so launcher can reach control-plane auth. +*/}} +{{- define "airbyte.cluster.dataPlane.controlPlaneAuthEndpoint" }} + {{- $explicit := .Values.global.cluster.dataPlane.controlPlaneAuthEndpoint | default .Values.global.airbyteUrl }} + {{- if $explicit }} + {{- $explicit }} + {{- else }} + {{- printf "http://%s" (include "airbyte.common.server.host" .) }} + {{- end }} +{{- end }} + +{{/* +Renders the cluster.dataPlane.controlPlaneAuthEndpoint environment variable +*/}} +{{- define "airbyte.cluster.dataPlane.controlPlaneAuthEndpoint.env" }} +- name: CONTROL_PLANE_AUTH_ENDPOINT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONTROL_PLANE_AUTH_ENDPOINT +{{- end }} + +{{/* +Renders the set of all cluster environment variables +*/}} +{{- define "airbyte.cluster.envs" }} +{{- include "airbyte.cluster.type.env" . }} +{{- $opt := (include "airbyte.cluster.type" .) }} + +{{- if eq $opt "control-plane" }} +{{- end }} + +{{- if eq $opt "data-plane" }} +{{- include "airbyte.cluster.dataPlane.controlPlaneAuthEndpoint.env" . }} +{{- end }} + +{{- if eq $opt "hybrid" }} +{{- include "airbyte.cluster.dataPlane.controlPlaneAuthEndpoint.env" . }} +{{- end }} + +{{- end }} + +{{/* +Renders the set of all cluster config map variables +*/}} +{{- define "airbyte.cluster.configVars" }} +CLUSTER_TYPE: {{ include "airbyte.cluster.type" . | quote }} +{{- $opt := (include "airbyte.cluster.type" .) }} + +{{- if eq $opt "control-plane" }} +{{- end }} + +{{- if eq $opt "data-plane" }} +CONTROL_PLANE_AUTH_ENDPOINT: {{ include "airbyte.cluster.dataPlane.controlPlaneAuthEndpoint" . | quote }} +{{- end }} + +{{- if eq $opt "hybrid" }} +CONTROL_PLANE_AUTH_ENDPOINT: {{ include "airbyte.cluster.dataPlane.controlPlaneAuthEndpoint" . | quote }} +{{- end }} + +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_common.tpl b/helm/airbyte-v2/templates/config/_common.tpl new file mode 100644 index 00000000..15f55eb6 --- /dev/null +++ b/helm/airbyte-v2/templates/config/_common.tpl @@ -0,0 +1,334 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Common Configuration +*/}} + +{{/* +Renders the global.edition value +*/}} +{{- define "airbyte.common.edition" }} + {{- .Values.global.edition | default "community" }} +{{- end }} + +{{/* +Renders the common.edition environment variable +*/}} +{{- define "airbyte.common.edition.env" }} +- name: AIRBYTE_EDITION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_EDITION +{{- end }} + +{{/* +Renders the global.version value (non-empty default to avoid "Unsetting empty" VERSION in workload-api-server). +*/}} +{{- define "airbyte.common.version" }} + {{- .Values.global.version | default .Chart.AppVersion | default "2.0.0" }} +{{- end }} + +{{/* +Renders the common.version environment variable +*/}} +{{- define "airbyte.common.version.env" }} +- name: AIRBYTE_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_VERSION +{{- end }} + +{{/* +Renders the VERSION environment variable (same value as AIRBYTE_VERSION; avoids "Unsetting empty" in workload-api-server). +*/}} +{{- define "airbyte.common.version.env.asVERSION" }} +- name: VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: VERSION +{{- end }} + +{{/* +Renders the global.cluster.type value +*/}} +{{- define "airbyte.common.cluster.type" }} + {{- .Values.global.cluster.type }} +{{- end }} + +{{/* +Renders the common.cluster.type environment variable +*/}} +{{- define "airbyte.common.cluster.type.env" }} +- name: AIRBYTE_CLUSTER_TYPE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_CLUSTER_TYPE +{{- end }} + +{{/* +Renders the global.cluster.name value ( non-empty default to avoid "Unsetting empty" in workload-api-server). +*/}} +{{- define "airbyte.common.cluster.name" }} + {{- .Values.global.cluster.name | default "default" }} +{{- end }} + +{{/* +Renders the common.cluster.name environment variable +*/}} +{{- define "airbyte.common.cluster.name.env" }} +- name: AIRBYTE_CLUSTER_NAME + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_CLUSTER_NAME +{{- end }} + +{{/* +Renders the global.airbyteUrl value +*/}} +{{- define "airbyte.common.airbyteUrl" }} + {{- .Values.global.airbyteUrl }} +{{- end }} + +{{/* +Renders the common.airbyteUrl environment variable +*/}} +{{- define "airbyte.common.airbyteUrl.env" }} +- name: AIRBYTE_URL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_URL +{{- end }} + +{{/* +Renders the global.api.host value (AIRBYTE_API_HOST). When global.api.host or global.airbyteUrl is set, use it; otherwise default to internal server URL so in-cluster components (e.g. workload-launcher) can reach the API. +*/}} +{{- define "airbyte.common.api.host" }} + {{- if .Values.global.api.host }} + {{- .Values.global.api.host }} + {{- else if .Values.global.airbyteUrl }} + {{- printf "%s/api/public" (trimSuffix "/" .Values.global.airbyteUrl) }} + {{- else }} + {{- printf "http://%s" (include "airbyte.common.server.host" .) }} + {{- end }} +{{- end }} + +{{/* +Renders the common.api.host environment variable +*/}} +{{- define "airbyte.common.api.host.env" }} +- name: AIRBYTE_API_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_API_HOST +{{- end }} + +{{/* +Renders the global.api.authHeaderName value +*/}} +{{- define "airbyte.common.api.authHeaderName" }} + {{- .Values.global.api.authHeaderName | default "X-Airbyte-Auth" }} +{{- end }} + +{{/* +Renders the internal API auth header value (AIRBYTE_API_AUTH_HEADER_VALUE). +Server and workload-launcher must use the same value so the token endpoint accepts launcher requests. +*/}} +{{- define "airbyte.common.api.internalAuthHeaderValue" }} + {{- .Values.global.api.internalAuthHeaderValue | default "Internal server" }} +{{- end }} + +{{/* +Renders the common.api.authHeaderName environment variable +*/}} +{{- define "airbyte.common.api.authHeaderName.env" }} +- name: AIRBYTE_API_AUTH_HEADER_NAME + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_API_AUTH_HEADER_NAME +{{- end }} + +{{/* +Renders the global.server.host value +*/}} +{{- define "airbyte.common.server.host" }} + {{- (printf "%s-airbyte-server-svc.%s:%d" .Release.Name .Release.Namespace (int .Values.server.service.port)) }} +{{- end }} + +{{/* +Renders the common.server.host environment variable +*/}} +{{- define "airbyte.common.server.host.env" }} +- name: AIRBYTE_SERVER_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_SERVER_HOST +{{- end }} + +{{/* +Renders the global.auth.enabled value +*/}} +{{- define "airbyte.common.auth.enabled" }} + {{- if eq .Values.global.auth.enabled nil }} + {{- true }} + {{- else }} + {{- .Values.global.auth.enabled }} + {{- end }} +{{- end }} + +{{/* +Renders the common.auth.enabled environment variable +*/}} +{{- define "airbyte.common.auth.enabled.env" }} +- name: API_AUTHORIZATION_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: API_AUTHORIZATION_ENABLED +{{- end }} + +{{/* +Renders the global.connectorBuilderServer.apiHost value +*/}} +{{- define "airbyte.common.connectorBuilderServer.apiHost" }} + {{- (printf "http://%s-airbyte-connector-builder-server-svc.%s:%d" .Release.Name .Release.Namespace (int .Values.connectorBuilderServer.service.port)) }} +{{- end }} + +{{/* +Renders the common.connectorBuilderServer.apiHost environment variable +*/}} +{{- define "airbyte.common.connectorBuilderServer.apiHost.env" }} +- name: CONNECTOR_BUILDER_SERVER_API_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONNECTOR_BUILDER_SERVER_API_HOST +{{- end }} + +{{/* +Renders the global.deploymentEnv value deploymentMode "oss"; non-empty default to avoid "Unsetting empty"). +*/}} +{{- define "airbyte.common.deploymentEnv" }} + {{- .Values.global.deploymentEnv | default "oss" }} +{{- end }} + +{{/* +Renders the common.deploymentEnv environment variable +*/}} +{{- define "airbyte.common.deploymentEnv.env" }} +- name: DEPLOYMENT_ENV + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DEPLOYMENT_ENV +{{- end }} + +{{/* +Renders the global.api.internalHost value +*/}} +{{- define "airbyte.common.api.internalHost" }} + {{- ternary (include "airbyte.common.airbyteUrl" .) (printf "http://%s-airbyte-server-svc.%s:%d" .Release.Name .Release.Namespace (int .Values.server.service.port)) (eq (include "airbyte.common.cluster.type" .) "data-plane") }} +{{- end }} + +{{/* +Renders the common.api.internalHost environment variable +*/}} +{{- define "airbyte.common.api.internalHost.env" }} +- name: INTERNAL_API_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: INTERNAL_API_HOST +{{- end }} + +{{/* +Renders the global.local value +*/}} +{{- define "airbyte.common.local" }} + {{- if eq .Values.global.local nil }} + {{- false }} + {{- else }} + {{- .Values.global.local }} + {{- end }} +{{- end }} + +{{/* +Renders the common.local environment variable +*/}} +{{- define "airbyte.common.local.env" }} +- name: LOCAL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: LOCAL +{{- end }} + +{{/* +Renders the global.webapp.url value +*/}} +{{- define "airbyte.common.webapp.url" }} + {{- (include "airbyte.common.airbyteUrl" .) }} +{{- end }} + +{{/* +Renders the common.webapp.url environment variable +*/}} +{{- define "airbyte.common.webapp.url.env" }} +- name: WEBAPP_URL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WEBAPP_URL +{{- end }} + +{{/* +Renders the set of all common environment variables +*/}} +{{- define "airbyte.common.envs" }} +{{- include "airbyte.common.edition.env" . }} +{{- include "airbyte.common.version.env" . }} +{{- include "airbyte.common.version.env.asVERSION" . }} +{{- include "airbyte.common.cluster.type.env" . }} +{{- include "airbyte.common.cluster.name.env" . }} +{{- include "airbyte.common.airbyteUrl.env" . }} +{{- include "airbyte.common.api.host.env" . }} +{{- include "airbyte.common.api.authHeaderName.env" . }} +{{- include "airbyte.common.server.host.env" . }} +{{- include "airbyte.common.auth.enabled.env" . }} +{{- include "airbyte.common.connectorBuilderServer.apiHost.env" . }} +{{- include "airbyte.common.deploymentEnv.env" . }} +{{- include "airbyte.common.api.internalHost.env" . }} +{{- include "airbyte.common.local.env" . }} +{{- include "airbyte.common.webapp.url.env" . }} +{{- end }} + +{{/* +Renders the set of all common config map variables +*/}} +{{- define "airbyte.common.configVars" }} +AIRBYTE_EDITION: {{ include "airbyte.common.edition" . | quote }} +AIRBYTE_VERSION: {{ include "airbyte.common.version" . | quote }} +VERSION: {{ include "airbyte.common.version" . | quote }} +AIRBYTE_CLUSTER_TYPE: {{ include "airbyte.common.cluster.type" . | quote }} +AIRBYTE_CLUSTER_NAME: {{ include "airbyte.common.cluster.name" . | quote }} +AIRBYTE_URL: {{ include "airbyte.common.airbyteUrl" . | quote }} +AIRBYTE_API_HOST: {{ include "airbyte.common.api.host" . | quote }} +AIRBYTE_API_AUTH_HEADER_NAME: {{ include "airbyte.common.api.authHeaderName" . | quote }} +AIRBYTE_API_AUTH_HEADER_VALUE: {{ include "airbyte.common.api.internalAuthHeaderValue" . | quote }} +AIRBYTE_SERVER_HOST: {{ include "airbyte.common.server.host" . | quote }} +API_AUTHORIZATION_ENABLED: {{ include "airbyte.common.auth.enabled" . | quote }} +CONNECTOR_BUILDER_SERVER_API_HOST: {{ include "airbyte.common.connectorBuilderServer.apiHost" . | quote }} +DEPLOYMENT_ENV: {{ include "airbyte.common.deploymentEnv" . | quote }} +INTERNAL_API_HOST: {{ include "airbyte.common.api.internalHost" . | quote }} +LOCAL: {{ include "airbyte.common.local" . | quote }} +WEBAPP_URL: {{ include "airbyte.common.webapp.url" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_connector.tpl b/helm/airbyte-v2/templates/config/_connector.tpl new file mode 100644 index 00000000..2113b37f --- /dev/null +++ b/helm/airbyte-v2/templates/config/_connector.tpl @@ -0,0 +1,58 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Connector Configuration +*/}} + +{{/* +Renders the global.connectorRegistry.seedProvider value +*/}} +{{- define "airbyte.connector.seedProvider" }} + {{- .Values.global.connectorRegistry.seedProvider }} +{{- end }} + +{{/* +Renders the connector.seedProvider environment variable +*/}} +{{- define "airbyte.connector.seedProvider.env" }} +- name: CONNECTOR_REGISTRY_SEED_PROVIDER + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONNECTOR_REGISTRY_SEED_PROVIDER +{{- end }} + +{{/* +Renders the global.connectorRegistry.enterpriseSourceStubsUrl value +*/}} +{{- define "airbyte.connector.enterpriseSourceStubsUrl" }} + {{- .Values.global.connectorRegistry.enterpriseSourceStubsUrl | default "https://connectors.airbyte.com/files/resources/connector_stubs/v1/connector_stubs.json" }} +{{- end }} + +{{/* +Renders the connector.enterpriseSourceStubsUrl environment variable +*/}} +{{- define "airbyte.connector.enterpriseSourceStubsUrl.env" }} +- name: ENTERPRISE_SOURCE_STUBS_URL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: ENTERPRISE_SOURCE_STUBS_URL +{{- end }} + +{{/* +Renders the set of all connector environment variables +*/}} +{{- define "airbyte.connector.envs" }} +{{- include "airbyte.connector.seedProvider.env" . }} +{{- include "airbyte.connector.enterpriseSourceStubsUrl.env" . }} +{{- end }} + +{{/* +Renders the set of all connector config map variables +*/}} +{{- define "airbyte.connector.configVars" }} +CONNECTOR_REGISTRY_SEED_PROVIDER: {{ include "airbyte.connector.seedProvider" . | quote }} +ENTERPRISE_SOURCE_STUBS_URL: {{ include "airbyte.connector.enterpriseSourceStubsUrl" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_connectorBuilder.tpl b/helm/airbyte-v2/templates/config/_connectorBuilder.tpl new file mode 100644 index 00000000..71ad604f --- /dev/null +++ b/helm/airbyte-v2/templates/config/_connectorBuilder.tpl @@ -0,0 +1,57 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Connectorbuilder Configuration +*/}} + +{{/* +Renders the connectorBuilderServer.aiAssistUrlBase value +*/}} +{{- define "airbyte.connectorBuilder.aiAssistUrlBase" }} + {{- .Values.connectorBuilderServer.aiAssistUrlBase }} +{{- end }} + +{{/* +Renders the connectorBuilder.aiAssistUrlBase environment variable +*/}} +{{- define "airbyte.connectorBuilder.aiAssistUrlBase.env" }} +- name: AI_ASSIST_URL_BASE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AI_ASSIST_URL_BASE +{{- end }} + +{{/* +Renders the connectorBuilder.manifestServerApi.basePath value (URL of connector-builder-server for ManifestServerApiClient). +Required by server: airbyte.manifest-server-api.base-path must match pattern .+ +*/}} +{{- define "airbyte.connectorBuilder.manifestServerApi.basePath" }} + {{- .Values.connectorBuilderServer.manifestServerApi.basePath | default (include "airbyte.common.connectorBuilderServer.apiHost" .) }} +{{- end }} + +{{/* +Renders the connectorBuilder.manifestServerApi.basePath environment variable. +use literal value (same as CONNECTOR_BUILDER_SERVER_API_HOST) so server always has non-empty base-path. +*/}} +{{- define "airbyte.connectorBuilder.manifestServerApi.basePath.env" }} +- name: AIRBYTE_MANIFEST_SERVER_API_BASE_PATH + value: {{ include "airbyte.connectorBuilder.manifestServerApi.basePath" . | quote }} +{{- end }} + +{{/* +Renders the set of all connectorBuilder environment variables +*/}} +{{- define "airbyte.connectorBuilder.envs" }} +{{- include "airbyte.connectorBuilder.aiAssistUrlBase.env" . }} +{{- include "airbyte.connectorBuilder.manifestServerApi.basePath.env" . }} +{{- end }} + +{{/* +Renders the set of all connectorBuilder config map variables +*/}} +{{- define "airbyte.connectorBuilder.configVars" }} +AI_ASSIST_URL_BASE: {{ include "airbyte.connectorBuilder.aiAssistUrlBase" . | quote }} +AIRBYTE_MANIFEST_SERVER_API_BASE_PATH: {{ include "airbyte.connectorBuilder.manifestServerApi.basePath" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_connectorRollout.tpl b/helm/airbyte-v2/templates/config/_connectorRollout.tpl new file mode 100644 index 00000000..3954a0fb --- /dev/null +++ b/helm/airbyte-v2/templates/config/_connectorRollout.tpl @@ -0,0 +1,141 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Connectorrollout Configuration +*/}} + +{{/* +Renders the global.connectorRollout.expirationSeconds value +*/}} +{{- define "airbyte.connectorRollout.expirationSeconds" }} + {{- .Values.global.connectorRollout.expirationSeconds | default 7200 }} +{{- end }} + +{{/* +Renders the connectorRollout.expirationSeconds environment variable +*/}} +{{- define "airbyte.connectorRollout.expirationSeconds.env" }} +- name: CONNECTOR_ROLLOUT_EXPIRATION_SECONDS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONNECTOR_ROLLOUT_EXPIRATION_SECONDS +{{- end }} + +{{/* +Renders the global.connectorRollout.waitBetweenRolloutSeconds value +*/}} +{{- define "airbyte.connectorRollout.waitBetweenRolloutSeconds" }} + {{- .Values.global.connectorRollout.waitBetweenRolloutSeconds | default 60 }} +{{- end }} + +{{/* +Renders the connectorRollout.waitBetweenRolloutSeconds environment variable +*/}} +{{- define "airbyte.connectorRollout.waitBetweenRolloutSeconds.env" }} +- name: CONNECTOR_ROLLOUT_WAIT_BETWEEN_ROLLOUTS_SECONDS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONNECTOR_ROLLOUT_WAIT_BETWEEN_ROLLOUTS_SECONDS +{{- end }} + +{{/* +Renders the global.connectorRollout.waitBetweenSyncResultsQueriesSeconds value +*/}} +{{- define "airbyte.connectorRollout.waitBetweenSyncResultsQueriesSeconds" }} + {{- .Values.global.connectorRollout.waitBetweenSyncResultsQueriesSeconds | default 10 }} +{{- end }} + +{{/* +Renders the connectorRollout.waitBetweenSyncResultsQueriesSeconds environment variable +*/}} +{{- define "airbyte.connectorRollout.waitBetweenSyncResultsQueriesSeconds.env" }} +- name: CONNECTOR_ROLLOUT_WAIT_BETWEEN_SYNC_RESULTS_QUERIES_SECONDS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONNECTOR_ROLLOUT_WAIT_BETWEEN_SYNC_RESULTS_QUERIES_SECONDS +{{- end }} + +{{/* +Renders the set of all connectorRollout environment variables +*/}} +{{- define "airbyte.connectorRollout.envs" }} +{{- include "airbyte.connectorRollout.expirationSeconds.env" . }} +{{- include "airbyte.connectorRollout.waitBetweenRolloutSeconds.env" . }} +{{- include "airbyte.connectorRollout.waitBetweenSyncResultsQueriesSeconds.env" . }} +{{- end }} + +{{/* +Renders the set of all connectorRollout config map variables +*/}} +{{- define "airbyte.connectorRollout.configVars" }} +CONNECTOR_ROLLOUT_EXPIRATION_SECONDS: {{ include "airbyte.connectorRollout.expirationSeconds" . | quote }} +CONNECTOR_ROLLOUT_WAIT_BETWEEN_ROLLOUTS_SECONDS: {{ include "airbyte.connectorRollout.waitBetweenRolloutSeconds" . | quote }} +CONNECTOR_ROLLOUT_WAIT_BETWEEN_SYNC_RESULTS_QUERIES_SECONDS: {{ include "airbyte.connectorRollout.waitBetweenSyncResultsQueriesSeconds" . | quote }} +{{- end }} + +{{/* +Renders the connectorRollout.temporal secret name +*/}} +{{- define "airbyte.connectorRollout.temporal.secretName" }} +{{- if .Values.global.temporal.secretName }} + {{- .Values.global.temporal.secretName }} +{{- else }} + {{- .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} +{{- end }} +{{- end }} + +{{/* +Renders the global.temporal.cloud.connectorRollout.host value +*/}} +{{- define "airbyte.connectorRollout.temporal.host" }} + {{- .Values.global.temporal.cloud.connectorRollout.host }} +{{- end }} + +{{/* +Renders the connectorRollout.temporal.host environment variable +*/}} +{{- define "airbyte.connectorRollout.temporal.host.env" }} +- name: TEMPORAL_CLOUD_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TEMPORAL_CLOUD_HOST_CONNECTOR_ROLLOUT +{{- end }} + +{{/* +Renders the global.temporal.cloud.connectorRollout.namespace value +*/}} +{{- define "airbyte.connectorRollout.temporal.namespace" }} + {{- .Values.global.temporal.cloud.connectorRollout.namespace }} +{{- end }} + +{{/* +Renders the connectorRollout.temporal.namespace environment variable +*/}} +{{- define "airbyte.connectorRollout.temporal.namespace.env" }} +- name: TEMPORAL_CLOUD_NAMESPACE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TEMPORAL_CLOUD_NAMESPACE_CONNECTOR_ROLLOUT +{{- end }} + +{{/* +Renders the set of all connectorRollout.temporal environment variables +*/}} +{{- define "airbyte.connectorRollout.temporal.envs" }} +{{- include "airbyte.connectorRollout.temporal.host.env" . }} +{{- include "airbyte.connectorRollout.temporal.namespace.env" . }} +{{- end }} + +{{/* +Renders the set of all connectorRollout.temporal config map variables +*/}} +{{- define "airbyte.connectorRollout.temporal.configVars" }} +TEMPORAL_CLOUD_HOST_CONNECTOR_ROLLOUT: {{ include "airbyte.connectorRollout.temporal.host" . | quote }} +TEMPORAL_CLOUD_NAMESPACE_CONNECTOR_ROLLOUT: {{ include "airbyte.connectorRollout.temporal.namespace" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_cron.tpl b/helm/airbyte-v2/templates/config/_cron.tpl new file mode 100644 index 00000000..545de572 --- /dev/null +++ b/helm/airbyte-v2/templates/config/_cron.tpl @@ -0,0 +1,38 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Cron Configuration +*/}} + +{{/* +Renders the cron.jobs.updateDefinitions.enabled value +*/}} +{{- define "airbyte.cron.jobs.updateDefinitions.enabled" }} + {{- .Values.cron.jobs.updateDefinitions.enabled }} +{{- end }} + +{{/* +Renders the cron.jobs.updateDefinitions.enabled environment variable +*/}} +{{- define "airbyte.cron.jobs.updateDefinitions.enabled.env" }} +- name: UPDATE_DEFINITIONS_CRON_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: UPDATE_DEFINITIONS_CRON_ENABLED +{{- end }} + +{{/* +Renders the set of all cron environment variables +*/}} +{{- define "airbyte.cron.envs" }} +{{- include "airbyte.cron.jobs.updateDefinitions.enabled.env" . }} +{{- end }} + +{{/* +Renders the set of all cron config map variables +*/}} +{{- define "airbyte.cron.configVars" }} +UPDATE_DEFINITIONS_CRON_ENABLED: {{ include "airbyte.cron.jobs.updateDefinitions.enabled" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_customerio.tpl b/helm/airbyte-v2/templates/config/_customerio.tpl new file mode 100644 index 00000000..cbbbdde2 --- /dev/null +++ b/helm/airbyte-v2/templates/config/_customerio.tpl @@ -0,0 +1,56 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Customerio Configuration +*/}} + +{{/* +Renders the customerio secret name +*/}} +{{- define "airbyte.customerio.secretName" }} +{{- if .Values.global.customerio.secretName }} + {{- .Values.global.customerio.secretName }} +{{- else }} + {{- .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} +{{- end }} +{{- end }} + +{{/* +Renders the global.customerio.apiKey value +*/}} +{{- define "airbyte.customerio.apiKey" }} + {{- .Values.global.customerio.apiKey }} +{{- end }} + +{{/* +Renders the customerio.apiKey secret key +*/}} +{{- define "airbyte.customerio.apiKey.secretKey" }} + {{- .Values.global.customerio.apiKeySecretKey | default "CUSTOMERIO_API_KEY" }} +{{- end }} + +{{/* +Renders the customerio.apiKey environment variable +*/}} +{{- define "airbyte.customerio.apiKey.env" }} +- name: CUSTOMERIO_API_KEY + valueFrom: + secretKeyRef: + name: {{ include "airbyte.customerio.secretName" . }} + key: {{ include "airbyte.customerio.apiKey.secretKey" . }} +{{- end }} + +{{/* +Renders the set of all customerio environment variables +*/}} +{{- define "airbyte.customerio.envs" }} +{{- include "airbyte.customerio.apiKey.env" . }} +{{- end }} + +{{/* +Renders the set of all customerio secret variables +*/}} +{{- define "airbyte.customerio.secrets" }} +CUSTOMERIO_API_KEY: {{ include "airbyte.customerio.apiKey" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_database.tpl b/helm/airbyte-v2/templates/config/_database.tpl new file mode 100644 index 00000000..b090fa20 --- /dev/null +++ b/helm/airbyte-v2/templates/config/_database.tpl @@ -0,0 +1,257 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Database Configuration +*/}} + +{{/* +Renders the database secret name +*/}} +{{- define "airbyte.database.secretName" }} +{{- if .Values.global.database.secretName }} + {{- .Values.global.database.secretName }} +{{- else }} + {{- .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} +{{- end }} +{{- end }} + +{{/* +Renders the global.database.host value +*/}} +{{- define "airbyte.database.host" }} + {{- .Values.global.database.host | default (printf "airbyte-db-svc.%s.svc.cluster.local" .Release.Namespace) }} +{{- end }} + +{{/* +Renders the database.host environment variable +*/}} +{{- define "airbyte.database.host.env" }} +- name: DATABASE_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATABASE_HOST +{{- end }} + +{{/* +Renders the global.database.port value +*/}} +{{- define "airbyte.database.port" }} + {{- .Values.global.database.port | default 5432 }} +{{- end }} + +{{/* +Renders the database.port environment variable +*/}} +{{- define "airbyte.database.port.env" }} +- name: DATABASE_PORT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATABASE_PORT +{{- end }} + +{{/* +Renders the global.database.url value +*/}} +{{- define "airbyte.database.url" }} + {{- (printf "jdbc:postgresql://%s:%d/%s" (include "airbyte.database.host" .) (int (include "airbyte.database.port" .)) (include "airbyte.database.name" .)) }} +{{- end }} + +{{/* +Renders the database.url environment variable +*/}} +{{- define "airbyte.database.url.env" }} +- name: DATABASE_URL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATABASE_URL +{{- end }} + +{{/* +Renders the global.database.user value. +When global.secretSeed is set, default is env-specific (dbuser-); else "airbyte". +*/}} +{{- define "airbyte.database.user" }} + {{- .Values.global.database.user | default (ternary (printf "dbuser-%s" (include "airbyte.secretSeed.hashBase" . | sha256sum | nospace | trunc 12)) "airbyte" (ne (toString .Values.global.secretSeed) "")) }} +{{- end }} + +{{/* +Renders the database.user secret key +*/}} +{{- define "airbyte.database.user.secretKey" }} + {{- .Values.global.database.userSecretKey | default "DATABASE_USER" }} +{{- end }} + +{{/* +Renders the database.user environment variable +*/}} +{{- define "airbyte.database.user.env" }} +- name: DATABASE_USER + valueFrom: + secretKeyRef: + name: {{ include "airbyte.database.secretName" . }} + key: {{ include "airbyte.database.user.secretKey" . }} +{{- end }} + +{{/* +Renders the global.database.password value. +When global.secretSeed is set, default is env-specific; else "airbyte". +*/}} +{{- define "airbyte.database.password" }} + {{- .Values.global.database.password | default (ternary (printf "db-%s" (include "airbyte.secretSeed.hashBase" . | sha256sum | nospace | trunc 24)) "airbyte" (ne (toString .Values.global.secretSeed) "")) }} +{{- end }} + +{{/* +Renders the database.password secret key +*/}} +{{- define "airbyte.database.password.secretKey" }} + {{- .Values.global.database.passwordSecretKey | default "DATABASE_PASSWORD" }} +{{- end }} + +{{/* +Renders the database.password environment variable +*/}} +{{- define "airbyte.database.password.env" }} +- name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "airbyte.database.secretName" . }} + key: {{ include "airbyte.database.password.secretKey" . }} +{{- end }} + +{{/* +Renders the global.database.name value +*/}} +{{- define "airbyte.database.name" }} + {{- .Values.global.database.name | default "db-airbyte" }} +{{- end }} + +{{/* +Renders the database.name environment variable +*/}} +{{- define "airbyte.database.name.env" }} +- name: DATABASE_DB + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATABASE_DB +{{- end }} + +{{/* +Renders the set of all database environment variables +*/}} +{{- define "airbyte.database.envs" }} +{{- include "airbyte.database.host.env" . }} +{{- include "airbyte.database.port.env" . }} +{{- include "airbyte.database.url.env" . }} +{{- include "airbyte.database.user.env" . }} +{{- include "airbyte.database.password.env" . }} +{{- include "airbyte.database.name.env" . }} +{{- end }} + +{{/* +Renders the set of all database config map variables +*/}} +{{- define "airbyte.database.configVars" }} +DATABASE_HOST: {{ include "airbyte.database.host" . | quote }} +DATABASE_PORT: {{ include "airbyte.database.port" . | quote }} +DATABASE_URL: {{ include "airbyte.database.url" . | quote }} +DATABASE_DB: {{ include "airbyte.database.name" . | quote }} +DATABASE_USER: {{ include "airbyte.database.user" . | quote }} +DATABASE_PASSWORD: {{ include "airbyte.database.password" . | quote }} +{{- end }} + +{{/* +Renders the set of all database secret variables +*/}} +{{- define "airbyte.database.secrets" }} +DATABASE_USER: {{ include "airbyte.database.user" . | quote }} +DATABASE_PASSWORD: {{ include "airbyte.database.password" . | quote }} +{{- end }} + +{{/* +Renders secret vars for Micronaut datasource "local-secrets" (workload-init-container when SECRET_PERSISTENCE=testing_config_db_table). Same credentials as main DB. +*/}} +{{- define "airbyte.database.localSecretsDatasource.secrets" }} +DATASOURCES_LOCALSECRETS_USERNAME: {{ include "airbyte.database.user" . | quote }} +DATASOURCES_LOCALSECRETS_PASSWORD: {{ include "airbyte.database.password" . | quote }} +{{- end }} + +{{/* +Renders the global.migrations.runAtStartup value +*/}} +{{- define "airbyte.database.migrations.runAtStartup" }} + {{- if eq .Values.global.migrations.runAtStartup nil }} + {{- true }} + {{- else }} + {{- .Values.global.migrations.runAtStartup }} + {{- end }} +{{- end }} + +{{/* +Renders the database.migrations.runAtStartup environment variable +*/}} +{{- define "airbyte.database.migrations.runAtStartup.env" }} +- name: RUN_DATABASE_MIGRATION_ON_STARTUP + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: RUN_DATABASE_MIGRATION_ON_STARTUP +{{- end }} + +{{/* +Renders the global.migrations.configDb.minimumFlywayMigrationVersion value +*/}} +{{- define "airbyte.database.migrations.configDb.minimumFlywayMigrationVersion" }} + {{- .Values.global.migrations.configDb.minimumFlywayMigrationVersion | default "0.35.15.001" }} +{{- end }} + +{{/* +Renders the database.migrations.configDb.minimumFlywayMigrationVersion environment variable +*/}} +{{- define "airbyte.database.migrations.configDb.minimumFlywayMigrationVersion.env" }} +- name: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION +{{- end }} + +{{/* +Renders the global.migrations.jobsDb.minimumFlywayMigrationVersion value +*/}} +{{- define "airbyte.database.migrations.jobsDb.minimumFlywayMigrationVersion" }} + {{- .Values.global.migrations.jobsDb.minimumFlywayMigrationVersion | default "0.29.15.001" }} +{{- end }} + +{{/* +Renders the database.migrations.jobsDb.minimumFlywayMigrationVersion environment variable +*/}} +{{- define "airbyte.database.migrations.jobsDb.minimumFlywayMigrationVersion.env" }} +- name: JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION +{{- end }} + +{{/* +Renders the set of all database.migrations environment variables +*/}} +{{- define "airbyte.database.migrations.envs" }} +{{- include "airbyte.database.migrations.runAtStartup.env" . }} +{{- include "airbyte.database.migrations.configDb.minimumFlywayMigrationVersion.env" . }} +{{- include "airbyte.database.migrations.jobsDb.minimumFlywayMigrationVersion.env" . }} +{{- end }} + +{{/* +Renders the set of all database.migrations config map variables +*/}} +{{- define "airbyte.database.migrations.configVars" }} +RUN_DATABASE_MIGRATION_ON_STARTUP: {{ include "airbyte.database.migrations.runAtStartup" . | quote }} +CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION: {{ include "airbyte.database.migrations.configDb.minimumFlywayMigrationVersion" . | quote }} +JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION: {{ include "airbyte.database.migrations.jobsDb.minimumFlywayMigrationVersion" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_datadog.tpl b/helm/airbyte-v2/templates/config/_datadog.tpl new file mode 100644 index 00000000..cfda707d --- /dev/null +++ b/helm/airbyte-v2/templates/config/_datadog.tpl @@ -0,0 +1,402 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Datadog Configuration +*/}} + +{{/* +Renders the global.datadog.agentHost value +*/}} +{{- define "airbyte.datadog.agentHost" }} + {{- .Values.global.datadog.agentHost }} +{{- end }} + +{{/* +Renders the datadog.agentHost environment variable +*/}} +{{- define "airbyte.datadog.agentHost.env" }} +- name: DD_AGENT_HOST + valueFrom: + fieldRef: + fieldPath: status.hostIP + +{{- end }} + +{{/* +Renders the global.datadog.enabled value +*/}} +{{- define "airbyte.datadog.enabled" }} + {{- if eq .Values.global.datadog.enabled nil }} + {{- false }} + {{- else }} + {{- .Values.global.datadog.enabled }} + {{- end }} +{{- end }} + +{{/* +Renders the datadog.enabled environment variable +*/}} +{{- define "airbyte.datadog.enabled.env" }} +- name: DD_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DD_ENABLED +{{- end }} + +{{/* +Renders the global.datadog.env value +*/}} +{{- define "airbyte.datadog.env" }} + {{- .Values.global.datadog.env }} +{{- end }} + +{{/* +Renders the datadog.env environment variable +*/}} +{{- define "airbyte.datadog.env.env" }} +- name: DD_ENV + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DD_ENV +{{- end }} + +{{/* +Renders the global.datadog.version value +*/}} +{{- define "airbyte.datadog.version" }} + {{- .Values.global.image.tag }} +{{- end }} + +{{/* +Renders the datadog.version environment variable +*/}} +{{- define "airbyte.datadog.version.env" }} +- name: DD_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DD_VERSION +{{- end }} + +{{/* +Renders the global.datadog.statsd.port value +*/}} +{{- define "airbyte.datadog.statsd.port" }} + {{- .Values.global.datadog.statsd.port | default 8125 }} +{{- end }} + +{{/* +Renders the datadog.statsd.port environment variable +*/}} +{{- define "airbyte.datadog.statsd.port.env" }} +- name: DD_DOGSTATSD_PORT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DD_DOGSTATSD_PORT +{{- end }} + +{{/* +Renders the global.datadog.traceAgentPort value +*/}} +{{- define "airbyte.datadog.traceAgentPort" }} + {{- .Values.global.datadog.traceAgentPort | default 8126 }} +{{- end }} + +{{/* +Renders the datadog.traceAgentPort environment variable +*/}} +{{- define "airbyte.datadog.traceAgentPort.env" }} +- name: DD_TRACE_AGENT_PORT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DD_TRACE_AGENT_PORT +{{- end }} + +{{/* +Renders the global.datadog.integrations.dbm.enabled value +*/}} +{{- define "airbyte.datadog.integrations.dbm.enabled" }} + {{- if eq .Values.global.datadog.integrations.dbm.enabled nil }} + {{- false }} + {{- else }} + {{- .Values.global.datadog.integrations.dbm.enabled }} + {{- end }} +{{- end }} + +{{/* +Renders the datadog.integrations.dbm.enabled environment variable +*/}} +{{- define "airbyte.datadog.integrations.dbm.enabled.env" }} +- name: DD_INTEGRATION_DBM_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DD_INTEGRATION_DBM_ENABLED +{{- end }} + +{{/* +Renders the global.datadog.integrations.dbm.propagationMode value +*/}} +{{- define "airbyte.datadog.integrations.dbm.propagationMode" }} + {{- .Values.global.datadog.integrations.dbm.propagationMode | default "full" }} +{{- end }} + +{{/* +Renders the datadog.integrations.dbm.propagationMode environment variable +*/}} +{{- define "airbyte.datadog.integrations.dbm.propagationMode.env" }} +- name: DD_INTEGRATION_DBM_PROPAGATION_MODE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DD_INTEGRATION_DBM_PROPAGATION_MODE +{{- end }} + +{{/* +Renders the global.datadog.integrations.googleHttpClient.enabled value +*/}} +{{- define "airbyte.datadog.integrations.googleHttpClient.enabled" }} + {{- if eq .Values.global.datadog.integrations.googleHttpClient.enabled nil }} + {{- false }} + {{- else }} + {{- .Values.global.datadog.integrations.googleHttpClient.enabled }} + {{- end }} +{{- end }} + +{{/* +Renders the datadog.integrations.googleHttpClient.enabled environment variable +*/}} +{{- define "airbyte.datadog.integrations.googleHttpClient.enabled.env" }} +- name: DD_INTEGRATION_GOOGLE_HTTP_CLIENT_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DD_INTEGRATION_GOOGLE_HTTP_CLIENT_ENABLED +{{- end }} + +{{/* +Renders the global.datadog.integrations.grpc.enabled value +*/}} +{{- define "airbyte.datadog.integrations.grpc.enabled" }} + {{- if eq .Values.global.datadog.integrations.grpc.enabled nil }} + {{- false }} + {{- else }} + {{- .Values.global.datadog.integrations.grpc.enabled }} + {{- end }} +{{- end }} + +{{/* +Renders the datadog.integrations.grpc.enabled environment variable +*/}} +{{- define "airbyte.datadog.integrations.grpc.enabled.env" }} +- name: DD_INTEGRATION_GRPC_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DD_INTEGRATION_GRPC_ENABLED +{{- end }} + +{{/* +Renders the global.datadog.integrations.grpc.clientEnabled value +*/}} +{{- define "airbyte.datadog.integrations.grpc.clientEnabled" }} + {{- if eq .Values.global.datadog.integrations.grpc.clientEnabled nil }} + {{- false }} + {{- else }} + {{- .Values.global.datadog.integrations.grpc.clientEnabled }} + {{- end }} +{{- end }} + +{{/* +Renders the datadog.integrations.grpc.clientEnabled environment variable +*/}} +{{- define "airbyte.datadog.integrations.grpc.clientEnabled.env" }} +- name: DD_INTEGRATION_GRPC_CLIENT_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DD_INTEGRATION_GRPC_CLIENT_ENABLED +{{- end }} + +{{/* +Renders the global.datadog.integrations.grpc.serverEnabled value +*/}} +{{- define "airbyte.datadog.integrations.grpc.serverEnabled" }} + {{- if eq .Values.global.datadog.integrations.grpc.serverEnabled nil }} + {{- false }} + {{- else }} + {{- .Values.global.datadog.integrations.grpc.serverEnabled }} + {{- end }} +{{- end }} + +{{/* +Renders the datadog.integrations.grpc.serverEnabled environment variable +*/}} +{{- define "airbyte.datadog.integrations.grpc.serverEnabled.env" }} +- name: DD_INTEGRATION_GRPC_SERVER_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DD_INTEGRATION_GRPC_SERVER_ENABLED +{{- end }} + +{{/* +Renders the global.datadog.integrations.httpUrlConnection.enabled value +*/}} +{{- define "airbyte.datadog.integrations.httpUrlConnection.enabled" }} + {{- if eq .Values.global.datadog.integrations.httpUrlConnection.enabled nil }} + {{- false }} + {{- else }} + {{- .Values.global.datadog.integrations.httpUrlConnection.enabled }} + {{- end }} +{{- end }} + +{{/* +Renders the datadog.integrations.httpUrlConnection.enabled environment variable +*/}} +{{- define "airbyte.datadog.integrations.httpUrlConnection.enabled.env" }} +- name: DD_INTEGRATION_HTTPURLCONNECTION_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DD_INTEGRATION_HTTPURLCONNECTION_ENABLED +{{- end }} + +{{/* +Renders the global.datadog.integrations.kotlinCoroutineExperimental.enabled value +*/}} +{{- define "airbyte.datadog.integrations.kotlinCoroutineExperimental.enabled" }} + {{- if eq .Values.global.datadog.integrations.kotlinCoroutineExperimental.enabled nil }} + {{- false }} + {{- else }} + {{- .Values.global.datadog.integrations.kotlinCoroutineExperimental.enabled }} + {{- end }} +{{- end }} + +{{/* +Renders the datadog.integrations.kotlinCoroutineExperimental.enabled environment variable +*/}} +{{- define "airbyte.datadog.integrations.kotlinCoroutineExperimental.enabled.env" }} +- name: DD_INTEGRATION_KOTLIN_COROUTINE_EXPERIMENTAL_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DD_INTEGRATION_KOTLIN_COROUTINE_EXPERIMENTAL_ENABLED +{{- end }} + +{{/* +Renders the global.datadog.integrations.netty.enabled value +*/}} +{{- define "airbyte.datadog.integrations.netty.enabled" }} + {{- if eq .Values.global.datadog.integrations.netty.enabled nil }} + {{- false }} + {{- else }} + {{- .Values.global.datadog.integrations.netty.enabled }} + {{- end }} +{{- end }} + +{{/* +Renders the datadog.integrations.netty.enabled environment variable +*/}} +{{- define "airbyte.datadog.integrations.netty.enabled.env" }} +- name: DD_INTEGRATION_NETTY_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DD_INTEGRATION_NETTY_ENABLED +{{- end }} + +{{/* +Renders the global.datadog.integrations.netty41.enabled value +*/}} +{{- define "airbyte.datadog.integrations.netty41.enabled" }} + {{- if eq .Values.global.datadog.integrations.netty41.enabled nil }} + {{- false }} + {{- else }} + {{- .Values.global.datadog.integrations.netty41.enabled }} + {{- end }} +{{- end }} + +{{/* +Renders the datadog.integrations.netty41.enabled environment variable +*/}} +{{- define "airbyte.datadog.integrations.netty41.enabled.env" }} +- name: DD_INTEGRATION_NETTY_4_1_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DD_INTEGRATION_NETTY_4_1_ENABLED +{{- end }} + +{{/* +Renders the global.datadog.integrations.urlConnection.enabled value +*/}} +{{- define "airbyte.datadog.integrations.urlConnection.enabled" }} + {{- if eq .Values.global.datadog.integrations.urlConnection.enabled nil }} + {{- false }} + {{- else }} + {{- .Values.global.datadog.integrations.urlConnection.enabled }} + {{- end }} +{{- end }} + +{{/* +Renders the datadog.integrations.urlConnection.enabled environment variable +*/}} +{{- define "airbyte.datadog.integrations.urlConnection.enabled.env" }} +- name: DD_INTEGRATION_URLCONNECTION_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DD_INTEGRATION_URLCONNECTION_ENABLED +{{- end }} + +{{/* +Renders the set of all datadog environment variables +*/}} +{{- define "airbyte.datadog.envs" }} +{{- include "airbyte.datadog.agentHost.env" . }} +{{- include "airbyte.datadog.enabled.env" . }} +{{- include "airbyte.datadog.env.env" . }} +{{- include "airbyte.datadog.version.env" . }} +{{- include "airbyte.datadog.statsd.port.env" . }} +{{- include "airbyte.datadog.traceAgentPort.env" . }} +{{- include "airbyte.datadog.integrations.dbm.enabled.env" . }} +{{- include "airbyte.datadog.integrations.dbm.propagationMode.env" . }} +{{- include "airbyte.datadog.integrations.googleHttpClient.enabled.env" . }} +{{- include "airbyte.datadog.integrations.grpc.enabled.env" . }} +{{- include "airbyte.datadog.integrations.grpc.clientEnabled.env" . }} +{{- include "airbyte.datadog.integrations.grpc.serverEnabled.env" . }} +{{- include "airbyte.datadog.integrations.httpUrlConnection.enabled.env" . }} +{{- include "airbyte.datadog.integrations.kotlinCoroutineExperimental.enabled.env" . }} +{{- include "airbyte.datadog.integrations.netty.enabled.env" . }} +{{- include "airbyte.datadog.integrations.netty41.enabled.env" . }} +{{- include "airbyte.datadog.integrations.urlConnection.enabled.env" . }} +{{- end }} + +{{/* +Renders the set of all datadog config map variables +*/}} +{{- define "airbyte.datadog.configVars" }} +DD_AGENT_HOST: {{ include "airbyte.datadog.agentHost" . | quote }} +DD_ENABLED: {{ include "airbyte.datadog.enabled" . | quote }} +DD_ENV: {{ include "airbyte.datadog.env" . | quote }} +DD_VERSION: {{ include "airbyte.datadog.version" . | quote }} +DD_DOGSTATSD_PORT: {{ include "airbyte.datadog.statsd.port" . | quote }} +DD_TRACE_AGENT_PORT: {{ include "airbyte.datadog.traceAgentPort" . | quote }} +DD_INTEGRATION_DBM_ENABLED: {{ include "airbyte.datadog.integrations.dbm.enabled" . | quote }} +DD_INTEGRATION_DBM_PROPAGATION_MODE: {{ include "airbyte.datadog.integrations.dbm.propagationMode" . | quote }} +DD_INTEGRATION_GOOGLE_HTTP_CLIENT_ENABLED: {{ include "airbyte.datadog.integrations.googleHttpClient.enabled" . | quote }} +DD_INTEGRATION_GRPC_ENABLED: {{ include "airbyte.datadog.integrations.grpc.enabled" . | quote }} +DD_INTEGRATION_GRPC_CLIENT_ENABLED: {{ include "airbyte.datadog.integrations.grpc.clientEnabled" . | quote }} +DD_INTEGRATION_GRPC_SERVER_ENABLED: {{ include "airbyte.datadog.integrations.grpc.serverEnabled" . | quote }} +DD_INTEGRATION_HTTPURLCONNECTION_ENABLED: {{ include "airbyte.datadog.integrations.httpUrlConnection.enabled" . | quote }} +DD_INTEGRATION_KOTLIN_COROUTINE_EXPERIMENTAL_ENABLED: {{ include "airbyte.datadog.integrations.kotlinCoroutineExperimental.enabled" . | quote }} +DD_INTEGRATION_NETTY_ENABLED: {{ include "airbyte.datadog.integrations.netty.enabled" . | quote }} +DD_INTEGRATION_NETTY_4_1_ENABLED: {{ include "airbyte.datadog.integrations.netty41.enabled" . | quote }} +DD_INTEGRATION_URLCONNECTION_ENABLED: {{ include "airbyte.datadog.integrations.urlConnection.enabled" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_enterprise.tpl b/helm/airbyte-v2/templates/config/_enterprise.tpl new file mode 100644 index 00000000..3392afbe --- /dev/null +++ b/helm/airbyte-v2/templates/config/_enterprise.tpl @@ -0,0 +1,56 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Enterprise Configuration +*/}} + +{{/* +Renders the enterprise secret name +*/}} +{{- define "airbyte.enterprise.secretName" }} +{{- if .Values.global.enterprise.secretName }} + {{- .Values.global.enterprise.secretName }} +{{- else }} + {{- .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} +{{- end }} +{{- end }} + +{{/* +Renders the global.enterprise.licenseKey value +*/}} +{{- define "airbyte.enterprise.licenseKey" }} + {{- .Values.global.enterprise.licenseKey }} +{{- end }} + +{{/* +Renders the enterprise.licenseKey secret key +*/}} +{{- define "airbyte.enterprise.licenseKey.secretKey" }} + {{- .Values.global.enterprise.licenseKeySecretKey | default "AIRBYTE_LICENSE_KEY" }} +{{- end }} + +{{/* +Renders the enterprise.licenseKey environment variable +*/}} +{{- define "airbyte.enterprise.licenseKey.env" }} +- name: AIRBYTE_LICENSE_KEY + valueFrom: + secretKeyRef: + name: {{ include "airbyte.enterprise.secretName" . }} + key: {{ include "airbyte.enterprise.licenseKey.secretKey" . }} +{{- end }} + +{{/* +Renders the set of all enterprise environment variables +*/}} +{{- define "airbyte.enterprise.envs" }} +{{- include "airbyte.enterprise.licenseKey.env" . }} +{{- end }} + +{{/* +Renders the set of all enterprise secret variables +*/}} +{{- define "airbyte.enterprise.secrets" }} +AIRBYTE_LICENSE_KEY: {{ include "airbyte.enterprise.licenseKey" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_featureFlags.tpl b/helm/airbyte-v2/templates/config/_featureFlags.tpl new file mode 100644 index 00000000..31b01212 --- /dev/null +++ b/helm/airbyte-v2/templates/config/_featureFlags.tpl @@ -0,0 +1,126 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Featureflags Configuration +*/}} + +{{/* +Renders the featureFlags secret name +*/}} +{{- define "airbyte.featureFlags.secretName" }} +{{- if .Values.global.featureFlags.secretName }} + {{- .Values.global.featureFlags.secretName }} +{{- else }} + {{- .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} +{{- end }} +{{- end }} + +{{/* +Renders the global.featureFlags.client value +*/}} +{{- define "airbyte.featureFlags.client" }} + {{- .Values.global.featureFlags.client | default "configfile" }} +{{- end }} + +{{/* +Renders the featureFlags.client environment variable +*/}} +{{- define "airbyte.featureFlags.client.env" }} +- name: FEATURE_FLAG_CLIENT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: FEATURE_FLAG_CLIENT +{{- end }} + +{{/* +Renders the global.featureFlags.configfile.path value +*/}} +{{- define "airbyte.featureFlags.configfile.path" }} + {{- .Values.global.featureFlags.configfile.path | default "/etc/launchdarkly/flags.yml" }} +{{- end }} + +{{/* +Renders the featureFlags.configfile.path environment variable +*/}} +{{- define "airbyte.featureFlags.configfile.path.env" }} +- name: FEATURE_FLAG_PATH + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: FEATURE_FLAG_PATH +{{- end }} + +{{/* +Renders the global.featureFlags.launchdarkly.key value +*/}} +{{- define "airbyte.featureFlags.launchdarkly.key" }} + {{- .Values.global.featureFlags.launchdarkly.key }} +{{- end }} + +{{/* +Renders the featureFlags.launchdarkly.key secret key +*/}} +{{- define "airbyte.featureFlags.launchdarkly.key.secretKey" }} + {{- .Values.global.featureFlags.launchdarkly.keySecretKey | default "LAUNCHDARKLY_KEY" }} +{{- end }} + +{{/* +Renders the featureFlags.launchdarkly.key environment variable +*/}} +{{- define "airbyte.featureFlags.launchdarkly.key.env" }} +- name: LAUNCHDARKLY_KEY + valueFrom: + secretKeyRef: + name: {{ include "airbyte.featureFlags.secretName" . }} + key: {{ include "airbyte.featureFlags.launchdarkly.key.secretKey" . }} +{{- end }} + +{{/* +Renders the set of all featureFlags environment variables +*/}} +{{- define "airbyte.featureFlags.envs" }} +{{- include "airbyte.featureFlags.client.env" . }} +{{- $opt := (include "airbyte.featureFlags.client" .) }} + +{{- if eq $opt "configfile" }} +{{- include "airbyte.featureFlags.configfile.path.env" . }} +{{- end }} + +{{- if eq $opt "launchdarkly" }} +{{- include "airbyte.featureFlags.launchdarkly.key.env" . }} +{{- end }} + +{{- end }} + +{{/* +Renders the set of all featureFlags config map variables +*/}} +{{- define "airbyte.featureFlags.configVars" }} +FEATURE_FLAG_CLIENT: {{ include "airbyte.featureFlags.client" . | quote }} +{{- $opt := (include "airbyte.featureFlags.client" .) }} + +{{- if eq $opt "configfile" }} +FEATURE_FLAG_PATH: {{ include "airbyte.featureFlags.configfile.path" . | quote }} +{{- end }} + +{{- if eq $opt "launchdarkly" }} +{{- end }} + +{{- end }} + +{{/* +Renders the set of all featureFlags secret variables +*/}} +{{- define "airbyte.featureFlags.secrets" }} +{{- $opt := (include "airbyte.featureFlags.client" .) }} + +{{- if eq $opt "configfile" }} +{{- end }} + +{{- if eq $opt "launchdarkly" }} +LAUNCHDARKLY_KEY: {{ include "airbyte.featureFlags.launchdarkly.key" . | quote }} +{{- end }} + +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_java.tpl b/helm/airbyte-v2/templates/config/_java.tpl new file mode 100644 index 00000000..e957ba83 --- /dev/null +++ b/helm/airbyte-v2/templates/config/_java.tpl @@ -0,0 +1,38 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Java Configuration +*/}} + +{{/* +Renders the global.java.opts value +*/}} +{{- define "airbyte.java.opts" }} + {{- join " " .Values.global.java.opts }} +{{- end }} + +{{/* +Renders the java.opts environment variable +*/}} +{{- define "airbyte.java.opts.env" }} +- name: JAVA_TOOL_OPTIONS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JAVA_TOOL_OPTIONS +{{- end }} + +{{/* +Renders the set of all java environment variables +*/}} +{{- define "airbyte.java.envs" }} +{{- include "airbyte.java.opts.env" . }} +{{- end }} + +{{/* +Renders the set of all java config map variables +*/}} +{{- define "airbyte.java.configVars" }} +JAVA_TOOL_OPTIONS: {{ include "airbyte.java.opts" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_jobs.tpl b/helm/airbyte-v2/templates/config/_jobs.tpl new file mode 100644 index 00000000..8c1b5c03 --- /dev/null +++ b/helm/airbyte-v2/templates/config/_jobs.tpl @@ -0,0 +1,422 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Jobs Configuration +*/}} + +{{/* +Renders the service account used for job pods (JOB_KUBE_SERVICEACCOUNT). Uses effectiveServiceAccountName (e.g. RELEASE_NAME-worker-sa) when global.serviceAccountName is not set. +*/}} +{{- define "airbyte.jobs.kube.serviceAccount" }} + {{- include "airbyte.effectiveServiceAccountName" . }} +{{- end }} + +{{/* +Renders the jobs.kube.serviceAccount environment variable +*/}} +{{- define "airbyte.jobs.kube.serviceAccount.env" }} +- name: JOB_KUBE_SERVICEACCOUNT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_KUBE_SERVICEACCOUNT +{{- end }} + +{{/* +Renders the global.jobs.kube.namespace value +*/}} +{{- define "airbyte.jobs.kube.namespace" }} + {{- .Values.global.jobs.kube.namespace }} +{{- end }} + +{{/* +Renders the jobs.kube.namespace environment variable +*/}} +{{- define "airbyte.jobs.kube.namespace.env" }} +- name: JOB_KUBE_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + +{{- end }} + +{{/* +Renders the global.jobs.kube.localVolume.enabled value +*/}} +{{- define "airbyte.jobs.kube.localVolume.enabled" }} + {{- if eq .Values.global.jobs.kube.localVolume.enabled nil }} + {{- false }} + {{- else }} + {{- .Values.global.jobs.kube.localVolume.enabled }} + {{- end }} +{{- end }} + +{{/* +Renders the jobs.kube.localVolume.enabled environment variable +*/}} +{{- define "airbyte.jobs.kube.localVolume.enabled.env" }} +- name: JOB_KUBE_LOCAL_VOLUME_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_KUBE_LOCAL_VOLUME_ENABLED +{{- end }} + +{{/* +Renders the global.jobs.kube.mainContainerImagePullSecret value +*/}} +{{- define "airbyte.jobs.kube.mainContainerImagePullSecret" }} + {{- (include "airbyte.imagePullSecretNames" (dict "secrets" .Values.global.imagePullSecrets "extra" (list .Values.global.jobs.kube.mainContainerImagePullSecret))) }} +{{- end }} + +{{/* +Renders the jobs.kube.mainContainerImagePullSecret environment variable +*/}} +{{- define "airbyte.jobs.kube.mainContainerImagePullSecret.env" }} +- name: JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET +{{- end }} + +{{/* +Renders the global.jobs.kube.annotations value +*/}} +{{- define "airbyte.jobs.kube.annotations" }} + {{- .Values.global.jobs.kube.annotations | include "airbyte.flattenMap" }} +{{- end }} + +{{/* +Renders the jobs.kube.annotations environment variable +*/}} +{{- define "airbyte.jobs.kube.annotations.env" }} +- name: JOB_KUBE_ANNOTATIONS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_KUBE_ANNOTATIONS +{{- end }} + +{{/* +Renders the global.jobs.kube.labels value +*/}} +{{- define "airbyte.jobs.kube.labels" }} + {{- .Values.global.jobs.kube.labels | include "airbyte.flattenMap" }} +{{- end }} + +{{/* +Renders the jobs.kube.labels environment variable +*/}} +{{- define "airbyte.jobs.kube.labels.env" }} +- name: JOB_KUBE_LABELS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_KUBE_LABELS +{{- end }} + +{{/* +Renders the global.jobs.kube.nodeSelector value +*/}} +{{- define "airbyte.jobs.kube.nodeSelector" }} + {{- .Values.global.jobs.kube.nodeSelector | include "airbyte.flattenMap" }} +{{- end }} + +{{/* +Renders the jobs.kube.nodeSelector environment variable +*/}} +{{- define "airbyte.jobs.kube.nodeSelector.env" }} +- name: JOB_KUBE_NODE_SELECTORS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_KUBE_NODE_SELECTORS +{{- end }} + +{{/* +Renders the global.jobs.kube.tolerations value +*/}} +{{- define "airbyte.jobs.kube.tolerations" }} + {{- .Values.global.jobs.kube.tolerations | include "airbyte.flattenArrayMap" }} +{{- end }} + +{{/* +Renders the jobs.kube.tolerations environment variable +*/}} +{{- define "airbyte.jobs.kube.tolerations.env" }} +- name: JOB_KUBE_TOLERATIONS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_KUBE_TOLERATIONS +{{- end }} + +{{/* +Renders the set of all jobs environment variables +*/}} +{{- define "airbyte.jobs.envs" }} +{{- include "airbyte.jobs.kube.serviceAccount.env" . }} +{{- include "airbyte.jobs.kube.namespace.env" . }} +{{- include "airbyte.jobs.kube.localVolume.enabled.env" . }} +{{- include "airbyte.jobs.kube.mainContainerImagePullSecret.env" . }} +{{- include "airbyte.jobs.kube.annotations.env" . }} +{{- include "airbyte.jobs.kube.labels.env" . }} +{{- include "airbyte.jobs.kube.nodeSelector.env" . }} +{{- include "airbyte.jobs.kube.tolerations.env" . }} +{{- end }} + +{{/* +Renders the set of all jobs config map variables +*/}} +{{- define "airbyte.jobs.configVars" }} +JOB_KUBE_SERVICEACCOUNT: {{ include "airbyte.jobs.kube.serviceAccount" . | quote }} +JOB_KUBE_NAMESPACE: {{ include "airbyte.jobs.kube.namespace" . | quote }} +JOB_KUBE_LOCAL_VOLUME_ENABLED: {{ include "airbyte.jobs.kube.localVolume.enabled" . | quote }} +JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET: {{ include "airbyte.jobs.kube.mainContainerImagePullSecret" . | quote }} +JOB_KUBE_ANNOTATIONS: {{ include "airbyte.jobs.kube.annotations" . | quote }} +JOB_KUBE_LABELS: {{ include "airbyte.jobs.kube.labels" . | quote }} +JOB_KUBE_NODE_SELECTORS: {{ include "airbyte.jobs.kube.nodeSelector" . | quote }} +JOB_KUBE_TOLERATIONS: {{ include "airbyte.jobs.kube.tolerations" . | quote }} +{{- end }} + +{{/* +Renders the global.jobs.errors.reportingStrategy value +*/}} +{{- define "airbyte.jobs.errors.reportingStrategy" }} + {{- .Values.global.jobs.errors.reportingStrategy | default "logging" }} +{{- end }} + +{{/* +Renders the jobs.errors.reportingStrategy environment variable +*/}} +{{- define "airbyte.jobs.errors.reportingStrategy.env" }} +- name: JOB_ERROR_REPORTING_STRATEGY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_ERROR_REPORTING_STRATEGY +{{- end }} + +{{/* +Renders the global.jobs.errors.sentry.dsn value +*/}} +{{- define "airbyte.jobs.errors.sentry.dsn" }} + {{- .Values.global.jobs.errors.sentry.dsn }} +{{- end }} + +{{/* +Renders the jobs.errors.sentry.dsn environment variable +*/}} +{{- define "airbyte.jobs.errors.sentry.dsn.env" }} +- name: JOB_ERROR_REPORTING_SENTRY_DSN + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_ERROR_REPORTING_SENTRY_DSN +{{- end }} + +{{/* +Renders the set of all jobs.errors environment variables +*/}} +{{- define "airbyte.jobs.errors.envs" }} +{{- include "airbyte.jobs.errors.reportingStrategy.env" . }} +{{- $opt := (include "airbyte.jobs.errors.reportingStrategy" .) }} + +{{- if eq $opt "logging" }} +{{- end }} + +{{- if eq $opt "sentry" }} +{{- include "airbyte.jobs.errors.sentry.dsn.env" . }} +{{- end }} + +{{- end }} + +{{/* +Renders the set of all jobs.errors config map variables +*/}} +{{- define "airbyte.jobs.errors.configVars" }} +JOB_ERROR_REPORTING_STRATEGY: {{ include "airbyte.jobs.errors.reportingStrategy" . | quote }} +{{- $opt := (include "airbyte.jobs.errors.reportingStrategy" .) }} + +{{- if eq $opt "logging" }} +{{- end }} + +{{- if eq $opt "sentry" }} +JOB_ERROR_REPORTING_SENTRY_DSN: {{ include "airbyte.jobs.errors.sentry.dsn" . | quote }} +{{- end }} + +{{- end }} + +{{/* +Renders the global.jobs.kube.scheduling.check.nodeSelectors value +*/}} +{{- define "airbyte.jobs.scheduling.check.nodeSelectors" }} + {{- .Values.global.jobs.kube.scheduling.check.nodeSelectors | include "airbyte.flattenMap" }} +{{- end }} + +{{/* +Renders the jobs.scheduling.check.nodeSelectors environment variable +*/}} +{{- define "airbyte.jobs.scheduling.check.nodeSelectors.env" }} +- name: CHECK_JOB_KUBE_NODE_SELECTORS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CHECK_JOB_KUBE_NODE_SELECTORS +{{- end }} + +{{/* +Renders the global.jobs.kube.scheduling.check.runtimeClassName value +*/}} +{{- define "airbyte.jobs.scheduling.check.runtimeClassName" }} + {{- .Values.global.jobs.kube.scheduling.check.runtimeClassName }} +{{- end }} + +{{/* +Renders the jobs.scheduling.check.runtimeClassName environment variable +*/}} +{{- define "airbyte.jobs.scheduling.check.runtimeClassName.env" }} +- name: CHECK_JOB_KUBE_RUNTIME_CLASS_NAME + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CHECK_JOB_KUBE_RUNTIME_CLASS_NAME +{{- end }} + +{{/* +Renders the global.jobs.kube.scheduling.discover.nodeSelectors value +*/}} +{{- define "airbyte.jobs.scheduling.discover.nodeSelectors" }} + {{- .Values.global.jobs.kube.scheduling.discover.nodeSelectors | include "airbyte.flattenMap" }} +{{- end }} + +{{/* +Renders the jobs.scheduling.discover.nodeSelectors environment variable +*/}} +{{- define "airbyte.jobs.scheduling.discover.nodeSelectors.env" }} +- name: DISCOVER_JOB_KUBE_NODE_SELECTORS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DISCOVER_JOB_KUBE_NODE_SELECTORS +{{- end }} + +{{/* +Renders the global.jobs.kube.scheduling.discover.runtimeClassName value +*/}} +{{- define "airbyte.jobs.scheduling.discover.runtimeClassName" }} + {{- .Values.global.jobs.kube.scheduling.discover.runtimeClassName }} +{{- end }} + +{{/* +Renders the jobs.scheduling.discover.runtimeClassName environment variable +*/}} +{{- define "airbyte.jobs.scheduling.discover.runtimeClassName.env" }} +- name: DISCOVER_JOB_KUBE_RUNTIME_CLASS_NAME + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DISCOVER_JOB_KUBE_RUNTIME_CLASS_NAME +{{- end }} + +{{/* +Renders the global.jobs.kube.scheduling.isolated.nodeSelectors value +*/}} +{{- define "airbyte.jobs.scheduling.isolated.nodeSelectors" }} + {{- .Values.global.jobs.kube.scheduling.isolated.nodeSelectors | include "airbyte.flattenMap" }} +{{- end }} + +{{/* +Renders the jobs.scheduling.isolated.nodeSelectors environment variable +*/}} +{{- define "airbyte.jobs.scheduling.isolated.nodeSelectors.env" }} +- name: JOB_ISOLATED_KUBE_NODE_SELECTORS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_ISOLATED_KUBE_NODE_SELECTORS +{{- end }} + +{{/* +Renders the global.jobs.kube.scheduling.isolated.runtimeClassName value +*/}} +{{- define "airbyte.jobs.scheduling.isolated.runtimeClassName" }} + {{- .Values.global.jobs.kube.scheduling.isolated.runtimeClassName }} +{{- end }} + +{{/* +Renders the jobs.scheduling.isolated.runtimeClassName environment variable +*/}} +{{- define "airbyte.jobs.scheduling.isolated.runtimeClassName.env" }} +- name: JOB_ISOLATED_KUBE_RUNTIME_CLASS_NAME + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_ISOLATED_KUBE_RUNTIME_CLASS_NAME +{{- end }} + +{{/* +Renders the global.jobs.kube.scheduling.sourceDeclarativeManifest.nodeSelectors value +*/}} +{{- define "airbyte.jobs.scheduling.sourceDeclarativeManifest.nodeSelectors" }} + {{- .Values.global.jobs.kube.scheduling.sourceDeclarativeManifest.nodeSelectors | include "airbyte.flattenMap" }} +{{- end }} + +{{/* +Renders the jobs.scheduling.sourceDeclarativeManifest.nodeSelectors environment variable +*/}} +{{- define "airbyte.jobs.scheduling.sourceDeclarativeManifest.nodeSelectors.env" }} +- name: JOB_SOURCE_DECLARATIVE_MANIFEST_KUBE_NODE_SELECTORS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_SOURCE_DECLARATIVE_MANIFEST_KUBE_NODE_SELECTORS +{{- end }} + +{{/* +Renders the global.jobs.kube.scheduling.sourceDeclarativeManifest.runtimeClassName value +*/}} +{{- define "airbyte.jobs.scheduling.sourceDeclarativeManifest.runtimeClassName" }} + {{- .Values.global.jobs.kube.scheduling.sourceDeclarativeManifest.runtimeClassName }} +{{- end }} + +{{/* +Renders the jobs.scheduling.sourceDeclarativeManifest.runtimeClassName environment variable +*/}} +{{- define "airbyte.jobs.scheduling.sourceDeclarativeManifest.runtimeClassName.env" }} +- name: JOB_SOURCE_DECLARATIVE_MANIFEST_KUBE_RUNTIME_CLASS_NAME + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_SOURCE_DECLARATIVE_MANIFEST_KUBE_RUNTIME_CLASS_NAME +{{- end }} + +{{/* +Renders the set of all jobs.scheduling environment variables +*/}} +{{- define "airbyte.jobs.scheduling.envs" }} +{{- include "airbyte.jobs.scheduling.check.nodeSelectors.env" . }} +{{- include "airbyte.jobs.scheduling.check.runtimeClassName.env" . }} +{{- include "airbyte.jobs.scheduling.discover.nodeSelectors.env" . }} +{{- include "airbyte.jobs.scheduling.discover.runtimeClassName.env" . }} +{{- include "airbyte.jobs.scheduling.isolated.nodeSelectors.env" . }} +{{- include "airbyte.jobs.scheduling.isolated.runtimeClassName.env" . }} +{{- include "airbyte.jobs.scheduling.sourceDeclarativeManifest.nodeSelectors.env" . }} +{{- include "airbyte.jobs.scheduling.sourceDeclarativeManifest.runtimeClassName.env" . }} +{{- end }} + +{{/* +Renders the set of all jobs.scheduling config map variables +*/}} +{{- define "airbyte.jobs.scheduling.configVars" }} +CHECK_JOB_KUBE_NODE_SELECTORS: {{ include "airbyte.jobs.scheduling.check.nodeSelectors" . | quote }} +CHECK_JOB_KUBE_RUNTIME_CLASS_NAME: {{ include "airbyte.jobs.scheduling.check.runtimeClassName" . | quote }} +DISCOVER_JOB_KUBE_NODE_SELECTORS: {{ include "airbyte.jobs.scheduling.discover.nodeSelectors" . | quote }} +DISCOVER_JOB_KUBE_RUNTIME_CLASS_NAME: {{ include "airbyte.jobs.scheduling.discover.runtimeClassName" . | quote }} +JOB_ISOLATED_KUBE_NODE_SELECTORS: {{ include "airbyte.jobs.scheduling.isolated.nodeSelectors" . | quote }} +JOB_ISOLATED_KUBE_RUNTIME_CLASS_NAME: {{ include "airbyte.jobs.scheduling.isolated.runtimeClassName" . | quote }} +JOB_SOURCE_DECLARATIVE_MANIFEST_KUBE_NODE_SELECTORS: {{ include "airbyte.jobs.scheduling.sourceDeclarativeManifest.nodeSelectors" . | quote }} +JOB_SOURCE_DECLARATIVE_MANIFEST_KUBE_RUNTIME_CLASS_NAME: {{ include "airbyte.jobs.scheduling.sourceDeclarativeManifest.runtimeClassName" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_keycloak.tpl b/helm/airbyte-v2/templates/config/_keycloak.tpl new file mode 100644 index 00000000..cc41ab80 --- /dev/null +++ b/helm/airbyte-v2/templates/config/_keycloak.tpl @@ -0,0 +1,546 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Keycloak Configuration +*/}} + +{{/* +Renders the keycloak.url value +*/}} +{{- define "airbyte.keycloak.url" }} + {{- ternary (printf "%s/auth" .Values.global.airbyteUrl) "" (ne .Values.global.edition "enterprise") }} +{{- end }} + +{{/* +Renders the keycloak.url environment variable +*/}} +{{- define "airbyte.keycloak.url.env" }} +- name: KEYCLOAK_HOSTNAME_URL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: KEYCLOAK_HOSTNAME_URL +{{- end }} + +{{/* +Renders the keycloak.host value +*/}} +{{- define "airbyte.keycloak.host" }} + {{- (get (urlParse .Values.global.airbyteUrl) "host") }} +{{- end }} + +{{/* +Renders the keycloak.host environment variable +*/}} +{{- define "airbyte.keycloak.host.env" }} +- name: KEYCLOAK_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: KEYCLOAK_HOST +{{- end }} + +{{/* +Renders the keycloak.service.port value +*/}} +{{- define "airbyte.keycloak.service.port" }} + {{- .Values.keycloak.service.port | default 8180 }} +{{- end }} + +{{/* +Renders the keycloak.service.port environment variable +*/}} +{{- define "airbyte.keycloak.service.port.env" }} +- name: KEYCLOAK_PORT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: KEYCLOAK_PORT +{{- end }} + +{{/* +Renders the keycloak.protocol value +*/}} +{{- define "airbyte.keycloak.protocol" }} + {{- .Values.keycloak.protocol | default "https" }} +{{- end }} + +{{/* +Renders the keycloak.protocol environment variable +*/}} +{{- define "airbyte.keycloak.protocol.env" }} +- name: KEYCLOAK_PROTOCOL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: KEYCLOAK_PROTOCOL +{{- end }} + +{{/* +Renders the keycloak.javaOpts value +*/}} +{{- define "airbyte.keycloak.javaOpts" }} + {{- (printf "-Djgroups.dns.query=%s-airbyte-keycloak-headless-svc" .Release.Name) }} +{{- end }} + +{{/* +Renders the keycloak.javaOpts environment variable +*/}} +{{- define "airbyte.keycloak.javaOpts.env" }} +- name: JAVA_OPTS_APPEND + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JAVA_OPTS_APPEND +{{- end }} + +{{/* +Renders the set of all keycloak environment variables +*/}} +{{- define "airbyte.keycloak.envs" }} +{{- include "airbyte.keycloak.url.env" . }} +{{- include "airbyte.keycloak.host.env" . }} +{{- include "airbyte.keycloak.service.port.env" . }} +{{- include "airbyte.keycloak.protocol.env" . }} +{{- include "airbyte.keycloak.javaOpts.env" . }} +{{- end }} + +{{/* +Renders the set of all keycloak config map variables +*/}} +{{- define "airbyte.keycloak.configVars" }} +KEYCLOAK_HOSTNAME_URL: {{ include "airbyte.keycloak.url" . | quote }} +KEYCLOAK_HOST: {{ include "airbyte.keycloak.host" . | quote }} +KEYCLOAK_PORT: {{ include "airbyte.keycloak.service.port" . | quote }} +KEYCLOAK_PROTOCOL: {{ include "airbyte.keycloak.protocol" . | quote }} +JAVA_OPTS_APPEND: {{ include "airbyte.keycloak.javaOpts" . | quote }} +{{- end }} + +{{/* +Renders the keycloak.auth.adminRealm value +*/}} +{{- define "airbyte.keycloak.admin.client.auth.adminRealm" }} + {{- .Values.keycloak.auth.adminRealm }} +{{- end }} + +{{/* +Renders the keycloak.admin.client.auth.adminRealm environment variable +*/}} +{{- define "airbyte.keycloak.admin.client.auth.adminRealm.env" }} +- name: KEYCLOAK_ADMIN_REALM + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: KEYCLOAK_ADMIN_REALM +{{- end }} + +{{/* +Renders the keycloak.auth.adminCliClientId value +*/}} +{{- define "airbyte.keycloak.admin.client.auth.adminCliClientId" }} + {{- .Values.keycloak.auth.adminCliClientId | default "admin-cli" }} +{{- end }} + +{{/* +Renders the keycloak.admin.client.auth.adminCliClientId environment variable +*/}} +{{- define "airbyte.keycloak.admin.client.auth.adminCliClientId.env" }} +- name: KEYCLOAK_ADMIN_CLI_CLIENT_ID + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: KEYCLOAK_ADMIN_CLI_CLIENT_ID +{{- end }} + +{{/* +Renders the keycloak.clientRealm value +*/}} +{{- define "airbyte.keycloak.admin.client.clientRealm" }} + {{- .Values.keycloak.clientRealm | default "_airbyte-application-clients" }} +{{- end }} + +{{/* +Renders the keycloak.admin.client.clientRealm environment variable +*/}} +{{- define "airbyte.keycloak.admin.client.clientRealm.env" }} +- name: KEYCLOAK_CLIENT_REALM + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: KEYCLOAK_CLIENT_REALM +{{- end }} + +{{/* +Renders the keycloak.internalBasePath value +*/}} +{{- define "airbyte.keycloak.admin.client.internalBasePath" }} + {{- .Values.keycloak.internalBasePath | default "/auth" }} +{{- end }} + +{{/* +Renders the keycloak.admin.client.internalBasePath environment variable +*/}} +{{- define "airbyte.keycloak.admin.client.internalBasePath.env" }} +- name: KEYCLOAK_INTERNAL_BASE_PATH + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: KEYCLOAK_INTERNAL_BASE_PATH +{{- end }} + +{{/* +Renders the keycloak.internalHost value +*/}} +{{- define "airbyte.keycloak.admin.client.internalHost" }} + {{- .Values.keycloak.internalHost | default (ternary (printf "%s-airbyte-keycloak-svc.%s:%d" .Release.Name .Release.Namespace (int .Values.keycloak.service.port)) "localhost" (or (eq .Values.global.edition "enterprise") (ne .Values.global.edition "community"))) }} +{{- end }} + +{{/* +Renders the keycloak.admin.client.internalHost environment variable +*/}} +{{- define "airbyte.keycloak.admin.client.internalHost.env" }} +- name: KEYCLOAK_INTERNAL_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: KEYCLOAK_INTERNAL_HOST +{{- end }} + +{{/* +Renders the keycloak.internalProtocol value +*/}} +{{- define "airbyte.keycloak.admin.client.internalProtocol" }} + {{- .Values.keycloak.internalProtocol | default "http" }} +{{- end }} + +{{/* +Renders the keycloak.admin.client.internalProtocol environment variable +*/}} +{{- define "airbyte.keycloak.admin.client.internalProtocol.env" }} +- name: KEYCLOAK_INTERNAL_PROTOCOL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: KEYCLOAK_INTERNAL_PROTOCOL +{{- end }} + +{{/* +Renders the keycloak.internalRealm value +*/}} +{{- define "airbyte.keycloak.admin.client.internalRealm" }} + {{- .Values.keycloak.internalRealm | default "_airbyte-internal" }} +{{- end }} + +{{/* +Renders the keycloak.admin.client.internalRealm environment variable +*/}} +{{- define "airbyte.keycloak.admin.client.internalRealm.env" }} +- name: KEYCLOAK_INTERNAL_REALM + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: KEYCLOAK_INTERNAL_REALM +{{- end }} + +{{/* +Renders the set of all keycloak.admin.client environment variables +*/}} +{{- define "airbyte.keycloak.admin.client.envs" }} +{{- include "airbyte.keycloak.admin.client.auth.adminRealm.env" . }} +{{- include "airbyte.keycloak.admin.client.auth.adminCliClientId.env" . }} +{{- include "airbyte.keycloak.admin.client.clientRealm.env" . }} +{{- include "airbyte.keycloak.admin.client.internalBasePath.env" . }} +{{- include "airbyte.keycloak.admin.client.internalHost.env" . }} +{{- include "airbyte.keycloak.admin.client.internalProtocol.env" . }} +{{- include "airbyte.keycloak.admin.client.internalRealm.env" . }} +{{- end }} + +{{/* +Renders the set of all keycloak.admin.client config map variables +*/}} +{{- define "airbyte.keycloak.admin.client.configVars" }} +KEYCLOAK_ADMIN_REALM: {{ include "airbyte.keycloak.admin.client.auth.adminRealm" . | quote }} +KEYCLOAK_ADMIN_CLI_CLIENT_ID: {{ include "airbyte.keycloak.admin.client.auth.adminCliClientId" . | quote }} +KEYCLOAK_CLIENT_REALM: {{ include "airbyte.keycloak.admin.client.clientRealm" . | quote }} +KEYCLOAK_INTERNAL_BASE_PATH: {{ include "airbyte.keycloak.admin.client.internalBasePath" . | quote }} +KEYCLOAK_INTERNAL_HOST: {{ include "airbyte.keycloak.admin.client.internalHost" . | quote }} +KEYCLOAK_INTERNAL_PROTOCOL: {{ include "airbyte.keycloak.admin.client.internalProtocol" . | quote }} +KEYCLOAK_INTERNAL_REALM: {{ include "airbyte.keycloak.admin.client.internalRealm" . | quote }} +{{- end }} + +{{/* +Renders the keycloak.admin.user secret name +*/}} +{{- define "airbyte.keycloak.admin.user.secretName" }} +{{- if .Values.keycloak.secretName }} + {{- .Values.keycloak.secretName }} +{{- else }} + {{- .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} +{{- end }} +{{- end }} + +{{/* +Renders the keycloak.auth.adminUsername value. +When global.secretSeed is set, default is env-specific; else no default (must set in values for enterprise). +*/}} +{{- define "airbyte.keycloak.admin.user.auth.adminUsername" }} + {{- .Values.keycloak.auth.adminUsername | default (ternary (printf "keycloak-%s" (include "airbyte.secretSeed.hashBase" . | sha256sum | nospace | trunc 12)) "" (ne (toString .Values.global.secretSeed) "")) }} +{{- end }} + +{{/* +Renders the keycloak.admin.user.auth.adminUsername secret key +*/}} +{{- define "airbyte.keycloak.admin.user.auth.adminUsername.secretKey" }} + {{- .Values.keycloak.auth.adminUsernameSecretKey | default "KEYCLOAK_ADMIN_USER" }} +{{- end }} + +{{/* +Renders the keycloak.admin.user.auth.adminUsername environment variable +*/}} +{{- define "airbyte.keycloak.admin.user.auth.adminUsername.env" }} +- name: KEYCLOAK_ADMIN_USER + valueFrom: + secretKeyRef: + name: {{ include "airbyte.keycloak.admin.user.secretName" . }} + key: {{ include "airbyte.keycloak.admin.user.auth.adminUsername.secretKey" . }} +{{- end }} + +{{/* +Renders the keycloak.auth.adminPassword value. +When global.secretSeed is set, default is env-specific; else no default. +*/}} +{{- define "airbyte.keycloak.admin.user.auth.adminPassword" }} + {{- .Values.keycloak.auth.adminPassword | default (ternary (printf "kc-%s" (include "airbyte.secretSeed.hashBase" . | sha256sum | nospace | trunc 24)) "" (ne (toString .Values.global.secretSeed) "")) }} +{{- end }} + +{{/* +Renders the keycloak.admin.user.auth.adminPassword secret key +*/}} +{{- define "airbyte.keycloak.admin.user.auth.adminPassword.secretKey" }} + {{- .Values.keycloak.auth.adminPasswordSecretKey | default "KEYCLOAK_ADMIN_PASSWORD" }} +{{- end }} + +{{/* +Renders the keycloak.admin.user.auth.adminPassword environment variable +*/}} +{{- define "airbyte.keycloak.admin.user.auth.adminPassword.env" }} +- name: KEYCLOAK_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "airbyte.keycloak.admin.user.secretName" . }} + key: {{ include "airbyte.keycloak.admin.user.auth.adminPassword.secretKey" . }} +{{- end }} + +{{/* +Renders the set of all keycloak.admin.user environment variables +*/}} +{{- define "airbyte.keycloak.admin.user.envs" }} +{{- include "airbyte.keycloak.admin.user.auth.adminUsername.env" . }} +{{- include "airbyte.keycloak.admin.user.auth.adminPassword.env" . }} +{{- end }} + +{{/* +Renders the set of all keycloak.admin.user secret variables +*/}} +{{- define "airbyte.keycloak.admin.user.secrets" }} +KEYCLOAK_ADMIN_USER: {{ include "airbyte.keycloak.admin.user.auth.adminUsername" . | quote }} +KEYCLOAK_ADMIN_PASSWORD: {{ include "airbyte.keycloak.admin.user.auth.adminPassword" . | quote }} +{{- end }} + +{{/* +Renders the keycloak.realmIssuer value +*/}} +{{- define "airbyte.keycloak.client.realmIssuer" }} + {{- .Values.keycloak.realmIssuer | default (ternary (printf "%s/auth/realms/%s" .Values.global.airbyteUrl (include "airbyte.keycloak.admin.client.internalRealm" .)) (printf "%s-airbyte-keycloak-svc.%s:%d/auth/realms/%s" .Release.Name .Release.Namespace (int (include "airbyte.keycloak.service.port" .)) (include "airbyte.keycloak.admin.client.internalRealm" .)) (eq (include "airbyte.common.cluster.type" .) "data-plane")) }} +{{- end }} + +{{/* +Renders the keycloak.client.realmIssuer environment variable +*/}} +{{- define "airbyte.keycloak.client.realmIssuer.env" }} +- name: KEYCLOAK_INTERNAL_REALM_ISSUER + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: KEYCLOAK_INTERNAL_REALM_ISSUER +{{- end }} + +{{/* +Renders the set of all keycloak.client environment variables +*/}} +{{- define "airbyte.keycloak.client.envs" }} +{{- include "airbyte.keycloak.client.realmIssuer.env" . }} +{{- end }} + +{{/* +Renders the set of all keycloak.client config map variables +*/}} +{{- define "airbyte.keycloak.client.configVars" }} +KEYCLOAK_INTERNAL_REALM_ISSUER: {{ include "airbyte.keycloak.client.realmIssuer" . | quote }} +{{- end }} + +{{/* +Renders the keycloak.database secret name +*/}} +{{- define "airbyte.keycloak.database.secretName" }} +{{- if .Values.keycloak.database.secretName }} + {{- .Values.keycloak.database.secretName }} +{{- else }} + {{- .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} +{{- end }} +{{- end }} + +{{/* +Renders the keycloak.database.name value. default to global.database.name so Keycloak uses the same DB. +*/}} +{{- define "airbyte.keycloak.database.name" }} + {{- .Values.keycloak.database.name | default (include "airbyte.database.name" .) }} +{{- end }} + +{{/* +Renders the keycloak.database.name environment variable +*/}} +{{- define "airbyte.keycloak.database.name.env" }} +- name: KEYCLOAK_DATABASE_NAME + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: KEYCLOAK_DATABASE_NAME +{{- end }} + +{{/* +Renders the keycloak.database.host value. default to global.database.host so Keycloak uses the same DB. +*/}} +{{- define "airbyte.keycloak.database.host" }} + {{- .Values.keycloak.database.host | default (include "airbyte.database.host" .) }} +{{- end }} + +{{/* +Renders the keycloak.database.host environment variable +*/}} +{{- define "airbyte.keycloak.database.host.env" }} +- name: KEYCLOAK_DATABASE_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: KEYCLOAK_DATABASE_HOST +{{- end }} + +{{/* +Renders the keycloak.database.port value. default to global.database.port. +*/}} +{{- define "airbyte.keycloak.database.port" }} + {{- .Values.keycloak.database.port | default (include "airbyte.database.port" .) }} +{{- end }} + +{{/* +Renders the keycloak.database.port environment variable +*/}} +{{- define "airbyte.keycloak.database.port.env" }} +- name: KEYCLOAK_DATABASE_PORT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: KEYCLOAK_DATABASE_PORT +{{- end }} + +{{/* +Renders the keycloak.database.user value. +When global.secretSeed is set, default is env-specific; else "airbyte". +*/}} +{{- define "airbyte.keycloak.database.user" }} + {{- .Values.keycloak.database.user | default (ternary (printf "kcdb-%s" (include "airbyte.secretSeed.hashBase" . | sha256sum | nospace | trunc 12)) "airbyte" (ne (toString .Values.global.secretSeed) "")) }} +{{- end }} + +{{/* +Renders the keycloak.database.user secret key +*/}} +{{- define "airbyte.keycloak.database.user.secretKey" }} + {{- .Values.keycloak.database.userSecretKey | default "KEYCLOAK_DATABASE_USERNAME" }} +{{- end }} + +{{/* +Renders the keycloak.database.user environment variable +*/}} +{{- define "airbyte.keycloak.database.user.env" }} +- name: KEYCLOAK_DATABASE_USERNAME + valueFrom: + secretKeyRef: + name: {{ include "airbyte.keycloak.database.secretName" . }} + key: {{ include "airbyte.keycloak.database.user.secretKey" . }} +{{- end }} + +{{/* +Renders the keycloak.database.password value. +When global.secretSeed is set, default is env-specific; else "airbyte". +*/}} +{{- define "airbyte.keycloak.database.password" }} + {{- .Values.keycloak.database.password | default (ternary (printf "kcdb-%s" (include "airbyte.secretSeed.hashBase" . | sha256sum | nospace | trunc 24)) "airbyte" (ne (toString .Values.global.secretSeed) "")) }} +{{- end }} + +{{/* +Renders the keycloak.database.password secret key +*/}} +{{- define "airbyte.keycloak.database.password.secretKey" }} + {{- .Values.keycloak.database.passwordSecretKey | default "KEYCLOAK_DATABASE_PASSWORD" }} +{{- end }} + +{{/* +Renders the keycloak.database.password environment variable +*/}} +{{- define "airbyte.keycloak.database.password.env" }} +- name: KEYCLOAK_DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "airbyte.keycloak.database.secretName" . }} + key: {{ include "airbyte.keycloak.database.password.secretKey" . }} +{{- end }} + +{{/* +Renders the keycloak.database.url value.use global.database (host, port, name) so Keycloak uses the same DB as Airbyte with schema keycloak. +*/}} +{{- define "airbyte.keycloak.database.url" }} + {{- (printf "jdbc:postgresql://%s:%s/%s?currentSchema=keycloak" (include "airbyte.database.host" .) (include "airbyte.database.port" .) (include "airbyte.database.name" .)) }} +{{- end }} + +{{/* +Renders the keycloak.database.url environment variable +*/}} +{{- define "airbyte.keycloak.database.url.env" }} +- name: KEYCLOAK_DATABASE_URL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: KEYCLOAK_DATABASE_URL +{{- end }} + +{{/* +Renders the set of all keycloak.database environment variables +*/}} +{{- define "airbyte.keycloak.database.envs" }} +{{- include "airbyte.keycloak.database.name.env" . }} +{{- include "airbyte.keycloak.database.host.env" . }} +{{- include "airbyte.keycloak.database.port.env" . }} +{{- include "airbyte.keycloak.database.user.env" . }} +{{- include "airbyte.keycloak.database.password.env" . }} +{{- include "airbyte.keycloak.database.url.env" . }} +{{- end }} + +{{/* +Renders the set of all keycloak.database config map variables +*/}} +{{- define "airbyte.keycloak.database.configVars" }} +KEYCLOAK_DATABASE_NAME: {{ include "airbyte.keycloak.database.name" . | quote }} +KEYCLOAK_DATABASE_HOST: {{ include "airbyte.keycloak.database.host" . | quote }} +KEYCLOAK_DATABASE_PORT: {{ include "airbyte.keycloak.database.port" . | quote }} +KEYCLOAK_DATABASE_URL: {{ include "airbyte.keycloak.database.url" . | quote }} +{{- end }} + +{{/* +Renders the set of all keycloak.database secret variables +*/}} +{{- define "airbyte.keycloak.database.secrets" }} +KEYCLOAK_DATABASE_USERNAME: {{ include "airbyte.keycloak.database.user" . | quote }} +KEYCLOAK_DATABASE_PASSWORD: {{ include "airbyte.keycloak.database.password" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_logging.tpl b/helm/airbyte-v2/templates/config/_logging.tpl new file mode 100644 index 00000000..e8d4a8d2 --- /dev/null +++ b/helm/airbyte-v2/templates/config/_logging.tpl @@ -0,0 +1,38 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Logging Configuration +*/}} + +{{/* +Renders the global.logging.level value +*/}} +{{- define "airbyte.logging.level" }} + {{- .Values.global.logging.level | default "INFO" }} +{{- end }} + +{{/* +Renders the logging.level environment variable +*/}} +{{- define "airbyte.logging.level.env" }} +- name: LOG_LEVEL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: LOG_LEVEL +{{- end }} + +{{/* +Renders the set of all logging environment variables +*/}} +{{- define "airbyte.logging.envs" }} +{{- include "airbyte.logging.level.env" . }} +{{- end }} + +{{/* +Renders the set of all logging config map variables +*/}} +{{- define "airbyte.logging.configVars" }} +LOG_LEVEL: {{ include "airbyte.logging.level" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_metrics.tpl b/helm/airbyte-v2/templates/config/_metrics.tpl new file mode 100644 index 00000000..36e5248d --- /dev/null +++ b/helm/airbyte-v2/templates/config/_metrics.tpl @@ -0,0 +1,214 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Metrics Configuration +*/}} + +{{/* +Renders the global.metrics.enabled value +*/}} +{{- define "airbyte.metrics.enabled" }} + {{- if eq .Values.global.metrics.enabled nil }} + {{- false }} + {{- else }} + {{- .Values.global.metrics.enabled }} + {{- end }} +{{- end }} + +{{/* +Renders the metrics.enabled environment variable +*/}} +{{- define "airbyte.metrics.enabled.env" }} +- name: MICROMETER_METRICS_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MICROMETER_METRICS_ENABLED +{{- end }} + +{{/* +Renders the global.metrics.step value +*/}} +{{- define "airbyte.metrics.step" }} + {{- .Values.global.metrics.step | default "PT1M" }} +{{- end }} + +{{/* +Renders the metrics.step environment variable +*/}} +{{- define "airbyte.metrics.step.env" }} +- name: MICROMETER_METRICS_STEP + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MICROMETER_METRICS_STEP +{{- end }} + +{{/* +Renders the global.metrics.statsd.flavor value +*/}} +{{- define "airbyte.metrics.statsd.flavor" }} + {{- .Values.global.metrics.statsd.flavor | default "datadog" }} +{{- end }} + +{{/* +Renders the metrics.statsd.flavor environment variable +*/}} +{{- define "airbyte.metrics.statsd.flavor.env" }} +- name: MICROMETER_METRICS_STATSD_FLAVOR + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MICROMETER_METRICS_STATSD_FLAVOR +{{- end }} + +{{/* +Renders the global.metrics.otlp.enabled value +*/}} +{{- define "airbyte.metrics.otlp.enabled" }} + {{- if eq .Values.global.metrics.otlp.enabled nil }} + {{- false }} + {{- else }} + {{- .Values.global.metrics.otlp.enabled }} + {{- end }} +{{- end }} + +{{/* +Renders the metrics.otlp.enabled environment variable +*/}} +{{- define "airbyte.metrics.otlp.enabled.env" }} +- name: MICROMETER_METRICS_OTLP_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MICROMETER_METRICS_OTLP_ENABLED +{{- end }} + +{{/* +Renders the global.metrics.statsd.enabled value +*/}} +{{- define "airbyte.metrics.statsd.enabled" }} + {{- if eq .Values.global.metrics.statsd.enabled nil }} + {{- false }} + {{- else }} + {{- .Values.global.metrics.statsd.enabled }} + {{- end }} +{{- end }} + +{{/* +Renders the metrics.statsd.enabled environment variable +*/}} +{{- define "airbyte.metrics.statsd.enabled.env" }} +- name: MICROMETER_METRICS_STATSD_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MICROMETER_METRICS_STATSD_ENABLED +{{- end }} + +{{/* +Renders the global.metrics.otlp.collectorEndpoint value (non-empty default to avoid "Unsetting empty" OTEL_COLLECTOR_ENDPOINT). +*/}} +{{- define "airbyte.metrics.otlp.collectorEndpoint" }} + {{- .Values.global.metrics.otlp.collectorEndpoint | default "http://localhost:4317" }} +{{- end }} + +{{/* +Renders the metrics.otlp.collectorEndpoint environment variable +*/}} +{{- define "airbyte.metrics.otlp.collectorEndpoint.env" }} +- name: OTEL_COLLECTOR_ENDPOINT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: OTEL_COLLECTOR_ENDPOINT +{{- end }} + +{{/* +Renders the global.metrics.publish value +*/}} +{{- define "airbyte.metrics.publish" }} + {{- if eq .Values.global.metrics.publish nil }} + {{- true }} + {{- else }} + {{- .Values.global.metrics.publish }} + {{- end }} +{{- end }} + +{{/* +Renders the metrics.publish environment variable +*/}} +{{- define "airbyte.metrics.publish.env" }} +- name: PUBLISH_METRICS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: PUBLISH_METRICS +{{- end }} + +{{/* +Renders the global.metrics.statsd.host value +*/}} +{{- define "airbyte.metrics.statsd.host" }} + {{- .Values.global.metrics.statsd.host }} +{{- end }} + +{{/* +Renders the metrics.statsd.host environment variable +*/}} +{{- define "airbyte.metrics.statsd.host.env" }} +- name: STATSD_HOST + valueFrom: + fieldRef: + fieldPath: status.hostIP + +{{- end }} + +{{/* +Renders the global.metrics.statsd.port value +*/}} +{{- define "airbyte.metrics.statsd.port" }} + {{- .Values.global.metrics.statsd.port | default 8125 }} +{{- end }} + +{{/* +Renders the metrics.statsd.port environment variable +*/}} +{{- define "airbyte.metrics.statsd.port.env" }} +- name: STATSD_PORT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: STATSD_PORT +{{- end }} + +{{/* +Renders the set of all metrics environment variables +*/}} +{{- define "airbyte.metrics.envs" }} +{{- include "airbyte.metrics.enabled.env" . }} +{{- include "airbyte.metrics.step.env" . }} +{{- include "airbyte.metrics.statsd.flavor.env" . }} +{{- include "airbyte.metrics.otlp.enabled.env" . }} +{{- include "airbyte.metrics.statsd.enabled.env" . }} +{{- include "airbyte.metrics.otlp.collectorEndpoint.env" . }} +{{- include "airbyte.metrics.publish.env" . }} +{{- include "airbyte.metrics.statsd.host.env" . }} +{{- include "airbyte.metrics.statsd.port.env" . }} +{{- end }} + +{{/* +Renders the set of all metrics config map variables +*/}} +{{- define "airbyte.metrics.configVars" }} +MICROMETER_METRICS_ENABLED: {{ include "airbyte.metrics.enabled" . | quote }} +MICROMETER_METRICS_STEP: {{ include "airbyte.metrics.step" . | quote }} +MICROMETER_METRICS_STATSD_FLAVOR: {{ include "airbyte.metrics.statsd.flavor" . | quote }} +MICROMETER_METRICS_OTLP_ENABLED: {{ include "airbyte.metrics.otlp.enabled" . | quote }} +MICROMETER_METRICS_STATSD_ENABLED: {{ include "airbyte.metrics.statsd.enabled" . | quote }} +OTEL_COLLECTOR_ENDPOINT: {{ include "airbyte.metrics.otlp.collectorEndpoint" . | quote }} +PUBLISH_METRICS: {{ include "airbyte.metrics.publish" . | quote }} +STATSD_HOST: {{ include "airbyte.metrics.statsd.host" . | quote }} +STATSD_PORT: {{ include "airbyte.metrics.statsd.port" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_micronaut.tpl b/helm/airbyte-v2/templates/config/_micronaut.tpl new file mode 100644 index 00000000..48073b16 --- /dev/null +++ b/helm/airbyte-v2/templates/config/_micronaut.tpl @@ -0,0 +1,43 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Micronaut Configuration +*/}} + +{{/* +Renders the global.micronaut.environments value +*/}} +{{- define "airbyte.micronaut.environments" }} + {{- join "," ( + concat .Values.global.micronaut.environments + (list (ternary "control-plane" (include "airbyte.common.cluster.type" .) (eq (include "airbyte.common.cluster.type" .) "hybrid"))) + (list (ternary "edition-community" (include "airbyte.auth.identityProvider.type" .) (eq (include "airbyte.auth.identityProvider.type" .) "simple"))) + ) + }} +{{- end }} + +{{/* +Renders the micronaut.environments environment variable +*/}} +{{- define "airbyte.micronaut.environments.env" }} +- name: MICRONAUT_ENVIRONMENTS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MICRONAUT_ENVIRONMENTS +{{- end }} + +{{/* +Renders the set of all micronaut environment variables +*/}} +{{- define "airbyte.micronaut.envs" }} +{{- include "airbyte.micronaut.environments.env" . }} +{{- end }} + +{{/* +Renders the set of all micronaut config map variables +*/}} +{{- define "airbyte.micronaut.configVars" }} +MICRONAUT_ENVIRONMENTS: {{ include "airbyte.micronaut.environments" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_minio.tpl b/helm/airbyte-v2/templates/config/_minio.tpl new file mode 100644 index 00000000..dd65a149 --- /dev/null +++ b/helm/airbyte-v2/templates/config/_minio.tpl @@ -0,0 +1,50 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Minio Configuration +*/}} + +{{/* +Renders the minio.rootUser secret key +*/}} +{{- define "airbyte.minio.rootUser.secretKey" }} + {{- .Values.minio.rootUserSecretKey | default "MINIO_ROOT_USER" }} +{{- end }} + +{{/* +Renders the minio.rootUser environment variable +*/}} +{{- define "airbyte.minio.rootUser.env" }} +- name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: {{ include "airbyte.storage.secretName" . }} + key: {{ include "airbyte.minio.rootUser.secretKey" . }} +{{- end }} + +{{/* +Renders the minio.rootPassword secret key +*/}} +{{- define "airbyte.minio.rootPassword.secretKey" }} + {{- .Values.minio.rootPasswordSecretKey | default "MINIO_ROOT_PASSWORD" }} +{{- end }} + +{{/* +Renders the minio.rootPassword environment variable +*/}} +{{- define "airbyte.minio.rootPassword.env" }} +- name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "airbyte.storage.secretName" . }} + key: {{ include "airbyte.minio.rootPassword.secretKey" . }} +{{- end }} + +{{/* +Renders the set of all minio environment variables +*/}} +{{- define "airbyte.minio.envs" }} +{{- include "airbyte.minio.rootUser.env" . }} +{{- include "airbyte.minio.rootPassword.env" . }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_otel.tpl b/helm/airbyte-v2/templates/config/_otel.tpl new file mode 100644 index 00000000..2cca7175 --- /dev/null +++ b/helm/airbyte-v2/templates/config/_otel.tpl @@ -0,0 +1,138 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Otel Configuration +*/}} + +{{/* +Renders the global.metrics.otel.exporter.endpoint value +*/}} +{{- define "airbyte.otel.exporter.endpoint" }} + {{- .Values.global.metrics.otel.exporter.endpoint | default (ternary "http://${DD_AGENT_HOST}:4317" "" .Values.global.datadog.enabled) }} +{{- end }} + +{{/* +Renders the otel.exporter.endpoint environment variable +*/}} +{{- define "airbyte.otel.exporter.endpoint.env" }} +- name: OTEL_EXPORTER_OTLP_ENDPOINT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: OTEL_EXPORTER_OTLP_ENDPOINT +{{- end }} + +{{/* +Renders the global.metrics.otel.exporter.protocol value +*/}} +{{- define "airbyte.otel.exporter.protocol" }} + {{- .Values.global.metrics.otel.exporter.protocol | default "grpc" }} +{{- end }} + +{{/* +Renders the otel.exporter.protocol environment variable +*/}} +{{- define "airbyte.otel.exporter.protocol.env" }} +- name: OTEL_EXPORTER_OTLP_PROTOCOL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: OTEL_EXPORTER_OTLP_PROTOCOL +{{- end }} + +{{/* +Renders the global.metrics.otel.exporter.timeout value +*/}} +{{- define "airbyte.otel.exporter.timeout" }} + {{- .Values.global.metrics.otel.exporter.timeout | default 30000 }} +{{- end }} + +{{/* +Renders the otel.exporter.timeout environment variable +*/}} +{{- define "airbyte.otel.exporter.timeout.env" }} +- name: OTEL_EXPORTER_OTLP_TIMEOUT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: OTEL_EXPORTER_OTLP_TIMEOUT +{{- end }} + +{{/* +Renders the global.metrics.otel.exporter.metricExportInterval value +*/}} +{{- define "airbyte.otel.exporter.metricExportInterval" }} + {{- .Values.global.metrics.otel.exporter.metricExportInterval | default 10000 }} +{{- end }} + +{{/* +Renders the otel.exporter.metricExportInterval environment variable +*/}} +{{- define "airbyte.otel.exporter.metricExportInterval.env" }} +- name: OTEL_METRIC_EXPORT_INTERVAL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: OTEL_METRIC_EXPORT_INTERVAL +{{- end }} + +{{/* +Renders the global.metrics.otel.exporter.name value +*/}} +{{- define "airbyte.otel.exporter.name" }} + {{- .Values.global.metrics.otel.exporter.name | default "otlp" }} +{{- end }} + +{{/* +Renders the otel.exporter.name environment variable +*/}} +{{- define "airbyte.otel.exporter.name.env" }} +- name: OTEL_METRICS_EXPORTER + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: OTEL_METRICS_EXPORTER +{{- end }} + +{{/* +Renders the global.metrics.otel.resourceAttributes value +*/}} +{{- define "airbyte.otel.resourceAttributes" }} + {{- .Values.global.metrics.otel.resourceAttributes | default (printf "service.name=%s,deployment.environment=%s,service.version=%s" (include "airbyte.componentName" .) .Values.global.env (include "airbyte.common.version" .)) }} +{{- end }} + +{{/* +Renders the otel.resourceAttributes environment variable +*/}} +{{- define "airbyte.otel.resourceAttributes.env" }} +- name: OTEL_RESOURCE_ATTRIBUTES + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: OTEL_RESOURCE_ATTRIBUTES +{{- end }} + +{{/* +Renders the set of all otel environment variables +*/}} +{{- define "airbyte.otel.envs" }} +{{- include "airbyte.otel.exporter.endpoint.env" . }} +{{- include "airbyte.otel.exporter.protocol.env" . }} +{{- include "airbyte.otel.exporter.timeout.env" . }} +{{- include "airbyte.otel.exporter.metricExportInterval.env" . }} +{{- include "airbyte.otel.exporter.name.env" . }} +{{- include "airbyte.otel.resourceAttributes.env" . }} +{{- end }} + +{{/* +Renders the set of all otel config map variables +*/}} +{{- define "airbyte.otel.configVars" }} +OTEL_EXPORTER_OTLP_ENDPOINT: {{ include "airbyte.otel.exporter.endpoint" . | quote }} +OTEL_EXPORTER_OTLP_PROTOCOL: {{ include "airbyte.otel.exporter.protocol" . | quote }} +OTEL_EXPORTER_OTLP_TIMEOUT: {{ include "airbyte.otel.exporter.timeout" . | quote }} +OTEL_METRIC_EXPORT_INTERVAL: {{ include "airbyte.otel.exporter.metricExportInterval" . | quote }} +OTEL_METRICS_EXPORTER: {{ include "airbyte.otel.exporter.name" . | quote }} +OTEL_RESOURCE_ATTRIBUTES: {{ include "airbyte.otel.resourceAttributes" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_secretsManager.tpl b/helm/airbyte-v2/templates/config/_secretsManager.tpl new file mode 100644 index 00000000..332b6d1d --- /dev/null +++ b/helm/airbyte-v2/templates/config/_secretsManager.tpl @@ -0,0 +1,551 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Secretsmanager Configuration +*/}} + +{{/* +Renders the secretsManager secret name +*/}} +{{- define "airbyte.secretsManager.secretName" }} +{{- if .Values.global.secretsManager.secretName }} + {{- .Values.global.secretsManager.secretName }} +{{- else }} + {{- .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} +{{- end }} +{{- end }} + +{{/* +Renders the global.secretsManager.type value. +Must be a value the Airbyte server accepts: testing_config_db_table, no_op, aws_secret_manager, google_secret_manager, vault, azure_key_vault (not "kubernetes"). +Default "no_op" so job pod init container works without needing datasources.local-secrets (workload launcher may not inject DATASOURCES_LOCALSECRETS_*). Set explicitly to "testing_config_db_table" for connector secrets in config DB. +*/}} +{{- define "airbyte.secretsManager.type" }} + {{- .Values.global.secretsManager.type | default "no_op" }} +{{- end }} + +{{/* +Renders the secretsManager.type environment variable +*/}} +{{- define "airbyte.secretsManager.type.env" }} +- name: SECRET_PERSISTENCE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: SECRET_PERSISTENCE +{{- end }} + +{{/* +Renders the global.secretsManager.awsSecretManager.accessKeyId value +*/}} +{{- define "airbyte.secretsManager.awsSecretManager.accessKeyId" }} + {{- .Values.global.secretsManager.awsSecretManager.accessKeyId }} +{{- end }} + +{{/* +Renders the secretsManager.awsSecretManager.accessKeyId secret key +*/}} +{{- define "airbyte.secretsManager.awsSecretManager.accessKeyId.secretKey" }} + {{- .Values.global.secretsManager.awsSecretManager.accessKeyIdSecretKey | default "AWS_SECRET_MANAGER_ACCESS_KEY_ID" }} +{{- end }} + +{{/* +Renders the secretsManager.awsSecretManager.accessKeyId environment variable +*/}} +{{- define "airbyte.secretsManager.awsSecretManager.accessKeyId.env" }} +- name: AWS_SECRET_MANAGER_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretsManager.secretName" . }} + key: {{ include "airbyte.secretsManager.awsSecretManager.accessKeyId.secretKey" . }} +{{- end }} + +{{/* +Renders the global.secretsManager.awsSecretManager.secretAccessKey value +*/}} +{{- define "airbyte.secretsManager.awsSecretManager.secretAccessKey" }} + {{- .Values.global.secretsManager.awsSecretManager.secretAccessKey }} +{{- end }} + +{{/* +Renders the secretsManager.awsSecretManager.secretAccessKey secret key +*/}} +{{- define "airbyte.secretsManager.awsSecretManager.secretAccessKey.secretKey" }} + {{- .Values.global.secretsManager.awsSecretManager.secretAccessKeySecretKey | default "AWS_SECRET_MANAGER_SECRET_ACCESS_KEY" }} +{{- end }} + +{{/* +Renders the secretsManager.awsSecretManager.secretAccessKey environment variable +*/}} +{{- define "airbyte.secretsManager.awsSecretManager.secretAccessKey.env" }} +- name: AWS_SECRET_MANAGER_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretsManager.secretName" . }} + key: {{ include "airbyte.secretsManager.awsSecretManager.secretAccessKey.secretKey" . }} +{{- end }} + +{{/* +Renders the global.secretsManager.awsSecretManager.region value +*/}} +{{- define "airbyte.secretsManager.awsSecretManager.region" }} + {{- .Values.global.secretsManager.awsSecretManager.region }} +{{- end }} + +{{/* +Renders the secretsManager.awsSecretManager.region environment variable +*/}} +{{- define "airbyte.secretsManager.awsSecretManager.region.env" }} +- name: AWS_SECRET_MANAGER_REGION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AWS_SECRET_MANAGER_REGION +{{- end }} + +{{/* +Renders the global.secretsManager.awsSecretManager.tags value +*/}} +{{- define "airbyte.secretsManager.awsSecretManager.tags" }} + {{- include "airbyte.tagsToString" .Values.global.secretsManager.awsSecretManager.tags }} +{{- end }} + +{{/* +Renders the secretsManager.awsSecretManager.tags environment variable +*/}} +{{- define "airbyte.secretsManager.awsSecretManager.tags.env" }} +- name: AWS_SECRET_MANAGER_SECRET_TAGS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AWS_SECRET_MANAGER_SECRET_TAGS +{{- end }} + +{{/* +Renders the global.secretsManager.awsSecretManager.kmsArn value +*/}} +{{- define "airbyte.secretsManager.awsSecretManager.kmsArn" }} + {{- .Values.global.secretsManager.awsSecretManager.kmsArn | default "" }} +{{- end }} + +{{/* +Renders the secretsManager.awsSecretManager.kmsArn environment variable +*/}} +{{- define "airbyte.secretsManager.awsSecretManager.kmsArn.env" }} +- name: AWS_KMS_KEY_ARN + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AWS_KMS_KEY_ARN +{{- end }} + +{{/* +Renders the global.secretsManager.azureKeyVault.clientId value +*/}} +{{- define "airbyte.secretsManager.azureKeyVault.clientId" }} + {{- .Values.global.secretsManager.azureKeyVault.clientId }} +{{- end }} + +{{/* +Renders the secretsManager.azureKeyVault.clientId secret key +*/}} +{{- define "airbyte.secretsManager.azureKeyVault.clientId.secretKey" }} + {{- .Values.global.secretsManager.azureKeyVault.clientIdSecretKey | default "AB_AZURE_KEY_VAULT_CLIENT_ID" }} +{{- end }} + +{{/* +Renders the secretsManager.azureKeyVault.clientId environment variable +*/}} +{{- define "airbyte.secretsManager.azureKeyVault.clientId.env" }} +- name: AB_AZURE_KEY_VAULT_CLIENT_ID + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretsManager.secretName" . }} + key: {{ include "airbyte.secretsManager.azureKeyVault.clientId.secretKey" . }} +{{- end }} + +{{/* +Renders the global.secretsManager.azureKeyVault.clientSecret value +*/}} +{{- define "airbyte.secretsManager.azureKeyVault.clientSecret" }} + {{- .Values.global.secretsManager.azureKeyVault.clientSecret }} +{{- end }} + +{{/* +Renders the secretsManager.azureKeyVault.clientSecret secret key +*/}} +{{- define "airbyte.secretsManager.azureKeyVault.clientSecret.secretKey" }} + {{- .Values.global.secretsManager.azureKeyVault.clientSecretSecretKey | default "AB_AZURE_KEY_VAULT_CLIENT_SECRET" }} +{{- end }} + +{{/* +Renders the secretsManager.azureKeyVault.clientSecret environment variable +*/}} +{{- define "airbyte.secretsManager.azureKeyVault.clientSecret.env" }} +- name: AB_AZURE_KEY_VAULT_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretsManager.secretName" . }} + key: {{ include "airbyte.secretsManager.azureKeyVault.clientSecret.secretKey" . }} +{{- end }} + +{{/* +Renders the global.secretsManager.azureKeyVault.tenantId value +*/}} +{{- define "airbyte.secretsManager.azureKeyVault.tenantId" }} + {{- .Values.global.secretsManager.azureKeyVault.tenantId }} +{{- end }} + +{{/* +Renders the secretsManager.azureKeyVault.tenantId environment variable +*/}} +{{- define "airbyte.secretsManager.azureKeyVault.tenantId.env" }} +- name: AB_AZURE_KEY_VAULT_TENANT_ID + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AB_AZURE_KEY_VAULT_TENANT_ID +{{- end }} + +{{/* +Renders the global.secretsManager.azureKeyVault.vaultUrl value +*/}} +{{- define "airbyte.secretsManager.azureKeyVault.vaultUrl" }} + {{- .Values.global.secretsManager.azureKeyVault.vaultUrl }} +{{- end }} + +{{/* +Renders the secretsManager.azureKeyVault.vaultUrl environment variable +*/}} +{{- define "airbyte.secretsManager.azureKeyVault.vaultUrl.env" }} +- name: AB_AZURE_KEY_VAULT_VAULT_URL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AB_AZURE_KEY_VAULT_VAULT_URL +{{- end }} + +{{/* +Renders the global.secretsManager.azureKeyVault.clientIdRefName value +*/}} +{{- define "airbyte.secretsManager.azureKeyVault.clientIdRefName" }} + {{- include "airbyte.secretsManager.secretName" . }} +{{- end }} + +{{/* +Renders the secretsManager.azureKeyVault.clientIdRefName environment variable +*/}} +{{- define "airbyte.secretsManager.azureKeyVault.clientIdRefName.env" }} +- name: AB_AZURE_KEY_CLIENT_ID_REF_NAME + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AB_AZURE_KEY_CLIENT_ID_REF_NAME +{{- end }} + +{{/* +Renders the global.secretsManager.azureKeyVault.clientIdRefKey value +*/}} +{{- define "airbyte.secretsManager.azureKeyVault.clientIdRefKey" }} + {{- .Values.global.secretsManager.azureKeyVault.clientIdSecretKey | default "AB_AZURE_KEY_VAULT_CLIENT_ID" }} +{{- end }} + +{{/* +Renders the secretsManager.azureKeyVault.clientIdRefKey environment variable +*/}} +{{- define "airbyte.secretsManager.azureKeyVault.clientIdRefKey.env" }} +- name: AB_AZURE_KEY_CLIENT_ID_REF_KEY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AB_AZURE_KEY_CLIENT_ID_REF_KEY +{{- end }} + +{{/* +Renders the global.secretsManager.azureKeyVault.clientSecretRefName value +*/}} +{{- define "airbyte.secretsManager.azureKeyVault.clientSecretRefName" }} + {{- include "airbyte.secretsManager.secretName" . }} +{{- end }} + +{{/* +Renders the secretsManager.azureKeyVault.clientSecretRefName environment variable +*/}} +{{- define "airbyte.secretsManager.azureKeyVault.clientSecretRefName.env" }} +- name: AB_AZURE_KEY_CLIENT_SECRET_REF_NAME + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AB_AZURE_KEY_CLIENT_SECRET_REF_NAME +{{- end }} + +{{/* +Renders the global.secretsManager.azureKeyVault.clientSecretRefKey value +*/}} +{{- define "airbyte.secretsManager.azureKeyVault.clientSecretRefKey" }} + {{- .Values.global.secretsManager.azureKeyVault.clientSecretSecretKey | default "AB_AZURE_KEY_VAULT_CLIENT_SECRET" }} +{{- end }} + +{{/* +Renders the secretsManager.azureKeyVault.clientSecretRefKey environment variable +*/}} +{{- define "airbyte.secretsManager.azureKeyVault.clientSecretRefKey.env" }} +- name: AB_AZURE_KEY_CLIENT_SECRET_REF_KEY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AB_AZURE_KEY_CLIENT_SECRET_REF_KEY +{{- end }} + +{{/* +Renders the global.secretsManager.googleSecretManager.projectId value +*/}} +{{- define "airbyte.secretsManager.googleSecretManager.projectId" }} + {{- .Values.global.secretsManager.googleSecretManager.projectId }} +{{- end }} + +{{/* +Renders the secretsManager.googleSecretManager.projectId environment variable +*/}} +{{- define "airbyte.secretsManager.googleSecretManager.projectId.env" }} +- name: SECRET_STORE_GCP_PROJECT_ID + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: SECRET_STORE_GCP_PROJECT_ID +{{- end }} + +{{/* +Renders the global.secretsManager.googleSecretManager.credentials value +*/}} +{{- define "airbyte.secretsManager.googleSecretManager.credentials" }} + {{- .Values.global.secretsManager.googleSecretManager.credentials }} +{{- end }} + +{{/* +Renders the secretsManager.googleSecretManager.credentials secret key +*/}} +{{- define "airbyte.secretsManager.googleSecretManager.credentials.secretKey" }} + {{- .Values.global.secretsManager.googleSecretManager.credentialsSecretKey | default "SECRET_STORE_GCP_CREDENTIALS" }} +{{- end }} + +{{/* +Renders the secretsManager.googleSecretManager.credentials environment variable +*/}} +{{- define "airbyte.secretsManager.googleSecretManager.credentials.env" }} +- name: SECRET_STORE_GCP_CREDENTIALS + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretsManager.secretName" . }} + key: {{ include "airbyte.secretsManager.googleSecretManager.credentials.secretKey" . }} +{{- end }} + +{{/* +Renders the global.secretsManager.googleSecretManager.credentialsSecretKey value +*/}} +{{- define "airbyte.secretsManager.googleSecretManager.credentialsSecretKey" }} + {{- (include "airbyte.secretsManager.googleSecretManager.credentials.secretKey" .) }} +{{- end }} + +{{/* +Renders the secretsManager.googleSecretManager.credentialsSecretKey environment variable +*/}} +{{- define "airbyte.secretsManager.googleSecretManager.credentialsSecretKey.env" }} +- name: SECRET_STORE_GCP_SECRET_KEY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: SECRET_STORE_GCP_SECRET_KEY +{{- end }} + +{{/* +Renders the global.secretsManager.googleSecretManager.credentailsSecretName value +*/}} +{{- define "airbyte.secretsManager.googleSecretManager.credentailsSecretName" }} + {{- (include "airbyte.secretsManager.secretName" .) }} +{{- end }} + +{{/* +Renders the secretsManager.googleSecretManager.credentailsSecretName environment variable +*/}} +{{- define "airbyte.secretsManager.googleSecretManager.credentailsSecretName.env" }} +- name: SECRET_STORE_GCP_SECRET_NAME + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: SECRET_STORE_GCP_SECRET_NAME +{{- end }} + +{{/* +Renders the global.secretsManager.vault.address value +*/}} +{{- define "airbyte.secretsManager.vault.address" }} + {{- .Values.global.secretsManager.vault.address | default (printf "http://airbyte-vault-svc.%s:8200" .Release.Namespace) }} +{{- end }} + +{{/* +Renders the secretsManager.vault.address environment variable +*/}} +{{- define "airbyte.secretsManager.vault.address.env" }} +- name: VAULT_ADDRESS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: VAULT_ADDRESS +{{- end }} + +{{/* +Renders the global.secretsManager.vault.prefix value +*/}} +{{- define "airbyte.secretsManager.vault.prefix" }} + {{- .Values.global.secretsManager.vault.prefix | default "secret/" }} +{{- end }} + +{{/* +Renders the secretsManager.vault.prefix environment variable +*/}} +{{- define "airbyte.secretsManager.vault.prefix.env" }} +- name: VAULT_PREFIX + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: VAULT_PREFIX +{{- end }} + +{{/* +Renders the global.secretsManager.vault.authToken value +*/}} +{{- define "airbyte.secretsManager.vault.authToken" }} + {{- .Values.global.secretsManager.vault.authToken }} +{{- end }} + +{{/* +Renders the secretsManager.vault.authToken secret key +*/}} +{{- define "airbyte.secretsManager.vault.authToken.secretKey" }} + {{- .Values.global.secretsManager.vault.authTokenSecretKey | default "VAULT_AUTH_TOKEN" }} +{{- end }} + +{{/* +Renders the secretsManager.vault.authToken environment variable +*/}} +{{- define "airbyte.secretsManager.vault.authToken.env" }} +- name: VAULT_AUTH_TOKEN + valueFrom: + secretKeyRef: + name: {{ include "airbyte.secretsManager.secretName" . }} + key: {{ include "airbyte.secretsManager.vault.authToken.secretKey" . }} +{{- end }} + +{{/* +Renders the set of all secretsManager environment variables +*/}} +{{- define "airbyte.secretsManager.envs" }} +{{- include "airbyte.secretsManager.type.env" . }} +{{- $opt := (include "airbyte.secretsManager.type" .) }} + +{{- if eq $opt "AWS_SECRET_MANAGER" }} +{{- include "airbyte.secretsManager.awsSecretManager.accessKeyId.env" . }} +{{- include "airbyte.secretsManager.awsSecretManager.secretAccessKey.env" . }} +{{- include "airbyte.secretsManager.awsSecretManager.region.env" . }} +{{- include "airbyte.secretsManager.awsSecretManager.tags.env" . }} +{{- include "airbyte.secretsManager.awsSecretManager.kmsArn.env" . }} +{{- end }} + +{{- if eq $opt "AZURE_KEY_VAULT" }} +{{- include "airbyte.secretsManager.azureKeyVault.clientId.env" . }} +{{- include "airbyte.secretsManager.azureKeyVault.clientSecret.env" . }} +{{- include "airbyte.secretsManager.azureKeyVault.tenantId.env" . }} +{{- include "airbyte.secretsManager.azureKeyVault.vaultUrl.env" . }} +{{- include "airbyte.secretsManager.azureKeyVault.clientIdRefName.env" . }} +{{- include "airbyte.secretsManager.azureKeyVault.clientIdRefKey.env" . }} +{{- include "airbyte.secretsManager.azureKeyVault.clientSecretRefName.env" . }} +{{- include "airbyte.secretsManager.azureKeyVault.clientSecretRefKey.env" . }} +{{- end }} + +{{- if eq $opt "GOOGLE_SECRET_MANAGER" }} +{{- include "airbyte.secretsManager.googleSecretManager.projectId.env" . }} +{{- include "airbyte.secretsManager.googleSecretManager.credentials.env" . }} +{{- include "airbyte.secretsManager.googleSecretManager.credentialsSecretKey.env" . }} +{{- include "airbyte.secretsManager.googleSecretManager.credentailsSecretName.env" . }} +{{- end }} + +{{- if eq $opt "TESTING_CONFIG_DB_TABLE" }} +{{- end }} + +{{- if eq $opt "VAULT" }} +{{- include "airbyte.secretsManager.vault.address.env" . }} +{{- include "airbyte.secretsManager.vault.prefix.env" . }} +{{- include "airbyte.secretsManager.vault.authToken.env" . }} +{{- end }} + +{{- end }} + +{{/* +Renders the set of all secretsManager config map variables +*/}} +{{- define "airbyte.secretsManager.configVars" }} +SECRET_PERSISTENCE: {{ include "airbyte.secretsManager.type" . | quote }} +{{- $opt := (include "airbyte.secretsManager.type" .) }} + +{{- if eq $opt "AWS_SECRET_MANAGER" }} +AWS_SECRET_MANAGER_REGION: {{ include "airbyte.secretsManager.awsSecretManager.region" . | quote }} +AWS_SECRET_MANAGER_SECRET_TAGS: {{ include "airbyte.tagsToString" .Values.global.secretsManager.awsSecretManager.tags | quote }} +AWS_KMS_KEY_ARN: {{ include "airbyte.secretsManager.awsSecretManager.kmsArn" . | quote }} +{{- end }} + +{{- if eq $opt "AZURE_KEY_VAULT" }} +AB_AZURE_KEY_VAULT_TENANT_ID: {{ include "airbyte.secretsManager.azureKeyVault.tenantId" . | quote }} +AB_AZURE_KEY_VAULT_VAULT_URL: {{ include "airbyte.secretsManager.azureKeyVault.vaultUrl" . | quote }} +AB_AZURE_KEY_CLIENT_ID_REF_NAME: {{ include "airbyte.secretsManager.secretName" . | quote }} +AB_AZURE_KEY_CLIENT_ID_REF_KEY: {{ .Values.global.secretsManager.azureKeyVault.clientIdSecretKey | default "AB_AZURE_KEY_VAULT_CLIENT_ID" | quote }} +AB_AZURE_KEY_CLIENT_SECRET_REF_NAME: {{ include "airbyte.secretsManager.secretName" . | quote }} +AB_AZURE_KEY_CLIENT_SECRET_REF_KEY: {{ .Values.global.secretsManager.azureKeyVault.clientSecretSecretKey | default "AB_AZURE_KEY_VAULT_CLIENT_SECRET" | quote }} +{{- end }} + +{{- if eq $opt "GOOGLE_SECRET_MANAGER" }} +SECRET_STORE_GCP_PROJECT_ID: {{ include "airbyte.secretsManager.googleSecretManager.projectId" . | quote }} +SECRET_STORE_GCP_SECRET_KEY: {{ (include "airbyte.secretsManager.googleSecretManager.credentials.secretKey" .) | quote }} +SECRET_STORE_GCP_SECRET_NAME: {{ (include "airbyte.secretsManager.secretName" .) | quote }} +{{- end }} + +{{- if or (eq $opt "TESTING_CONFIG_DB_TABLE") (eq $opt "testing_config_db_table") }} +DATASOURCES_LOCALSECRETS_URL: {{ include "airbyte.database.url" . | quote }} +{{- end }} + +{{- if eq $opt "VAULT" }} +VAULT_ADDRESS: {{ include "airbyte.secretsManager.vault.address" . | quote }} +VAULT_PREFIX: {{ include "airbyte.secretsManager.vault.prefix" . | quote }} +{{- end }} + +{{- end }} + +{{/* +Renders the set of all secretsManager secret variables +*/}} +{{- define "airbyte.secretsManager.secrets" }} +{{- $opt := (include "airbyte.secretsManager.type" .) }} + +{{- if eq $opt "AWS_SECRET_MANAGER" }} +AWS_SECRET_MANAGER_ACCESS_KEY_ID: {{ include "airbyte.secretsManager.awsSecretManager.accessKeyId" . | quote }} +AWS_SECRET_MANAGER_SECRET_ACCESS_KEY: {{ include "airbyte.secretsManager.awsSecretManager.secretAccessKey" . | quote }} +{{- end }} + +{{- if eq $opt "AZURE_KEY_VAULT" }} +AB_AZURE_KEY_VAULT_CLIENT_ID: {{ include "airbyte.secretsManager.azureKeyVault.clientId" . | quote }} +AB_AZURE_KEY_VAULT_CLIENT_SECRET: {{ include "airbyte.secretsManager.azureKeyVault.clientSecret" . | quote }} +{{- end }} + +{{- if eq $opt "GOOGLE_SECRET_MANAGER" }} +SECRET_STORE_GCP_CREDENTIALS: {{ include "airbyte.secretsManager.googleSecretManager.credentials" . | quote }} +{{- end }} + +{{- if or (eq $opt "TESTING_CONFIG_DB_TABLE") (eq $opt "testing_config_db_table") }} +{{- end }} + +{{- if eq $opt "VAULT" }} +VAULT_AUTH_TOKEN: {{ include "airbyte.secretsManager.vault.authToken" . | quote }} +{{- end }} + +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_server.tpl b/helm/airbyte-v2/templates/config/_server.tpl new file mode 100644 index 00000000..ce43bf45 --- /dev/null +++ b/helm/airbyte-v2/templates/config/_server.tpl @@ -0,0 +1,38 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Server Configuration +*/}} + +{{/* +Renders the server.applications value +*/}} +{{- define "airbyte.server.applications" }} + {{- .Values.server.applications | default ternary "database" "" (eq (include "airbyte.auth.identityProvider.type" .) "generic-oidc") }} +{{- end }} + +{{/* +Renders the server.applications environment variable +*/}} +{{- define "airbyte.server.applications.env" }} +- name: AB_AIRBYTE_APPLICATIONS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AB_AIRBYTE_APPLICATIONS +{{- end }} + +{{/* +Renders the set of all server environment variables +*/}} +{{- define "airbyte.server.envs" }} +{{- include "airbyte.server.applications.env" . }} +{{- end }} + +{{/* +Renders the set of all server config map variables +*/}} +{{- define "airbyte.server.configVars" }} +AB_AIRBYTE_APPLICATIONS: {{ include "airbyte.server.applications" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_shopify.tpl b/helm/airbyte-v2/templates/config/_shopify.tpl new file mode 100644 index 00000000..5d0934ec --- /dev/null +++ b/helm/airbyte-v2/templates/config/_shopify.tpl @@ -0,0 +1,83 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Shopify Configuration +*/}} + +{{/* +Renders the shopify secret name +*/}} +{{- define "airbyte.shopify.secretName" }} +{{- if .Values.global.shopify.secretName }} + {{- .Values.global.shopify.secretName }} +{{- else }} + {{- .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} +{{- end }} +{{- end }} + +{{/* +Renders the global.shopify.clientId value +*/}} +{{- define "airbyte.shopify.clientId" }} + {{- .Values.global.shopify.clientId }} +{{- end }} + +{{/* +Renders the shopify.clientId secret key +*/}} +{{- define "airbyte.shopify.clientId.secretKey" }} + {{- .Values.global.shopify.clientIdSecretKey | default "SHOPIFY_CLIENT_ID" }} +{{- end }} + +{{/* +Renders the shopify.clientId environment variable +*/}} +{{- define "airbyte.shopify.clientId.env" }} +- name: SHOPIFY_CLIENT_ID + valueFrom: + secretKeyRef: + name: {{ include "airbyte.shopify.secretName" . }} + key: {{ include "airbyte.shopify.clientId.secretKey" . }} +{{- end }} + +{{/* +Renders the global.shopify.apiSecretKey value +*/}} +{{- define "airbyte.shopify.apiSecretKey" }} + {{- .Values.global.shopify.apiSecretKey }} +{{- end }} + +{{/* +Renders the shopify.apiSecretKey secret key +*/}} +{{- define "airbyte.shopify.apiSecretKey.secretKey" }} + {{- .Values.global.shopify.apiSecretKeySecretKey | default "SHOPIFY_API_SECRET_KEY" }} +{{- end }} + +{{/* +Renders the shopify.apiSecretKey environment variable +*/}} +{{- define "airbyte.shopify.apiSecretKey.env" }} +- name: SHOPIFY_API_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ include "airbyte.shopify.secretName" . }} + key: {{ include "airbyte.shopify.apiSecretKey.secretKey" . }} +{{- end }} + +{{/* +Renders the set of all shopify environment variables +*/}} +{{- define "airbyte.shopify.envs" }} +{{- include "airbyte.shopify.clientId.env" . }} +{{- include "airbyte.shopify.apiSecretKey.env" . }} +{{- end }} + +{{/* +Renders the set of all shopify secret variables +*/}} +{{- define "airbyte.shopify.secrets" }} +SHOPIFY_CLIENT_ID: {{ include "airbyte.shopify.clientId" . | quote }} +SHOPIFY_API_SECRET_KEY: {{ include "airbyte.shopify.apiSecretKey" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_storage.tpl b/helm/airbyte-v2/templates/config/_storage.tpl new file mode 100644 index 00000000..ab4a2be4 --- /dev/null +++ b/helm/airbyte-v2/templates/config/_storage.tpl @@ -0,0 +1,492 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Storage Configuration +*/}} + +{{/* +Renders the storage secret name +*/}} +{{- define "airbyte.storage.secretName" }} +{{- if .Values.global.storage.secretName }} + {{- .Values.global.storage.secretName }} +{{- else }} + {{- .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} +{{- end }} +{{- end }} + +{{/* +Renders the global.storage.type value. Default "s3" deployment. +*/}} +{{- define "airbyte.storage.type" }} + {{- .Values.global.storage.type | default "s3" }} +{{- end }} + +{{/* +Renders the storage.type environment variable +*/}} +{{- define "airbyte.storage.type.env" }} +- name: STORAGE_TYPE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: STORAGE_TYPE +{{- end }} + +{{/* +Renders the global.storage.bucket.activityPayload value +*/}} +{{- define "airbyte.storage.bucket.activityPayload" }} + {{- .Values.global.storage.bucket.activityPayload | default "airbyte-storage" }} +{{- end }} + +{{/* +Renders the storage.bucket.activityPayload environment variable +*/}} +{{- define "airbyte.storage.bucket.activityPayload.env" }} +- name: STORAGE_BUCKET_ACTIVITY_PAYLOAD + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: STORAGE_BUCKET_ACTIVITY_PAYLOAD +{{- end }} + +{{/* +Renders the global.storage.bucket.log value +*/}} +{{- define "airbyte.storage.bucket.log" }} + {{- .Values.global.storage.bucket.log | default "airbyte-storage" }} +{{- end }} + +{{/* +Renders the storage.bucket.log environment variable +*/}} +{{- define "airbyte.storage.bucket.log.env" }} +- name: STORAGE_BUCKET_LOG + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: STORAGE_BUCKET_LOG +{{- end }} + +{{/* +Renders the global.storage.bucket.state value +*/}} +{{- define "airbyte.storage.bucket.state" }} + {{- .Values.global.storage.bucket.state | default "airbyte-storage" }} +{{- end }} + +{{/* +Renders the storage.bucket.state environment variable +*/}} +{{- define "airbyte.storage.bucket.state.env" }} +- name: STORAGE_BUCKET_STATE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: STORAGE_BUCKET_STATE +{{- end }} + +{{/* +Renders the global.storage.bucket.workloadOutput value +*/}} +{{- define "airbyte.storage.bucket.workloadOutput" }} + {{- .Values.global.storage.bucket.workloadOutput | default "airbyte-storage" }} +{{- end }} + +{{/* +Renders the storage.bucket.workloadOutput environment variable +*/}} +{{- define "airbyte.storage.bucket.workloadOutput.env" }} +- name: STORAGE_BUCKET_WORKLOAD_OUTPUT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: STORAGE_BUCKET_WORKLOAD_OUTPUT +{{- end }} + +{{/* +Renders the global.storage.bucket.auditLogging value +*/}} +{{- define "airbyte.storage.bucket.auditLogging" }} + {{- .Values.global.storage.bucket.auditLogging | default "airbyte-storage" }} +{{- end }} + +{{/* +Renders the storage.bucket.auditLogging environment variable +*/}} +{{- define "airbyte.storage.bucket.auditLogging.env" }} +- name: STORAGE_BUCKET_AUDIT_LOGGING + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: STORAGE_BUCKET_AUDIT_LOGGING +{{- end }} + +{{/* +Renders the global.storage.s3.region value +*/}} +{{- define "airbyte.storage.s3.region" }} + {{- .Values.global.storage.s3.region }} +{{- end }} + +{{/* +Renders the storage.s3.region environment variable +*/}} +{{- define "airbyte.storage.s3.region.env" }} +- name: AWS_DEFAULT_REGION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AWS_DEFAULT_REGION +{{- end }} + +{{/* +Renders the global.storage.s3.authenticationType value +*/}} +{{- define "airbyte.storage.s3.authenticationType" }} + {{- .Values.global.storage.s3.authenticationType | default "credentials" }} +{{- end }} + +{{/* +Renders the storage.s3.authenticationType environment variable +*/}} +{{- define "airbyte.storage.s3.authenticationType.env" }} +- name: AWS_AUTHENTICATION_TYPE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AWS_AUTHENTICATION_TYPE +{{- end }} + +{{/* +Renders the global.storage.s3.accessKeyId value +*/}} +{{- define "airbyte.storage.s3.accessKeyId" }} + {{- .Values.global.storage.s3.accessKeyId }} +{{- end }} + +{{/* +Renders the storage.s3.accessKeyId secret key +*/}} +{{- define "airbyte.storage.s3.accessKeyId.secretKey" }} + {{- .Values.global.storage.s3.accessKeyIdSecretKey | default "AWS_ACCESS_KEY_ID" }} +{{- end }} + +{{/* +Renders the storage.s3.accessKeyId environment variable +*/}} +{{- define "airbyte.storage.s3.accessKeyId.env" }} +- name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ include "airbyte.storage.secretName" . }} + key: {{ include "airbyte.storage.s3.accessKeyId.secretKey" . }} +{{- end }} + +{{/* +Renders the global.storage.s3.secretAccessKey value +*/}} +{{- define "airbyte.storage.s3.secretAccessKey" }} + {{- .Values.global.storage.s3.secretAccessKey }} +{{- end }} + +{{/* +Renders the storage.s3.secretAccessKey secret key +*/}} +{{- define "airbyte.storage.s3.secretAccessKey.secretKey" }} + {{- .Values.global.storage.s3.secretAccessKeySecretKey | default "AWS_SECRET_ACCESS_KEY" }} +{{- end }} + +{{/* +Renders the storage.s3.secretAccessKey environment variable +*/}} +{{- define "airbyte.storage.s3.secretAccessKey.env" }} +- name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ include "airbyte.storage.secretName" . }} + key: {{ include "airbyte.storage.s3.secretAccessKey.secretKey" . }} +{{- end }} + +{{/* +Renders the global.storage.azure.connectionString value +*/}} +{{- define "airbyte.storage.azure.connectionString" }} + {{- .Values.global.storage.azure.connectionString }} +{{- end }} + +{{/* +Renders the storage.azure.connectionString secret key +*/}} +{{- define "airbyte.storage.azure.connectionString.secretKey" }} + {{- .Values.global.storage.azure.connectionStringSecretKey | default "AZURE_STORAGE_CONNECTION_STRING" }} +{{- end }} + +{{/* +Renders the storage.azure.connectionString environment variable +*/}} +{{- define "airbyte.storage.azure.connectionString.env" }} +- name: AZURE_STORAGE_CONNECTION_STRING + valueFrom: + secretKeyRef: + name: {{ include "airbyte.storage.secretName" . }} + key: {{ include "airbyte.storage.azure.connectionString.secretKey" . }} +{{- end }} + +{{/* +Renders the global.storage.gcs.credentialsJson value +*/}} +{{- define "airbyte.storage.gcs.credentialsJson" }} + {{- .Values.global.storage.gcs.credentialsJson }} +{{- end }} + +{{/* +Renders the storage.gcs.credentialsJson secret key +*/}} +{{- define "airbyte.storage.gcs.credentialsJson.secretKey" }} + {{- .Values.global.storage.gcs.credentialsJsonSecretKey | default "GOOGLE_APPLICATION_CREDENTIALS_JSON" }} +{{- end }} + +{{/* +Renders the storage.gcs.credentialsJson environment variable +*/}} +{{- define "airbyte.storage.gcs.credentialsJson.env" }} +- name: GOOGLE_APPLICATION_CREDENTIALS_JSON + valueFrom: + secretKeyRef: + name: {{ include "airbyte.storage.secretName" . }} + key: {{ include "airbyte.storage.gcs.credentialsJson.secretKey" . }} +{{- end }} + +{{/* +Renders the global.storage.gcs.credentialsJsonPath value +*/}} +{{- define "airbyte.storage.gcs.credentialsJsonPath" }} + {{- .Values.global.storage.gcs.credentialsJsonPath | default "/secrets/gcp-creds/gcp.json" }} +{{- end }} + +{{/* +Renders the storage.gcs.credentialsJsonPath environment variable +*/}} +{{- define "airbyte.storage.gcs.credentialsJsonPath.env" }} +- name: GOOGLE_APPLICATION_CREDENTIALS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: GOOGLE_APPLICATION_CREDENTIALS +{{- end }} + +{{/* +Renders the global.storage.minio.accessKeyId value. +When global.secretSeed is set, default is env-specific; else "minio". +*/}} +{{- define "airbyte.storage.minio.accessKeyId" }} + {{- .Values.global.storage.minio.accessKeyId | default (ternary (printf "minio-%s" (include "airbyte.secretSeed.hashBase" . | sha256sum | nospace | trunc 12)) "minio" (ne (toString .Values.global.secretSeed) "")) }} +{{- end }} + +{{/* +Renders the storage.minio.accessKeyId secret key +*/}} +{{- define "airbyte.storage.minio.accessKeyId.secretKey" }} + {{- .Values.global.storage.minio.accessKeyIdSecretKey | default "AWS_ACCESS_KEY_ID" }} +{{- end }} + +{{/* +Renders the storage.minio.accessKeyId environment variable +*/}} +{{- define "airbyte.storage.minio.accessKeyId.env" }} +- name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ include "airbyte.storage.secretName" . }} + key: {{ include "airbyte.storage.minio.accessKeyId.secretKey" . }} +{{- end }} + +{{/* +Renders the global.storage.minio.secretAccessKey value. +When global.secretSeed is set, default is env-specific; else "minio123". +*/}} +{{- define "airbyte.storage.minio.secretAccessKey" }} + {{- .Values.global.storage.minio.secretAccessKey | default (ternary (printf "minio-secret-%s" (include "airbyte.secretSeed.hashBase" . | sha256sum | nospace | trunc 24)) "minio123" (ne (toString .Values.global.secretSeed) "")) }} +{{- end }} + +{{/* +Renders the storage.minio.secretAccessKey secret key +*/}} +{{- define "airbyte.storage.minio.secretAccessKey.secretKey" }} + {{- .Values.global.storage.minio.secretAccessKeySecretKey | default "AWS_SECRET_ACCESS_KEY" }} +{{- end }} + +{{/* +Renders the storage.minio.secretAccessKey environment variable +*/}} +{{- define "airbyte.storage.minio.secretAccessKey.env" }} +- name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ include "airbyte.storage.secretName" . }} + key: {{ include "airbyte.storage.minio.secretAccessKey.secretKey" . }} +{{- end }} + +{{/* +Renders the global.storage.minio.endpoint value +*/}} +{{- define "airbyte.storage.minio.endpoint" }} + {{- .Values.global.storage.minio.endpoint | default "http://airbyte-minio-svc:9000" }} +{{- end }} + +{{/* +Renders the storage.minio.endpoint environment variable +*/}} +{{- define "airbyte.storage.minio.endpoint.env" }} +- name: MINIO_ENDPOINT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MINIO_ENDPOINT +{{- end }} + +{{/* +Renders the global.storage.minio.s3PathStyleAccess value +*/}} +{{- define "airbyte.storage.minio.s3PathStyleAccess" }} + {{- if eq .Values.global.storage.minio.s3PathStyleAccess nil }} + {{- true }} + {{- else }} + {{- .Values.global.storage.minio.s3PathStyleAccess }} + {{- end }} +{{- end }} + +{{/* +Renders S3_PATH_STYLE_ACCESS value for configmap (always "true"). +*/}} +{{- define "airbyte.storage.s3PathStyleAccess.value" }} +{{- "true" }} +{{- end }} + +{{/* +Renders the storage.minio.s3PathStyleAccess environment variable +*/}} +{{- define "airbyte.storage.minio.s3PathStyleAccess.env" }} +- name: S3_PATH_STYLE_ACCESS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: S3_PATH_STYLE_ACCESS +{{- end }} + +{{/* +Renders the set of all storage environment variables +*/}} +{{- define "airbyte.storage.envs" }} +{{- include "airbyte.storage.type.env" . }} +{{- include "airbyte.storage.bucket.activityPayload.env" . }} +{{- include "airbyte.storage.bucket.log.env" . }} +{{- include "airbyte.storage.bucket.state.env" . }} +{{- include "airbyte.storage.bucket.workloadOutput.env" . }} +{{- include "airbyte.storage.bucket.auditLogging.env" . }} +{{- $opt := (include "airbyte.storage.type" .) }} + +{{- if eq $opt "azure" }} +{{- include "airbyte.storage.azure.connectionString.env" . }} +{{- end }} + +{{- if eq $opt "gcs" }} +{{- include "airbyte.storage.gcs.credentialsJson.env" . }} +{{- include "airbyte.storage.gcs.credentialsJsonPath.env" . }} +{{- end }} + +{{- if eq $opt "minio" }} +{{- include "airbyte.storage.minio.accessKeyId.env" . }} +{{- include "airbyte.storage.minio.secretAccessKey.env" . }} +{{- include "airbyte.storage.minio.endpoint.env" . }} +{{- include "airbyte.storage.minio.s3PathStyleAccess.env" . }} +{{- end }} + +{{- if eq $opt "s3" }} +{{- include "airbyte.storage.s3.region.env" . }} +{{- include "airbyte.storage.s3.authenticationType.env" . }} +{{- if ne (trim (default "credentials" .Values.global.storage.s3.authenticationType)) "instanceProfile" }} +{{- include "airbyte.storage.s3.accessKeyId.env" . }} +{{- include "airbyte.storage.s3.secretAccessKey.env" . }} +{{- end }} +{{- end }} + +{{- /* S3_PATH_STYLE_ACCESS always in configmap; include env ref so all components get it (avoids "Unsetting empty") */}} +{{- include "airbyte.storage.minio.s3PathStyleAccess.env" . }} +{{- end }} + +{{/* +Renders the set of all storage config map variables +*/}} +{{- define "airbyte.storage.configVars" }} +STORAGE_TYPE: {{ include "airbyte.storage.type" . | quote }} +STORAGE_BUCKET_ACTIVITY_PAYLOAD: {{ include "airbyte.storage.bucket.activityPayload" . | quote }} +STORAGE_BUCKET_LOG: {{ include "airbyte.storage.bucket.log" . | quote }} +STORAGE_BUCKET_STATE: {{ include "airbyte.storage.bucket.state" . | quote }} +STORAGE_BUCKET_WORKLOAD_OUTPUT: {{ include "airbyte.storage.bucket.workloadOutput" . | quote }} +STORAGE_BUCKET_AUDIT_LOGGING: {{ include "airbyte.storage.bucket.auditLogging" . | quote }} +{{- /* airbyte.cloud.storage.bucket.* (StorageBucketConfig) - same values as STORAGE_BUCKET_* */}} +AIRBYTE_CLOUD_STORAGE_BUCKET_LOG: {{ include "airbyte.storage.bucket.log" . | quote }} +AIRBYTE_CLOUD_STORAGE_BUCKET_STATE: {{ include "airbyte.storage.bucket.state" . | quote }} +AIRBYTE_CLOUD_STORAGE_BUCKET_WORKLOAD_OUTPUT: {{ include "airbyte.storage.bucket.workloadOutput" . | quote }} +AIRBYTE_CLOUD_STORAGE_BUCKET_ACTIVITY_PAYLOAD: {{ include "airbyte.storage.bucket.activityPayload" . | quote }} +AIRBYTE_CLOUD_STORAGE_BUCKET_AUDIT_LOGGING: {{ include "airbyte.storage.bucket.auditLogging" . | quote }} +AIRBYTE_CLOUD_STORAGE_BUCKET_PROFILER_OUTPUT: {{ include "airbyte.storage.bucket.log" . | quote }} +{{- $opt := (include "airbyte.storage.type" .) }} + +{{- if eq $opt "azure" }} +{{- end }} + +{{- if eq $opt "gcs" }} +GOOGLE_APPLICATION_CREDENTIALS: {{ include "airbyte.storage.gcs.credentialsJsonPath" . | quote }} +{{- end }} + +{{- if eq $opt "minio" }} +MINIO_ENDPOINT: {{ include "airbyte.storage.minio.endpoint" . | quote }} +{{- end }} + +{{- /* When s3, MinIO is still deployed (create-bucket job); include MINIO_ENDPOINT for that pod */}} +{{- if eq $opt "s3" }} +MINIO_ENDPOINT: {{ include "airbyte.storage.minio.endpoint" . | quote }} +AWS_DEFAULT_REGION: {{ include "airbyte.storage.s3.region" . | quote }} +AWS_AUTHENTICATION_TYPE: {{ include "airbyte.storage.s3.authenticationType" . | quote }} +{{- end }} + +{{- /* Always set S3_PATH_STYLE_ACCESS so workload-api-server never sees empty (avoids "Unsetting empty") */}} +S3_PATH_STYLE_ACCESS: {{ include "airbyte.storage.s3PathStyleAccess.value" . | quote }} +{{- end }} + +{{/* +Renders the set of all storage secret variables +*/}} +{{- define "airbyte.storage.secrets" }} +{{- $opt := (include "airbyte.storage.type" .) }} + +{{- if eq $opt "azure" }} +AZURE_STORAGE_CONNECTION_STRING: {{ include "airbyte.storage.azure.connectionString" . | quote }} +{{- end }} + +{{- if eq $opt "gcs" }} +GOOGLE_APPLICATION_CREDENTIALS_JSON: {{ include "airbyte.storage.gcs.credentialsJson" . | quote }} +{{- end }} + +{{- if eq $opt "minio" }} +AWS_ACCESS_KEY_ID: {{ include "airbyte.storage.minio.accessKeyId" . | quote }} +AWS_SECRET_ACCESS_KEY: {{ include "airbyte.storage.minio.secretAccessKey" . | quote }} +{{- end }} + +{{- if and (eq $opt "s3") (ne (trim (default "credentials" .Values.global.storage.s3.authenticationType)) "instanceProfile") }} +AWS_ACCESS_KEY_ID: {{ include "airbyte.storage.s3.accessKeyId" . | quote }} +AWS_SECRET_ACCESS_KEY: {{ include "airbyte.storage.s3.secretAccessKey" . | quote }} +{{- end }} + +{{- /* Dedicated MinIO keys so instanceProfile S3 does not need AWS_ACCESS_KEY_ID in this secret. */}} +{{- if .Values.minio.enabled }} +MINIO_ROOT_USER: {{ include "airbyte.storage.minio.accessKeyId" . | quote }} +MINIO_ROOT_PASSWORD: {{ include "airbyte.storage.minio.secretAccessKey" . | quote }} +{{- end }} + +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_temporal.tpl b/helm/airbyte-v2/templates/config/_temporal.tpl new file mode 100644 index 00000000..446e2952 --- /dev/null +++ b/helm/airbyte-v2/templates/config/_temporal.tpl @@ -0,0 +1,800 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Temporal Configuration +*/}} + +{{/* +Renders the temporal.autoSetup value +*/}} +{{- define "airbyte.temporal.autoSetup" }} + {{- if eq .Values.temporal.autoSetup nil }} + {{- true }} + {{- else }} + {{- .Values.temporal.autoSetup }} + {{- end }} +{{- end }} + +{{/* +Renders the temporal.autoSetup environment variable +*/}} +{{- define "airbyte.temporal.autoSetup.env" }} +- name: AUTO_SETUP + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AUTO_SETUP +{{- end }} + +{{/* +Renders the temporal.host value. +When temporal.host is set, use it (external or separate cluster). Else Temporal Cloud host or in-cluster Release.Name-temporal:port. +*/}} +{{- define "airbyte.temporal.host" }} + {{- if .Values.temporal.host }} + {{- .Values.temporal.host }} + {{- else if eq (include "airbyte.temporal.cloud.enabled" .) "true" }} + {{- include "airbyte.temporal.cloud.host" . }} + {{- else }} + {{- printf "%s-temporal:%d" .Release.Name (int .Values.temporal.service.port) }} + {{- end }} +{{- end }} + +{{/* +Renders the temporal.host environment variable +*/}} +{{- define "airbyte.temporal.host.env" }} +- name: TEMPORAL_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TEMPORAL_HOST +{{- end }} + +{{/* +Renders the temporal.configFilePath value +*/}} +{{- define "airbyte.temporal.configFilePath" }} + {{- .Values.temporal.configFilePath | default "config/dynamicconfig/development.yaml" }} +{{- end }} + +{{/* +Renders the temporal.configFilePath environment variable +*/}} +{{- define "airbyte.temporal.configFilePath.env" }} +- name: DYNAMIC_CONFIG_FILE_PATH + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DYNAMIC_CONFIG_FILE_PATH +{{- end }} + +{{/* +Renders the temporal.prometheus.endpoint value +*/}} +{{- define "airbyte.temporal.prometheus.endpoint" }} + {{- .Values.temporal.prometheus.endpoint | default "0.0.0.0:9090" }} +{{- end }} + +{{/* +Renders the temporal.prometheus.endpoint environment variable +*/}} +{{- define "airbyte.temporal.prometheus.endpoint.env" }} +- name: PROMETHEUS_ENDPOINT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: PROMETHEUS_ENDPOINT +{{- end }} + +{{/* +Renders the set of all temporal environment variables +*/}} +{{- define "airbyte.temporal.envs" }} +{{- include "airbyte.temporal.autoSetup.env" . }} +{{- include "airbyte.temporal.host.env" . }} +{{- include "airbyte.temporal.configFilePath.env" . }} +{{- include "airbyte.temporal.prometheus.endpoint.env" . }} +{{- end }} + +{{/* +Renders the set of all temporal config map variables +*/}} +{{- define "airbyte.temporal.configVars" }} +AUTO_SETUP: {{ include "airbyte.temporal.autoSetup" . | quote }} +TEMPORAL_HOST: {{ include "airbyte.temporal.host" . | quote }} +DYNAMIC_CONFIG_FILE_PATH: {{ include "airbyte.temporal.configFilePath" . | quote }} +PROMETHEUS_ENDPOINT: {{ include "airbyte.temporal.prometheus.endpoint" . | quote }} +{{- end }} + +{{/* +Renders the temporal.cli secret name +*/}} +{{- define "airbyte.temporal.cli.secretName" }} +{{- if .Values.global.temporal.secretName }} + {{- .Values.global.temporal.secretName }} +{{- else }} + {{- .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} +{{- end }} +{{- end }} + +{{/* +Renders the global.temporal.cli.address value +*/}} +{{- define "airbyte.temporal.cli.address" }} + {{- .Values.global.temporal.cli.address }} +{{- end }} + +{{/* +Renders the temporal.cli.address environment variable +*/}} +{{- define "airbyte.temporal.cli.address.env" }} +- name: TEMPORAL_CLI_ADDRESS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TEMPORAL_CLI_ADDRESS +{{- end }} + +{{/* +Renders the global.temporal.cli.namespace value +*/}} +{{- define "airbyte.temporal.cli.namespace" }} + {{- .Values.global.temporal.cli.namespace }} +{{- end }} + +{{/* +Renders the temporal.cli.namespace environment variable +*/}} +{{- define "airbyte.temporal.cli.namespace.env" }} +- name: TEMPORAL_CLI_NAMESPACE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TEMPORAL_CLI_NAMESPACE +{{- end }} + +{{/* +Renders the global.temporal.cli.tlsCert value +*/}} +{{- define "airbyte.temporal.cli.tlsCert" }} + {{- .Values.global.temporal.cli.tlsCert }} +{{- end }} + +{{/* +Renders the temporal.cli.tlsCert secret key +*/}} +{{- define "airbyte.temporal.cli.tlsCert.secretKey" }} + {{- .Values.global.temporal.cli.tlsCertSecretKey | default "TEMPORAL_CLI_TLS_CERT" }} +{{- end }} + +{{/* +Renders the temporal.cli.tlsCert environment variable +*/}} +{{- define "airbyte.temporal.cli.tlsCert.env" }} +- name: TEMPORAL_CLI_TLS_CERT + valueFrom: + secretKeyRef: + name: {{ include "airbyte.temporal.cli.secretName" . }} + key: {{ include "airbyte.temporal.cli.tlsCert.secretKey" . }} +{{- end }} + +{{/* +Renders the global.temporal.cli.tlsKey value +*/}} +{{- define "airbyte.temporal.cli.tlsKey" }} + {{- .Values.global.temporal.cli.tlsKey }} +{{- end }} + +{{/* +Renders the temporal.cli.tlsKey secret key +*/}} +{{- define "airbyte.temporal.cli.tlsKey.secretKey" }} + {{- .Values.global.temporal.cli.tlsKeySecretKey | default "TEMPORAL_CLI_TLS_KEY" }} +{{- end }} + +{{/* +Renders the temporal.cli.tlsKey environment variable +*/}} +{{- define "airbyte.temporal.cli.tlsKey.env" }} +- name: TEMPORAL_CLI_TLS_KEY + valueFrom: + secretKeyRef: + name: {{ include "airbyte.temporal.cli.secretName" . }} + key: {{ include "airbyte.temporal.cli.tlsKey.secretKey" . }} +{{- end }} + +{{/* +Renders the set of all temporal.cli environment variables +*/}} +{{- define "airbyte.temporal.cli.envs" }} +{{- include "airbyte.temporal.cli.address.env" . }} +{{- include "airbyte.temporal.cli.namespace.env" . }} +{{- include "airbyte.temporal.cli.tlsCert.env" . }} +{{- include "airbyte.temporal.cli.tlsKey.env" . }} +{{- end }} + +{{/* +Renders the set of all temporal.cli config map variables +*/}} +{{- define "airbyte.temporal.cli.configVars" }} +TEMPORAL_CLI_ADDRESS: {{ include "airbyte.temporal.cli.address" . | quote }} +TEMPORAL_CLI_NAMESPACE: {{ include "airbyte.temporal.cli.namespace" . | quote }} +{{- end }} + +{{/* +Renders the set of all temporal.cli secret variables +*/}} +{{- define "airbyte.temporal.cli.secrets" }} +TEMPORAL_CLI_TLS_CERT: {{ include "airbyte.temporal.cli.tlsCert" . | quote }} +TEMPORAL_CLI_TLS_KEY: {{ include "airbyte.temporal.cli.tlsKey" . | quote }} +{{- end }} + +{{/* +Renders the temporal.cloud secret name +*/}} +{{- define "airbyte.temporal.cloud.secretName" }} +{{- if .Values.global.temporal.secretName }} + {{- .Values.global.temporal.secretName }} +{{- else }} + {{- .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} +{{- end }} +{{- end }} + +{{/* +Renders the global.temporal.cloud.enabled value +*/}} +{{- define "airbyte.temporal.cloud.enabled" }} + {{- if eq .Values.global.temporal.cloud.enabled nil }} + {{- false }} + {{- else }} + {{- .Values.global.temporal.cloud.enabled }} + {{- end }} +{{- end }} + +{{/* +Renders the temporal.cloud.enabled environment variable +*/}} +{{- define "airbyte.temporal.cloud.enabled.env" }} +- name: TEMPORAL_CLOUD_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TEMPORAL_CLOUD_ENABLED +{{- end }} + +{{/* +Renders the global.temporal.cloud.host value +*/}} +{{- define "airbyte.temporal.cloud.host" }} + {{- .Values.global.temporal.cloud.host }} +{{- end }} + +{{/* +Renders the temporal.cloud.host environment variable +*/}} +{{- define "airbyte.temporal.cloud.host.env" }} +- name: TEMPORAL_CLOUD_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TEMPORAL_CLOUD_HOST +{{- end }} + +{{/* +Renders the global.temporal.cloud.namespace value +*/}} +{{- define "airbyte.temporal.cloud.namespace" }} + {{- .Values.global.temporal.cloud.namespace }} +{{- end }} + +{{/* +Renders the temporal.cloud.namespace environment variable +*/}} +{{- define "airbyte.temporal.cloud.namespace.env" }} +- name: TEMPORAL_CLOUD_NAMESPACE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TEMPORAL_CLOUD_NAMESPACE +{{- end }} + +{{/* +Renders the set of all temporal.cloud environment variables +*/}} +{{- define "airbyte.temporal.cloud.envs" }} +{{- include "airbyte.temporal.cloud.enabled.env" . }} +{{- include "airbyte.temporal.cloud.host.env" . }} +{{- include "airbyte.temporal.cloud.namespace.env" . }} +{{- end }} + +{{/* +Renders the set of all temporal.cloud config map variables +*/}} +{{- define "airbyte.temporal.cloud.configVars" }} +TEMPORAL_CLOUD_ENABLED: {{ include "airbyte.temporal.cloud.enabled" . | quote }} +TEMPORAL_CLOUD_HOST: {{ include "airbyte.temporal.cloud.host" . | quote }} +TEMPORAL_CLOUD_NAMESPACE: {{ include "airbyte.temporal.cloud.namespace" . | quote }} +{{- end }} + +{{/* +Renders the temporal.cloud.billing secret name +*/}} +{{- define "airbyte.temporal.cloud.billing.secretName" }} +{{- if .Values.global.temporal.secretName }} + {{- .Values.global.temporal.secretName }} +{{- else }} + {{- .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} +{{- end }} +{{- end }} + +{{/* +Renders the global.temporal.cloud.billing.host value +*/}} +{{- define "airbyte.temporal.cloud.billing.host" }} + {{- .Values.global.temporal.cloud.billing.host }} +{{- end }} + +{{/* +Renders the temporal.cloud.billing.host environment variable +*/}} +{{- define "airbyte.temporal.cloud.billing.host.env" }} +- name: TEMPORAL_CLOUD_BILLING_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TEMPORAL_CLOUD_BILLING_HOST +{{- end }} + +{{/* +Renders the global.temporal.cloud.billing.namespace value +*/}} +{{- define "airbyte.temporal.cloud.billing.namespace" }} + {{- .Values.global.temporal.cloud.billing.namespace }} +{{- end }} + +{{/* +Renders the temporal.cloud.billing.namespace environment variable +*/}} +{{- define "airbyte.temporal.cloud.billing.namespace.env" }} +- name: TEMPORAL_CLOUD_BILLING_NAMESPACE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TEMPORAL_CLOUD_BILLING_NAMESPACE +{{- end }} + +{{/* +Renders the set of all temporal.cloud.billing environment variables +*/}} +{{- define "airbyte.temporal.cloud.billing.envs" }} +{{- include "airbyte.temporal.cloud.billing.host.env" . }} +{{- include "airbyte.temporal.cloud.billing.namespace.env" . }} +{{- end }} + +{{/* +Renders the set of all temporal.cloud.billing config map variables +*/}} +{{- define "airbyte.temporal.cloud.billing.configVars" }} +TEMPORAL_CLOUD_BILLING_HOST: {{ include "airbyte.temporal.cloud.billing.host" . | quote }} +TEMPORAL_CLOUD_BILLING_NAMESPACE: {{ include "airbyte.temporal.cloud.billing.namespace" . | quote }} +{{- end }} + +{{/* +Renders the temporal.cloud.connectorRollout secret name +*/}} +{{- define "airbyte.temporal.cloud.connectorRollout.secretName" }} +{{- if .Values.global.temporal.secretName }} + {{- .Values.global.temporal.secretName }} +{{- else }} + {{- .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} +{{- end }} +{{- end }} + +{{/* +Renders the global.temporal.cloud.connectorRollout.host value +*/}} +{{- define "airbyte.temporal.cloud.connectorRollout.host" }} + {{- .Values.global.temporal.cloud.connectorRollout.host }} +{{- end }} + +{{/* +Renders the temporal.cloud.connectorRollout.host environment variable +*/}} +{{- define "airbyte.temporal.cloud.connectorRollout.host.env" }} +- name: TEMPORAL_CLOUD_HOST_CONNECTOR_ROLLOUT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TEMPORAL_CLOUD_HOST_CONNECTOR_ROLLOUT +{{- end }} + +{{/* +Renders the global.temporal.cloud.connectorRollout.namespace value +*/}} +{{- define "airbyte.temporal.cloud.connectorRollout.namespace" }} + {{- .Values.global.temporal.cloud.connectorRollout.namespace }} +{{- end }} + +{{/* +Renders the temporal.cloud.connectorRollout.namespace environment variable +*/}} +{{- define "airbyte.temporal.cloud.connectorRollout.namespace.env" }} +- name: TEMPORAL_CLOUD_NAMESPACE_CONNECTOR_ROLLOUT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TEMPORAL_CLOUD_NAMESPACE_CONNECTOR_ROLLOUT +{{- end }} + +{{/* +Renders the set of all temporal.cloud.connectorRollout environment variables +*/}} +{{- define "airbyte.temporal.cloud.connectorRollout.envs" }} +{{- include "airbyte.temporal.cloud.connectorRollout.host.env" . }} +{{- include "airbyte.temporal.cloud.connectorRollout.namespace.env" . }} +{{- end }} + +{{/* +Renders the set of all temporal.cloud.connectorRollout config map variables +*/}} +{{- define "airbyte.temporal.cloud.connectorRollout.configVars" }} +TEMPORAL_CLOUD_HOST_CONNECTOR_ROLLOUT: {{ include "airbyte.temporal.cloud.connectorRollout.host" . | quote }} +TEMPORAL_CLOUD_NAMESPACE_CONNECTOR_ROLLOUT: {{ include "airbyte.temporal.cloud.connectorRollout.namespace" . | quote }} +{{- end }} + +{{/* +Renders the temporal.cloud.credentials secret name +*/}} +{{- define "airbyte.temporal.cloud.credentials.secretName" }} +{{- if .Values.global.temporal.secretName }} + {{- .Values.global.temporal.secretName }} +{{- else }} + {{- .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} +{{- end }} +{{- end }} + +{{/* +Renders the global.temporal.cloud.clientCert value +*/}} +{{- define "airbyte.temporal.cloud.credentials.clientCert" }} + {{- .Values.global.temporal.cloud.clientCert }} +{{- end }} + +{{/* +Renders the temporal.cloud.credentials.clientCert secret key +*/}} +{{- define "airbyte.temporal.cloud.credentials.clientCert.secretKey" }} + {{- .Values.global.temporal.cloud.clientCertSecretKey | default "TEMPORAL_CLOUD_CLIENT_CERT" }} +{{- end }} + +{{/* +Renders the temporal.cloud.credentials.clientCert environment variable +*/}} +{{- define "airbyte.temporal.cloud.credentials.clientCert.env" }} +- name: TEMPORAL_CLOUD_CLIENT_CERT + valueFrom: + secretKeyRef: + name: {{ include "airbyte.temporal.cloud.secretName" . }} + key: {{ include "airbyte.temporal.cloud.credentials.clientCert.secretKey" . }} +{{- end }} + +{{/* +Renders the global.temporal.cloud.clientKey value +*/}} +{{- define "airbyte.temporal.cloud.credentials.clientKey" }} + {{- .Values.global.temporal.cloud.clientKey }} +{{- end }} + +{{/* +Renders the temporal.cloud.credentials.clientKey secret key +*/}} +{{- define "airbyte.temporal.cloud.credentials.clientKey.secretKey" }} + {{- .Values.global.temporal.cloud.clientKeySecretKey | default "TEMPORAL_CLOUD_CLIENT_KEY" }} +{{- end }} + +{{/* +Renders the temporal.cloud.credentials.clientKey environment variable +*/}} +{{- define "airbyte.temporal.cloud.credentials.clientKey.env" }} +- name: TEMPORAL_CLOUD_CLIENT_KEY + valueFrom: + secretKeyRef: + name: {{ include "airbyte.temporal.cloud.secretName" . }} + key: {{ include "airbyte.temporal.cloud.credentials.clientKey.secretKey" . }} +{{- end }} + +{{/* +Renders the set of all temporal.cloud.credentials environment variables +*/}} +{{- define "airbyte.temporal.cloud.credentials.envs" }} +{{- include "airbyte.temporal.cloud.credentials.clientCert.env" . }} +{{- include "airbyte.temporal.cloud.credentials.clientKey.env" . }} +{{- end }} + +{{/* +Renders the set of all temporal.cloud.credentials secret variables +*/}} +{{- define "airbyte.temporal.cloud.credentials.secrets" }} +TEMPORAL_CLOUD_CLIENT_CERT: {{ include "airbyte.temporal.cloud.credentials.clientCert" . | quote }} +TEMPORAL_CLOUD_CLIENT_KEY: {{ include "airbyte.temporal.cloud.credentials.clientKey" . | quote }} +{{- end }} + +{{/* +Renders the temporal.database.engine value +*/}} +{{- define "airbyte.temporal.database.engine" }} + {{- .Values.temporal.database.engine | default "postgresql" }} +{{- end }} + +{{/* +Renders the temporal.database.engine environment variable +*/}} +{{- define "airbyte.temporal.database.engine.env" }} +- name: DB + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DB +{{- end }} + +{{/* +Renders the temporal.database.host environment variable +*/}} +{{- define "airbyte.temporal.database.host.env" }} +- name: POSTGRES_SEEDS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATABASE_HOST +{{- end }} + +{{/* +Renders the temporal.database.port environment variable +*/}} +{{- define "airbyte.temporal.database.port.env" }} +- name: DB_PORT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATABASE_PORT +{{- end }} + +{{/* +Renders the temporal.database.user secret key +*/}} +{{- define "airbyte.temporal.database.user.secretKey" }} + {{- .Values.temporal.database.userSecretKey | default "DATABASE_USER" }} +{{- end }} + +{{/* +Renders the temporal.database.user environment variable +*/}} +{{- define "airbyte.temporal.database.user.env" }} +- name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: {{ include "airbyte.database.secretName" . }} + key: {{ include "airbyte.temporal.database.user.secretKey" . }} +{{- end }} + +{{/* +Renders the temporal.database.password secret key +*/}} +{{- define "airbyte.temporal.database.password.secretKey" }} + {{- .Values.temporal.database.passwordSecretKey | default "DATABASE_PASSWORD" }} +{{- end }} + +{{/* +Renders the temporal.database.password environment variable +*/}} +{{- define "airbyte.temporal.database.password.env" }} +- name: POSTGRES_PWD + valueFrom: + secretKeyRef: + name: {{ include "airbyte.database.secretName" . }} + key: {{ include "airbyte.temporal.database.password.secretKey" . }} +{{- end }} + +{{/* +Renders the temporal.database.tlsEnabled value +*/}} +{{- define "airbyte.temporal.database.tlsEnabled" }} + {{- ternary "true" "false" (eq .Values.global.database.type "external") }} +{{- end }} + +{{/* +Renders the temporal.database.tlsEnabled environment variable +*/}} +{{- define "airbyte.temporal.database.tlsEnabled.env" }} +- name: POSTGRES_TLS_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: POSTGRES_TLS_ENABLED +{{- end }} + +{{/* +Renders the temporal.database.tlsDisableHostVerification value +*/}} +{{- define "airbyte.temporal.database.tlsDisableHostVerification" }} + {{- ternary "true" "false" (eq .Values.global.database.type "external") }} +{{- end }} + +{{/* +Renders the temporal.database.tlsDisableHostVerification environment variable +*/}} +{{- define "airbyte.temporal.database.tlsDisableHostVerification.env" }} +- name: POSTGRES_TLS_DISABLE_HOST_VERIFICATION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: POSTGRES_TLS_DISABLE_HOST_VERIFICATION +{{- end }} + +{{/* +Renders the temporal.database.sqlTlsEnabled value +*/}} +{{- define "airbyte.temporal.database.sqlTlsEnabled" }} + {{- ternary "true" "false" (eq .Values.global.database.type "external") }} +{{- end }} + +{{/* +Renders the temporal.database.sqlTlsEnabled environment variable +*/}} +{{- define "airbyte.temporal.database.sqlTlsEnabled.env" }} +- name: SQL_TLS_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: SQL_TLS_ENABLED +{{- end }} + +{{/* +Renders the temporal.database.sqlTlsDisableHostVerification value +*/}} +{{- define "airbyte.temporal.database.sqlTlsDisableHostVerification" }} + {{- ternary "true" "false" (eq .Values.global.database.type "external") }} +{{- end }} + +{{/* +Renders the temporal.database.sqlTlsDisableHostVerification environment variable +*/}} +{{- define "airbyte.temporal.database.sqlTlsDisableHostVerification.env" }} +- name: SQL_TLS_DISABLE_HOST_VERIFICATION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: SQL_TLS_DISABLE_HOST_VERIFICATION +{{- end }} + +{{/* +Renders the set of all temporal.database environment variables +*/}} +{{- define "airbyte.temporal.database.envs" }} +{{- include "airbyte.temporal.database.engine.env" . }} +{{- include "airbyte.temporal.database.host.env" . }} +{{- include "airbyte.temporal.database.port.env" . }} +{{- include "airbyte.temporal.database.user.env" . }} +{{- include "airbyte.temporal.database.password.env" . }} +{{- include "airbyte.temporal.database.tlsEnabled.env" . }} +{{- include "airbyte.temporal.database.tlsDisableHostVerification.env" . }} +{{- include "airbyte.temporal.database.sqlTlsEnabled.env" . }} +{{- include "airbyte.temporal.database.sqlTlsDisableHostVerification.env" . }} +{{- end }} + +{{/* +Renders the set of all temporal.database config map variables +*/}} +{{- define "airbyte.temporal.database.configVars" }} +DB: {{ include "airbyte.temporal.database.engine" . | quote }} +POSTGRES_TLS_ENABLED: {{ include "airbyte.temporal.database.tlsEnabled" . | quote }} +POSTGRES_TLS_DISABLE_HOST_VERIFICATION: {{ include "airbyte.temporal.database.tlsDisableHostVerification" . | quote }} +SQL_TLS_ENABLED: {{ include "airbyte.temporal.database.sqlTlsEnabled" . | quote }} +SQL_TLS_DISABLE_HOST_VERIFICATION: {{ include "airbyte.temporal.database.sqlTlsDisableHostVerification" . | quote }} +{{- end }} + +{{/* +Renders the temporal.sdk.rpc.timeout value +*/}} +{{- define "airbyte.temporal.sdk.rpc.timeout" }} + {{- .Values.temporal.sdk.rpc.timeout | default "120s" }} +{{- end }} + +{{/* +Renders the temporal.sdk.rpc.timeout environment variable +*/}} +{{- define "airbyte.temporal.sdk.rpc.timeout.env" }} +- name: TEMPORAL_SDK_RPC_TIMEOUT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TEMPORAL_SDK_RPC_TIMEOUT +{{- end }} + +{{/* +Renders the temporal.sdk.rpc.longPollTimeout value +*/}} +{{- define "airbyte.temporal.sdk.rpc.longPollTimeout" }} + {{- .Values.temporal.sdk.rpc.longPollTimeout | default "140s" }} +{{- end }} + +{{/* +Renders the temporal.sdk.rpc.longPollTimeout environment variable +*/}} +{{- define "airbyte.temporal.sdk.rpc.longPollTimeout.env" }} +- name: TEMPORAL_SDK_RPC_LONG_POLL_TIMEOUT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TEMPORAL_SDK_RPC_LONG_POLL_TIMEOUT +{{- end }} + +{{/* +Renders the temporal.sdk.rpc.queryTimeout value +*/}} +{{- define "airbyte.temporal.sdk.rpc.queryTimeout" }} + {{- .Values.temporal.sdk.rpc.queryTimeout | default "20s" }} +{{- end }} + +{{/* +Renders the temporal.sdk.rpc.queryTimeout environment variable +*/}} +{{- define "airbyte.temporal.sdk.rpc.queryTimeout.env" }} +- name: TEMPORAL_SDK_RPC_QUERY_TIMEOUT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TEMPORAL_SDK_RPC_QUERY_TIMEOUT +{{- end }} + +{{/* +Renders the set of all temporal.sdk environment variables +*/}} +{{- define "airbyte.temporal.sdk.envs" }} +{{- include "airbyte.temporal.sdk.rpc.timeout.env" . }} +{{- include "airbyte.temporal.sdk.rpc.longPollTimeout.env" . }} +{{- include "airbyte.temporal.sdk.rpc.queryTimeout.env" . }} +{{- end }} + +{{/* +Renders the set of all temporal.sdk config map variables +*/}} +{{- define "airbyte.temporal.sdk.configVars" }} +TEMPORAL_SDK_RPC_TIMEOUT: {{ include "airbyte.temporal.sdk.rpc.timeout" . | quote }} +TEMPORAL_SDK_RPC_LONG_POLL_TIMEOUT: {{ include "airbyte.temporal.sdk.rpc.longPollTimeout" . | quote }} +TEMPORAL_SDK_RPC_QUERY_TIMEOUT: {{ include "airbyte.temporal.sdk.rpc.queryTimeout" . | quote }} +{{- end }} + +{{/* +Renders the temporal.worker.ports value +*/}} +{{- define "airbyte.temporal.worker.ports" }} + {{- join "," (seq 9000 9040) }} +{{- end }} + +{{/* +Renders the temporal.worker.ports environment variable +*/}} +{{- define "airbyte.temporal.worker.ports.env" }} +- name: TEMPORAL_WORKER_PORTS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TEMPORAL_WORKER_PORTS +{{- end }} + +{{/* +Renders the set of all temporal.worker environment variables +*/}} +{{- define "airbyte.temporal.worker.envs" }} +{{- include "airbyte.temporal.worker.ports.env" . }} +{{- end }} + +{{/* +Renders the set of all temporal.worker config map variables +*/}} +{{- define "airbyte.temporal.worker.configVars" }} +TEMPORAL_WORKER_PORTS: {{ include "airbyte.temporal.worker.ports" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_topology.tpl b/helm/airbyte-v2/templates/config/_topology.tpl new file mode 100644 index 00000000..863455e6 --- /dev/null +++ b/helm/airbyte-v2/templates/config/_topology.tpl @@ -0,0 +1,98 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Topology Configuration +*/}} + +{{/* +Renders the global.topology.nodeSelectorLabel value +*/}} +{{- define "airbyte.topology.nodeSelectorLabel" }} + {{- .Values.global.topology.nodeSelectorLabel | default "airbyte/node-pool" }} +{{- end }} + +{{/* +Renders the topology.nodeSelectorLabel environment variable +*/}} +{{- define "airbyte.topology.nodeSelectorLabel.env" }} +- name: NODE_SELECTOR_LABEL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: NODE_SELECTOR_LABEL +{{- end }} + +{{/* +Renders the global.topology.nodeSelectors.mainNodePool value +*/}} +{{- define "airbyte.topology.nodeSelectors.mainNodePool" }} + {{- .Values.global.topology.nodeSelectors.mainNodePool | default "main" }} +{{- end }} + +{{/* +Renders the topology.nodeSelectors.mainNodePool environment variable +*/}} +{{- define "airbyte.topology.nodeSelectors.mainNodePool.env" }} +- name: NODE_SELECTOR_LABEL_MAIN_NODE_POOL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: NODE_SELECTOR_LABEL_MAIN_NODE_POOL +{{- end }} + +{{/* +Renders the global.topology.nodeSelectors.jobsNodePool value +*/}} +{{- define "airbyte.topology.nodeSelectors.jobsNodePool" }} + {{- .Values.global.topology.nodeSelectors.jobsNodePool | default "jobs" }} +{{- end }} + +{{/* +Renders the topology.nodeSelectors.jobsNodePool environment variable +*/}} +{{- define "airbyte.topology.nodeSelectors.jobsNodePool.env" }} +- name: NODE_SELECTOR_LABEL_JOBS_NODE_POOL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: NODE_SELECTOR_LABEL_JOBS_NODE_POOL +{{- end }} + +{{/* +Renders the global.topology.nodeSelectors.quickJobsNodePool value +*/}} +{{- define "airbyte.topology.nodeSelectors.quickJobsNodePool" }} + {{- .Values.global.topology.nodeSelectors.quickJobsNodePool | default "quick-jobs" }} +{{- end }} + +{{/* +Renders the topology.nodeSelectors.quickJobsNodePool environment variable +*/}} +{{- define "airbyte.topology.nodeSelectors.quickJobsNodePool.env" }} +- name: NODE_SELECTOR_LABEL_QUICK_JOBS_NODE_POOL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: NODE_SELECTOR_LABEL_QUICK_JOBS_NODE_POOL +{{- end }} + +{{/* +Renders the set of all topology environment variables +*/}} +{{- define "airbyte.topology.envs" }} +{{- include "airbyte.topology.nodeSelectorLabel.env" . }} +{{- include "airbyte.topology.nodeSelectors.mainNodePool.env" . }} +{{- include "airbyte.topology.nodeSelectors.jobsNodePool.env" . }} +{{- include "airbyte.topology.nodeSelectors.quickJobsNodePool.env" . }} +{{- end }} + +{{/* +Renders the set of all topology config map variables +*/}} +{{- define "airbyte.topology.configVars" }} +NODE_SELECTOR_LABEL: {{ include "airbyte.topology.nodeSelectorLabel" . | quote }} +NODE_SELECTOR_LABEL_MAIN_NODE_POOL: {{ include "airbyte.topology.nodeSelectors.mainNodePool" . | quote }} +NODE_SELECTOR_LABEL_JOBS_NODE_POOL: {{ include "airbyte.topology.nodeSelectors.jobsNodePool" . | quote }} +NODE_SELECTOR_LABEL_QUICK_JOBS_NODE_POOL: {{ include "airbyte.topology.nodeSelectors.quickJobsNodePool" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_tracking.tpl b/helm/airbyte-v2/templates/config/_tracking.tpl new file mode 100644 index 00000000..367e48ab --- /dev/null +++ b/helm/airbyte-v2/templates/config/_tracking.tpl @@ -0,0 +1,82 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Tracking Configuration +*/}} + +{{/* +Renders the global.tracking.enabled value +*/}} +{{- define "airbyte.tracking.enabled" }} + {{- if eq .Values.global.tracking.enabled nil }} + {{- true }} + {{- else }} + {{- .Values.global.tracking.enabled }} + {{- end }} +{{- end }} + +{{/* +Renders the tracking.enabled environment variable +*/}} +{{- define "airbyte.tracking.enabled.env" }} +- name: TRACKING_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TRACKING_ENABLED +{{- end }} + +{{/* +Renders the global.tracking.strategy value +*/}} +{{- define "airbyte.tracking.strategy" }} + {{- .Values.global.tracking.strategy | default "logging" }} +{{- end }} + +{{/* +Renders the tracking.strategy environment variable +*/}} +{{- define "airbyte.tracking.strategy.env" }} +- name: TRACKING_STRATEGY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: TRACKING_STRATEGY +{{- end }} + +{{/* +Renders the global.tracking.segment.writeKeySecretKey value +*/}} +{{- define "airbyte.tracking.segment.writeKeySecretKey" }} + {{- .Values.global.tracking.segment.writeKeySecretKey | default "7UDdp5K55CyiGgsauOr2pNNujGvmhaeu" }} +{{- end }} + +{{/* +Renders the tracking.segment.writeKeySecretKey environment variable +*/}} +{{- define "airbyte.tracking.segment.writeKeySecretKey.env" }} +- name: SEGMENT_WRITE_KEY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: SEGMENT_WRITE_KEY +{{- end }} + +{{/* +Renders the set of all tracking environment variables +*/}} +{{- define "airbyte.tracking.envs" }} +{{- include "airbyte.tracking.enabled.env" . }} +{{- include "airbyte.tracking.strategy.env" . }} +{{- include "airbyte.tracking.segment.writeKeySecretKey.env" . }} +{{- end }} + +{{/* +Renders the set of all tracking config map variables +*/}} +{{- define "airbyte.tracking.configVars" }} +TRACKING_ENABLED: {{ include "airbyte.tracking.enabled" . | quote }} +TRACKING_STRATEGY: {{ include "airbyte.tracking.strategy" . | quote }} +SEGMENT_WRITE_KEY: {{ include "airbyte.tracking.segment.writeKeySecretKey" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_webapp.tpl b/helm/airbyte-v2/templates/config/_webapp.tpl new file mode 100644 index 00000000..8688ffeb --- /dev/null +++ b/helm/airbyte-v2/templates/config/_webapp.tpl @@ -0,0 +1,84 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Webapp Configuration +*/}} + +{{/* +Renders the webapp.api.url value. Airbyte v2 unified server exposes the public API at /api/public/v1/ (not /api/v1/); wrong path causes 404 or unexpected responses and UI errors like "Cannot read properties of undefined (reading 'sort')". +*/}} +{{- define "airbyte.webapp.api.url" }} + {{- .Values.webapp.api.url | default "/api/public/v1/" }} +{{- end }} + +{{/* +Renders the webapp.api.url environment variable +*/}} +{{- define "airbyte.webapp.api.url.env" }} +- name: API_URL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: API_URL +{{- end }} + +{{/* +Renders the webapp.connectorBuilderServer.url value required by webapp or it may throw e.g. "Cannot read properties of undefined (reading 'sort')". +*/}} +{{- define "airbyte.webapp.connectorBuilderServer.url" }} + {{- trimSuffix "/" (index (index .Values.webapp "connectorBuilderServer" | default dict) "url" | default "/connector-builder-api") }} +{{- end }} + +{{/* +Renders the webapp CONNECTOR_BUILDER_API_HOST: must be a host:port for nginx upstream. When webapp.connectorBuilderServer.url is a path (e.g. /connector-builder-api), use internal service host so nginx is valid; when url is a full host/URL use it; otherwise internal service host. +*/}} +{{- define "airbyte.webapp.connectorBuilderServer.host" }} + {{- $url := trimSuffix "/" (index (index .Values.webapp "connectorBuilderServer" | default dict) "url" | default "") }} + {{- $internalHost := printf "%s-airbyte-connector-builder-server-svc.%s:%d" .Release.Name .Release.Namespace (int .Values.connectorBuilderServer.service.port) }} + {{- if and $url (not (hasPrefix "/" $url)) }} + {{- $url }} + {{- else }} + {{- $internalHost }} + {{- end }} +{{- end }} + +{{/* +Renders the webapp.connectorBuilderServer.host environment variable +*/}} +{{- define "airbyte.webapp.connectorBuilderServer.host.env" }} +- name: CONNECTOR_BUILDER_API_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONNECTOR_BUILDER_API_HOST +{{- end }} + +{{/* +Renders the webapp.connectorBuilderServer.url environment variable (CONNECTOR_BUILDER_API_URL, same as airbyte-v1). +*/}} +{{- define "airbyte.webapp.connectorBuilderServer.url.env" }} +- name: CONNECTOR_BUILDER_API_URL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONNECTOR_BUILDER_API_URL +{{- end }} + +{{/* +Renders the set of all webapp environment variables +*/}} +{{- define "airbyte.webapp.envs" }} +{{- include "airbyte.webapp.api.url.env" . }} +{{- include "airbyte.webapp.connectorBuilderServer.host.env" . }} +{{- include "airbyte.webapp.connectorBuilderServer.url.env" . }} +{{- end }} + +{{/* +Renders the set of all webapp config map variables +*/}} +{{- define "airbyte.webapp.configVars" }} +API_URL: {{ include "airbyte.webapp.api.url" . | quote }} +CONNECTOR_BUILDER_API_HOST: {{ include "airbyte.webapp.connectorBuilderServer.host" . | quote }} +CONNECTOR_BUILDER_API_URL: {{ include "airbyte.webapp.connectorBuilderServer.url" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_worker.tpl b/helm/airbyte-v2/templates/config/_worker.tpl new file mode 100644 index 00000000..b5b1b698 --- /dev/null +++ b/helm/airbyte-v2/templates/config/_worker.tpl @@ -0,0 +1,346 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Worker Configuration +*/}} + +{{/* +Renders the worker.activityMaxAttempt value +*/}} +{{- define "airbyte.worker.activityMaxAttempt" }} + {{- .Values.worker.activityMaxAttempt }} +{{- end }} + +{{/* +Renders the worker.activityMaxAttempt environment variable +*/}} +{{- define "airbyte.worker.activityMaxAttempt.env" }} +- name: ACTIVITY_MAX_ATTEMPT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: ACTIVITY_MAX_ATTEMPT +{{- end }} + +{{/* +Renders the worker.activityInitialDelayBetweenAttemptsSeconds value +*/}} +{{- define "airbyte.worker.activityInitialDelayBetweenAttemptsSeconds" }} + {{- .Values.worker.activityInitialDelayBetweenAttemptsSeconds }} +{{- end }} + +{{/* +Renders the worker.activityInitialDelayBetweenAttemptsSeconds environment variable +*/}} +{{- define "airbyte.worker.activityInitialDelayBetweenAttemptsSeconds.env" }} +- name: ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS +{{- end }} + +{{/* +Renders the worker.activityMaxDelayBetweenAttemptsSeconds value +*/}} +{{- define "airbyte.worker.activityMaxDelayBetweenAttemptsSeconds" }} + {{- .Values.worker.activityMaxDelayBetweenAttemptsSeconds }} +{{- end }} + +{{/* +Renders the worker.activityMaxDelayBetweenAttemptsSeconds environment variable +*/}} +{{- define "airbyte.worker.activityMaxDelayBetweenAttemptsSeconds.env" }} +- name: ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS +{{- end }} + +{{/* +Renders the worker.configRoot value +*/}} +{{- define "airbyte.worker.configRoot" }} + {{- .Values.worker.configRoot | default "/configs" }} +{{- end }} + +{{/* +Renders the worker.configRoot environment variable +*/}} +{{- define "airbyte.worker.configRoot.env" }} +- name: CONFIG_ROOT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONFIG_ROOT +{{- end }} + +{{/* +Renders the worker.maxCheckWorkers value +*/}} +{{- define "airbyte.worker.maxCheckWorkers" }} + {{- .Values.worker.maxCheckWorkers | default 30 }} +{{- end }} + +{{/* +Renders the worker.maxCheckWorkers environment variable +*/}} +{{- define "airbyte.worker.maxCheckWorkers.env" }} +- name: MAX_CHECK_WORKERS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MAX_CHECK_WORKERS +{{- end }} + +{{/* +Renders the worker.maxNotifyWorkers value +*/}} +{{- define "airbyte.worker.maxNotifyWorkers" }} + {{- .Values.worker.maxNotifyWorkers | default 30 }} +{{- end }} + +{{/* +Renders the worker.maxNotifyWorkers environment variable +*/}} +{{- define "airbyte.worker.maxNotifyWorkers.env" }} +- name: MAX_NOTIFY_WORKERS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MAX_NOTIFY_WORKERS +{{- end }} + +{{/* +Renders the worker.maxSyncWorkers value +*/}} +{{- define "airbyte.worker.maxSyncWorkers" }} + {{- .Values.worker.maxSyncWorkers | default 5 }} +{{- end }} + +{{/* +Renders the worker.maxSyncWorkers environment variable +*/}} +{{- define "airbyte.worker.maxSyncWorkers.env" }} +- name: MAX_SYNC_WORKERS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: MAX_SYNC_WORKERS +{{- end }} + +{{/* +Renders the worker.syncJobMaxAttempts value +*/}} +{{- define "airbyte.worker.syncJobMaxAttempts" }} + {{- .Values.worker.syncJobMaxAttempts | default 3 }} +{{- end }} + +{{/* +Renders the worker.syncJobMaxAttempts environment variable +*/}} +{{- define "airbyte.worker.syncJobMaxAttempts.env" }} +- name: SYNC_JOB_MAX_ATTEMPTS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: SYNC_JOB_MAX_ATTEMPTS +{{- end }} + +{{/* +Renders the worker.syncJobMaxTimeoutDays value +*/}} +{{- define "airbyte.worker.syncJobMaxTimeoutDays" }} + {{- .Values.worker.syncJobMaxTimeoutDays | default 3 }} +{{- end }} + +{{/* +Renders the worker.syncJobMaxTimeoutDays environment variable +*/}} +{{- define "airbyte.worker.syncJobMaxTimeoutDays.env" }} +- name: SYNC_JOB_MAX_TIMEOUT_DAYS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: SYNC_JOB_MAX_TIMEOUT_DAYS +{{- end }} + +{{/* +Renders the worker.syncJobInitRetryTimeoutMinutes value +*/}} +{{- define "airbyte.worker.syncJobInitRetryTimeoutMinutes" }} + {{- .Values.worker.syncJobInitRetryTimeoutMinutes | default 5 }} +{{- end }} + +{{/* +Renders the worker.syncJobInitRetryTimeoutMinutes environment variable +*/}} +{{- define "airbyte.worker.syncJobInitRetryTimeoutMinutes.env" }} +- name: SYNC_JOB_INIT_RETRY_TIMEOUT_MINUTES + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: SYNC_JOB_INIT_RETRY_TIMEOUT_MINUTES +{{- end }} + +{{/* +Renders the worker.useCustomNodeSelector value +*/}} +{{- define "airbyte.worker.useCustomNodeSelector" }} + {{- if eq .Values.worker.useCustomNodeSelector nil }} + {{- false }} + {{- else }} + {{- .Values.worker.useCustomNodeSelector }} + {{- end }} +{{- end }} + +{{/* +Renders the worker.useCustomNodeSelector environment variable +*/}} +{{- define "airbyte.worker.useCustomNodeSelector.env" }} +- name: USE_CUSTOM_NODE_SELECTOR + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: USE_CUSTOM_NODE_SELECTOR +{{- end }} + +{{/* +Renders the worker.useStreamCapableState value +*/}} +{{- define "airbyte.worker.useStreamCapableState" }} + {{- if eq .Values.worker.useStreamCapableState nil }} + {{- true }} + {{- else }} + {{- .Values.worker.useStreamCapableState }} + {{- end }} +{{- end }} + +{{/* +Renders the worker.useStreamCapableState environment variable +*/}} +{{- define "airbyte.worker.useStreamCapableState.env" }} +- name: USE_STREAM_CAPABLE_STATE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: USE_STREAM_CAPABLE_STATE +{{- end }} + +{{/* +Renders the worker.workflowFailureRestartDelaySeconds value +*/}} +{{- define "airbyte.worker.workflowFailureRestartDelaySeconds" }} + {{- .Values.worker.workflowFailureRestartDelaySeconds }} +{{- end }} + +{{/* +Renders the worker.workflowFailureRestartDelaySeconds environment variable +*/}} +{{- define "airbyte.worker.workflowFailureRestartDelaySeconds.env" }} +- name: WORKFLOW_FAILURE_RESTART_DELAY_SECONDS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKFLOW_FAILURE_RESTART_DELAY_SECONDS +{{- end }} + +{{/* +Renders the worker.workspaceDockerMount value +*/}} +{{- define "airbyte.worker.workspaceDockerMount" }} + {{- .Values.worker.workspaceDockerMount | default "airbyte_workspace" }} +{{- end }} + +{{/* +Renders the worker.workspaceDockerMount environment variable +*/}} +{{- define "airbyte.worker.workspaceDockerMount.env" }} +- name: WORKSPACE_DOCKER_MOUNT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKSPACE_DOCKER_MOUNT +{{- end }} + +{{/* +Renders the worker.workspaceRoot value +*/}} +{{- define "airbyte.worker.workspaceRoot" }} + {{- .Values.worker.workspaceRoot | default "/workspace" }} +{{- end }} + +{{/* +Renders the worker.workspaceRoot environment variable +*/}} +{{- define "airbyte.worker.workspaceRoot.env" }} +- name: WORKSPACE_ROOT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKSPACE_ROOT +{{- end }} + +{{/* +Renders the worker.environment value +*/}} +{{- define "airbyte.worker.environment" }} + {{- "kubernetes" }} +{{- end }} + +{{/* +Renders the worker.environment environment variable +*/}} +{{- define "airbyte.worker.environment.env" }} +- name: WORKER_ENVIRONMENT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKER_ENVIRONMENT +{{- end }} + +{{/* +Renders the set of all worker environment variables +*/}} +{{- define "airbyte.worker.envs" }} +{{- include "airbyte.worker.activityMaxAttempt.env" . }} +{{- include "airbyte.worker.activityInitialDelayBetweenAttemptsSeconds.env" . }} +{{- include "airbyte.worker.activityMaxDelayBetweenAttemptsSeconds.env" . }} +{{- include "airbyte.worker.configRoot.env" . }} +{{- include "airbyte.worker.maxCheckWorkers.env" . }} +{{- include "airbyte.worker.maxNotifyWorkers.env" . }} +{{- include "airbyte.worker.maxSyncWorkers.env" . }} +{{- include "airbyte.worker.syncJobMaxAttempts.env" . }} +{{- include "airbyte.worker.syncJobMaxTimeoutDays.env" . }} +{{- include "airbyte.worker.syncJobInitRetryTimeoutMinutes.env" . }} +{{- include "airbyte.worker.useCustomNodeSelector.env" . }} +{{- include "airbyte.worker.useStreamCapableState.env" . }} +{{- include "airbyte.worker.workflowFailureRestartDelaySeconds.env" . }} +{{- include "airbyte.worker.workspaceDockerMount.env" . }} +{{- include "airbyte.worker.workspaceRoot.env" . }} +{{- include "airbyte.worker.environment.env" . }} +{{- end }} + +{{/* +Renders the set of all worker config map variables +*/}} +{{- define "airbyte.worker.configVars" }} +ACTIVITY_MAX_ATTEMPT: {{ include "airbyte.worker.activityMaxAttempt" . | quote }} +ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS: {{ include "airbyte.worker.activityInitialDelayBetweenAttemptsSeconds" . | quote }} +ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS: {{ include "airbyte.worker.activityMaxDelayBetweenAttemptsSeconds" . | quote }} +CONFIG_ROOT: {{ include "airbyte.worker.configRoot" . | quote }} +MAX_CHECK_WORKERS: {{ include "airbyte.worker.maxCheckWorkers" . | quote }} +MAX_NOTIFY_WORKERS: {{ include "airbyte.worker.maxNotifyWorkers" . | quote }} +MAX_SYNC_WORKERS: {{ include "airbyte.worker.maxSyncWorkers" . | quote }} +SYNC_JOB_MAX_ATTEMPTS: {{ include "airbyte.worker.syncJobMaxAttempts" . | quote }} +SYNC_JOB_MAX_TIMEOUT_DAYS: {{ include "airbyte.worker.syncJobMaxTimeoutDays" . | quote }} +SYNC_JOB_INIT_RETRY_TIMEOUT_MINUTES: {{ include "airbyte.worker.syncJobInitRetryTimeoutMinutes" . | quote }} +USE_CUSTOM_NODE_SELECTOR: {{ include "airbyte.worker.useCustomNodeSelector" . | quote }} +USE_STREAM_CAPABLE_STATE: {{ include "airbyte.worker.useStreamCapableState" . | quote }} +WORKFLOW_FAILURE_RESTART_DELAY_SECONDS: {{ include "airbyte.worker.workflowFailureRestartDelaySeconds" . | quote }} +WORKSPACE_DOCKER_MOUNT: {{ include "airbyte.worker.workspaceDockerMount" . | quote }} +WORKSPACE_ROOT: {{ include "airbyte.worker.workspaceRoot" . | quote }} +WORKER_ENVIRONMENT: {{ include "airbyte.worker.environment" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_workloadApiServer.tpl b/helm/airbyte-v2/templates/config/_workloadApiServer.tpl new file mode 100644 index 00000000..a8259f7f --- /dev/null +++ b/helm/airbyte-v2/templates/config/_workloadApiServer.tpl @@ -0,0 +1,153 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Workloadapiserver Configuration +*/}} + +{{/* +Renders the workloadApiServer secret name +*/}} +{{- define "airbyte.workloadApiServer.secretName" }} +{{- if .Values.workloadApiServer.secretName }} + {{- .Values.workloadApiServer.secretName }} +{{- else }} + {{- .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} +{{- end }} +{{- end }} + +{{/* +Renders the workloadApiServer.enabled value +*/}} +{{- define "airbyte.workloadApiServer.enabled" }} + {{- if eq .Values.workloadApiServer.enabled nil }} + {{- true }} + {{- else }} + {{- .Values.workloadApiServer.enabled }} + {{- end }} +{{- end }} + +{{/* +Renders the workloadApiServer.enabled environment variable +*/}} +{{- define "airbyte.workloadApiServer.enabled.env" }} +- name: WORKLOAD_API_SERVER_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKLOAD_API_SERVER_ENABLED +{{- end }} + +{{/* +Renders the workloadApiServer.host value +*/}} +{{- define "airbyte.workloadApiServer.host" }} + {{- ternary (include "airbyte.common.airbyteUrl" .) (printf "http://%s-workload-api-server-svc.%s:%d" .Release.Name .Release.Namespace (int .Values.workloadApiServer.service.port)) (eq (include "airbyte.cluster.type" .) "data-plane") }} +{{- end }} + +{{/* +Renders the workloadApiServer.host environment variable +*/}} +{{- define "airbyte.workloadApiServer.host.env" }} +- name: WORKLOAD_API_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKLOAD_API_HOST +{{- end }} + +{{/* +Release-unique default workload API bearer token (stable across upgrades). +*/}} +{{- define "airbyte.workloadApiServer.bearerToken.default" }} + {{- printf "workload-api-%s" (include "airbyte.secretSeed.hashBase" . | sha256sum | trunc 32) }} +{{- end }} + +{{/* +Renders the workloadApiServer.bearerToken value +*/}} +{{- define "airbyte.workloadApiServer.bearerToken" }} + {{- .Values.workloadApiServer.bearerToken | default (include "airbyte.workloadApiServer.bearerToken.default" .) }} +{{- end }} + +{{/* +Renders the workloadApiServer.bearerToken secret key +*/}} +{{- define "airbyte.workloadApiServer.bearerToken.secretKey" }} + {{- .Values.workloadApiServer.bearerTokenSecretKey | default "WORKLOAD_API_BEARER_TOKEN" }} +{{- end }} + +{{/* +Renders the workloadApiServer.bearerToken environment variable +*/}} +{{- define "airbyte.workloadApiServer.bearerToken.env" }} +- name: WORKLOAD_API_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: {{ include "airbyte.workloadApiServer.secretName" . }} + key: {{ include "airbyte.workloadApiServer.bearerToken.secretKey" . }} +{{- end }} + +{{/* +Renders the workloadApiServer.bearerTokenSecretName value +*/}} +{{- define "airbyte.workloadApiServer.bearerTokenSecretName" }} + {{- .Values.workloadApiServer.bearerTokenSecretName | default (include "airbyte.workloadApiServer.secretName" .) }} +{{- end }} + +{{/* +Renders the workloadApiServer.bearerTokenSecretName environment variable +*/}} +{{- define "airbyte.workloadApiServer.bearerTokenSecretName.env" }} +- name: WORKLOAD_API_BEARER_TOKEN_SECRET_NAME + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKLOAD_API_BEARER_TOKEN_SECRET_NAME +{{- end }} + +{{/* +Renders the workloadApiServer.bearerTokenSecretKey value +*/}} +{{- define "airbyte.workloadApiServer.bearerTokenSecretKey" }} + {{- .Values.workloadApiServer.bearerTokenSecretKey | default "WORKLOAD_API_BEARER_TOKEN" }} +{{- end }} + +{{/* +Renders the workloadApiServer.bearerTokenSecretKey environment variable +*/}} +{{- define "airbyte.workloadApiServer.bearerTokenSecretKey.env" }} +- name: WORKLOAD_API_BEARER_TOKEN_SECRET_KEY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKLOAD_API_BEARER_TOKEN_SECRET_KEY +{{- end }} + +{{/* +Renders the set of all workloadApiServer environment variables +*/}} +{{- define "airbyte.workloadApiServer.envs" }} +{{- include "airbyte.workloadApiServer.enabled.env" . }} +{{- include "airbyte.workloadApiServer.host.env" . }} +{{- include "airbyte.workloadApiServer.bearerToken.env" . }} +{{- include "airbyte.workloadApiServer.bearerTokenSecretName.env" . }} +{{- include "airbyte.workloadApiServer.bearerTokenSecretKey.env" . }} +{{- end }} + +{{/* +Renders the set of all workloadApiServer config map variables +*/}} +{{- define "airbyte.workloadApiServer.configVars" }} +WORKLOAD_API_SERVER_ENABLED: {{ include "airbyte.workloadApiServer.enabled" . | quote }} +WORKLOAD_API_HOST: {{ include "airbyte.workloadApiServer.host" . | quote }} +WORKLOAD_API_BEARER_TOKEN_SECRET_NAME: {{ include "airbyte.workloadApiServer.bearerTokenSecretName" . | quote }} +WORKLOAD_API_BEARER_TOKEN_SECRET_KEY: {{ include "airbyte.workloadApiServer.bearerTokenSecretKey" . | quote }} +{{- end }} + +{{/* +Renders the set of all workloadApiServer secret variables +*/}} +{{- define "airbyte.workloadApiServer.secrets" }} +WORKLOAD_API_BEARER_TOKEN: {{ include "airbyte.workloadApiServer.bearerToken" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_workloadLauncher.tpl b/helm/airbyte-v2/templates/config/_workloadLauncher.tpl new file mode 100644 index 00000000..453e8f9e --- /dev/null +++ b/helm/airbyte-v2/templates/config/_workloadLauncher.tpl @@ -0,0 +1,404 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Workloadlauncher Configuration +*/}} + +{{/* +Override with workloadLauncher.controlPlane.tokenEndpoint in values if needed. +*/}} +{{- define "airbyte.workloadLauncher.controlPlane.tokenEndpoint" }} + {{- $internalBase := printf "http://%s-airbyte-server-svc.%s:%d" .Release.Name .Release.Namespace (int .Values.server.service.port) }} + {{- $internalUrl := printf "%s/api/v1/dataplanes/token" $internalBase }} + {{- $publicUrl := printf "%s/api/v1/dataplanes/token" (include "airbyte.common.airbyteUrl" .) }} + {{- $clusterType := include "airbyte.common.cluster.type" . }} + {{- $useInternal := and (ne $clusterType "data-plane") (ne .Values.workloadLauncher.controlPlane.useInternalUrl false) }} + {{- if .Values.workloadLauncher.controlPlane.tokenEndpoint }} + {{- $base := .Values.workloadLauncher.controlPlane.tokenEndpoint | trimSuffix "/" }} + {{- if hasSuffix "/api/v1/dataplanes/token" $base }}{{- $base }}{{- else }}{{- printf "%s/api/v1/dataplanes/token" $base }}{{- end }} + {{- else if $useInternal }} + {{- $internalUrl }} + {{- else }} + {{- $publicUrl }} + {{- end }} +{{- end }} + +{{/* +Renders the workloadLauncher.controlPlane.tokenEndpoint environment variable +*/}} +{{- define "airbyte.workloadLauncher.controlPlane.tokenEndpoint.env" }} +- name: CONTROL_PLANE_TOKEN_ENDPOINT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONTROL_PLANE_TOKEN_ENDPOINT +{{- end }} + +{{/* +Renders the workloadLauncher.enabled value +*/}} +{{- define "airbyte.workloadLauncher.enabled" }} + {{- if eq .Values.workloadLauncher.enabled nil }} + {{- true }} + {{- else }} + {{- .Values.workloadLauncher.enabled }} + {{- end }} +{{- end }} + +{{/* +Renders the workloadLauncher.enabled environment variable +*/}} +{{- define "airbyte.workloadLauncher.enabled.env" }} +- name: WORKLOAD_LAUNCHER_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKLOAD_LAUNCHER_ENABLED +{{- end }} + +{{/* +Renders the workloadLauncher.parallelism value +*/}} +{{- define "airbyte.workloadLauncher.parallelism" }} + {{- .Values.workloadLauncher.parallelism | default 10 }} +{{- end }} + +{{/* +Renders the workloadLauncher.parallelism environment variable +*/}} +{{- define "airbyte.workloadLauncher.parallelism.env" }} +- name: WORKLOAD_LAUNCHER_PARALLELISM + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKLOAD_LAUNCHER_PARALLELISM +{{- end }} + +{{/* +Renders the set of all workloadLauncher environment variables +*/}} +{{- define "airbyte.workloadLauncher.envs" }} +{{- include "airbyte.workloadLauncher.controlPlane.tokenEndpoint.env" . }} +{{- include "airbyte.workloadLauncher.enabled.env" . }} +{{- include "airbyte.workloadLauncher.parallelism.env" . }} +{{- end }} + +{{/* +Renders the set of all workloadLauncher config map variables +*/}} +{{- define "airbyte.workloadLauncher.configVars" }} +CONTROL_PLANE_TOKEN_ENDPOINT: {{ include "airbyte.workloadLauncher.controlPlane.tokenEndpoint" . | quote }} +WORKLOAD_LAUNCHER_ENABLED: {{ include "airbyte.workloadLauncher.enabled" . | quote }} +WORKLOAD_LAUNCHER_PARALLELISM: {{ include "airbyte.workloadLauncher.parallelism" . | quote }} +{{- end }} + +{{/* +Renders the workloadLauncher.dataPlane secret name (where to read dataplane-client-id and dataplane-client-secret). +When existingSecretName is set, use it so credentials come from that secret (no hardcoding in values). +*/}} +{{- define "airbyte.workloadLauncher.dataPlane.secretName" }} +{{- if .Values.workloadLauncher.dataPlane.existingSecretName }} + {{- .Values.workloadLauncher.dataPlane.existingSecretName }} +{{- else if .Values.workloadLauncher.dataPlane.secretName }} + {{- .Values.workloadLauncher.dataPlane.secretName }} +{{- else }} + {{- .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} +{{- end }} +{{- end }} + +{{/* +True when dataplane credentials are read from an existing secret (don't write them into the chart secret). +*/}} +{{- define "airbyte.workloadLauncher.dataPlane.useExistingSecret" }} + {{- .Values.workloadLauncher.dataPlane.existingSecretName }} +{{- end }} + +{{/* +Release-unique default dataplane clientId (stable across upgrades). +Server DataplaneTokenServiceDataImpl expects a valid UUID; format 32 hex chars as 8-4-4-4-12. +*/}} +{{- define "airbyte.workloadLauncher.dataPlane.clientId.default" }} + {{- $h := (printf "id-%s" (include "airbyte.secretSeed.hashBase" .) | sha256sum | nospace | trunc 32) }} + {{- $p1 := trunc 8 $h }}{{ $r1 := trimPrefix $p1 $h }}{{ $p2 := trunc 4 $r1 }}{{ $r2 := trimPrefix $p2 $r1 }}{{ $p3 := trunc 4 $r2 }}{{ $r3 := trimPrefix $p3 $r2 }}{{ $p4 := trunc 4 $r3 }}{{ $r4 := trimPrefix $p4 $r3 }}{{ $p5 := trunc 12 $r4 }} + {{- $p1 }}-{{ $p2 }}-{{ $p3 }}-{{ $p4 }}-{{ $p5 }} +{{- end }} + +{{/* +Renders the workloadLauncher.dataPlane.clientId value. +Use or so empty string is treated as unset and default (UUID) is used; set explicitly to match service_accounts.id in DB. +*/}} +{{- define "airbyte.workloadLauncher.dataPlane.clientId" }} + {{- or .Values.workloadLauncher.dataPlane.clientId (include "airbyte.workloadLauncher.dataPlane.clientId.default" .) }} +{{- end }} + +{{/* +Renders the workloadLauncher.dataPlane.clientId secret key. +Default "dataplane-client-id" so server/launcher read the key that matches service_accounts in DB (secret may also use key dataplane-client-id). +*/}} +{{- define "airbyte.workloadLauncher.dataPlane.clientId.secretKey" }} + {{- .Values.workloadLauncher.dataPlane.clientIdSecretKey | default "dataplane-client-id" }} +{{- end }} + +{{/* +Renders the workloadLauncher.dataPlane.clientId environment variable +*/}} +{{- define "airbyte.workloadLauncher.dataPlane.clientId.env" }} +- name: DATAPLANE_CLIENT_ID + valueFrom: + secretKeyRef: + name: {{ include "airbyte.workloadLauncher.dataPlane.secretName" . }} + key: {{ include "airbyte.workloadLauncher.dataPlane.clientId.secretKey" . }} +{{- end }} + +{{/* +Renders the workloadLauncher.dataPlane.clientIdSecretName value (secret that holds dataplane-client-id). +*/}} +{{- define "airbyte.workloadLauncher.dataPlane.clientIdSecretName" }} + {{- .Values.workloadLauncher.dataPlane.clientIdSecretName | default (include "airbyte.workloadLauncher.dataPlane.secretName" .) }} +{{- end }} + +{{/* +Renders the workloadLauncher.dataPlane.clientIdSecretName environment variable +*/}} +{{- define "airbyte.workloadLauncher.dataPlane.clientIdSecretName.env" }} +- name: DATAPLANE_CLIENT_ID_SECRET_NAME + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATAPLANE_CLIENT_ID_SECRET_NAME +{{- end }} + +{{/* +Renders the workloadLauncher.dataPlane.clientIdSecretKey value +*/}} +{{- define "airbyte.workloadLauncher.dataPlane.clientIdSecretKey" }} + {{- .Values.workloadLauncher.dataPlane.clientIdSecretKey | default (include "airbyte.workloadLauncher.dataPlane.clientId.secretKey" .) }} +{{- end }} + +{{/* +Renders the workloadLauncher.dataPlane.clientIdSecretKey environment variable +*/}} +{{- define "airbyte.workloadLauncher.dataPlane.clientIdSecretKey.env" }} +- name: DATAPLANE_CLIENT_ID_SECRET_KEY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATAPLANE_CLIENT_ID_SECRET_KEY +{{- end }} + +{{/* +Release-unique default dataplane clientSecret (stable across upgrades). +*/}} +{{- define "airbyte.workloadLauncher.dataPlane.clientSecret.default" }} + {{- printf "dataplane-%s" (printf "secret-%s" (include "airbyte.secretSeed.hashBase" .) | sha256sum | trunc 32) }} +{{- end }} + +{{/* +Renders the workloadLauncher.dataPlane.clientSecret value. +Use or so empty string is treated as unset; set explicitly to match the secret for the service account in DB. +*/}} +{{- define "airbyte.workloadLauncher.dataPlane.clientSecret" }} + {{- or .Values.workloadLauncher.dataPlane.clientSecret (include "airbyte.workloadLauncher.dataPlane.clientSecret.default" .) }} +{{- end }} + +{{/* +Renders the workloadLauncher.dataPlane.clientSecret secret key +*/}} +{{- define "airbyte.workloadLauncher.dataPlane.clientSecret.secretKey" }} + {{- .Values.workloadLauncher.dataPlane.clientSecretSecretKey | default "dataplane-client-secret" }} +{{- end }} + +{{/* +Renders the workloadLauncher.dataPlane.clientSecret environment variable +*/}} +{{- define "airbyte.workloadLauncher.dataPlane.clientSecret.env" }} +- name: DATAPLANE_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: {{ include "airbyte.workloadLauncher.dataPlane.secretName" . }} + key: {{ include "airbyte.workloadLauncher.dataPlane.clientSecret.secretKey" . }} +{{- end }} + +{{/* +Renders the workloadLauncher.dataPlane.clientSecretSecretName value (secret that holds dataplane-client-secret). +*/}} +{{- define "airbyte.workloadLauncher.dataPlane.clientSecretSecretName" }} + {{- .Values.workloadLauncher.dataPlane.clientSecretSecretName | default (include "airbyte.workloadLauncher.dataPlane.secretName" .) }} +{{- end }} + +{{/* +Renders the workloadLauncher.dataPlane.clientSecretSecretName environment variable +*/}} +{{- define "airbyte.workloadLauncher.dataPlane.clientSecretSecretName.env" }} +- name: DATAPLANE_CLIENT_SECRET_SECRET_NAME + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATAPLANE_CLIENT_SECRET_SECRET_NAME +{{- end }} + +{{/* +Renders the workloadLauncher.dataPlane.clientSecretSecretKey value +*/}} +{{- define "airbyte.workloadLauncher.dataPlane.clientSecretSecretKey" }} + {{- .Values.workloadLauncher.dataPlane.clientSecretSecretKey | default (include "airbyte.workloadLauncher.dataPlane.clientSecret.secretKey" .) }} +{{- end }} + +{{/* +Renders the workloadLauncher.dataPlane.clientSecretSecretKey environment variable +*/}} +{{- define "airbyte.workloadLauncher.dataPlane.clientSecretSecretKey.env" }} +- name: DATAPLANE_CLIENT_SECRET_SECRET_KEY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATAPLANE_CLIENT_SECRET_SECRET_KEY +{{- end }} + +{{/* +Renders the set of all workloadLauncher.dataPlane environment variables +*/}} +{{- define "airbyte.workloadLauncher.dataPlane.envs" }} +{{- include "airbyte.workloadLauncher.dataPlane.clientId.env" . }} +{{- include "airbyte.workloadLauncher.dataPlane.clientIdSecretName.env" . }} +{{- include "airbyte.workloadLauncher.dataPlane.clientIdSecretKey.env" . }} +{{- include "airbyte.workloadLauncher.dataPlane.clientSecret.env" . }} +{{- include "airbyte.workloadLauncher.dataPlane.clientSecretSecretName.env" . }} +{{- include "airbyte.workloadLauncher.dataPlane.clientSecretSecretKey.env" . }} +{{- end }} + +{{/* +Renders the set of all workloadLauncher.dataPlane config map variables +*/}} +{{- define "airbyte.workloadLauncher.dataPlane.configVars" }} +DATAPLANE_CLIENT_ID_SECRET_NAME: {{ include "airbyte.workloadLauncher.dataPlane.clientIdSecretName" . | quote }} +DATAPLANE_CLIENT_ID_SECRET_KEY: {{ include "airbyte.workloadLauncher.dataPlane.clientIdSecretKey" . | quote }} +DATAPLANE_CLIENT_SECRET_SECRET_NAME: {{ include "airbyte.workloadLauncher.dataPlane.clientSecretSecretName" . | quote }} +DATAPLANE_CLIENT_SECRET_SECRET_KEY: {{ include "airbyte.workloadLauncher.dataPlane.clientSecretSecretKey" . | quote }} +{{- end }} + +{{/* +Renders the set of all workloadLauncher.dataPlane secret variables. +- dataplane-client-id / dataplane-client-secret: secret keys used by default for secretKeyRef; launcher calls token endpoint with these credentials (matches Airbyte server token endpoint / service_accounts). +- DATAPLANE_CLIENT_ID / DATAPLANE_CLIENT_SECRET: same values under env-style keys so existing secrets or tools that use these key names also work. +Pods get env vars DATAPLANE_CLIENT_ID and DATAPLANE_CLIENT_SECRET whose values are read from secret keys dataplane-client-id and dataplane-client-secret. +*/}} +{{- define "airbyte.workloadLauncher.dataPlane.secrets" }} +DATAPLANE_CLIENT_ID: {{ include "airbyte.workloadLauncher.dataPlane.clientId" . | quote }} +DATAPLANE_CLIENT_SECRET: {{ include "airbyte.workloadLauncher.dataPlane.clientSecret" . | quote }} +dataplane-client-id: {{ include "airbyte.workloadLauncher.dataPlane.clientId" . | quote }} +dataplane-client-secret: {{ include "airbyte.workloadLauncher.dataPlane.clientSecret" . | quote }} +{{- end }} + +{{/* +Renders the workloadLauncher.connectorProfiler.image value +*/}} +{{- define "airbyte.workloadLauncher.images.connectorProfiler.image" }} + {{- include "imageUrl" (list .Values.workloadLauncher.connectorProfiler.image $) }} +{{- end }} + +{{/* +Renders the workloadLauncher.images.connectorProfiler.image environment variable +*/}} +{{- define "airbyte.workloadLauncher.images.connectorProfiler.image.env" }} +- name: CONNECTOR_PROFILER_IMAGE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONNECTOR_PROFILER_IMAGE +{{- end }} + +{{/* +Renders the workloadLauncher.connectorSidecar.image value +*/}} +{{- define "airbyte.workloadLauncher.images.connectorSidecar.image" }} + {{- include "imageUrl" (list .Values.workloadLauncher.connectorSidecar.image $) }} +{{- end }} + +{{/* +Renders the workloadLauncher.images.connectorSidecar.image environment variable +*/}} +{{- define "airbyte.workloadLauncher.images.connectorSidecar.image.env" }} +- name: CONNECTOR_SIDECAR_IMAGE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONNECTOR_SIDECAR_IMAGE +{{- end }} + +{{/* +Renders the workloadLauncher.containerOrchestrator.enabled value +*/}} +{{- define "airbyte.workloadLauncher.images.containerOrchestrator.enabled" }} + {{- .Values.workloadLauncher.containerOrchestrator.enabled }} +{{- end }} + +{{/* +Renders the workloadLauncher.images.containerOrchestrator.enabled environment variable +*/}} +{{- define "airbyte.workloadLauncher.images.containerOrchestrator.enabled.env" }} +- name: CONTAINER_ORCHESTRATOR_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONTAINER_ORCHESTRATOR_ENABLED +{{- end }} + +{{/* +Renders the workloadLauncher.containerOrchestrator.image value +*/}} +{{- define "airbyte.workloadLauncher.images.containerOrchestrator.image" }} + {{- include "imageUrl" (list .Values.workloadLauncher.containerOrchestrator.image $) }} +{{- end }} + +{{/* +Renders the workloadLauncher.images.containerOrchestrator.image environment variable +*/}} +{{- define "airbyte.workloadLauncher.images.containerOrchestrator.image.env" }} +- name: CONTAINER_ORCHESTRATOR_IMAGE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONTAINER_ORCHESTRATOR_IMAGE +{{- end }} + +{{/* +Renders the workloadLauncher.workloadInit.image value +*/}} +{{- define "airbyte.workloadLauncher.images.workloadInit.image" }} + {{- include "imageUrl" (list .Values.workloadLauncher.workloadInit.image $) }} +{{- end }} + +{{/* +Renders the workloadLauncher.images.workloadInit.image environment variable +*/}} +{{- define "airbyte.workloadLauncher.images.workloadInit.image.env" }} +- name: WORKLOAD_INIT_IMAGE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKLOAD_INIT_IMAGE +{{- end }} + +{{/* +Renders the set of all workloadLauncher.images environment variables +*/}} +{{- define "airbyte.workloadLauncher.images.envs" }} +{{- include "airbyte.workloadLauncher.images.connectorProfiler.image.env" . }} +{{- include "airbyte.workloadLauncher.images.connectorSidecar.image.env" . }} +{{- include "airbyte.workloadLauncher.images.containerOrchestrator.enabled.env" . }} +{{- include "airbyte.workloadLauncher.images.containerOrchestrator.image.env" . }} +{{- include "airbyte.workloadLauncher.images.workloadInit.image.env" . }} +{{- end }} + +{{/* +Renders the set of all workloadLauncher.images config map variables +*/}} +{{- define "airbyte.workloadLauncher.images.configVars" }} +CONNECTOR_PROFILER_IMAGE: {{ include "airbyte.workloadLauncher.images.connectorProfiler.image" . | quote }} +CONNECTOR_SIDECAR_IMAGE: {{ include "airbyte.workloadLauncher.images.connectorSidecar.image" . | quote }} +CONTAINER_ORCHESTRATOR_ENABLED: {{ include "airbyte.workloadLauncher.images.containerOrchestrator.enabled" . | quote }} +CONTAINER_ORCHESTRATOR_IMAGE: {{ include "airbyte.workloadLauncher.images.containerOrchestrator.image" . | quote }} +WORKLOAD_INIT_IMAGE: {{ include "airbyte.workloadLauncher.images.workloadInit.image" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/config/_workloads.tpl b/helm/airbyte-v2/templates/config/_workloads.tpl new file mode 100644 index 00000000..8fcffc78 --- /dev/null +++ b/helm/airbyte-v2/templates/config/_workloads.tpl @@ -0,0 +1,791 @@ + +{{/* DO NOT EDIT: This file was autogenerated. */}} + +{{/* + Workloads Configuration +*/}} + +{{/* +Renders the global.workloads.containerOrchestrator.secretName value +*/}} +{{- define "airbyte.workloads.containerOrchestrator.secretName" }} + {{- .Values.global.workloads.containerOrchestrator.secretName | default .Values.global.secretName }} +{{- end }} + +{{/* +Renders the workloads.containerOrchestrator.secretName environment variable +*/}} +{{- define "airbyte.workloads.containerOrchestrator.secretName.env" }} +- name: CONTAINER_ORCHESTRATOR_SECRET_NAME + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONTAINER_ORCHESTRATOR_SECRET_NAME +{{- end }} + +{{/* +Renders the global.workloads.containerOrchestrator.dataPlane.secretMountPath value +*/}} +{{- define "airbyte.workloads.containerOrchestrator.dataPlane.secretMountPath" }} + {{- .Values.global.workloads.containerOrchestrator.dataPlane.secretMountPath | default "/secrets/dataplane-creds" }} +{{- end }} + +{{/* +Renders the workloads.containerOrchestrator.dataPlane.secretMountPath environment variable +*/}} +{{- define "airbyte.workloads.containerOrchestrator.dataPlane.secretMountPath.env" }} +- name: CONTAINER_ORCHESTRATOR_DATA_PLANE_CREDS_SECRET_MOUNT_PATH + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONTAINER_ORCHESTRATOR_DATA_PLANE_CREDS_SECRET_MOUNT_PATH +{{- end }} + +{{/* +Renders the global.workloads.containerOrchestrator.dataPlane.secretName value. +When unset or empty, defaults to chart secret so launcher has a non-empty value (avoids "Unsetting empty" and job pods can mount creds). +*/}} +{{- define "airbyte.workloads.containerOrchestrator.dataPlane.secretName" }} + {{- or .Values.global.workloads.containerOrchestrator.dataPlane.secretName (include "airbyte.auth.bootstrap.managedSecretName" .) }} +{{- end }} + +{{/* +Renders the workloads.containerOrchestrator.dataPlane.secretName environment variable +*/}} +{{- define "airbyte.workloads.containerOrchestrator.dataPlane.secretName.env" }} +- name: CONTAINER_ORCHESTRATOR_DATA_PLANE_CREDS_SECRET_NAME + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONTAINER_ORCHESTRATOR_DATA_PLANE_CREDS_SECRET_NAME +{{- end }} + +{{/* +Renders the global.workloads.containerOrchestrator.dataPlane.secretKey value +*/}} +{{- define "airbyte.workloads.containerOrchestrator.dataPlane.secretKey" }} + {{- .Values.global.workloads.containerOrchestrator.dataPlane.secretKey | default "sa.json" }} +{{- end }} + +{{/* +Renders the workloads.containerOrchestrator.dataPlane.secretKey environment variable +*/}} +{{- define "airbyte.workloads.containerOrchestrator.dataPlane.secretKey.env" }} +- name: CONTAINER_ORCHESTRATOR_DATA_PLANE_CREDS_SECRET_KEY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONTAINER_ORCHESTRATOR_DATA_PLANE_CREDS_SECRET_KEY +{{- end }} + +{{/* +Renders the global.workloads.containerOrchestrator.javaOpts value +*/}} +{{- define "airbyte.workloads.containerOrchestrator.javaOpts" }} + {{- .Values.global.workloads.containerOrchestrator.javaOpts | default "-XX:+ExitOnOutOfMemoryError -XX:MaxRAMPercentage=75.0 -javaagent:/app/dd-java-agent.jar -Ddd.profiling.enabled=true -XX:FlightRecorderOptions=stackdepth=256 -Ddd.trace.sample.rate=0.5 -Ddd.trace.request_header.tags=User-Agent:http.useragent" }} +{{- end }} + +{{/* +Renders the workloads.containerOrchestrator.javaOpts environment variable +*/}} +{{- define "airbyte.workloads.containerOrchestrator.javaOpts.env" }} +- name: CONTAINER_ORCHESTRATOR_JAVA_OPTS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONTAINER_ORCHESTRATOR_JAVA_OPTS +{{- end }} + +{{/* +Renders the global.workloads.containerOrchestrator.secretMountPath value +*/}} +{{- define "airbyte.workloads.containerOrchestrator.secretMountPath" }} + {{- .Values.global.workloads.containerOrchestrator.secretMountPath | default "/secrets/gcp-creds" }} +{{- end }} + +{{/* +Renders the workloads.containerOrchestrator.secretMountPath environment variable +*/}} +{{- define "airbyte.workloads.containerOrchestrator.secretMountPath.env" }} +- name: CONTAINER_ORCHESTRATOR_SECRET_MOUNT_PATH + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONTAINER_ORCHESTRATOR_SECRET_MOUNT_PATH +{{- end }} + +{{/* +Renders the global.workloads.kubernetesClientMaxIdleConnections value +*/}} +{{- define "airbyte.workloads.kubernetesClientMaxIdleConnections" }} + {{- .Values.global.workloads.kubernetesClientMaxIdleConnections | default 100 }} +{{- end }} + +{{/* +Renders the workloads.kubernetesClientMaxIdleConnections environment variable +*/}} +{{- define "airbyte.workloads.kubernetesClientMaxIdleConnections.env" }} +- name: KUBERNETES_CLIENT_MAX_IDLE_CONNECTIONS + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: KUBERNETES_CLIENT_MAX_IDLE_CONNECTIONS +{{- end }} + +{{/* +Renders the global.workloads.kuberentesClientMaxRetries value +*/}} +{{- define "airbyte.workloads.kuberentesClientMaxRetries" }} + {{- .Values.global.workloads.kuberentesClientMaxRetries | default "" }} +{{- end }} + +{{/* +Renders the workloads.kuberentesClientMaxRetries environment variable +*/}} +{{- define "airbyte.workloads.kuberentesClientMaxRetries.env" }} +- name: KUBERNETES_CLIENT_MAX_RETRIES + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: KUBERNETES_CLIENT_MAX_RETRIES +{{- end }} + +{{/* +Renders the global.workloads.namespace value +*/}} +{{- define "airbyte.workloads.namespace" }} + {{- .Values.global.workloads.namespace | default .Release.Namespace }} +{{- end }} + +{{/* +Renders the workloads.namespace environment variable +*/}} +{{- define "airbyte.workloads.namespace.env" }} +- name: WORKLOADS_NAMESPACE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: WORKLOADS_NAMESPACE +{{- end }} + +{{/* +Renders the global.workloads.pubSub.enabled value +*/}} +{{- define "airbyte.workloads.pubSub.enabled" }} + {{- if eq .Values.global.workloads.pubSub.enabled nil }} + {{- false }} + {{- else }} + {{- .Values.global.workloads.pubSub.enabled }} + {{- end }} +{{- end }} + +{{/* +Renders the workloads.pubSub.enabled environment variable +*/}} +{{- define "airbyte.workloads.pubSub.enabled.env" }} +- name: PUB_SUB_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: PUB_SUB_ENABLED +{{- end }} + +{{/* +Renders the global.workloads.pubSub.topicName value +*/}} +{{- define "airbyte.workloads.pubSub.topicName" }} + {{- .Values.global.workloads.pubSub.topicName | default "" }} +{{- end }} + +{{/* +Renders the workloads.pubSub.topicName environment variable +*/}} +{{- define "airbyte.workloads.pubSub.topicName.env" }} +- name: PUB_SUB_TOPIC_NAME + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: PUB_SUB_TOPIC_NAME +{{- end }} + +{{/* +Renders the set of all workloads environment variables +*/}} +{{- define "airbyte.workloads.envs" }} +{{- include "airbyte.workloads.containerOrchestrator.secretName.env" . }} +{{- include "airbyte.workloads.containerOrchestrator.dataPlane.secretMountPath.env" . }} +{{- include "airbyte.workloads.containerOrchestrator.dataPlane.secretName.env" . }} +{{- include "airbyte.workloads.containerOrchestrator.dataPlane.secretKey.env" . }} +{{- include "airbyte.workloads.containerOrchestrator.javaOpts.env" . }} +{{- include "airbyte.workloads.containerOrchestrator.secretMountPath.env" . }} +{{- include "airbyte.workloads.kubernetesClientMaxIdleConnections.env" . }} +{{- include "airbyte.workloads.kuberentesClientMaxRetries.env" . }} +{{- include "airbyte.workloads.namespace.env" . }} +{{- include "airbyte.workloads.pubSub.enabled.env" . }} +{{- include "airbyte.workloads.pubSub.topicName.env" . }} +{{- end }} + +{{/* +Renders the set of all workloads config map variables +*/}} +{{- define "airbyte.workloads.configVars" }} +CONTAINER_ORCHESTRATOR_SECRET_NAME: {{ include "airbyte.workloads.containerOrchestrator.secretName" . | quote }} +CONTAINER_ORCHESTRATOR_DATA_PLANE_CREDS_SECRET_MOUNT_PATH: {{ include "airbyte.workloads.containerOrchestrator.dataPlane.secretMountPath" . | quote }} +CONTAINER_ORCHESTRATOR_DATA_PLANE_CREDS_SECRET_NAME: {{ include "airbyte.workloads.containerOrchestrator.dataPlane.secretName" . | quote }} +CONTAINER_ORCHESTRATOR_DATA_PLANE_CREDS_SECRET_KEY: {{ include "airbyte.workloads.containerOrchestrator.dataPlane.secretKey" . | quote }} +CONTAINER_ORCHESTRATOR_JAVA_OPTS: {{ include "airbyte.workloads.containerOrchestrator.javaOpts" . | quote }} +CONTAINER_ORCHESTRATOR_SECRET_MOUNT_PATH: {{ include "airbyte.workloads.containerOrchestrator.secretMountPath" . | quote }} +KUBERNETES_CLIENT_MAX_IDLE_CONNECTIONS: {{ include "airbyte.workloads.kubernetesClientMaxIdleConnections" . | quote }} +KUBERNETES_CLIENT_MAX_RETRIES: {{ include "airbyte.workloads.kuberentesClientMaxRetries" . | quote }} +WORKLOADS_NAMESPACE: {{ include "airbyte.workloads.namespace" . | quote }} +PUB_SUB_ENABLED: {{ include "airbyte.workloads.pubSub.enabled" . | quote }} +PUB_SUB_TOPIC_NAME: {{ include "airbyte.workloads.pubSub.topicName" . | quote }} +{{- end }} + +{{/* +Renders the global.workloads.queues.check value +*/}} +{{- define "airbyte.workloads.queues.check" }} + {{- join " " .Values.global.workloads.queues.check }} +{{- end }} + +{{/* +Renders the workloads.queues.check environment variable +*/}} +{{- define "airbyte.workloads.queues.check.env" }} +- name: DATA_CHECK_TASK_QUEUES + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATA_CHECK_TASK_QUEUES +{{- end }} + +{{/* +Renders the global.workloads.queues.discover value +*/}} +{{- define "airbyte.workloads.queues.discover" }} + {{- join " " .Values.global.workloads.queues.discover }} +{{- end }} + +{{/* +Renders the workloads.queues.discover environment variable +*/}} +{{- define "airbyte.workloads.queues.discover.env" }} +- name: DATA_DISCOVER_TASK_QUEUES + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATA_DISCOVER_TASK_QUEUES +{{- end }} + +{{/* +Renders the global.workloads.queues.sync value +*/}} +{{- define "airbyte.workloads.queues.sync" }} + {{- join " " .Values.global.workloads.queues.sync }} +{{- end }} + +{{/* +Renders the workloads.queues.sync environment variable +*/}} +{{- define "airbyte.workloads.queues.sync.env" }} +- name: DATA_SYNC_TASK_QUEUES + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATA_SYNC_TASK_QUEUES +{{- end }} + +{{/* +Renders the set of all workloads.queues environment variables +*/}} +{{- define "airbyte.workloads.queues.envs" }} +{{- include "airbyte.workloads.queues.check.env" . }} +{{- include "airbyte.workloads.queues.discover.env" . }} +{{- include "airbyte.workloads.queues.sync.env" . }} +{{- end }} + +{{/* +Renders the set of all workloads.queues config map variables +*/}} +{{- define "airbyte.workloads.queues.configVars" }} +DATA_CHECK_TASK_QUEUES: {{ include "airbyte.workloads.queues.check" . | quote }} +DATA_DISCOVER_TASK_QUEUES: {{ include "airbyte.workloads.queues.discover" . | quote }} +DATA_SYNC_TASK_QUEUES: {{ include "airbyte.workloads.queues.sync" . | quote }} +{{- end }} + +{{/* +Renders the global.workloads.resources.useConnectorResourceDefaults value +*/}} +{{- define "airbyte.workloads.resources.useConnectorResourceDefaults" }} + {{- if eq .Values.global.workloads.resources.useConnectorResourceDefaults nil }} + {{- true }} + {{- else }} + {{- .Values.global.workloads.resources.useConnectorResourceDefaults }} + {{- end }} +{{- end }} + +{{/* +Renders the workloads.resources.useConnectorResourceDefaults environment variable +*/}} +{{- define "airbyte.workloads.resources.useConnectorResourceDefaults.env" }} +- name: CONNECTOR_SPECIFIC_RESOURCE_DEFAULTS_ENABLED + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONNECTOR_SPECIFIC_RESOURCE_DEFAULTS_ENABLED +{{- end }} + +{{/* +Renders the global.workloads.resources.mainContainer.cpu.limit value +*/}} +{{- define "airbyte.workloads.resources.mainContainer.cpu.limit" }} + {{- .Values.global.workloads.resources.mainContainer.cpu.limit | default "" }} +{{- end }} + +{{/* +Renders the workloads.resources.mainContainer.cpu.limit environment variable +*/}} +{{- define "airbyte.workloads.resources.mainContainer.cpu.limit.env" }} +- name: JOB_MAIN_CONTAINER_CPU_LIMIT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_MAIN_CONTAINER_CPU_LIMIT +{{- end }} + +{{/* +Renders the global.workloads.resources.mainContainer.cpu.request value +*/}} +{{- define "airbyte.workloads.resources.mainContainer.cpu.request" }} + {{- .Values.global.workloads.resources.mainContainer.cpu.request | default "" }} +{{- end }} + +{{/* +Renders the workloads.resources.mainContainer.cpu.request environment variable +*/}} +{{- define "airbyte.workloads.resources.mainContainer.cpu.request.env" }} +- name: JOB_MAIN_CONTAINER_CPU_REQUEST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_MAIN_CONTAINER_CPU_REQUEST +{{- end }} + +{{/* +Renders the global.workloads.resources.mainContainer.memory.limit value +*/}} +{{- define "airbyte.workloads.resources.mainContainer.memory.limit" }} + {{- .Values.global.workloads.resources.mainContainer.memory.limit | default "" }} +{{- end }} + +{{/* +Renders the workloads.resources.mainContainer.memory.limit environment variable +*/}} +{{- define "airbyte.workloads.resources.mainContainer.memory.limit.env" }} +- name: JOB_MAIN_CONTAINER_MEMORY_LIMIT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_MAIN_CONTAINER_MEMORY_LIMIT +{{- end }} + +{{/* +Renders the global.workloads.resources.mainContainer.memory.request value +*/}} +{{- define "airbyte.workloads.resources.mainContainer.memory.request" }} + {{- .Values.global.workloads.resources.mainContainer.memory.request | default "" }} +{{- end }} + +{{/* +Renders the workloads.resources.mainContainer.memory.request environment variable +*/}} +{{- define "airbyte.workloads.resources.mainContainer.memory.request.env" }} +- name: JOB_MAIN_CONTAINER_MEMORY_REQUEST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: JOB_MAIN_CONTAINER_MEMORY_REQUEST +{{- end }} + +{{/* +Renders the global.workloads.resources.check.cpu.limit value +*/}} +{{- define "airbyte.workloads.resources.check.cpu.limit" }} + {{- .Values.global.workloads.resources.check.cpu.limit | default "" }} +{{- end }} + +{{/* +Renders the workloads.resources.check.cpu.limit environment variable +*/}} +{{- define "airbyte.workloads.resources.check.cpu.limit.env" }} +- name: CHECK_JOB_MAIN_CONTAINER_CPU_LIMIT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CHECK_JOB_MAIN_CONTAINER_CPU_LIMIT +{{- end }} + +{{/* +Renders the global.workloads.resources.check.cpu.request value +*/}} +{{- define "airbyte.workloads.resources.check.cpu.request" }} + {{- .Values.global.workloads.resources.check.cpu.request | default "" }} +{{- end }} + +{{/* +Renders the workloads.resources.check.cpu.request environment variable +*/}} +{{- define "airbyte.workloads.resources.check.cpu.request.env" }} +- name: CHECK_JOB_MAIN_CONTAINER_CPU_REQUEST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CHECK_JOB_MAIN_CONTAINER_CPU_REQUEST +{{- end }} + +{{/* +Renders the global.workloads.resources.check.memory.limit value +*/}} +{{- define "airbyte.workloads.resources.check.memory.limit" }} + {{- .Values.global.workloads.resources.check.memory.limit | default "" }} +{{- end }} + +{{/* +Renders the workloads.resources.check.memory.limit environment variable +*/}} +{{- define "airbyte.workloads.resources.check.memory.limit.env" }} +- name: CHECK_JOB_MAIN_CONTAINER_MEMORY_LIMIT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CHECK_JOB_MAIN_CONTAINER_MEMORY_LIMIT +{{- end }} + +{{/* +Renders the global.workloads.resources.check.memory.request value +*/}} +{{- define "airbyte.workloads.resources.check.memory.request" }} + {{- .Values.global.workloads.resources.check.memory.request | default "" }} +{{- end }} + +{{/* +Renders the workloads.resources.check.memory.request environment variable +*/}} +{{- define "airbyte.workloads.resources.check.memory.request.env" }} +- name: CHECK_JOB_MAIN_CONTAINER_MEMORY_REQUEST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CHECK_JOB_MAIN_CONTAINER_MEMORY_REQUEST +{{- end }} + +{{/* +Renders the global.workloads.resources.discover.cpu.limit value +*/}} +{{- define "airbyte.workloads.resources.discover.cpu.limit" }} + {{- .Values.global.workloads.resources.discover.cpu.limit | default "" }} +{{- end }} + +{{/* +Renders the workloads.resources.discover.cpu.limit environment variable +*/}} +{{- define "airbyte.workloads.resources.discover.cpu.limit.env" }} +- name: DISCOVER_JOB_MAIN_CONTAINER_CPU_LIMIT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DISCOVER_JOB_MAIN_CONTAINER_CPU_LIMIT +{{- end }} + +{{/* +Renders the global.workloads.resources.discover.cpu.request value +*/}} +{{- define "airbyte.workloads.resources.discover.cpu.request" }} + {{- .Values.global.workloads.resources.discover.cpu.request | default "" }} +{{- end }} + +{{/* +Renders the workloads.resources.discover.cpu.request environment variable +*/}} +{{- define "airbyte.workloads.resources.discover.cpu.request.env" }} +- name: DISCOVER_JOB_MAIN_CONTAINER_CPU_REQUEST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DISCOVER_JOB_MAIN_CONTAINER_CPU_REQUEST +{{- end }} + +{{/* +Renders the global.workloads.resources.discover.memory.limit value +*/}} +{{- define "airbyte.workloads.resources.discover.memory.limit" }} + {{- .Values.global.workloads.resources.discover.memory.limit | default "" }} +{{- end }} + +{{/* +Renders the workloads.resources.discover.memory.limit environment variable +*/}} +{{- define "airbyte.workloads.resources.discover.memory.limit.env" }} +- name: DISCOVER_JOB_MAIN_CONTAINER_MEMORY_LIMIT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DISCOVER_JOB_MAIN_CONTAINER_MEMORY_LIMIT +{{- end }} + +{{/* +Renders the global.workloads.resources.discover.memory.request value +*/}} +{{- define "airbyte.workloads.resources.discover.memory.request" }} + {{- .Values.global.workloads.resources.discover.memory.request | default "" }} +{{- end }} + +{{/* +Renders the workloads.resources.discover.memory.request environment variable +*/}} +{{- define "airbyte.workloads.resources.discover.memory.request.env" }} +- name: DISCOVER_JOB_MAIN_CONTAINER_MEMORY_REQUEST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DISCOVER_JOB_MAIN_CONTAINER_MEMORY_REQUEST +{{- end }} + +{{/* +Renders the global.workloads.resources.fileTransfer.storage.limit value +*/}} +{{- define "airbyte.workloads.resources.fileTransfer.storage.limit" }} + {{- .Values.global.workloads.resources.fileTransfer.storage.limit | default "" }} +{{- end }} + +{{/* +Renders the workloads.resources.fileTransfer.storage.limit environment variable +*/}} +{{- define "airbyte.workloads.resources.fileTransfer.storage.limit.env" }} +- name: FILE_TRANSFER_EPHEMERAL_STORAGE_LIMIT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: FILE_TRANSFER_EPHEMERAL_STORAGE_LIMIT +{{- end }} + +{{/* +Renders the global.workloads.resources.fileTransfer.storage.request value +*/}} +{{- define "airbyte.workloads.resources.fileTransfer.storage.request" }} + {{- .Values.global.workloads.resources.fileTransfer.storage.request | default "" }} +{{- end }} + +{{/* +Renders the workloads.resources.fileTransfer.storage.request environment variable +*/}} +{{- define "airbyte.workloads.resources.fileTransfer.storage.request.env" }} +- name: FILE_TRANSFER_EPHEMERAL_STORAGE_REQUEST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: FILE_TRANSFER_EPHEMERAL_STORAGE_REQUEST +{{- end }} + +{{/* +Renders the global.workloads.resources.replication.cpu.limit value +*/}} +{{- define "airbyte.workloads.resources.replication.cpu.limit" }} + {{- .Values.global.workloads.resources.replication.cpu.limit | default "" }} +{{- end }} + +{{/* +Renders the workloads.resources.replication.cpu.limit environment variable +*/}} +{{- define "airbyte.workloads.resources.replication.cpu.limit.env" }} +- name: REPLICATION_ORCHESTRATOR_CPU_LIMIT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: REPLICATION_ORCHESTRATOR_CPU_LIMIT +{{- end }} + +{{/* +Renders the global.workloads.resources.replication.cpu.request value +*/}} +{{- define "airbyte.workloads.resources.replication.cpu.request" }} + {{- .Values.global.workloads.resources.replication.cpu.request | default "" }} +{{- end }} + +{{/* +Renders the workloads.resources.replication.cpu.request environment variable +*/}} +{{- define "airbyte.workloads.resources.replication.cpu.request.env" }} +- name: REPLICATION_ORCHESTRATOR_CPU_REQUEST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: REPLICATION_ORCHESTRATOR_CPU_REQUEST +{{- end }} + +{{/* +Renders the global.workloads.resources.replication.memory.limit value +*/}} +{{- define "airbyte.workloads.resources.replication.memory.limit" }} + {{- .Values.global.workloads.resources.replication.memory.limit | default "" }} +{{- end }} + +{{/* +Renders the workloads.resources.replication.memory.limit environment variable +*/}} +{{- define "airbyte.workloads.resources.replication.memory.limit.env" }} +- name: REPLICATION_ORCHESTRATOR_MEMORY_LIMIT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: REPLICATION_ORCHESTRATOR_MEMORY_LIMIT +{{- end }} + +{{/* +Renders the global.workloads.resources.replication.memory.request value +*/}} +{{- define "airbyte.workloads.resources.replication.memory.request" }} + {{- .Values.global.workloads.resources.replication.memory.request | default "" }} +{{- end }} + +{{/* +Renders the workloads.resources.replication.memory.request environment variable +*/}} +{{- define "airbyte.workloads.resources.replication.memory.request.env" }} +- name: REPLICATION_ORCHESTRATOR_MEMORY_REQUEST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: REPLICATION_ORCHESTRATOR_MEMORY_REQUEST +{{- end }} + +{{/* +Renders the global.workloads.resources.sidecar.cpu.limit value +*/}} +{{- define "airbyte.workloads.resources.sidecar.cpu.limit" }} + {{- .Values.global.workloads.resources.sidecar.cpu.limit | default "" }} +{{- end }} + +{{/* +Renders the workloads.resources.sidecar.cpu.limit environment variable +*/}} +{{- define "airbyte.workloads.resources.sidecar.cpu.limit.env" }} +- name: SIDECAR_MAIN_CONTAINER_CPU_LIMIT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: SIDECAR_MAIN_CONTAINER_CPU_LIMIT +{{- end }} + +{{/* +Renders the global.workloads.resources.sidecar.cpu.request value +*/}} +{{- define "airbyte.workloads.resources.sidecar.cpu.request" }} + {{- .Values.global.workloads.resources.sidecar.cpu.request | default "" }} +{{- end }} + +{{/* +Renders the workloads.resources.sidecar.cpu.request environment variable +*/}} +{{- define "airbyte.workloads.resources.sidecar.cpu.request.env" }} +- name: SIDECAR_MAIN_CONTAINER_CPU_REQUEST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: SIDECAR_MAIN_CONTAINER_CPU_REQUEST +{{- end }} + +{{/* +Renders the global.workloads.resources.sidecar.memory.limit value +*/}} +{{- define "airbyte.workloads.resources.sidecar.memory.limit" }} + {{- .Values.global.workloads.resources.sidecar.memory.limit | default "" }} +{{- end }} + +{{/* +Renders the workloads.resources.sidecar.memory.limit environment variable +*/}} +{{- define "airbyte.workloads.resources.sidecar.memory.limit.env" }} +- name: SIDECAR_MAIN_CONTAINER_MEMORY_LIMIT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: SIDECAR_MAIN_CONTAINER_MEMORY_LIMIT +{{- end }} + +{{/* +Renders the global.workloads.resources.sidecar.memory.request value +*/}} +{{- define "airbyte.workloads.resources.sidecar.memory.request" }} + {{- .Values.global.workloads.resources.sidecar.memory.request | default "" }} +{{- end }} + +{{/* +Renders the workloads.resources.sidecar.memory.request environment variable +*/}} +{{- define "airbyte.workloads.resources.sidecar.memory.request.env" }} +- name: SIDECAR_MAIN_CONTAINER_MEMORY_REQUEST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: SIDECAR_MAIN_CONTAINER_MEMORY_REQUEST +{{- end }} + +{{/* +Renders the set of all workloads.resources environment variables +*/}} +{{- define "airbyte.workloads.resources.envs" }} +{{- include "airbyte.workloads.resources.useConnectorResourceDefaults.env" . }} +{{- include "airbyte.workloads.resources.mainContainer.cpu.limit.env" . }} +{{- include "airbyte.workloads.resources.mainContainer.cpu.request.env" . }} +{{- include "airbyte.workloads.resources.mainContainer.memory.limit.env" . }} +{{- include "airbyte.workloads.resources.mainContainer.memory.request.env" . }} +{{- include "airbyte.workloads.resources.check.cpu.limit.env" . }} +{{- include "airbyte.workloads.resources.check.cpu.request.env" . }} +{{- include "airbyte.workloads.resources.check.memory.limit.env" . }} +{{- include "airbyte.workloads.resources.check.memory.request.env" . }} +{{- include "airbyte.workloads.resources.discover.cpu.limit.env" . }} +{{- include "airbyte.workloads.resources.discover.cpu.request.env" . }} +{{- include "airbyte.workloads.resources.discover.memory.limit.env" . }} +{{- include "airbyte.workloads.resources.discover.memory.request.env" . }} +{{- include "airbyte.workloads.resources.fileTransfer.storage.limit.env" . }} +{{- include "airbyte.workloads.resources.fileTransfer.storage.request.env" . }} +{{- include "airbyte.workloads.resources.replication.cpu.limit.env" . }} +{{- include "airbyte.workloads.resources.replication.cpu.request.env" . }} +{{- include "airbyte.workloads.resources.replication.memory.limit.env" . }} +{{- include "airbyte.workloads.resources.replication.memory.request.env" . }} +{{- include "airbyte.workloads.resources.sidecar.cpu.limit.env" . }} +{{- include "airbyte.workloads.resources.sidecar.cpu.request.env" . }} +{{- include "airbyte.workloads.resources.sidecar.memory.limit.env" . }} +{{- include "airbyte.workloads.resources.sidecar.memory.request.env" . }} +{{- end }} + +{{/* +Renders the set of all workloads.resources config map variables +*/}} +{{- define "airbyte.workloads.resources.configVars" }} +CONNECTOR_SPECIFIC_RESOURCE_DEFAULTS_ENABLED: {{ include "airbyte.workloads.resources.useConnectorResourceDefaults" . | quote }} +JOB_MAIN_CONTAINER_CPU_LIMIT: {{ include "airbyte.workloads.resources.mainContainer.cpu.limit" . | quote }} +JOB_MAIN_CONTAINER_CPU_REQUEST: {{ include "airbyte.workloads.resources.mainContainer.cpu.request" . | quote }} +JOB_MAIN_CONTAINER_MEMORY_LIMIT: {{ include "airbyte.workloads.resources.mainContainer.memory.limit" . | quote }} +JOB_MAIN_CONTAINER_MEMORY_REQUEST: {{ include "airbyte.workloads.resources.mainContainer.memory.request" . | quote }} +CHECK_JOB_MAIN_CONTAINER_CPU_LIMIT: {{ include "airbyte.workloads.resources.check.cpu.limit" . | quote }} +CHECK_JOB_MAIN_CONTAINER_CPU_REQUEST: {{ include "airbyte.workloads.resources.check.cpu.request" . | quote }} +CHECK_JOB_MAIN_CONTAINER_MEMORY_LIMIT: {{ include "airbyte.workloads.resources.check.memory.limit" . | quote }} +CHECK_JOB_MAIN_CONTAINER_MEMORY_REQUEST: {{ include "airbyte.workloads.resources.check.memory.request" . | quote }} +DISCOVER_JOB_MAIN_CONTAINER_CPU_LIMIT: {{ include "airbyte.workloads.resources.discover.cpu.limit" . | quote }} +DISCOVER_JOB_MAIN_CONTAINER_CPU_REQUEST: {{ include "airbyte.workloads.resources.discover.cpu.request" . | quote }} +DISCOVER_JOB_MAIN_CONTAINER_MEMORY_LIMIT: {{ include "airbyte.workloads.resources.discover.memory.limit" . | quote }} +DISCOVER_JOB_MAIN_CONTAINER_MEMORY_REQUEST: {{ include "airbyte.workloads.resources.discover.memory.request" . | quote }} +FILE_TRANSFER_EPHEMERAL_STORAGE_LIMIT: {{ include "airbyte.workloads.resources.fileTransfer.storage.limit" . | quote }} +FILE_TRANSFER_EPHEMERAL_STORAGE_REQUEST: {{ include "airbyte.workloads.resources.fileTransfer.storage.request" . | quote }} +REPLICATION_ORCHESTRATOR_CPU_LIMIT: {{ include "airbyte.workloads.resources.replication.cpu.limit" . | quote }} +REPLICATION_ORCHESTRATOR_CPU_REQUEST: {{ include "airbyte.workloads.resources.replication.cpu.request" . | quote }} +REPLICATION_ORCHESTRATOR_MEMORY_LIMIT: {{ include "airbyte.workloads.resources.replication.memory.limit" . | quote }} +REPLICATION_ORCHESTRATOR_MEMORY_REQUEST: {{ include "airbyte.workloads.resources.replication.memory.request" . | quote }} +SIDECAR_MAIN_CONTAINER_CPU_LIMIT: {{ include "airbyte.workloads.resources.sidecar.cpu.limit" . | quote }} +SIDECAR_MAIN_CONTAINER_CPU_REQUEST: {{ include "airbyte.workloads.resources.sidecar.cpu.request" . | quote }} +SIDECAR_MAIN_CONTAINER_MEMORY_LIMIT: {{ include "airbyte.workloads.resources.sidecar.memory.limit" . | quote }} +SIDECAR_MAIN_CONTAINER_MEMORY_REQUEST: {{ include "airbyte.workloads.resources.sidecar.memory.request" . | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/env-configmap.yaml b/helm/airbyte-v2/templates/env-configmap.yaml new file mode 100644 index 00000000..fbc13ceb --- /dev/null +++ b/helm/airbyte-v2/templates/env-configmap.yaml @@ -0,0 +1,63 @@ +{{- $airbyteYmlDict := (default "" .Values.global.airbyteYml | fromYaml) }} # default to empty dict if airbyteYml is undefined +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-airbyte-env + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" + labels: + {{- include "airbyte.labels" . | nindent 4 }} +data: + {{- include "airbyte.auth.bootstrap.configVars" . | nindent 2 }} + {{- include "airbyte.auth.identityProvider.configVars" . | nindent 2 }} + {{- include "airbyte.auth.instanceAdmin.enterprise.configVars" . | nindent 2 }} + {{- include "airbyte.auth.security.configVars" . | nindent 2 }} + {{- include "airbyte.cluster.configVars" . | nindent 2 }} + {{- include "airbyte.common.configVars" . | nindent 2 }} + {{- include "airbyte.connector.configVars" . | nindent 2 }} + {{- include "airbyte.connectorBuilder.configVars" . | nindent 2 }} + {{- include "airbyte.connectorRollout.configVars" . | nindent 2 }} + {{- include "airbyte.cron.configVars" . | nindent 2 }} + {{- include "airbyte.database.configVars" . | nindent 2 }} + {{- include "airbyte.database.migrations.configVars" . | nindent 2 }} + {{- include "airbyte.datadog.configVars" . | nindent 2 }} + {{- include "airbyte.featureFlags.configVars" . | nindent 2 }} + {{- include "airbyte.java.configVars" . | nindent 2 }} + {{- include "airbyte.jobs.configVars" . | nindent 2 }} + {{- include "airbyte.jobs.scheduling.configVars" . | nindent 2 }} + {{- include "airbyte.jobs.errors.configVars" . | nindent 2 }} + {{- include "airbyte.keycloak.admin.client.configVars" . | nindent 2 }} + {{- include "airbyte.keycloak.client.configVars" . | nindent 2 }} + {{- include "airbyte.keycloak.configVars" . | nindent 2 }} + {{- include "airbyte.keycloak.database.configVars" . | nindent 2 }} + {{- include "airbyte.logging.configVars" . | nindent 2 }} + {{- include "airbyte.metrics.configVars" . | nindent 2 }} + {{- include "airbyte.micronaut.configVars" . | nindent 2 }} + {{- include "airbyte.otel.configVars" . | nindent 2 }} + {{- include "airbyte.secretsManager.configVars" . | nindent 2 }} + {{- include "airbyte.storage.configVars" . | nindent 2 }} + {{- include "airbyte.temporal.cli.configVars" . | nindent 2 }} + {{- include "airbyte.temporal.cloud.configVars" . | nindent 2 }} + {{- include "airbyte.temporal.configVars" . | nindent 2 }} + {{- include "airbyte.temporal.database.configVars" . | nindent 2 }} + {{- include "airbyte.temporal.sdk.configVars" . | nindent 2 }} + {{- include "airbyte.temporal.worker.configVars" . | nindent 2 }} + {{- include "airbyte.topology.configVars" . | nindent 2 }} + {{- include "airbyte.tracking.configVars" . | nindent 2 }} + {{- include "airbyte.webapp.configVars" . | nindent 2 }} + {{- include "airbyte.worker.configVars" . | nindent 2 }} + {{- include "airbyte.workloadApiServer.configVars" . | nindent 2 }} + {{- include "airbyte.workloadLauncher.configVars" . | nindent 2 }} + {{- include "airbyte.workloadLauncher.dataPlane.configVars" . | nindent 2 }} + {{- include "airbyte.workloadLauncher.images.configVars" . | nindent 2 }} + {{- include "airbyte.workloads.configVars" . | nindent 2 }} + {{- include "airbyte.workloads.queues.configVars" . | nindent 2 }} + {{- include "airbyte.workloads.resources.configVars" . | nindent 2 }} + + {{- include "airbyte.extra.configVars" . | nindent 2 }} + + LAUNCHER_MICRONAUT_ENVIRONMENTS: "control-plane,oss" + WORKERS_MICRONAUT_ENVIRONMENTS: "control-plane" + CRON_MICRONAUT_ENVIRONMENTS: "control-plane" + SERVER_MICRONAUT_ENVIRONMENTS: "control-plane" diff --git a/helm/airbyte-v2/templates/gcs-log-creds-secret.yaml b/helm/airbyte-v2/templates/gcs-log-creds-secret.yaml new file mode 100644 index 00000000..af6bfc2d --- /dev/null +++ b/helm/airbyte-v2/templates/gcs-log-creds-secret.yaml @@ -0,0 +1,18 @@ +{{- if and + (or (eq .Values.global.edition "community") (eq .Values.global.edition "enterprise")) + (eq (lower (default "" .Values.global.storage.type)) "gcs") + (not .Values.global.storage.secretName) + (not .Values.global.storage.storageSecretName) +}} +{{ $gcsConfig := required "You must set 'global.storage.gcs' when setting 'global.storage.type' to 'gcs'" .Values.global.storage.gcs }} +{{ $credentialsJson := required "You must set 'global.storage.gcs.credentialsJson' when 'global.storage.secretName' is not set" .Values.global.storage.gcs.credentialsJson }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-gcs-log-creds + labels: + {{- include "airbyte.labels" . | nindent 4 }} +type: Opaque +data: + gcp.json: {{ $credentialsJson | default "" | quote }} +{{- end }} diff --git a/helm/airbyte-v2/templates/instance-admin-email-job.yaml b/helm/airbyte-v2/templates/instance-admin-email-job.yaml new file mode 100644 index 00000000..d4e62ea5 --- /dev/null +++ b/helm/airbyte-v2/templates/instance-admin-email-job.yaml @@ -0,0 +1,71 @@ +{{- if and .Values.global.auth.enabled (include "airbyte.auth.instanceAdmin.email" .) }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-set-instance-admin-email + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "10" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +spec: + backoffLimit: 6 + ttlSecondsAfterFinished: 300 + template: + metadata: + labels: + {{- include "airbyte.selectorLabels" . | nindent 8 }} + spec: + restartPolicy: OnFailure + serviceAccountName: {{ include "airbyte.effectiveServiceAccountName" . }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + - name: {{ .name }} + {{- end }} + {{- end }} + containers: + - name: psql + image: postgres:15-alpine + env: + - name: AB_ADMIN_EMAIL + value: {{ include "airbyte.auth.instanceAdmin.email" . | quote }} + - name: PGHOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATABASE_HOST + - name: PGPORT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATABASE_PORT + - name: PGDATABASE + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATABASE_DB + - name: PGUSER + valueFrom: + secretKeyRef: + name: {{ include "airbyte.database.secretName" . }} + key: {{ include "airbyte.database.user.secretKey" . }} + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: {{ include "airbyte.database.secretName" . }} + key: {{ include "airbyte.database.password.secretKey" . }} + command: + - sh + - -c + - | + set -eu + until psql -c 'SELECT 1' >/dev/null 2>&1; do + echo "waiting for postgres..." + sleep 3 + done + psql -v ON_ERROR_STOP=1 -c "UPDATE organization SET email = '${AB_ADMIN_EMAIL}' WHERE id = '00000000-0000-0000-0000-000000000000'" + psql -v ON_ERROR_STOP=1 -c "UPDATE \"user\" SET email = '${AB_ADMIN_EMAIL}' WHERE id = '00000000-0000-0000-0000-000000000000'" +{{- end }} diff --git a/helm/airbyte-v2/templates/minio.yaml b/helm/airbyte-v2/templates/minio.yaml new file mode 100644 index 00000000..7b3b9010 --- /dev/null +++ b/helm/airbyte-v2/templates/minio.yaml @@ -0,0 +1,152 @@ +{{- /* Deploy MinIO (and PVC) when minio.enabled and storage type is s3 or minio, same as airbyte-v1 */}} +{{- if .Values.minio.enabled }} +{{- $storageType := lower (default "s3" .Values.global.storage.type) }} +{{- if or (eq $storageType "minio") (eq $storageType "s3") }} +apiVersion: apps/v1 # for k8s versions before 1.9.0 use apps/v1beta2 and before 1.8.0 use extensions/v1beta1 +kind: StatefulSet +metadata: + # This name uniquely identifies the Deployment + name: airbyte-minio + annotations: + helm.sh/hook: pre-install + helm.sh/hook-weight: "-1" + labels: + {{ include "airbyte.minioLabels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{ include "airbyte.minioSelectorLabels" . | nindent 6 }} + serviceName: airbyte-minio-svc + volumeClaimTemplates: + - metadata: + name: airbyte-minio-pv-claim + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{ .Values.minio.storage.volumeClaimValue }} + template: + metadata: + labels: + # Label is used as selector in the service. + {{ include "airbyte.minioSelectorLabels" . | nindent 8 }} + spec: + # Refer to the PVC created earlier + securityContext: + fsGroup: 1000 + containers: + - name: airbyte-minio + image: {{ include "imageUrl" (list .Values.minio.image $) }} + args: + - server + - /storage + env: + {{- include "airbyte.minio.envs" . | nindent 12 }} + ports: + - containerPort: 9000 + resources: + requests: + memory: {{ .Values.minio.resources.requests.memory }} + cpu: {{ .Values.minio.resources.requests.cpu }} + limits: + memory: {{ .Values.minio.resources.limits.memory }} + cpu: {{ .Values.minio.resources.limits.cpu }} + # Mount the volume into the pod + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + volumeMounts: + - name: airbyte-minio-pv-claim # must match the volume name, above + mountPath: "/storage" + {{- with .Values.minio.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.minio.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.minio.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + +--- +apiVersion: v1 +kind: Service +metadata: + name: airbyte-minio-svc + labels: + {{ include "airbyte.minioLabels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install + helm.sh/hook-weight: "-1" +spec: + ports: + - port: 9000 + targetPort: 9000 + protocol: TCP + selector: + {{ include "airbyte.minioSelectorLabels" . | nindent 4 }} +--- +# This pod creates the state-storage bucket in the minio server, +# which the local cloud deployment requires to store its state. +apiVersion: v1 +kind: Pod +metadata: + name: airbyte-minio-create-bucket + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded, before-hook-creation +spec: + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + restartPolicy: OnFailure + securityContext: + fsGroup: 1000 + containers: + - name: minio-mc + image: {{ include "imageUrl" (list .Values.minio.image $) }} + command: ["/bin/sh", "-c", + "until (/bin/mc config host add myminio $MINIO_ENDPOINT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY) do echo '...waiting...' && sleep 1; done; + /bin/mc mb --ignore-existing myminio/state-storage; + /bin/mc policy set public myminio/state-storage; + /bin/mc mb --ignore-existing myminio/airbyte-dev-logs; + /bin/mc policy set public myminio/airbyte-dev-logs;"] + securityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + env: + {{- include "airbyte.minio.envs" . | nindent 8 }} + {{- include "airbyte.storage.minio.endpoint.env" . | nindent 8 }} +{{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/secret.yaml b/helm/airbyte-v2/templates/secret.yaml new file mode 100644 index 00000000..41e08934 --- /dev/null +++ b/helm/airbyte-v2/templates/secret.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-airbyte-secrets + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" + labels: + {{- include "airbyte.labels" . | nindent 4 }} +type: Opaque +stringData: + {{- /* v1 order: database, storage, workload API, then auth/keycloak/rest */}} + {{- include "airbyte.database.secrets" . | nindent 2 }} + {{- if eq (include "airbyte.secretsManager.type" .) "testing_config_db_table" }} + {{- include "airbyte.database.localSecretsDatasource.secrets" . | nindent 2 }} + {{- end }} + {{- include "airbyte.storage.secrets" . | nindent 2 }} + {{- include "airbyte.workloadApiServer.secrets" . | nindent 2 }} + {{- include "airbyte.auth.bootstrap.secrets" . | nindent 2 }} + {{- include "airbyte.auth.identityProvider.secrets" . | nindent 2 }} + {{- include "airbyte.auth.instanceAdmin.enterprise.secrets" . | nindent 2 }} + {{- include "airbyte.customerio.secrets" . | nindent 2 }} + {{- include "airbyte.enterprise.secrets" . | nindent 2 }} + {{- include "airbyte.featureFlags.secrets" . | nindent 2 }} + {{- include "airbyte.keycloak.admin.user.secrets" . | nindent 2 }} + {{- include "airbyte.keycloak.database.secrets" . | nindent 2 }} + {{- include "airbyte.secretsManager.secrets" . | nindent 2 }} + {{- include "airbyte.temporal.cli.secrets" . | nindent 2 }} + {{- include "airbyte.temporal.cloud.credentials.secrets" . | nindent 2 }} + {{- if not (include "airbyte.workloadLauncher.dataPlane.useExistingSecret" .) }} + {{- include "airbyte.workloadLauncher.dataPlane.secrets" . | nindent 2 }} + {{- end }} + + {{- include "airbyte.extra.secrets" . | nindent 2 }} diff --git a/helm/airbyte-v2/templates/serviceaccount.yaml b/helm/airbyte-v2/templates/serviceaccount.yaml new file mode 100644 index 00000000..a88a25c1 --- /dev/null +++ b/helm/airbyte-v2/templates/serviceaccount.yaml @@ -0,0 +1,89 @@ +{{- if or (eq .Values.global.edition "community") (eq .Values.global.edition "cloud") (eq .Values.global.edition "enterprise") }} +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "airbyte.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install + helm.sh/hook-weight: "-10" + {{- with .Values.serviceAccount.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "airbyte.serviceAccountName" . }}-role + namespace: {{ .Release.Namespace }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install + helm.sh/hook-weight: "-5" +rules: + - apiGroups: [""] + resources: ["jobs", "pods", "pods/log", "pods/exec", "pods/attach", "pods/status", "secrets"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "airbyte.serviceAccountName" . }}-binding + namespace: {{ .Release.Namespace }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + helm.sh/hook: pre-install + helm.sh/hook-weight: "-3" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "airbyte.serviceAccountName" . }}-role +subjects: + - kind: ServiceAccount + name: {{ include "airbyte.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +--- +# Pod launcher Role: allows the job-pod service account to create/manage pods (sync, check, etc.) +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: airbyte-pod-launcher + namespace: {{ .Release.Namespace }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: + - pods + - pods/log + - pods/exec + - pods/status + verbs: + - create + - get + - list + - watch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: airbyte-pod-launcher-binding + namespace: {{ .Release.Namespace }} + labels: + {{- include "airbyte.labels" . | nindent 4 }} +roleRef: + kind: Role + name: airbyte-pod-launcher + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + name: {{ include "airbyte.jobPodServiceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} +{{- end }} diff --git a/helm/airbyte-v2/templates/tests/test-webapp.yaml b/helm/airbyte-v2/templates/tests/test-webapp.yaml new file mode 100644 index 00000000..35a8e648 --- /dev/null +++ b/helm/airbyte-v2/templates/tests/test-webapp.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: "{{ .Release.Name }}-test-connection" + labels: + {{- include "airbyte.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 8 }} + {{- end }} + {{- end }} + containers: + - name: webapp-test + image: {{ include "imageUrl" (list .Values.testWebapp.image $) }} + command: ['curl'] + args: ['-s', '{{ .Release.Name }}-airbyte-webapp-svc:{{ .Values.webapp.service.port }}', '>', '/dev/null'] + resources: + requests: + memory: "64Mi" + cpu: "100m" + limits: + memory: "128Mi" + cpu: "200m" + restartPolicy: Never diff --git a/helm/airbyte-v2/values.yaml b/helm/airbyte-v2/values.yaml new file mode 100644 index 00000000..85568f51 --- /dev/null +++ b/helm/airbyte-v2/values.yaml @@ -0,0 +1,2892 @@ +global: + # -- Service account name override. Leave empty to use release-namespaced SA (e.g. airbyte-v2-worker-sa) so worker and job pods share one SA with RBAC. + serviceAccountName: "" + # -- Edition; "community" or "enterprise" + edition: community + + # --AWS keys + aws_access_key: "" + aws_secret_key: "" + # -- Resource limits and requests for workloads (global defaults for pods without explicit resource blocks) + resource_cpu_limit: "2" + resource_cpu_request: "250m" + resource_memory_limit: "2Gi" + resource_memory_request: "512Mi" + + # -- Secret name for auth/bootstrap and other components that read from a single secret. + # Set to your existing secret (e.g. airbyte-v2-airbyte-secrets) so pods look up instance-admin-password etc. there. + secretName: "airbyte-v2-airbyte-secrets" + + # -- Optional seed (e.g. cluster name or infra ID) so chart-generated default secrets differ per infrastructure. + # When set, same release name + namespace on two clusters will get different dynamic values (instance admin, dataplane-client-id / dataplane-client-secret, JWT, workload API bearer token). Leave empty for deterministic defaults per release+namespace. + secretSeed: "" + + local: false + + cluster: + type: hybrid # "control-plane" | "data-plane" | "hybrid" + secretName: "" + + controlPlane: + bootstrap: + dataPlane: + secretName: "" + clientIdSecretKey: "" + clientSecretSecretKey: "" + + dataPlane: + # -- CONTROL_PLANE_AUTH_ENDPOINT. Leave empty to use global.airbyteUrl or, when that is also empty, internal server URL (http://RELEASE-airbyte-server-svc.NAMESPACE:8001). + controlPlaneAuthEndpoint: "" + + enterprise: + # -- Secret name where an Airbyte license key is stored + secretName: "airbyte-v2-airbyte-secrets" + # -- The key within `licenseKeySecretName` where the Airbyte license key is stored + licenseKeySecretKey: "license-key" + + # -- The URL where Airbyte will be reached; This should match your Ingress host + airbyteUrl: "" + + # Docker image config that will apply to all images. + image: + # Docker registry to pull platform images from, e.g. http://my-registry:8000/ + registry: "" + # Image tag to use for airbyte images. + # Does not include non-airbyte images such as temporal, minio, etc. + tag: "" + + # Docker image pull secret + imagePullSecrets: + - name: regcred + + # -- Auth configuration + auth: + # -- Whether auth is enabled + enabled: true + # -- Secret name for auth bootstrap ( auth.secretName). Bootloader writes here; server reads from here. Use chart-created secret name. + secretName: "airbyte-v2-airbyte-secrets" + # -- Same as secretName; bootloader AB_KUBERNETES_SECRET_NAME. Defaults to secretName if unset. + managedSecretName: "airbyte-v2-airbyte-secrets" + + # -- Admin user configuration + instanceAdmin: + # -- Secret name where the instanceAdmin configuration is stored + secretName: "airbyte-v2-airbyte-secrets" + # -- The first name of the initial user + firstName: "Airbyte" + # -- The last name of the initial user + lastName: "Admin" + # -- Login email / username for community auth (must match organization.email after setup, or AB_INSTANCE_ADMIN_USERNAME) + email: "devops@blotout.io" + # -- The key within `emailSecretName` where the initial user's email is stored + emailSecretKey: "instance-admin-email" + # -- The key within `passwordSecretName` where the initial user's password is stored + passwordSecretKey: "instance-admin-password" + # -- Instance admin password (app reads key instance-admin-password). When unset, a release-unique default is generated; set for production. Retrieve: kubectl get secret -n -o jsonpath='{.data.instance-admin-password}' | base64 -d + password: "" + + # -- SSO Identify Provider configuration; (requires Enterprise) + identityProvider: + # -- Secret name where the OIDC configuration is stored + secretName: "" + # -- The identity provider type (e.g. oidc) + type: "" # one of: "oidc" or "generic-oidc" + + # -- OIDC configuration (required if `auth.identityProvider.type` is "oidc") + #oidc: + # # -- OIDC application domain + # domain: "" + # # -- OIDC application name + # appName: "" + # # -- The key within `clientIdSecretName` where the OIDC client id is stored + # clientIdSecretKey: "" + # # -- The key within `clientSecretSecretName` where the OIDC client secret is stored + # clientSecretSecretKey: "" + + # -- Generic OIDC configuration (required if `auth.identityProvider.type` is "generic-oidc") + #genericOidc: + # clientId: "" + # audience: "" + # issuer: "" + # endpoints: + # authorizationServerEndpoint: "" + # jwksEndpoint: "" + # fields: + # subject: sub + # email: email + # name: name + # issuer: iss + + # -- Security configuration + security: + secretName: "" + + cookieSecureSetting: true + # Lax: cookie sent on same-site and top-level navigations; avoids 401 on API calls after login when Strict blocks in some cases. + cookieSameSiteSetting: Lax + + jwtSignatureSecretKey: "" + + # Dataplane auth: server validates workload-launcher token using these keys from the chart secret. + # Defaults match workloadLauncher.dataPlane: secret keys dataplane-client-id, dataplane-client-secret. + dataPlane: + clientIdSecretKey: "" # default: dataplane-client-id + clientSecretSecretKey: "" # default: dataplane-client-secret + + api: + # -- AIRBYTE_API_HOST. Leave empty to derive: from global.airbyteUrl when set (e.g. https://airbytev2.blotout.io/api/public); otherwise defaults to internal server URL (http://RELEASE-airbyte-server-svc.NAMESPACE:8001). + host: "" + authHeaderName: X-Airbyte-Auth + # -- AIRBYTE_API_AUTH_HEADER_VALUE for server and workload-launcher (internal API auth). Must match so the dataplane token endpoint accepts launcher. + internalAuthHeaderValue: "Internal server" + + # -- Environment variables (avoid setting SECRET_PERSISTENCE/S3_PATH_STYLE_ACCESS to "" or they override configmap and trigger "Unsetting empty") + env_vars: {} + + # -- Secrets + secrets: {} + + # -- Database configuration + database: + type: external # "external" + + # -- Secret name where database credentials are stored + secretName: "airbyte-v2-airbyte-secrets" # e.g. "airbyte-config-secrets" + + # -- The database host + host: "airbyte-host" + + # -- The database port + port: 3306 + + # -- The database name + database: "airbyte-v1" + + # -- The database user + user: "rds_admin" + # -- The key within `secretName` where the user is stored + #userSecretKey: "" # e.g. "database-user" + + # -- The database password + password: "pwd" + # -- The key within `secretName` where the password is stored + #passwordSecretKey: "" # e.g."database-password" + + migrations: + runAtStartup: true + + configDb: + minimumFlywayMigrationVersion: "" + + jobsDb: + minimumFlywayMigrationVersion: "" + + storage: + secretName: "airbyte-v2-airbyte-secrets" + # -- The storage backend type. Supports s3 (default, same as airbyte-v1), gcs, azure, minio + type: s3 + + # Minio + minio: + accessKeyId: minio + secretAccessKey: minio123 + + bucket: + log: b-multitenantdev-dev-airbyte-v1-logs + state: b-multitenantdev-dev-airbyte-v1-logs + workloadOutput: b-multitenantdev-dev-airbyte-v1-logs + activityPayload: b-multitenantdev-dev-airbyte-v1-logs + # Required by AirbyteStorageConfig (STORAGE_BUCKET_AUDIT_LOGGING) + auditLogging: b-multitenantdev-dev-airbyte-v1-logs + + # S3 + s3: + region: "us-east-1" ## e.g. us-east-1 + authenticationType: credentials ## Use "credentials" or "instanceProfile" + accessKeyId: "" + secretAccessKey: "" + + # GCS + #gcs: + # projectId: + # credentialsJson: + # credentialsJsonPath: /secrets/gcs-log-creds/gcp.json + + # Azure + #azure: + # # one of the following: connectionString, connectionStringSecretKey + # connectionString: + # connectionStringSecretKey: + + secretsManager: + enabled: false + # type: no_op (default), testing_config_db_table, aws_secret_manager, google_secret_manager, vault, azure_key_vault. Not "kubernetes". + # Default no_op avoids job pod init "datasources.local-secrets" failure. Set to "testing_config_db_table" for connector secrets in config DB (requires launcher to pass DATASOURCES_LOCALSECRETS_* to job pods). + type: "" + #secretName: "airbyte-config-secrets" + + #awsSecretManager: + # region: + # authenticationType: credentials ## Use "credentials" or "instanceProfile" + # tags: ## Optional - You may add tags to new secrets created by Airbyte. + # - key: ## e.g. team + # value: ## e.g. deployments + # - key: business-unit + # value: engineering + # kms: ## Optional - ARN for KMS Decryption. + + #googleSecretManager: + # projectId: + # credentialsSecretKey: gcp.json + + #azureKeyVault: + # tenantId: "" + # vaultUrl: "" + # clientId: "" + # clientIdSecretKey: "" + # clientSecret: "" + # clientSecretSecretKey: "" + # tags: "" + + #vault: + # address: "" + # prefix: "" + # authToken: "" + # authTokenSecretKey: "" + + logging: + level: info + httpAccessLogsEnabled: false + log4jConfigFile: "" + + connectorRegistry: + seedProvider: local + + connectorRollout: + expirationSeconds: + waitBetweenRolloutSeconds: + waitBetweenSyncResultsQueriesSeconds: + + workloads: + secretName: "" + namespace: jobs + + images: + workloadInit: "" + connectorSideCar: "" + containerOrchestrator: "" + + containerOrchestrator: + secretName: "" + dataPlane: + secretName: "" + secretMountPath: "" + + queues: + check: [CHECK_CONNECTION] + discover: [DISCOVER_SCHEMA] + sync: [SYNC] + + pubSub: + enabled: false + topicName: "" + + resources: + useConnectorResourceDefaults: true + + mainContainer: + cpu: + request: "1" + limit: "6" + memory: + request: "5Gi" + limit: "10Gi" + check: + cpu: {} + memory: {} + discover: + cpu: {} + memory: {} + replication: + cpu: {} + memory: {} + sidecar: + cpu: {} + memory: {} + + fileTransfer: + storage: + request: 5G + limit: 10G + + featureFlags: + secretName: "" + client: configfile + + configfile: {} + + #launchdarkly: + # key: "" + + java: + opts: [] + + metrics: + enabled: false + step: "" + + otlp: + enabled: false + # -- The open-telemetry-collector endpoint that metrics will be sent to + collectorEndpoint: "" + + statsd: + enabled: false + flavor: "" + host: "" + port: "" + + otel: + resourceAttributes: {} + + collector: + endpoint: "" + + exporter: + name: otlp + protocol: grpc + timeout: 30000 + metricExportInterval: 10000 + + # Jobs resource requests and limits, see http://kubernetes.io/docs/user-guide/compute-resources/ + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. + jobs: + resources: + ## Example: + ## requests: + ## memory: 256Mi + ## cpu: 250m + # -- Job resource requests + requests: {} + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- Job resource limits + limits: {} + + kube: + ## JOB_KUBE_ANNOTATIONS + # pod annotations of the sync job and the default pod annotations fallback for others jobs + # -- key/value annotations applied to kube jobs + annotations: {} + + ## JOB_KUBE_LABELS + ## pod labels of the sync job and the default pod labels fallback for others jobs + # -- key/value labels applied to kube jobs + labels: {} + + ## JOB_KUBE_NODE_SELECTORS + ## pod node selector of the sync job and the default pod node selector fallback for others jobs + # -- Node labels for pod assignment + nodeSelector: {} + + ## JOB_KUBE_TOLERATIONS + # -- Node tolerations for pod assignment + # Any boolean values should be quoted to ensure the value is passed through as a string. + tolerations: [] + + ## JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET + # -- image pull secret to use for job pod + mainContainerImagePullSecret: "" + + ## JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY + # -- image pull policy to use for job pod + mainContainerImagePullPolicy: "" + + localVolume: + enabled: false + + scheduling: + check: + nodeSelectors: {} + runtimeClassName: "" + discover: + nodeSelectors: {} + runtimeClassName: "" + isolated: + nodeSelectors: {} + runtimeClassName: "" + sourceDeclarativeManifest: + nodeSelectors: {} + runtimeClassName: "" + + errors: + reportingStrategy: logging + + #sentry: + # dsn: "" + + topology: + nodeSelectorLabel: airbyte/node-pool + nodeSelectors: + mainNodePool: main + jobsNodePool: jobs + quickJobsNodePool: quick-jobs + + temporal: + secretName: "" + + cli: + address: "" + namespace: "" + tlsCert: "" + tlsCertSecretKey: TEMPORAL_CLOUD_CLIENT_CERT + tlsKey: "" + tlsKeySecretKey: TEMPORAL_CLOUD_CLIENT_KEY + + cloud: + enabled: false + host: "" + namespace: "" + clientCert: "" + clientCertSecretKey: TEMPORAL_CLOUD_CLIENT_CERT + clientKey: "" + clientKeySecretKey: TEMPORAL_CLOUD_CLIENT_KEY + + sdk: + rpc: + timeout: 120s + longPollTimeout: 140s + queryTimeout: 20s + + datadog: + enabled: false + env: dev + traceAgentPort: 8126 + + statsd: + port: 8125 + + integrations: + dbm: + enabled: false + propagationMode: full + + grpc: + enabled: false + clientEnabled: false + serverEnabled: false + + googleHttpClient: + enabled: false + + httpUrlConnection: + enabled: false + + kotlinCoroutineExperimental: + enabled: false + + urlConnection: + enabled: false + + netty: + enabled: false + + netty41: + enabled: false + + customerio: + secretName: "" + apiKeySecretKey: "" + + tracking: + enabled: true + secretName: "" + # Use "logging" to avoid loading Segment from cdn.segment.com (fails in restricted networks and can cause UI errors). Set to "segment" only if your environment can reach cdn.segment.com. + strategy: logging # one of: logging, segment + + segment: + writeKey: "" + writeKeySecretKey: "" + + micronaut: + environments: [] + + extraSelectorLabels: {} + extraInitContainers: [] + extraContainers: [] + +## @section Common Parameters + +# -- String to partially override airbyte.fullname template with a string (will prepend the release name) +nameOverride: "" +# -- String to fully override airbyte.fullname template with a string +fullnameOverride: "" + +# Pods Service Account, see https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +serviceAccount: + # -- Specifies whether a ServiceAccount should be created + create: true + # -- Annotations for service account. Evaluated as a template. Only used if `create` is `true`. + annotations: {} + # -- Name of the service account to use. If not set and create is true, uses RELEASE_NAME-worker-sa (e.g. airbyte-v2-worker-sa). + name: "" + +# -- Sets the AIRBYTE_VERSION environment variable. Defaults to Chart.AppVersion. +## If changing the image tags below, you should probably also update this. +version: "" + +## @section Webapp Parameters + +webapp: + enabled: false + # -- Number of webapp replicas + replicaCount: 1 + + serviceAccountName: + + ## webapp.image.repository + ## webapp.image.pullPolicy + ## webapp.image.tag The airbyte webapp image tag. Defaults to the chart's AppVersion + image: + # -- The repository to use for the airbyte webapp image + repository: airbyte/webapp + # -- The pull policy to use for the airbyte webapp image + pullPolicy: IfNotPresent + + annotations: {} + + # -- Add extra annotations to the webapp pods + podAnnotations: {} + + # -- webapp.podLabels [object] Add extra labels to the webapp pods + podLabels: {} + + # -- Security context for the container + podSecurityContext: + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + ## Configure extra options for the webapp containers' liveness and readiness probes, + ## see https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes + livenessProbe: + # -- Enable livenessProbe on the webapp + enabled: true + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 30 + # -- Period seconds for livenessProbe + periodSeconds: 10 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 1 + # -- Failure threshold for livenessProbe + failureThreshold: 3 + # -- Success threshold for livenessProbe + successThreshold: 1 + + readinessProbe: + # -- Enable readinessProbe on the webapp + enabled: true + # -- Initial delay seconds for readinessProbe + initialDelaySeconds: 10 + # -- Period seconds for readinessProbe + periodSeconds: 10 + # -- Timeout seconds for readinessProbe + timeoutSeconds: 1 + # -- Failure threshold for readinessProbe + failureThreshold: 3 + # -- Success threshold for readinessProbe + successThreshold: 1 + + service: + # -- The service type to use for the webapp service + type: ClusterIP + # -- The service port to expose the webapp on + port: 80 + # -- Annotations for the webapp service resource + annotations: {} + + ## Web app resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- The resources limits for the Web container + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + # -- The requested resources for the Web container + requests: {} + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for webapp pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for webapp pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + ## Configure the ingress resource that allows you to access the Airbyte installation. + ## ref: http://kubernetes.io/docs/user-guide/ingress/ + ingress: + # -- Set to true to enable ingress record generation + enabled: true + # -- Specifies ingressClassName for clusters >= 1.18+ + className: "nginx" + # -- Ingress annotations. Defaults include higher proxy timeouts to avoid 504 when creating sources/destinations (discovery and check can be slow). Override with webapp.ingress.annotations to replace. + annotations: + nginx.ingress.kubernetes.io/proxy-read-timeout: "600" + nginx.ingress.kubernetes.io/proxy-send-timeout: "600" + nginx.ingress.kubernetes.io/proxy-connect-timeout: "300" + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + # -- The list of hostnames to be covered with this ingress record. + hosts: + - host: + paths: + # pathType ImplementationSpecific to match airbyte-v1 + - path: / + pathType: ImplementationSpecific + # - host: chart-example.local + # paths: + # - path: / + # pathType: ImplementationSpecific + # -- Custom ingress TLS configuration + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + + api: + # -- The webapp API url. Airbyte v2 unified server uses /api/public/v1/ (default). Do not use /api/v1/ or the UI may get wrong responses and throw e.g. "Cannot read properties of undefined (reading 'sort')". + url: /api/v1/ + + # -- Connector Builder Server URL path (same as v1). When set (e.g. /connector-builder-api), webapp uses this path on the same origin and ingress routes it to connector-builder-server. Default matches airbyte-v1. + connectorBuilderServer: + url: "/connector-builder-api" + + fullstory: + # -- Whether or not to enable fullstory + enabled: false + + ## Examples (when using `webapp.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: var-run + ## mountPath: /var/run/ + ## - name: var-cache-nginx + ## mountPath: /var/cache/nginx + ## - mountPath: /etc/nginx/conf.d + ## name: nginx-conf-d + ## + # -- Additional volumeMounts for webapp containers + extraVolumeMounts: [] + + ## Examples (when using `webapp.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: var-run + ## emptyDir: {} + ## - name: var-cache-nginx + ## emptyDir: {} + ## - name: nginx-conf-d + ## emptyDir: {} + # -- Additional volumes for webapp pods + extraVolumes: [] + + ## Example: + # extraContainers: + # - name: otel_collector + # image: somerepo/someimage:sometag + # args: [ + # "--important-args" + # ] + # ports: + # - containerPort: 443 + # volumeMounts: + # - name: volumeMountCool + # mountPath: /some/path + # readOnly: true + # -- Additional container for server pods + extraContainers: [] + + # -- When using in-cluster Temporal (temporal.enabled and not Temporal Cloud), the server runs an init container + # that waits for Temporal to be reachable before starting. Override the image if busybox is not suitable. + temporalWaitInitContainer: + image: "busybox:1.36" + + ## Example: + # extraInitContainers: + # - name: sleepy + # image: alpine + # command: ['sleep', '60'] + # -- Additional init containers for server pods + extraInitContainers: [] + + ## Example: (With default env vars and values taken from generated config map) + # extraEnv: + # - name: POSTGRES_USER + # valueFrom: + # secretKeyRef: + # name: airbyte-secrets + # key: DATABASE_USER + # - name: POSTGRES_PWD + # valueFrom: + # secretKeyRef: + # name: airbyte-secrets + # key: DATABASE_PASSWORD + # - name: DYNAMIC_CONFIG_FILE_PATH + # value: "config/dynamicconfig/development.yaml" + # - name: DB + # value: "postgresql" + # - name: DB_PORT + # valueFrom: + # configMapKeyRef: + # name: airbyte-env + # key: DATABASE_PORT + # - name: POSTGRES_SEEDS + # valueFrom: + # configMapKeyRef: + # name: airbyte-env + # key: DATABASE_HOST + # -- Supply extra env variables to main container using full notation + extraEnv: [] + ## Example: + ## secrets: + ## DATABASE_PASSWORD: strong-password + ## DATABASE_USER: my-db-user + # -- Supply additional secrets to container + secrets: {} + + ## Example: + ## env_vars: + ## DATABASE_HOST: airbyte-db + ## DATABASE_PORT: 5432 + # -- Supply extra env variables to main container using simplified notation + env_vars: {} + +## @section Server parameters + +server: + enabled: true + # -- Number of server replicas (2 for 10-pipeline workload) + replicaCount: 1 + + serviceAccountName: + + deploymentStrategyType: Recreate + + applications: "" + + image: + # -- The repository to use for the airbyte server image. + repository: airbyte/server + # -- the pull policy to use for the airbyte server image + pullPolicy: IfNotPresent + + ## service.type The service type to use for the API server + ## service.port The service port to expose the API server on + service: + type: ClusterIP + port: 8001 + annotations: {} + + annotations: {} + + # -- Add extra annotations to the server pods + podAnnotations: {} + + # -- Add extra labels to the server pods + podLabels: {} + + # -- Security context for the container + podSecurityContext: + # gid=1000(airbyte) + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + ## Configure extra options for the server containers' liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes + livenessProbe: + # -- Enable livenessProbe on the server + enabled: true + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 30 + # -- Period seconds for livenessProbe + periodSeconds: 10 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 10 + # -- Failure threshold for livenessProbe + failureThreshold: 3 + # -- Success threshold for livenessProbe + successThreshold: 1 + + readinessProbe: + # -- Enable readinessProbe on the server + enabled: true + # -- Initial delay seconds for readinessProbe + initialDelaySeconds: 10 + # -- Period seconds for readinessProbe + periodSeconds: 10 + # -- Timeout seconds for readinessProbe + timeoutSeconds: 10 + # -- Failure threshold for readinessProbe + failureThreshold: 3 + # -- Success threshold for readinessProbe + successThreshold: 1 + + ## server resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- The resources limits for the server container (observed: 3m CPU, 951Mi mem) + limits: + cpu: 2 + memory: 4Gi + # -- The requested resources for the server container + requests: + cpu: 250m + memory: 1Gi + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for server pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for server pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + log: + # -- The log level to log at + level: INFO + + ## Examples (when using `server.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: tmpdir + ## mountPath: /tmp + # -- Additional volumeMounts for server containers + extraVolumeMounts: [] + + ## Examples (when using `server.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: tmpdir + ## emptyDir: {} + # -- Additional volumes for server pods + extraVolumes: [] + + ## Example: + # extraContainers: + # - name: otel_collector + # image: somerepo/someimage:sometag + # args: [ + # "--important-args" + # ] + # ports: + # - containerPort: 443 + # volumeMounts: + # - name: volumeMountCool + # mountPath: /some/path + # readOnly: true + # -- Additional container for server pods + extraContainers: [] + + ## Example: + # extraInitContainers: + # - name: sleepy + # image: alpine + # command: ['sleep', '60'] + # -- Additional init containers for server pods + extraInitContainers: [] + + ## Example: (With default env vars and values taken from generated config map) + # extraEnv: + # - name: AIRBYTE_VERSION + # valueFrom: + # configMapKeyRef: + # name: airbyte-env + # key: AIRBYTE_VERSION + # - name: API_URL + # valueFrom: + # configMapKeyRef: + # name: airbyte-env + # key: API_URL + # - name: TRACKING_STRATEGY + # valueFrom: + # configMapKeyRef: + # name: airbyte-env + # key: TRACKING_STRATEGY + # - name: FULLSTORY + # valueFrom: + # configMapKeyRef: + # name: airbyte-env + # key: FULLSTORY + # - name: INTERNAL_API_HOST + # valueFrom: + # configMapKeyRef: + # name: airbyte-env + # key: INTERNAL_API_HOST + ## + # -- Supply extra env variables to main container using full notation + extraEnv: [] + ## Example: + ## secrets: + ## DATABASE_PASSWORD: strong-password + ## DATABASE_USER: my-db-user + # -- Supply additional secrets to container + secrets: {} + ## Example: + ## env_vars: + ## DATABASE_HOST: airbyte-db + ## DATABASE_PORT: 5432 + # -- Supply extra env variables to main container using simplified notation + env_vars: {} + + debug: + enabled: false + remoteDebugPort: 5005 + +## @section Worker Parameters + +worker: + enabled: true + # -- Number of worker replicas (3 for 10-pipeline workload) + replicaCount: 1 + + serviceAccountName: + + image: + # -- The repository to use for the airbyte worker image. + repository: airbyte/worker + # -- the pull policy to use for the airbyte worker image + pullPolicy: IfNotPresent + + annotations: {} + + # -- Add extra annotations to the worker pods + podAnnotations: {} + + # -- Add extra labels to the worker pods + podLabels: {} + + # -- Security context for the container + podSecurityContext: + # gid=1000(airbyte) + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + livenessProbe: + # -- Enable livenessProbe on the worker + enabled: true + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 30 + # -- Period seconds for livenessProbe + periodSeconds: 10 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 1 + # -- Failure threshold for livenessProbe + failureThreshold: 3 + # -- Success threshold for livenessProbe + successThreshold: 1 + + readinessProbe: + # -- Enable readinessProbe on the worker + enabled: true + # -- Initial delay seconds for readinessProbe + initialDelaySeconds: 10 + # -- Period seconds for readinessProbe + periodSeconds: 10 + # -- Timeout seconds for readinessProbe + timeoutSeconds: 1 + # -- Failure threshold for readinessProbe + failureThreshold: 3 + # -- Success threshold for readinessProbe + successThreshold: 1 + + ## worker resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- The resources limits for the worker container (observed: 3m CPU, 520Mi mem) + limits: + cpu: 2 + memory: 4Gi + # -- The requested resources for the worker container + requests: + cpu: 250m + memory: 768Mi + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for worker pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for worker pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + log: + #! -- The log level to log at. + level: "INFO" + + ## Example: + ## + ## extraEnv: + ## - name: JOB_KUBE_TOLERATIONS + ## value: "key=airbyte-server,operator=Equals,value=true,effect=NoSchedule" + # -- Additional env vars for worker pods + extraEnv: [] + + ## Examples (when using `worker.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: tmpdir + ## mountPath: /tmp + # -- Additional volumeMounts for worker containers + extraVolumeMounts: [] + + ## Examples (when using `worker.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: tmpdir + ## emptyDir: {} + # -- Additional volumes for worker pods + extraVolumes: [] + # -- Additional container for worker pods + extraInitContainers: [] + extraContainers: [] + + hpa: + enabled: false + + debug: + enabled: false + remoteDebugPort: 5005 + + ## current no exist documentations + activityMaxAttempt: "" + activityInitialDelayBetweenAttemptsSeconds: "" + activityMaxDelayBetweenAttemptsSeconds: "" + maxNotifyWorkers: "" + maxCheckWorkers: "" + maxSyncWorkers: "" + + workloadApi: {} + + env_vars: + CONTAINER_ORCHESTRATOR_ENABLED: true + +## @section Workload Launcher Parameters + +workloadLauncher: + enabled: true + # -- Number of workload launcher replicas + replicaCount: 1 + + serviceAccountName: + + image: + # -- The repository to use for the airbyte workload launcher image. + repository: airbyte/workload-launcher + # -- The pull policy to use for the airbyte workload launcher image + pullPolicy: IfNotPresent + + controlPlane: + # -- When true (default), use internal server URL for token endpoint in control-plane/hybrid so the launcher hits the server directly (avoids 500 from session requirement on public ingress). Set to false to use global.airbyteUrl. + useInternalUrl: true + # -- Override token endpoint (base URL, e.g. http://airbyte-v2-airbyte-server-svc.airbyte-v2:8001). When set, /api/v1/dataplanes/token is appended. Empty = use internal or public URL per useInternalUrl and cluster type. + tokenEndpoint: "" + + # -- Data plane ID (v1: DATA_PLANE_ID=local for same-cluster launcher). Default "local". + dataPlaneId: "" + + # -- Workspace docker mount (v1: WORKSPACE_DOCKER_MOUNT=workspace). Default "workspace". + workspaceDockerMount: "" + + # Set existingSecretName to read dataplane creds from that secret (no hardcoding in values). Keys: dataplane-client-id, dataplane-client-secret. + # Launcher uses these secret key names to create token; use lowercase keys so it reads dataplane-client-id / dataplane-client-secret (not DATAPLANE_CLIENT_ID / DATAPLANE_CLIENT_SECRET). + dataPlane: + existingSecretName: "" # e.g. "airbyte-dataplane-creds" – create this secret with id/secret matching service_accounts in DB + secretName: "" + clientId: "" + clientIdSecretName: "" + clientIdSecretKey: "dataplane-client-id" # secret key to read for token; keep lowercase so launcher uses same key as server + clientSecret: "" + clientSecretSecretName: "" + clientSecretSecretKey: "dataplane-client-secret" # secret key to read for token; keep lowercase so launcher uses same key as server + + env_vars: {} + + annotations: {} + + # -- Add extra annotations to the workload launcher pods + podAnnotations: {} + + # -- Add extra labels to the workload launcher pods + podLabels: {} + + # -- Security context for the container + podSecurityContext: + # gid=1000(airbyte) + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + livenessProbe: + # -- Enable livenessProbe on the workload launcher (tcpSocket on 8016 to avoid HTTP timeout/401) + enabled: true + # -- Initial delay seconds for livenessProbe (increase if kube-api-access mount or startup is slow) + initialDelaySeconds: 60 + # -- Period seconds for livenessProbe + periodSeconds: 10 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 1 + # -- Failure threshold for livenessProbe + failureThreshold: 3 + # -- Success threshold for livenessProbe + successThreshold: 1 + + readinessProbe: + # -- Enable readinessProbe on the workload launcher (tcpSocket on 8016) + enabled: true + # -- Initial delay seconds for readinessProbe + initialDelaySeconds: 30 + # -- Period seconds for readinessProbe + periodSeconds: 10 + # -- Timeout seconds for readinessProbe + timeoutSeconds: 1 + # -- Failure threshold for readinessProbe + failureThreshold: 3 + # -- Success threshold for readinessProbe + successThreshold: 1 + + ## workload launcher resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- The resources limits for the workload launcher container (observed: 3m CPU, 440Mi mem) + limits: + cpu: 2 + memory: 4Gi + # -- The requested resources for the workload launcher container + requests: + cpu: 250m + memory: 512Mi + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for workload launcher pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # Affinity and anti-affinity for workload launcher pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + log: + # -- The log level to log at + level: INFO + + ## Example: + ## + ## extraEnv: + ## - name: JOB_KUBE_TOLERATIONS + ## value: "key=airbyte-server,operator=Equals,value=true,effect=NoSchedule" + # -- Additional env vars for workload launcher pods + extraEnv: [] + + ## Examples (when using `workload launcher.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: tmpdir + ## mountPath: /tmp + # -- Additional volumeMounts for workload launcher containers + extraVolumeMounts: [] + + ## Examples (when using `workload-launcher.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: tmpdir + ## emptyDir: {} + # -- Additional volumes for workload launcher pods + extraVolumes: [] + + extraInitContainers: [] + extraContainers: [] + + hpa: + enabled: false + + debug: + enabled: false + remoteDebugPort: 5005 + + containerOrchestrator: + # -- Enable or disable Orchestrator + enabled: true + + secretName: "" + + # -- Orchestrator image + # This is a template string that will be passed to the "tpl" helper. + image: "airbyte/container-orchestrator:{{ default .Chart.AppVersion .Values.global.image.tag }}" + + connectorSidecar: + # -- Connector Sidecar image + # This is a template string that will be passed to the "tpl" helper. + image: "airbyte/connector-sidecar:{{ default .Chart.AppVersion .Values.global.image.tag }}" + + workloadInit: + # -- Workload init image + # This is a template string that will be passed to the "tpl" helper. + image: "airbyte/workload-init-container:{{ default .Chart.AppVersion .Values.global.image.tag }}" + + connectorProfiler: + # -- Workload init image + # This is a template string that will be passed to the "tpl" helper. + image: "airbyte/async-profiler:{{ default .Chart.AppVersion .Values.global.image.tag }}" + + ## current no exist documentations + activityMaxAttempt: "" + activityInitialDelayBetweenAttemptsSeconds: "" + activityMaxDelayBetweenAttemptsSeconds: "" + + workloadApi: {} + +## @section Rollout Worker Parameters + +connectorRolloutWorker: + enabled: false + # -- Number of connector rollout worker replicas + replicaCount: 1 + + serviceAccountName: + + image: + # -- The repository to use for the airbyte connector rollout worker image. + repository: airbyte/connector-rollout-worker + # -- The pull policy to use for the airbyte connector rollout worker image + pullPolicy: IfNotPresent + + annotations: {} + + # -- Add extra annotations to the connector rollout worker pods + podAnnotations: {} + + # -- Add extra labels to the connector rollout worker pods + podLabels: {} + + # -- Security context for the container + podSecurityContext: + # gid=1000(airbyte) + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + livenessProbe: + # -- Enable livenessProbe on the connector rollout worker + enabled: true + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 30 + # -- Period seconds for livenessProbe + periodSeconds: 10 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 1 + # -- Failure threshold for livenessProbe + failureThreshold: 3 + # -- Success threshold for livenessProbe + successThreshold: 1 + + readinessProbe: + # -- Enable readinessProbe on the connector rollout worker + enabled: true + # -- Initial delay seconds for readinessProbe + initialDelaySeconds: 10 + # -- Period seconds for readinessProbe + periodSeconds: 10 + # -- Timeout seconds for readinessProbe + timeoutSeconds: 1 + # -- Failure threshold for readinessProbe + failureThreshold: 3 + # -- Success threshold for readinessProbe + successThreshold: 1 + + ## connector rollout worker resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- The resources limits for the connector rollout worker container + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + # -- The requested resources for the connector rollout worker container + requests: {} + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for connector rollout worker pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # Affinity and anti-affinity for connector rollout worker pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + log: + # -- The log level to log at + level: INFO + + ## Example: + ## + ## extraEnv: + ## - name: JOB_KUBE_TOLERATIONS + ## value: "key=airbyte-server,operator=Equals,value=true,effect=NoSchedule" + # -- Additional env vars for connector rollout worker pods + extraEnv: [] + + ## Examples (when using `connector rollout worker.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: tmpdir + ## mountPath: /tmp + # -- Additional volumeMounts for connector rollout worker containers + extraVolumeMounts: [] + + ## Examples (when using `connector-rollout-worker.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: tmpdir + ## emptyDir: {} + # -- Additional volumes for connector rollout worker pods + extraVolumes: [] + + extraInitContainers: [] + extraContainers: [] + + env_vars: {} + + hpa: + enabled: false + + debug: + enabled: false + remoteDebugPort: 5005 + + containerOrchestrator: + # -- Enable or disable Orchestrator + enabled: true + # -- Orchestrator image + image: "" + + ## current no exist documentations + activityMaxAttempt: "" + activityInitialDelayBetweenAttemptsSeconds: "" + activityMaxDelayBetweenAttemptsSeconds: "" + +## @section Metrics parameters +metrics: + enabled: false + + # -- Number of metrics-reporter replicas + replicaCount: 1 + + serviceAccountName: + + image: + # -- The repository to use for the airbyte metrics-reporter image. + repository: airbyte/metrics-reporter + # -- The pull policy to use for the airbyte metrics-reporter image + pullPolicy: IfNotPresent + + annotations: {} + + # -- Add extra annotations to the metrics-reporter pod + podAnnotations: {} + + # -- Add extra labels to the metrics-reporter pod + podLabels: {} + + # -- Security context for the container + podSecurityContext: + # gid=1000(airbyte) + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + ## metrics-reporter app resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- The resources limits for the metrics-reporter container + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + # -- The requested resources for the metrics-reporter container + requests: {} + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for metrics-reporter pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for metrics-reporter pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + ## Example: + ## + ## extraEnv: + ## - name: SAMPLE_ENV_VAR + ## value: "key=sample-value" + # -- Additional env vars for metrics-reporter pods + extraEnv: [] + + ## Examples (when using `metrics.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: tmpdir + ## mountPath: /tmp + # -- Additional volumeMounts for metrics-reporter containers + extraVolumeMounts: [] + + ## Examples (when using `metrics.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: tmpdir + ## emptyDir: {} + # -- Additional volumes for metrics-reporter pods + extraVolumes: [] + + extraInitContainers: [] + extraContainers: [] + + secrets: {} + + env_vars: {} + +## @section Bootloader Parameters + +airbyteBootloader: + enabled: true + + serviceAccountName: + + image: + # -- The repository to use for the airbyte bootloader image. + repository: airbyte/bootloader + # -- The pull policy to use for the airbyte bootloader image + pullPolicy: IfNotPresent + + annotations: {} + + # -- Add extra annotations to the bootloader pod + podAnnotations: {} + + # -- Add extra labels to the bootloader pod + podLabels: {} + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for worker pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + ## Bootloader resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## requests: + ## memory: 256Mi + ## cpu: 250m + # -- The requested resources for the airbyte bootloader image + requests: {} + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- The resources limits for the airbyte bootloader image + limits: {} + + # -- Affinity and anti-affinity for bootloader pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + # -- Security context for the container + podSecurityContext: + # gid=1000(airbyte) + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + ## Example: (With default env vars and values taken from generated config map) + ## extraEnv: + ## - name: AIRBYTE_VERSION + ## valueFrom: + ## configMapKeyRef: + ## name: airbyte-env + ## key: AIRBYTE_VERSION + ## - name: DATABASE_HOST + ## valueFrom: + ## configMapKeyRef: + ## name: airbyte-env + ## key: DATABASE_HOST + ## - name: DATABASE_PORT + ## valueFrom: + ## configMapKeyRef: + ## name: airbyte-env + ## key: DATABASE_PORT + ## - name: DATABASE_PASSWORD + ## valueFrom: + ## secretKeyRef: + ## name: airbyte-secrets + ## key: DATABASE_PASSWORD + ## - name: DATABASE_URL + ## valueFrom: + ## configMapKeyRef: + ## name: airbyte-env + ## key: DATABASE_URL + ## - name: DATABASE_USER + ## valueFrom: + ## secretKeyRef: + ## name: airbyte-secrets + ## key: DATABASE_USER + # -- Supply extra env variables to main container using full notation + extraEnv: [] + ## Example: + ## secrets: + ## DATABASE_PASSWORD: strong-password + ## DATABASE_USER: my-db-user + # -- Supply additional secrets to container + secrets: {} + + ## Example: + ## env_vars: + ## DATABASE_HOST: airbyte-db + ## DATABASE_PORT: 5432 + # -- Supply extra env variables to main container using simplified notation + env_vars: {} + ## Example: + # extraContainers: + # - name: otel_collector + # image: somerepo/someimage:sometag + # args: [ + # "--important-args" + # ] + # ports: + # - containerPort: 443 + # volumeMounts: + # - name: volumeMountCool + # mountPath: /some/path + # readOnly: true + # -- Additional container for server pod(s) + extraContainers: [] + + ## Example: + # extraInitContainers: + # - name: sleepy + # image: alpine + # command: ['sleep', '60'] + # -- Additional init containers for server pods + extraInitContainers: [] + + ## Examples (when using `containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: tmpdir + ## mountPath: /tmp + # -- Additional volumeMounts for server containers + extraVolumeMounts: [] + + ## Examples (when using `containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: tmpdir + ## emptyDir: {} + # -- Additional volumes for server pods + extraVolumes: [] + +## @section Temporal parameters + +temporal: + enabled: true + # -- The number of temporal replicas to deploy + replicaCount: 1 + + serviceAccountName: + + image: + # -- The temporal image repository to use + repository: temporalio/auto-setup + # -- The pull policy for the temporal image + pullPolicy: IfNotPresent + # -- The temporal image tag to use + tag: "1.27.2" + + # -- Override container command. Leave empty/nil to use image default (required for temporalio/auto-setup; it has its own entrypoint that runs /etc/temporal/auto-setup.sh). Set only when using airbyte/temporal image: ["/bin/sh", "-c", "exec ./entrypoint.sh autosetup"]. + command: [] + + # -- Override container args (used when command is set; leave empty to use image default). + args: [] + + secretName: "" + + autoSetup: true + + database: + engine: postgres12 + host: "" + port: "" + user: "" + userSecretKey: "" + password: "" + passwordSecretKey: "" + + # -- Override Temporal server address (host:port). When set, all components use this instead of in-cluster Release.Name-temporal. + # Use for a separate Temporal cluster for v2: set to your v2 Temporal address (e.g. "temporal-v2.other-ns.svc.cluster.local:7233"), and set temporal.enabled: false so this chart does not deploy Temporal. + host: "" + + sdk: + rpc: + timeout: 120s + longPollTimeout: 140s + queryTimeout: 20s + + worker: + ports: "" + + service: + # -- The Kubernetes Service Type + type: ClusterIP + # -- The temporal port and exposed kubernetes port + port: 7233 + + prometheus: + endpoint: "" + + annotations: {} + + # -- Add extra annotations to the temporal pod + podAnnotations: {} + + # -- Add extra labels to the temporal pod + podLabels: {} + + # -- Security context for the container + podSecurityContext: + # gid=1000(temporal) + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(temporal) + runAsUser: 1000 + # gid=1000(temporal) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + ## Examples (when using `temporal.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraInitContainers: + ## - name: config-loader + ## image: temporalio/auto-setup:1.27.2 + ## command: + ## - /bin/sh + ## - -c + ## - >- + ## find /etc/temporal/config/ -maxdepth 1 -mindepth 1 -exec cp -ar {} /config/ \; + ## volumeMounts: + ## - name: config + ## mountPath: /config + # -- Additional InitContainers to initialize the pod + extraInitContainers: [] + + livenessProbe: + # -- Enable livenessProbe on the temporal + enabled: false + + readinessProbe: + # -- Enable readinessProbe on the temporal + enabled: false + + # -- Node labels for temporal pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for temporal pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for temporal pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + ## Example: + ## + ## extraEnv: + ## - name: SAMPLE_ENV_VAR + ## value: "key=sample-value" + # -- Additional env vars for temporal pod(s). + extraEnv: [] + + ## Examples (when using `temporal.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: tmpdir + ## mountPath: /tmp + ## - name: config + ## mountPath: /etc/temporal/config + # -- Additional volumeMounts for temporal containers + extraVolumeMounts: [] + + ## Examples (when using `temporal.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: tmpdir + ## emptyDir: {} + ## - name: config + ## emptyDir: {} + # -- Additional volumes for temporal pods + extraVolumes: [] + + ## Temporal resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## requests: + ## memory: 256Mi + ## cpu: 250m + # -- The requested resources for temporal pods (observed: 30m CPU, 146Mi mem) + requests: + cpu: 100m + memory: 256Mi + # -- The resources limits for temporal pods + limits: + cpu: 500m + memory: 512Mi + + extraContainers: [] + + env_vars: {} + +## @section Temporal parameters +## TODO: Move to consuming temporal from a dedicated helm chart + +temporalUi: + enabled: false + # -- The number of temporal-ui replicas to deploy + replicaCount: 1 + + serviceAccountName: + + image: + # -- The temporal-ui image repository to use + repository: temporalio/ui + # -- The pull policy for the temporal-ui image + pullPolicy: IfNotPresent + tag: "2.30.1" + + service: + # -- The Kubernetes Service Type + type: ClusterIP + # -- The temporal-ui port and exposed kubernetes port + port: 8088 + + # -- Add extra annotations to the temporal-ui pod + podAnnotations: {} + + # -- Add extra labels to the temporal-ui pod + podLabels: {} + + # -- Security context for the container + podSecurityContext: + # gid=1000(temporal-ui) + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: false + # uid=1000(temporal-ui) + runAsUser: 0 + # gid=1000(temporal-ui) + runAsGroup: 0 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + ## Examples (when using `temporal.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraInitContainers: + ## - name: config-loader + ## image: temporalio/ui:2.30.1 + ## command: + ## - /bin/sh + ## - -c + ## - >- + ## find /etc/temporal/config/ -maxdepth 1 -mindepth 1 -exec cp -ar {} /config/ \; + ## volumeMounts: + ## - name: config + ## mountPath: /config + # -- Additional InitContainers to initialize the pod + extraInitContainers: [] + + livenessProbe: + # -- Enable livenessProbe on the temporal-ui + enabled: false + + readinessProbe: + # -- Enable readinessProbe on the temporal-ui + enabled: false + + # -- Node labels for temporal-ui pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for temporal-ui pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for temporal-ui pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + ## Example: + ## + ## extraEnv: + ## value: "key=sample-value" + # -- Additional env vars for temporal-ui pod(s). + extraEnv: [] + + ## Examples (when using `temporal.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: tmpdir + ## mountPath: /tmp + ## - name: config + ## mountPath: /etc/temporal/config + # -- Additional volumeMounts for temporal containers + extraVolumeMounts: [] + + ## Examples (when using `temporal.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: tmpdir + ## emptyDir: {} + ## - name: config + ## emptyDir: {} + # -- Additional volumes for temporal-ui pods + extraVolumes: [] + + ## Temporal UI resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## requests: + ## memory: 256Mi + ## cpu: 250m + # -- The requested resources for temporal-ui pods + requests: {} + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- The resources limits for temporal-ui pods + limits: {} + + extraContainers: [] + + env_vars: {} + + debug: + enabled: false + remoteDebugPort: 5005 + + + +## @section Airbyte Database parameters + +# PostgreSQL chart configuration, see https://github.com/bitnami/charts/blob/master/bitnami/postgresql/values.yaml +postgresql: + # -- Switch to enable or disable the PostgreSQL helm chart + enabled: false + + ## image.repository Repository for airbyte-db statefulset + image: + repository: airbyte/db + + # -- Airbyte Postgresql username + postgresqlUsername: airbyte + # -- Airbyte Postgresql password + postgresqlPassword: airbyte + # -- Airbyte Postgresql database + postgresqlDatabase: db-airbyte + # fullnameOverride: *db-hostname + ## This secret is used in case of postgresql.enabled=true and we would like to specify password for newly created postgresql instance + # -- Name of an existing secret containing the PostgreSQL password ('postgresql-password' key) + existingSecret: "" + podSecurityContext: + # gid=70(postgres) + fsGroup: 70 + containerSecurityContext: + # -- Ensures the container will run with a non-root user + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=70(postgres) + runAsUser: 70 + # gid=70(postgres) + runAsGroup: 70 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + commonAnnotations: + # -- It will determine when the hook should be rendered + helm.sh/hook: pre-install + # -- The order in which the hooks are executed. If weight is lower, it has higher priority + helm.sh/hook-weight: "-1" + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for postgresql pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for postgresql pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + +minio: + # Deploy MinIO (and PVC) for both s3 and minio storage types when true, same as airbyte-v1 + enabled: true + secretName: "" + + image: + # -- Minio image used by Minio helm chart + repository: minio/minio + # -- Minio tag image + tag: RELEASE.2023-11-20T22-40-07Z + + rootUser: "" + rootPassword: "" + + storage: + volumeClaimValue: 5Gi + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + # -- Tolerations for minio pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + + # -- Affinity and anti-affinity for minio pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + resources: + # observed: 1m CPU, 224Mi mem + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 1Gi +## @section cron parameters +cron: + enabled: true + # -- Number of cron replicas + replicaCount: 1 + + serviceAccountName: + + image: + # -- The repository to use for the airbyte cron image. + repository: airbyte/cron + # -- The pull policy to use for the airbyte cron image + pullPolicy: IfNotPresent + + jobs: + updateDefinitions: + enabled: false + + ## workloadApi: + ## bearerTokenSecretName: workload-api-bearer-token-name kubernetes secret name where bearer token is stored + ## bearerTokenSecretKey: workload-api-bearer-token-key kubernetes secret key where bearer token is stored + workloadApi: {} + + annotations: {} + + # -- Add extra annotations to the cron pods + podAnnotations: {} + + # -- Add extra labels to the cron pods + podLabels: {} + + # -- Security context for the container + podSecurityContext: + # uid=1000(airbyte) + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + livenessProbe: + # -- Enable livenessProbe on the cron + enabled: true + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 30 + # -- Period seconds for livenessProbe + periodSeconds: 10 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 1 + # -- Failure threshold for livenessProbe + failureThreshold: 3 + # -- Success threshold for livenessProbe + successThreshold: 1 + + readinessProbe: + # -- Enable readinessProbe on the cron + enabled: true + # -- Initial delay seconds for readinessProbe + initialDelaySeconds: 10 + # -- Period seconds for readinessProbe + periodSeconds: 10 + # -- Timeout seconds for readinessProbe + timeoutSeconds: 1 + # -- Failure threshold for readinessProbe + failureThreshold: 3 + # -- Success threshold for readinessProbe + successThreshold: 1 + + ## cron resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- The resources limits for the cron container (observed: 2m CPU, 583Mi mem) + limits: + cpu: 700m + memory: 1Gi + # -- The requested resources for the cron container + requests: + cpu: 100m + memory: 512Mi + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for cron pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for cron pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + log: + # -- The log level to log at. + level: INFO + + ## Examples (when using `cron.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: tmpdir + ## mountPath: /tmp + # -- Additional volumeMounts for cron containers + extraVolumeMounts: [] + + ## Examples (when using `cron.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: tmpdir + ## emptyDir: {} + # -- Additional volumes for cron pods + extraVolumes: [] + + ## Example: + # extraContainers: + # - name: otel_collector + # image: somerepo/someimage:sometag + # args: [ + # "--important-args" + # ] + # ports: + # - containerPort: 443 + # volumeMounts: + # - name: volumeMountCool + # mountPath: /some/path + # readOnly: true + # -- Additional container for cron pods + extraContainers: [] + + ## Example: + # extraInitContainers: + # - name: sleepy + # image: alpine + # command: ['sleep', '60'] + # -- Additional init containers for cron pods + extraInitContainers: [] + + ## Example: (With default env vars and values taken from generated config map) + # extraEnv: + # - name: AIRBYTE_VERSION + # valueFrom: + # configMapKeyRef: + # name: airbyte-env + # key: AIRBYTE_VERSION + # - name: API_URL + # valueFrom: + # configMapKeyRef: + # name: airbyte-env + # key: API_URL + # - name: TRACKING_STRATEGY + # valueFrom: + # configMapKeyRef: + # name: airbyte-env + # key: TRACKING_STRATEGY + # - name: FULLSTORY + # valueFrom: + # configMapKeyRef: + # name: airbyte-env + # key: FULLSTORY + # - name: INTERNAL_API_HOST + # valueFrom: + # configMapKeyRef: + # name: airbyte-env + # key: INTERNAL_API_HOST + # -- Supply extra env variables to main container using full notation + extraEnv: [] + ## Example: + ## secrets: + ## DATABASE_PASSWORD: strong-password + ## DATABASE_USER: my-db-user + # -- Supply additional secrets to container + secrets: {} + ## Example: + ## env_vars: + ## DATABASE_HOST: airbyte-db + ## DATABASE_PORT: 5432 + # -- Supply extra env variables to main container using simplified notation + env_vars: {} + +connectorBuilderServer: + enabled: true + # -- Number of connector-builder-server replicas + replicaCount: 1 + + # -- Base URL for ManifestServerApiClient (airbyte.manifest-server-api.base-path). Defaults to connector-builder-server service URL. + manifestServerApi: + basePath: "" + + serviceAccountName: + + image: + # -- The repository to use for the airbyte connector-builder-server image. + repository: airbyte/connector-builder-server + # -- The pull policy to use for the airbyte connector-builder-server image + pullPolicy: IfNotPresent + + annotations: {} + + # -- Security context for the container + podSecurityContext: + # gid=1000(airbyte) + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + livenessProbe: + # -- Enable livenessProbe on the server + enabled: true + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 30 + # -- Period seconds for livenessProbe + periodSeconds: 10 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 10 + # -- Failure threshold for livenessProbe + failureThreshold: 3 + # -- Success threshold for livenessProbe + successThreshold: 1 + + readinessProbe: + # -- Enable readinessProbe on the server + enabled: true + # -- Initial delay seconds for readinessProbe + initialDelaySeconds: 10 + # -- Period seconds for readinessProbe + periodSeconds: 10 + # -- Timeout seconds for readinessProbe + timeoutSeconds: 10 + # -- Failure threshold for readinessProbe + failureThreshold: 3 + # -- Success threshold for readinessProbe + successThreshold: 1 + + ## connector-builder-server resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- The resources limits for the connector-builder-server container (observed: 2m CPU, 439Mi mem) + limits: + cpu: 500m + memory: 1Gi + # -- The requested resources for the connector-builder-server container + requests: + cpu: 100m + memory: 512Mi + + log: + # -- The log level to log at. + level: INFO + + env_vars: {} + + service: + type: NodePort + port: 80 + annotations: {} + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for webapp pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for webapp pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + extraInitContainers: [] + extraContainers: [] + + debug: + enabled: false + remoteDebugPort: 5005 + + aiAssistUrlBase: "" + +keycloak: + enabled: false + env_vars: {} + + secretName: "" + + url: "" + internalHost: + internalProtocol: http + internalBasePath: /auth + + clientRealm: "" + realmIssuer: "" + + database: + name: "" + host: "" + + user: "" + userSecretKey: "" + + password: "" + passwordSecretKey: "" + + auth: + adminUsername: airbyteAdmin + adminUsernameSecretKey: "" + + adminPassword: airbyte123 + adminPasswordSecretKey: "" + + adminRealm: "" + adminRealmSecretKey: "" + + adminCliClientId: "" + + serviceAccountName: + + image: + repository: airbyte/keycloak + pullPolicy: IfNotPresent + + service: + type: ClusterIP + port: 8180 + annotations: {} + + headlessService: + annotations: {} + + # -- Security context for the container + podSecurityContext: + # gid=0(root) + fsGroup: 1000 + + initContainers: + initDb: + image: "postgres:13-alpine" + + initContainerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(keycloak) + runAsUser: 1000 + # gid=0(root) + runAsGroup: 0 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for webapp pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for webapp pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + ## Configure extra options for the keycloak containers' liveness and readiness probes + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes + ## livenessProbe.enabled Enable livenessProbe on keycloak + ## livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe + ## livenessProbe.periodSeconds Period seconds for livenessProbe + ## livenessProbe.timeoutSeconds Timeout seconds for livenessProbe + ## livenessProbe.failureThreshold Failure threshold for livenessProbe + ## livenessProbe.successThreshold Success threshold for livenessProbe + ## + livenessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + + ## readinessProbe.enabled Enable readinessProbe on keycloak + ## readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe + ## readinessProbe.periodSeconds Period seconds for readinessProbe + ## readinessProbe.timeoutSeconds Timeout seconds for readinessProbe + ## readinessProbe.failureThreshold Failure threshold for readinessProbe + ## readinessProbe.successThreshold Success threshold for readinessProbe + ## + readinessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + successThreshold: 1 + + startupProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 5 + timeoutSeconds: 1 + failureThreshold: 60 + successThreshold: 1 + + extraInitContainers: [] + extraContainers: [] + +keycloakSetup: + enabled: false + env_vars: {} + + serviceAccountName: + + image: + repository: airbyte/keycloak-setup + pullPolicy: IfNotPresent + + annotations: {} + + # -- Security context for the container + podSecurityContext: + # gid=1000(airbyte) + fsGroup: 1000 + + initContainerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for webapp pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for webapp pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + debug: + enabled: false + remoteDebugPort: 5005 + + extraInitContainers: [] + extraContainers: [] + +workloadApiServer: + enabled: true + + bearerToken: token + bearerTokenSecretName: "" + + # -- workload-api-server replicas + replicaCount: 1 + + serviceAccountName: + + image: + # -- The repository to use for the airbyte-workload-api-server image. + repository: airbyte/workload-api-server + # -- The pull policy to use for the airbyte-workload-api-server image + pullPolicy: IfNotPresent + + annotations: {} + + # -- Security context for the container + podSecurityContext: + # gid=1000(airbyte) + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + livenessProbe: + # -- Enable livenessProbe on the server + enabled: true + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 30 + # -- Period seconds for livenessProbe + periodSeconds: 10 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 10 + # -- Failure threshold for livenessProbe + failureThreshold: 3 + # -- Success threshold for livenessProbe + successThreshold: 1 + + readinessProbe: + # -- Enable readinessProbe on the server + enabled: true + # -- Initial delay seconds for readinessProbe + initialDelaySeconds: 10 + # -- Period seconds for readinessProbe + periodSeconds: 10 + # -- Timeout seconds for readinessProbe + timeoutSeconds: 10 + # -- Failure threshold for readinessProbe + failureThreshold: 3 + # -- Success threshold for readinessProbe + successThreshold: 1 + + ## airbyte-workload-api-server resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- The resources limits for the airbyte-workload-api-server container (observed: 9m CPU, 436Mi mem) + limits: + cpu: 2 + memory: 4Gi + # -- The requested resources for the airbyte-workload-api-server container + requests: + cpu: 250m + memory: 512Mi + + log: + # -- The log level at which to log + level: INFO + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for webapp pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for webapp pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + env_vars: {} + + service: + type: ClusterIP + port: 8001 + annotations: {} + + # Configure the ingress resource that allows you to access the Airbyte Workload API, see http://kubernetes.io/docs/user-guide/ingress/ + ingress: + # -- Set to true to enable ingress record generation + enabled: false + # -- Specifies ingressClassName for clusters >= 1.18+ + className: "" + # -- Ingress annotations done as key:value pairs + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + # -- The list of hostnames to be covered with this ingress record + hosts: [] + # - host: chart-example.local + # paths: + # - path: / + # pathType: ImplementationSpecific + # -- Custom ingress TLS configuration + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + + debug: + enabled: false + remoteDebugPort: 5005 + + extraInitContainers: [] + extraContainers: [] + + workloadApi: {} + +featureflagServer: + enabled: false + + # -- workload-api-server replicas + replicaCount: 1 + + serviceAccountName: + + image: + # -- The repository to use for the airbyte-workload-api-server image. + repository: airbyte/featureflag-server + # -- The pull policy to use for the airbyte-workload-api-server image + pullPolicy: IfNotPresent + + annotations: {} + + # -- Security context for the container + podSecurityContext: + # gid=1000(airbyte) + fsGroup: 1000 + + containerSecurityContext: + allowPrivilegeEscalation: false + runAsNonRoot: true + # uid=1000(airbyte) + runAsUser: 1000 + # gid=1000(airbyte) + runAsGroup: 1000 + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + seccompProfile: + type: RuntimeDefault + + livenessProbe: + # -- Enable livenessProbe on the server + enabled: true + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 30 + # -- Period seconds for livenessProbe + periodSeconds: 10 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 10 + # -- Failure threshold for livenessProbe + failureThreshold: 3 + # -- Success threshold for livenessProbe + successThreshold: 1 + + readinessProbe: + # -- Enable readinessProbe on the server + enabled: true + # -- Initial delay seconds for readinessProbe + initialDelaySeconds: 10 + # -- Period seconds for readinessProbe + periodSeconds: 10 + # -- Timeout seconds for readinessProbe + timeoutSeconds: 10 + # -- Failure threshold for readinessProbe + failureThreshold: 3 + # -- Success threshold for readinessProbe + successThreshold: 1 + + ## airbyte-workload-api-server resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + # -- The resources limits for the featureflag-server container + limits: + cpu: 500m + memory: 1Gi + # -- The requested resources for the featureflag-server container + requests: + cpu: 100m + memory: 256Mi + + log: + # -- The log level at which to log + level: INFO + + # -- Node labels for pod assignment, see https://kubernetes.io/docs/user-guide/node-selection/ + nodeSelector: {} + + # -- Tolerations for webapp pod assignment, see https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + # -- Affinity and anti-affinity for webapp pod assignment, see + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + + env_vars: {} + service: + port: 8007 + + # Configure the ingress resource that allows you to access the Airbyte Workload API, see http://kubernetes.io/docs/user-guide/ingress/ + ingress: + # -- Set to true to enable ingress record generation + enabled: false + # -- Specifies ingressClassName for clusters >= 1.18+ + className: "" + # -- Ingress annotations done as key:value pairs + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + # -- The list of hostnames to be covered with this ingress record + hosts: [] + # - host: chart-example.local + # paths: + # - path: / + # pathType: ImplementationSpecific + # -- Custom ingress TLS configuration + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + + extraInitContainers: [] + extraContainers: [] + + debug: + enabled: false + remoteDebugPort: 5005 + +testWebapp: + image: + repository: airbyte/utils diff --git a/helm/backend-sdk/Chart.yaml b/helm/backend-sdk/Chart.yaml index 0cc2e75d..889e2d26 100644 --- a/helm/backend-sdk/Chart.yaml +++ b/helm/backend-sdk/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: v0.26.0-alpha.4 +version: v0.27.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/backend-sdk/templates/hpa.yaml b/helm/backend-sdk/templates/hpa.yaml index dc44c178..7361a4cc 100644 --- a/helm/backend-sdk/templates/hpa.yaml +++ b/helm/backend-sdk/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta2 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ include "backend-sdk.fullname" . }} diff --git a/helm/dashboard-backend/Chart.yaml b/helm/dashboard-backend/Chart.yaml index da57e48f..04297a60 100644 --- a/helm/dashboard-backend/Chart.yaml +++ b/helm/dashboard-backend/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: v0.26.0-alpha.3 +version: v0.27.2 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/dashboard-backend/templates/deployment.yaml b/helm/dashboard-backend/templates/deployment.yaml index 810750e1..5512c91d 100644 --- a/helm/dashboard-backend/templates/deployment.yaml +++ b/helm/dashboard-backend/templates/deployment.yaml @@ -38,8 +38,8 @@ spec: args: - -Xms512m - -Xmx3024m - - -XX:PermSize=64m - - -XX:MaxPermSize=500m + - -XX:MetaspaceSize=64m + - -XX:MaxMetaspaceSize=500m - -Xss1024k - -jar - blotout-0.0.1.jar diff --git a/helm/dashboard-backend/templates/hpa.yaml b/helm/dashboard-backend/templates/hpa.yaml index 36959440..d4bc6c1f 100644 --- a/helm/dashboard-backend/templates/hpa.yaml +++ b/helm/dashboard-backend/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta2 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ include "dashboard-backend.fullname" . }} diff --git a/helm/dashboard-backend/templates/ingress.yaml b/helm/dashboard-backend/templates/ingress.yaml index af8f59ce..6106ae1b 100644 --- a/helm/dashboard-backend/templates/ingress.yaml +++ b/helm/dashboard-backend/templates/ingress.yaml @@ -44,7 +44,7 @@ spec: {{- range .paths }} - path: {{ .path }} {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} + pathType: ImplementationSpecific {{- end }} backend: {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} @@ -58,4 +58,4 @@ spec: {{- end }} {{- end }} {{- end }} -{{- end }} +{{- end }} \ No newline at end of file diff --git a/helm/dashboard-backend/templates/swagger-ui-custom-html.yaml b/helm/dashboard-backend/templates/swagger-ui-custom-html.yaml index 2b8cb985..6db95b95 100644 --- a/helm/dashboard-backend/templates/swagger-ui-custom-html.yaml +++ b/helm/dashboard-backend/templates/swagger-ui-custom-html.yaml @@ -42,18 +42,19 @@ spec: http: paths: - path: /swagger-ui-custom.html(/|$)(.*) - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} pathType: ImplementationSpecific + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} backend: {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} {{- end }} {{- end }} - {{- end }} -{{- end }} \ No newline at end of file + {{- end }} \ No newline at end of file diff --git a/helm/dashboard-backend/templates/swagger-ui.yaml b/helm/dashboard-backend/templates/swagger-ui.yaml index 42e60d48..c70be868 100644 --- a/helm/dashboard-backend/templates/swagger-ui.yaml +++ b/helm/dashboard-backend/templates/swagger-ui.yaml @@ -42,18 +42,19 @@ spec: http: paths: - path: /swagger-ui(/|$)(.*) - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} pathType: ImplementationSpecific + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} backend: {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} {{- end }} {{- end }} - {{- end }} -{{- end }} \ No newline at end of file + {{- end }} \ No newline at end of file diff --git a/helm/dashboard-backend/templates/swagger-v3.yaml b/helm/dashboard-backend/templates/swagger-v3.yaml index ca4d1189..a354ed50 100644 --- a/helm/dashboard-backend/templates/swagger-v3.yaml +++ b/helm/dashboard-backend/templates/swagger-v3.yaml @@ -42,18 +42,19 @@ spec: http: paths: - path: /v3(/|$)(.*) - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} pathType: ImplementationSpecific + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} backend: {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} {{- end }} {{- end }} - {{- end }} -{{- end }} \ No newline at end of file + {{- end }} \ No newline at end of file diff --git a/helm/dashboard-frontend/Chart.yaml b/helm/dashboard-frontend/Chart.yaml index f3e43223..c70c98a3 100644 --- a/helm/dashboard-frontend/Chart.yaml +++ b/helm/dashboard-frontend/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: v0.26.0-alpha.3 +version: v0.27.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm/dashboard-frontend/templates/hpa.yaml b/helm/dashboard-frontend/templates/hpa.yaml index 4aa998c7..2c60d8a0 100644 --- a/helm/dashboard-frontend/templates/hpa.yaml +++ b/helm/dashboard-frontend/templates/hpa.yaml @@ -1,5 +1,5 @@ {{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta2 +apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler metadata: name: {{ include "dashboard-frontend.fullname" . }}