From 559013c17957ca8f17c7d55c315ebadc7fb06514 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Thu, 27 Aug 2026 14:13:42 -0600 Subject: [PATCH 1/4] =?UTF-8?q?copy=20clickhouse=202.8.0=20=E2=86=92=203.0?= =?UTF-8?q?.0=20(verbatim)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5 --- clickhouse/versions/3.0.0/Chart.yaml | 18 + clickhouse/versions/3.0.0/README.md | 263 ++++++++++++ .../versions/3.0.0/templates/_helpers.tpl | 201 +++++++++ clickhouse/versions/3.0.0/templates/gvc.yaml | 11 + .../versions/3.0.0/templates/identity.yaml | 15 + .../versions/3.0.0/templates/policy.yaml | 37 ++ .../3.0.0/templates/secret-azure-config.yaml | 51 +++ .../3.0.0/templates/secret-gcs-config.yaml | 52 +++ .../templates/secret-hetzner-config.yaml | 49 +++ .../3.0.0/templates/secret-keeper-config.yaml | 104 +++++ .../3.0.0/templates/secret-s3-config.yaml | 52 +++ .../3.0.0/templates/secret-server-config.yaml | 389 ++++++++++++++++++ .../versions/3.0.0/templates/volumeset.yaml | 30 ++ .../3.0.0/templates/workload-keeper.yaml | 80 ++++ .../3.0.0/templates/workload-server.yaml | 126 ++++++ clickhouse/versions/3.0.0/values.yaml | 82 ++++ 16 files changed, 1560 insertions(+) create mode 100644 clickhouse/versions/3.0.0/Chart.yaml create mode 100644 clickhouse/versions/3.0.0/README.md create mode 100644 clickhouse/versions/3.0.0/templates/_helpers.tpl create mode 100644 clickhouse/versions/3.0.0/templates/gvc.yaml create mode 100644 clickhouse/versions/3.0.0/templates/identity.yaml create mode 100644 clickhouse/versions/3.0.0/templates/policy.yaml create mode 100644 clickhouse/versions/3.0.0/templates/secret-azure-config.yaml create mode 100644 clickhouse/versions/3.0.0/templates/secret-gcs-config.yaml create mode 100644 clickhouse/versions/3.0.0/templates/secret-hetzner-config.yaml create mode 100644 clickhouse/versions/3.0.0/templates/secret-keeper-config.yaml create mode 100644 clickhouse/versions/3.0.0/templates/secret-s3-config.yaml create mode 100644 clickhouse/versions/3.0.0/templates/secret-server-config.yaml create mode 100644 clickhouse/versions/3.0.0/templates/volumeset.yaml create mode 100644 clickhouse/versions/3.0.0/templates/workload-keeper.yaml create mode 100644 clickhouse/versions/3.0.0/templates/workload-server.yaml create mode 100644 clickhouse/versions/3.0.0/values.yaml diff --git a/clickhouse/versions/3.0.0/Chart.yaml b/clickhouse/versions/3.0.0/Chart.yaml new file mode 100644 index 00000000..fbf65bac --- /dev/null +++ b/clickhouse/versions/3.0.0/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v2 +name: clickhouse +description: Fast, open source column-oriented database management system + +type: application +version: 2.8.0 +appVersion: "25.10" + +annotations: + created: "2025-11-21" + lastModified: "2026-08-24" + category: "database" + createsGvc: true + +dependencies: + - name: cpln-common + version: 1.0.0 + repository: "oci://ghcr.io/controlplane-com/templates" \ No newline at end of file diff --git a/clickhouse/versions/3.0.0/README.md b/clickhouse/versions/3.0.0/README.md new file mode 100644 index 00000000..fd755552 --- /dev/null +++ b/clickhouse/versions/3.0.0/README.md @@ -0,0 +1,263 @@ +# ClickHouse + +> **Object-storage credentials are a prerequisite secret (2.7.0+)** for the `gcp` and `hetzner` providers. +> They reach ClickHouse as `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY` and are read through +> ``, because a `cpln://` reference inside the disk XML is never resolved. +> `aws` was already keyless via cloud identity and `azure` uses an account key, so neither changes. +> An upgrade still carrying `accessKeyId`/`secretAccessKey` is refused at render. + + +This template deploys ClickHouse in either **single-node** or **cluster** mode depending on how locations are configured in `values.yaml`. All deployments use object storage (AWS S3, GCS, Azure Blob Storage, or Hetzner Object Storage) as the primary data store. + +ClickHouse is a high-performance column-oriented analytical database designed for real-time querying and data warehousing at scale. Storage includes: + +- **Primary object storage** — long-term scalable storage (AWS S3, GCS, Azure Blob Storage, or Hetzner Object Storage) +- **Scratch volume** — fast local read cache for performance +- **Volumeset** — persistent metadata, state, and system files + +## Architecture + +- **GVC** — created by this template, named by `gvc.name`. Give every install its own. +- **Stateful ClickHouse Server workload** — the database, with configurable replicas per location. +- **Stateful ClickHouse Keeper workload** *(cluster mode only)* — coordination, one replica per location across the first three. +- **Volume sets** — server metadata and state, plus Keeper state in cluster mode. Primary data lives in object storage; the volume is a read cache. +- **Secrets** — startup scripts for Server and Keeper, and one storage-configuration secret for the selected provider. +- **Identity and policy** — `reveal` on this template's secrets plus the credentials secret you create, and cloud access to the bucket. + +## Deployment Modes + +### Single-Node +Specify exactly **1 location with `replicas: 1`**. No ClickHouse Keeper is deployed. Ideal for development, staging, or lower-traffic workloads where high availability is not required. + +### Single-Shard Cluster +Specify **1 location with `replicas` > 1**. Deploys a single shard with multiple replicas in one location. ClickHouse Keeper is deployed for replication coordination. + +### Multi-Shard Cluster +Specify **3 or more locations**. Deploys a shard per location with configurable replicas. ClickHouse Keeper is deployed across the first 3 locations for quorum. Recommended for production workloads requiring high availability and geographic distribution. + +> **Note:** 2 locations is not supported. Use 1 location (single-node or single-shard) or 3+. + +**Important**: To minimize network egress costs, deploy all locations in the same cloud provider and keep object storage in the same region(s). Using 1 replica per location for ClickHouse server is sufficient for most cluster deployments. + +## Prerequisites + +**One `dictionary` secret must exist BEFORE you install.** This is the password you put in every client connection, so it is not a value — putting it in values would leave it in the Helm release. + +```bash +cpln secret create-dictionary --name my-clickhouse-credentials \ + --entry password='YOUR-STRONG-PASSWORD' \ + --entry database=mydatabase +``` + +Set `database.credentialsSecretName` to the name you used. Secret names are organization-wide, so give each release its own. + +There is no `username` key: ClickHouse authenticates as its built-in `default` user here, so the secret holds only `password` and `database`. + +**If the secret does not exist at install time, the deployment wedges silently.** `cpln logs` returns **zero lines** — the container never starts, so it has nothing to log. Read `status.versions[].message` instead: + +```bash +cpln workload get-deployments RELEASE_NAME-clickhouse-server --gvc GVC_NAME -o yaml +``` + +Note this is `get-deployments` — plain `cpln workload get` has no `versions` field. + +Upgrading from 2.5.x: delete `database.password` and `database.name` from your values and create the secret instead, using the password the database already has — it was applied on first initialisation and a new value in the secret will not change it. An upgrade that still carries either key is refused at render. + +## Configuration + +Before installing, update `values.yaml` with the parameters relevant to your environment: + +- **GVC name**: Assign a name for the Global Virtual Cloud. +- **Locations**: Set 1 location with `replicas: 1` for single-node, or configure 3+ locations for a cluster. +- **Cluster Name**: Assign a cluster name. Used in distributed DDL queries (cluster mode only). +- **Storage**: Choose a provider (`aws`, `gcp`, `azure`, or `hetzner`) and fill in the configuration values under that section. + +**Note on GVC Naming** + - This template creates a GVC automatically with a name defined in `values.yaml`. If deploying multiple independent ClickHouse instances, **you must use a unique GVC name** for each deployment. + +## Setting Up Storage + +Object storage is required for all deployment modes. Choose one of the supported providers below. + +### AWS S3 + + +Upgrading from 2.6.0: this version removes aws::ReadOnlyAccess from the backup identity. +That managed policy granted read access to every bucket in your AWS account and contained no write actions, +so it was never carrying the backup itself — but it was silently supplying any read action your +bucket-scoped policy happened to omit. Update your IAM policy to the full action list in this section before +upgrading; if it already matches, no action is needed. Nothing else changes. + + +For ClickHouse to have access to a S3 bucket, ensure the following prerequisites are completed in your AWS account before installing: + +1. Create your bucket. Update the value `bucket` to include its name and `region` to include its region. + +2. If you do not have a Cloud Account set up, refer to the docs to [Create a Cloud Account](https://docs.controlplane.com/guides/create-cloud-account). Update the value `cloudAccountName`. + +3. Create a new policy with the following JSON (replace `YOUR_BUCKET_NAME`) + +```JSON +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:ListBucket", + "s3:GetObjectVersion", + "s3:DeleteObjectVersion", + "s3:GetBucketLocation", + "s3:AbortMultipartUpload", + "s3:ListBucketMultipartUploads", + "s3:ListMultipartUploadParts" + ], + "Resource": [ + "arn:aws:s3:::YOUR_BUCKET_NAME", + "arn:aws:s3:::YOUR_BUCKET_NAME/*" + ] + } + ] +} +``` + +4. Update `cloudAccountName` in your values file with the name of your Cloud Account. + +5. Set `policyName` to match the policy created in step 3. + +### GCS + +For ClickHouse to have access to a GCS bucket, ensure the following prerequisites are completed in your GCP account before installing: + +**Note**: ClickHouse requires S3-compatible HMAC authentication. You must provide an interoperability HMAC key. A Cloud Account is not required. + +1. Create your bucket. Update the value `bucket` to include its name. + +2. Navigate to Settings > Interoperability and click `Create a key for a service account`. + +3. Click `Create new account` and name your service account. + +4. Under `Permissions`, assign the role `Storage Object Admin` and click `Done`. + +5. You will be provided a new HMAC key. Store it in a `dictionary` secret and set `gcp.credentialsSecretName` to that secret's name: + +```bash +cpln secret create-dictionary --name my-clickhouse-gcs-credentials \ + --entry accessKeyId=YOUR_HMAC_ACCESS_KEY \ + --entry secretAccessKey=YOUR_HMAC_SECRET +``` + +To configure using the CLI: + +```BASH +gcloud config set project YOUR_PROJECT_ID + +# To specify another dual region, replace NAM4 +gcloud storage buckets create gs://YOUR_BUCKET_NAME \ + --location=NAM4 + +gcloud iam service-accounts create clickhouse-storage + +gcloud projects add-iam-policy-binding $(gcloud config get-value project) \ + --member="serviceAccount:clickhouse-storage@$(gcloud config get-value project).iam.gserviceaccount.com" \ + --role="roles/storage.objectAdmin" + +gsutil hmac create clickhouse-storage@$(gcloud config get-value project).iam.gserviceaccount.com +``` + +### Azure Blob Storage + +ClickHouse uses Azure's native Blob Storage SDK. A Cloud Account is not required — authentication uses a storage account access key directly. + +1. In the [Azure Portal](https://portal.azure.com), go to **Storage accounts → Create**. + - Performance: Standard + - Redundancy: LRS + - Leave hierarchical namespace off + +2. Inside the storage account, go to **Containers → + Container** and create a container (e.g. `clickhouse-data`). Set access level to **Private**. + +3. Go to **Security + networking → Access keys** and copy either `key1` or `key2`. + +4. Update `values.yaml`: + - `azure.storageAccount` — the storage account name + - `azure.container` — the container name from step 2 + - `azure.credentialsSecretName` — the name of a `dictionary` secret holding the access key from step 3: + + ```bash + cpln secret create-dictionary --name my-clickhouse-azure-credentials \ + --entry accountKey=YOUR_ACCOUNT_KEY + ``` + + The key is never a value: it is read from this secret into the container's environment and + referenced from the storage config, so it never lands in the Helm release. + +To configure using the CLI: + +```BASH +az storage account create \ + --name YOUR_STORAGE_ACCOUNT \ + --resource-group YOUR_RESOURCE_GROUP \ + --sku Standard_LRS + +az storage container create \ + --name clickhouse-data \ + --account-name YOUR_STORAGE_ACCOUNT + +az storage account keys list \ + --account-name YOUR_STORAGE_ACCOUNT \ + --resource-group YOUR_RESOURCE_GROUP \ + --query "[0].value" -o tsv +``` + +### Hetzner Object Storage + +Hetzner Object Storage is S3-compatible. A Cloud Account is not required — authentication uses an access key pair. + +Available regions: +- `nbg1` — Nuremberg, Germany +- `hel1` — Helsinki, Finland +- `fsn1` — Falkenstein, Germany + +1. In the Hetzner Cloud console, go to **Object Storage** and create a bucket. Note the bucket name and region. + +2. Go to **Security → S3 Credentials** and click **Generate credentials**. Save the **Access Key** and **Secret Key** immediately — the secret will not be shown again. + +3. Update `values.yaml`: + - `hetzner.bucket` — the bucket name + - `hetzner.region` — the region (e.g. `nbg1`) + - `hetzner.credentialsSecretName` — the name of a `dictionary` secret holding the key pair from step 2: + + ```bash + cpln secret create-dictionary --name my-clickhouse-hetzner-credentials \ + --entry accessKeyId=YOUR_ACCESS_KEY \ + --entry secretAccessKey=YOUR_SECRET_KEY + ``` + +## Connecting to ClickHouse + +To connect using the ClickHouse client from within the same GVC: + +```SH +clickhouse-client --host $WORKLOAD_NAME --password $PASSWORD +``` + +### Supported External Services + +- [ClickHouse Documentation](https://clickhouse.com/docs/) +- [Cloud Accounts Documentation](https://docs.controlplane.com/guides/create-cloud-account#overview) +- [ClickHouse with S3](https://clickhouse.com/docs/integrations/s3) +- [ClickHouse with GCS](https://clickhouse.com/docs/integrations/gcs) +- [ClickHouse with Azure Blob Storage](https://clickhouse.com/docs/engines/table-engines/integrations/azureBlobStorage) +- [ClickHouse with S3-compatible storage](https://clickhouse.com/docs/integrations/s3#s3-compatible-storage) + +## Important Notes + +- **This template creates its own GVC.** Never point `gvc.name` at an existing shared GVC — the chart adopts a GVC that already exists, and uninstalling then deletes it along with everything else in it. Give each install a unique name. +- **Object storage is required in every mode**, including single-node. There is no local-only shape. +- **2 locations is not supported.** Use 1 (single-node or single-shard) or 3 or more. +- **The credentials secret has no `username` key.** ClickHouse authenticates as its built-in `default` user, so the secret holds only `password` and `database`. +- **Credentials apply on first initialization only.** Changing the secret afterwards does not change the cluster — rotate inside ClickHouse first, then update the secret. +- **Keep locations and buckets in the same provider and region family.** Cross-region traffic to object storage is billed on every query that misses the local cache. \ No newline at end of file diff --git a/clickhouse/versions/3.0.0/templates/_helpers.tpl b/clickhouse/versions/3.0.0/templates/_helpers.tpl new file mode 100644 index 00000000..9fb3c4dc --- /dev/null +++ b/clickhouse/versions/3.0.0/templates/_helpers.tpl @@ -0,0 +1,201 @@ +{{/* Resource Naming */}} + +{{/* +Clickhouse Keeper Workload Name +*/}} +{{- define "clickhouse.keeper.name" -}} +{{- printf "%s-clickhouse-keeper" .Release.Name }} +{{- end }} + +{{/* +Clickhouse Server Workload Name +*/}} +{{- define "clickhouse.server.name" -}} +{{- printf "%s-clickhouse-server" .Release.Name }} +{{- end }} + +{{/* +Clickhouse Secret Database Config Name +*/}} +{{- define "clickhouse.secretDatabase.name" -}} +{{- printf "%s-clickhouse-db-config" .Release.Name }} +{{- end }} + +{{/* +Clickhouse Secret Keeper Config Name +*/}} +{{- define "clickhouse.secretKeeper.name" -}} +{{- printf "%s-clickhouse-keeper-startup" .Release.Name }} +{{- end }} + +{{/* +Clickhouse Secret Server Config Name +*/}} +{{- define "clickhouse.secretServer.name" -}} +{{- printf "%s-clickhouse-server-startup" .Release.Name }} +{{- end }} + +{{/* +Clickhouse Secret GCS Config Name +*/}} +{{- define "clickhouse.secretGCS.name" -}} +{{- printf "%s-clickhouse-gcs-config" .Release.Name }} +{{- end }} + +{{/* +Clickhouse Secret S3 Config Name +*/}} +{{- define "clickhouse.secretS3.name" -}} +{{- printf "%s-clickhouse-s3-config" .Release.Name }} +{{- end }} + +{{/* +Clickhouse Secret Azure Config Name +*/}} +{{- define "clickhouse.secretAzure.name" -}} +{{- printf "%s-clickhouse-azure-config" .Release.Name }} +{{- end }} + +{{/* +Clickhouse Secret Hetzner Config Name +*/}} +{{- define "clickhouse.secretHetzner.name" -}} +{{- printf "%s-clickhouse-hetzner-config" .Release.Name }} +{{- end }} + +{{/* +Clickhouse Identity Name +*/}} +{{- define "clickhouse.identity.name" -}} +{{- printf "%s-clickhouse-identity" .Release.Name }} +{{- end }} + +{{/* +Clickhouse Policy Name +*/}} +{{- define "clickhouse.policy.name" -}} +{{- printf "%s-clickhouse-policy" .Release.Name }} +{{- end }} + +{{/* +Clickhouse Volume Set Server Name +*/}} +{{- define "clickhouse.volumeServer.name" -}} +{{- printf "%s-clickhouse-server-vs" .Release.Name }} +{{- end }} + +{{/* +Clickhouse Volume Set Keeper Name +*/}} +{{- define "clickhouse.volumeKeeper.name" -}} +{{- printf "%s-clickhouse-keeper-vs" .Release.Name }} +{{- end }} + + +{{/* Validation */}} + +{{/* +Determine if this is a single-node deployment. +Requires exactly 1 location with exactly 1 replica. +1 location with >1 replica is a single-shard cluster and still requires Keeper. +*/}} +{{- define "clickhouse.isSingleNode" -}} +{{- if and (eq (len .Values.gvc.locations) 1) (eq ((index .Values.gvc.locations 0).replicas | int) 1) -}} +true +{{- end -}} +{{- end -}} + +{{- define "clickhouse.validateStorage" -}} +{{- $provider := .Values.provider -}} +{{- if not (or (eq $provider "aws") (eq $provider "gcp") (eq $provider "azure") (eq $provider "hetzner")) -}} + {{- fail "provider must be set to 'aws', 'gcp', 'azure', or 'hetzner'." -}} +{{- end -}} +{{- if eq $provider "aws" -}} + {{- if not .Values.aws.bucket -}} + {{- fail "All fields are required for AWS. Missing: aws.bucket" -}} + {{- end -}} + {{- if not .Values.aws.region -}} + {{- fail "All fields are required for AWS. Missing: aws.region" -}} + {{- end -}} + {{- if not .Values.aws.cloudAccountName -}} + {{- fail "All fields are required for AWS. Missing: aws.cloudAccountName" -}} + {{- end -}} + {{- if not .Values.aws.policyName -}} + {{- fail "All fields are required for AWS. Missing: aws.policyName" -}} + {{- end -}} +{{- end -}} +{{- if eq $provider "gcp" -}} + {{- if not .Values.gcp.bucket -}} + {{- fail "All fields are required for GCP. Missing: gcp.bucket" -}} + {{- end -}} + {{- end -}} +{{- if eq $provider "azure" -}} + {{- if not .Values.azure.storageAccount -}} + {{- fail "All fields are required for Azure. Missing: azure.storageAccount" -}} + {{- end -}} + {{- if not .Values.azure.container -}} + {{- fail "All fields are required for Azure. Missing: azure.container" -}} + {{- end -}} + {{- if not .Values.azure.credentialsSecretName -}} + {{- fail "All fields are required for Azure. Missing: azure.credentialsSecretName" -}} + {{- end -}} +{{- end -}} +{{- if eq $provider "hetzner" -}} + {{- if not .Values.hetzner.bucket -}} + {{- fail "All fields are required for Hetzner. Missing: hetzner.bucket" -}} + {{- end -}} + {{- if not .Values.hetzner.region -}} + {{- fail "All fields are required for Hetzner. Missing: hetzner.region" -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{- define "clickhouse.validateLocations" -}} +{{- $count := len .Values.gvc.locations -}} +{{- if eq $count 2 -}} + {{- fail "2 locations is not supported. Use 1 location for single-node mode or 3+ locations for cluster mode." -}} +{{- end -}} +{{- end -}} + + +{{/* Labeling */}} + +{{/* +Common labels - delegated to cpln-common +*/}} +{{- define "clickhouse.tags" -}} +{{- include "cpln-common.tags" . }} +{{- end }} +{{/* Validation */}} +{{- define "clickhouse.validate" -}} +{{- include "clickhouse.validateStorageCreds" . -}} +{{- if hasKey .Values.azure "accountKey" -}} +{{- fail "clickhouse: azure.accountKey was REMOVED — it is now a `dictionary` secret you create, named by azure.credentialsSecretName, holding the key `accountKey`. Delete it from your values. See Prerequisites in the README." -}} +{{- end -}} +{{- if or (hasKey .Values.database "password") (hasKey .Values.database "name") -}} +{{- fail "clickhouse: database.password and database.name were REMOVED — they are now a `dictionary` secret you create, named by database.credentialsSecretName, holding the keys `password` and `database`. Delete them from your values. See Prerequisites in the README." -}} +{{- end -}} +{{- if not .Values.database.credentialsSecretName -}} +{{- fail "clickhouse: database.credentialsSecretName is required — it names the `dictionary` secret holding `password` and `database`. Create it BEFORE installing; see Prerequisites in the README." -}} +{{- end -}} +{{- end -}} + +{{/* +Object-storage credentials for gcp and hetzner moved to prerequisite secrets. They +are supplied as AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY and read via +, because a cpln:// reference inside the disk XML is +never resolved by the platform. +*/}} +{{- define "clickhouse.validateStorageCreds" -}} +{{- range $p := list "gcp" "hetzner" -}} +{{- if eq $.Values.provider $p -}} +{{- $c := index $.Values $p -}} +{{- if or $c.accessKeyId $c.secretAccessKey -}} +{{- fail (printf "clickhouse: %s.accessKeyId and %s.secretAccessKey were REMOVED — they are now a `dictionary` secret you create, named by %s.credentialsSecretName, holding those two keys. Delete them from your values." $p $p $p) -}} +{{- end -}} +{{- if not $c.credentialsSecretName -}} +{{- fail (printf "clickhouse: %s.credentialsSecretName is required — it names the `dictionary` secret holding accessKeyId and secretAccessKey. Create it BEFORE installing." $p) -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/clickhouse/versions/3.0.0/templates/gvc.yaml b/clickhouse/versions/3.0.0/templates/gvc.yaml new file mode 100644 index 00000000..2c2c9579 --- /dev/null +++ b/clickhouse/versions/3.0.0/templates/gvc.yaml @@ -0,0 +1,11 @@ +kind: gvc +name: {{ .Values.gvc.name }} +description: {{ .Values.gvc.name }} +tags: {{- include "clickhouse.tags" . | nindent 4 }} +spec: + endpointNamingFormat: org + staticPlacement: + locationLinks: + {{- range .Values.gvc.locations }} + - //location/{{ .name }} + {{- end }} diff --git a/clickhouse/versions/3.0.0/templates/identity.yaml b/clickhouse/versions/3.0.0/templates/identity.yaml new file mode 100644 index 00000000..0ecac0c3 --- /dev/null +++ b/clickhouse/versions/3.0.0/templates/identity.yaml @@ -0,0 +1,15 @@ +{{- include "clickhouse.validate" . -}} +{{ include "clickhouse.validateStorage" . }} +--- +kind: identity +gvc: {{ .Values.gvc.name }} +name: {{ include "clickhouse.identity.name" . }} +description: Clickhouse identity +tags: {{- include "clickhouse.tags" . | nindent 4 }} +{{- if eq .Values.provider "aws" }} +aws: + cloudAccountLink: //cloudaccount/{{ .Values.aws.cloudAccountName }} + policyRefs: + - cpln-connector + - "{{ .Values.aws.policyName }}" +{{- end }} diff --git a/clickhouse/versions/3.0.0/templates/policy.yaml b/clickhouse/versions/3.0.0/templates/policy.yaml new file mode 100644 index 00000000..90f91019 --- /dev/null +++ b/clickhouse/versions/3.0.0/templates/policy.yaml @@ -0,0 +1,37 @@ +{{ include "clickhouse.validateStorage" . }} +--- +kind: policy +name: {{ include "clickhouse.policy.name" . }} +description: Clickhouse policy +tags: {{- include "clickhouse.tags" . | nindent 4 }} +bindings: + - permissions: + - reveal + principalLinks: + - //gvc/{{ .Values.gvc.name }}/identity/{{ include "clickhouse.identity.name" . }} +targetKind: secret +targetLinks: + - //secret/{{ .Values.database.credentialsSecretName }} + - //secret/{{ include "clickhouse.secretKeeper.name" . }} + - //secret/{{ include "clickhouse.secretServer.name" . }} + {{- if eq .Values.provider "gcp" }} + - //secret/{{ .Values.gcp.credentialsSecretName }} + {{- end }} + {{- if eq .Values.provider "hetzner" }} + - //secret/{{ .Values.hetzner.credentialsSecretName }} + {{- end }} + {{- if eq .Values.provider "azure" }} + - //secret/{{ .Values.azure.credentialsSecretName }} + {{- end }} + {{- if eq .Values.provider "aws" }} + - //secret/{{ include "clickhouse.secretS3.name" . }} + {{- end }} + {{- if eq .Values.provider "gcp" }} + - //secret/{{ include "clickhouse.secretGCS.name" . }} + {{- end }} + {{- if eq .Values.provider "azure" }} + - //secret/{{ include "clickhouse.secretAzure.name" . }} + {{- end }} + {{- if eq .Values.provider "hetzner" }} + - //secret/{{ include "clickhouse.secretHetzner.name" . }} + {{- end }} diff --git a/clickhouse/versions/3.0.0/templates/secret-azure-config.yaml b/clickhouse/versions/3.0.0/templates/secret-azure-config.yaml new file mode 100644 index 00000000..7fd0b587 --- /dev/null +++ b/clickhouse/versions/3.0.0/templates/secret-azure-config.yaml @@ -0,0 +1,51 @@ +{{ include "clickhouse.validateStorage" . }} +--- +{{- if eq .Values.provider "azure" }} +kind: secret +name: {{ include "clickhouse.secretAzure.name" . }} +description: Clickhouse Azure Blob Storage config +tags: {{- include "clickhouse.tags" . | nindent 4 }} +type: opaque +data: + encoding: plain + payload: |- + + + + + azure_blob_storage + https://{{ .Values.azure.storageAccount }}.blob.core.windows.net + {{ .Values.azure.container }} + {{ .Values.azure.storageAccount }} + + /var/lib/clickhouse/disks/azure_disk_metadata/ + + + + cache + azure_disk + azure/ + 1Gi + + + + + + +
+ azure_cache_disk +
+ + + + + + + /var/lib/clickhouse_cache/ + + + + azure_main + + +{{- end }} diff --git a/clickhouse/versions/3.0.0/templates/secret-gcs-config.yaml b/clickhouse/versions/3.0.0/templates/secret-gcs-config.yaml new file mode 100644 index 00000000..e668aedb --- /dev/null +++ b/clickhouse/versions/3.0.0/templates/secret-gcs-config.yaml @@ -0,0 +1,52 @@ +{{ include "clickhouse.validateStorage" . }} +--- +{{- if eq .Values.provider "gcp" }} +kind: secret +name: {{ include "clickhouse.secretGCS.name" . }} +description: Clickhouse GCS config +tags: {{- include "clickhouse.tags" . | nindent 4 }} +type: opaque +data: + encoding: plain + payload: |- + + + + + s3 + https://storage.googleapis.com/{{ .Values.gcp.bucket }}/ + 1 + + {{ .Values.gcp.bucket }} + + /var/lib/clickhouse/disks/gcs_disk_metadata/ + + + + cache + gcs_disk + gcs/ + 1Gi + + + + + + +
+ gcs_cache_disk +
+ + + + + + + /var/lib/clickhouse_cache/ + + + + gcs_main + + +{{- end }} \ No newline at end of file diff --git a/clickhouse/versions/3.0.0/templates/secret-hetzner-config.yaml b/clickhouse/versions/3.0.0/templates/secret-hetzner-config.yaml new file mode 100644 index 00000000..c1504190 --- /dev/null +++ b/clickhouse/versions/3.0.0/templates/secret-hetzner-config.yaml @@ -0,0 +1,49 @@ +{{ include "clickhouse.validateStorage" . }} +--- +{{- if eq .Values.provider "hetzner" }} +kind: secret +name: {{ include "clickhouse.secretHetzner.name" . }} +description: Clickhouse Hetzner Object Storage config +tags: {{- include "clickhouse.tags" . | nindent 4 }} +type: opaque +data: + encoding: plain + payload: |- + + + + + s3 + https://{{ .Values.hetzner.region }}.your-objectstorage.com/{{ .Values.hetzner.bucket }}/data/ + 1 + /var/lib/clickhouse/disks/hetzner_disk_metadata/ + + + + cache + hetzner_disk + hetzner/ + 1Gi + + + + + + +
+ hetzner_cache_disk +
+ + + + + + + /var/lib/clickhouse_cache/ + + + + hetzner_main + + +{{- end }} diff --git a/clickhouse/versions/3.0.0/templates/secret-keeper-config.yaml b/clickhouse/versions/3.0.0/templates/secret-keeper-config.yaml new file mode 100644 index 00000000..203711ec --- /dev/null +++ b/clickhouse/versions/3.0.0/templates/secret-keeper-config.yaml @@ -0,0 +1,104 @@ +{{- if not (include "clickhouse.isSingleNode" .) }} +--- +kind: secret +name: {{ include "clickhouse.secretKeeper.name" . }} +description: Clickhouse keeper startup script +tags: {{- include "clickhouse.tags" . | nindent 4 }} +type: opaque +data: + encoding: plain + payload: |- + #!/usr/bin/env bash + set -euo pipefail + + # --- Replica info --- + LOCATION=$(basename "${CPLN_LOCATION:-}") + HOSTNAME=${HOSTNAME:-} + GVC="{{ .Values.gvc.name }}" + WORKLOAD_NAME="{{ include "clickhouse.keeper.name" . }}" + REPLICA_INDEX=$(echo "${HOSTNAME}" | awk -F'-' '{print $NF}') + SELF_FQDN="replica-${REPLICA_INDEX}.${WORKLOAD_NAME}.${LOCATION}.${GVC}.cpln.local" + + # --- Directories --- + CONFIG_DIR="/etc/clickhouse-keeper" + CONFIG_FILE="$CONFIG_DIR/config.xml" + LOG_DIR="/var/log/clickhouse-keeper" + DATA_DIR="/var/lib/clickhouse/coordination" + + mkdir -p "$CONFIG_DIR" "$LOG_DIR" "$DATA_DIR/logs" "$DATA_DIR/snapshots" + + # --- Prepare locations from Helm values --- + IFS=',' read -r -a LOCATIONS <<< "${LOCATIONS_STR:-}" + + # --- Compute CURRENT_SERVER_ID --- + CURRENT_SERVER_ID=0 + SERVER_ID=1 + for LOC_PAIR in "${LOCATIONS[@]:0:3}"; do # only first 3 locations + LOC="${LOC_PAIR%%:*}" + if [[ "$LOC" == "$LOCATION" && "$REPLICA_INDEX" == "0" ]]; then + CURRENT_SERVER_ID=$SERVER_ID + fi + SERVER_ID=$((SERVER_ID+1)) + done + + # --- Start config.xml --- + cat > "$CONFIG_FILE" < + + information + 1000M + 3 + + + 0.0.0.0 + + 9181 + $CURRENT_SERVER_ID + $DATA_DIR/log + $DATA_DIR/snapshots + + + 20000 + 100000 + information + 20000 + 100000 + + + + EOF + + # --- Generate only 3 keeper replicas (replica-0 for first 3 locations) --- + SERVER_ID=1 + for LOC_PAIR in "${LOCATIONS[@]:0:3}"; do + LOC="${LOC_PAIR%%:*}" + FQDN="replica-0.${WORKLOAD_NAME}.${LOC}.${GVC}.cpln.local" + + cat >> "$CONFIG_FILE" < + $SERVER_ID + $FQDN + 9234 + + EOL + + SERVER_ID=$((SERVER_ID+1)) + done + + # --- Write myid file --- + MYID_FILE="$DATA_DIR/myid" + echo "$CURRENT_SERVER_ID" > "$MYID_FILE" + echo "Created myid file with ID $CURRENT_SERVER_ID at $MYID_FILE" + + # --- Close XML --- + cat >> "$CONFIG_FILE" < + + + EOF + + echo "Generated Keeper config at $CONFIG_FILE:" + cat "$CONFIG_FILE" + + exec clickhouse-keeper --config-file="$CONFIG_FILE" +{{- end }} diff --git a/clickhouse/versions/3.0.0/templates/secret-s3-config.yaml b/clickhouse/versions/3.0.0/templates/secret-s3-config.yaml new file mode 100644 index 00000000..fcddfd62 --- /dev/null +++ b/clickhouse/versions/3.0.0/templates/secret-s3-config.yaml @@ -0,0 +1,52 @@ +{{ include "clickhouse.validateStorage" . }} +--- +{{- if eq .Values.provider "aws" }} +kind: secret +name: {{ include "clickhouse.secretS3.name" . }} +description: Clickhouse s3 config +tags: {{- include "clickhouse.tags" . | nindent 4 }} +type: opaque +data: + encoding: plain + payload: |- + + + + + s3 + https://{{ .Values.aws.bucket }}.s3.{{ .Values.aws.region }}.amazonaws.com/data/ + {{ .Values.aws.bucket }} + {{ .Values.aws.region }} + 1 + 1 + /var/lib/clickhouse/disks/s3_disk_metadata/ + + + + cache + s3_disk + s3/ + 1Gi + + + + + + +
+ s3_cache_disk +
+ + + + + + + /var/lib/clickhouse_cache/ + + + + s3_main + + +{{- end }} \ No newline at end of file diff --git a/clickhouse/versions/3.0.0/templates/secret-server-config.yaml b/clickhouse/versions/3.0.0/templates/secret-server-config.yaml new file mode 100644 index 00000000..45397aab --- /dev/null +++ b/clickhouse/versions/3.0.0/templates/secret-server-config.yaml @@ -0,0 +1,389 @@ +{{- if include "clickhouse.isSingleNode" . }} +--- +kind: secret +name: {{ include "clickhouse.secretServer.name" . }} +description: Clickhouse server startup script (single-node) +tags: {{- include "clickhouse.tags" . | nindent 4 }} +type: opaque +data: + encoding: plain + payload: |- + #!/usr/bin/env bash + set -euo pipefail + + # --- Directories --- + CONFIG_DIR="/etc/clickhouse-server" + LOG_DIR="/var/log/clickhouse-server" + DATA_DIR="/var/lib/clickhouse" + + mkdir -p "$CONFIG_DIR" "$LOG_DIR" "$DATA_DIR" + + # --- Generate config.xml --- + cat > "$CONFIG_DIR/config.xml" < + + + /etc/clickhouse-server/users.xml + + + information + + + + /var/lib/clickhouse-tmp + + + 1073741824 + 536870912 + + + 21474836480 + + + 9000 + 8123 + + + 0.0.0.0 + :: + + + clickhouse-standalone + + + EOF + + echo "Generated ClickHouse config at $CONFIG_DIR/config.xml" + + # --- Generate users.xml dynamically --- + cat > "$CONFIG_DIR/users.xml" < + + + ${CLICKHOUSE_PASSWORD} + default + default + + + + + + 10000000000 + 0 + in_order + 1 + + + + EOF + + echo "Generated ClickHouse users.xml at $CONFIG_DIR/users.xml" + +{{- if eq .Values.provider "aws" }} + mkdir -p /var/lib/clickhouse/disks/s3_disk_metadata + mkdir -p /var/lib/clickhouse_cache/s3 + echo "1" > /var/lib/clickhouse/disks/s3_disk_metadata/format_version.txt + echo "Created paths for S3" +{{- end }} +{{- if eq .Values.provider "azure" }} + mkdir -p /var/lib/clickhouse/disks/azure_disk_metadata + mkdir -p /var/lib/clickhouse_cache/azure + echo "1" > /var/lib/clickhouse/disks/azure_disk_metadata/format_version.txt + echo "Created paths for Azure" +{{- end }} +{{- if eq .Values.provider "hetzner" }} + mkdir -p /var/lib/clickhouse/disks/hetzner_disk_metadata + mkdir -p /var/lib/clickhouse_cache/hetzner + echo "1" > /var/lib/clickhouse/disks/hetzner_disk_metadata/format_version.txt + echo "Created paths for Hetzner" +{{- end }} + + # --- Start ClickHouse server in the background --- + clickhouse-server --config-file="$CONFIG_DIR/config.xml" & + CH_PID=$! + trap 'kill -TERM $CH_PID; wait $CH_PID' TERM INT + + # Wait for server to accept connections + echo "Waiting for ClickHouse to accept connections..." + until clickhouse-client --user default --password "$CLICKHOUSE_PASSWORD" --query "SELECT 1" &>/dev/null; do + sleep 1 + done + + echo "ClickHouse is ready!" + + # --- Create database --- + echo "Creating database ${CLICKHOUSE_DATABASE}..." + clickhouse-client --user default --password "$CLICKHOUSE_PASSWORD" \ + --query "CREATE DATABASE IF NOT EXISTS ${CLICKHOUSE_DATABASE} ENGINE = Atomic;" + + echo "Database ${CLICKHOUSE_DATABASE} created." + + # Bring ClickHouse server to foreground + wait $CH_PID +{{- else }} +--- +kind: secret +name: {{ include "clickhouse.secretServer.name" . }} +description: Clickhouse server startup script +tags: {{- include "clickhouse.tags" . | nindent 4 }} +type: opaque +data: + encoding: plain + payload: |- + #!/usr/bin/env bash + set -euo pipefail + + # --- Runtime variables --- + LOCATION=$(basename "${CPLN_LOCATION:-}") + HOSTNAME=${HOSTNAME:-} + GVC="{{ .Values.gvc.name }}" + WORKLOAD_NAME="{{ include "clickhouse.server.name" . }}" + KEEPER_WORKLOAD_NAME="{{ include "clickhouse.keeper.name" . }}" + REPLICA_INDEX=$(echo "${HOSTNAME}" | awk -F'-' '{print $NF}') + SELF_FQDN="replica-${REPLICA_INDEX}.${WORKLOAD_NAME}.${LOCATION}.${GVC}.cpln.local" + + echo "Starting ClickHouse server for replica ${SELF_FQDN}" + + # --- Directories --- + CONFIG_DIR="/etc/clickhouse-server" + LOG_DIR="/var/log/clickhouse-server" + DATA_DIR="/var/lib/clickhouse" + + mkdir -p "$CONFIG_DIR" "$LOG_DIR" "$DATA_DIR" + + # --- Parse locations and replicas --- + IFS=',' read -r -a LOCATIONS <<< "${LOCATIONS_STR:-}" + + # --- Compute SHARD_INDEX for this node (per location, not per replica) --- + SHARD_INDEX=1 + for LOC_PAIR in "${LOCATIONS[@]}"; do + LOC="${LOC_PAIR%%:*}" + if [[ "$LOC" == "$LOCATION" ]]; then + break + fi + SHARD_INDEX=$((SHARD_INDEX+1)) + done + + # --- Generate config.xml --- + cat > "$CONFIG_DIR/config.xml" < + + + /etc/clickhouse-server/users.xml + + + information + + + + /var/lib/clickhouse-tmp + + + 1073741824 + 536870912 + + + 21474836480 + + + 9000 + 8123 + 9009 + $SELF_FQDN + + + 0.0.0.0 + :: + + + $WORKLOAD_NAME node $REPLICA_INDEX + + /clickhouse/task_queue/ddl + default + + + + <{{ .Values.clusterName }}> + EOF + + # --- Add shards and replicas --- + for LOC_PAIR in "${LOCATIONS[@]}"; do + LOC="${LOC_PAIR%%:*}" + REPLICAS="${LOC_PAIR##*:}" + { + echo " " + echo " true" + for IDX in $(seq 0 $((REPLICAS-1))); do + FQDN="replica-$IDX.${WORKLOAD_NAME}.${LOC}.${GVC}.cpln.local" + echo " " + echo " $FQDN" + echo " 9000" + echo " default" + echo " ${CLICKHOUSE_PASSWORD}" + echo " ${CLICKHOUSE_DATABASE}" + echo " " + done + echo " " + } >> "$CONFIG_DIR/config.xml" + done + + # --- Close remote_servers --- + { + echo " " + echo " " + } >> "$CONFIG_DIR/config.xml" + + # --- Zookeeper nodes --- + { + echo " " + for LOC in "${LOCATIONS[@]:0:3}"; do # only first 3 locations + LOC_NAME="${LOC%%:*}" + FQDN="replica-0.${KEEPER_WORKLOAD_NAME}.${LOC_NAME}.${GVC}.cpln.local" + echo " " + echo " $FQDN" + echo " 9181" + echo " " + done + echo " " + } >> "$CONFIG_DIR/config.xml" + + # --- Macros --- + { + echo " " + printf " %d\n" "$SHARD_INDEX" + printf " %d\n" "$((REPLICA_INDEX + 1))" + echo " " + } >> "$CONFIG_DIR/config.xml" + + # --- Add system logging for filesystem cache and query visibility --- + cat >> "$CONFIG_DIR/config.xml" < + + + system + filesystem_cache_log
+ Log + 7500 + + + EOF + + # --- Close XML --- + echo "" >> "$CONFIG_DIR/config.xml" + + echo "Generated ClickHouse config at $CONFIG_DIR/config.xml" + + # --- Generate users.xml dynamically --- + cat > "$CONFIG_DIR/users.xml" < + + + ${CLICKHOUSE_PASSWORD} + default + default + + + + + + 10000000000 + 0 + in_order + 1 + + + + EOF + + echo "Generated ClickHouse users.xml at $CONFIG_DIR/users.xml" + +{{- if eq .Values.provider "aws" }} + mkdir -p /var/lib/clickhouse/disks/s3_disk_metadata + mkdir -p /var/lib/clickhouse_cache/s3 + echo "1" > /var/lib/clickhouse/disks/s3_disk_metadata/format_version.txt + echo "Created paths for S3" +{{- end }} +{{- if eq .Values.provider "azure" }} + mkdir -p /var/lib/clickhouse/disks/azure_disk_metadata + mkdir -p /var/lib/clickhouse_cache/azure + echo "1" > /var/lib/clickhouse/disks/azure_disk_metadata/format_version.txt + echo "Created paths for Azure" +{{- end }} +{{- if eq .Values.provider "hetzner" }} + mkdir -p /var/lib/clickhouse/disks/hetzner_disk_metadata + mkdir -p /var/lib/clickhouse_cache/hetzner + echo "1" > /var/lib/clickhouse/disks/hetzner_disk_metadata/format_version.txt + echo "Created paths for Hetzner" +{{- end }} + + # --- Force ClickHouse to resolve local FQDN as localhost --- + LOCAL_FQDN="$SELF_FQDN" + + echo "Applying hostname mapping workaround..." + echo "127.0.0.1 $LOCAL_FQDN" >> /etc/hosts + + echo "Hosts file updated:" + cat /etc/hosts + + # --- Start ClickHouse server in the background --- + clickhouse-server --config-file="$CONFIG_DIR/config.xml" & + CH_PID=$! + trap 'kill -TERM $CH_PID; wait $CH_PID' TERM INT + + # Wait for server to accept connections using default superuser + echo "Waiting for ClickHouse to accept connections..." + until clickhouse-client --user default --password "$CLICKHOUSE_PASSWORD" --query "SELECT 1" &>/dev/null; do + sleep 1 + done + + echo "ClickHouse is ready!" + + # --- Compute if this node is the bootstrap node for cluster-wide DB creation --- + FIRST_LOC="{{ (index .Values.gvc.locations 0).name }}" + FIRST_LOCATION="false" + + if [[ "$LOCATION" == "$FIRST_LOC" && "$REPLICA_INDEX" == "0" ]]; then + FIRST_LOCATION="true" + fi + + # --- Wait for Keeper quorum before creating database --- + if [[ "$FIRST_LOCATION" == "true" ]]; then + echo "Waiting for Keeper quorum before bootstrapping DB..." + until clickhouse-client --query "SELECT * FROM system.zookeeper WHERE path='/'" >/dev/null 2>&1; do + echo "Keeper not ready yet, waiting..." + sleep 2 + done + echo "Keeper quorum detected!" + fi + + # --- Create database using default user (only on first location / first replica) --- + if [[ "$FIRST_LOCATION" == "true" ]]; then + echo "Bootstrapping ClickHouse database across cluster (ON CLUSTER)..." + clickhouse-client --user default --password "$CLICKHOUSE_PASSWORD" \ + --query "CREATE DATABASE IF NOT EXISTS ${CLICKHOUSE_DATABASE} ON CLUSTER {{ .Values.clusterName }} ENGINE = Atomic;" + else + echo "Skipping DB creation on this replica (not the bootstrap node)." + fi + + # Bring ClickHouse server to foreground + wait $CH_PID +{{- end }} diff --git a/clickhouse/versions/3.0.0/templates/volumeset.yaml b/clickhouse/versions/3.0.0/templates/volumeset.yaml new file mode 100644 index 00000000..578987c8 --- /dev/null +++ b/clickhouse/versions/3.0.0/templates/volumeset.yaml @@ -0,0 +1,30 @@ +kind: volumeset +name: {{ include "clickhouse.volumeServer.name" . }} +description: Clickhouse server volumeset +tags: + {{- include "clickhouse.tags" . | nindent 2 }} + workload: {{ include "clickhouse.server.name" . }} +spec: + fileSystemType: ext4 + initialCapacity: {{ .Values.volumeset.server.capacity }} + performanceClass: general-purpose-ssd + snapshots: + createFinalSnapshot: true + retentionDuration: 7d + +{{- if not (include "clickhouse.isSingleNode" .) }} +--- +kind: volumeset +name: {{ include "clickhouse.volumeKeeper.name" . }} +description: Clickhouse keeper volumeset +tags: + {{- include "clickhouse.tags" . | nindent 2 }} + workload: {{ include "clickhouse.keeper.name" . }} +spec: + fileSystemType: ext4 + initialCapacity: {{ .Values.volumeset.keeper.capacity }} + performanceClass: general-purpose-ssd + snapshots: + createFinalSnapshot: true + retentionDuration: 7d +{{- end }} \ No newline at end of file diff --git a/clickhouse/versions/3.0.0/templates/workload-keeper.yaml b/clickhouse/versions/3.0.0/templates/workload-keeper.yaml new file mode 100644 index 00000000..f897dcd5 --- /dev/null +++ b/clickhouse/versions/3.0.0/templates/workload-keeper.yaml @@ -0,0 +1,80 @@ +{{- if not (include "clickhouse.isSingleNode" .) }} +{{ include "clickhouse.validateLocations" . }} +kind: workload +name: {{ include "clickhouse.keeper.name" . }} +description: clickhouse keeper +tags: {{- include "clickhouse.tags" . | nindent 4 }} +spec: + type: stateful + containers: + - name: keeper + cpu: {{ .Values.keeper.resources.cpu | quote }} + image: {{ .Values.keeper.image }} + inheritEnv: false + env: + - name: LOCATIONS_STR + value: "{{- range $i, $loc := .Values.gvc.locations -}}{{ $loc.name }}:{{ $loc.replicas }}{{ if lt $i (sub (len $.Values.gvc.locations) 1) }},{{ end }}{{- end }}" + memory: {{ .Values.keeper.resources.memory | quote }} + command: "/bin/bash" + args: + - "/scripts/keeper-startup.sh" + ports: + - number: 9181 + protocol: tcp + - number: 9234 + protocol: tcp + volumes: + - path: /var/lib/clickhouse + recoveryPolicy: retain + uri: cpln://volumeset/{{ include "clickhouse.volumeKeeper.name" . }} + - path: /scripts/keeper-startup.sh + uri: cpln://secret/{{ include "clickhouse.secretKeeper.name" . }} + identityLink: //gvc/{{ .Values.gvc.name }}/identity/{{ include "clickhouse.identity.name" . }} + defaultOptions: + autoscaling: + keda: {} + maxConcurrency: 0 + maxScale: 1 + metric: disabled + minScale: 1 + scaleToZeroDelay: 300 + target: 100 + capacityAI: false + debug: false + suspend: false + timeoutSeconds: 5 + {{- $locs := .Values.gvc.locations }} + localOptions: + {{- range $i, $loc := $locs }} + - autoscaling: + maxConcurrency: 0 + metric: disabled + minScale: 1 + maxScale: 1 + scaleToZeroDelay: 300 + target: 100 + capacityAI: false + debug: false + location: //location/{{ $loc.name }} + suspend: {{ if ge $i 3 }}true{{ else }}false{{ end }} + timeoutSeconds: 5 + {{- end }} + firewallConfig: + external: + inboundAllowCIDR: [] + inboundBlockedCIDR: [] + outboundAllowCIDR: + - 0.0.0.0/0 + outboundAllowHostname: [] + outboundAllowPort: [] + outboundBlockedCIDR: [] + internal: + inboundAllowType: {{ .Values.keeper.internal_access.type }} + inboundAllowWorkload: [] + loadBalancer: + direct: + enabled: false + ports: [] + replicaDirect: true + supportDynamicTags: false +{{- end }} diff --git a/clickhouse/versions/3.0.0/templates/workload-server.yaml b/clickhouse/versions/3.0.0/templates/workload-server.yaml new file mode 100644 index 00000000..09558b92 --- /dev/null +++ b/clickhouse/versions/3.0.0/templates/workload-server.yaml @@ -0,0 +1,126 @@ +{{ include "clickhouse.validateLocations" . }} +{{ include "clickhouse.validateStorage" . }} +kind: workload +name: {{ include "clickhouse.server.name" . }} +description: clickhouse server +tags: {{- include "clickhouse.tags" . | nindent 4 }} +spec: + type: stateful + containers: + - name: server + cpu: {{ .Values.server.resources.cpu | quote }} + image: {{ .Values.server.image }} + inheritEnv: false + env: + {{- if eq .Values.provider "gcp" }} + - name: AWS_ACCESS_KEY_ID + value: cpln://secret/{{ .Values.gcp.credentialsSecretName }}.accessKeyId + - name: AWS_SECRET_ACCESS_KEY + value: cpln://secret/{{ .Values.gcp.credentialsSecretName }}.secretAccessKey + {{- end }} + {{- if eq .Values.provider "hetzner" }} + - name: AWS_ACCESS_KEY_ID + value: cpln://secret/{{ .Values.hetzner.credentialsSecretName }}.accessKeyId + - name: AWS_SECRET_ACCESS_KEY + value: cpln://secret/{{ .Values.hetzner.credentialsSecretName }}.secretAccessKey + {{- end }} + {{- if eq .Values.provider "azure" }} + - name: AZURE_STORAGE_ACCOUNT_KEY + value: cpln://secret/{{ .Values.azure.credentialsSecretName }}.accountKey + {{- end }} + - name: LOCATIONS_STR + value: "{{- range $i, $loc := .Values.gvc.locations -}}{{ $loc.name }}:{{ $loc.replicas }}{{ if lt $i (sub (len $.Values.gvc.locations) 1) }},{{ end }}{{- end }}" + - name: CLICKHOUSE_PASSWORD + value: cpln://secret/{{ .Values.database.credentialsSecretName }}.password + - name: CLICKHOUSE_DATABASE + value: cpln://secret/{{ .Values.database.credentialsSecretName }}.database + memory: {{ .Values.server.resources.memory | quote }} + command: "/bin/bash" + args: + - "/scripts/server-startup.sh" + ports: + - number: 8123 + protocol: http + - number: 9000 + protocol: tcp + - number: 9009 + protocol: http + volumes: + - path: /var/lib/clickhouse + recoveryPolicy: retain + uri: cpln://volumeset/{{ include "clickhouse.volumeServer.name" . }} + - path: /scripts/server-startup.sh + uri: cpln://secret/{{ include "clickhouse.secretServer.name" . }} + - path: /var/lib/clickhouse_cache + recoveryPolicy: retain + uri: scratch://cache-vol + - path: /var/lib/clickhouse-tmp + recoveryPolicy: retain + uri: scratch://tmp-vol + {{- if eq .Values.provider "aws" }} + - path: /etc/clickhouse-server/config.d/storage.xml + recoveryPolicy: retain + uri: cpln://secret/{{ include "clickhouse.secretS3.name" . }} + {{- end }} + {{- if eq .Values.provider "gcp" }} + - path: /etc/clickhouse-server/config.d/storage.xml + recoveryPolicy: retain + uri: cpln://secret/{{ include "clickhouse.secretGCS.name" . }} + {{- end }} + {{- if eq .Values.provider "azure" }} + - path: /etc/clickhouse-server/config.d/storage.xml + recoveryPolicy: retain + uri: cpln://secret/{{ include "clickhouse.secretAzure.name" . }} + {{- end }} + {{- if eq .Values.provider "hetzner" }} + - path: /etc/clickhouse-server/config.d/storage.xml + recoveryPolicy: retain + uri: cpln://secret/{{ include "clickhouse.secretHetzner.name" . }} + {{- end }} + identityLink: //gvc/{{ .Values.gvc.name }}/identity/{{ include "clickhouse.identity.name" . }} + defaultOptions: + autoscaling: + keda: {} + maxConcurrency: 0 + maxScale: {{ if eq ((index .Values.gvc.locations 0).replicas | int) 0 }}1{{ else }}{{ (index .Values.gvc.locations 0).replicas | int }}{{ end }} + metric: disabled + minScale: 1 + scaleToZeroDelay: 300 + target: 100 + capacityAI: false + debug: false + suspend: false + timeoutSeconds: 5 + localOptions: + {{- range $location := .Values.gvc.locations }} + - autoscaling: + maxConcurrency: 0 + maxScale: {{ if eq ($location.replicas | int) 0 }}1{{ else }}{{ $location.replicas | int }}{{ end }} + metric: disabled + minScale: {{ if eq ($location.replicas | int) 0 }}0{{ else }}{{ $location.replicas | int }}{{ end }} + scaleToZeroDelay: 300 + target: 100 + capacityAI: false + debug: false + location: //location/{{ $location.name }} + suspend: {{ if eq ($location.replicas | int) 0 }}true{{ else }}false{{ end }} + timeoutSeconds: 10 + {{- end }} + firewallConfig: + external: + inboundAllowCIDR: [] + inboundBlockedCIDR: [] + outboundAllowCIDR: + - 0.0.0.0/0 + outboundAllowHostname: [] + outboundAllowPort: [] + outboundBlockedCIDR: [] + internal: + inboundAllowType: {{ .Values.server.internal_access.type }} + inboundAllowWorkload: [] + loadBalancer: + direct: + enabled: false + ports: [] + replicaDirect: true + supportDynamicTags: false \ No newline at end of file diff --git a/clickhouse/versions/3.0.0/values.yaml b/clickhouse/versions/3.0.0/values.yaml new file mode 100644 index 00000000..5a5bd26b --- /dev/null +++ b/clickhouse/versions/3.0.0/values.yaml @@ -0,0 +1,82 @@ +gvc: + name: clickhouse-gvc + # Single-node mode: exactly 1 location with replicas: 1 — no Keeper, uses S3/GCS for storage + # Single-shard cluster: 1 location with replicas > 1 — Keeper required + # Multi-shard cluster: 3 or more locations — Keeper required + # Note: 2 locations is not supported + locations: + - name: aws-us-east-2 + replicas: 1 + - name: aws-us-west-2 + replicas: 1 + - name: aws-us-east-1 + replicas: 1 + +provider: aws # Options: aws, gcp, azure, or hetzner + +aws: # If enabled, all fields below are required - See README for guidance + bucket: clickhouse-s3-bucket # Name of your S3 bucket + region: us-east-1 # Region of your S3 bucket + cloudAccountName: clickhouse-s3-cloudaccount # Name of your Cloud Account + policyName: clickhouse-s3-policy # Name of your pre-created policy to allow access to the S3 bucket + +gcp: # If enabled, all fields below are required - See README for guidance + bucket: clickhouse-gcs-bucket # Name of your GCS bucket + # REQUIRED PREREQUISITE SECRET — a `dictionary` secret holding exactly + # `accessKeyId` and `secretAccessKey` (the GCS interoperability HMAC pair). + # They reach ClickHouse as AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY rather + # than being written into the disk XML, because a cpln:// reference inside a + # mounted config file is never resolved. + credentialsSecretName: my-clickhouse-gcs-credentials + +azure: # If enabled, all fields below are required - See README for guidance + storageAccount: myclickhousestorage # Name of your Azure Storage Account + container: clickhouse-data # Name of your Blob Storage container + # REQUIRED PREREQUISITE SECRET — a `dictionary` secret holding exactly + # `accountKey` for your Azure Storage Account. + credentialsSecretName: my-clickhouse-azure-credentials + +hetzner: # If enabled, all fields below are required - See README for guidance + bucket: clickhouse-hetzner-bucket # Name of your Hetzner Object Storage bucket + region: nbg1 # Region of your bucket. Options: nbg1, hel1, fsn1 + # REQUIRED PREREQUISITE SECRET — a `dictionary` secret holding exactly + # `accessKeyId` and `secretAccessKey` for your Hetzner Object Storage. + credentialsSecretName: my-clickhouse-hetzner-credentials + +clusterName: my_cluster # Used in cluster mode only + +database: # Automatically create a database on initialization using the default user + # REQUIRED PREREQUISITE SECRET — CREATE IT BEFORE YOU INSTALL. + # A `dictionary` secret holding exactly two keys: `password` and `database`. + # ClickHouse has no separate username here — it uses the built-in `default` + # user, so there is no `username` key. If the secret does not exist at install + # time the deployment WEDGES silently; see Prerequisites in the README. + credentialsSecretName: my-clickhouse-credentials + +volumeset: + server: + capacity: 10 # initial capacity in GiB (minimum is 10) + keeper: + capacity: 10 # initial capacity in GiB (minimum is 10) - cluster mode only + +server: + image: clickhouse/clickhouse-server:25.10 + resources: + cpu: 2 + memory: 2Gi + internal_access: + type: same-gvc # options: same-gvc, same-org, workload-list + workloads: # Note: can only be used if type is same-gvc or workload-list + #- //gvc/GVC_NAME/workload/WORKLOAD_NAME + #- //gvc/GVC_NAME/workload/WORKLOAD_NAME + +keeper: # cluster mode only + image: clickhouse/clickhouse-keeper:25.10 + resources: + cpu: 2 + memory: 2Gi + internal_access: + type: same-gvc # options: same-gvc, same-org, workload-list + workloads: # Note: can only be used if type is same-gvc or workload-list + #- //gvc/GVC_NAME/workload/WORKLOAD_NAME + #- //gvc/GVC_NAME/workload/WORKLOAD_NAME From 5093fc6819233c0924eeaa62a4650027bab17555 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Thu, 27 Aug 2026 14:33:38 -0600 Subject: [PATCH 2/4] clickhouse 3.0.0: stop creating a GVC, and make a topology mismatch loud The chart no longer declares `kind: gvc`. It deploys into the GVC you install into; `gvc.name` is gone and `gvc.locations` becomes the top-level `locations`. For clickhouse that list is the TOPOLOGY, not placement: its length selects the mode, a location's position in it is that shard's number and its `` identity, and the first three entries are the Keeper Raft group. With the GVC no longer under the chart's control the list and reality can disagree in both directions, and in 2.8.0 both were silent. Three layers close that, copied from pgedge 2.0.0: - `defaultOptions.minScale/maxScale: 0` on both workloads, with `localOptions` supplying the real per-location counts. An undeclared GVC location gets `desiredScale: None` and starts nothing -- the phantom-shard case is closed by construction, not by detection. - `SHARD_INDEX` sentinel is now 0, and 0 is fatal. 2.8.0 let a location that never matched fall out of the loop at len(locations)+1: a shard number in no `` entry, which joined Keeper, registered ReplicatedMergeTree paths under that phantom shard, and was invisible to every distributed query. The guard fires before config.xml is written and before the server launches. - A boot-time GVC read (`view` on the one install GVC, via a new `policy-gvc.yaml`) closes the other direction, which the platform does not validate at all. Hard-fails a FRESH data directory, warns an initialised one, and skips entirely if the read fails -- a control-plane hiccup must never stop a live cluster. Two clickhouse-specific fixes the audit demanded: - Keeper quorum is decided arithmetically before Keeper starts. Below a majority of the first-3 locations the container exits naming the count instead of forming an election that can never complete. `CURRENT_SERVER_ID == 0` is now fatal too; 2.8.0 wrote Raft server_id 0 and started anyway. - The `Keeper not ready yet, waiting...` loop was infinite. It is now bounded at 300s, skipped entirely once the database exists, and on expiry prints the configured Keeper members and the GVC's real locations before taking the server down with it. Other changes: - `internal_access` -> `internalAccess`, and `workloads` is finally wired into `inboundAllowWorkload`. It was hardcoded to `[]`, so choosing `type: workload-list` silently blocked all internal traffic. - Keeper `localOptions` render only for the first three locations. That removes the chart's only use of `localOptions[].suspend`, which permanently breaks a workload's inbound reachability from other locations. - `replicas: 0` was an undocumented special case that suspended a location while still consuming a shard index and a Raft slot. Refused at render instead. - New render-time validation: duplicate locations, empty list, and `clusterName` as a bare identifier (it is an XML element name AND unquoted `ON CLUSTER` DDL, so `my-cluster` produced DDL that does not parse). - Validators consolidated into one `clickhouse.validate`, invoked from both `identity.yaml` and `workload-server.yaml` -- Helm renders templates in reverse-lexical order, so wiring it only to identity.yaml let a malformed `locations` crash with a Go-template error before the friendly message fired. - Default `locations` is now a single location x 1 replica. A multi-location default is N simultaneous guesses about someone else's GVC, and every wrong guess is a silent no-op. - README gains Migrating from 2.x, a Prerequisites section covering the GVC requirement and its diagnostics, and a Connecting table using fully-qualified service DNS. The old example used the bare workload name, which is not reliable on this platform. Neither image ships curl, so the GVC read uses only the wget options GNU (server, ubuntu) and BusyBox (keeper, alpine) both accept. Verified byte-identical output in both images, with 403 and DNS-failure negative controls. Co-Authored-By: Claude Opus 5 --- briefings/clickhouse.md | 113 ++++-- clickhouse/versions/3.0.0/Chart.yaml | 8 +- clickhouse/versions/3.0.0/README.md | 350 ++++++++++++------ .../versions/3.0.0/templates/_helpers.tpl | 241 +++++++++++- clickhouse/versions/3.0.0/templates/gvc.yaml | 11 - .../versions/3.0.0/templates/identity.yaml | 3 +- .../versions/3.0.0/templates/policy-gvc.yaml | 19 + .../versions/3.0.0/templates/policy.yaml | 4 +- .../3.0.0/templates/secret-azure-config.yaml | 1 - .../3.0.0/templates/secret-gcs-config.yaml | 1 - .../templates/secret-hetzner-config.yaml | 1 - .../3.0.0/templates/secret-keeper-config.yaml | 46 ++- .../3.0.0/templates/secret-s3-config.yaml | 1 - .../3.0.0/templates/secret-server-config.yaml | 183 +++++++-- .../3.0.0/templates/workload-keeper.yaml | 35 +- .../3.0.0/templates/workload-server.yaml | 41 +- clickhouse/versions/3.0.0/values.yaml | 67 ++-- 17 files changed, 849 insertions(+), 276 deletions(-) delete mode 100644 clickhouse/versions/3.0.0/templates/gvc.yaml create mode 100644 clickhouse/versions/3.0.0/templates/policy-gvc.yaml diff --git a/briefings/clickhouse.md b/briefings/clickhouse.md index 01ff51b9..9390e89a 100644 --- a/briefings/clickhouse.md +++ b/briefings/clickhouse.md @@ -1,53 +1,84 @@ # ClickHouse — maintainer briefing -**What it is.** ClickHouse, the column-oriented analytical database, backed by object storage as its primary -data store with a local volume as read cache. **This template creates its own GVC.** +**What it is.** ClickHouse, the column-oriented analytical database (Apache-2.0 — fully free to self-host, +no paid edition), backed by object storage as its primary data store with a local volume for metadata and +read cache. **From 3.0.0 it deploys into an existing GVC and creates none.** -**Common use cases.** Real-time analytics, event and clickstream warehousing, product metrics — read-heavy -aggregate queries over large append-mostly datasets. +**Common use cases.** +- Real-time analytics and product metrics over large append-mostly datasets +- Event / clickstream warehousing where reads dominate writes +- A cheap warehouse tier: table data sits in S3/GCS/Azure/Hetzner, not on expensive block storage +- Multi-region sharding when query volume outgrows one node -## Architecture +## Architecture on cpln -| Resource | Notes | +| Resource | Purpose | |---|---| -| **gvc** | created by the chart, named by `gvc.name` | -| workload `-clickhouse-server` (stateful) | the database; replicas per location | -| workload `-clickhouse-keeper` (stateful) | coordination; **cluster mode only**, one replica per location across the first 3 | -| volumesets | server metadata/state, and keeper state in cluster mode | -| secrets | server + keeper startup scripts, and one storage-config secret per provider | -| identity + policy | `reveal` on this release's secrets plus the user's credentials secret; cloud binding for the bucket | +| workload `-clickhouse-server` (stateful, replicaDirect) | the database; replicas per location | +| workload `-clickhouse-keeper` (stateful, replicaDirect) | Raft coordination; **cluster modes only**, replica-0 of the first 3 locations | +| volumesets (server, keeper) | metadata, `store/`, Keeper state — NOT the table data | +| secrets | server + keeper startup scripts, one storage-config XML per provider | +| identity + `-policy` | `reveal` on this release's secrets and the user's prerequisite secrets; AWS cloud binding for the bucket | +| **`-gvc-policy` (new in 3.0.0)** | `view` on **only** the install GVC, so containers can read their own location list at boot | -**Deployment mode is derived from `gvc.locations`, not a knob:** 1 location with `replicas: 1` is single-node -(no Keeper); 1 location with more replicas is a single shard; 3+ locations is multi-shard. **2 locations is -not supported.** +- **Mode is derived from `locations`, not a knob:** 1 location × 1 replica = single-node (no Keeper); + 1 location × N = one shard, N replicas; 3+ locations = one shard per location. **2 locations is refused.** +- **Three-layer defence against a GVC/values mismatch:** `defaultOptions` scale 0/0 (an undeclared GVC + location runs nothing), a `SHARD_INDEX` sentinel that exits rather than inventing a shard number, and a + boot-time GVC read that hard-fails a fresh node and only warns an initialised one. +- Neither image ships `curl`, so the GVC read uses the `wget` options GNU **and** BusyBox both accept + (`-q -O - -T --header=…`), retried in shell. Verified byte-identical in both images. ## Key knobs (shipped defaults) | Knob | Default | Notes | |---|---|---| -| `gvc.name` | `clickhouse-gvc` | **unique per install** — the chart creates this GVC | -| `provider` | `aws` | `aws`, `gcp`, `azure`, `hetzner` | -| `database.credentialsSecretName` | `my-clickhouse-credentials` | **prerequisite** `dictionary` secret (2.6.0+) | -| `clusterName` | `my_cluster` | distributed DDL, cluster mode only | -| `server.image` | `clickhouse/clickhouse-server:25.10` | | - -## Troubleshooting traps - -- **It creates a GVC, so it can destroy one.** Never point `gvc.name` at an existing shared GVC — a - `createsGvc` chart adopts one that already exists and `helm uninstall` then deletes it, taking every - unrelated workload with it. Uninstall against the GVC you **installed into**, not the one the resources - live in, or the policy hook blocks the cleanup. In-container verification is also unavailable, because the - hook denies `exec` against the created GVC. -- **The credentials secret has no `username`.** ClickHouse authenticates as its built-in `default` user, so - the prerequisite secret holds only `password` and `database` — the one datastore in the catalog with that - shape. An upgrade still carrying `database.password`/`database.name` is refused at render. -- **Object storage is required in every mode**, including single-node — there is no local-only shape. -- **GCS uses S3-compatible HMAC keys, not a Cloud Account**; Azure and Hetzner use account keys directly. - Only AWS is keyless via a Cloud Account. -- **Keep locations in one provider and region family.** Cross-region traffic to object storage is billed and - the README says so; a 3-location cluster spread across providers will be slow and expensive. -- **`gcp` and `hetzner` object-storage credentials are a prerequisite secret from 2.7.0.** They are supplied - as `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY` and read via ``, because a - `cpln://` reference inside the disk XML is never resolved. Proven against a live S3-compatible endpoint - before the design was settled: 11 objects written through the disk with correct env credentials, and none - with wrong ones. `aws` was already keyless; `azure` uses an account key and is unaffected. +| `locations` | `[{name: aws-us-east-1, replicas: 1}]` | **must already exist in your GVC**; sets the mode | +| `provider` | `aws` | `aws` (keyless, Cloud Account) / `gcp` / `azure` / `hetzner` | +| `database.credentialsSecretName` | `my-clickhouse-credentials` | **prerequisite** `dictionary` secret: `password` + `database`, **no `username`** | +| `clusterName` | `my_cluster` | bare identifier only — becomes an XML tag and unquoted `ON CLUSTER` DDL; validated at render | +| `server.image` / `keeper.image` | `clickhouse/clickhouse-{server,keeper}:25.10` | | +| `server.resources` / `keeper.resources` | `cpu: 2`, `memory: 2Gi` | limit only, so bare names | +| `volumeset.{server,keeper}.capacity` | `10` | GiB | +| `server.internalAccess.type` | `same-gvc` | renamed from `internal_access` in 3.0.0; the `workloads` list is now actually applied — in 2.x it was hardcoded to `[]`, so `workload-list` silently blocked everything | + +## Troubleshooting / considerations + +- **Never `helm upgrade` a 2.x release onto 3.0.0.** The upgrade drops `kind: gvc` and Helm prunes what a + chart no longer declares — that deletes the GVC and everything in it, in seconds, while printing + "upgraded successfully". The chart refuses at render if the old `gvc` key is present, but an upgrade run + with **no values at all** slips past the guard. Migration is: existing GVC → new release → reload data → + uninstall the old release **against the GVC you originally installed it into**. +- **Prerequisite secret missing = silent wedge.** `cpln logs` returns **zero lines** (the container never + starts). The only honest diagnostic is `status.versions[].message` from + `cpln workload get-deployments {rel}-clickhouse-server` — note `get-deployments`, not `get`. Self-heals in + ~5.5–10.5 min once the secret exists, or `force-redeployment` to skip the wait. +- **A location in the values that the GVC lacks fails on FRESH nodes only.** An already-initialised node + warns and keeps serving — deliberate, so a control-plane hiccup or a deliberate GVC shrink can never + crash a live cluster. Freshness marker is `/var/lib/clickhouse/metadata` (absent on a new volume, created + on first successful start, survives a clean shutdown — `status` does not, which is why it is not used). +- **If EVERY declared location is absent from the GVC, nothing starts and nothing complains** — no container + boots, so no guard can fire. Symptom: zero replicas, `desiredScale: None`, empty logs. Check + `cpln gvc get NAME -o json` → `spec.staticPlacement.locationLinks` **before** installing. +- **Keeper quorum is checked arithmetically before Keeper starts.** Below a majority of the first-3 + locations the container exits with a named error instead of waiting for an election that can never + complete. On the server the same arithmetic runs **before** the location check, so "quorum is + unreachable" — the more specific diagnosis — is what the user sees when both are true. +- **The bootstrap DDL runs on `locations[0]`/replica-0 only**, is skipped entirely once the database + exists, and otherwise waits up to **300 s** for Keeper before exiting with a named failure that prints the + configured Keeper members and the GVC's real locations. 2.8.0 looped forever printing + `Keeper not ready yet, waiting...` and never created the database. +- **Use `ReplicatedMergeTree` + a `Distributed` table.** A plain `MergeTree` in a multi-shard cluster is + single-copy; the cluster's availability story does not cover it. +- **A `helm upgrade` restarts every replica in every location at once.** `maxUnavailableReplicas` is + silently dropped on stateful workloads, so there is no way to serialize it — expect a measured outage on + upgrade rather than a rolling one. The chart renders no `rolloutOptions`, so no availability claim rests + on that field. +- **The credentials secret has no `username`** — ClickHouse authenticates as its built-in `default` user. + Credentials apply on **first initialisation only**; rotate inside ClickHouse first, then update the + secret, then force a redeployment (a `cpln://` secret rotation does **not** redeploy by itself). +- **An identity's cloud binding is never removed once set** (the API merges on update), so switching + `provider` on an existing release leaves the old cloud-account binding attached. Switch providers with a + fresh install. +- **Keep locations and the bucket in one region family.** Distributed queries fan out cross-region and every + cache miss pulls from object storage — both are billed. diff --git a/clickhouse/versions/3.0.0/Chart.yaml b/clickhouse/versions/3.0.0/Chart.yaml index fbf65bac..82814908 100644 --- a/clickhouse/versions/3.0.0/Chart.yaml +++ b/clickhouse/versions/3.0.0/Chart.yaml @@ -3,16 +3,16 @@ name: clickhouse description: Fast, open source column-oriented database management system type: application -version: 2.8.0 +version: 3.0.0 appVersion: "25.10" annotations: created: "2025-11-21" - lastModified: "2026-08-24" + lastModified: "2026-08-27" category: "database" - createsGvc: true + createsGvc: false dependencies: - name: cpln-common version: 1.0.0 - repository: "oci://ghcr.io/controlplane-com/templates" \ No newline at end of file + repository: "oci://ghcr.io/controlplane-com/templates" diff --git a/clickhouse/versions/3.0.0/README.md b/clickhouse/versions/3.0.0/README.md index fd755552..2132acf5 100644 --- a/clickhouse/versions/3.0.0/README.md +++ b/clickhouse/versions/3.0.0/README.md @@ -1,47 +1,62 @@ # ClickHouse -> **Object-storage credentials are a prerequisite secret (2.7.0+)** for the `gcp` and `hetzner` providers. -> They reach ClickHouse as `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY` and are read through -> ``, because a `cpln://` reference inside the disk XML is never resolved. -> `aws` was already keyless via cloud identity and `azure` uses an account key, so neither changes. -> An upgrade still carrying `accessKeyId`/`secretAccessKey` is refused at render. +ClickHouse is a column-oriented analytical database for real-time querying and data warehousing at scale. +This template deploys it in **single-node** or **cluster** mode — the mode is derived from the `locations` +list — with object storage (AWS S3, GCS, Azure Blob Storage, or Hetzner Object Storage) as the primary data +store. **From 3.0.0 the chart deploys into the GVC you install into and creates none of its own.** +## Architecture -This template deploys ClickHouse in either **single-node** or **cluster** mode depending on how locations are configured in `values.yaml`. All deployments use object storage (AWS S3, GCS, Azure Blob Storage, or Hetzner Object Storage) as the primary data store. +- **ClickHouse Server** — stateful workload, the database itself, with configurable replicas per location. +- **ClickHouse Keeper** *(cluster modes only)* — stateful workload providing Raft coordination, one replica in each of the first three locations. +- **Volume sets** — server metadata and state, plus Keeper state in cluster modes. Table data lives in object storage; the volume is metadata and read cache. +- **Scratch volumes** — local filesystem cache and temporary spill. +- **Secrets** — startup scripts for Server and Keeper, plus one storage-configuration secret for the selected provider. +- **Identity + two policies** — `reveal` on this release's secrets and your credentials secret, plus `view` on the one GVC you install into so each container can confirm at boot that the GVC really has every location you listed. -ClickHouse is a high-performance column-oriented analytical database designed for real-time querying and data warehousing at scale. Storage includes: +This template does **not** create a GVC. Every resource lands in the GVC you pass to `--gvc`, so +`cpln workload exec`, `cpln logs` and `cpln helm uninstall` all work against that GVC, and uninstalling +can never delete it. -- **Primary object storage** — long-term scalable storage (AWS S3, GCS, Azure Blob Storage, or Hetzner Object Storage) -- **Scratch volume** — fast local read cache for performance -- **Volumeset** — persistent metadata, state, and system files +## Deployment Modes -## Architecture +The `locations` list is the topology, not just placement — its length selects the mode, and a location's +position in it is that shard's number. -- **GVC** — created by this template, named by `gvc.name`. Give every install its own. -- **Stateful ClickHouse Server workload** — the database, with configurable replicas per location. -- **Stateful ClickHouse Keeper workload** *(cluster mode only)* — coordination, one replica per location across the first three. -- **Volume sets** — server metadata and state, plus Keeper state in cluster mode. Primary data lives in object storage; the volume is a read cache. -- **Secrets** — startup scripts for Server and Keeper, and one storage-configuration secret for the selected provider. -- **Identity and policy** — `reveal` on this template's secrets plus the credentials secret you create, and cloud access to the bucket. +| `locations` | Mode | Keeper | +|---|---|---| +| 1 location, `replicas: 1` | **Single-node** — development, staging, lower-traffic workloads | none | +| 1 location, `replicas: N > 1` | **Single-shard cluster** — one shard, N replicas, survives a replica loss | 1 member (no fault tolerance) | +| 3 or more locations | **Multi-shard cluster** — one shard per location | 3 members across the first three locations, quorum 2 | +| 2 locations | **Not supported** — refused at render | — | -## Deployment Modes +To minimize network egress costs, keep all locations in the same cloud provider and the bucket in the same +region family. One server replica per location is enough for most cluster deployments. -### Single-Node -Specify exactly **1 location with `replicas: 1`**. No ClickHouse Keeper is deployed. Ideal for development, staging, or lower-traffic workloads where high availability is not required. +## Prerequisites -### Single-Shard Cluster -Specify **1 location with `replicas` > 1**. Deploys a single shard with multiple replicas in one location. ClickHouse Keeper is deployed for replication coordination. +**A GVC must already exist, and it must contain every location you list in `locations`.** +The requirement is one-directional: the GVC may have *more* locations than you list — nothing +ClickHouse-related runs in those. Check what a GVC has before you install: -### Multi-Shard Cluster -Specify **3 or more locations**. Deploys a shard per location with configurable replicas. ClickHouse Keeper is deployed across the first 3 locations for quorum. Recommended for production workloads requiring high availability and geographic distribution. +```bash +cpln gvc get GVC_NAME -o json +``` -> **Note:** 2 locations is not supported. Use 1 location (single-node or single-shard) or 3+. +The locations are under `spec.staticPlacement.locationLinks`. If you list a location the GVC does not +have, `helm install` still succeeds — the platform does not validate it — and the containers then refuse +to initialise with a named error in `cpln logs`: -**Important**: To minimize network egress costs, deploy all locations in the same cloud provider and keep object storage in the same region(s). Using 1 replica per location for ClickHouse server is sufficient for most cluster deployments. +``` +[clickhouse] FATAL: locations declared in values are not in GVC 'my-gvc': aws-eu-central-1 +``` -## Prerequisites +If **every** location you list is absent from the GVC, nothing starts at all and there is no container to +log anything: `cpln workload get-deployments` shows zero replicas and `desiredScale: None` in every +location. That is why the pre-flight check above matters. -**One `dictionary` secret must exist BEFORE you install.** This is the password you put in every client connection, so it is not a value — putting it in values would leave it in the Helm release. +**One `dictionary` secret must exist BEFORE you install.** This is the password you put in every client +connection, so it is not a value — putting it in values would leave it in the Helm release. ```bash cpln secret create-dictionary --name my-clickhouse-credentials \ @@ -49,53 +64,155 @@ cpln secret create-dictionary --name my-clickhouse-credentials \ --entry database=mydatabase ``` -Set `database.credentialsSecretName` to the name you used. Secret names are organization-wide, so give each release its own. - -There is no `username` key: ClickHouse authenticates as its built-in `default` user here, so the secret holds only `password` and `database`. +Set `database.credentialsSecretName` to the name you used. Secret names are organization-wide, so give each +release its own. There is no `username` key: ClickHouse authenticates as its built-in `default` user, so the +secret holds only `password` and `database`. -**If the secret does not exist at install time, the deployment wedges silently.** `cpln logs` returns **zero lines** — the container never starts, so it has nothing to log. Read `status.versions[].message` instead: +**If the secret does not exist at install time, the deployment wedges silently.** `cpln logs` returns +**zero lines** — the container never starts, so it has nothing to log. Read `status.versions[].message`: ```bash cpln workload get-deployments RELEASE_NAME-clickhouse-server --gvc GVC_NAME -o yaml ``` -Note this is `get-deployments` — plain `cpln workload get` has no `versions` field. +Note this is `get-deployments` — plain `cpln workload get` has no `versions` field. Creating the secret +repairs the deployment on its own in roughly 5.5 to 10.5 minutes, or force a redeployment to skip the wait. + +**Object storage is required in every mode**, including single-node. See [Storage setup](#storage-setup) for +the per-provider steps; the `gcp`, `azure` and `hetzner` providers each need their own prerequisite secret. -Upgrading from 2.5.x: delete `database.password` and `database.name` from your values and create the secret instead, using the password the database already has — it was applied on first initialisation and a new value in the secret will not change it. An upgrade that still carries either key is refused at render. +## Migrating from 2.x + +**Never `helm upgrade` a 2.x release onto 3.0.0.** Versions through 2.8.0 created their own GVC, so that +GVC is part of the 2.x release's manifest. 3.0.0 does not declare it — and Helm deletes what a chart stops +declaring. The upgrade would therefore **delete the GVC and every workload, volume set and identity inside +it**, including your ClickHouse metadata and Keeper state, in seconds, while printing `upgraded +successfully`. + +The chart refuses to render if your values still carry a `gvc:` key, so an upgrade that passes your old +values file fails before touching anything. A 2.x install made on pure defaults has no such key and is +**not** protected — nothing at render time can see it. Migrate instead: + +1. Create (or pick) the GVC you want 3.0.0 to live in, with the locations you intend to use. +2. Install 3.0.0 as a **new release** with a **new release name** into that GVC. Point it at the **same + bucket with a different prefix**, or at a new bucket. Secret names are organization-wide and would + otherwise collide with the 2.x release's. +3. Re-ingest your data into the new cluster. Do **not** try to adopt the old release's volume set: it holds + metadata whose `` identity and Keeper paths belong to the old topology, and it cannot be + moved between releases. +4. Cut your applications over to the new endpoint. +5. Uninstall the old release **against the GVC you originally installed it into**, not the GVC it created. + That is where Helm tracks the release, and it takes the created GVC with it. + +Renamed in 3.0.0: `gvc.locations` is now the top-level `locations`, and `server.internal_access` / +`keeper.internal_access` are now `server.internalAccess` / `keeper.internalAccess`. The `workloads` list +under those keys is now actually applied — in 2.x it was ignored, so `type: workload-list` silently blocked +all internal traffic. ## Configuration -Before installing, update `values.yaml` with the parameters relevant to your environment: +### Locations -- **GVC name**: Assign a name for the Global Virtual Cloud. -- **Locations**: Set 1 location with `replicas: 1` for single-node, or configure 3+ locations for a cluster. -- **Cluster Name**: Assign a cluster name. Used in distributed DDL queries (cluster mode only). -- **Storage**: Choose a provider (`aws`, `gcp`, `azure`, or `hetzner`) and fill in the configuration values under that section. +```yaml +# Every location listed here MUST already exist in the GVC you install into. +# Extra locations in the GVC are fine: nothing ClickHouse-related runs in them. +# The list length selects the mode — see Deployment Modes above. +locations: + - name: aws-us-east-1 + replicas: 1 +``` -**Note on GVC Naming** - - This template creates a GVC automatically with a name defined in `values.yaml`. If deploying multiple independent ClickHouse instances, **you must use a unique GVC name** for each deployment. +### Cluster -## Setting Up Storage +```yaml +# Used in cluster modes only. Must be a bare identifier: letters, digits and +# underscores, not starting with a digit — it becomes an XML element name and is +# used unquoted in `ON CLUSTER` DDL. +clusterName: my_cluster -Object storage is required for all deployment modes. Choose one of the supported providers below. +database: + # REQUIRED PREREQUISITE SECRET — create it before you install (see Prerequisites). + # A `dictionary` secret holding exactly `password` and `database`. No `username`. + credentialsSecretName: my-clickhouse-credentials +``` -### AWS S3 +### Object storage + +```yaml +provider: aws # Options: aws, gcp, azure, or hetzner + +aws: # If enabled, all fields below are required - See README for guidance + bucket: my-clickhouse-bucket # Name of your S3 bucket + region: us-east-1 # Region of your S3 bucket + cloudAccountName: my-clickhouse-cloudaccount # Name of your Cloud Account + policyName: my-clickhouse-s3-policy # Name of your pre-created policy to allow access to the S3 bucket + +gcp: # If enabled, all fields below are required - See README for guidance + bucket: my-clickhouse-gcs-bucket # Name of your GCS bucket + # REQUIRED PREREQUISITE SECRET — a `dictionary` secret holding `accessKeyId` + # and `secretAccessKey` (the GCS interoperability HMAC pair). + credentialsSecretName: my-clickhouse-gcs-credentials + +azure: # If enabled, all fields below are required - See README for guidance + storageAccount: myclickhousestorage # Name of your Azure Storage Account + container: clickhouse-data # Name of your Blob Storage container + # REQUIRED PREREQUISITE SECRET — a `dictionary` secret holding `accountKey`. + credentialsSecretName: my-clickhouse-azure-credentials + +hetzner: # If enabled, all fields below are required - See README for guidance + bucket: my-clickhouse-hetzner-bucket # Name of your Hetzner Object Storage bucket + region: nbg1 # Region of your bucket. Options: nbg1, hel1, fsn1 + # REQUIRED PREREQUISITE SECRET — a `dictionary` secret holding `accessKeyId` + # and `secretAccessKey`. + credentialsSecretName: my-clickhouse-hetzner-credentials +``` +### Storage volumes -Upgrading from 2.6.0: this version removes aws::ReadOnlyAccess from the backup identity. -That managed policy granted read access to every bucket in your AWS account and contained no write actions, -so it was never carrying the backup itself — but it was silently supplying any read action your -bucket-scoped policy happened to omit. Update your IAM policy to the full action list in this section before -upgrading; if it already matches, no action is needed. Nothing else changes. +```yaml +volumeset: + server: + capacity: 10 # initial capacity in GiB (minimum is 10) + keeper: + capacity: 10 # initial capacity in GiB (minimum is 10) - cluster modes only +``` +### Server and Keeper + +```yaml +server: + image: clickhouse/clickhouse-server:25.10 + resources: + cpu: 2 + memory: 2Gi + internalAccess: + type: same-gvc # options: same-gvc, same-org, workload-list, none + workloads: [] # required when type is workload-list; e.g. //gvc/GVC_NAME/workload/WORKLOAD_NAME + +keeper: # cluster modes only + image: clickhouse/clickhouse-keeper:25.10 + resources: + cpu: 2 + memory: 2Gi + internalAccess: + type: same-gvc # options: same-gvc, same-org, workload-list, none + workloads: [] # required when type is workload-list; e.g. //gvc/GVC_NAME/workload/WORKLOAD_NAME +``` -For ClickHouse to have access to a S3 bucket, ensure the following prerequisites are completed in your AWS account before installing: +An access-knob change takes up to about five minutes to propagate — re-test before concluding it did not +apply. -1. Create your bucket. Update the value `bucket` to include its name and `region` to include its region. +## Storage setup -2. If you do not have a Cloud Account set up, refer to the docs to [Create a Cloud Account](https://docs.controlplane.com/guides/create-cloud-account). Update the value `cloudAccountName`. +Object storage is required in every deployment mode. Choose one of the providers below. -3. Create a new policy with the following JSON (replace `YOUR_BUCKET_NAME`) +### AWS S3 + +1. Create your bucket. Set `aws.bucket` to its name and `aws.region` to its region. + +2. If you do not have a Cloud Account set up, refer to the docs to [Create a Cloud Account](https://docs.controlplane.com/guides/create-cloud-account). Set `aws.cloudAccountName`. + +3. Create a new IAM policy with the following JSON (replace `YOUR_BUCKET_NAME`), and set `aws.policyName` to its name: ```JSON { @@ -124,17 +241,15 @@ For ClickHouse to have access to a S3 bucket, ensure the following prerequisites } ``` -4. Update `cloudAccountName` in your values file with the name of your Cloud Account. - -5. Set `policyName` to match the policy created in step 3. +AWS is the only keyless provider here — access comes from the Cloud Account through the workload identity, +so there is no key to store. ### GCS -For ClickHouse to have access to a GCS bucket, ensure the following prerequisites are completed in your GCP account before installing: - -**Note**: ClickHouse requires S3-compatible HMAC authentication. You must provide an interoperability HMAC key. A Cloud Account is not required. +ClickHouse reaches GCS over its S3-compatible interface, which requires an interoperability HMAC key. A +Cloud Account is not required. -1. Create your bucket. Update the value `bucket` to include its name. +1. Create your bucket. Set `gcp.bucket` to its name. 2. Navigate to Settings > Interoperability and click `Create a key for a service account`. @@ -142,7 +257,7 @@ For ClickHouse to have access to a GCS bucket, ensure the following prerequisite 4. Under `Permissions`, assign the role `Storage Object Admin` and click `Done`. -5. You will be provided a new HMAC key. Store it in a `dictionary` secret and set `gcp.credentialsSecretName` to that secret's name: +5. Store the HMAC key in a `dictionary` secret and set `gcp.credentialsSecretName` to that secret's name: ```bash cpln secret create-dictionary --name my-clickhouse-gcs-credentials \ @@ -170,29 +285,24 @@ gsutil hmac create clickhouse-storage@$(gcloud config get-value project).iam.gse ### Azure Blob Storage -ClickHouse uses Azure's native Blob Storage SDK. A Cloud Account is not required — authentication uses a storage account access key directly. +ClickHouse uses Azure's native Blob Storage SDK. A Cloud Account is not required — authentication uses a +storage account access key. 1. In the [Azure Portal](https://portal.azure.com), go to **Storage accounts → Create**. - Performance: Standard - Redundancy: LRS - Leave hierarchical namespace off -2. Inside the storage account, go to **Containers → + Container** and create a container (e.g. `clickhouse-data`). Set access level to **Private**. +2. Inside the storage account, go to **Containers → + Container** and create a container (e.g. `clickhouse-data`). Set access level to **Private**. Set `azure.storageAccount` and `azure.container`. 3. Go to **Security + networking → Access keys** and copy either `key1` or `key2`. -4. Update `values.yaml`: - - `azure.storageAccount` — the storage account name - - `azure.container` — the container name from step 2 - - `azure.credentialsSecretName` — the name of a `dictionary` secret holding the access key from step 3: - - ```bash - cpln secret create-dictionary --name my-clickhouse-azure-credentials \ - --entry accountKey=YOUR_ACCOUNT_KEY - ``` +4. Store the key in a `dictionary` secret and set `azure.credentialsSecretName` to that secret's name: - The key is never a value: it is read from this secret into the container's environment and - referenced from the storage config, so it never lands in the Helm release. +```bash +cpln secret create-dictionary --name my-clickhouse-azure-credentials \ + --entry accountKey=YOUR_ACCOUNT_KEY +``` To configure using the CLI: @@ -214,50 +324,76 @@ az storage account keys list \ ### Hetzner Object Storage -Hetzner Object Storage is S3-compatible. A Cloud Account is not required — authentication uses an access key pair. - -Available regions: -- `nbg1` — Nuremberg, Germany -- `hel1` — Helsinki, Finland -- `fsn1` — Falkenstein, Germany +Hetzner Object Storage is S3-compatible. A Cloud Account is not required — authentication uses an access +key pair. Available regions: `nbg1` (Nuremberg), `hel1` (Helsinki), `fsn1` (Falkenstein). -1. In the Hetzner Cloud console, go to **Object Storage** and create a bucket. Note the bucket name and region. +1. In the Hetzner Cloud console, go to **Object Storage** and create a bucket. Set `hetzner.bucket` and `hetzner.region`. 2. Go to **Security → S3 Credentials** and click **Generate credentials**. Save the **Access Key** and **Secret Key** immediately — the secret will not be shown again. -3. Update `values.yaml`: - - `hetzner.bucket` — the bucket name - - `hetzner.region` — the region (e.g. `nbg1`) - - `hetzner.credentialsSecretName` — the name of a `dictionary` secret holding the key pair from step 2: +3. Store the pair in a `dictionary` secret and set `hetzner.credentialsSecretName` to that secret's name: + +```bash +cpln secret create-dictionary --name my-clickhouse-hetzner-credentials \ + --entry accessKeyId=YOUR_ACCESS_KEY \ + --entry secretAccessKey=YOUR_SECRET_KEY +``` - ```bash - cpln secret create-dictionary --name my-clickhouse-hetzner-credentials \ - --entry accessKeyId=YOUR_ACCESS_KEY \ - --entry secretAccessKey=YOUR_SECRET_KEY - ``` +## Connecting -## Connecting to ClickHouse +| What | Where | +|---|---| +| Public endpoint | **None.** This template exposes no public access — `inboundAllowCIDR` is empty and there is no direct load balancer | +| Native protocol (clients, `clickhouse-client`) | `RELEASE_NAME-clickhouse-server.GVC_NAME.cpln.local:9000` | +| HTTP interface | `RELEASE_NAME-clickhouse-server.GVC_NAME.cpln.local:8123` | +| A specific replica | `replica-INDEX.RELEASE_NAME-clickhouse-server.LOCATION.GVC_NAME.cpln.local:9000` | +| Keeper (cluster modes) | `replica-0.RELEASE_NAME-clickhouse-keeper.LOCATION.GVC_NAME.cpln.local:9181` | +| Username | `default` — there is no other user | +| Password / database name | the `password` and `database` entries of your credentials secret | -To connect using the ClickHouse client from within the same GVC: +From another workload in the same GVC: -```SH -clickhouse-client --host $WORKLOAD_NAME --password $PASSWORD +```bash +clickhouse-client --host RELEASE_NAME-clickhouse-server.GVC_NAME.cpln.local \ + --port 9000 --user default --password 'YOUR-PASSWORD' ``` -### Supported External Services +Always use the fully qualified `.GVC_NAME.cpln.local` form. The bare workload name is not reliable on this +platform — whether it resolves depends on the workload type. -- [ClickHouse Documentation](https://clickhouse.com/docs/) -- [Cloud Accounts Documentation](https://docs.controlplane.com/guides/create-cloud-account#overview) -- [ClickHouse with S3](https://clickhouse.com/docs/integrations/s3) -- [ClickHouse with GCS](https://clickhouse.com/docs/integrations/gcs) -- [ClickHouse with Azure Blob Storage](https://clickhouse.com/docs/engines/table-engines/integrations/azureBlobStorage) -- [ClickHouse with S3-compatible storage](https://clickhouse.com/docs/integrations/s3#s3-compatible-storage) +### Tables in cluster modes + +Use `ReplicatedMergeTree` plus a `Distributed` table. A plain `MergeTree` in a multi-shard cluster is +single-copy and is not covered by the cluster's availability story. + +```sql +CREATE TABLE events_local ON CLUSTER my_cluster (id UInt64, ts DateTime) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/events', '{replica}') +ORDER BY id; + +CREATE TABLE events ON CLUSTER my_cluster AS events_local +ENGINE = Distributed(my_cluster, currentDatabase(), events_local, rand()); +``` + +`{shard}` and `{replica}` come from each node's ``, which the chart derives from the location's +position in `locations`. ## Important Notes -- **This template creates its own GVC.** Never point `gvc.name` at an existing shared GVC — the chart adopts a GVC that already exists, and uninstalling then deletes it along with everything else in it. Give each install a unique name. -- **Object storage is required in every mode**, including single-node. There is no local-only shape. -- **2 locations is not supported.** Use 1 (single-node or single-shard) or 3 or more. -- **The credentials secret has no `username` key.** ClickHouse authenticates as its built-in `default` user, so the secret holds only `password` and `database`. -- **Credentials apply on first initialization only.** Changing the secret afterwards does not change the cluster — rotate inside ClickHouse first, then update the secret. -- **Keep locations and buckets in the same provider and region family.** Cross-region traffic to object storage is billed on every query that misses the local cache. \ No newline at end of file +- **Never `helm upgrade` a 2.x release onto 3.0.0** — it deletes the GVC the 2.x chart created and everything in it. See [Migrating from 2.x](#migrating-from-2x) +- **The GVC must contain every location you list**, and may contain more. A missing one is not caught at install: the container exits with `FATAL: locations declared in values are not in GVC …`. An already-initialised node logs a `WARNING` instead and keeps serving, so this can never stop a running cluster +- **2 locations is not supported.** Use 1 (single-node or single-shard) or 3 or more +- **Object storage is required in every mode**, including single-node. There is no local-only shape +- **The credentials secret has no `username` key**, and credentials apply on first initialization only — rotate inside ClickHouse first, then update the secret, then force a redeployment. Updating a `cpln://` secret does not restart the workload by itself +- **Switch object-storage providers with a fresh install, not an upgrade.** An identity's cloud binding is never removed once set, so an existing release keeps the old provider's binding attached +- **Keeper is the availability floor.** Three members tolerate one loss; the single-shard shape has one member and tolerates none. If a majority of Keeper locations are missing from the GVC, the containers exit with a named error rather than waiting for an election that can never complete +- **`helm upgrade` restarts every replica in every location at once** — nothing serialises a rolling restart on a stateful workload, so treat an upgrade as a planned query interruption +- **Keep locations and the bucket in the same provider and region family.** Cross-region traffic to object storage is billed on every query that misses the local cache + +## Links + +- [ClickHouse Documentation](https://clickhouse.com/docs/) +- [ClickHouse Keeper](https://clickhouse.com/docs/guides/sre/keeper/clickhouse-keeper) +- [Data Replication (ReplicatedMergeTree)](https://clickhouse.com/docs/engines/table-engines/mergetree-family/replication) +- [Distributed Table Engine](https://clickhouse.com/docs/engines/table-engines/special/distributed) +- [Create a Cloud Account](https://docs.controlplane.com/guides/create-cloud-account) diff --git a/clickhouse/versions/3.0.0/templates/_helpers.tpl b/clickhouse/versions/3.0.0/templates/_helpers.tpl index 9fb3c4dc..220bdf83 100644 --- a/clickhouse/versions/3.0.0/templates/_helpers.tpl +++ b/clickhouse/versions/3.0.0/templates/_helpers.tpl @@ -77,6 +77,13 @@ Clickhouse Policy Name {{- printf "%s-clickhouse-policy" .Release.Name }} {{- end }} +{{/* +Clickhouse GVC-read Policy Name +*/}} +{{- define "clickhouse.policy.gvc.name" -}} +{{- printf "%s-clickhouse-gvc-policy" .Release.Name }} +{{- end }} + {{/* Clickhouse Volume Set Server Name */}} @@ -100,7 +107,8 @@ Requires exactly 1 location with exactly 1 replica. 1 location with >1 replica is a single-shard cluster and still requires Keeper. */}} {{- define "clickhouse.isSingleNode" -}} -{{- if and (eq (len .Values.gvc.locations) 1) (eq ((index .Values.gvc.locations 0).replicas | int) 1) -}} +{{- $locs := .Values.locations | default (list) -}} +{{- if and (eq (len $locs) 1) (eq ((index $locs 0).replicas | int) 1) -}} true {{- end -}} {{- end -}} @@ -150,33 +158,68 @@ true {{- end -}} {{- end -}} +{{/* +`locations` is the topology, not just placement: its length selects the mode, a +node's position in it is its shard number, and the first three entries are the +Keeper Raft group. Every shape check therefore has to pass before anything else +in the chart reads the list. +*/}} {{- define "clickhouse.validateLocations" -}} -{{- $count := len .Values.gvc.locations -}} +{{- if not .Values.locations -}} + {{- fail "clickhouse: `locations` must contain at least 1 location. It is a top-level values key from 3.0.0 (it was `gvc.locations` in 2.x) and every location listed must already exist in the GVC you install into." -}} +{{- end -}} +{{- $count := len .Values.locations -}} {{- if eq $count 2 -}} {{- fail "2 locations is not supported. Use 1 location for single-node mode or 3+ locations for cluster mode." -}} {{- end -}} +{{- $seen := dict -}} +{{- range .Values.locations -}} + {{- if not .name -}} + {{- fail "clickhouse: every entry in `locations` needs a `name`." -}} + {{- end -}} + {{- if hasKey $seen .name -}} + {{- fail (printf "clickhouse: location '%s' is listed more than once in `locations`. Each location is one shard, and the shard index is the location's position in the list -- a duplicate emits two shards with identical replica hostnames, and the second is aliased onto the first and unreachable. List each location exactly once." .name) -}} + {{- end -}} + {{- $_ := set $seen .name true -}} +{{- end -}} {{- end -}} - -{{/* Labeling */}} +{{/* +2.8.0 quietly turned `replicas: 0` into a suspended location. That was never +documented, and with defaultOptions pinned to 0/0 the branch is dead code that +would still consume a shard index and a Keeper Raft slot. Refuse it instead. +*/}} +{{- define "clickhouse.validateReplicas" -}} +{{- range .Values.locations -}} +{{- if lt (.replicas | int) 1 -}} +{{- fail (printf "clickhouse: location '%s' must have at least 1 replica. To stop running in a location, remove it from `locations` (and from the GVC if nothing else uses it) -- a location with 0 replicas would still take a shard number and a Keeper Raft slot." .name) -}} +{{- end -}} +{{- end -}} +{{- end -}} {{/* -Common labels - delegated to cpln-common +clusterName becomes an XML element name in AND is interpolated +unquoted into `ON CLUSTER ` DDL, so anything that is not a bare identifier +produces a config that parses as something else or DDL that does not parse. */}} -{{- define "clickhouse.tags" -}} -{{- include "cpln-common.tags" . }} -{{- end }} -{{/* Validation */}} -{{- define "clickhouse.validate" -}} -{{- include "clickhouse.validateStorageCreds" . -}} -{{- if hasKey .Values.azure "accountKey" -}} -{{- fail "clickhouse: azure.accountKey was REMOVED — it is now a `dictionary` secret you create, named by azure.credentialsSecretName, holding the key `accountKey`. Delete it from your values. See Prerequisites in the README." -}} +{{- define "clickhouse.validateClusterName" -}} +{{- if not .Values.clusterName -}} +{{- fail "clickhouse: clusterName is required." -}} {{- end -}} -{{- if or (hasKey .Values.database "password") (hasKey .Values.database "name") -}} -{{- fail "clickhouse: database.password and database.name were REMOVED — they are now a `dictionary` secret you create, named by database.credentialsSecretName, holding the keys `password` and `database`. Delete them from your values. See Prerequisites in the README." -}} +{{- if not (regexMatch "^[A-Za-z_][A-Za-z0-9_]*$" (.Values.clusterName | toString)) -}} +{{- fail (printf "clickhouse: clusterName '%s' is not a bare identifier. It becomes an XML element name and is used unquoted in `ON CLUSTER` DDL, so it must match ^[A-Za-z_][A-Za-z0-9_]*$ -- letters, digits and underscores, not starting with a digit. 'my_cluster' is valid; 'my-cluster' is not." .Values.clusterName) -}} {{- end -}} -{{- if not .Values.database.credentialsSecretName -}} -{{- fail "clickhouse: database.credentialsSecretName is required — it names the `dictionary` secret holding `password` and `database`. Create it BEFORE installing; see Prerequisites in the README." -}} +{{- end -}} + +{{/* +The chart stopped creating a GVC in 3.0.0. Refuse to render if the values still +carry the 2.x `gvc` key -- an in-place `helm upgrade` from 2.x would drop +`kind: gvc` from the manifest, and Helm deletes what a chart no longer declares, +taking the GVC and everything inside it. +*/}} +{{- define "clickhouse.validateNoLegacyGvc" -}} +{{- if hasKey .Values "gvc" -}} +{{- fail "clickhouse 3.0.0: the `gvc` values key was REMOVED. This chart no longer creates a GVC -- it deploys into the GVC you install into, and `gvc.locations` moved to the top-level `locations`. DO NOT `helm upgrade` a 2.x release onto 3.0.0: the upgrade drops `kind: gvc` from the manifest and Helm deletes what a chart no longer declares, which DESTROYS that GVC and every workload, volumeset and identity inside it -- including your ClickHouse metadata and Keeper state. Install 3.0.0 as a NEW release against an existing GVC, reload your data, then uninstall the old release. See `Migrating from 2.x` in the README." -}} {{- end -}} {{- end -}} @@ -191,11 +234,171 @@ never resolved by the platform. {{- if eq $.Values.provider $p -}} {{- $c := index $.Values $p -}} {{- if or $c.accessKeyId $c.secretAccessKey -}} -{{- fail (printf "clickhouse: %s.accessKeyId and %s.secretAccessKey were REMOVED — they are now a `dictionary` secret you create, named by %s.credentialsSecretName, holding those two keys. Delete them from your values." $p $p $p) -}} +{{- fail (printf "clickhouse: %s.accessKeyId and %s.secretAccessKey were REMOVED -- they are now a `dictionary` secret you create, named by %s.credentialsSecretName, holding those two keys. Delete them from your values." $p $p $p) -}} {{- end -}} {{- if not $c.credentialsSecretName -}} -{{- fail (printf "clickhouse: %s.credentialsSecretName is required — it names the `dictionary` secret holding accessKeyId and secretAccessKey. Create it BEFORE installing." $p) -}} +{{- fail (printf "clickhouse: %s.credentialsSecretName is required -- it names the `dictionary` secret holding accessKeyId and secretAccessKey. Create it BEFORE installing." $p) -}} +{{- end -}} {{- end -}} {{- end -}} {{- end -}} + +{{- define "clickhouse.validateCredentials" -}} +{{- if hasKey .Values.azure "accountKey" -}} +{{- fail "clickhouse: azure.accountKey was REMOVED -- it is now a `dictionary` secret you create, named by azure.credentialsSecretName, holding the key `accountKey`. Delete it from your values. See Prerequisites in the README." -}} +{{- end -}} +{{- if or (hasKey .Values.database "password") (hasKey .Values.database "name") -}} +{{- fail "clickhouse: database.password and database.name were REMOVED -- they are now a `dictionary` secret you create, named by database.credentialsSecretName, holding the keys `password` and `database`. Delete them from your values. See Prerequisites in the README." -}} +{{- end -}} +{{- if not .Values.database.credentialsSecretName -}} +{{- fail "clickhouse: database.credentialsSecretName is required -- it names the `dictionary` secret holding `password` and `database`. Create it BEFORE installing; see Prerequisites in the README." -}} {{- end -}} +{{- end -}} + +{{/* +Single aggregate validator. Invoked once, from identity.yaml, which is +unconditionally rendered in every mode -- so `is validation still wired up?` is +one grep. 2.8.0 scattered validateStorage across four files and still managed to +leave validateLocations unreachable in single-node mode. +*/}} +{{- define "clickhouse.validate" -}} +{{- include "clickhouse.validateNoLegacyGvc" . -}} +{{- include "clickhouse.validateLocations" . -}} +{{- include "clickhouse.validateReplicas" . -}} +{{- include "clickhouse.validateClusterName" . -}} +{{- include "clickhouse.validateStorage" . -}} +{{- include "clickhouse.validateStorageCreds" . -}} +{{- include "clickhouse.validateCredentials" . -}} +{{- end -}} + + +{{/* Topology */}} + +{{/* +The topology, rendered ONCE for the whole chart and consumed by both startup +scripts, so the server and the Keeper can never disagree about the cluster. +`loc:replicas` pairs, comma separated. +*/}} +{{- define "clickhouse.locationsEnv" -}} +- name: LOCATIONS_STR + value: "{{- $locs := .Values.locations | default (list) -}}{{- range $i, $loc := $locs -}}{{ $loc.name }}:{{ $loc.replicas }}{{ if lt $i (sub (len $locs) 1) }},{{ end }}{{- end }}" +{{- end -}} + +{{/* +Shared boot-time GVC reconciliation prelude, emitted into all three startup +script bodies (server single-node, server cluster, keeper). + +Why it exists: the platform accepts a localOptions entry naming a location the +GVC does not have. It is stored verbatim, is inert, and nothing reports it -- so +a typo in `locations` yields a cluster whose declared shards can never exist, +and (before 3.0.0) a bootstrap node that printed "Keeper not ready yet" forever. +Helm cannot see a live GVC at render time, so ask the GVC itself at boot. + +Portability: neither image ships curl (server is ubuntu + GNU wget, keeper is +alpine + BusyBox wget), so this uses only the wget options both accept -- no +--tries, no --timeout=, no -w. HTTP failure surfaces as a non-zero exit and the +retry is done in shell. Measured byte-identical output under both. + +Requires LOG_PREFIX to be set by the caller, so the Keeper's lines are not +attributed to the server. +Sets: GVC, CONFIGURED_LOCS, GVC_READ_OK, GVC_LOCS, MISSING. The caller decides +what to do about MISSING -- the disposition differs per script. +*/}} +{{- define "clickhouse.gvcLocationGuard" -}} +# --- GVC location reconciliation --- +# GVC comes from the runtime built-in, not from Helm, so it cannot drift from +# the GVC the workload is actually running in. +GVC="${CPLN_GVC}" + +CONFIGURED_LOCS="" +for _pair in $(printf '%s' "${LOCATIONS_STR:-}" | tr ',' ' '); do + CONFIGURED_LOCS="${CONFIGURED_LOCS}${_pair%%:*} " +done + +GVC_LOCS="" +GVC_READ_OK="false" +for _try in 1 2 3; do + if GVC_JSON=$(wget -q -O - -T 10 \ + --header="Authorization: ${CPLN_TOKEN:-}" \ + "${CPLN_ENDPOINT:-http://api.cpln.io}/org/${CPLN_ORG:-}/gvc/${GVC}" 2>/dev/null); then + GVC_LOCS=$(printf '%s' "$GVC_JSON" | tr -d ' \n' \ + | sed -n 's/.*"locationLinks":\[\([^]]*\)\].*/\1/p' \ + | tr ',' '\n' | sed 's#.*/##;s/"//g' | tr '\n' ' ') + if [ -n "$GVC_LOCS" ]; then + GVC_READ_OK="true" + break + fi + fi + sleep 2 +done + +MISSING="" +if [ "$GVC_READ_OK" != "true" ]; then + # A control-plane hiccup must never be the reason a database refuses to + # start. Skip the API-backed checks; the values-only guards still apply. + echo "${LOG_PREFIX} WARNING: could not read the location list of GVC '${GVC}' -- skipping the GVC location check." >&2 +else + for _loc in ${CONFIGURED_LOCS}; do + case " ${GVC_LOCS} " in + *" ${_loc} "*) : ;; + *) MISSING="${MISSING}${_loc} " ;; + esac + done + EXTRA="" + for _loc in ${GVC_LOCS}; do + case " ${CONFIGURED_LOCS} " in + *" ${_loc} "*) : ;; + *) EXTRA="${EXTRA}${_loc} " ;; + esac + done + if [ -n "${EXTRA}" ]; then + echo "${LOG_PREFIX} GVC '${GVC}' also has locations this release does not use: ${EXTRA% } -- nothing ClickHouse-related runs there." + fi + if [ -z "${MISSING}" ]; then + echo "${LOG_PREFIX} GVC location check OK -- '${GVC}' has every configured location (${CONFIGURED_LOCS% })" + fi +fi +{{- end -}} + +{{/* +Keeper quorum feasibility, shared by the keeper (unconditional) and the server +(fresh data directory only). Keeper members are replica-0 of the first +min(3, len(locations)) declared locations; a Raft group of TOTAL needs +floor(TOTAL/2)+1 members to elect at all. Requires the guard above to have run. +Sets: KEEPER_MEMBER_LOCS, KEEPER_TOTAL, KEEPER_PRESENT, KEEPER_QUORUM_OK. +*/}} +{{- define "clickhouse.keeperQuorumArithmetic" -}} +# --- Keeper quorum feasibility --- +KEEPER_MEMBER_LOCS="" +_n=0 +for _loc in ${CONFIGURED_LOCS}; do + if [ "$_n" -ge 3 ]; then break; fi + KEEPER_MEMBER_LOCS="${KEEPER_MEMBER_LOCS}${_loc} " + _n=$((_n+1)) +done +KEEPER_TOTAL=$_n +KEEPER_PRESENT=0 +KEEPER_QUORUM_OK="unknown" +if [ "$GVC_READ_OK" = "true" ]; then + for _loc in ${KEEPER_MEMBER_LOCS}; do + case " ${GVC_LOCS} " in + *" ${_loc} "*) KEEPER_PRESENT=$((KEEPER_PRESENT+1)) ;; + esac + done + if [ $((2 * KEEPER_PRESENT)) -gt "$KEEPER_TOTAL" ]; then + KEEPER_QUORUM_OK="true" + else + KEEPER_QUORUM_OK="false" + fi +fi +{{- end -}} + + +{{/* Labeling */}} + +{{/* +Common labels - delegated to cpln-common +*/}} +{{- define "clickhouse.tags" -}} +{{- include "cpln-common.tags" . }} +{{- end }} diff --git a/clickhouse/versions/3.0.0/templates/gvc.yaml b/clickhouse/versions/3.0.0/templates/gvc.yaml deleted file mode 100644 index 2c2c9579..00000000 --- a/clickhouse/versions/3.0.0/templates/gvc.yaml +++ /dev/null @@ -1,11 +0,0 @@ -kind: gvc -name: {{ .Values.gvc.name }} -description: {{ .Values.gvc.name }} -tags: {{- include "clickhouse.tags" . | nindent 4 }} -spec: - endpointNamingFormat: org - staticPlacement: - locationLinks: - {{- range .Values.gvc.locations }} - - //location/{{ .name }} - {{- end }} diff --git a/clickhouse/versions/3.0.0/templates/identity.yaml b/clickhouse/versions/3.0.0/templates/identity.yaml index 0ecac0c3..caa2ba37 100644 --- a/clickhouse/versions/3.0.0/templates/identity.yaml +++ b/clickhouse/versions/3.0.0/templates/identity.yaml @@ -1,8 +1,7 @@ {{- include "clickhouse.validate" . -}} -{{ include "clickhouse.validateStorage" . }} --- kind: identity -gvc: {{ .Values.gvc.name }} +gvc: {{ .Values.global.cpln.gvc }} name: {{ include "clickhouse.identity.name" . }} description: Clickhouse identity tags: {{- include "clickhouse.tags" . | nindent 4 }} diff --git a/clickhouse/versions/3.0.0/templates/policy-gvc.yaml b/clickhouse/versions/3.0.0/templates/policy-gvc.yaml new file mode 100644 index 00000000..b51ddb63 --- /dev/null +++ b/clickhouse/versions/3.0.0/templates/policy-gvc.yaml @@ -0,0 +1,19 @@ +{{- /* + Each ClickHouse container reads its own GVC at boot to confirm the GVC really + has every location this release declares -- the platform does not validate + localOptions locations, so a typo is otherwise accepted silently and simply + never runs. Scoped to the ONE install GVC; never `target: all`, which would + grant view on every GVC in the org. +*/}} +kind: policy +name: {{ include "clickhouse.policy.gvc.name" . }} +description: Clickhouse GVC read policy +tags: {{- include "clickhouse.tags" . | nindent 4 }} +bindings: + - permissions: + - view + principalLinks: + - //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "clickhouse.identity.name" . }} +targetKind: gvc +targetLinks: + - //gvc/{{ .Values.global.cpln.gvc }} diff --git a/clickhouse/versions/3.0.0/templates/policy.yaml b/clickhouse/versions/3.0.0/templates/policy.yaml index 90f91019..03c45458 100644 --- a/clickhouse/versions/3.0.0/templates/policy.yaml +++ b/clickhouse/versions/3.0.0/templates/policy.yaml @@ -1,5 +1,3 @@ -{{ include "clickhouse.validateStorage" . }} ---- kind: policy name: {{ include "clickhouse.policy.name" . }} description: Clickhouse policy @@ -8,7 +6,7 @@ bindings: - permissions: - reveal principalLinks: - - //gvc/{{ .Values.gvc.name }}/identity/{{ include "clickhouse.identity.name" . }} + - //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "clickhouse.identity.name" . }} targetKind: secret targetLinks: - //secret/{{ .Values.database.credentialsSecretName }} diff --git a/clickhouse/versions/3.0.0/templates/secret-azure-config.yaml b/clickhouse/versions/3.0.0/templates/secret-azure-config.yaml index 7fd0b587..f8b7754f 100644 --- a/clickhouse/versions/3.0.0/templates/secret-azure-config.yaml +++ b/clickhouse/versions/3.0.0/templates/secret-azure-config.yaml @@ -1,4 +1,3 @@ -{{ include "clickhouse.validateStorage" . }} --- {{- if eq .Values.provider "azure" }} kind: secret diff --git a/clickhouse/versions/3.0.0/templates/secret-gcs-config.yaml b/clickhouse/versions/3.0.0/templates/secret-gcs-config.yaml index e668aedb..2246bd11 100644 --- a/clickhouse/versions/3.0.0/templates/secret-gcs-config.yaml +++ b/clickhouse/versions/3.0.0/templates/secret-gcs-config.yaml @@ -1,4 +1,3 @@ -{{ include "clickhouse.validateStorage" . }} --- {{- if eq .Values.provider "gcp" }} kind: secret diff --git a/clickhouse/versions/3.0.0/templates/secret-hetzner-config.yaml b/clickhouse/versions/3.0.0/templates/secret-hetzner-config.yaml index c1504190..03045094 100644 --- a/clickhouse/versions/3.0.0/templates/secret-hetzner-config.yaml +++ b/clickhouse/versions/3.0.0/templates/secret-hetzner-config.yaml @@ -1,4 +1,3 @@ -{{ include "clickhouse.validateStorage" . }} --- {{- if eq .Values.provider "hetzner" }} kind: secret diff --git a/clickhouse/versions/3.0.0/templates/secret-keeper-config.yaml b/clickhouse/versions/3.0.0/templates/secret-keeper-config.yaml index 203711ec..df53ac97 100644 --- a/clickhouse/versions/3.0.0/templates/secret-keeper-config.yaml +++ b/clickhouse/versions/3.0.0/templates/secret-keeper-config.yaml @@ -11,13 +11,15 @@ data: #!/usr/bin/env bash set -euo pipefail + LOG_PREFIX="[clickhouse-keeper]" + # --- Replica info --- + # GVC comes from the runtime built-in (set by the shared guard below), not + # from Helm, so it cannot drift from the GVC this workload actually runs in. LOCATION=$(basename "${CPLN_LOCATION:-}") HOSTNAME=${HOSTNAME:-} - GVC="{{ .Values.gvc.name }}" WORKLOAD_NAME="{{ include "clickhouse.keeper.name" . }}" REPLICA_INDEX=$(echo "${HOSTNAME}" | awk -F'-' '{print $NF}') - SELF_FQDN="replica-${REPLICA_INDEX}.${WORKLOAD_NAME}.${LOCATION}.${GVC}.cpln.local" # --- Directories --- CONFIG_DIR="/etc/clickhouse-keeper" @@ -25,12 +27,13 @@ data: LOG_DIR="/var/log/clickhouse-keeper" DATA_DIR="/var/lib/clickhouse/coordination" - mkdir -p "$CONFIG_DIR" "$LOG_DIR" "$DATA_DIR/logs" "$DATA_DIR/snapshots" - # --- Prepare locations from Helm values --- IFS=',' read -r -a LOCATIONS <<< "${LOCATIONS_STR:-}" # --- Compute CURRENT_SERVER_ID --- + # Values-only, so it runs before the API is consulted and can never be + # skipped. 2.8.0 wrote server_id 0 -- an invalid Raft id -- and started + # anyway when this replica was not a configured Keeper member. CURRENT_SERVER_ID=0 SERVER_ID=1 for LOC_PAIR in "${LOCATIONS[@]:0:3}"; do # only first 3 locations @@ -40,6 +43,38 @@ data: fi SERVER_ID=$((SERVER_ID+1)) done + if [[ "$CURRENT_SERVER_ID" -eq 0 ]]; then + echo "[clickhouse-keeper] FATAL: this replica (location '${LOCATION}', replica index '${REPLICA_INDEX}') is not a configured Keeper member." >&2 + echo "[clickhouse-keeper] Keeper members are replica-0 of the first three entries in 'locations' (${LOCATIONS_STR:-})." >&2 + echo "[clickhouse-keeper] Refusing to start with Raft server_id 0, which is not a valid id. Remove this location from the GVC, or add it to 'locations' in your values." >&2 + exit 1 + fi + + mkdir -p "$CONFIG_DIR" "$LOG_DIR" "$DATA_DIR/logs" "$DATA_DIR/snapshots" + + + {{- include "clickhouse.gvcLocationGuard" . | nindent 4 }} + + SELF_FQDN="replica-${REPLICA_INDEX}.${WORKLOAD_NAME}.${LOCATION}.${GVC}.cpln.local" + + {{- include "clickhouse.keeperQuorumArithmetic" . | nindent 4 }} + + # --- Check C: Keeper quorum must be arithmetically possible --- + # Unlike the server's location check this is NOT fresh/initialised + # asymmetric: below a majority the Raft group cannot elect at all, so the + # Keeper is useless whether or not it has data. It only fires on a + # successful GVC read, so a control-plane hiccup can never trigger it. + if [ "$KEEPER_QUORUM_OK" = "false" ]; then + echo "[clickhouse-keeper] FATAL: Keeper quorum is unreachable. ${KEEPER_PRESENT} of ${KEEPER_TOTAL} configured Keeper locations (${KEEPER_MEMBER_LOCS% }) exist in GVC '${GVC}' (which has: ${GVC_LOCS% })." >&2 + echo "[clickhouse-keeper] A Raft group of ${KEEPER_TOTAL} needs floor(${KEEPER_TOTAL}/2)+1 members. Refusing to start rather than waiting for an election that can never complete." >&2 + echo "[clickhouse-keeper] Add the missing location(s) to the GVC, or remove them from 'locations' in your values." >&2 + exit 1 + fi + if [ "$KEEPER_QUORUM_OK" = "true" ] && [ "$KEEPER_PRESENT" -lt "$KEEPER_TOTAL" ]; then + echo "[clickhouse-keeper] WARNING: quorum is possible but degraded -- only ${KEEPER_PRESENT} of ${KEEPER_TOTAL} configured Keeper locations (${KEEPER_MEMBER_LOCS% }) exist in GVC '${GVC}'. There is no fault tolerance left." >&2 + fi + + echo "[clickhouse-keeper] Starting Keeper ${SELF_FQDN} as Raft server_id ${CURRENT_SERVER_ID} of ${KEEPER_TOTAL}" # --- Start config.xml --- cat > "$CONFIG_FILE" <> "$CONFIG_FILE" <&2 + echo "[clickhouse] Refusing to start a second, unrelated ClickHouse under the same service name. Remove this location from the GVC, or add it to 'locations' in your values." >&2 + exit 1 ;; + esac + + + {{- include "clickhouse.gvcLocationGuard" . | nindent 4 }} + + # --- Check B: every declared location must exist in the GVC --- + # Asymmetric on purpose: hard-fail only on a FRESH data directory. An + # already-initialised node's own data is still correct, and hard-failing it + # would turn "the GVC shrank" into "the database is down". + if [ -n "${MISSING}" ]; then + if [ "${FRESH}" = "true" ]; then + echo "[clickhouse] FATAL: locations declared in values are not in GVC '${GVC}': ${MISSING% }" >&2 + echo "[clickhouse] GVC '${GVC}' has: ${GVC_LOCS% }" >&2 + echo "[clickhouse] Refusing to initialise. Add the missing location(s) to the GVC, or remove them from 'locations' in your values." >&2 + exit 1 + else + echo "[clickhouse] WARNING: locations declared in values are not in GVC '${GVC}': ${MISSING% }" >&2 + echo "[clickhouse] GVC '${GVC}' has: ${GVC_LOCS% }" >&2 + echo "[clickhouse] This node is already initialised, so it keeps serving." >&2 + fi + fi + mkdir -p "$CONFIG_DIR" "$LOG_DIR" "$DATA_DIR" # --- Generate config.xml --- @@ -142,36 +190,99 @@ data: #!/usr/bin/env bash set -euo pipefail + LOG_PREFIX="[clickhouse]" + # --- Runtime variables --- + # GVC comes from the runtime built-in (set by the shared guard below), not + # from Helm, so it cannot drift from the GVC this workload actually runs in. LOCATION=$(basename "${CPLN_LOCATION:-}") HOSTNAME=${HOSTNAME:-} - GVC="{{ .Values.gvc.name }}" WORKLOAD_NAME="{{ include "clickhouse.server.name" . }}" KEEPER_WORKLOAD_NAME="{{ include "clickhouse.keeper.name" . }}" REPLICA_INDEX=$(echo "${HOSTNAME}" | awk -F'-' '{print $NF}') - SELF_FQDN="replica-${REPLICA_INDEX}.${WORKLOAD_NAME}.${LOCATION}.${GVC}.cpln.local" - - echo "Starting ClickHouse server for replica ${SELF_FQDN}" # --- Directories --- CONFIG_DIR="/etc/clickhouse-server" LOG_DIR="/var/log/clickhouse-server" DATA_DIR="/var/lib/clickhouse" - mkdir -p "$CONFIG_DIR" "$LOG_DIR" "$DATA_DIR" + # Freshness marker, read BEFORE this script creates anything. + # clickhouse-server creates $DATA_DIR/metadata on its first successful start + # and it persists on the volumeset across a clean shutdown -- unlike + # $DATA_DIR/status, which is removed on exit. + if [ -d "${DATA_DIR}/metadata" ]; then FRESH="false"; else FRESH="true"; fi # --- Parse locations and replicas --- IFS=',' read -r -a LOCATIONS <<< "${LOCATIONS_STR:-}" # --- Compute SHARD_INDEX for this node (per location, not per replica) --- - SHARD_INDEX=1 + # The sentinel is 0, and 0 is fatal. 2.8.0 started at 1 and let a location + # that never matched fall out of the loop at len(locations)+1 -- a shard + # number present in no entry, written into + # and used for every ReplicatedMergeTree path. This runs BEFORE config.xml is + # written and before clickhouse-server launches, so a node in an undeclared + # location can never register a replicated path under a phantom shard. + SHARD_INDEX=0 + _i=1 for LOC_PAIR in "${LOCATIONS[@]}"; do - LOC="${LOC_PAIR%%:*}" - if [[ "$LOC" == "$LOCATION" ]]; then + if [[ "${LOC_PAIR%%:*}" == "$LOCATION" ]]; then + SHARD_INDEX=$_i break fi - SHARD_INDEX=$((SHARD_INDEX+1)) + _i=$((_i+1)) done + if [[ "$SHARD_INDEX" -eq 0 ]]; then + echo "[clickhouse] FATAL: this replica is running in location '${LOCATION}', which is not in the configured location list (${LOCATIONS_STR:-})." >&2 + echo "[clickhouse] Refusing to start: it would take a shard number that appears in no entry, register ReplicatedMergeTree paths under that phantom shard, and be invisible to every distributed query." >&2 + echo "[clickhouse] Remove this location from the GVC, or add it to 'locations' in your values." >&2 + exit 1 + fi + + + {{- include "clickhouse.gvcLocationGuard" . | nindent 4 }} + + SELF_FQDN="replica-${REPLICA_INDEX}.${WORKLOAD_NAME}.${LOCATION}.${GVC}.cpln.local" + + {{- include "clickhouse.keeperQuorumArithmetic" . | nindent 4 }} + + # --- Check C-prime: Keeper quorum must be arithmetically possible --- + # Fresh only, and deliberately BEFORE check B: when a location is missing AND + # that costs Keeper its majority, both checks are true, and 'quorum is + # unreachable' is the more specific and more actionable of the two. It also + # turns the bounded Keeper wait below into an immediate, named failure. + # The degraded warning below is NOT fresh-gated -- an initialised node that + # has lost fault tolerance should say so on every boot. + if [ "${FRESH}" = "true" ] && [ "$KEEPER_QUORUM_OK" = "false" ]; then + echo "[clickhouse] FATAL: Keeper quorum is unreachable. ${KEEPER_PRESENT} of ${KEEPER_TOTAL} configured Keeper locations (${KEEPER_MEMBER_LOCS% }) exist in GVC '${GVC}' (which has: ${GVC_LOCS% })." >&2 + echo "[clickhouse] A Raft group of ${KEEPER_TOTAL} needs floor(${KEEPER_TOTAL}/2)+1 members. Refusing to initialise rather than waiting for an election that can never complete." >&2 + exit 1 + fi + if [ "$KEEPER_QUORUM_OK" = "true" ] && [ "$KEEPER_PRESENT" -lt "$KEEPER_TOTAL" ]; then + echo "[clickhouse] WARNING: Keeper is degraded -- only ${KEEPER_PRESENT} of ${KEEPER_TOTAL} configured Keeper locations (${KEEPER_MEMBER_LOCS% }) exist in GVC '${GVC}'. Quorum is still possible, but there is no fault tolerance left." >&2 + fi + + # --- Check B: every declared location must exist in the GVC --- + # Asymmetric on purpose: hard-fail only on a FRESH data directory. A fresh + # node is being asked to build a cluster whose shards can never exist. An + # already-initialised node's own data and are still correct; + # the only damage is that some OTHER shard is unreachable, and hard-failing + # would turn "one shard down" into "all shards down". + if [ -n "${MISSING}" ]; then + if [ "${FRESH}" = "true" ]; then + echo "[clickhouse] FATAL: locations declared in values are not in GVC '${GVC}': ${MISSING% }" >&2 + echo "[clickhouse] GVC '${GVC}' has: ${GVC_LOCS% }" >&2 + echo "[clickhouse] Refusing to initialise a cluster whose declared shards can never exist. Add the missing location(s) to the GVC, or remove them from 'locations' in your values." >&2 + exit 1 + else + echo "[clickhouse] WARNING: locations declared in values are not in GVC '${GVC}': ${MISSING% }" >&2 + echo "[clickhouse] GVC '${GVC}' has: ${GVC_LOCS% }" >&2 + echo "[clickhouse] This node is already initialised, so it keeps serving its own shard -- but shards in the missing location(s) will never appear." >&2 + fi + fi + + echo "[clickhouse] Starting ClickHouse server for replica ${SELF_FQDN} (shard ${SHARD_INDEX})" + + mkdir -p "$CONFIG_DIR" "$LOG_DIR" "$DATA_DIR" # --- Generate config.xml --- cat > "$CONFIG_DIR/config.xml" <" - for LOC in "${LOCATIONS[@]:0:3}"; do # only first 3 locations - LOC_NAME="${LOC%%:*}" + for LOC_NAME in ${KEEPER_MEMBER_LOCS}; do # replica-0 of the first 3 locations FQDN="replica-0.${KEEPER_WORKLOAD_NAME}.${LOC_NAME}.${GVC}.cpln.local" echo " " echo " $FQDN" @@ -358,28 +468,51 @@ data: echo "ClickHouse is ready!" # --- Compute if this node is the bootstrap node for cluster-wide DB creation --- - FIRST_LOC="{{ (index .Values.gvc.locations 0).name }}" + FIRST_LOC="{{ (index .Values.locations 0).name }}" FIRST_LOCATION="false" if [[ "$LOCATION" == "$FIRST_LOC" && "$REPLICA_INDEX" == "0" ]]; then FIRST_LOCATION="true" fi - # --- Wait for Keeper quorum before creating database --- if [[ "$FIRST_LOCATION" == "true" ]]; then - echo "Waiting for Keeper quorum before bootstrapping DB..." - until clickhouse-client --query "SELECT * FROM system.zookeeper WHERE path='/'" >/dev/null 2>&1; do - echo "Keeper not ready yet, waiting..." - sleep 2 - done - echo "Keeper quorum detected!" - fi + # Skip the Keeper wait entirely when the database is already there: + # neither the wait nor the ON CLUSTER DDL is needed, and this is what + # keeps an initialised bootstrap node from crash-looping through a + # Keeper outage. + DB_EXISTS=$(clickhouse-client --user default --password "$CLICKHOUSE_PASSWORD" \ + --query "SELECT count() FROM system.databases WHERE name = '${CLICKHOUSE_DATABASE}'" 2>/dev/null || echo "0") + + if [[ "$DB_EXISTS" != "0" ]]; then + echo "[clickhouse] Database '${CLICKHOUSE_DATABASE}' already exists -- skipping the Keeper wait and the bootstrap DDL." + else + # --- Wait for Keeper quorum before creating the database --- + # BOUNDED. 2.8.0 looped forever printing "Keeper not ready yet", + # which turned a configuration error into a workload that looked + # healthy and never created the database. 300s is ~3x the worst + # legitimate Raft election in this chart (20-100s bounds). + echo "[clickhouse] Waiting for Keeper quorum before bootstrapping the database..." + SECONDS=0 + until clickhouse-client --user default --password "$CLICKHOUSE_PASSWORD" \ + --query "SELECT count() FROM system.zookeeper WHERE path='/'" >/dev/null 2>&1; do + if (( SECONDS >= 300 )); then + echo "[clickhouse] FATAL: Keeper quorum was not reachable within 300s." >&2 + echo "[clickhouse] Configured Keeper members: ${KEEPER_MEMBER_LOCS% } (replica-0 of each)." >&2 + echo "[clickhouse] GVC '${GVC}' locations: ${GVC_LOCS:-}." >&2 + echo "[clickhouse] Database '${CLICKHOUSE_DATABASE}' was NOT created. Check that every Keeper location exists in the GVC and that its replica-0 is ready." >&2 + kill -TERM "$CH_PID" 2>/dev/null || true + wait "$CH_PID" 2>/dev/null || true + exit 1 + fi + echo "[clickhouse] Keeper not ready yet, waiting... (${SECONDS}s of 300s)" + sleep 5 + done + echo "[clickhouse] Keeper quorum detected!" - # --- Create database using default user (only on first location / first replica) --- - if [[ "$FIRST_LOCATION" == "true" ]]; then - echo "Bootstrapping ClickHouse database across cluster (ON CLUSTER)..." - clickhouse-client --user default --password "$CLICKHOUSE_PASSWORD" \ - --query "CREATE DATABASE IF NOT EXISTS ${CLICKHOUSE_DATABASE} ON CLUSTER {{ .Values.clusterName }} ENGINE = Atomic;" + echo "[clickhouse] Bootstrapping ClickHouse database across cluster (ON CLUSTER)..." + clickhouse-client --user default --password "$CLICKHOUSE_PASSWORD" \ + --query "CREATE DATABASE IF NOT EXISTS ${CLICKHOUSE_DATABASE} ON CLUSTER {{ .Values.clusterName }} ENGINE = Atomic;" + fi else echo "Skipping DB creation on this replica (not the bootstrap node)." fi diff --git a/clickhouse/versions/3.0.0/templates/workload-keeper.yaml b/clickhouse/versions/3.0.0/templates/workload-keeper.yaml index f897dcd5..552a44ef 100644 --- a/clickhouse/versions/3.0.0/templates/workload-keeper.yaml +++ b/clickhouse/versions/3.0.0/templates/workload-keeper.yaml @@ -1,5 +1,4 @@ {{- if not (include "clickhouse.isSingleNode" .) }} -{{ include "clickhouse.validateLocations" . }} kind: workload name: {{ include "clickhouse.keeper.name" . }} description: clickhouse keeper @@ -12,8 +11,7 @@ spec: image: {{ .Values.keeper.image }} inheritEnv: false env: - - name: LOCATIONS_STR - value: "{{- range $i, $loc := .Values.gvc.locations -}}{{ $loc.name }}:{{ $loc.replicas }}{{ if lt $i (sub (len $.Values.gvc.locations) 1) }},{{ end }}{{- end }}" + {{- include "clickhouse.locationsEnv" . | nindent 6 }} memory: {{ .Values.keeper.resources.memory | quote }} command: "/bin/bash" args: @@ -29,23 +27,31 @@ spec: uri: cpln://volumeset/{{ include "clickhouse.volumeKeeper.name" . }} - path: /scripts/keeper-startup.sh uri: cpln://secret/{{ include "clickhouse.secretKeeper.name" . }} - identityLink: //gvc/{{ .Values.gvc.name }}/identity/{{ include "clickhouse.identity.name" . }} + identityLink: //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "clickhouse.identity.name" . }} defaultOptions: autoscaling: keda: {} maxConcurrency: 0 - maxScale: 1 + maxScale: 0 metric: disabled - minScale: 1 + minScale: 0 scaleToZeroDelay: 300 target: 100 capacityAI: false debug: false suspend: false timeoutSeconds: 5 - {{- $locs := .Values.gvc.locations }} + {{- /* + Keeper members are replica-0 of the FIRST THREE locations only. 2.8.0 + rendered a localOptions entry for every location and set `suspend: true` + on index >= 3; with defaultOptions pinned to 0/0 that entry is unnecessary, + and dropping it removes this chart's only use of localOptions[].suspend -- + a field that permanently breaks a workload's inbound reachability from + other locations. + */}} localOptions: - {{- range $i, $loc := $locs }} + {{- range $i, $loc := .Values.locations }} + {{- if lt $i 3 }} - autoscaling: maxConcurrency: 0 metric: disabled @@ -56,9 +62,10 @@ spec: capacityAI: false debug: false location: //location/{{ $loc.name }} - suspend: {{ if ge $i 3 }}true{{ else }}false{{ end }} + suspend: false timeoutSeconds: 5 {{- end }} + {{- end }} firewallConfig: external: inboundAllowCIDR: [] @@ -69,8 +76,16 @@ spec: outboundAllowPort: [] outboundBlockedCIDR: [] internal: - inboundAllowType: {{ .Values.keeper.internal_access.type }} + inboundAllowType: {{ .Values.keeper.internalAccess.type }} + {{- $wl := .Values.keeper.internalAccess.workloads | default (list) }} + {{- if $wl }} + inboundAllowWorkload: + {{- range $wl }} + - {{ . }} + {{- end }} + {{- else }} inboundAllowWorkload: [] + {{- end }} loadBalancer: direct: enabled: false diff --git a/clickhouse/versions/3.0.0/templates/workload-server.yaml b/clickhouse/versions/3.0.0/templates/workload-server.yaml index 09558b92..8f2155e8 100644 --- a/clickhouse/versions/3.0.0/templates/workload-server.yaml +++ b/clickhouse/versions/3.0.0/templates/workload-server.yaml @@ -1,5 +1,13 @@ -{{ include "clickhouse.validateLocations" . }} -{{ include "clickhouse.validateStorage" . }} +{{- /* + The aggregate validator is invoked from BOTH here and identity.yaml. It is + defined once and emits nothing, so the second call costs nothing -- but + Helm renders templates in reverse-lexical order within a directory, which + puts workload-server.yaml FIRST and identity.yaml LAST. Wiring it only to + identity.yaml meant a malformed `locations` (empty, or missing a name) + crashed here with a Go-template error before the friendly message could + fire. Do not rely on which of the two Helm reaches first. +*/ -}} +{{- include "clickhouse.validate" . -}} kind: workload name: {{ include "clickhouse.server.name" . }} description: clickhouse server @@ -28,8 +36,7 @@ spec: - name: AZURE_STORAGE_ACCOUNT_KEY value: cpln://secret/{{ .Values.azure.credentialsSecretName }}.accountKey {{- end }} - - name: LOCATIONS_STR - value: "{{- range $i, $loc := .Values.gvc.locations -}}{{ $loc.name }}:{{ $loc.replicas }}{{ if lt $i (sub (len $.Values.gvc.locations) 1) }},{{ end }}{{- end }}" + {{- include "clickhouse.locationsEnv" . | nindent 6 }} - name: CLICKHOUSE_PASSWORD value: cpln://secret/{{ .Values.database.credentialsSecretName }}.password - name: CLICKHOUSE_DATABASE @@ -77,14 +84,14 @@ spec: recoveryPolicy: retain uri: cpln://secret/{{ include "clickhouse.secretHetzner.name" . }} {{- end }} - identityLink: //gvc/{{ .Values.gvc.name }}/identity/{{ include "clickhouse.identity.name" . }} + identityLink: //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "clickhouse.identity.name" . }} defaultOptions: autoscaling: keda: {} maxConcurrency: 0 - maxScale: {{ if eq ((index .Values.gvc.locations 0).replicas | int) 0 }}1{{ else }}{{ (index .Values.gvc.locations 0).replicas | int }}{{ end }} + maxScale: 0 metric: disabled - minScale: 1 + minScale: 0 scaleToZeroDelay: 300 target: 100 capacityAI: false @@ -92,18 +99,18 @@ spec: suspend: false timeoutSeconds: 5 localOptions: - {{- range $location := .Values.gvc.locations }} + {{- range $location := .Values.locations }} - autoscaling: maxConcurrency: 0 - maxScale: {{ if eq ($location.replicas | int) 0 }}1{{ else }}{{ $location.replicas | int }}{{ end }} + maxScale: {{ $location.replicas | int }} metric: disabled - minScale: {{ if eq ($location.replicas | int) 0 }}0{{ else }}{{ $location.replicas | int }}{{ end }} + minScale: {{ $location.replicas | int }} scaleToZeroDelay: 300 target: 100 capacityAI: false debug: false location: //location/{{ $location.name }} - suspend: {{ if eq ($location.replicas | int) 0 }}true{{ else }}false{{ end }} + suspend: false timeoutSeconds: 10 {{- end }} firewallConfig: @@ -116,11 +123,19 @@ spec: outboundAllowPort: [] outboundBlockedCIDR: [] internal: - inboundAllowType: {{ .Values.server.internal_access.type }} + inboundAllowType: {{ .Values.server.internalAccess.type }} + {{- $wl := .Values.server.internalAccess.workloads | default (list) }} + {{- if $wl }} + inboundAllowWorkload: + {{- range $wl }} + - {{ . }} + {{- end }} + {{- else }} inboundAllowWorkload: [] + {{- end }} loadBalancer: direct: enabled: false ports: [] replicaDirect: true - supportDynamicTags: false \ No newline at end of file + supportDynamicTags: false diff --git a/clickhouse/versions/3.0.0/values.yaml b/clickhouse/versions/3.0.0/values.yaml index 5a5bd26b..63d52573 100644 --- a/clickhouse/versions/3.0.0/values.yaml +++ b/clickhouse/versions/3.0.0/values.yaml @@ -1,27 +1,29 @@ -gvc: - name: clickhouse-gvc - # Single-node mode: exactly 1 location with replicas: 1 — no Keeper, uses S3/GCS for storage - # Single-shard cluster: 1 location with replicas > 1 — Keeper required - # Multi-shard cluster: 3 or more locations — Keeper required - # Note: 2 locations is not supported - locations: - - name: aws-us-east-2 - replicas: 1 - - name: aws-us-west-2 - replicas: 1 - - name: aws-us-east-1 - replicas: 1 +# ─── Locations ──────────────────────────────────────────────────────────────── +# This chart deploys into the GVC you install into — it does NOT create one. +# Every location listed here MUST already exist in that GVC, or the nodes refuse +# to initialise with a named error (see Prerequisites in the README). +# Extra locations in the GVC are fine: nothing ClickHouse-related runs in them. +# +# Deployment mode is derived from this list: +# 1 location, replicas: 1 → single-node (no Keeper) +# 1 location, replicas: >1 → single-shard cluster (Keeper required) +# 3 or more locations → multi-shard cluster, one shard per location +# 2 locations → NOT supported +locations: + - name: aws-us-east-1 + replicas: 1 +# ─── Object Storage ─────────────────────────────────────────────────────────── provider: aws # Options: aws, gcp, azure, or hetzner aws: # If enabled, all fields below are required - See README for guidance - bucket: clickhouse-s3-bucket # Name of your S3 bucket + bucket: my-clickhouse-bucket # Name of your S3 bucket region: us-east-1 # Region of your S3 bucket - cloudAccountName: clickhouse-s3-cloudaccount # Name of your Cloud Account - policyName: clickhouse-s3-policy # Name of your pre-created policy to allow access to the S3 bucket + cloudAccountName: my-clickhouse-cloudaccount # Name of your Cloud Account + policyName: my-clickhouse-s3-policy # Name of your pre-created policy to allow access to the S3 bucket gcp: # If enabled, all fields below are required - See README for guidance - bucket: clickhouse-gcs-bucket # Name of your GCS bucket + bucket: my-clickhouse-gcs-bucket # Name of your GCS bucket # REQUIRED PREREQUISITE SECRET — a `dictionary` secret holding exactly # `accessKeyId` and `secretAccessKey` (the GCS interoperability HMAC pair). # They reach ClickHouse as AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY rather @@ -37,13 +39,17 @@ azure: # If enabled, all fields below are required - See README for guidance credentialsSecretName: my-clickhouse-azure-credentials hetzner: # If enabled, all fields below are required - See README for guidance - bucket: clickhouse-hetzner-bucket # Name of your Hetzner Object Storage bucket + bucket: my-clickhouse-hetzner-bucket # Name of your Hetzner Object Storage bucket region: nbg1 # Region of your bucket. Options: nbg1, hel1, fsn1 # REQUIRED PREREQUISITE SECRET — a `dictionary` secret holding exactly # `accessKeyId` and `secretAccessKey` for your Hetzner Object Storage. credentialsSecretName: my-clickhouse-hetzner-credentials -clusterName: my_cluster # Used in cluster mode only +# ─── Cluster ────────────────────────────────────────────────────────────────── +# Used in cluster modes only. Must be a bare identifier: letters, digits and +# underscores, not starting with a digit — it becomes an XML element name and is +# used unquoted in `ON CLUSTER` DDL. +clusterName: my_cluster database: # Automatically create a database on initialization using the default user # REQUIRED PREREQUISITE SECRET — CREATE IT BEFORE YOU INSTALL. @@ -53,30 +59,29 @@ database: # Automatically create a database on initialization using the default # time the deployment WEDGES silently; see Prerequisites in the README. credentialsSecretName: my-clickhouse-credentials +# ─── Storage ────────────────────────────────────────────────────────────────── volumeset: server: capacity: 10 # initial capacity in GiB (minimum is 10) keeper: - capacity: 10 # initial capacity in GiB (minimum is 10) - cluster mode only + capacity: 10 # initial capacity in GiB (minimum is 10) - cluster modes only +# ─── Server ─────────────────────────────────────────────────────────────────── server: image: clickhouse/clickhouse-server:25.10 resources: cpu: 2 memory: 2Gi - internal_access: - type: same-gvc # options: same-gvc, same-org, workload-list - workloads: # Note: can only be used if type is same-gvc or workload-list - #- //gvc/GVC_NAME/workload/WORKLOAD_NAME - #- //gvc/GVC_NAME/workload/WORKLOAD_NAME + internalAccess: + type: same-gvc # options: same-gvc, same-org, workload-list, none + workloads: [] # required when type is workload-list; e.g. //gvc/GVC_NAME/workload/WORKLOAD_NAME -keeper: # cluster mode only +# ─── Keeper (cluster modes only) ────────────────────────────────────────────── +keeper: # cluster modes only image: clickhouse/clickhouse-keeper:25.10 resources: cpu: 2 memory: 2Gi - internal_access: - type: same-gvc # options: same-gvc, same-org, workload-list - workloads: # Note: can only be used if type is same-gvc or workload-list - #- //gvc/GVC_NAME/workload/WORKLOAD_NAME - #- //gvc/GVC_NAME/workload/WORKLOAD_NAME + internalAccess: + type: same-gvc # options: same-gvc, same-org, workload-list, none + workloads: [] # required when type is workload-list; e.g. //gvc/GVC_NAME/workload/WORKLOAD_NAME From 548b565ef82170e8ebb1912c712026e36a3c0056 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Thu, 27 Aug 2026 14:43:44 -0600 Subject: [PATCH 3/4] clickhouse 3.0.0: bound the GVC read, which could stall a boot for ~10 minutes Found by the conventions review. The guard's own contract says "a control-plane hiccup must never be the reason a database refuses to start", and it did not hold: `-T 10` is a per-operation timeout, not a retry cap. GNU wget on the server image defaults to --tries=20 and retries connect and DNS failures, so a single call to an unreachable API can burn ~200s, and the three-attempt loop can reach ~600s -- past CLAUDE.md's five-minute readiness gate, in precisely the transient failure this code exists to tolerate. The spec deliberately excluded --tries because BusyBox wget (the keeper image) has no such option, so a portable bound has to come from outside wget. `timeout` is present in both images. Measured with controls, in the real images: server timeout 15 -> rc=124 at 15s (vs 33s for only THREE retries) keeper stops at its own -T 10 positive control: a fast connection-refused still returns immediately, so the bound does not penalise the healthy path Worst case is now 45s across the loop. Verified every rendered GVC read carries the prefix -- 2 occurrences in a 3-location render, 0 unbounded. The review's only warning; nothing here changes a path any functional row exercises, so the in-flight test round's completed rows stay valid. Co-Authored-By: Claude Opus 5 --- briefings/clickhouse.md | 1 + clickhouse/versions/3.0.0/templates/_helpers.tpl | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/briefings/clickhouse.md b/briefings/clickhouse.md index 9390e89a..5c03c32b 100644 --- a/briefings/clickhouse.md +++ b/briefings/clickhouse.md @@ -82,3 +82,4 @@ read cache. **From 3.0.0 it deploys into an existing GVC and creates none.** fresh install. - **Keep locations and the bucket in one region family.** Distributed queries fan out cross-region and every cache miss pulls from object storage — both are billed. +- **The boot-time GVC read is bounded with `timeout`, and that bound is load-bearing.** `-T` is a per-operation timeout, not a retry cap: GNU wget (the server image) defaults to `--tries=20` and retries connect and DNS failures, so an unreachable API could consume ~200 s per invocation and ~600 s across the three-attempt loop — past the platform's readiness budget, in exactly the transient failure the guard exists to tolerate. BusyBox wget (keeper) has no `--tries`, so the bound cannot come from wget itself. `timeout` is present in both images. Measured: server 15 s / rc=124 (vs 33 s for only three retries); keeper stops at its own `-T 10`. Worst case is now 45 s. Do not remove the `timeout` prefix. diff --git a/clickhouse/versions/3.0.0/templates/_helpers.tpl b/clickhouse/versions/3.0.0/templates/_helpers.tpl index 220bdf83..39642a7c 100644 --- a/clickhouse/versions/3.0.0/templates/_helpers.tpl +++ b/clickhouse/versions/3.0.0/templates/_helpers.tpl @@ -318,7 +318,15 @@ done GVC_LOCS="" GVC_READ_OK="false" for _try in 1 2 3; do - if GVC_JSON=$(wget -q -O - -T 10 \ + # `timeout` bounds the WHOLE call. -T is a per-operation timeout, not a retry + # cap: GNU wget (the server image) defaults to --tries=20 and retries connect + # and DNS failures, so an unreachable API can consume ~200s per invocation and + # ~600s across this loop -- past the platform's readiness budget, in exactly + # the transient failure this guard exists to tolerate. BusyBox wget (keeper) + # has no --tries, so a portable bound has to come from outside wget. + # Measured: server 15s rc=124 (vs 33s for only three retries); keeper stops at + # its own -T 10. `timeout` is present in both images. + if GVC_JSON=$(timeout 15 wget -q -O - -T 10 \ --header="Authorization: ${CPLN_TOKEN:-}" \ "${CPLN_ENDPOINT:-http://api.cpln.io}/org/${CPLN_ORG:-}/gvc/${GVC}" 2>/dev/null); then GVC_LOCS=$(printf '%s' "$GVC_JSON" | tr -d ' \n' \ From 403509d1bb7e6a8e7b5cc006bdc9e02df27a9916 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Thu, 27 Aug 2026 17:00:37 -0600 Subject: [PATCH 4/4] clickhouse 3.0.0: workload-list cut the cluster off from itself Same defect class etcd-ml hit in this batch, found independently. The internal firewall list governs intra-release traffic too -- Keeper's Raft on 9234, the servers' calls to Keeper on 9181, and server-to-server replication and distributed queries -- so a list naming only clients silently destroys the cluster. Measured by following the README: rbac_access_denied_matched_policy[none] ... "10.0.84.230:9234" Code: 279 ... Connection reset by peer ...while get-deployments reported every replica `ready=true restarts=0`. Control proving the mechanism: adding the keeper's own workload link restored `zk_followers 2 / zk_synced_followers 2`. Both workloads now carry the release's own server and keeper links. Both are included on both rather than the minimum each strictly needs: the over-grant is one release's workloads reaching each other inside one GVC, while the under-grant is a silently destroyed cluster, and getting the direction wrong is exactly how this shipped. Rendered across four controls -- unchanged `[]` on the same-gvc default; both own links plus the client in cluster mode on both workloads; no keeper link in single-node, where no Keeper exists; and no duplicates when the user names them too. Also gates the policy's reveal on the Keeper startup secret behind cluster mode. Single-node renders no Keeper, so 2.8.0 and the untouched 3.0.0 both granted reveal on a secret that is never created -- inert, but a dangling grant. Four README corrections the round measured: - the all-absent symptom is `desiredScale: 0`, not `None` (users grep for it) - a 3-shard rolling upgrade is ~83 s of total unavailability; a Keeper setting change ~60 s - renaming `clusterName` orphans existing Distributed tables (`Code: 701`) - with one replica per shard, losing a shard fails EVERY distributed query Co-Authored-By: Claude Opus 5 --- clickhouse/versions/3.0.0/README.md | 12 +++++-- .../versions/3.0.0/templates/policy.yaml | 3 ++ .../3.0.0/templates/workload-keeper.yaml | 32 ++++++++++++++++++- .../3.0.0/templates/workload-server.yaml | 32 ++++++++++++++++++- clickhouse/versions/3.0.0/values.yaml | 6 ++-- 5 files changed, 78 insertions(+), 7 deletions(-) diff --git a/clickhouse/versions/3.0.0/README.md b/clickhouse/versions/3.0.0/README.md index 2132acf5..176ffd2c 100644 --- a/clickhouse/versions/3.0.0/README.md +++ b/clickhouse/versions/3.0.0/README.md @@ -52,7 +52,7 @@ to initialise with a named error in `cpln logs`: ``` If **every** location you list is absent from the GVC, nothing starts at all and there is no container to -log anything: `cpln workload get-deployments` shows zero replicas and `desiredScale: None` in every +log anything: `cpln workload get-deployments` shows zero replicas and `desiredScale: 0` in every location. That is why the pre-flight check above matters. **One `dictionary` secret must exist BEFORE you install.** This is the password you put in every client @@ -187,7 +187,8 @@ server: memory: 2Gi internalAccess: type: same-gvc # options: same-gvc, same-org, workload-list, none - workloads: [] # required when type is workload-list; e.g. //gvc/GVC_NAME/workload/WORKLOAD_NAME + workloads: [] # required when type is workload-list; list only your clients -- this release's + # own server and keeper workloads are added automatically, e.g. //gvc/GVC_NAME/workload/WORKLOAD_NAME keeper: # cluster modes only image: clickhouse/clickhouse-keeper:25.10 @@ -196,7 +197,8 @@ keeper: # cluster modes only memory: 2Gi internalAccess: type: same-gvc # options: same-gvc, same-org, workload-list, none - workloads: [] # required when type is workload-list; e.g. //gvc/GVC_NAME/workload/WORKLOAD_NAME + workloads: [] # required when type is workload-list; list only your clients -- this release's + # own server and keeper workloads are added automatically, e.g. //gvc/GVC_NAME/workload/WORKLOAD_NAME ``` An access-knob change takes up to about five minutes to propagate — re-test before concluding it did not @@ -389,6 +391,10 @@ position in `locations`. - **Keeper is the availability floor.** Three members tolerate one loss; the single-shard shape has one member and tolerates none. If a majority of Keeper locations are missing from the GVC, the containers exit with a named error rather than waiting for an election that can never complete - **`helm upgrade` restarts every replica in every location at once** — nothing serialises a rolling restart on a stateful workload, so treat an upgrade as a planned query interruption - **Keep locations and the bucket in the same provider and region family.** Cross-region traffic to object storage is billed on every query that misses the local cache +- **With `workload-list`, list only your clients.** The server and Keeper reach each other over the same internal firewall, so the chart adds this release's own workloads to the list for you +- **A rolling upgrade of a 3-shard cluster is about 83 seconds of total unavailability** (measured), and changing a Keeper setting costs roughly 60 seconds of coordination outage. Plan both as query interruptions +- **Renaming `clusterName` orphans existing `Distributed` tables** — they keep pointing at the old cluster and fail with `Code: 701`. Recreate them after a rename +- **With one replica per shard, losing a shard fails every distributed query** (~60 s to surface), not just the rows on that shard. Add replicas if partial results are not acceptable ## Links diff --git a/clickhouse/versions/3.0.0/templates/policy.yaml b/clickhouse/versions/3.0.0/templates/policy.yaml index 03c45458..6c782969 100644 --- a/clickhouse/versions/3.0.0/templates/policy.yaml +++ b/clickhouse/versions/3.0.0/templates/policy.yaml @@ -10,7 +10,10 @@ bindings: targetKind: secret targetLinks: - //secret/{{ .Values.database.credentialsSecretName }} + {{- if not (include "clickhouse.isSingleNode" .) }} + {{- /* single-node renders no Keeper, so granting reveal on its secret is a dangling grant */}} - //secret/{{ include "clickhouse.secretKeeper.name" . }} + {{- end }} - //secret/{{ include "clickhouse.secretServer.name" . }} {{- if eq .Values.provider "gcp" }} - //secret/{{ .Values.gcp.credentialsSecretName }} diff --git a/clickhouse/versions/3.0.0/templates/workload-keeper.yaml b/clickhouse/versions/3.0.0/templates/workload-keeper.yaml index 552a44ef..89715e41 100644 --- a/clickhouse/versions/3.0.0/templates/workload-keeper.yaml +++ b/clickhouse/versions/3.0.0/templates/workload-keeper.yaml @@ -78,7 +78,37 @@ spec: internal: inboundAllowType: {{ .Values.keeper.internalAccess.type }} {{- $wl := .Values.keeper.internalAccess.workloads | default (list) }} - {{- if $wl }} + {{- if eq .Values.keeper.internalAccess.type "workload-list" }} + {{- /* + This list governs ALL inbound internal traffic, INCLUDING traffic between + the workloads of THIS release: Keeper's Raft on 9234, the servers' calls + to Keeper on 9181, and server-to-server replication and distributed + queries on 9000/9009. A list naming only clients cuts the cluster off + from itself -- measured 2026-08-27 by following the README: Raft denied + (`rbac_access_denied_matched_policy[none] ... "10.0.84.230:9234"`), + quorum lost, cross-shard queries failing with `Code: 279`, while + get-deployments reported every replica `ready=true restarts=0`. Adding + the release's own links restored `zk_synced_followers 2`. + + Both own links are included on both workloads rather than the minimum + each strictly needs. The over-grant is one release's workloads reaching + each other inside one GVC; the under-grant is a silently destroyed + cluster, and getting the direction wrong is exactly how this shipped. + */}} + {{- $own := list (printf "//gvc/%s/workload/%s" .Values.global.cpln.gvc (include "clickhouse.server.name" .)) }} + {{- if not (include "clickhouse.isSingleNode" .) }} + {{- $own = append $own (printf "//gvc/%s/workload/%s" .Values.global.cpln.gvc (include "clickhouse.keeper.name" .)) }} + {{- end }} + inboundAllowWorkload: + {{- range $own }} + - {{ . }} + {{- end }} + {{- range $wl }} + {{- if not (has . $own) }} + - {{ . }} + {{- end }} + {{- end }} + {{- else if $wl }} inboundAllowWorkload: {{- range $wl }} - {{ . }} diff --git a/clickhouse/versions/3.0.0/templates/workload-server.yaml b/clickhouse/versions/3.0.0/templates/workload-server.yaml index 8f2155e8..4324a172 100644 --- a/clickhouse/versions/3.0.0/templates/workload-server.yaml +++ b/clickhouse/versions/3.0.0/templates/workload-server.yaml @@ -125,7 +125,37 @@ spec: internal: inboundAllowType: {{ .Values.server.internalAccess.type }} {{- $wl := .Values.server.internalAccess.workloads | default (list) }} - {{- if $wl }} + {{- if eq .Values.server.internalAccess.type "workload-list" }} + {{- /* + This list governs ALL inbound internal traffic, INCLUDING traffic between + the workloads of THIS release: Keeper's Raft on 9234, the servers' calls + to Keeper on 9181, and server-to-server replication and distributed + queries on 9000/9009. A list naming only clients cuts the cluster off + from itself -- measured 2026-08-27 by following the README: Raft denied + (`rbac_access_denied_matched_policy[none] ... "10.0.84.230:9234"`), + quorum lost, cross-shard queries failing with `Code: 279`, while + get-deployments reported every replica `ready=true restarts=0`. Adding + the release's own links restored `zk_synced_followers 2`. + + Both own links are included on both workloads rather than the minimum + each strictly needs. The over-grant is one release's workloads reaching + each other inside one GVC; the under-grant is a silently destroyed + cluster, and getting the direction wrong is exactly how this shipped. + */}} + {{- $own := list (printf "//gvc/%s/workload/%s" .Values.global.cpln.gvc (include "clickhouse.server.name" .)) }} + {{- if not (include "clickhouse.isSingleNode" .) }} + {{- $own = append $own (printf "//gvc/%s/workload/%s" .Values.global.cpln.gvc (include "clickhouse.keeper.name" .)) }} + {{- end }} + inboundAllowWorkload: + {{- range $own }} + - {{ . }} + {{- end }} + {{- range $wl }} + {{- if not (has . $own) }} + - {{ . }} + {{- end }} + {{- end }} + {{- else if $wl }} inboundAllowWorkload: {{- range $wl }} - {{ . }} diff --git a/clickhouse/versions/3.0.0/values.yaml b/clickhouse/versions/3.0.0/values.yaml index 63d52573..077479c7 100644 --- a/clickhouse/versions/3.0.0/values.yaml +++ b/clickhouse/versions/3.0.0/values.yaml @@ -74,7 +74,8 @@ server: memory: 2Gi internalAccess: type: same-gvc # options: same-gvc, same-org, workload-list, none - workloads: [] # required when type is workload-list; e.g. //gvc/GVC_NAME/workload/WORKLOAD_NAME + workloads: [] # required when type is workload-list; list only your clients -- this release's + # own server and keeper workloads are added automatically, e.g. //gvc/GVC_NAME/workload/WORKLOAD_NAME # ─── Keeper (cluster modes only) ────────────────────────────────────────────── keeper: # cluster modes only @@ -84,4 +85,5 @@ keeper: # cluster modes only memory: 2Gi internalAccess: type: same-gvc # options: same-gvc, same-org, workload-list, none - workloads: [] # required when type is workload-list; e.g. //gvc/GVC_NAME/workload/WORKLOAD_NAME + workloads: [] # required when type is workload-list; list only your clients -- this release's + # own server and keeper workloads are added automatically, e.g. //gvc/GVC_NAME/workload/WORKLOAD_NAME