diff --git a/changelog/16.feature.md b/changelog/16.feature.md new file mode 100644 index 0000000..a650c57 --- /dev/null +++ b/changelog/16.feature.md @@ -0,0 +1 @@ +Add first-class ``concurrency`` and free-form ``extraArgs`` values to provider workers, rendered as Celery worker options after the ``--`` separator, and pin the esmvaltool worker to ``concurrency: 1`` to prevent node OOM kills when its Dask cluster fans out. diff --git a/helm/templates/providers/deployment.yaml b/helm/templates/providers/deployment.yaml index 31e158a..33d796d 100644 --- a/helm/templates/providers/deployment.yaml +++ b/helm/templates/providers/deployment.yaml @@ -2,6 +2,11 @@ {{- $spec := merge (deepCopy $.Values.defaults) $spec -}} {{- $args := list "celery" "start-worker" "--loglevel" "DEBUG" -}} {{- if ne $provider "orchestrator" -}} {{- $args = concat $args (list "--provider" $provider) -}} {{- end -}} +{{- /* Celery worker options passed after the `--` separator (e.g. --concurrency, --pool). */ -}} +{{- $workerArgs := list -}} +{{- if $spec.concurrency -}} {{- $workerArgs = append $workerArgs (printf "--concurrency=%v" $spec.concurrency) -}} {{- end -}} +{{- with $spec.extraArgs -}} {{- $workerArgs = concat $workerArgs . -}} {{- end -}} +{{- if $workerArgs -}} {{- $args = concat $args (list "--") $workerArgs -}} {{- end -}} {{- /* Chart-managed esmvaltool config: auto-mount when providers.esmvaltool.config is set. */ -}} {{- $volumes := $spec.volumes | default list -}} {{- $volumeMounts := $spec.volumeMounts | default list -}} diff --git a/helm/values.yaml b/helm/values.yaml index ab2474e..62041d1 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -173,6 +173,16 @@ defaults: replicaCount: 1 + # Number of Celery worker child processes per pod (passed as `--concurrency`). + # Leave unset to use Celery's default (one per CPU core). Set to 1 to pin a + # provider to a single worker process, e.g. for memory-heavy diagnostics that + # OOM when several run concurrently on the same node. + concurrency: null + + # Additional arguments appended to the Celery worker after the `--` separator, + # for options not exposed as first-class values (e.g. ["--pool=solo"]). + extraArgs: [] + image: repository: ghcr.io/climate-ref/climate-ref pullPolicy: IfNotPresent @@ -235,6 +245,12 @@ defaults: providers: orchestrator: {} esmvaltool: + # Pin the esmvaltool worker to a single Celery child process. Each esmvaltool + # execution already fans out via its own Dask cluster (see `config` below); + # running multiple Celery workers per pod multiplies that memory footprint and + # OOM-kills the node. + # See https://climate-ref.readthedocs.io/en/latest/how-to-guides/control-memory-use/ + concurrency: 1 # Esmvalcore configuration written to a ConfigMap and mounted at # /etc/esmvaltool/config.yaml. Without this, esmvalcore auto-detects all # available cores and runs unbounded dask workers, which routinely OOMs