From deac033ab6e946c5ec0c1bc1633ee986afe52ad5 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 11 Jun 2026 07:11:06 +0000 Subject: [PATCH 1/9] Add Helm chart and GHCR image publish workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Şahin Olut --- .github/workflows/docker-publish.yml | 58 +++++++ .github/workflows/helm.yml | 36 ++++ deploy/helm/gitmirrorcache/Chart.yaml | 14 ++ deploy/helm/gitmirrorcache/README.md | 82 ++++++++++ .../helm/gitmirrorcache/templates/NOTES.txt | 20 +++ .../gitmirrorcache/templates/_helpers.tpl | 151 +++++++++++++++++ .../gitmirrorcache/templates/configmap.yaml | 11 ++ .../templates/cronjob-compaction.yaml | 63 +++++++ .../gitmirrorcache/templates/ingress.yaml | 41 +++++ .../gitmirrorcache/templates/service.yaml | 15 ++ .../templates/serviceaccount.yaml | 12 ++ .../gitmirrorcache/templates/statefulset.yaml | 100 ++++++++++++ deploy/helm/gitmirrorcache/values.yaml | 154 ++++++++++++++++++ docs/deployment.md | 9 + 14 files changed, 766 insertions(+) create mode 100644 .github/workflows/docker-publish.yml create mode 100644 .github/workflows/helm.yml create mode 100644 deploy/helm/gitmirrorcache/Chart.yaml create mode 100644 deploy/helm/gitmirrorcache/README.md create mode 100644 deploy/helm/gitmirrorcache/templates/NOTES.txt create mode 100644 deploy/helm/gitmirrorcache/templates/_helpers.tpl create mode 100644 deploy/helm/gitmirrorcache/templates/configmap.yaml create mode 100644 deploy/helm/gitmirrorcache/templates/cronjob-compaction.yaml create mode 100644 deploy/helm/gitmirrorcache/templates/ingress.yaml create mode 100644 deploy/helm/gitmirrorcache/templates/service.yaml create mode 100644 deploy/helm/gitmirrorcache/templates/serviceaccount.yaml create mode 100644 deploy/helm/gitmirrorcache/templates/statefulset.yaml create mode 100644 deploy/helm/gitmirrorcache/values.yaml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000..749ddaa --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,58 @@ +name: Publish Docker Image + +on: + push: + branches: [main] + tags: ["v*"] + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + publish: + name: Build & push multi-arch image + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract image metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha + type=raw,value=latest,enable={{is_default_branch}} + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml new file mode 100644 index 0000000..7c97e82 --- /dev/null +++ b/.github/workflows/helm.yml @@ -0,0 +1,36 @@ +name: Helm Chart + +on: + push: + branches: [main] + paths: ["deploy/helm/**"] + pull_request: + branches: [main] + paths: ["deploy/helm/**"] + +jobs: + lint: + name: Lint & template + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + + - uses: azure/setup-helm@v4 + + - name: helm lint + run: helm lint deploy/helm/gitmirrorcache --set config.objectStore.s3.bucket=ci-bucket + + - name: helm template (s3 defaults) + run: | + helm template git-cache deploy/helm/gitmirrorcache \ + --set config.objectStore.s3.bucket=ci-bucket \ + --set aws.region=us-west-2 \ + --set ingress.enabled=true + + - name: helm template (local store, no persistence) + run: | + helm template git-cache deploy/helm/gitmirrorcache \ + --set config.objectStore.kind=local \ + --set persistence.enabled=false \ + --set compaction.enabled=false diff --git a/deploy/helm/gitmirrorcache/Chart.yaml b/deploy/helm/gitmirrorcache/Chart.yaml new file mode 100644 index 0000000..42c1d7d --- /dev/null +++ b/deploy/helm/gitmirrorcache/Chart.yaml @@ -0,0 +1,14 @@ +apiVersion: v2 +name: gitmirrorcache +description: Read-through Git mirroring and caching service backed by S3-compatible object storage. +type: application +version: 0.1.0 +appVersion: "latest" +home: https://github.com/0lut/gitmirrorcache +sources: + - https://github.com/0lut/gitmirrorcache +keywords: + - git + - cache + - mirror + - s3 diff --git a/deploy/helm/gitmirrorcache/README.md b/deploy/helm/gitmirrorcache/README.md new file mode 100644 index 0000000..c60be35 --- /dev/null +++ b/deploy/helm/gitmirrorcache/README.md @@ -0,0 +1,82 @@ +# gitmirrorcache Helm Chart + +Deploys the gitmirrorcache read-through Git caching service to Kubernetes. + +The server runs as a StatefulSet with a persistent volume mounted at `/cache` +(the hot cache). An S3-compatible object store remains the durable source of +truth, so the cache volume is disposable: losing it only forces rehydration. +An hourly CronJob runs `git-cache compact --all`, mirroring the EventBridge +compaction rule from the AWS deployment. + +## Install + +```sh +helm install git-cache deploy/helm/gitmirrorcache \ + --set config.objectStore.s3.bucket=my-git-cache-bucket \ + --set aws.region=us-west-2 +``` + +## S3 credentials + +Prefer workload identity over static keys: + +```yaml +serviceAccount: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/gitmirrorcache +``` + +Or reference an existing Secret with `AWS_ACCESS_KEY_ID` / +`AWS_SECRET_ACCESS_KEY` keys: + +```yaml +aws: + region: us-west-2 + existingSecret: git-cache-aws +``` + +For S3-compatible stores (MinIO, Cloudflare R2, ...), set +`config.objectStore.s3.endpoint`. + +## Upstream authentication + +To warm and serve private repositories, point `upstreamAuth.existingSecret` +at a Secret containing a token (key `token` by default): + +```yaml +upstreamAuth: + existingSecret: git-cache-github-token +``` + +## Key values + +| Value | Default | Description | +| --- | --- | --- | +| `image.repository` | `ghcr.io/0lut/gitmirrorcache` | Container image | +| `config.objectStore.kind` | `s3` | `s3` or `local` (testing only) | +| `config.objectStore.s3.bucket` | – | Required for `s3` | +| `config.allowedUpstreamHosts` | `[github.com]` | Upstream allowlist | +| `config.gitRemote.enabled` | `true` | Serve `/git/{host}/{owner}/{repo}.git` | +| `config.disk.quotaBytes` | 100 GiB | Hot-cache disk quota | +| `persistence.size` | `100Gi` | PVC size (keep ≥ disk quota) | +| `persistence.enabled` | `true` | Use a PVC; `false` falls back to emptyDir | +| `compaction.enabled` | `true` | Hourly `git-cache compact --all` CronJob | +| `configFile` | `""` | Optional full TOML config (see `config/production.example.toml`) | +| `config.extraEnv` | `[]` | Extra `GIT_CACHE_*` env vars | + +See `values.yaml` for the full list. + +## Using the cache + +```sh +kubectl port-forward svc/git-cache-gitmirrorcache 8080:80 +curl http://localhost:8080/healthz +git clone http://localhost:8080/git/github.com//.git +``` + +## Scaling + +Each replica keeps its own hot cache on its own PVC. Replicas coordinate only +through the object store; compaction/publish uses conditional PUTs on the +generation head, so multiple replicas and the compaction CronJob are safe to +run concurrently. diff --git a/deploy/helm/gitmirrorcache/templates/NOTES.txt b/deploy/helm/gitmirrorcache/templates/NOTES.txt new file mode 100644 index 0000000..e6c38d1 --- /dev/null +++ b/deploy/helm/gitmirrorcache/templates/NOTES.txt @@ -0,0 +1,20 @@ +gitmirrorcache has been deployed. + +Service endpoint (in-cluster): + http://{{ include "gitmirrorcache.fullname" . }}.{{ .Release.Namespace }}.svc:{{ .Values.service.port }} + +Quick checks: + kubectl -n {{ .Release.Namespace }} port-forward svc/{{ include "gitmirrorcache.fullname" . }} 8080:{{ .Values.service.port }} + curl http://localhost:8080/healthz + +{{- if .Values.config.gitRemote.enabled }} + +Clone through the cache: + git clone http://localhost:8080/git/github.com//.git +{{- end }} +{{- if and (eq .Values.config.objectStore.kind "s3") (not .Values.aws.existingSecret) }} + +NOTE: objectStore.kind is "s3" and no aws.existingSecret was provided. +Make sure pods can reach S3 via IRSA/pod identity (serviceAccount.annotations) +or node instance roles. +{{- end }} diff --git a/deploy/helm/gitmirrorcache/templates/_helpers.tpl b/deploy/helm/gitmirrorcache/templates/_helpers.tpl new file mode 100644 index 0000000..7da27ba --- /dev/null +++ b/deploy/helm/gitmirrorcache/templates/_helpers.tpl @@ -0,0 +1,151 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "gitmirrorcache.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +*/}} +{{- define "gitmirrorcache.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Chart label. +*/}} +{{- define "gitmirrorcache.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels. +*/}} +{{- define "gitmirrorcache.labels" -}} +helm.sh/chart: {{ include "gitmirrorcache.chart" . }} +{{ include "gitmirrorcache.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels. +*/}} +{{- define "gitmirrorcache.selectorLabels" -}} +app.kubernetes.io/name: {{ include "gitmirrorcache.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Service account name. +*/}} +{{- define "gitmirrorcache.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "gitmirrorcache.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Image reference. +*/}} +{{- define "gitmirrorcache.image" -}} +{{- printf "%s:%s" .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} +{{- end }} + +{{/* +Shared GIT_CACHE_* environment variables used by both the server and the +compaction CronJob. +*/}} +{{- define "gitmirrorcache.env" -}} +- name: GIT_CACHE_BIND_ADDR + value: {{ .Values.config.bindAddr | quote }} +- name: GIT_CACHE_ROOT + value: {{ .Values.config.cacheRoot | quote }} +- name: GIT_CACHE_GIT_BINARY + value: {{ .Values.config.gitBinary | quote }} +- name: GIT_CACHE_GIT_TIMEOUT_SECONDS + value: {{ .Values.config.gitTimeoutSeconds | quote }} +- name: GIT_CACHE_MAX_GIT_OUTPUT_BYTES + value: {{ .Values.config.maxGitOutputBytes | quote }} +- name: GIT_CACHE_RATE_LIMIT_PER_MINUTE + value: {{ .Values.config.rateLimitPerMinute | quote }} +- name: GIT_CACHE_ALLOWED_UPSTREAM_HOSTS + value: {{ join "," .Values.config.allowedUpstreamHosts | quote }} +- name: GIT_CACHE_MAX_CONCURRENT_GIT_PROCESSES + value: {{ .Values.config.maxConcurrentGitProcesses | quote }} +- name: GIT_CACHE_DISK_QUOTA_BYTES + value: {{ .Values.config.disk.quotaBytes | quote }} +- name: GIT_CACHE_DISK_MIN_FREE_BYTES + value: {{ .Values.config.disk.minFreeBytes | quote }} +- name: GIT_CACHE_GIT_REMOTE_ENABLED + value: {{ .Values.config.gitRemote.enabled | quote }} +- name: GIT_CACHE_COMPACTION_CHAIN_DEPTH_THRESHOLD + value: {{ .Values.config.compaction.chainDepthThreshold | quote }} +- name: GIT_CACHE_COMPACTION_INLINE + value: {{ .Values.config.compaction.inline | quote }} +- name: RUST_LOG + value: {{ .Values.config.logLevel | quote }} +{{- if eq .Values.config.objectStore.kind "s3" }} +- name: GIT_CACHE_OBJECT_STORE_KIND + value: "s3" +- name: GIT_CACHE_S3_BUCKET + value: {{ required "config.objectStore.s3.bucket is required when objectStore.kind is s3" .Values.config.objectStore.s3.bucket | quote }} +- name: GIT_CACHE_S3_PREFIX + value: {{ .Values.config.objectStore.s3.prefix | quote }} +{{- if .Values.config.objectStore.s3.endpoint }} +- name: GIT_CACHE_S3_ENDPOINT + value: {{ .Values.config.objectStore.s3.endpoint | quote }} +{{- end }} +{{- else }} +- name: GIT_CACHE_OBJECT_STORE_KIND + value: "local" +- name: GIT_CACHE_OBJECT_STORE_ROOT + value: {{ .Values.config.objectStore.local.root | quote }} +{{- end }} +{{- if .Values.aws.region }} +- name: AWS_REGION + value: {{ .Values.aws.region | quote }} +{{- end }} +{{- if .Values.aws.existingSecret }} +- name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ .Values.aws.existingSecret }} + key: AWS_ACCESS_KEY_ID +- name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.aws.existingSecret }} + key: AWS_SECRET_ACCESS_KEY +{{- end }} +{{- if .Values.upstreamAuth.existingSecret }} +- name: GIT_CACHE_UPSTREAM_AUTH_TOKEN_ENV + value: {{ .Values.upstreamAuth.tokenEnv | quote }} +- name: {{ .Values.upstreamAuth.tokenEnv }} + valueFrom: + secretKeyRef: + name: {{ .Values.upstreamAuth.existingSecret }} + key: {{ .Values.upstreamAuth.secretKey }} +{{- end }} +{{- if .Values.configFile }} +- name: GIT_CACHE_CONFIG + value: /etc/gitmirrorcache/config.toml +{{- end }} +{{- with .Values.config.extraEnv }} +{{ toYaml . }} +{{- end }} +{{- end }} diff --git a/deploy/helm/gitmirrorcache/templates/configmap.yaml b/deploy/helm/gitmirrorcache/templates/configmap.yaml new file mode 100644 index 0000000..eb0deee --- /dev/null +++ b/deploy/helm/gitmirrorcache/templates/configmap.yaml @@ -0,0 +1,11 @@ +{{- if .Values.configFile -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "gitmirrorcache.fullname" . }} + labels: + {{- include "gitmirrorcache.labels" . | nindent 4 }} +data: + config.toml: | + {{- .Values.configFile | nindent 4 }} +{{- end }} diff --git a/deploy/helm/gitmirrorcache/templates/cronjob-compaction.yaml b/deploy/helm/gitmirrorcache/templates/cronjob-compaction.yaml new file mode 100644 index 0000000..b00f1f8 --- /dev/null +++ b/deploy/helm/gitmirrorcache/templates/cronjob-compaction.yaml @@ -0,0 +1,63 @@ +{{- if .Values.compaction.enabled -}} +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "gitmirrorcache.fullname" . }}-compaction + labels: + {{- include "gitmirrorcache.labels" . | nindent 4 }} +spec: + schedule: {{ .Values.compaction.schedule | quote }} + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: {{ .Values.compaction.backoffLimit }} + template: + metadata: + labels: + {{- include "gitmirrorcache.selectorLabels" . | nindent 12 }} + app.kubernetes.io/component: compaction + spec: + restartPolicy: {{ .Values.compaction.restartPolicy }} + serviceAccountName: {{ include "gitmirrorcache.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 12 }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 12 }} + {{- end }} + containers: + - name: compaction + image: {{ include "gitmirrorcache.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + {{- toYaml .Values.securityContext | nindent 16 }} + command: ["/usr/local/bin/git-cache", "compact", "--all"] + env: + {{- include "gitmirrorcache.env" . | nindent 16 }} + # Compaction works against the object store; use scratch disk + # so the job does not contend with the server's hot cache. + - name: GIT_CACHE_ROOT + value: /tmp/compaction-cache + {{- with .Values.compaction.resources }} + resources: + {{- toYaml . | nindent 16 }} + {{- end }} + volumeMounts: + {{- if .Values.configFile }} + - name: config + mountPath: /etc/gitmirrorcache + readOnly: true + {{- end }} + - name: scratch + mountPath: /tmp/compaction-cache + volumes: + - name: scratch + emptyDir: {} + {{- if .Values.configFile }} + - name: config + configMap: + name: {{ include "gitmirrorcache.fullname" . }} + {{- end }} +{{- end }} diff --git a/deploy/helm/gitmirrorcache/templates/ingress.yaml b/deploy/helm/gitmirrorcache/templates/ingress.yaml new file mode 100644 index 0000000..b33fd84 --- /dev/null +++ b/deploy/helm/gitmirrorcache/templates/ingress.yaml @@ -0,0 +1,41 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "gitmirrorcache.fullname" . }} + labels: + {{- include "gitmirrorcache.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ include "gitmirrorcache.fullname" $ }} + port: + name: http + {{- end }} + {{- end }} +{{- end }} diff --git a/deploy/helm/gitmirrorcache/templates/service.yaml b/deploy/helm/gitmirrorcache/templates/service.yaml new file mode 100644 index 0000000..28196a6 --- /dev/null +++ b/deploy/helm/gitmirrorcache/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "gitmirrorcache.fullname" . }} + labels: + {{- include "gitmirrorcache.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "gitmirrorcache.selectorLabels" . | nindent 4 }} diff --git a/deploy/helm/gitmirrorcache/templates/serviceaccount.yaml b/deploy/helm/gitmirrorcache/templates/serviceaccount.yaml new file mode 100644 index 0000000..1e779a3 --- /dev/null +++ b/deploy/helm/gitmirrorcache/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "gitmirrorcache.serviceAccountName" . }} + labels: + {{- include "gitmirrorcache.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/deploy/helm/gitmirrorcache/templates/statefulset.yaml b/deploy/helm/gitmirrorcache/templates/statefulset.yaml new file mode 100644 index 0000000..20f3699 --- /dev/null +++ b/deploy/helm/gitmirrorcache/templates/statefulset.yaml @@ -0,0 +1,100 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "gitmirrorcache.fullname" . }} + labels: + {{- include "gitmirrorcache.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + serviceName: {{ include "gitmirrorcache.fullname" . }} + selector: + matchLabels: + {{- include "gitmirrorcache.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- if .Values.configFile }} + checksum/config: {{ .Values.configFile | sha256sum }} + {{- end }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "gitmirrorcache.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "gitmirrorcache.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: git-cache-api + image: {{ include "gitmirrorcache.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + ports: + - name: http + containerPort: {{ (split ":" .Values.config.bindAddr)._1 | int }} + protocol: TCP + env: + {{- include "gitmirrorcache.env" . | nindent 12 }} + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: cache + mountPath: {{ .Values.config.cacheRoot }} + {{- if .Values.configFile }} + - name: config + mountPath: /etc/gitmirrorcache + readOnly: true + {{- end }} + volumes: + {{- if .Values.configFile }} + - name: config + configMap: + name: {{ include "gitmirrorcache.fullname" . }} + {{- end }} + {{- if not .Values.persistence.enabled }} + - name: cache + emptyDir: {} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.persistence.enabled }} + volumeClaimTemplates: + - metadata: + name: cache + {{- with .Values.persistence.annotations }} + annotations: + {{- toYaml . | nindent 10 }} + {{- end }} + spec: + accessModes: + {{- toYaml .Values.persistence.accessModes | nindent 10 }} + {{- if .Values.persistence.storageClass }} + storageClassName: {{ .Values.persistence.storageClass | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + {{- end }} diff --git a/deploy/helm/gitmirrorcache/values.yaml b/deploy/helm/gitmirrorcache/values.yaml new file mode 100644 index 0000000..a44e4ad --- /dev/null +++ b/deploy/helm/gitmirrorcache/values.yaml @@ -0,0 +1,154 @@ +replicaCount: 1 + +image: + repository: ghcr.io/0lut/gitmirrorcache + pullPolicy: IfNotPresent + # Defaults to the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + create: true + # Useful for EKS IRSA, e.g. + # eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/gitmirrorcache + annotations: {} + name: "" + +podAnnotations: {} +podLabels: {} + +podSecurityContext: + fsGroup: 1000 + +securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + className: "" + annotations: {} + hosts: + - host: git-cache.example.com + paths: + - path: / + pathType: Prefix + tls: [] + +resources: {} +# resources: +# requests: +# cpu: "2" +# memory: 4Gi +# limits: +# memory: 8Gi + +nodeSelector: {} +tolerations: [] +affinity: {} + +# Persistent hot cache mounted at /cache. The cache is disposable from a +# correctness perspective (S3 is the durable source); losing the volume only +# forces rehydration. Disable persistence to fall back to an emptyDir. +persistence: + enabled: true + storageClass: "" + accessModes: + - ReadWriteOnce + size: 100Gi + annotations: {} + +# Application configuration. Each field maps to a GIT_CACHE_* environment +# variable understood by the server and CLI. +config: + bindAddr: "0.0.0.0:8080" + cacheRoot: /cache + gitBinary: git + gitTimeoutSeconds: 120 + maxGitOutputBytes: 16777216 + rateLimitPerMinute: 120 + allowedUpstreamHosts: + - github.com + maxConcurrentGitProcesses: 64 + logLevel: info + objectStore: + # "s3" or "local" + kind: s3 + s3: + bucket: "" + # Runtime appends the v2 schema suffix (e.g. "repos" stores under repos-v2). + prefix: repos + # Optional custom endpoint for S3-compatible stores (MinIO, R2, ...). + endpoint: "" + # Only used when kind is "local"; stored on the cache volume. + local: + root: /cache/object-store + disk: + quotaBytes: 107374182400 # 100 GiB; keep below the persistence size + minFreeBytes: 10737418240 # 10 GiB + gitRemote: + # Serve git clone/fetch directly via /git/{host}/{owner}/{repo}.git + enabled: true + compaction: + chainDepthThreshold: 10 + inline: false + # Extra environment variables appended verbatim to the container, e.g. to + # set GIT_CACHE_* knobs not surfaced above. + extraEnv: [] + # extraEnv: + # - name: GIT_CACHE_GIT_REMOTE_PROXY_ON_MISS_BY_DEFAULT + # value: "false" + +# Optional full TOML config mounted into the pod and pointed to by +# GIT_CACHE_CONFIG. When set, file values take precedence over env defaults +# for keys present in the file (see config/production.example.toml). +configFile: "" + +# Token used to authenticate against upstream hosts when warming/serving +# private repositories. +upstreamAuth: + # Name of an existing Secret containing the token. + existingSecret: "" + secretKey: token + # Environment variable the server reads the token from. + tokenEnv: GITHUB_TOKEN + +# Static AWS credentials for S3. Prefer IRSA / pod identity via +# serviceAccount.annotations and leave existingSecret empty. +aws: + region: "" + # Existing Secret with AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY keys. + existingSecret: "" + +# Hourly generation-chain compaction (git-cache compact --all), mirroring the +# EventBridge rule used by the AWS deployment. +compaction: + enabled: true + schedule: "0 * * * *" + resources: {} + # Concurrency-safe: publish uses conditional PUT on the generation head. + restartPolicy: Never + backoffLimit: 0 + +livenessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + +readinessProbe: + httpGet: + path: /healthz + port: http + initialDelaySeconds: 2 + periodSeconds: 5 diff --git a/docs/deployment.md b/docs/deployment.md index c056d43..341be1b 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -1,5 +1,14 @@ # Deployment Notes +## Kubernetes (Helm) + +A Helm chart for deploying to any Kubernetes cluster lives at +[`deploy/helm/gitmirrorcache`](../deploy/helm/gitmirrorcache/README.md). It runs +the server as a StatefulSet with a persistent `/cache` volume, an hourly +`git-cache compact --all` CronJob, and S3 (or any S3-compatible store) as the +durable source. Container images are published to +`ghcr.io/0lut/gitmirrorcache` by the `Publish Docker Image` workflow. + ## Current AWS Deployment Path Use **ECS on Graviton EC2 with host-mounted EBS** for large repository deployments. From 960409572006f19da3e86e52a86c5561192578c8 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 11 Jun 2026 07:17:50 +0000 Subject: [PATCH 2/9] Keep GIT_CACHE_CONFIG out of compaction CronJob env MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Şahin Olut --- .github/workflows/helm.yml | 10 ++++++++++ deploy/helm/gitmirrorcache/templates/_helpers.tpl | 8 +++----- .../gitmirrorcache/templates/cronjob-compaction.yaml | 12 ++---------- .../helm/gitmirrorcache/templates/statefulset.yaml | 4 ++++ deploy/helm/gitmirrorcache/values.yaml | 8 +++++--- 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 7c97e82..b95d5d3 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -34,3 +34,13 @@ jobs: --set config.objectStore.kind=local \ --set persistence.enabled=false \ --set compaction.enabled=false + + - name: helm template (configFile set; CronJob must stay env-configured) + run: | + out=$(helm template git-cache deploy/helm/gitmirrorcache \ + --set config.objectStore.s3.bucket=ci-bucket \ + --set-string configFile='cache_root = "/cache"') + echo "$out" | grep -q 'GIT_CACHE_CONFIG' || { echo "server missing GIT_CACHE_CONFIG"; exit 1; } + cronjob=$(echo "$out" | awk '/^kind: CronJob$/{f=1} /^---$/{f=0} f') + echo "$cronjob" | grep -q 'GIT_CACHE_CONFIG' && { echo "CronJob must not set GIT_CACHE_CONFIG"; exit 1; } + echo "ok" diff --git a/deploy/helm/gitmirrorcache/templates/_helpers.tpl b/deploy/helm/gitmirrorcache/templates/_helpers.tpl index 7da27ba..87fb09e 100644 --- a/deploy/helm/gitmirrorcache/templates/_helpers.tpl +++ b/deploy/helm/gitmirrorcache/templates/_helpers.tpl @@ -68,7 +68,9 @@ Image reference. {{/* Shared GIT_CACHE_* environment variables used by both the server and the -compaction CronJob. +compaction CronJob. GIT_CACHE_CONFIG is deliberately not set here: when it is +set the application reads the entire config from the TOML file and ignores +env vars, so only the server (which mounts the ConfigMap) opts into it. */}} {{- define "gitmirrorcache.env" -}} - name: GIT_CACHE_BIND_ADDR @@ -141,10 +143,6 @@ compaction CronJob. name: {{ .Values.upstreamAuth.existingSecret }} key: {{ .Values.upstreamAuth.secretKey }} {{- end }} -{{- if .Values.configFile }} -- name: GIT_CACHE_CONFIG - value: /etc/gitmirrorcache/config.toml -{{- end }} {{- with .Values.config.extraEnv }} {{ toYaml . }} {{- end }} diff --git a/deploy/helm/gitmirrorcache/templates/cronjob-compaction.yaml b/deploy/helm/gitmirrorcache/templates/cronjob-compaction.yaml index b00f1f8..d446853 100644 --- a/deploy/helm/gitmirrorcache/templates/cronjob-compaction.yaml +++ b/deploy/helm/gitmirrorcache/templates/cronjob-compaction.yaml @@ -34,6 +34,8 @@ spec: securityContext: {{- toYaml .Values.securityContext | nindent 16 }} command: ["/usr/local/bin/git-cache", "compact", "--all"] + # The CLI is configured purely via env vars (no GIT_CACHE_CONFIG + # here: a TOML config would override the scratch cache root). env: {{- include "gitmirrorcache.env" . | nindent 16 }} # Compaction works against the object store; use scratch disk @@ -45,19 +47,9 @@ spec: {{- toYaml . | nindent 16 }} {{- end }} volumeMounts: - {{- if .Values.configFile }} - - name: config - mountPath: /etc/gitmirrorcache - readOnly: true - {{- end }} - name: scratch mountPath: /tmp/compaction-cache volumes: - name: scratch emptyDir: {} - {{- if .Values.configFile }} - - name: config - configMap: - name: {{ include "gitmirrorcache.fullname" . }} - {{- end }} {{- end }} diff --git a/deploy/helm/gitmirrorcache/templates/statefulset.yaml b/deploy/helm/gitmirrorcache/templates/statefulset.yaml index 20f3699..5511fa3 100644 --- a/deploy/helm/gitmirrorcache/templates/statefulset.yaml +++ b/deploy/helm/gitmirrorcache/templates/statefulset.yaml @@ -44,6 +44,10 @@ spec: protocol: TCP env: {{- include "gitmirrorcache.env" . | nindent 12 }} + {{- if .Values.configFile }} + - name: GIT_CACHE_CONFIG + value: /etc/gitmirrorcache/config.toml + {{- end }} livenessProbe: {{- toYaml .Values.livenessProbe | nindent 12 }} readinessProbe: diff --git a/deploy/helm/gitmirrorcache/values.yaml b/deploy/helm/gitmirrorcache/values.yaml index a44e4ad..a2bacef 100644 --- a/deploy/helm/gitmirrorcache/values.yaml +++ b/deploy/helm/gitmirrorcache/values.yaml @@ -108,9 +108,11 @@ config: # - name: GIT_CACHE_GIT_REMOTE_PROXY_ON_MISS_BY_DEFAULT # value: "false" -# Optional full TOML config mounted into the pod and pointed to by -# GIT_CACHE_CONFIG. When set, file values take precedence over env defaults -# for keys present in the file (see config/production.example.toml). +# Optional full TOML config mounted into the server pod and pointed to by +# GIT_CACHE_CONFIG. When set, the server reads its ENTIRE config from this +# file and ignores GIT_CACHE_* env vars (see config/production.example.toml). +# The compaction CronJob always uses env-var config so it can keep its +# scratch cache root. configFile: "" # Token used to authenticate against upstream hosts when warming/serving From a20707ec0480358ca2c37b2d19a78b01ed81ae60 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 11 Jun 2026 07:18:19 +0000 Subject: [PATCH 3/9] Tighten CronJob GIT_CACHE_CONFIG CI assertion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Şahin Olut --- .github/workflows/helm.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index b95d5d3..3ed6d1f 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -40,7 +40,7 @@ jobs: out=$(helm template git-cache deploy/helm/gitmirrorcache \ --set config.objectStore.s3.bucket=ci-bucket \ --set-string configFile='cache_root = "/cache"') - echo "$out" | grep -q 'GIT_CACHE_CONFIG' || { echo "server missing GIT_CACHE_CONFIG"; exit 1; } + echo "$out" | grep -q 'name: GIT_CACHE_CONFIG' || { echo "server missing GIT_CACHE_CONFIG"; exit 1; } cronjob=$(echo "$out" | awk '/^kind: CronJob$/{f=1} /^---$/{f=0} f') - echo "$cronjob" | grep -q 'GIT_CACHE_CONFIG' && { echo "CronJob must not set GIT_CACHE_CONFIG"; exit 1; } + echo "$cronjob" | grep -q 'name: GIT_CACHE_CONFIG' && { echo "CronJob must not set GIT_CACHE_CONFIG"; exit 1; } echo "ok" From 2a7809de3d0a1e66498d65ee39dece78f2a622f6 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 11 Jun 2026 07:47:22 +0000 Subject: [PATCH 4/9] Set numeric runAsUser/fsGroup 999 to satisfy runAsNonRoot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Şahin Olut --- deploy/helm/gitmirrorcache/values.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/deploy/helm/gitmirrorcache/values.yaml b/deploy/helm/gitmirrorcache/values.yaml index a2bacef..776acae 100644 --- a/deploy/helm/gitmirrorcache/values.yaml +++ b/deploy/helm/gitmirrorcache/values.yaml @@ -21,10 +21,14 @@ podAnnotations: {} podLabels: {} podSecurityContext: - fsGroup: 1000 + fsGroup: 999 securityContext: runAsNonRoot: true + # The image runs as the system user "git-cache" (uid/gid 999). A numeric + # runAsUser is required for runAsNonRoot verification with named users. + runAsUser: 999 + runAsGroup: 999 allowPrivilegeEscalation: false capabilities: drop: ["ALL"] From 0b17556d929904e1e04baf9abc19c47a80bec3ae Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 11 Jun 2026 07:54:33 +0000 Subject: [PATCH 5/9] Dedupe CronJob GIT_CACHE_ROOT and render large ints with int64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Şahin Olut --- .../gitmirrorcache/templates/_helpers.tpl | 28 +++++++++++-------- .../templates/cronjob-compaction.yaml | 8 ++---- .../gitmirrorcache/templates/statefulset.yaml | 2 +- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/deploy/helm/gitmirrorcache/templates/_helpers.tpl b/deploy/helm/gitmirrorcache/templates/_helpers.tpl index 87fb09e..b9734fd 100644 --- a/deploy/helm/gitmirrorcache/templates/_helpers.tpl +++ b/deploy/helm/gitmirrorcache/templates/_helpers.tpl @@ -68,35 +68,40 @@ Image reference. {{/* Shared GIT_CACHE_* environment variables used by both the server and the -compaction CronJob. GIT_CACHE_CONFIG is deliberately not set here: when it is -set the application reads the entire config from the TOML file and ignores -env vars, so only the server (which mounts the ConfigMap) opts into it. +compaction CronJob. Takes a dict: "ctx" (the root context) and an optional +"cacheRoot" override so the CronJob can use a scratch directory. GIT_CACHE_CONFIG +is deliberately not set here: when it is set the application reads the entire +config from the TOML file and ignores env vars, so only the server (which +mounts the ConfigMap) opts into it. */}} {{- define "gitmirrorcache.env" -}} +{{- $ctx := .ctx -}} +{{- $cacheRoot := default $ctx.Values.config.cacheRoot .cacheRoot -}} +{{- with $ctx }} - name: GIT_CACHE_BIND_ADDR value: {{ .Values.config.bindAddr | quote }} - name: GIT_CACHE_ROOT - value: {{ .Values.config.cacheRoot | quote }} + value: {{ $cacheRoot | quote }} - name: GIT_CACHE_GIT_BINARY value: {{ .Values.config.gitBinary | quote }} - name: GIT_CACHE_GIT_TIMEOUT_SECONDS - value: {{ .Values.config.gitTimeoutSeconds | quote }} + value: {{ .Values.config.gitTimeoutSeconds | int64 | quote }} - name: GIT_CACHE_MAX_GIT_OUTPUT_BYTES - value: {{ .Values.config.maxGitOutputBytes | quote }} + value: {{ .Values.config.maxGitOutputBytes | int64 | quote }} - name: GIT_CACHE_RATE_LIMIT_PER_MINUTE - value: {{ .Values.config.rateLimitPerMinute | quote }} + value: {{ .Values.config.rateLimitPerMinute | int64 | quote }} - name: GIT_CACHE_ALLOWED_UPSTREAM_HOSTS value: {{ join "," .Values.config.allowedUpstreamHosts | quote }} - name: GIT_CACHE_MAX_CONCURRENT_GIT_PROCESSES - value: {{ .Values.config.maxConcurrentGitProcesses | quote }} + value: {{ .Values.config.maxConcurrentGitProcesses | int64 | quote }} - name: GIT_CACHE_DISK_QUOTA_BYTES - value: {{ .Values.config.disk.quotaBytes | quote }} + value: {{ .Values.config.disk.quotaBytes | int64 | quote }} - name: GIT_CACHE_DISK_MIN_FREE_BYTES - value: {{ .Values.config.disk.minFreeBytes | quote }} + value: {{ .Values.config.disk.minFreeBytes | int64 | quote }} - name: GIT_CACHE_GIT_REMOTE_ENABLED value: {{ .Values.config.gitRemote.enabled | quote }} - name: GIT_CACHE_COMPACTION_CHAIN_DEPTH_THRESHOLD - value: {{ .Values.config.compaction.chainDepthThreshold | quote }} + value: {{ .Values.config.compaction.chainDepthThreshold | int64 | quote }} - name: GIT_CACHE_COMPACTION_INLINE value: {{ .Values.config.compaction.inline | quote }} - name: RUST_LOG @@ -147,3 +152,4 @@ env vars, so only the server (which mounts the ConfigMap) opts into it. {{ toYaml . }} {{- end }} {{- end }} +{{- end }} diff --git a/deploy/helm/gitmirrorcache/templates/cronjob-compaction.yaml b/deploy/helm/gitmirrorcache/templates/cronjob-compaction.yaml index d446853..80eed14 100644 --- a/deploy/helm/gitmirrorcache/templates/cronjob-compaction.yaml +++ b/deploy/helm/gitmirrorcache/templates/cronjob-compaction.yaml @@ -36,12 +36,10 @@ spec: command: ["/usr/local/bin/git-cache", "compact", "--all"] # The CLI is configured purely via env vars (no GIT_CACHE_CONFIG # here: a TOML config would override the scratch cache root). + # Compaction works against the object store; use scratch disk + # so the job does not contend with the server's hot cache. env: - {{- include "gitmirrorcache.env" . | nindent 16 }} - # Compaction works against the object store; use scratch disk - # so the job does not contend with the server's hot cache. - - name: GIT_CACHE_ROOT - value: /tmp/compaction-cache + {{- include "gitmirrorcache.env" (dict "ctx" . "cacheRoot" "/tmp/compaction-cache") | nindent 16 }} {{- with .Values.compaction.resources }} resources: {{- toYaml . | nindent 16 }} diff --git a/deploy/helm/gitmirrorcache/templates/statefulset.yaml b/deploy/helm/gitmirrorcache/templates/statefulset.yaml index 5511fa3..de04810 100644 --- a/deploy/helm/gitmirrorcache/templates/statefulset.yaml +++ b/deploy/helm/gitmirrorcache/templates/statefulset.yaml @@ -43,7 +43,7 @@ spec: containerPort: {{ (split ":" .Values.config.bindAddr)._1 | int }} protocol: TCP env: - {{- include "gitmirrorcache.env" . | nindent 12 }} + {{- include "gitmirrorcache.env" (dict "ctx" .) | nindent 12 }} {{- if .Values.configFile }} - name: GIT_CACHE_CONFIG value: /etc/gitmirrorcache/config.toml From 23e7a9825c92cb9abe3f52cdab1977ccd3855353 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 11 Jun 2026 08:39:55 +0000 Subject: [PATCH 6/9] Add graceful shutdown: SIGTERM drain with readiness gate and chart grace period MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Şahin Olut --- config/production.example.toml | 7 ++ crates/git-cache-api/src/lib.rs | 79 ++++++++++++++-- crates/git-cache-api/src/main.rs | 91 ++++++++++++++++++- crates/git-cache-api/tests/support/mod.rs | 1 + crates/git-cache-core/src/config.rs | 42 +++++++++ crates/git-cache-core/src/lib.rs | 2 +- .../src/materializer/tests/mod.rs | 1 + crates/git-cache-domain/src/state.rs | 1 + deploy/helm/gitmirrorcache/README.md | 3 + .../gitmirrorcache/templates/_helpers.tpl | 4 + .../gitmirrorcache/templates/statefulset.yaml | 1 + deploy/helm/gitmirrorcache/values.yaml | 12 +++ 12 files changed, 234 insertions(+), 10 deletions(-) diff --git a/config/production.example.toml b/config/production.example.toml index 6f034df..2a53e7c 100644 --- a/config/production.example.toml +++ b/config/production.example.toml @@ -23,3 +23,10 @@ min_free_bytes = 107374182400 [compaction] chain_depth_threshold = 10 inline = false + +# Graceful shutdown: after SIGTERM the server fails /healthz readiness for +# readiness_delay_seconds, then drains in-flight requests for up to +# drain_timeout_seconds before exiting. +[shutdown] +readiness_delay_seconds = 5 +drain_timeout_seconds = 60 diff --git a/crates/git-cache-api/src/lib.rs b/crates/git-cache-api/src/lib.rs index 09c2ed5..65b3fab 100644 --- a/crates/git-cache-api/src/lib.rs +++ b/crates/git-cache-api/src/lib.rs @@ -22,7 +22,7 @@ use sha2::{Digest, Sha256}; use std::collections::{HashMap, HashSet}; use std::future::Future; use std::pin::Pin; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use std::task::{Context, Poll}; use std::time::{Duration, Instant}; @@ -54,9 +54,28 @@ pub fn app_result(config: AppConfig) -> CoreResult { } pub async fn app_result_async(config: AppConfig) -> CoreResult { + Ok(app_with_shutdown_async(config).await?.0) +} + +/// Like [`app_result_async`], but also returns a [`ReadinessGate`] that the +/// caller can flip during shutdown so `/healthz` starts failing and load +/// balancers stop routing new traffic while in-flight requests drain. +pub async fn app_with_shutdown_async(config: AppConfig) -> CoreResult<(Router, ReadinessGate)> { let git_remote_enabled = config.git_remote.enabled; let state = Arc::new(ApiState::try_new_async(config).await?); - router(git_remote_enabled, state) + let gate = ReadinessGate(Arc::clone(&state.shutting_down)); + Ok((router(git_remote_enabled, state)?, gate)) +} + +/// Handle that marks the server as shutting down; once flipped, `/healthz` +/// returns 503 so orchestrators stop sending new traffic. +#[derive(Clone, Debug)] +pub struct ReadinessGate(Arc); + +impl ReadinessGate { + pub fn begin_shutdown(&self) { + self.0.store(true, Ordering::SeqCst); + } } fn router(git_remote_enabled: bool, state: Arc) -> CoreResult { @@ -86,6 +105,7 @@ struct ApiState { upstream_http: reqwest::Client, metrics: Arc, rate_limiter: Arc, + shutting_down: Arc, } impl ApiState { @@ -124,6 +144,7 @@ impl ApiState { upstream_http, metrics: Arc::new(Metrics::default()), rate_limiter: Arc::new(rate_limiter), + shutting_down: Arc::new(AtomicBool::new(false)), }) } @@ -245,11 +266,17 @@ fn hex_lower(bytes: &[u8]) -> String { out } -async fn healthz() -> Json { - Json(HealthResponse { - ok: true, +async fn healthz(State(state): State>) -> Response { + let shutting_down = state.shutting_down.load(Ordering::SeqCst); + let body = Json(HealthResponse { + ok: !shutting_down, checked_at: chrono::Utc::now(), - }) + }); + if shutting_down { + (StatusCode::SERVICE_UNAVAILABLE, body).into_response() + } else { + body.into_response() + } } async fn metrics(State(state): State>) -> Response { @@ -1866,6 +1893,7 @@ mod tests { }, git_remote: Default::default(), compaction: Default::default(), + shutdown: Default::default(), max_concurrent_git_processes: git_cache_core::default_max_concurrent_git_processes(), async_materialize_concurrency: git_cache_core::default_async_materialize_concurrency(), use_gitoxide: true, @@ -1890,6 +1918,45 @@ mod tests { } } + #[tokio::test] + async fn healthz_fails_after_shutdown_begins() { + let tmp = TempDir::new().unwrap(); + let config = AppConfig { + bind_addr: "127.0.0.1:0".parse::().unwrap(), + cache_root: tmp.path().join("cache"), + upstream_root: None, + git_binary: PathBuf::from("git"), + git_timeout_seconds: 60, + max_git_output_bytes: 16 * 1024 * 1024, + object_store: ObjectStoreConfig::Local { + root: tmp.path().join("objects"), + }, + upstream_auth_token_env: None, + rate_limit_per_minute: 0, + allowed_upstream_hosts: vec!["github.com".into()], + disk: git_cache_core::DiskConfig { + quota_bytes: 1024 * 1024 * 1024, + min_free_bytes: 0, + }, + git_remote: Default::default(), + compaction: Default::default(), + shutdown: Default::default(), + max_concurrent_git_processes: git_cache_core::default_max_concurrent_git_processes(), + async_materialize_concurrency: git_cache_core::default_async_materialize_concurrency(), + use_gitoxide: true, + }; + let state = Arc::new(ApiState::try_new(config).unwrap()); + let gate = ReadinessGate(Arc::clone(&state.shutting_down)); + + let response = healthz(State(Arc::clone(&state))).await; + assert_eq!(response.status(), StatusCode::OK); + + gate.begin_shutdown(); + + let response = healthz(State(state)).await; + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + } + #[tokio::test] async fn upload_pack_stream_times_out_when_reader_stays_pending() { let (reader, _writer) = duplex(64); diff --git a/crates/git-cache-api/src/main.rs b/crates/git-cache-api/src/main.rs index 4e28157..b527139 100644 --- a/crates/git-cache-api/src/main.rs +++ b/crates/git-cache-api/src/main.rs @@ -1,7 +1,8 @@ -use git_cache_api::app_result_async; +use git_cache_api::{app_with_shutdown_async, ReadinessGate}; use git_cache_core::AppConfig; +use std::time::Duration; use tokio::net::TcpListener; -use tracing::info; +use tracing::{info, warn}; #[tokio::main] async fn main() -> anyhow::Result<()> { @@ -10,9 +11,93 @@ async fn main() -> anyhow::Result<()> { .init(); let config = AppConfig::from_env()?; + let shutdown_config = config.shutdown.clone(); let listener = TcpListener::bind(config.bind_addr).await?; info!(addr = %config.bind_addr, "starting git-cache-api"); - axum::serve(listener, app_result_async(config).await?).await?; + let (app, readiness) = app_with_shutdown_async(config).await?; + + let readiness_delay = Duration::from_secs(shutdown_config.readiness_delay_seconds); + let drain_timeout = Duration::from_secs(shutdown_config.drain_timeout_seconds); + let (drain_deadline_tx, drain_deadline_rx) = tokio::sync::oneshot::channel::<()>(); + + let server = axum::serve(listener, app).with_graceful_shutdown(graceful_shutdown( + readiness, + readiness_delay, + drain_timeout, + drain_deadline_tx, + )); + + tokio::select! { + result = server => result?, + _ = wait_for_drain_deadline(drain_deadline_rx, drain_timeout) => { + warn!( + drain_timeout_seconds = drain_timeout.as_secs(), + "drain timeout elapsed with requests still in flight; exiting" + ); + } + } Ok(()) } + +/// Resolves once the process should stop accepting new connections: after a +/// SIGTERM/SIGINT is received, readiness is failed, and the configured +/// readiness propagation delay has passed. Signals `drain_deadline_tx` so the +/// caller can bound the remaining in-flight drain. +async fn graceful_shutdown( + readiness: ReadinessGate, + readiness_delay: Duration, + drain_timeout: Duration, + drain_deadline_tx: tokio::sync::oneshot::Sender<()>, +) { + shutdown_signal().await; + readiness.begin_shutdown(); + info!( + readiness_delay_seconds = readiness_delay.as_secs(), + drain_timeout_seconds = drain_timeout.as_secs(), + "shutdown signal received; failing readiness, then draining in-flight requests" + ); + tokio::time::sleep(readiness_delay).await; + let _ = drain_deadline_tx.send(()); +} + +async fn wait_for_drain_deadline( + drain_started: tokio::sync::oneshot::Receiver<()>, + drain_timeout: Duration, +) { + if drain_started.await.is_err() { + // Server finished before shutdown began; never force-exit. + std::future::pending::<()>().await; + } + tokio::time::sleep(drain_timeout).await; +} + +async fn shutdown_signal() { + let ctrl_c = async { + if let Err(err) = tokio::signal::ctrl_c().await { + warn!(%err, "failed to install SIGINT handler"); + std::future::pending::<()>().await; + } + }; + + #[cfg(unix)] + let terminate = async { + match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) { + Ok(mut signal) => { + signal.recv().await; + } + Err(err) => { + warn!(%err, "failed to install SIGTERM handler"); + std::future::pending::<()>().await; + } + } + }; + + #[cfg(not(unix))] + let terminate = std::future::pending::<()>(); + + tokio::select! { + _ = ctrl_c => {}, + _ = terminate => {}, + } +} diff --git a/crates/git-cache-api/tests/support/mod.rs b/crates/git-cache-api/tests/support/mod.rs index 86da39f..a5909b3 100644 --- a/crates/git-cache-api/tests/support/mod.rs +++ b/crates/git-cache-api/tests/support/mod.rs @@ -36,6 +36,7 @@ pub fn test_config_with_upstream( ..Default::default() }, compaction: Default::default(), + shutdown: Default::default(), max_concurrent_git_processes: git_cache_core::default_max_concurrent_git_processes(), async_materialize_concurrency: git_cache_core::default_async_materialize_concurrency(), use_gitoxide: true, diff --git a/crates/git-cache-core/src/config.rs b/crates/git-cache-core/src/config.rs index 8912649..35343a0 100644 --- a/crates/git-cache-core/src/config.rs +++ b/crates/git-cache-core/src/config.rs @@ -28,6 +28,8 @@ pub struct AppConfig { pub git_remote: GitRemoteConfig, #[serde(default)] pub compaction: CompactionConfig, + #[serde(default)] + pub shutdown: ShutdownConfig, #[serde(default = "default_max_concurrent_git_processes")] pub max_concurrent_git_processes: usize, #[serde(default = "default_async_materialize_concurrency")] @@ -113,6 +115,16 @@ impl AppConfig { )?, inline: parse_bool_env("GIT_CACHE_COMPACTION_INLINE", false)?, }, + shutdown: ShutdownConfig { + readiness_delay_seconds: parse_env( + "GIT_CACHE_SHUTDOWN_READINESS_DELAY_SECONDS", + default_shutdown_readiness_delay_seconds(), + )?, + drain_timeout_seconds: parse_env( + "GIT_CACHE_SHUTDOWN_DRAIN_TIMEOUT_SECONDS", + default_shutdown_drain_timeout_seconds(), + )?, + }, max_concurrent_git_processes: parse_env( "GIT_CACHE_MAX_CONCURRENT_GIT_PROCESSES", default_max_concurrent_git_processes(), @@ -205,6 +217,28 @@ impl Default for CompactionConfig { } } +/// Graceful shutdown behavior for the API server. On SIGTERM/SIGINT the +/// server first fails readiness (`/healthz` returns 503) for +/// `readiness_delay_seconds` so load balancers stop routing new traffic, +/// then stops accepting connections and drains in-flight requests for up to +/// `drain_timeout_seconds` before exiting. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ShutdownConfig { + #[serde(default = "default_shutdown_readiness_delay_seconds")] + pub readiness_delay_seconds: u64, + #[serde(default = "default_shutdown_drain_timeout_seconds")] + pub drain_timeout_seconds: u64, +} + +impl Default for ShutdownConfig { + fn default() -> Self { + Self { + readiness_delay_seconds: default_shutdown_readiness_delay_seconds(), + drain_timeout_seconds: default_shutdown_drain_timeout_seconds(), + } + } +} + fn default_compaction_threshold() -> u32 { 10 } @@ -278,6 +312,14 @@ pub fn default_use_gitoxide() -> bool { true } +fn default_shutdown_readiness_delay_seconds() -> u64 { + 5 +} + +fn default_shutdown_drain_timeout_seconds() -> u64 { + 60 +} + fn parse_env(name: &str, default: T) -> crate::Result where T::Err: std::fmt::Display, diff --git a/crates/git-cache-core/src/lib.rs b/crates/git-cache-core/src/lib.rs index d42c6ae..d9e6c93 100644 --- a/crates/git-cache-core/src/lib.rs +++ b/crates/git-cache-core/src/lib.rs @@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize}; pub use auth::{SecretString, UpstreamAuth, UpstreamAuthorizationMode}; pub use config::{ default_async_materialize_concurrency, default_max_concurrent_git_processes, AppConfig, - CompactionConfig, DiskConfig, GitRemoteConfig, ObjectStoreConfig, + CompactionConfig, DiskConfig, GitRemoteConfig, ObjectStoreConfig, ShutdownConfig, }; pub use error::{GitCacheError, Result}; pub use manifest::{ diff --git a/crates/git-cache-domain/src/materializer/tests/mod.rs b/crates/git-cache-domain/src/materializer/tests/mod.rs index 06c92ca..23cda06 100644 --- a/crates/git-cache-domain/src/materializer/tests/mod.rs +++ b/crates/git-cache-domain/src/materializer/tests/mod.rs @@ -137,6 +137,7 @@ impl GitFixture { }, git_remote: Default::default(), compaction: Default::default(), + shutdown: Default::default(), max_concurrent_git_processes: git_cache_core::default_max_concurrent_git_processes(), async_materialize_concurrency: 2, use_gitoxide: true, diff --git a/crates/git-cache-domain/src/state.rs b/crates/git-cache-domain/src/state.rs index b683dd5..008f6b8 100644 --- a/crates/git-cache-domain/src/state.rs +++ b/crates/git-cache-domain/src/state.rs @@ -396,6 +396,7 @@ mod tests { }, git_remote: Default::default(), compaction: Default::default(), + shutdown: Default::default(), max_concurrent_git_processes: 1, async_materialize_concurrency: 2, use_gitoxide: true, diff --git a/deploy/helm/gitmirrorcache/README.md b/deploy/helm/gitmirrorcache/README.md index c60be35..ca7916b 100644 --- a/deploy/helm/gitmirrorcache/README.md +++ b/deploy/helm/gitmirrorcache/README.md @@ -63,6 +63,9 @@ upstreamAuth: | `compaction.enabled` | `true` | Hourly `git-cache compact --all` CronJob | | `configFile` | `""` | Optional full TOML config (see `config/production.example.toml`) | | `config.extraEnv` | `[]` | Extra `GIT_CACHE_*` env vars | +| `config.shutdown.readinessDelaySeconds` | `5` | Failing-readiness window after SIGTERM before draining | +| `config.shutdown.drainTimeoutSeconds` | `60` | Max in-flight drain time before exit | +| `terminationGracePeriodSeconds` | `75` | Keep > readiness delay + drain timeout | See `values.yaml` for the full list. diff --git a/deploy/helm/gitmirrorcache/templates/_helpers.tpl b/deploy/helm/gitmirrorcache/templates/_helpers.tpl index b9734fd..a3b1212 100644 --- a/deploy/helm/gitmirrorcache/templates/_helpers.tpl +++ b/deploy/helm/gitmirrorcache/templates/_helpers.tpl @@ -104,6 +104,10 @@ mounts the ConfigMap) opts into it. value: {{ .Values.config.compaction.chainDepthThreshold | int64 | quote }} - name: GIT_CACHE_COMPACTION_INLINE value: {{ .Values.config.compaction.inline | quote }} +- name: GIT_CACHE_SHUTDOWN_READINESS_DELAY_SECONDS + value: {{ .Values.config.shutdown.readinessDelaySeconds | int64 | quote }} +- name: GIT_CACHE_SHUTDOWN_DRAIN_TIMEOUT_SECONDS + value: {{ .Values.config.shutdown.drainTimeoutSeconds | int64 | quote }} - name: RUST_LOG value: {{ .Values.config.logLevel | quote }} {{- if eq .Values.config.objectStore.kind "s3" }} diff --git a/deploy/helm/gitmirrorcache/templates/statefulset.yaml b/deploy/helm/gitmirrorcache/templates/statefulset.yaml index de04810..bff9107 100644 --- a/deploy/helm/gitmirrorcache/templates/statefulset.yaml +++ b/deploy/helm/gitmirrorcache/templates/statefulset.yaml @@ -30,6 +30,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} serviceAccountName: {{ include "gitmirrorcache.serviceAccountName" . }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} containers: diff --git a/deploy/helm/gitmirrorcache/values.yaml b/deploy/helm/gitmirrorcache/values.yaml index 776acae..5ced3cb 100644 --- a/deploy/helm/gitmirrorcache/values.yaml +++ b/deploy/helm/gitmirrorcache/values.yaml @@ -60,6 +60,10 @@ nodeSelector: {} tolerations: [] affinity: {} +# Must exceed config.shutdown.readinessDelaySeconds + drainTimeoutSeconds so +# Kubernetes does not SIGKILL the pod while it is still draining. +terminationGracePeriodSeconds: 75 + # Persistent hot cache mounted at /cache. The cache is disposable from a # correctness perspective (S3 is the durable source); losing the volume only # forces rehydration. Disable persistence to fall back to an emptyDir. @@ -105,6 +109,14 @@ config: compaction: chainDepthThreshold: 10 inline: false + # Graceful shutdown: on SIGTERM the server fails /healthz readiness for + # readinessDelaySeconds (so endpoints drop out of rotation), then stops + # accepting connections and drains in-flight requests for up to + # drainTimeoutSeconds before exiting. Keep terminationGracePeriodSeconds + # above readinessDelaySeconds + drainTimeoutSeconds. + shutdown: + readinessDelaySeconds: 5 + drainTimeoutSeconds: 60 # Extra environment variables appended verbatim to the container, e.g. to # set GIT_CACHE_* knobs not surfaced above. extraEnv: [] From 38ecf2ed22fdf11d3783eb93f6c971868ca1c93b Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 11 Jun 2026 18:27:12 +0000 Subject: [PATCH 7/9] Set default server resources and document sizing guidance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Şahin Olut --- deploy/helm/gitmirrorcache/README.md | 9 +++++++++ deploy/helm/gitmirrorcache/values.yaml | 18 +++++++++++------- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/deploy/helm/gitmirrorcache/README.md b/deploy/helm/gitmirrorcache/README.md index c60be35..59811c1 100644 --- a/deploy/helm/gitmirrorcache/README.md +++ b/deploy/helm/gitmirrorcache/README.md @@ -74,6 +74,15 @@ curl http://localhost:8080/healthz git clone http://localhost:8080/git/github.com//.git ``` +## Sizing + +Serving clones spawns `git pack-objects` children, which are CPU- and +memory-intensive (bounded by `config.maxConcurrentGitProcesses`, default 64). +The chart defaults to requests of 2 vCPU / 4 GiB with an 8 GiB memory limit — +treat that as the floor. The production ECS deployment runs 8 vCPU / 24 GiB; +scale CPU with concurrent clone traffic and tune +`config.maxConcurrentGitProcesses` to match the allocation. + ## Scaling Each replica keeps its own hot cache on its own PVC. Replicas coordinate only diff --git a/deploy/helm/gitmirrorcache/values.yaml b/deploy/helm/gitmirrorcache/values.yaml index 776acae..726cd08 100644 --- a/deploy/helm/gitmirrorcache/values.yaml +++ b/deploy/helm/gitmirrorcache/values.yaml @@ -48,13 +48,17 @@ ingress: pathType: Prefix tls: [] -resources: {} -# resources: -# requests: -# cpu: "2" -# memory: 4Gi -# limits: -# memory: 8Gi +# Serving upload-pack spawns git pack-objects children (CPU- and memory-heavy, +# up to config.maxConcurrentGitProcesses at once), so do not run this small. +# The production ECS deployment uses 8 vCPU / 24 GiB (docs/deployment.md); +# treat ~2 vCPU / 4 GiB as the floor for light workloads and scale CPU with +# concurrent clone traffic. +resources: + requests: + cpu: "2" + memory: 4Gi + limits: + memory: 8Gi nodeSelector: {} tolerations: [] From ed81453f4c71cfa02b122d2f3fee195af96fafe9 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 11 Jun 2026 18:41:39 +0000 Subject: [PATCH 8/9] Add shutdown field to fuzz test AppConfig MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Şahin Olut --- crates/git-cache-fuzz/tests/materializer_fuzz.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/git-cache-fuzz/tests/materializer_fuzz.rs b/crates/git-cache-fuzz/tests/materializer_fuzz.rs index 02481d9..3b51ac6 100644 --- a/crates/git-cache-fuzz/tests/materializer_fuzz.rs +++ b/crates/git-cache-fuzz/tests/materializer_fuzz.rs @@ -54,6 +54,7 @@ impl Fixture { }, git_remote: Default::default(), compaction: Default::default(), + shutdown: Default::default(), max_concurrent_git_processes: 4, async_materialize_concurrency: 2, use_gitoxide: true, From 2ff739ab85542388cee0fb95eaa9a59bc5061c51 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 11 Jun 2026 20:18:16 +0000 Subject: [PATCH 9/9] Add integration tests for graceful shutdown drain behavior MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Şahin Olut --- crates/git-cache-api/src/lib.rs | 135 ++++++++++++++++++++++++++++++-- 1 file changed, 128 insertions(+), 7 deletions(-) diff --git a/crates/git-cache-api/src/lib.rs b/crates/git-cache-api/src/lib.rs index 81c1ec5..7729576 100644 --- a/crates/git-cache-api/src/lib.rs +++ b/crates/git-cache-api/src/lib.rs @@ -197,6 +197,33 @@ pub async fn serve(listener: tokio::net::TcpListener, config: AppConfig) -> Core let readiness_delay = Duration::from_secs(shutdown_config.readiness_delay_seconds); let drain_timeout = Duration::from_secs(shutdown_config.drain_timeout_seconds); + run_until_shutdown( + listener, + app, + readiness, + readiness_delay, + drain_timeout, + shutdown_signal(), + ) + .await?; + + if let Err(err) = domain.disk.flush_repo_accesses().await { + warn!(error = %err, "failed to flush repo access timestamps during shutdown"); + } + Ok(()) +} + +/// Serve `app` on `listener` until `signal` resolves, then drain: fail +/// readiness, wait `readiness_delay`, stop accepting connections, and let +/// in-flight requests finish for at most `drain_timeout` before returning. +async fn run_until_shutdown( + listener: tokio::net::TcpListener, + app: Router, + readiness: ReadinessGate, + readiness_delay: Duration, + drain_timeout: Duration, + signal: impl std::future::Future + Send + 'static, +) -> CoreResult<()> { let (drain_deadline_tx, drain_deadline_rx) = tokio::sync::oneshot::channel::<()>(); let server = axum::serve(listener, app).with_graceful_shutdown(graceful_shutdown( @@ -204,6 +231,7 @@ pub async fn serve(listener: tokio::net::TcpListener, config: AppConfig) -> Core readiness_delay, drain_timeout, drain_deadline_tx, + signal, )); tokio::select! { @@ -217,15 +245,11 @@ pub async fn serve(listener: tokio::net::TcpListener, config: AppConfig) -> Core ); } } - - if let Err(err) = domain.disk.flush_repo_accesses().await { - warn!(error = %err, "failed to flush repo access timestamps during shutdown"); - } Ok(()) } -/// Resolves once the process should stop accepting new connections: after a -/// SIGTERM/SIGINT is received, readiness is failed, and the configured +/// Resolves once the process should stop accepting new connections: after the +/// shutdown signal is received, readiness is failed, and the configured /// readiness propagation delay has passed. Signals `drain_deadline_tx` so the /// caller can bound the remaining in-flight drain. async fn graceful_shutdown( @@ -233,8 +257,9 @@ async fn graceful_shutdown( readiness_delay: Duration, drain_timeout: Duration, drain_deadline_tx: tokio::sync::oneshot::Sender<()>, + signal: impl std::future::Future, ) { - shutdown_signal().await; + signal.await; readiness.begin_shutdown(); info!( readiness_delay_seconds = readiness_delay.as_secs(), @@ -2088,6 +2113,102 @@ mod tests { assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); } + /// Starts `run_until_shutdown` on an ephemeral port with a single `/slow` + /// route that sleeps for `handler_delay` before responding. Returns the + /// bound address, the shutdown flag readable by the test, a sender that + /// triggers shutdown, and the server task handle. + async fn spawn_drain_server( + handler_delay: Duration, + drain_timeout: Duration, + ) -> ( + SocketAddr, + Arc, + tokio::sync::oneshot::Sender<()>, + tokio::task::JoinHandle>, + ) { + let shutting_down = Arc::new(AtomicBool::new(false)); + let gate = ReadinessGate(Arc::clone(&shutting_down)); + let app = Router::new().route( + "/slow", + get(move || async move { + tokio::time::sleep(handler_delay).await; + "done" + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>(); + let server = tokio::spawn(run_until_shutdown( + listener, + app, + gate, + Duration::ZERO, + drain_timeout, + async move { + let _ = shutdown_rx.await; + }, + )); + (addr, shutting_down, shutdown_tx, server) + } + + #[tokio::test] + async fn graceful_shutdown_lets_in_flight_request_finish_within_drain_timeout() { + let (addr, shutting_down, shutdown_tx, server) = + spawn_drain_server(Duration::from_millis(300), Duration::from_secs(5)).await; + + let request = + tokio::spawn(async move { reqwest::get(format!("http://{addr}/slow")).await }); + + // Let the request reach the handler, then trigger shutdown mid-flight. + tokio::time::sleep(Duration::from_millis(50)).await; + shutdown_tx.send(()).unwrap(); + + let response = tokio::time::timeout(Duration::from_secs(3), request) + .await + .expect("request should finish well before the drain timeout") + .unwrap() + .expect("in-flight request must not be killed by graceful shutdown"); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.text().await.unwrap(), "done"); + assert!(shutting_down.load(Ordering::SeqCst)); + + tokio::time::timeout(Duration::from_secs(3), server) + .await + .expect("server should stop promptly once the request drains") + .unwrap() + .unwrap(); + } + + #[tokio::test] + async fn graceful_shutdown_kills_request_still_in_flight_after_drain_timeout() { + let (addr, shutting_down, shutdown_tx, server) = + spawn_drain_server(Duration::from_secs(60), Duration::from_millis(200)).await; + + let request = + tokio::spawn(async move { reqwest::get(format!("http://{addr}/slow")).await }); + + tokio::time::sleep(Duration::from_millis(50)).await; + shutdown_tx.send(()).unwrap(); + + // The server must exit once the drain timeout elapses, without waiting + // for the 60s handler. + tokio::time::timeout(Duration::from_secs(3), server) + .await + .expect("server should force-exit at the drain deadline") + .unwrap() + .unwrap(); + assert!(shutting_down.load(Ordering::SeqCst)); + + // In production the process exits at this point, cutting the request. + // In-process we can only assert the server gave up on it: the request + // is still in flight when run_until_shutdown returns. + assert!( + !request.is_finished(), + "request should still be in flight when the server force-exits" + ); + request.abort(); + } + #[tokio::test] async fn upload_pack_stream_times_out_when_reader_stays_pending() { let (reader, _writer) = duplex(64);