From 083d8126091f35bbd027dd21feda57e6bfcaee1f Mon Sep 17 00:00:00 2001 From: Xe Iaso Date: Wed, 23 Sep 2026 15:55:37 -0400 Subject: [PATCH 1/4] feat: add kubernetes deployment, healthz endpoint, and image ci Add a kustomize deployment under manifest/: one objgitd pod with a 16Gi pack-cache PVC on the default storage class, a Service for HTTP, git://, and SSH, a second Service for metrics, and probes against a new GET /healthz endpoint on the metrics listener. The metrics mux moves into newMetricsMux so the health endpoint is testable. NewPackCache now sweeps objgit-packs-* directories left by a run that died without Cleanup, which otherwise starve a persistent pack-cache volume. The Dockerfile cross-compiles through TARGETOS and TARGETARCH with the build stage pinned to BUILDPLATFORM, and docker.yml builds and publishes multi-arch images to ghcr.io on every commit, with a smoke test before any push. Assisted-by: GLM 5.3 via OpenCode Signed-off-by: Xe Iaso --- .github/workflows/docker.yml | 87 ++++++++++++++ AGENTS.md | 44 +++---- Dockerfile | 12 +- README.md | 4 +- cmd/objgitd/main.go | 33 ++++-- cmd/objgitd/main_test.go | 70 +++++++++++ docs/architecture/metrics.md | 3 +- docs/usage/kubernetes.md | 134 ++++++++++++++++++++++ internal/storage/tigris/packcache.go | 30 +++++ internal/storage/tigris/packcache_test.go | 46 ++++++++ manifest/deployment.yaml | 133 +++++++++++++++++++++ manifest/kustomization.yaml | 78 +++++++++++++ manifest/namespace.yaml | 4 + manifest/pvc.yaml | 17 +++ manifest/service-metrics.yaml | 17 +++ manifest/service.yaml | 25 ++++ manifest/serviceaccount.yaml | 10 ++ 17 files changed, 713 insertions(+), 34 deletions(-) create mode 100644 .github/workflows/docker.yml create mode 100644 cmd/objgitd/main_test.go create mode 100644 docs/usage/kubernetes.md create mode 100644 manifest/deployment.yaml create mode 100644 manifest/kustomization.yaml create mode 100644 manifest/namespace.yaml create mode 100644 manifest/pvc.yaml create mode 100644 manifest/service-metrics.yaml create mode 100644 manifest/service.yaml create mode 100644 manifest/serviceaccount.yaml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..dbcfa2c --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,87 @@ +name: Docker + +on: + push: + pull_request: + branches: ["main"] + +permissions: + contents: read + packages: write + +concurrency: + group: docker-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + # A pull request from a branch of this repository skips this job. The + # push event for that branch already built and published the commit, so + # a second run doubles the cost and adds nothing. A pull request from a + # fork has no push event here, so it runs. + if: "${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository }}" + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + persist-credentials: false + + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Log in to the GitHub Container Registry + if: github.event_name == 'push' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Image tags + id: meta + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=sha + type=ref,event=branch + type=semver,pattern={{version}} + type=raw,value=latest,enable={{is_default_branch}} + + # Build one platform and load it, so the smoke test runs before any + # push. A broken image then never reaches the registry. + - name: Build the amd64 image + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + platforms: linux/amd64 + load: true + tags: objgitd:smoke + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Smoke test the health endpoint + run: | + docker run -d --name objgitd -e BUCKET=ci-dummy -p 127.0.0.1:9090:9090 objgitd:smoke + for i in $(seq 1 30); do + if curl -fsS http://127.0.0.1:9090/healthz; then + exit 0 + fi + sleep 1 + done + docker logs objgitd + exit 1 + + # The build stage is pinned to the runner platform (see the Dockerfile), + # so both targets compile without emulation. The builder cache already + # holds the amd64 layers from the smoke build. Push events publish. + # Pull requests build both platforms and push nothing. + - name: Build every platform + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name == 'push' }} + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/AGENTS.md b/AGENTS.md index 1a9757b..ed79741 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,8 +18,8 @@ One backend answers three transports: All three funnel authorization through one pluggable `internal/auth.Authorizer`. -A fourth listener serves Prometheus metrics at `/metrics` (`-metrics-bind`, -default `:9090`). An empty value disables it. +A fourth listener serves Prometheus metrics at `/metrics` and a `/healthz` +probe endpoint (`-metrics-bind`, default `:9090`). An empty value disables it. ## Commands @@ -50,36 +50,36 @@ Notes on the tests and on configuration: ## Where the code lives -| Path | Purpose | -| ----------------------------- | ---------------------------------------------------------------------- | -| `cmd/objgitd/main.go` | Builds the one `*daemon` and starts every listener. | -| `cmd/objgitd/git_protocol.go` | The git:// server. Also holds `operationFor` and `(*daemon).authorize`. | -| `cmd/objgitd/http.go` | Smart HTTP. `*daemon` is the `http.Handler` itself. | -| `cmd/objgitd/ssh.go` | The SSH server and its per-session dispatch. | -| `cmd/objgitd/receivepack.go` | The go-git fork that streams hook output, plus `writePack`. | -| `cmd/objgitd/hooks.go` | Ref diffing and the sandboxed hook run. | -| `internal/auth` | The one authorization interface. | -| `internal/repofs` | Maps a repository path to a `storage.Storer`. | -| `internal/storage/tigris` | Repository storage. A `storage.Storer` on the bucket. | -| `internal/bundler` | The async upload queue behind that storer. | -| `internal/s3fs` | Daemon-level state only, which is the SSH host key. | -| `internal/mountfs`, `internal/treefs`, `internal/kefkash` | The hook sandbox filesystem and shell wiring. | -| `internal/metrics` | Every Prometheus vector, plus thin helpers. | -| `internal/slog.go` | JSON handler init. | -| `cmd/membench/` | Push memory benchmark harness. Not shipped; see `docs/usage/memory-benchmark.md`. | +| Path | Purpose | +| --------------------------------------------------------- | --------------------------------------------------------------------------------- | +| `cmd/objgitd/main.go` | Builds the one `*daemon` and starts every listener. | +| `cmd/objgitd/git_protocol.go` | The git:// server. Also holds `operationFor` and `(*daemon).authorize`. | +| `cmd/objgitd/http.go` | Smart HTTP. `*daemon` is the `http.Handler` itself. | +| `cmd/objgitd/ssh.go` | The SSH server and its per-session dispatch. | +| `cmd/objgitd/receivepack.go` | The go-git fork that streams hook output, plus `writePack`. | +| `cmd/objgitd/hooks.go` | Ref diffing and the sandboxed hook run. | +| `internal/auth` | The one authorization interface. | +| `internal/repofs` | Maps a repository path to a `storage.Storer`. | +| `internal/storage/tigris` | Repository storage. A `storage.Storer` on the bucket. | +| `internal/bundler` | The async upload queue behind that storer. | +| `internal/s3fs` | Daemon-level state only, which is the SSH host key. | +| `internal/mountfs`, `internal/treefs`, `internal/kefkash` | The hook sandbox filesystem and shell wiring. | +| `internal/metrics` | Every Prometheus vector, plus thin helpers. | +| `internal/slog.go` | JSON handler init. | +| `cmd/membench/` | Push memory benchmark harness. Not shipped; see `docs/usage/memory-benchmark.md`. | ## Architecture Read [docs/architecture/README.md](docs/architecture/README.md) first. It describes the daemon and links to one page for each subsystem. -| Page | Read it before you change... | -| ------------------------------------------------------ | ---------------------------------------------------------------------- | +| Page | Read it before you change... | +| ------------------------------------------------------ | ----------------------------------------------------------------------- | | [transports.md](docs/architecture/transports.md) | Any transport. It holds two protocol points that are easy to get wrong. | | [auth.md](docs/architecture/auth.md) | Credentials, decisions, or a new `Authorizer`. | | [hooks.md](docs/architecture/hooks.md) | Push hooks, output streaming, or the sandbox. | | [metrics.md](docs/architecture/metrics.md) | Any metric or instrumentation seam. | -| [tigris-storer.md](docs/architecture/tigris-storer.md) | Object layout, refs, packs, the pack cache, or the upload path. | +| [tigris-storer.md](docs/architecture/tigris-storer.md) | Object layout, refs, packs, the pack cache, or the upload path. | | [s3fs.md](docs/architecture/s3fs.md) | The `billy.Filesystem` over the bucket. | Two more directories carry detail: diff --git a/Dockerfile b/Dockerfile index 7cc746d..ed8f9e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,14 @@ # syntax=docker/dockerfile:1 # --- build stage ------------------------------------------------------------- -FROM golang:1.26 AS build +# Pin the build stage to the platform of the runner ($BUILDPLATFORM) and +# cross-compile through TARGETOS and TARGETARCH. One native runner then +# builds every target platform, and no emulation is necessary. A plain +# `docker build` sets these values to the host platform, which keeps the +# previous behavior. +FROM --platform=$BUILDPLATFORM golang:1.26 AS build +ARG TARGETOS +ARG TARGETARCH WORKDIR /src @@ -16,7 +23,8 @@ COPY . . # protocol natively (no `git` binary at runtime). RUN --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/root/.cache/go-build \ - CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" \ + CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH \ + go build -trimpath -ldflags="-s -w" \ -o /objgitd ./cmd/objgitd # --- runtime stage ----------------------------------------------------------- diff --git a/README.md b/README.md index db34bef..62b18ce 100644 --- a/README.md +++ b/README.md @@ -123,4 +123,6 @@ To ssh://localhost:2222/xeiaso.net/objgit.git ### Deployment to production -TODO(Xe): re-evaluate performance-based life choices and write. +There is a kustomize deployment in [`manifest/`](./manifest/). See +[docs/usage/kubernetes.md](./docs/usage/kubernetes.md) for configuration and +caveats. diff --git a/cmd/objgitd/main.go b/cmd/objgitd/main.go index b970979..05cec55 100644 --- a/cmd/objgitd/main.go +++ b/cmd/objgitd/main.go @@ -170,14 +170,7 @@ func main() { os.Exit(1) } runtime.SetBlockProfileRate(100) - mux := http.NewServeMux() - mux.Handle("/metrics", promhttp.Handler()) - mux.HandleFunc("GET /debug/pprof/", pprof.Index) - mux.HandleFunc("GET /debug/pprof/cmdline", pprof.Cmdline) - mux.HandleFunc("GET /debug/pprof/profile", pprof.Profile) - mux.HandleFunc("GET /debug/pprof/symbol", pprof.Symbol) - mux.HandleFunc("GET /debug/pprof/trace", pprof.Trace) - srv := &http.Server{Handler: mux} + srv := &http.Server{Handler: newMetricsMux()} g.Go(func() error { if err := srv.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) { return err @@ -246,3 +239,27 @@ func main() { os.Exit(1) } } + +// newMetricsMux builds the metrics listener's handler: the Prometheus scrape +// endpoint, pprof, and /healthz for probes and load balancers. +func newMetricsMux() *http.ServeMux { + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + mux.HandleFunc("GET /healthz", handleHealthz) + mux.HandleFunc("GET /debug/pprof/", pprof.Index) + mux.HandleFunc("GET /debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("GET /debug/pprof/profile", pprof.Profile) + mux.HandleFunc("GET /debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("GET /debug/pprof/trace", pprof.Trace) + return mux +} + +// handleHealthz answers 200 while the process is serving. The daemon has no +// deeper signal to report. A dead transport listener fails the errgroup and +// exits the process, and the probe turns that exit into a restart. The +// handler reads no bucket state, so a Tigris outage does not fail the probes +// of every pod at the same time. +func handleHealthz(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + fmt.Fprintln(w, "ok") +} diff --git a/cmd/objgitd/main_test.go b/cmd/objgitd/main_test.go new file mode 100644 index 0000000..9dd31ea --- /dev/null +++ b/cmd/objgitd/main_test.go @@ -0,0 +1,70 @@ +package main + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" +) + +// TestMetricsMuxHealthz covers the health endpoint of the metrics listener. +// GET /healthz answers 200 with an ok body for probes. Other methods are +// rejected. /metrics keeps serving the scrape endpoint beside it. +func TestMetricsMuxHealthz(t *testing.T) { + ts := httptest.NewServer(newMetricsMux()) + t.Cleanup(ts.Close) + + for _, tt := range []struct { + name string + method string + path string + wantStatus int + wantBody string + }{ + { + name: "GET /healthz answers 200 ok", + method: http.MethodGet, + path: "/healthz", + wantStatus: http.StatusOK, + wantBody: "ok\n", + }, + { + name: "POST /healthz is not allowed", + method: http.MethodPost, + path: "/healthz", + wantStatus: http.StatusMethodNotAllowed, + }, + { + name: "GET /metrics still serves the scrape endpoint", + method: http.MethodGet, + path: "/metrics", + wantStatus: http.StatusOK, + }, + } { + t.Run(tt.name, func(t *testing.T) { + req, err := http.NewRequest(tt.method, ts.URL+tt.path, nil) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("Do: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != tt.wantStatus { + t.Errorf("status = %d, want %d", resp.StatusCode, tt.wantStatus) + } + if tt.wantBody == "" { + return + } + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + if string(body) != tt.wantBody { + t.Errorf("body = %q, want %q", body, tt.wantBody) + } + }) + } +} diff --git a/docs/architecture/metrics.md b/docs/architecture/metrics.md index cbea291..ba31fbe 100644 --- a/docs/architecture/metrics.md +++ b/docs/architecture/metrics.md @@ -7,7 +7,8 @@ the **default registry**. The Go-runtime and process collectors from `main.go` serves `promhttp.Handler()` on its own listener. The flag is `-metrics-bind`, the default is `:9090`, and an empty value disables the listener. It uses the same errgroup Serve and Shutdown idiom as the HTTP -transport. +transport. The same listener also answers `GET /healthz` with `200 ok` for +probes and load balancers, and serves pprof under `/debug/pprof/`. All series carry the prefix `objgit_`. There is no `repo` label, because repository names have unbounded cardinality. Git operations are keyed by diff --git a/docs/usage/kubernetes.md b/docs/usage/kubernetes.md new file mode 100644 index 0000000..ff89efc --- /dev/null +++ b/docs/usage/kubernetes.md @@ -0,0 +1,134 @@ +# Kubernetes deployment + +`manifest/` holds a kustomize deployment for `objgitd`. The pod stores all +repository data in a Tigris bucket. A persistent volume holds the pack cache. +An ephemeral volume holds upload scratch data. + +| Object | Purpose | +| ------------------------------------------ | -------------------------------------------------- | +| `Namespace/objgit` | Holds every object below. | +| `ServiceAccount/objgitd` | The pod identity. Users can annotate it for IRSA. | +| `Deployment/objgitd` | The daemon, with all three transports and metrics. | +| `PersistentVolumeClaim/objgitd-pack-cache` | 16Gi from the default StorageClass. | +| `Service/objgit` | HTTP (80), git:// (9418), SSH (22). | +| `Service/objgit-metrics` | Prometheus /metrics (9090), on its own Service. | + +## Configure + +The daemon reads all configuration from environment variables. `flagenv` maps +each flag to one variable (see the README section "Setup instructions"). The +literals live in `manifest/kustomization.yaml`: + +- `BUCKET` in the `objgitd-config` configMapGenerator is **required**. The + daemon does not start when this variable does not name a Tigris bucket. +- The `objgitd-tigris` secretGenerator holds a static Tigris keypair + (`TIGRIS_STORAGE_ACCESS_KEY_ID`, `TIGRIS_STORAGE_SECRET_ACCESS_KEY`). + Before you apply the configuration, replace both placeholder values. + +To avoid a long-lived keypair, use IRSA or workload identity: + +1. Delete the `objgitd-tigris` secretGenerator from + `manifest/kustomization.yaml`. +2. Delete the `secretRef` block from `manifest/deployment.yaml`. The block is + optional. +3. Annotate the `objgitd` ServiceAccount for EKS IRSA or GKE workload + identity. + +The AWS SDK default credential chain finds the role on its own. + +CI builds the image on every commit. The workflow lives in +`.github/workflows/docker.yml`. Every push event publishes these tags to +`ghcr.io/tigrisdata/objgit`: + +- `:latest` and `:main` from the main branch. +- `:sha-` from every branch and every release tag. +- One tag per release, such as `:1.5.1`. + +Every tag carries both `linux/amd64` and `linux/arm64` images. Pull requests +from a fork build both platforms and push nothing. The smoke test in the +workflow starts the image and reads `/healthz`. + +The Deployment references `:latest` and sets `imagePullPolicy: Always`, so +Kubernetes pulls the image on every pod start. For a fixed deployment, pin an +immutable `:sha-` tag through an `images:` entry in +`manifest/kustomization.yaml`. + +New GHCR packages start private. Make the package public, or configure an +imagePullSecret in the namespace. To use a different registry: build the repo +`Dockerfile`. Push the image to that registry: + +```text +docker build -t registry.example.com/objgit:1.5.1 . +docker push registry.example.com/objgit:1.5.1 +``` + +Then update the image reference in `manifest/deployment.yaml`. + +## Apply + +```text +kubectl apply -k manifest/ +``` + +## Use + +Both Services are of type ClusterIP. They are reachable only from inside the +cluster. objgitd has no authentication yet. For this reason, the Services +stay cluster-internal. + +The `objgit` Service exposes SSH on port 22 and HTTP on port 80. In the pod, +the daemon still binds port 2222 for SSH and port 8080 for HTTP. The named +target ports in the Service map one to the other: + +```text +git clone http://objgit.objgit.svc.cluster.local/org/repo.git +git clone git://objgit.objgit.svc.cluster.local:9418/org/repo.git +git clone ssh://git@objgit.objgit.svc.cluster.local/org/repo.git +``` + +Scrape metrics from `objgit-metrics.objgit.svc.cluster.local:9090/metrics`. +The same listener serves pprof under `/debug/pprof/`. + +## Health check + +The metrics listener answers `GET /healthz` with `200 ok`. The smart-HTTP mux +returns 404 for every non-git path. For this reason, the probes use the +metrics listener. + +The endpoint reads no bucket state. A Tigris outage therefore does not fail +the probes. + +The startup, liveness, and readiness probes in the manifest all send +`GET /healthz` to the metrics port. + +## Storage notes + +- The **pack cache** (`PACK_CACHE_DIR`) lives on the 16Gi + `objgitd-pack-cache` claim. `PACK_CACHE_BYTES` caps the cache at 14 GiB. + The unused space on the claim holds the temporary files of in-flight + downloads. This claim is the only persistent state. The repositories and + the SSH host key live in the Tigris bucket. On startup, the daemon sweeps + cache directories that an earlier run left behind, so a crash does not + leak disk space across restarts. +- **Upload scratch data** is separate from the pack cache. The tigris storer + stages pack writes to the OS temp directory through `os.CreateTemp`. The + root filesystem of the container is read-only, so the OS temp directory + needs its own `emptyDir` volume (4Gi `sizeLimit`) at `/tmp`. An exceeded + `sizeLimit` evicts the entire pod, which kills every push in flight. For + very large pushes, raise `sizeLimit`, or set `TMPDIR` to a path on the + pack-cache volume. +- The claim has the `ReadWriteOnce` access mode. A rolling update tries to + mount it on two pods at the same time. For this reason, the Deployment + uses the `Recreate` strategy and one replica. Scale-out needs + `ReadOnlyMany` storage or one cache per pod. Reference updates commit + through compare-and-swap on the bucket, so the ref store does not limit + the replica count. + +## Resources + +Each concurrent push costs approximately 400 MiB of resident set for a large +repository. `MAX_CONCURRENT_PUSHES` (4 by default) caps the number of +concurrent pushes. The 4Gi memory limit in the manifest covers four pushes +and fetch traffic. The memory limit and the cap must change together. See +[plans/bound-concurrent-pushes.md](../plans/bound-concurrent-pushes.md) for +the measurements. diff --git a/internal/storage/tigris/packcache.go b/internal/storage/tigris/packcache.go index eca7879..2415989 100644 --- a/internal/storage/tigris/packcache.go +++ b/internal/storage/tigris/packcache.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "runtime" + "strings" "sync" "sync/atomic" "time" @@ -81,6 +82,14 @@ func NewPackCache(parent string, maxBytes int64) (*PackCache, error) { if err != nil { return nil, fmt.Errorf("tigris: create pack cache directory: %w", err) } + // A run that died without Cleanup leaves its cache directory behind. On + // the OS temp directory that is harmless, but on a persistent parent + // (kubernetes -pack-cache-dir on a volume) the orphans survive every + // restart, hold packs no later run can read, and starve maxBytes. One + // daemon owns the parent in every supported deployment, so the sweep is + // safe; a reader from an older process keeps its descriptors through the + // unlink, exactly like an eviction. + sweepOrphanedCaches(parent, dir) return &PackCache{ dir: dir, maxBytes: maxBytes, @@ -88,6 +97,27 @@ func NewPackCache(parent string, maxBytes int64) (*PackCache, error) { }, nil } +// sweepOrphanedCaches removes the objgit-packs-* directories under parent +// that earlier runs left behind, and keeps keep, the directory this run just +// created. It is best effort: a ReadDir failure leaves everything alone, and +// a directory that cannot be removed stays for the next start to try again. +func sweepOrphanedCaches(parent, keep string) { + entries, err := os.ReadDir(parent) + if err != nil { + return + } + for _, e := range entries { + if !e.IsDir() || !strings.HasPrefix(e.Name(), "objgit-packs-") { + continue + } + dir := filepath.Join(parent, e.Name()) + if dir == keep { + continue + } + os.RemoveAll(dir) + } +} + // Cleanup removes the cache directory and everything in it. Call it once at // shutdown. Descriptors already handed out keep working. func (c *PackCache) Cleanup() error { diff --git a/internal/storage/tigris/packcache_test.go b/internal/storage/tigris/packcache_test.go index 568868c..4b435f0 100644 --- a/internal/storage/tigris/packcache_test.go +++ b/internal/storage/tigris/packcache_test.go @@ -69,6 +69,52 @@ func countCacheFiles(t *testing.T, c *PackCache) int { return len(ents) } +// TestPackCacheSweepsOrphans covers the startup sweep: a previous run that +// died without Cleanup leaves its objgit-packs-* directory on disk, and a +// persistent -pack-cache-dir must not accumulate them. +func TestPackCacheSweepsOrphans(t *testing.T) { + t.Parallel() + + parent := t.TempDir() + for _, name := range []string{"objgit-packs-stale", "objgit-packs-also-stale"} { + dir := filepath.Join(parent, name) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", dir, err) + } + if err := os.WriteFile(filepath.Join(dir, "orphan.bin"), []byte("junk"), 0o644); err != nil { + t.Fatalf("write orphan in %s: %v", name, err) + } + } + if err := os.MkdirAll(filepath.Join(parent, "not-a-cache"), 0o755); err != nil { + t.Fatalf("mkdir unrelated: %v", err) + } + + c, err := NewPackCache(parent, 0) + if err != nil { + t.Fatalf("NewPackCache: %v", err) + } + t.Cleanup(func() { _ = c.Cleanup() }) + + for _, tt := range []struct { + path string + want bool // want the path to still exist after the sweep + }{ + {path: filepath.Join(parent, "objgit-packs-stale"), want: false}, + {path: filepath.Join(parent, "objgit-packs-also-stale"), want: false}, + {path: filepath.Join(parent, "not-a-cache"), want: true}, + {path: c.dir, want: true}, + } { + _, err := os.Stat(tt.path) + exists := err == nil + if err != nil && !os.IsNotExist(err) { + t.Fatalf("stat %s: %v", tt.path, err) + } + if exists != tt.want { + t.Errorf("%s exists = %v, want %v", tt.path, exists, tt.want) + } + } +} + // TestPackCacheDownloadsOnce is the whole point of the cache: a second request // for a pack already on disk must not go back to the network. func TestPackCacheDownloadsOnce(t *testing.T) { diff --git a/manifest/deployment.yaml b/manifest/deployment.yaml new file mode 100644 index 0000000..1df24b6 --- /dev/null +++ b/manifest/deployment.yaml @@ -0,0 +1,133 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: objgitd + labels: + app.kubernetes.io/name: objgitd + app.kubernetes.io/component: server + app.kubernetes.io/part-of: objgit +spec: + # One replica. Repositories and the SSH host key live in the Tigris bucket. + # The pack cache is the only node-local state, and this claim is + # ReadWriteOnce. Scale-out needs ReadOnlyMany storage or one cache per pod. + replicas: 1 + strategy: + # A rolling update schedules the replacement pod while the old pod still + # holds the ReadWriteOnce volume. The new pod then blocks until the + # timeout. + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: objgitd + template: + metadata: + labels: + app.kubernetes.io/name: objgitd + app.kubernetes.io/component: server + app.kubernetes.io/part-of: objgit + spec: + serviceAccountName: objgitd + securityContext: + # The image is distroless/static nonroot, uid and gid 65532. fsGroup + # hands the pack-cache volume to that group so the daemon can + # MkdirTemp its cache directory under the mount. Storage classes with + # fsGroupPolicy: None skip this chown. Such classes need a + # securityContext.fsGroupChangePolicy or an initContainer. + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: objgitd + # CI builds this image on every commit and publishes it to + # ghcr.io/tigrisdata/objgit (see .github/workflows/docker.yml). The + # tags are :latest, :main, :sha-, and one per release. GHCR + # packages start private. Make the package public, or configure an + # imagePullSecret. To use a different registry, change this + # reference. + # + # imagePullPolicy: Always pulls on every pod start, so a cached + # :latest on a node cannot pin an old image. Pin an immutable + # :sha- tag through an `images:` entry in + # kustomization.yaml for a fixed deployment. + image: ghcr.io/tigrisdata/objgit:latest + imagePullPolicy: Always + envFrom: + - configMapRef: + name: objgitd-config + - secretRef: + name: objgitd-tigris + # Credentials can also come from the AWS SDK default chain + # (IRSA, workload identity, node profile). Then the Secret is + # never created. + optional: true + ports: + - name: http + containerPort: 8080 + - name: git + containerPort: 9418 + - name: ssh + containerPort: 2222 + - name: metrics + containerPort: 9090 + # /healthz on the metrics listener is the dedicated probe endpoint. + # The smart-HTTP mux 404s every non-git path, so the git ports + # cannot answer probes. + startupProbe: + httpGet: + path: /healthz + port: metrics + periodSeconds: 2 + failureThreshold: 30 + livenessProbe: + httpGet: + path: /healthz + port: metrics + periodSeconds: 10 + readinessProbe: + httpGet: + path: /healthz + port: metrics + periodSeconds: 5 + resources: + # Each concurrent push costs roughly 400 MiB of resident set for + # a large repository. The limit must cover MAX_CONCURRENT_PUSHES + # (4 by default) plus fetch traffic. See + # docs/plans/bound-concurrent-pushes.md. + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "2" + memory: 4Gi + volumeMounts: + - name: pack-cache + mountPath: /var/cache/objgit + # The tigris storer stages uploads in the OS temp directory + # through os.CreateTemp (internal/storage/tigris/writer.go and + # packwriter.go). Distroless ships a writable /tmp, but the root + # filesystem of the container is read-only, so the OS temp + # directory needs its own volume. + # + # CAUTION: Do not set sizeLimit below the largest expected push. + # A pack stages whole in /tmp, and an exceeded sizeLimit evicts + # the entire pod, which kills every push in flight. For very + # large pushes, raise sizeLimit, or set TMPDIR to a path on the + # pack-cache volume. + - name: tmp + mountPath: /tmp + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumes: + - name: pack-cache + persistentVolumeClaim: + claimName: objgitd-pack-cache + - name: tmp + emptyDir: + sizeLimit: 4Gi diff --git a/manifest/kustomization.yaml b/manifest/kustomization.yaml new file mode 100644 index 0000000..d3af8ce --- /dev/null +++ b/manifest/kustomization.yaml @@ -0,0 +1,78 @@ +# Kustomize deployment for objgitd, the objgit daemon. +# +# 1. CI builds an image on every commit and publishes it to +# ghcr.io/tigrisdata/objgit. Use those tags, or build the repo Dockerfile +# and push it to a registry you control. Then update the image reference +# in deployment.yaml. +# 2. Set BUCKET in the configMapGenerator below. +# 3. Set the Tigris credentials in the secretGenerator below. For IRSA or +# workload identity, see docs/usage/kubernetes.md. +# 4. kubectl apply -k manifest/ +# +# See docs/usage/kubernetes.md for the full walkthrough. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: objgit + +resources: + - namespace.yaml + - serviceaccount.yaml + - pvc.yaml + - deployment.yaml + - service.yaml + - service-metrics.yaml + +# Pin an immutable image tag for a fixed deployment. The Deployment references +# :latest, which Kubernetes pulls on every pod start. Uncomment and set the +# tag to a specific commit: +# +# images: +# - name: ghcr.io/tigrisdata/objgit +# newTag: sha-a1b2c3d + +configMapGenerator: + # Daemon configuration. flagenv maps every flag to UPPER_SNAKE_CASE, and + # environment variables override the flag defaults (README "Setup + # instructions"). + - name: objgitd-config + literals: + # Required: the Tigris bucket that holds every repository and the SSH + # host key. objgitd does not start when this value is empty. + - BUCKET=replace-me + # Transports. HTTP and git:// are on by default. SSH needs a bind value + # to turn on. Every bind is a pod port named below. + - HTTP_BIND=:8080 + - GIT_BIND=:9418 + - SSH_BIND=:2222 + # Prometheus /metrics (and pprof), exposed on the objgit-metrics + # Service. The probes in deployment.yaml read /healthz on this + # listener. An empty value therefore breaks every probe, because it + # disables the listener. + - METRICS_BIND=:9090 + # The pack cache lives on the objgitd-pack-cache PVC. 14 GiB (15032385536 + # bytes) leaves headroom under the 16Gi claim. + - PACK_CACHE_DIR=/var/cache/objgit + - PACK_CACHE_BYTES=15032385536 + # Anonymous pushes over every transport. Set false for a read-only + # mirror. There is no other authentication yet (README "Notable + # features"). Keep the Service cluster-internal, or put your own + # authentication in front. + - ALLOW_PUSH=true + # Run .objgit/hooks/receive-pack from the repository after a push. See + # docs/usage/hooks.md. + - ALLOW_HOOKS=false + - SLOG_LEVEL=INFO + +secretGenerator: + # Static Tigris credentials. The storage-go client prefers these over the + # AWS SDK default chain. Replace the placeholder values before you apply. + # + # To avoid a long-lived keypair: delete this generator and the secretRef in + # deployment.yaml (it is optional). Then annotate the objgitd + # ServiceAccount for IRSA or workload identity. The default chain finds + # the role on its own. + - name: objgitd-tigris + literals: + - TIGRIS_STORAGE_ACCESS_KEY_ID=replace-me + - TIGRIS_STORAGE_SECRET_ACCESS_KEY=replace-me diff --git a/manifest/namespace.yaml b/manifest/namespace.yaml new file mode 100644 index 0000000..be296ed --- /dev/null +++ b/manifest/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: objgit diff --git a/manifest/pvc.yaml b/manifest/pvc.yaml new file mode 100644 index 0000000..c9b57a6 --- /dev/null +++ b/manifest/pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: objgitd-pack-cache + labels: + app.kubernetes.io/name: objgitd + app.kubernetes.io/component: server + app.kubernetes.io/part-of: objgit +spec: + accessModes: + - ReadWriteOnce + # storageClassName is deliberately unset, so the cluster uses its default + # StorageClass. If the default is not the provisioner you want, name a + # class explicitly. + resources: + requests: + storage: 16Gi diff --git a/manifest/service-metrics.yaml b/manifest/service-metrics.yaml new file mode 100644 index 0000000..7e1f1f0 --- /dev/null +++ b/manifest/service-metrics.yaml @@ -0,0 +1,17 @@ +# Prometheus /metrics (and pprof), kept off the git Service on purpose. +apiVersion: v1 +kind: Service +metadata: + name: objgit-metrics + labels: + app.kubernetes.io/name: objgitd + app.kubernetes.io/component: metrics + app.kubernetes.io/part-of: objgit +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: objgitd + ports: + - name: metrics + port: 9090 + targetPort: metrics diff --git a/manifest/service.yaml b/manifest/service.yaml new file mode 100644 index 0000000..42e8e78 --- /dev/null +++ b/manifest/service.yaml @@ -0,0 +1,25 @@ +# The git transports: smart HTTP, git://, and SSH. Metrics is deliberately +# absent. It lives on the objgit-metrics Service, so a scrape ACL or a +# NetworkPolicy can gate it separately from git traffic. +apiVersion: v1 +kind: Service +metadata: + name: objgit + labels: + app.kubernetes.io/name: objgitd + app.kubernetes.io/component: server + app.kubernetes.io/part-of: objgit +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: objgitd + ports: + - name: http + port: 80 + targetPort: http + - name: git + port: 9418 + targetPort: git + - name: ssh + port: 22 + targetPort: ssh diff --git a/manifest/serviceaccount.yaml b/manifest/serviceaccount.yaml new file mode 100644 index 0000000..33ba30f --- /dev/null +++ b/manifest/serviceaccount.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: objgitd + # To use EKS IRSA or GKE workload identity instead of a static Tigris + # keypair: annotate this account, and delete the objgitd-tigris Secret + # reference from deployment.yaml. + # + # eks.amazonaws.com/role-arn: arn:aws:iam::ACCOUNT:role/objgitd +# annotations: {} From 597bb01d80ddb62c6bfefa12544a6c6807cc76e7 Mon Sep 17 00:00:00 2001 From: Xe Iaso Date: Wed, 23 Sep 2026 15:59:15 -0400 Subject: [PATCH 2/4] fix(manifest): use standard aws environment names for credentials The secretGenerator now sets AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY. The AWS SDK default chain reads these from the environment, which is also what the storage-go client falls back to when the TIGRIS_STORAGE_* overrides are unset. This matches the names every AWS tooling expects, and AWS_SESSION_TOKEN works for temporary credentials. Assisted-by: GLM 5.3 via OpenCode Signed-off-by: Xe Iaso --- docs/usage/kubernetes.md | 7 ++++--- manifest/kustomization.yaml | 10 ++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/usage/kubernetes.md b/docs/usage/kubernetes.md index ff89efc..b8979dd 100644 --- a/docs/usage/kubernetes.md +++ b/docs/usage/kubernetes.md @@ -21,9 +21,10 @@ literals live in `manifest/kustomization.yaml`: - `BUCKET` in the `objgitd-config` configMapGenerator is **required**. The daemon does not start when this variable does not name a Tigris bucket. -- The `objgitd-tigris` secretGenerator holds a static Tigris keypair - (`TIGRIS_STORAGE_ACCESS_KEY_ID`, `TIGRIS_STORAGE_SECRET_ACCESS_KEY`). - Before you apply the configuration, replace both placeholder values. +- The `objgitd-tigris` secretGenerator holds a static Tigris keypair under + the standard AWS names (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`). + Before you apply the configuration, replace both placeholder values. Add + `AWS_SESSION_TOKEN` for temporary credentials. To avoid a long-lived keypair, use IRSA or workload identity: diff --git a/manifest/kustomization.yaml b/manifest/kustomization.yaml index d3af8ce..4d54f4f 100644 --- a/manifest/kustomization.yaml +++ b/manifest/kustomization.yaml @@ -65,8 +65,10 @@ configMapGenerator: - SLOG_LEVEL=INFO secretGenerator: - # Static Tigris credentials. The storage-go client prefers these over the - # AWS SDK default chain. Replace the placeholder values before you apply. + # Static Tigris credentials under their standard AWS names. The AWS SDK + # default chain reads them from the environment. Replace the placeholder + # values before you apply. Add AWS_SESSION_TOKEN for temporary + # credentials. # # To avoid a long-lived keypair: delete this generator and the secretRef in # deployment.yaml (it is optional). Then annotate the objgitd @@ -74,5 +76,5 @@ secretGenerator: # the role on its own. - name: objgitd-tigris literals: - - TIGRIS_STORAGE_ACCESS_KEY_ID=replace-me - - TIGRIS_STORAGE_SECRET_ACCESS_KEY=replace-me + - AWS_ACCESS_KEY_ID=replace-me + - AWS_SECRET_ACCESS_KEY=replace-me From 994499a7472c79c4d7e2f35071772e1450ee56cf Mon Sep 17 00:00:00 2001 From: Xe Iaso Date: Wed, 23 Sep 2026 16:00:01 -0400 Subject: [PATCH 3/4] fix(manifest): point the aws sdk default chain at tigris The configMapGenerator now sets AWS_ENDPOINT_URL_S3, AWS_ENDPOINT_URL_IAM, and AWS_REGION=auto, so every AWS SDK client in the pod talks to Tigris. The storage-go wrapper pins the S3 endpoint itself, so the IAM endpoint matters most for the credential chain of IRSA. Assisted-by: GLM 5.3 via OpenCode Signed-off-by: Xe Iaso --- docs/usage/kubernetes.md | 3 +++ manifest/kustomization.yaml | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/docs/usage/kubernetes.md b/docs/usage/kubernetes.md index b8979dd..30f4553 100644 --- a/docs/usage/kubernetes.md +++ b/docs/usage/kubernetes.md @@ -21,6 +21,9 @@ literals live in `manifest/kustomization.yaml`: - `BUCKET` in the `objgitd-config` configMapGenerator is **required**. The daemon does not start when this variable does not name a Tigris bucket. +- The same configMapGenerator sets `AWS_ENDPOINT_URL_S3`, + `AWS_ENDPOINT_URL_IAM`, and `AWS_REGION`. These values point the AWS SDK + default chain at Tigris, in region `auto`. - The `objgitd-tigris` secretGenerator holds a static Tigris keypair under the standard AWS names (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`). Before you apply the configuration, replace both placeholder values. Add diff --git a/manifest/kustomization.yaml b/manifest/kustomization.yaml index 4d54f4f..a6ab567 100644 --- a/manifest/kustomization.yaml +++ b/manifest/kustomization.yaml @@ -50,6 +50,13 @@ configMapGenerator: # listener. An empty value therefore breaks every probe, because it # disables the listener. - METRICS_BIND=:9090 + # The AWS SDK default chain reads these three values. They point every + # SDK client at Tigris, in region auto. The storage-go wrapper pins + # the S3 endpoint itself, so the IAM endpoint matters most for the + # credential chain of IRSA. + - AWS_ENDPOINT_URL_S3=https://t3.storage.dev + - AWS_ENDPOINT_URL_IAM=https://iam.storage.dev + - AWS_REGION=auto # The pack cache lives on the objgitd-pack-cache PVC. 14 GiB (15032385536 # bytes) leaves headroom under the 16Gi claim. - PACK_CACHE_DIR=/var/cache/objgit From 17121c88d374f0f9144241a33151af0f4909f591 Mon Sep 17 00:00:00 2001 From: Xe Iaso Date: Wed, 23 Sep 2026 16:23:28 -0400 Subject: [PATCH 4/4] fix(manifest): read credentials from gitignored files The secretGenerator now reads AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY from files under manifest/secrets/, which .gitignore excludes, so real credentials cannot reach the repository. Kustomize refuses to build when the files are missing, which prevents an accidental deploy with placeholder values. The files must be written with printf '%s', because a trailing newline becomes part of the credential. Assisted-by: GLM 5.3 via OpenCode Signed-off-by: Xe Iaso --- .gitignore | 1 + docs/usage/kubernetes.md | 18 +++++++++++++++--- manifest/kustomization.yaml | 17 ++++++++++------- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 4aa8b6a..5df20b2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .env /objgitd +manifest/secrets/ node_modules diff --git a/docs/usage/kubernetes.md b/docs/usage/kubernetes.md index 30f4553..010713e 100644 --- a/docs/usage/kubernetes.md +++ b/docs/usage/kubernetes.md @@ -25,9 +25,21 @@ literals live in `manifest/kustomization.yaml`: `AWS_ENDPOINT_URL_IAM`, and `AWS_REGION`. These values point the AWS SDK default chain at Tigris, in region `auto`. - The `objgitd-tigris` secretGenerator holds a static Tigris keypair under - the standard AWS names (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`). - Before you apply the configuration, replace both placeholder values. Add - `AWS_SESSION_TOKEN` for temporary credentials. + the standard AWS names (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`). It + reads the values from files under `manifest/secrets/`, and `.gitignore` + excludes that directory, so the credentials never reach the repository. + Write each file with `printf`, because a trailing newline becomes part of + the credential: + + ```text + printf '%s' '' > manifest/secrets/aws-access-key-id + printf '%s' '' > manifest/secrets/aws-secret-access-key + ``` + + Kustomize refuses to build when the files are missing. For this reason, a + fresh clone cannot apply the manifest until it receives the files through + a secure side channel. For temporary credentials, add a third file line + for `AWS_SESSION_TOKEN`. To avoid a long-lived keypair, use IRSA or workload identity: diff --git a/manifest/kustomization.yaml b/manifest/kustomization.yaml index a6ab567..e094dd3 100644 --- a/manifest/kustomization.yaml +++ b/manifest/kustomization.yaml @@ -72,16 +72,19 @@ configMapGenerator: - SLOG_LEVEL=INFO secretGenerator: - # Static Tigris credentials under their standard AWS names. The AWS SDK - # default chain reads them from the environment. Replace the placeholder - # values before you apply. Add AWS_SESSION_TOKEN for temporary - # credentials. + # Static Tigris credentials under their standard AWS names, read from + # files under secrets/. That directory is in .gitignore, so the values + # never reach this repository. Write each file with printf '%s', because + # a trailing newline becomes part of the credential. Kustomize refuses to + # build when a file is missing, which prevents an accidental deploy with + # placeholder values. For temporary credentials, add a third file line + # for AWS_SESSION_TOKEN. # # To avoid a long-lived keypair: delete this generator and the secretRef in # deployment.yaml (it is optional). Then annotate the objgitd # ServiceAccount for IRSA or workload identity. The default chain finds # the role on its own. - name: objgitd-tigris - literals: - - AWS_ACCESS_KEY_ID=replace-me - - AWS_SECRET_ACCESS_KEY=replace-me + files: + - AWS_ACCESS_KEY_ID=secrets/aws-access-key-id + - AWS_SECRET_ACCESS_KEY=secrets/aws-secret-access-key