diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..dbcfa2c --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,87 @@ +name: Docker + +on: + push: + pull_request: + branches: ["main"] + +permissions: + contents: read + packages: write + +concurrency: + group: docker-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + # A pull request from a branch of this repository skips this job. The + # push event for that branch already built and published the commit, so + # a second run doubles the cost and adds nothing. A pull request from a + # fork has no push event here, so it runs. + if: "${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository }}" + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + persist-credentials: false + + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Log in to the GitHub Container Registry + if: github.event_name == 'push' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Image tags + id: meta + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 + with: + images: ghcr.io/${{ github.repository }} + tags: | + type=sha + type=ref,event=branch + type=semver,pattern={{version}} + type=raw,value=latest,enable={{is_default_branch}} + + # Build one platform and load it, so the smoke test runs before any + # push. A broken image then never reaches the registry. + - name: Build the amd64 image + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + platforms: linux/amd64 + load: true + tags: objgitd:smoke + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Smoke test the health endpoint + run: | + docker run -d --name objgitd -e BUCKET=ci-dummy -p 127.0.0.1:9090:9090 objgitd:smoke + for i in $(seq 1 30); do + if curl -fsS http://127.0.0.1:9090/healthz; then + exit 0 + fi + sleep 1 + done + docker logs objgitd + exit 1 + + # The build stage is pinned to the runner platform (see the Dockerfile), + # so both targets compile without emulation. The builder cache already + # holds the amd64 layers from the smoke build. Push events publish. + # Pull requests build both platforms and push nothing. + - name: Build every platform + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name == 'push' }} + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.gitignore b/.gitignore index 4aa8b6a..5df20b2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .env /objgitd +manifest/secrets/ node_modules diff --git a/AGENTS.md b/AGENTS.md index 1a9757b..ed79741 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,8 +18,8 @@ One backend answers three transports: All three funnel authorization through one pluggable `internal/auth.Authorizer`. -A fourth listener serves Prometheus metrics at `/metrics` (`-metrics-bind`, -default `:9090`). An empty value disables it. +A fourth listener serves Prometheus metrics at `/metrics` and a `/healthz` +probe endpoint (`-metrics-bind`, default `:9090`). An empty value disables it. ## Commands @@ -50,36 +50,36 @@ Notes on the tests and on configuration: ## Where the code lives -| Path | Purpose | -| ----------------------------- | ---------------------------------------------------------------------- | -| `cmd/objgitd/main.go` | Builds the one `*daemon` and starts every listener. | -| `cmd/objgitd/git_protocol.go` | The git:// server. Also holds `operationFor` and `(*daemon).authorize`. | -| `cmd/objgitd/http.go` | Smart HTTP. `*daemon` is the `http.Handler` itself. | -| `cmd/objgitd/ssh.go` | The SSH server and its per-session dispatch. | -| `cmd/objgitd/receivepack.go` | The go-git fork that streams hook output, plus `writePack`. | -| `cmd/objgitd/hooks.go` | Ref diffing and the sandboxed hook run. | -| `internal/auth` | The one authorization interface. | -| `internal/repofs` | Maps a repository path to a `storage.Storer`. | -| `internal/storage/tigris` | Repository storage. A `storage.Storer` on the bucket. | -| `internal/bundler` | The async upload queue behind that storer. | -| `internal/s3fs` | Daemon-level state only, which is the SSH host key. | -| `internal/mountfs`, `internal/treefs`, `internal/kefkash` | The hook sandbox filesystem and shell wiring. | -| `internal/metrics` | Every Prometheus vector, plus thin helpers. | -| `internal/slog.go` | JSON handler init. | -| `cmd/membench/` | Push memory benchmark harness. Not shipped; see `docs/usage/memory-benchmark.md`. | +| Path | Purpose | +| --------------------------------------------------------- | --------------------------------------------------------------------------------- | +| `cmd/objgitd/main.go` | Builds the one `*daemon` and starts every listener. | +| `cmd/objgitd/git_protocol.go` | The git:// server. Also holds `operationFor` and `(*daemon).authorize`. | +| `cmd/objgitd/http.go` | Smart HTTP. `*daemon` is the `http.Handler` itself. | +| `cmd/objgitd/ssh.go` | The SSH server and its per-session dispatch. | +| `cmd/objgitd/receivepack.go` | The go-git fork that streams hook output, plus `writePack`. | +| `cmd/objgitd/hooks.go` | Ref diffing and the sandboxed hook run. | +| `internal/auth` | The one authorization interface. | +| `internal/repofs` | Maps a repository path to a `storage.Storer`. | +| `internal/storage/tigris` | Repository storage. A `storage.Storer` on the bucket. | +| `internal/bundler` | The async upload queue behind that storer. | +| `internal/s3fs` | Daemon-level state only, which is the SSH host key. | +| `internal/mountfs`, `internal/treefs`, `internal/kefkash` | The hook sandbox filesystem and shell wiring. | +| `internal/metrics` | Every Prometheus vector, plus thin helpers. | +| `internal/slog.go` | JSON handler init. | +| `cmd/membench/` | Push memory benchmark harness. Not shipped; see `docs/usage/memory-benchmark.md`. | ## Architecture Read [docs/architecture/README.md](docs/architecture/README.md) first. It describes the daemon and links to one page for each subsystem. -| Page | Read it before you change... | -| ------------------------------------------------------ | ---------------------------------------------------------------------- | +| Page | Read it before you change... | +| ------------------------------------------------------ | ----------------------------------------------------------------------- | | [transports.md](docs/architecture/transports.md) | Any transport. It holds two protocol points that are easy to get wrong. | | [auth.md](docs/architecture/auth.md) | Credentials, decisions, or a new `Authorizer`. | | [hooks.md](docs/architecture/hooks.md) | Push hooks, output streaming, or the sandbox. | | [metrics.md](docs/architecture/metrics.md) | Any metric or instrumentation seam. | -| [tigris-storer.md](docs/architecture/tigris-storer.md) | Object layout, refs, packs, the pack cache, or the upload path. | +| [tigris-storer.md](docs/architecture/tigris-storer.md) | Object layout, refs, packs, the pack cache, or the upload path. | | [s3fs.md](docs/architecture/s3fs.md) | The `billy.Filesystem` over the bucket. | Two more directories carry detail: diff --git a/Dockerfile b/Dockerfile index 7cc746d..ed8f9e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,14 @@ # syntax=docker/dockerfile:1 # --- build stage ------------------------------------------------------------- -FROM golang:1.26 AS build +# Pin the build stage to the platform of the runner ($BUILDPLATFORM) and +# cross-compile through TARGETOS and TARGETARCH. One native runner then +# builds every target platform, and no emulation is necessary. A plain +# `docker build` sets these values to the host platform, which keeps the +# previous behavior. +FROM --platform=$BUILDPLATFORM golang:1.26 AS build +ARG TARGETOS +ARG TARGETARCH WORKDIR /src @@ -16,7 +23,8 @@ COPY . . # protocol natively (no `git` binary at runtime). RUN --mount=type=cache,target=/go/pkg/mod \ --mount=type=cache,target=/root/.cache/go-build \ - CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" \ + CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH \ + go build -trimpath -ldflags="-s -w" \ -o /objgitd ./cmd/objgitd # --- runtime stage ----------------------------------------------------------- diff --git a/README.md b/README.md index db34bef..62b18ce 100644 --- a/README.md +++ b/README.md @@ -123,4 +123,6 @@ To ssh://localhost:2222/xeiaso.net/objgit.git ### Deployment to production -TODO(Xe): re-evaluate performance-based life choices and write. +There is a kustomize deployment in [`manifest/`](./manifest/). See +[docs/usage/kubernetes.md](./docs/usage/kubernetes.md) for configuration and +caveats. diff --git a/cmd/objgitd/main.go b/cmd/objgitd/main.go index b970979..05cec55 100644 --- a/cmd/objgitd/main.go +++ b/cmd/objgitd/main.go @@ -170,14 +170,7 @@ func main() { os.Exit(1) } runtime.SetBlockProfileRate(100) - mux := http.NewServeMux() - mux.Handle("/metrics", promhttp.Handler()) - mux.HandleFunc("GET /debug/pprof/", pprof.Index) - mux.HandleFunc("GET /debug/pprof/cmdline", pprof.Cmdline) - mux.HandleFunc("GET /debug/pprof/profile", pprof.Profile) - mux.HandleFunc("GET /debug/pprof/symbol", pprof.Symbol) - mux.HandleFunc("GET /debug/pprof/trace", pprof.Trace) - srv := &http.Server{Handler: mux} + srv := &http.Server{Handler: newMetricsMux()} g.Go(func() error { if err := srv.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) { return err @@ -246,3 +239,27 @@ func main() { os.Exit(1) } } + +// newMetricsMux builds the metrics listener's handler: the Prometheus scrape +// endpoint, pprof, and /healthz for probes and load balancers. +func newMetricsMux() *http.ServeMux { + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + mux.HandleFunc("GET /healthz", handleHealthz) + mux.HandleFunc("GET /debug/pprof/", pprof.Index) + mux.HandleFunc("GET /debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("GET /debug/pprof/profile", pprof.Profile) + mux.HandleFunc("GET /debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("GET /debug/pprof/trace", pprof.Trace) + return mux +} + +// handleHealthz answers 200 while the process is serving. The daemon has no +// deeper signal to report. A dead transport listener fails the errgroup and +// exits the process, and the probe turns that exit into a restart. The +// handler reads no bucket state, so a Tigris outage does not fail the probes +// of every pod at the same time. +func handleHealthz(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + fmt.Fprintln(w, "ok") +} diff --git a/cmd/objgitd/main_test.go b/cmd/objgitd/main_test.go new file mode 100644 index 0000000..9dd31ea --- /dev/null +++ b/cmd/objgitd/main_test.go @@ -0,0 +1,70 @@ +package main + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" +) + +// TestMetricsMuxHealthz covers the health endpoint of the metrics listener. +// GET /healthz answers 200 with an ok body for probes. Other methods are +// rejected. /metrics keeps serving the scrape endpoint beside it. +func TestMetricsMuxHealthz(t *testing.T) { + ts := httptest.NewServer(newMetricsMux()) + t.Cleanup(ts.Close) + + for _, tt := range []struct { + name string + method string + path string + wantStatus int + wantBody string + }{ + { + name: "GET /healthz answers 200 ok", + method: http.MethodGet, + path: "/healthz", + wantStatus: http.StatusOK, + wantBody: "ok\n", + }, + { + name: "POST /healthz is not allowed", + method: http.MethodPost, + path: "/healthz", + wantStatus: http.StatusMethodNotAllowed, + }, + { + name: "GET /metrics still serves the scrape endpoint", + method: http.MethodGet, + path: "/metrics", + wantStatus: http.StatusOK, + }, + } { + t.Run(tt.name, func(t *testing.T) { + req, err := http.NewRequest(tt.method, ts.URL+tt.path, nil) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("Do: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != tt.wantStatus { + t.Errorf("status = %d, want %d", resp.StatusCode, tt.wantStatus) + } + if tt.wantBody == "" { + return + } + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + if string(body) != tt.wantBody { + t.Errorf("body = %q, want %q", body, tt.wantBody) + } + }) + } +} diff --git a/docs/architecture/metrics.md b/docs/architecture/metrics.md index cbea291..ba31fbe 100644 --- a/docs/architecture/metrics.md +++ b/docs/architecture/metrics.md @@ -7,7 +7,8 @@ the **default registry**. The Go-runtime and process collectors from `main.go` serves `promhttp.Handler()` on its own listener. The flag is `-metrics-bind`, the default is `:9090`, and an empty value disables the listener. It uses the same errgroup Serve and Shutdown idiom as the HTTP -transport. +transport. The same listener also answers `GET /healthz` with `200 ok` for +probes and load balancers, and serves pprof under `/debug/pprof/`. All series carry the prefix `objgit_`. There is no `repo` label, because repository names have unbounded cardinality. Git operations are keyed by diff --git a/docs/usage/kubernetes.md b/docs/usage/kubernetes.md new file mode 100644 index 0000000..010713e --- /dev/null +++ b/docs/usage/kubernetes.md @@ -0,0 +1,150 @@ +# Kubernetes deployment + +`manifest/` holds a kustomize deployment for `objgitd`. The pod stores all +repository data in a Tigris bucket. A persistent volume holds the pack cache. +An ephemeral volume holds upload scratch data. + +| Object | Purpose | +| ------------------------------------------ | -------------------------------------------------- | +| `Namespace/objgit` | Holds every object below. | +| `ServiceAccount/objgitd` | The pod identity. Users can annotate it for IRSA. | +| `Deployment/objgitd` | The daemon, with all three transports and metrics. | +| `PersistentVolumeClaim/objgitd-pack-cache` | 16Gi from the default StorageClass. | +| `Service/objgit` | HTTP (80), git:// (9418), SSH (22). | +| `Service/objgit-metrics` | Prometheus /metrics (9090), on its own Service. | + +## Configure + +The daemon reads all configuration from environment variables. `flagenv` maps +each flag to one variable (see the README section "Setup instructions"). The +literals live in `manifest/kustomization.yaml`: + +- `BUCKET` in the `objgitd-config` configMapGenerator is **required**. The + daemon does not start when this variable does not name a Tigris bucket. +- The same configMapGenerator sets `AWS_ENDPOINT_URL_S3`, + `AWS_ENDPOINT_URL_IAM`, and `AWS_REGION`. These values point the AWS SDK + default chain at Tigris, in region `auto`. +- The `objgitd-tigris` secretGenerator holds a static Tigris keypair under + the standard AWS names (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`). It + reads the values from files under `manifest/secrets/`, and `.gitignore` + excludes that directory, so the credentials never reach the repository. + Write each file with `printf`, because a trailing newline becomes part of + the credential: + + ```text + printf '%s' '' > manifest/secrets/aws-access-key-id + printf '%s' '' > manifest/secrets/aws-secret-access-key + ``` + + Kustomize refuses to build when the files are missing. For this reason, a + fresh clone cannot apply the manifest until it receives the files through + a secure side channel. For temporary credentials, add a third file line + for `AWS_SESSION_TOKEN`. + +To avoid a long-lived keypair, use IRSA or workload identity: + +1. Delete the `objgitd-tigris` secretGenerator from + `manifest/kustomization.yaml`. +2. Delete the `secretRef` block from `manifest/deployment.yaml`. The block is + optional. +3. Annotate the `objgitd` ServiceAccount for EKS IRSA or GKE workload + identity. + +The AWS SDK default credential chain finds the role on its own. + +CI builds the image on every commit. The workflow lives in +`.github/workflows/docker.yml`. Every push event publishes these tags to +`ghcr.io/tigrisdata/objgit`: + +- `:latest` and `:main` from the main branch. +- `:sha-` from every branch and every release tag. +- One tag per release, such as `:1.5.1`. + +Every tag carries both `linux/amd64` and `linux/arm64` images. Pull requests +from a fork build both platforms and push nothing. The smoke test in the +workflow starts the image and reads `/healthz`. + +The Deployment references `:latest` and sets `imagePullPolicy: Always`, so +Kubernetes pulls the image on every pod start. For a fixed deployment, pin an +immutable `:sha-` tag through an `images:` entry in +`manifest/kustomization.yaml`. + +New GHCR packages start private. Make the package public, or configure an +imagePullSecret in the namespace. To use a different registry: build the repo +`Dockerfile`. Push the image to that registry: + +```text +docker build -t registry.example.com/objgit:1.5.1 . +docker push registry.example.com/objgit:1.5.1 +``` + +Then update the image reference in `manifest/deployment.yaml`. + +## Apply + +```text +kubectl apply -k manifest/ +``` + +## Use + +Both Services are of type ClusterIP. They are reachable only from inside the +cluster. objgitd has no authentication yet. For this reason, the Services +stay cluster-internal. + +The `objgit` Service exposes SSH on port 22 and HTTP on port 80. In the pod, +the daemon still binds port 2222 for SSH and port 8080 for HTTP. The named +target ports in the Service map one to the other: + +```text +git clone http://objgit.objgit.svc.cluster.local/org/repo.git +git clone git://objgit.objgit.svc.cluster.local:9418/org/repo.git +git clone ssh://git@objgit.objgit.svc.cluster.local/org/repo.git +``` + +Scrape metrics from `objgit-metrics.objgit.svc.cluster.local:9090/metrics`. +The same listener serves pprof under `/debug/pprof/`. + +## Health check + +The metrics listener answers `GET /healthz` with `200 ok`. The smart-HTTP mux +returns 404 for every non-git path. For this reason, the probes use the +metrics listener. + +The endpoint reads no bucket state. A Tigris outage therefore does not fail +the probes. + +The startup, liveness, and readiness probes in the manifest all send +`GET /healthz` to the metrics port. + +## Storage notes + +- The **pack cache** (`PACK_CACHE_DIR`) lives on the 16Gi + `objgitd-pack-cache` claim. `PACK_CACHE_BYTES` caps the cache at 14 GiB. + The unused space on the claim holds the temporary files of in-flight + downloads. This claim is the only persistent state. The repositories and + the SSH host key live in the Tigris bucket. On startup, the daemon sweeps + cache directories that an earlier run left behind, so a crash does not + leak disk space across restarts. +- **Upload scratch data** is separate from the pack cache. The tigris storer + stages pack writes to the OS temp directory through `os.CreateTemp`. The + root filesystem of the container is read-only, so the OS temp directory + needs its own `emptyDir` volume (4Gi `sizeLimit`) at `/tmp`. An exceeded + `sizeLimit` evicts the entire pod, which kills every push in flight. For + very large pushes, raise `sizeLimit`, or set `TMPDIR` to a path on the + pack-cache volume. +- The claim has the `ReadWriteOnce` access mode. A rolling update tries to + mount it on two pods at the same time. For this reason, the Deployment + uses the `Recreate` strategy and one replica. Scale-out needs + `ReadOnlyMany` storage or one cache per pod. Reference updates commit + through compare-and-swap on the bucket, so the ref store does not limit + the replica count. + +## Resources + +Each concurrent push costs approximately 400 MiB of resident set for a large +repository. `MAX_CONCURRENT_PUSHES` (4 by default) caps the number of +concurrent pushes. The 4Gi memory limit in the manifest covers four pushes +and fetch traffic. The memory limit and the cap must change together. See +[plans/bound-concurrent-pushes.md](../plans/bound-concurrent-pushes.md) for +the measurements. diff --git a/internal/storage/tigris/packcache.go b/internal/storage/tigris/packcache.go index eca7879..2415989 100644 --- a/internal/storage/tigris/packcache.go +++ b/internal/storage/tigris/packcache.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "runtime" + "strings" "sync" "sync/atomic" "time" @@ -81,6 +82,14 @@ func NewPackCache(parent string, maxBytes int64) (*PackCache, error) { if err != nil { return nil, fmt.Errorf("tigris: create pack cache directory: %w", err) } + // A run that died without Cleanup leaves its cache directory behind. On + // the OS temp directory that is harmless, but on a persistent parent + // (kubernetes -pack-cache-dir on a volume) the orphans survive every + // restart, hold packs no later run can read, and starve maxBytes. One + // daemon owns the parent in every supported deployment, so the sweep is + // safe; a reader from an older process keeps its descriptors through the + // unlink, exactly like an eviction. + sweepOrphanedCaches(parent, dir) return &PackCache{ dir: dir, maxBytes: maxBytes, @@ -88,6 +97,27 @@ func NewPackCache(parent string, maxBytes int64) (*PackCache, error) { }, nil } +// sweepOrphanedCaches removes the objgit-packs-* directories under parent +// that earlier runs left behind, and keeps keep, the directory this run just +// created. It is best effort: a ReadDir failure leaves everything alone, and +// a directory that cannot be removed stays for the next start to try again. +func sweepOrphanedCaches(parent, keep string) { + entries, err := os.ReadDir(parent) + if err != nil { + return + } + for _, e := range entries { + if !e.IsDir() || !strings.HasPrefix(e.Name(), "objgit-packs-") { + continue + } + dir := filepath.Join(parent, e.Name()) + if dir == keep { + continue + } + os.RemoveAll(dir) + } +} + // Cleanup removes the cache directory and everything in it. Call it once at // shutdown. Descriptors already handed out keep working. func (c *PackCache) Cleanup() error { diff --git a/internal/storage/tigris/packcache_test.go b/internal/storage/tigris/packcache_test.go index 568868c..4b435f0 100644 --- a/internal/storage/tigris/packcache_test.go +++ b/internal/storage/tigris/packcache_test.go @@ -69,6 +69,52 @@ func countCacheFiles(t *testing.T, c *PackCache) int { return len(ents) } +// TestPackCacheSweepsOrphans covers the startup sweep: a previous run that +// died without Cleanup leaves its objgit-packs-* directory on disk, and a +// persistent -pack-cache-dir must not accumulate them. +func TestPackCacheSweepsOrphans(t *testing.T) { + t.Parallel() + + parent := t.TempDir() + for _, name := range []string{"objgit-packs-stale", "objgit-packs-also-stale"} { + dir := filepath.Join(parent, name) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", dir, err) + } + if err := os.WriteFile(filepath.Join(dir, "orphan.bin"), []byte("junk"), 0o644); err != nil { + t.Fatalf("write orphan in %s: %v", name, err) + } + } + if err := os.MkdirAll(filepath.Join(parent, "not-a-cache"), 0o755); err != nil { + t.Fatalf("mkdir unrelated: %v", err) + } + + c, err := NewPackCache(parent, 0) + if err != nil { + t.Fatalf("NewPackCache: %v", err) + } + t.Cleanup(func() { _ = c.Cleanup() }) + + for _, tt := range []struct { + path string + want bool // want the path to still exist after the sweep + }{ + {path: filepath.Join(parent, "objgit-packs-stale"), want: false}, + {path: filepath.Join(parent, "objgit-packs-also-stale"), want: false}, + {path: filepath.Join(parent, "not-a-cache"), want: true}, + {path: c.dir, want: true}, + } { + _, err := os.Stat(tt.path) + exists := err == nil + if err != nil && !os.IsNotExist(err) { + t.Fatalf("stat %s: %v", tt.path, err) + } + if exists != tt.want { + t.Errorf("%s exists = %v, want %v", tt.path, exists, tt.want) + } + } +} + // TestPackCacheDownloadsOnce is the whole point of the cache: a second request // for a pack already on disk must not go back to the network. func TestPackCacheDownloadsOnce(t *testing.T) { diff --git a/manifest/deployment.yaml b/manifest/deployment.yaml new file mode 100644 index 0000000..1df24b6 --- /dev/null +++ b/manifest/deployment.yaml @@ -0,0 +1,133 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: objgitd + labels: + app.kubernetes.io/name: objgitd + app.kubernetes.io/component: server + app.kubernetes.io/part-of: objgit +spec: + # One replica. Repositories and the SSH host key live in the Tigris bucket. + # The pack cache is the only node-local state, and this claim is + # ReadWriteOnce. Scale-out needs ReadOnlyMany storage or one cache per pod. + replicas: 1 + strategy: + # A rolling update schedules the replacement pod while the old pod still + # holds the ReadWriteOnce volume. The new pod then blocks until the + # timeout. + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: objgitd + template: + metadata: + labels: + app.kubernetes.io/name: objgitd + app.kubernetes.io/component: server + app.kubernetes.io/part-of: objgit + spec: + serviceAccountName: objgitd + securityContext: + # The image is distroless/static nonroot, uid and gid 65532. fsGroup + # hands the pack-cache volume to that group so the daemon can + # MkdirTemp its cache directory under the mount. Storage classes with + # fsGroupPolicy: None skip this chown. Such classes need a + # securityContext.fsGroupChangePolicy or an initContainer. + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: objgitd + # CI builds this image on every commit and publishes it to + # ghcr.io/tigrisdata/objgit (see .github/workflows/docker.yml). The + # tags are :latest, :main, :sha-, and one per release. GHCR + # packages start private. Make the package public, or configure an + # imagePullSecret. To use a different registry, change this + # reference. + # + # imagePullPolicy: Always pulls on every pod start, so a cached + # :latest on a node cannot pin an old image. Pin an immutable + # :sha- tag through an `images:` entry in + # kustomization.yaml for a fixed deployment. + image: ghcr.io/tigrisdata/objgit:latest + imagePullPolicy: Always + envFrom: + - configMapRef: + name: objgitd-config + - secretRef: + name: objgitd-tigris + # Credentials can also come from the AWS SDK default chain + # (IRSA, workload identity, node profile). Then the Secret is + # never created. + optional: true + ports: + - name: http + containerPort: 8080 + - name: git + containerPort: 9418 + - name: ssh + containerPort: 2222 + - name: metrics + containerPort: 9090 + # /healthz on the metrics listener is the dedicated probe endpoint. + # The smart-HTTP mux 404s every non-git path, so the git ports + # cannot answer probes. + startupProbe: + httpGet: + path: /healthz + port: metrics + periodSeconds: 2 + failureThreshold: 30 + livenessProbe: + httpGet: + path: /healthz + port: metrics + periodSeconds: 10 + readinessProbe: + httpGet: + path: /healthz + port: metrics + periodSeconds: 5 + resources: + # Each concurrent push costs roughly 400 MiB of resident set for + # a large repository. The limit must cover MAX_CONCURRENT_PUSHES + # (4 by default) plus fetch traffic. See + # docs/plans/bound-concurrent-pushes.md. + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "2" + memory: 4Gi + volumeMounts: + - name: pack-cache + mountPath: /var/cache/objgit + # The tigris storer stages uploads in the OS temp directory + # through os.CreateTemp (internal/storage/tigris/writer.go and + # packwriter.go). Distroless ships a writable /tmp, but the root + # filesystem of the container is read-only, so the OS temp + # directory needs its own volume. + # + # CAUTION: Do not set sizeLimit below the largest expected push. + # A pack stages whole in /tmp, and an exceeded sizeLimit evicts + # the entire pod, which kills every push in flight. For very + # large pushes, raise sizeLimit, or set TMPDIR to a path on the + # pack-cache volume. + - name: tmp + mountPath: /tmp + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + volumes: + - name: pack-cache + persistentVolumeClaim: + claimName: objgitd-pack-cache + - name: tmp + emptyDir: + sizeLimit: 4Gi diff --git a/manifest/kustomization.yaml b/manifest/kustomization.yaml new file mode 100644 index 0000000..e094dd3 --- /dev/null +++ b/manifest/kustomization.yaml @@ -0,0 +1,90 @@ +# Kustomize deployment for objgitd, the objgit daemon. +# +# 1. CI builds an image on every commit and publishes it to +# ghcr.io/tigrisdata/objgit. Use those tags, or build the repo Dockerfile +# and push it to a registry you control. Then update the image reference +# in deployment.yaml. +# 2. Set BUCKET in the configMapGenerator below. +# 3. Set the Tigris credentials in the secretGenerator below. For IRSA or +# workload identity, see docs/usage/kubernetes.md. +# 4. kubectl apply -k manifest/ +# +# See docs/usage/kubernetes.md for the full walkthrough. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: objgit + +resources: + - namespace.yaml + - serviceaccount.yaml + - pvc.yaml + - deployment.yaml + - service.yaml + - service-metrics.yaml + +# Pin an immutable image tag for a fixed deployment. The Deployment references +# :latest, which Kubernetes pulls on every pod start. Uncomment and set the +# tag to a specific commit: +# +# images: +# - name: ghcr.io/tigrisdata/objgit +# newTag: sha-a1b2c3d + +configMapGenerator: + # Daemon configuration. flagenv maps every flag to UPPER_SNAKE_CASE, and + # environment variables override the flag defaults (README "Setup + # instructions"). + - name: objgitd-config + literals: + # Required: the Tigris bucket that holds every repository and the SSH + # host key. objgitd does not start when this value is empty. + - BUCKET=replace-me + # Transports. HTTP and git:// are on by default. SSH needs a bind value + # to turn on. Every bind is a pod port named below. + - HTTP_BIND=:8080 + - GIT_BIND=:9418 + - SSH_BIND=:2222 + # Prometheus /metrics (and pprof), exposed on the objgit-metrics + # Service. The probes in deployment.yaml read /healthz on this + # listener. An empty value therefore breaks every probe, because it + # disables the listener. + - METRICS_BIND=:9090 + # The AWS SDK default chain reads these three values. They point every + # SDK client at Tigris, in region auto. The storage-go wrapper pins + # the S3 endpoint itself, so the IAM endpoint matters most for the + # credential chain of IRSA. + - AWS_ENDPOINT_URL_S3=https://t3.storage.dev + - AWS_ENDPOINT_URL_IAM=https://iam.storage.dev + - AWS_REGION=auto + # The pack cache lives on the objgitd-pack-cache PVC. 14 GiB (15032385536 + # bytes) leaves headroom under the 16Gi claim. + - PACK_CACHE_DIR=/var/cache/objgit + - PACK_CACHE_BYTES=15032385536 + # Anonymous pushes over every transport. Set false for a read-only + # mirror. There is no other authentication yet (README "Notable + # features"). Keep the Service cluster-internal, or put your own + # authentication in front. + - ALLOW_PUSH=true + # Run .objgit/hooks/receive-pack from the repository after a push. See + # docs/usage/hooks.md. + - ALLOW_HOOKS=false + - SLOG_LEVEL=INFO + +secretGenerator: + # Static Tigris credentials under their standard AWS names, read from + # files under secrets/. That directory is in .gitignore, so the values + # never reach this repository. Write each file with printf '%s', because + # a trailing newline becomes part of the credential. Kustomize refuses to + # build when a file is missing, which prevents an accidental deploy with + # placeholder values. For temporary credentials, add a third file line + # for AWS_SESSION_TOKEN. + # + # To avoid a long-lived keypair: delete this generator and the secretRef in + # deployment.yaml (it is optional). Then annotate the objgitd + # ServiceAccount for IRSA or workload identity. The default chain finds + # the role on its own. + - name: objgitd-tigris + files: + - AWS_ACCESS_KEY_ID=secrets/aws-access-key-id + - AWS_SECRET_ACCESS_KEY=secrets/aws-secret-access-key diff --git a/manifest/namespace.yaml b/manifest/namespace.yaml new file mode 100644 index 0000000..be296ed --- /dev/null +++ b/manifest/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: objgit diff --git a/manifest/pvc.yaml b/manifest/pvc.yaml new file mode 100644 index 0000000..c9b57a6 --- /dev/null +++ b/manifest/pvc.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: objgitd-pack-cache + labels: + app.kubernetes.io/name: objgitd + app.kubernetes.io/component: server + app.kubernetes.io/part-of: objgit +spec: + accessModes: + - ReadWriteOnce + # storageClassName is deliberately unset, so the cluster uses its default + # StorageClass. If the default is not the provisioner you want, name a + # class explicitly. + resources: + requests: + storage: 16Gi diff --git a/manifest/service-metrics.yaml b/manifest/service-metrics.yaml new file mode 100644 index 0000000..7e1f1f0 --- /dev/null +++ b/manifest/service-metrics.yaml @@ -0,0 +1,17 @@ +# Prometheus /metrics (and pprof), kept off the git Service on purpose. +apiVersion: v1 +kind: Service +metadata: + name: objgit-metrics + labels: + app.kubernetes.io/name: objgitd + app.kubernetes.io/component: metrics + app.kubernetes.io/part-of: objgit +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: objgitd + ports: + - name: metrics + port: 9090 + targetPort: metrics diff --git a/manifest/service.yaml b/manifest/service.yaml new file mode 100644 index 0000000..42e8e78 --- /dev/null +++ b/manifest/service.yaml @@ -0,0 +1,25 @@ +# The git transports: smart HTTP, git://, and SSH. Metrics is deliberately +# absent. It lives on the objgit-metrics Service, so a scrape ACL or a +# NetworkPolicy can gate it separately from git traffic. +apiVersion: v1 +kind: Service +metadata: + name: objgit + labels: + app.kubernetes.io/name: objgitd + app.kubernetes.io/component: server + app.kubernetes.io/part-of: objgit +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: objgitd + ports: + - name: http + port: 80 + targetPort: http + - name: git + port: 9418 + targetPort: git + - name: ssh + port: 22 + targetPort: ssh diff --git a/manifest/serviceaccount.yaml b/manifest/serviceaccount.yaml new file mode 100644 index 0000000..33ba30f --- /dev/null +++ b/manifest/serviceaccount.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: objgitd + # To use EKS IRSA or GKE workload identity instead of a static Tigris + # keypair: annotate this account, and delete the objgitd-tigris Secret + # reference from deployment.yaml. + # + # eks.amazonaws.com/role-arn: arn:aws:iam::ACCOUNT:role/objgitd +# annotations: {}