diff --git a/.gitignore b/.gitignore index 5df20b2..30e58fd 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ manifest/secrets/ node_modules + +# Claude Code local agent state: per-user settings and scratch worktrees. +.claude/ diff --git a/AGENTS.md b/AGENTS.md index 6dc41e1..ded02a4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,6 +18,10 @@ One backend answers three transports: All three funnel authorization through one pluggable `internal/auth.Authorizer`. +Git LFS rides on the Smart HTTP listener, behind `-allow-lfs`. It is HTTP even +for an `ssh://` remote: SSH only answers `git-lfs-authenticate`, which names the +HTTP endpoint. See [docs/architecture/lfs.md](docs/architecture/lfs.md). + A fourth listener serves Prometheus metrics at `/metrics` and a `/healthz` probe endpoint (`-metrics-bind`, default `:9090`). An empty value disables it. @@ -60,7 +64,9 @@ Notes on the tests and on configuration: | `cmd/objgitd/receivepack.go` | The go-git fork that streams hook output, plus `writePack`. | | `cmd/objgitd/hooks.go` | Ref diffing and the sandboxed hook run. | | `cmd/objgitd/snapshots.go` | The erofs snapshot run after a push. | +| `cmd/objgitd/lfs.go` | The Git LFS HTTP handlers and `git-lfs-authenticate` over SSH. | | `internal/auth` | The one authorization interface. | +| `internal/lfs` | Git LFS: protocol types, the bucket store, and the presigner. | | `internal/repofs` | Maps a repository path to a `storage.Storer`. | | `internal/storage/tigris` | Repository storage. A `storage.Storer` on the bucket. | | `internal/bundler` | The async upload queue behind that storer. | @@ -85,6 +91,7 @@ describes the daemon and links to one page for each subsystem. | [snapshots.md](docs/architecture/snapshots.md) | Snapshot images, the snapshot cache, or `runSnapshots`. | | [tigris-storer.md](docs/architecture/tigris-storer.md) | Object layout, refs, packs, the pack cache, or the upload path. | | [s3fs.md](docs/architecture/s3fs.md) | The `billy.Filesystem` over the bucket. | +| [lfs.md](docs/architecture/lfs.md) | Git LFS: the batch API, presigned transfers, locks, or the blob layout. | Two more directories carry detail: diff --git a/README.md b/README.md index 62b18ce..e46d5c6 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ Price: Free as in mattress. If you use this, it's your problem. - ssh, http, and git protocol support. - [post-receive hooks](./docs/usage/hooks.md) powered by userspace sandboxed shells and [Kefka](https://xeiaso.net/blog/2026/dancing-mad-sandboxing/). +- [Git LFS](./docs/usage/lfs.md) with presigned transfers, so large files move straight between the client and Tigris instead of through the daemon. Object bytes are deduplicated across every repository in the bucket. - basic prometheus metrics. - no authentication whatsoever, if this ends up being actually useful then authentication will be implemented. diff --git a/cmd/formatbench/bucket.go b/cmd/formatbench/bucket.go new file mode 100644 index 0000000..55d62da --- /dev/null +++ b/cmd/formatbench/bucket.go @@ -0,0 +1,164 @@ +package main + +import ( + "context" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + tstorage "github.com/tigrisdata/storage-go" +) + +// bucketUsage is what one repository costs in the bucket after a push: how many +// keys it left and how many bytes those keys hold. Both eras address a +// repository by the same key prefix, so one listing measures either of them. +// The "before" build reaches that prefix through go-git's loader over +// internal/s3fs; the "after" build reaches it through +// repofs.BucketResolver.Resolve. +type bucketUsage struct { + Keys int `json:"keys"` + Bytes int64 `json:"bytes"` +} + +// bucketClient reads the bucket the daemon under test writes to. It is the +// harness's own client, separate from the daemon's, so its calls never land in +// the daemon's own S3 counters. +type bucketClient struct { + api *tstorage.Client + bucket string +} + +func newBucketClient(ctx context.Context, bucket string) (*bucketClient, error) { + if bucket == "" { + return nil, fmt.Errorf("formatbench: -bucket is required (BUCKET in .env)") + } + + api, err := tstorage.New(ctx) + if err != nil { + return nil, fmt.Errorf("formatbench: can't create Tigris storage client: %w", err) + } + + return &bucketClient{api: api, bucket: bucket}, nil +} + +// keyPrefixes returns every bucket prefix a repository can live under, because +// the two builds disagree about the ".git" suffix. +// +// The "after" build parses the transport path through repofs.Parse, which +// strips a trailing ".git" (internal/repofs/repofs.go:47), so its keys sit +// under "//". The "before" build hands the raw URL path to go-git's +// server loader over internal/s3fs, which keeps the suffix, so its keys sit +// under "/.git/". Only one of the two exists for any measurement, +// so walking both and summing is safe and keeps the harness from having to +// know which era it is talking to. +func keyPrefixes(prefix string) []string { + prefix = strings.TrimSuffix(prefix, "/") + prefix = strings.TrimSuffix(prefix, ".git") + + return []string{prefix + "/", prefix + ".git/"} +} + +// usage walks every prefix a repository can live under and sums what it finds. +func (c *bucketClient) usage(ctx context.Context, prefix string) (bucketUsage, error) { + var out bucketUsage + + for _, p := range keyPrefixes(prefix) { + err := c.walk(ctx, p, func(key string, size int64) error { + out.Keys++ + out.Bytes += size + return nil + }) + if err != nil { + return out, err + } + } + + return out, nil +} + +// deleteBatch is how many keys one DeleteObjects call carries. S3 caps a single +// request at 1000 keys, so that cap is what the chunking below is for. +const deleteBatch = 1000 + +// remove deletes every key a repository left behind. It is only reached from +// -cleanup; a benchmark run never deletes anything on its own. +// +// The count it returns is how many keys the bucket confirmed gone, including on +// the error path, where it is the keys deleted before the failure and not the +// whole chunk that failure happened in. +func (c *bucketClient) remove(ctx context.Context, prefix string) (int, error) { + var keys []string + for _, p := range keyPrefixes(prefix) { + if err := c.walk(ctx, p, func(key string, _ int64) error { + keys = append(keys, key) + return nil + }); err != nil { + return 0, err + } + } + + deleted := 0 + for i := 0; i < len(keys); i += deleteBatch { + end := min(i+deleteBatch, len(keys)) + + objects := make([]types.ObjectIdentifier, 0, end-i) + for _, key := range keys[i:end] { + objects = append(objects, types.ObjectIdentifier{Key: aws.String(key)}) + } + + out, err := c.api.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &c.bucket, + Delete: &types.Delete{Objects: objects}, + }) + if err != nil { + return deleted, fmt.Errorf("formatbench: can't delete %d keys under %q: %w", len(objects), prefix, err) + } + + // DeleteObjects answers 200 even when some of the keys failed, so a + // per-key failure arrives in the body and never as err. + deleted += len(out.Deleted) + if len(out.Errors) > 0 { + bad := out.Errors[0] + return deleted, fmt.Errorf("formatbench: can't delete %q (%d of %d keys failed): %s: %s", + aws.ToString(bad.Key), len(out.Errors), len(objects), aws.ToString(bad.Code), aws.ToString(bad.Message)) + } + } + + return deleted, nil +} + +// walk pages one exact prefix and calls fn for every key. +func (c *bucketClient) walk(ctx context.Context, prefix string, fn func(key string, size int64) error) error { + token := "" + for { + in := &s3.ListObjectsV2Input{Bucket: &c.bucket, Prefix: &prefix} + if token != "" { + in.ContinuationToken = &token + } + + page, err := c.api.ListObjectsV2(ctx, in) + if err != nil { + return fmt.Errorf("formatbench: can't list %q: %w", prefix, err) + } + + for _, entry := range page.Contents { + if entry.Key == nil { + continue + } + var size int64 + if entry.Size != nil { + size = *entry.Size + } + if err := fn(*entry.Key, size); err != nil { + return err + } + } + + if page.IsTruncated == nil || !*page.IsTruncated || page.NextContinuationToken == nil { + return nil + } + token = *page.NextContinuationToken + } +} diff --git a/cmd/formatbench/bucket_test.go b/cmd/formatbench/bucket_test.go new file mode 100644 index 0000000..0ca516f --- /dev/null +++ b/cmd/formatbench/bucket_test.go @@ -0,0 +1,34 @@ +package main + +import ( + "slices" + "testing" +) + +// The two builds disagree about the ".git" suffix, so a measurement has to look +// under both spellings or the older one silently reports an empty bucket. +func TestKeyPrefixes(t *testing.T) { + t.Parallel() + + want := []string{"formatbench/before-objgit-abc/", "formatbench/before-objgit-abc.git/"} + + for _, tt := range []struct { + name string + in string + }{ + {"bare prefix", "formatbench/before-objgit-abc"}, + {"already carries the suffix", "formatbench/before-objgit-abc.git"}, + {"trailing slash", "formatbench/before-objgit-abc/"}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := keyPrefixes(tt.in) + if !slices.Equal(got, want) { + t.Logf("want: %v", want) + t.Logf("got: %v", got) + t.Error("prefix set is wrong") + } + }) + } +} diff --git a/cmd/formatbench/git.go b/cmd/formatbench/git.go new file mode 100644 index 0000000..880782e --- /dev/null +++ b/cmd/formatbench/git.go @@ -0,0 +1,148 @@ +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "os/exec" + "regexp" + "strconv" + "strings" + "time" +) + +// gitEnv isolates every git subprocess from the operator's own configuration, +// so a stray push.default or http.postBuffer in ~/.gitconfig cannot move a +// measurement. Same treatment tryGit gives the protocol tests. +var gitEnv = []string{ + "GIT_CONFIG_GLOBAL=/dev/null", + "GIT_CONFIG_SYSTEM=/dev/null", + "GIT_TERMINAL_PROMPT=0", + "GIT_ASKPASS=/bin/true", +} + +// opResult is one timed git operation against the daemon under test. +type opResult struct { + Wall time.Duration `json:"wall_ns"` + WallStr string `json:"wall"` + + S3 s3Counters `json:"s3"` + + // WireBytes is what git reported receiving. Clone only; a push does not + // print a comparable figure. + WireBytes int64 `json:"wire_bytes,omitempty"` + + // DNF marks an operation the harness stopped at its time limit. Its Wall is + // the limit, not a completion time, and the S3 counters are however far it + // got. A DNF is a result and gets reported as one; it is never extrapolated + // into a finish time. + DNF bool `json:"dnf,omitempty"` + + Err string `json:"err,omitempty"` +} + +func (r opResult) ok() bool { return r.Err == "" && !r.DNF } + +// runGit runs one git command under a time limit and returns its combined +// output. The boolean reports whether the limit was what stopped it. +func runGit(ctx context.Context, limit time.Duration, dir string, args ...string) (string, bool, error) { + runCtx, cancel := context.WithTimeout(ctx, limit) + defer cancel() + + cmd := exec.CommandContext(runCtx, "git", args...) + cmd.Dir = dir + cmd.Env = append(cmd.Environ(), gitEnv...) + + var buf bytes.Buffer + cmd.Stdout = &buf + cmd.Stderr = &buf + + err := cmd.Run() + out := strings.TrimSpace(buf.String()) + + // A timeout kills the child, which surfaces as a signal error rather than a + // deadline error, so ask the context which one happened. + if errors.Is(runCtx.Err(), context.DeadlineExceeded) { + return out, true, fmt.Errorf("git %s stopped at the %s limit", args[0], limit) + } + if err != nil { + return out, false, fmt.Errorf("git %s: %w: %s", args[0], err, out) + } + + return out, false, nil +} + +// gitPush sends the whole repository to a prefix that does not exist yet. +// +// --mirror rather than a heads-and-tags refspec, because the point is to +// measure storing an entire repository: a mirror clone of a GitHub project +// also carries refs/notes/* and refs/pull/*, and those hold commits that a +// heads-and-tags push leaves behind. Leaving them behind would understate the +// work on both sides of the comparison and would make every clone disagree +// with its mirror in verifyClone. +func gitPush(ctx context.Context, limit time.Duration, mirror, url string) (time.Duration, bool, error) { + start := time.Now() + _, dnf, err := runGit(ctx, limit, mirror, "push", "--quiet", "--mirror", url) + + return time.Since(start), dnf, err +} + +// gitClone pulls the repository back out. --progress forces the receive +// counters onto stderr even though the harness is not a terminal, which is +// where the wire byte count comes from. +func gitClone(ctx context.Context, limit time.Duration, url, dst string) (time.Duration, int64, bool, error) { + start := time.Now() + out, dnf, err := runGit(ctx, limit, "", "clone", "--mirror", "--progress", url, dst) + wall := time.Since(start) + + return wall, parseReceivedBytes(out), dnf, err +} + +// receivedRe matches the byte total git prints once a pack is fully received, +// for example "Receiving objects: 100% (59459/59459), 80.86 MiB | 4.20 MiB/s". +// A small pack is reported in bytes instead ("244 bytes"), so both spellings +// are accepted. Only the 100% line matches, so the partial progress lines that +// precede it on the same stream are ignored. +var receivedRe = regexp.MustCompile(`Receiving objects: *100%[^,]*, *([0-9.]+) *(bytes|[KMG]iB)`) + +// parseReceivedBytes pulls the pack size off git's progress output. It returns +// zero when the line is absent, which happens for an empty clone or a clone +// that failed before any data moved. +func parseReceivedBytes(out string) int64 { + m := receivedRe.FindStringSubmatch(out) + if m == nil { + return 0 + } + + n, err := strconv.ParseFloat(m[1], 64) + if err != nil { + return 0 + } + + switch m[2] { + case "KiB": + n *= 1 << 10 + case "MiB": + n *= 1 << 20 + case "GiB": + n *= 1 << 30 + } + + return int64(n) +} + +// mirrorClone copies a source repository once. Every push in the run reads from +// this immutable copy, so the harness never writes to whatever it was pointed +// at and every repetition sends byte-identical history. +// +// It goes through runGit for the -mirror-timeout limit: a hung fetch from a +// forge would otherwise block the whole run on the bare signal context, with +// nothing in the output to say which step stopped. +func mirrorClone(ctx context.Context, src, dst string) error { + if _, _, err := runGit(ctx, *mirrorTimeout, "", "clone", "--mirror", src, dst); err != nil { + return fmt.Errorf("formatbench: can't mirror %s: %w", src, err) + } + + return nil +} diff --git a/cmd/formatbench/git_test.go b/cmd/formatbench/git_test.go new file mode 100644 index 0000000..c14b86e --- /dev/null +++ b/cmd/formatbench/git_test.go @@ -0,0 +1,78 @@ +package main + +import "testing" + +// scale states an expected byte count the way git prints it, as a decimal +// times a power-of-two unit. A plain constant expression will not do: the +// products here are not whole numbers, and Go refuses to convert a +// non-integral untyped constant to int64. +func scale(n float64, unit int64) int64 { return int64(n * float64(unit)) } + +func TestParseReceivedBytes(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + out string + want int64 + }{ + { + name: "mebibytes", + out: "Receiving objects: 100% (59459/59459), 80.86 MiB | 4.20 MiB/s, done.", + want: scale(80.86, 1<<20), + }, + { + name: "kibibytes", + out: "Receiving objects: 100% (318/318), 200.50 KiB | 1.10 MiB/s, done.", + want: scale(200.50, 1<<10), + }, + { + name: "a small pack is reported in plain bytes", + out: "Receiving objects: 100% (3/3), 244 bytes | 244.00 KiB/s, done.", + want: 244, + }, + { + name: "partial progress lines are ignored, only the 100% line counts", + out: "Receiving objects: 47% (28000/59459), 40.00 MiB\r" + + "Receiving objects: 100% (59459/59459), 80.86 MiB | 4.20 MiB/s, done.", + want: scale(80.86, 1<<20), + }, + { + name: "no receive line at all", + out: "warning: You appear to have cloned an empty repository.", + want: 0, + }, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := parseReceivedBytes(tt.out); got != tt.want { + t.Logf("want: %d", tt.want) + t.Logf("got: %d", got) + t.Error("wire byte count is wrong") + } + }) + } +} + +func TestOpResultOK(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + in opResult + want bool + }{ + {"finished cleanly", opResult{}, true}, + {"hit its time limit", opResult{DNF: true}, false}, + {"failed", opResult{Err: "connection refused"}, false}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := tt.in.ok(); got != tt.want { + t.Errorf("want %v, got %v", tt.want, got) + } + }) + } +} diff --git a/cmd/formatbench/main.go b/cmd/formatbench/main.go new file mode 100644 index 0000000..91180aa --- /dev/null +++ b/cmd/formatbench/main.go @@ -0,0 +1,1007 @@ +// Command formatbench measures what objgit's storage format change cost and +// bought. It builds two daemons — one from the release before the change, one +// from the current checkout — and pushes the same real repositories into both, +// then clones each one back out. +// +// Before commits f4a1419 and ebfe4e3 (2026-08-27), objgitd stored git's native +// bare-repository layout as bucket keys through internal/s3fs. After them it +// stores packs/.bin plus packs/.cue containers through +// internal/storage/tigris. Both builds report the same Prometheus counter, +// objgit_s3_requests_total, with the same operation labels, so the request +// count that each format costs is comparable and no proxy is needed. +// +// Comparable, but not free: the counter is per process, not per request, so +// anything the daemon does in the background lands in it too. The older build +// re-lists hot prefixes on a wall-clock timer, which would charge it for how +// long an operation took rather than for what the operation did. quiesceArgs +// turns that off, and a build only gets a flag its own source defines. +// +// Every measurement gets a fresh repository prefix, a fresh pack cache +// directory, and a fresh daemon process, so nothing is ever measured warm by +// accident. Builds are interleaved rather than batched: network conditions +// drift over an afternoon, and running every "before" cell first would let that +// drift look like a result. +// +// The harness never deletes bucket data during a run. It writes repos.txt and +// leaves cleanup to a later -cleanup invocation. +package main + +import ( + "bytes" + "context" + "errors" + "flag" + "fmt" + "log/slog" + "net/http" + "os" + "os/exec" + "os/signal" + "path/filepath" + "runtime" + "slices" + "strconv" + "strings" + "syscall" + "time" + + "github.com/facebookgo/flagenv" + "github.com/tigrisdata/objgit/internal" + + _ "github.com/joho/godotenv/autoload" +) + +// corpus is the default set of repositories. The three shapes matter more than +// the three names: a small mostly-text repository, a medium one with a large +// object count, and one dominated by images, which do not compress. +var corpus = []repoSpec{ + {Name: "objgit", URL: "https://github.com/tigrisdata/objgit", Reps: 3}, + {Name: "x", URL: "https://github.com/Xe/x", Reps: 2}, + {Name: "tigris-blog", URL: "https://github.com/tigrisdata/tigris-blog", Reps: 3}, +} + +// repoSpec is one corpus entry. Reps is how many times it is pushed per build; +// a bigger repository gets fewer repetitions because each one is expensive. +type repoSpec struct { + Name string `json:"name"` + URL string `json:"url"` + Reps int `json:"reps"` + + Mirror string `json:"mirror"` + Commits string `json:"commits,omitempty"` + Pack int64 `json:"pack_bytes,omitempty"` +} + +// buildSpec is one side of the comparison. Ref is the git ref its binary is +// built from; an empty Ref means the current checkout. +type buildSpec struct { + Name string `json:"name"` + Ref string `json:"ref"` + + Bin string `json:"bin"` + + // Commit is the SHA the binary was built from, with "-dirty" appended when + // the source carried uncommitted changes. Ref pins nothing on its own: + // "after" is whatever the checkout happened to hold, and a branch named + // through = moves between runs. Without this a result cannot be + // checked after the fact. + Commit string `json:"commit,omitempty"` + + // Args are the extra daemon flags this build's own source defines. See + // quiesceArgs. + Args []string `json:"args,omitempty"` +} + +var ( + outBase = flag.String("out", "", "parent directory for run output; empty uses var/bench/out under the checkout") + org = flag.String("org", "formatbench", "org segment every benchmark repository is created under") + bucket = flag.String("bucket", "", "Tigris bucket the daemon under test writes to; read from BUCKET in .env when empty") + + repos = flag.String("repos", "", "comma-separated corpus entries to run; empty runs all of them") + builds = flag.String("builds", "before,after", "comma-separated sides to run; 'before' and 'after'") + beforeRef = flag.String("before-ref", "v1.0.2", "git ref the 'before' daemon is built from; the last release where repositories live in the single -bucket through internal/s3fs") + reps = flag.Int("reps", 0, "override the per-repository repetition count; 0 keeps each entry's own") + + mirrorDir = flag.String("mirror-dir", "", "directory holding the mirror clones, reused across runs; empty uses var/bench/mirrors") + worktreeDir = flag.String("worktree-dir", "", "directory holding the 'before' worktree; empty uses var/bench/src") + refreshMirrors = flag.Bool("refresh-mirrors", false, "fetch each mirror before the run; off by default so repeated runs measure identical history") + + pushTimeout = flag.Duration("push-timeout", 45*time.Minute, "time limit for one push; a push that hits it is recorded as DNF") + cloneTimeout = flag.Duration("clone-timeout", 30*time.Minute, "time limit for one clone; a clone that hits it is recorded as DNF") + // An hour, longer than the two above, because those cover a transfer + // against a daemon on localhost while this one pulls a whole repository + // from a forge. It is the slowest single git call the harness makes. + mirrorTimeout = flag.Duration("mirror-timeout", 60*time.Minute, "time limit for the one-time mirror clone of a source repository") + verify = flag.Bool("verify", true, "compare each clone against its mirror and mark the cell INVALID when they differ") + + httpBind = flag.String("http-bind", "127.0.0.1:8080", "address the daemon under test serves smart HTTP on") + metricsBind = flag.String("metrics-bind", "127.0.0.1:9090", "address the daemon under test serves /metrics on") + readyWait = flag.Duration("ready-wait", 60*time.Second, "how long to wait for the daemon to answer /metrics before giving up") + + cleanup = flag.String("cleanup", "", "path to a repos.txt from an earlier run; deletes every prefix in it and exits without benchmarking") + + slogLevel = flag.String("slog-level", "INFO", "log level (DEBUG, INFO, WARN, ERROR)") +) + +func main() { + flagenv.Parse() + flag.Parse() + + logger, err := internal.InitSlog(*slogLevel) + if err != nil { + fmt.Fprintf(os.Stderr, "bad -slog-level: %v\n", err) + os.Exit(1) + } + slog.SetDefault(logger) + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + if *cleanup != "" { + if err := runCleanup(ctx, *cleanup); err != nil { + slog.Error("cleanup failed", "err", err) + os.Exit(1) + } + return + } + + if err := run(ctx); err != nil { + slog.Error("benchmark failed", "err", err) + os.Exit(1) + } +} + +func run(ctx context.Context) error { + root, err := moduleRoot() + if err != nil { + return err + } + + bucketName := bucketFromFlagOrEnv() + bc, err := newBucketClient(ctx, bucketName) + if err != nil { + return err + } + + selected, err := pickRepos(*repos) + if err != nil { + return err + } + + sides, err := pickBuilds(*builds) + if err != nil { + return err + } + + startedAt := time.Now() + runDir := filepath.Join(outDir(root), startedAt.Format("20060102-150405")) + if err := os.MkdirAll(runDir, 0o755); err != nil { + return fmt.Errorf("formatbench: can't create %s: %w", runDir, err) + } + slog.Info("run directory", "path", runDir, "bucket", bucketName) + + for i := range sides { + if err := buildSide(ctx, root, runDir, &sides[i]); err != nil { + return err + } + } + + for i := range selected { + if err := prepareMirror(ctx, root, &selected[i]); err != nil { + return err + } + } + + meta := runMeta{ + StartedAt: startedAt, + Host: hostname(), + GoVersion: runtime.Version(), + GitVersion: gitVersion(ctx), + NumCPU: runtime.NumCPU(), + Bucket: bucketName, + Org: *org, + Builds: sides, + Repos: selected, + RunDir: runDir, + } + + cells, runErr := drive(ctx, bc, runDir, sides, selected) + meta.FinishedAt = time.Now() + + if err := writeResults(runDir, meta, cells); err != nil { + return errors.Join(runErr, err) + } + + fmt.Printf("\n%s\n\nRepositories left in the bucket are listed in %s.\nDelete them with: go run ./cmd/formatbench -cleanup %s\n", + filepath.Join(runDir, "results.md"), + filepath.Join(runDir, "repos.txt"), + filepath.Join(runDir, "repos.txt")) + + return runErr +} + +// drive walks the schedule. It returns whatever it recorded even when a cell +// fails, because a partial table is still worth reading. +func drive(ctx context.Context, bc *bucketClient, runDir string, sides []buildSpec, selected []repoSpec) ([]cell, error) { + var cells []cell + + for _, c := range schedule(sides, selected) { + if err := ctx.Err(); err != nil { + return cells, err + } + + name, err := newUUID() + if err != nil { + return cells, err + } + c.Prefix = fmt.Sprintf("%s/%s-%s-%s", *org, c.Build, c.Repo, name) + + slog.Info("cell", "build", c.Build, "repo", c.Repo, "rep", c.Rep, "prefix", c.Prefix) + runCell(ctx, bc, runDir, sides, selected, &c) + cells = append(cells, c) + + // Write after every cell. A run can take hours, and an interrupted one + // must still leave a readable table and a complete cleanup list. + if err := appendRepoList(filepath.Join(runDir, "repos.txt"), c.Prefix); err != nil { + slog.Warn("can't record prefix for cleanup", "prefix", c.Prefix, "err", err) + } + } + + return cells, nil +} + +// schedule interleaves the builds. Repetition is the outer loop, so the order +// is (rep 1: every repo on every build), then (rep 2: the same), rather than +// every "before" cell followed by every "after" cell. +func schedule(sides []buildSpec, selected []repoSpec) []cell { + most := 0 + for _, r := range selected { + most = max(most, repCount(r)) + } + + var out []cell + for rep := 1; rep <= most; rep++ { + for _, r := range selected { + if rep > repCount(r) { + continue + } + for _, s := range sides { + out = append(out, cell{Build: s.Name, Repo: r.Name, Rep: rep}) + } + } + } + + return out +} + +func repCount(r repoSpec) int { + if *reps > 0 { + return *reps + } + return r.Reps +} + +// runCell does one push and one clone, each against its own fresh daemon. The +// daemon restarts between them so the clone is cold in the process, on disk, +// and in the pack index that (*Storer).ensurePacksBuilt rebuilds per request. +func runCell(ctx context.Context, bc *bucketClient, runDir string, sides []buildSpec, selected []repoSpec, c *cell) { + side, ok := findBuild(sides, c.Build) + if !ok { + c.Push.Err = "unknown build " + c.Build + return + } + + repo, ok := findRepo(selected, c.Repo) + if !ok { + c.Push.Err = "unknown repo " + c.Repo + return + } + + root, err := moduleRoot() + if err != nil { + c.Push.Err = err.Error() + return + } + + url := fmt.Sprintf("http://%s/%s.git", *httpBind, c.Prefix) + tag := fmt.Sprintf("%s-%s-%d", c.Build, c.Repo, c.Rep) + + c.Push = measure(ctx, root, runDir, side, tag+"-push", func(runCtx context.Context) (time.Duration, int64, bool, error) { + wall, dnf, err := gitPush(runCtx, *pushTimeout, repo.Mirror, url) + return wall, 0, dnf, err + }) + + // A push that never finished still left keys behind, and how far it got is + // the interesting part of a DNF. Measure the bucket either way. + // + // usage returns whatever it summed before it failed, so a listing throttled + // partway through the tens of thousands of loose-object keys the old format + // writes comes back short and renders as a complete figure. That + // understates the older build, which is the direction that would turn the + // comparison around, so a failed listing invalidates the cell instead. The + // push timing goes with it: cell has one validity flag, and printing a + // timing beside a key count nobody can stand behind is what this is trying + // to avoid. + usage, err := bc.usage(ctx, c.Prefix) + if err != nil { + slog.Error("can't measure bucket usage", "prefix", c.Prefix, "err", err) + c.Usage = bucketUsage{} + if c.Push.Err == "" { + c.Push.Err = fmt.Sprintf("the push finished but the bucket listing did not: %v", err) + } + } else { + c.Usage = usage + } + + if !c.Push.ok() { + slog.Warn("skipping clone because the push did not finish", "build", c.Build, "repo", c.Repo, "dnf", c.Push.DNF, "err", c.Push.Err) + return + } + + dst := filepath.Join(runDir, "clones", tag+".git") + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + c.Clone.Err = err.Error() + return + } + defer os.RemoveAll(dst) + + c.Clone = measure(ctx, root, runDir, side, tag+"-clone", func(runCtx context.Context) (time.Duration, int64, bool, error) { + return gitClone(runCtx, *cloneTimeout, url, dst) + }) + + if *verify && c.Clone.ok() { + c.Verify = verifyClone(ctx, repo.Mirror, dst) + if !c.Verify.OK { + slog.Error("clone does not match the mirror", "build", c.Build, "repo", c.Repo, "problems", c.Verify.Problems) + } + } +} + +// measure starts a daemon with an empty pack cache, scrapes its S3 counters, +// runs one git operation, and scrapes again. The daemon is stopped and its +// cache deleted before it returns, so nothing carries into the next +// measurement. +func measure(ctx context.Context, root, runDir string, side buildSpec, tag string, op func(context.Context) (time.Duration, int64, bool, error)) opResult { + var out opResult + + packCache, err := os.MkdirTemp(runDir, "packcache-") + if err != nil { + out.Err = fmt.Sprintf("can't create a pack cache directory: %v", err) + return out + } + defer os.RemoveAll(packCache) + + args := []string{ + "-bucket", bucketFromFlagOrEnv(), + "-http-bind", *httpBind, + "-metrics-bind", *metricsBind, + "-ssh-bind=", + "-allow-push", + "-pack-cache-dir", packCache, + } + args = append(args, side.Args...) + + if err := ensureNoDaemon(ctx, *metricsBind); err != nil { + out.Err = err.Error() + return out + } + + daemon, err := startDaemon(root, side.Bin, filepath.Join(runDir, "daemon-"+tag+".log"), args) + if err != nil { + out.Err = err.Error() + return out + } + defer stopDaemon(daemon) + + if err := waitReady(ctx, daemon, *metricsBind, *readyWait); err != nil { + out.Err = err.Error() + return out + } + + before, err := scrapeS3(ctx, *metricsBind) + if err != nil { + out.Err = err.Error() + return out + } + + wall, wire, dnf, opErr := op(ctx) + + out.Wall = wall + out.WallStr = wall.Round(time.Millisecond).String() + out.WireBytes = wire + out.DNF = dnf + if opErr != nil { + out.Err = opErr.Error() + } + + // The closing reading is what turns two counter values into a request + // count; without it there is no count. Leaving out.S3 at its zero value + // would file a measurement of no S3 traffic at all, and ok() and valid() + // would both stay true, so it would sit in the median with nothing in the + // report marking it. A scrape that did not answer fails the measurement. + after, err := scrapeS3(ctx, *metricsBind) + if err != nil { + slog.Error("dropping the measurement: can't scrape metrics after the operation", "tag", tag, "err", err) + if out.Err == "" { + out.Err = err.Error() + } + return out + } + out.S3 = after.sub(before) + + slog.Info("measured", "tag", tag, + "wall", out.WallStr, + "s3_requests", out.S3.Total(), + "s3_seconds", fmt.Sprintf("%.1f", out.S3.TotalSeconds()), + "dnf", dnf, + ) + + return out +} + +// buildSide compiles one daemon. The "before" side is built from a worktree +// checked out at its ref, because its go.mod and its whole internal tree differ +// from the current checkout. +func buildSide(ctx context.Context, root, runDir string, side *buildSpec) error { + src := root + if side.Ref != "" { + var err error + if src, err = ensureWorktree(ctx, root, side.Ref); err != nil { + return err + } + } + + side.Bin = filepath.Join(runDir, "objgitd-"+side.Name) + side.Commit = resolveCommit(ctx, src) + side.Args = quiesceArgs(src) + + cmd := exec.CommandContext(ctx, "go", "build", "-o", side.Bin, "./cmd/objgitd") + cmd.Dir = src + + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("formatbench: can't build the %q daemon from %s: %w: %s", + side.Name, src, err, strings.TrimSpace(string(out))) + } + + slog.Info("built daemon", "side", side.Name, "ref", side.Ref, "commit", side.Commit, + "src", src, "bin", side.Bin, "args", strings.Join(side.Args, " ")) + return nil +} + +// quiesceArgs returns the flags that stop the daemon built from src doing S3 +// work on a wall-clock timer. +// +// v1.0.2 runs a listing-cache warmer that re-lists every hot prefix each +// -s3-cache-refresh, 30 seconds by default, and those listings land in +// objgit_s3_requests_total, the vector measure brackets. A 45 minute push +// absorbs about ninety warmer cycles the current build never pays, and the +// inflation grows with wall time, which is the axis the report ranks builds on. +// +// Only the warmer is switched off. -s3-cache-ttl is left at its default, +// because the listing cache answered every request that release served: turning +// it off would measure a daemon nobody ran and would charge the old format for +// listings it did not make. +// +// The source is read rather than the ref name matched, because objgitd exits on +// an unknown flag and the current checkout has no -s3-cache-refresh. A build +// only gets a flag its own source defines. +func quiesceArgs(src string) []string { + if !definesFlag(src, "s3-cache-refresh") { + return nil + } + return []string{"-s3-cache-refresh=0"} +} + +// definesFlag reports whether the daemon's source under src declares the named +// flag. +func definesFlag(src, name string) bool { + dir := filepath.Join(src, "cmd", "objgitd") + + entries, err := os.ReadDir(dir) + if err != nil { + slog.Warn("can't read the daemon's source; assuming it defines no extra flags", "dir", dir, "err", err) + return false + } + + needle := []byte(strconv.Quote(name)) + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".go") { + continue + } + + raw, err := os.ReadFile(filepath.Join(dir, e.Name())) + if err != nil { + slog.Warn("can't read a daemon source file", "path", filepath.Join(dir, e.Name()), "err", err) + continue + } + if bytes.Contains(raw, needle) { + return true + } + } + + return false +} + +// resolveCommit records which commit a build was compiled from, so a number in +// the report can be traced back to a binary months later. It never fails the +// run: a missing SHA makes a result harder to check, but a wrong one would be +// worse and there is no way to produce one here. +func resolveCommit(ctx context.Context, src string) string { + sha, _, err := runGit(ctx, 30*time.Second, src, "rev-parse", "HEAD") + if err != nil { + slog.Warn("can't resolve the commit a build was made from", "src", src, "err", err) + return "unknown" + } + out := strings.TrimSpace(sha) + + // A dirty tree is the normal case for the current checkout, and it means + // the SHA alone does not describe what was built. + status, _, err := runGit(ctx, 30*time.Second, src, "status", "--porcelain") + if err != nil { + slog.Warn("can't tell whether a build's source was dirty", "src", src, "err", err) + return out + } + if strings.TrimSpace(status) != "" { + out += "-dirty" + } + + return out +} + +// ensureWorktree checks a ref out beside the working tree. A worktree is used +// rather than a detached checkout so the operator's own tree is never touched +// and the build can run in parallel with editing. +// +// An existing worktree is moved to ref rather than trusted to already be there. +// The = spelling exists to benchmark a candidate fix, and a fix +// lives on a branch: rerunning after pushing new commits would otherwise build +// the old checkout and print the branch name over it. +func ensureWorktree(ctx context.Context, root, ref string) (string, error) { + dir := filepath.Join(srcDir(root), worktreeName(ref)) + + if _, err := os.Stat(filepath.Join(dir, ".git")); err == nil { + if _, _, err := runGit(ctx, 10*time.Minute, dir, "checkout", "--detach", "--force", ref); err != nil { + return "", fmt.Errorf("formatbench: can't move the worktree at %s to %s: %w", dir, ref, err) + } + slog.Info("reusing worktree", "ref", ref, "path", dir) + return dir, nil + } + + if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil { + return "", fmt.Errorf("formatbench: can't create %s: %w", filepath.Dir(dir), err) + } + + cmd := exec.CommandContext(ctx, "git", "worktree", "add", "--detach", dir, ref) + cmd.Dir = root + + if out, err := cmd.CombinedOutput(); err != nil { + return "", fmt.Errorf("formatbench: can't add a worktree for %s: %w: %s", ref, err, strings.TrimSpace(string(out))) + } + + slog.Info("added worktree", "ref", ref, "path", dir) + return dir, nil +} + +// prepareMirror makes sure the repository is mirrored locally and records what +// went in: the reachable commit count and the size of the mirror's own pack. +// Those two numbers are what the report's ratios are read against. +func prepareMirror(ctx context.Context, root string, r *repoSpec) error { + r.Mirror = filepath.Join(mirrorsDir(root), r.Name+".git") + + if _, err := os.Stat(filepath.Join(r.Mirror, "HEAD")); err != nil { + if err := os.MkdirAll(filepath.Dir(r.Mirror), 0o755); err != nil { + return fmt.Errorf("formatbench: can't create %s: %w", filepath.Dir(r.Mirror), err) + } + slog.Info("mirroring", "repo", r.Name, "url", r.URL, "path", r.Mirror) + if err := mirrorClone(ctx, r.URL, r.Mirror); err != nil { + return err + } + } else if *refreshMirrors { + slog.Info("refreshing mirror", "repo", r.Name, "path", r.Mirror) + if _, _, err := runGit(ctx, 30*time.Minute, r.Mirror, "remote", "update", "--prune"); err != nil { + return fmt.Errorf("formatbench: can't refresh the %s mirror: %w", r.Name, err) + } + } + + count, _, err := runGit(ctx, verifyLimit, r.Mirror, "rev-list", "--all", "--count") + if err != nil { + return fmt.Errorf("formatbench: can't count commits in the %s mirror: %w", r.Name, err) + } + r.Commits = strings.TrimSpace(count) + r.Pack = packBytes(r.Mirror) + + slog.Info("mirror ready", "repo", r.Name, "commits", r.Commits, "pack_bytes", r.Pack) + return nil +} + +// packBytes sums the mirror's packfiles. It is the closest local stand-in for +// "how big is this repository" and it is what the bucket byte counts get +// compared against. +func packBytes(gitDir string) int64 { + entries, err := os.ReadDir(filepath.Join(gitDir, "objects", "pack")) + if err != nil { + return 0 + } + + var total int64 + for _, e := range entries { + if !strings.HasSuffix(e.Name(), ".pack") { + continue + } + info, err := e.Info() + if err != nil { + continue + } + total += info.Size() + } + + return total +} + +func runCleanup(ctx context.Context, listPath string) error { + raw, err := os.ReadFile(listPath) + if err != nil { + return fmt.Errorf("formatbench: can't read %s: %w", listPath, err) + } + + bc, err := newBucketClient(ctx, bucketFromFlagOrEnv()) + if err != nil { + return err + } + + var failed error + for line := range strings.SplitSeq(string(raw), "\n") { + prefix := strings.TrimSpace(line) + if prefix == "" || strings.HasPrefix(prefix, "#") { + continue + } + + n, err := bc.remove(ctx, prefix) + if err != nil { + slog.Error("can't delete prefix", "prefix", prefix, "err", err) + failed = errors.Join(failed, err) + continue + } + slog.Info("deleted prefix", "prefix", prefix, "keys", n) + } + + return failed +} + +// daemonProc is one objgitd under test: the process, the log file it writes to, +// and a channel carrying the single result of waiting on it. The channel is +// what lets waitReady notice a daemon that died instead of polling an address +// something else might answer. +type daemonProc struct { + cmd *exec.Cmd + log *os.File + logPath string + + // wait carries cmd.Wait's result once and is then closed, so every later + // receive returns immediately whether or not the value was taken. + wait chan error +} + +// ensureNoDaemon fails when something already answers on the metrics address. +// +// A daemon left behind by an interrupted run keeps the metrics and HTTP ports. +// The one started next dies on bind within milliseconds, and every request the +// harness makes after that goes to the old process: the wrong binary, with a +// warm pack cache, whose counters sub() quietly absorbs. Nothing downstream +// looks wrong, so it has to be caught before the measurement starts. +func ensureNoDaemon(ctx context.Context, metricsAddr string) error { + probe, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + + if _, err := scrapeS3(probe, metricsAddr); err == nil { + return fmt.Errorf("formatbench: something already answers http://%s/metrics; stop it before benchmarking", metricsAddr) + } + + return nil +} + +// startDaemon runs objgitd with its working directory set to the module root, +// because objgitd loads its credentials from the .env file there. The context +// is deliberately not attached: shutdown goes through stopDaemon so the daemon +// gets the same SIGINT it would in production. +func startDaemon(root, bin, logPath string, args []string) (*daemonProc, error) { + log, err := os.Create(logPath) + if err != nil { + return nil, fmt.Errorf("formatbench: can't create %s: %w", logPath, err) + } + + cmd := exec.Command(bin, args...) + cmd.Dir = root + cmd.Stdout = log + cmd.Stderr = log + + if err := cmd.Start(); err != nil { + log.Close() + return nil, fmt.Errorf("formatbench: can't start %s: %w", bin, err) + } + + p := &daemonProc{cmd: cmd, log: log, logPath: logPath, wait: make(chan error, 1)} + go func() { + p.wait <- cmd.Wait() + close(p.wait) + }() + + return p, nil +} + +// stopDaemon ends the daemon and closes its log. Closing here is what bounds +// the descriptors: measure runs twice per cell, and the file is only flushed +// when it is closed, so a log left open is both a leak and an unreadable log. +func stopDaemon(p *daemonProc) { + defer p.log.Close() + + if p.cmd.Process == nil { + return + } + + // A daemon that already died, or that waitReady found dead, has nothing + // left to signal. + select { + case <-p.wait: + return + default: + } + + if err := p.cmd.Process.Signal(os.Interrupt); err != nil && !errors.Is(err, os.ErrProcessDone) { + slog.Warn("can't interrupt daemon", "err", err) + } + + select { + case <-p.wait: + case <-time.After(30 * time.Second): + slog.Warn("daemon did not exit on SIGINT, killing it") + _ = p.cmd.Process.Kill() + <-p.wait + } +} + +// waitReady polls /metrics until the daemon answers, and gives up early if the +// daemon it was given is no longer running. +// +// Watching the child is the point. A 200 from the metrics address only says +// that some process holds that port, and objgitd exits on a bind failure, so +// without this a daemon that lost the port would be measured as if it had won +// it. +func waitReady(ctx context.Context, p *daemonProc, metricsAddr string, limit time.Duration) error { + url := "http://" + metricsAddr + "/metrics" + client := &http.Client{Timeout: 2 * time.Second} + deadline := time.Now().Add(limit) + + for time.Now().Before(deadline) { + if err := ctx.Err(); err != nil { + return err + } + + select { + case err := <-p.wait: + if err == nil { + err = errors.New("it exited without an error") + } + return fmt.Errorf("formatbench: the daemon stopped before it answered %s: %w; see %s", url, err, p.logPath) + default: + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return fmt.Errorf("formatbench: can't build readiness request: %w", err) + } + + resp, err := client.Do(req) + if err == nil { + resp.Body.Close() + if resp.StatusCode == http.StatusOK { + return nil + } + } + + if err := idle(ctx, 250*time.Millisecond); err != nil { + return err + } + } + + return fmt.Errorf("formatbench: daemon did not answer %s within %s", url, limit) +} + +// idle sleeps, but gives up as soon as the run is cancelled. +func idle(ctx context.Context, d time.Duration) error { + t := time.NewTimer(d) + defer t.Stop() + + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + return nil + } +} + +// pickRepos resolves -repos against the corpus table. +func pickRepos(raw string) ([]repoSpec, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return slices.Clone(corpus), nil + } + + var out []repoSpec + for want := range strings.SplitSeq(raw, ",") { + want = strings.TrimSpace(want) + if want == "" { + continue + } + + found := false + for _, r := range corpus { + if r.Name == want { + out = append(out, r) + found = true + break + } + } + if !found { + return nil, fmt.Errorf("formatbench: %q is not in the corpus; known entries are %s", want, corpusNames()) + } + } + + if len(out) == 0 { + return nil, fmt.Errorf("formatbench: -repos selected nothing") + } + return out, nil +} + +// pickBuilds resolves -builds into the sides of the comparison. +// +// Three spellings. "before" and "after" are the two sides of the format change +// itself. Anything else is "=", which builds that ref and labels its +// column ; an empty ref means the current checkout. The third form is +// what lets one interleaved session compare more than two points, which is the +// only honest way to read a candidate fix against both the old and the current +// build: comparing against a run from another day measures the network as much +// as the code. +func pickBuilds(raw string) ([]buildSpec, error) { + var out []buildSpec + + for want := range strings.SplitSeq(raw, ",") { + want = strings.TrimSpace(want) + + switch { + case want == "": + case want == "before": + out = append(out, buildSpec{Name: "before", Ref: *beforeRef}) + case want == "after": + out = append(out, buildSpec{Name: "after"}) + default: + name, ref, ok := strings.Cut(want, "=") + name, ref = strings.TrimSpace(name), strings.TrimSpace(ref) + if !ok || name == "" { + return nil, fmt.Errorf("formatbench: %q is not a build; use 'before', 'after', or '='", want) + } + out = append(out, buildSpec{Name: name, Ref: ref}) + } + } + + if len(out) == 0 { + return nil, fmt.Errorf("formatbench: -builds selected nothing") + } + + seen := map[string]bool{} + for _, s := range out { + if seen[s.Name] { + return nil, fmt.Errorf("formatbench: -builds names %q twice; every column needs its own name", s.Name) + } + seen[s.Name] = true + } + + return out, nil +} + +// worktreeName turns a ref into one directory name. A branch ref carries +// slashes, and joining those straight onto the worktree parent would bury the +// checkout several directories down and collide with any other ref sharing the +// prefix. +func worktreeName(ref string) string { + return strings.Map(func(r rune) rune { + if r == '/' || r == '\\' || r == ':' || r == ' ' { + return '-' + } + return r + }, ref) +} + +func corpusNames() string { + names := make([]string, 0, len(corpus)) + for _, r := range corpus { + names = append(names, r.Name) + } + return strings.Join(names, ", ") +} + +func findBuild(sides []buildSpec, name string) (buildSpec, bool) { + for _, s := range sides { + if s.Name == name { + return s, true + } + } + return buildSpec{}, false +} + +func findRepo(selected []repoSpec, name string) (repoSpec, bool) { + for _, r := range selected { + if r.Name == name { + return r, true + } + } + return repoSpec{}, false +} + +func bucketFromFlagOrEnv() string { + if *bucket != "" { + return *bucket + } + return os.Getenv("BUCKET") +} + +func outDir(root string) string { return underRoot(root, *outBase, "out") } +func mirrorsDir(root string) string { return underRoot(root, *mirrorDir, "mirrors") } +func srcDir(root string) string { return underRoot(root, *worktreeDir, "src") } + +// underRoot resolves one directory flag against the checkout, and always +// returns an absolute path. +// +// Absolute is load-bearing, not tidiness. startDaemon sets cmd.Dir to the +// checkout root so the daemon finds .env, and a relative binary path is then +// resolved against that directory rather than against the harness's own +// working directory. A relative -out therefore produces a daemon path that +// does not exist and every cell fails with "no such file or directory". +func underRoot(root, flagVal, name string) string { + dir := flagVal + if dir == "" { + dir = filepath.Join(root, "var", "bench", name) + } + + abs, err := filepath.Abs(dir) + if err != nil { + return dir + } + return abs +} + +// moduleRoot walks up from the working directory to the directory holding +// go.mod. That is where objgitd's .env lives, and where `go build` has to run. +func moduleRoot() (string, error) { + dir, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("formatbench: can't read working directory: %w", err) + } + + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir, nil + } + parent := filepath.Dir(dir) + if parent == dir { + return "", fmt.Errorf("formatbench: no go.mod above the working directory; run this from the objgit checkout") + } + dir = parent + } +} + +func hostname() string { + h, err := os.Hostname() + if err != nil { + return "unknown" + } + return h +} + +func gitVersion(ctx context.Context) string { + out, _, err := runGit(ctx, 10*time.Second, "", "version") + if err != nil { + return "unknown" + } + return strings.TrimSpace(out) +} diff --git a/cmd/formatbench/main_test.go b/cmd/formatbench/main_test.go new file mode 100644 index 0000000..f51affa --- /dev/null +++ b/cmd/formatbench/main_test.go @@ -0,0 +1,137 @@ +package main + +import "testing" + +func TestPickBuilds(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + in string + want []buildSpec + wantErr bool + }{ + { + name: "the two sides of the format change", + in: "before,after", + want: []buildSpec{{Name: "before", Ref: "v1.0.2"}, {Name: "after"}}, + }, + { + name: "a named ref builds its own column", + in: "before,fixed=1bb4b3e", + want: []buildSpec{{Name: "before", Ref: "v1.0.2"}, {Name: "fixed", Ref: "1bb4b3e"}}, + }, + { + name: "an empty ref means the current checkout", + in: "working=", + want: []buildSpec{{Name: "working"}}, + }, + { + name: "whitespace is trimmed everywhere", + in: " before , fixed = 1bb4b3e ", + want: []buildSpec{{Name: "before", Ref: "v1.0.2"}, {Name: "fixed", Ref: "1bb4b3e"}}, + }, + { + name: "a duplicate column name is refused", + in: "after,after", + wantErr: true, + }, + { + name: "two refs cannot share one column name", + in: "x=main,x=v1.0.2", + wantErr: true, + }, + { + name: "a bare word that is not a known side", + in: "sideways", + wantErr: true, + }, + { + name: "a ref with no name", + in: "=main", + wantErr: true, + }, + { + name: "nothing selected", + in: " , ", + wantErr: true, + }, + } { + t.Run(tt.name, func(t *testing.T) { + // Not parallel: pickBuilds reads the -before-ref flag. + *beforeRef = "v1.0.2" + + got, err := pickBuilds(tt.in) + if tt.wantErr { + if err == nil { + t.Fatalf("wanted an error, got %v", got) + } + return + } + if err != nil { + t.Fatalf("pickBuilds: %v", err) + } + + if len(got) != len(tt.want) { + t.Fatalf("want %v, got %v", tt.want, got) + } + for i := range tt.want { + if got[i].Name != tt.want[i].Name || got[i].Ref != tt.want[i].Ref { + t.Errorf("column %d: want %+v, got %+v", i, tt.want[i], got[i]) + } + } + }) + } +} + +func TestWorktreeName(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + in string + want string + }{ + {"v1.0.2", "v1.0.2"}, + {"1bb4b3e", "1bb4b3e"}, + {"perf/shallow-lookup-cache", "perf-shallow-lookup-cache"}, + {"origin/main", "origin-main"}, + } { + t.Run(tt.in, func(t *testing.T) { + t.Parallel() + + if got := worktreeName(tt.in); got != tt.want { + t.Logf("want: %s", tt.want) + t.Logf("got: %s", got) + t.Error("a ref turned into a bad directory name") + } + }) + } +} + +func TestScheduleInterleavesBuilds(t *testing.T) { + t.Parallel() + + sides := []buildSpec{{Name: "before"}, {Name: "after"}} + selected := []repoSpec{{Name: "small", Reps: 2}, {Name: "big", Reps: 1}} + + *reps = 0 + got := schedule(sides, selected) + + want := []cell{ + {Build: "before", Repo: "small", Rep: 1}, + {Build: "after", Repo: "small", Rep: 1}, + {Build: "before", Repo: "big", Rep: 1}, + {Build: "after", Repo: "big", Rep: 1}, + {Build: "before", Repo: "small", Rep: 2}, + {Build: "after", Repo: "small", Rep: 2}, + } + + if len(got) != len(want) { + t.Fatalf("want %d cells, got %d: %+v", len(want), len(got), got) + } + for i := range want { + if got[i].Build != want[i].Build || got[i].Repo != want[i].Repo || got[i].Rep != want[i].Rep { + t.Fatalf("cell %d: want %+v, got %+v", i, want[i], got[i]) + } + } +} diff --git a/cmd/formatbench/metrics.go b/cmd/formatbench/metrics.go new file mode 100644 index 0000000..ff5aa2f --- /dev/null +++ b/cmd/formatbench/metrics.go @@ -0,0 +1,284 @@ +package main + +import ( + "bufio" + "context" + "fmt" + "io" + "net/http" + "sort" + "strconv" + "strings" + "time" +) + +// The two series this harness reads. Both builds under test report them with +// the same label values: internal/s3fs feeds metrics.ObserveS3 on the "before" +// side, and (*tigris.Storer).observe feeds the same function on the "after" +// side, so the operation vocabulary (GetObject, PutObject, HeadObject, +// ListObjectsV2, DeleteObject) matches across the format change. +const ( + seriesRequests = "objgit_s3_requests_total" + seriesSeconds = "objgit_s3_request_duration_seconds_sum" +) + +// s3Counters is one reading of the daemon's S3 counters. Requests is keyed by +// the operation label and sums every status; Errors counts only the samples +// whose status label is not "ok". Seconds is the total time the daemon spent +// inside S3 calls, which separates network cost from local work. +type s3Counters struct { + Requests map[string]uint64 `json:"requests"` + Errors map[string]uint64 `json:"errors,omitempty"` + Seconds map[string]float64 `json:"seconds"` +} + +func newS3Counters() s3Counters { + return s3Counters{ + Requests: map[string]uint64{}, + Errors: map[string]uint64{}, + Seconds: map[string]float64{}, + } +} + +// Total is the request count across every operation. +func (c s3Counters) Total() uint64 { + var n uint64 + for _, v := range c.Requests { + n += v + } + return n +} + +// TotalSeconds is the time spent in S3 calls across every operation. +func (c s3Counters) TotalSeconds() float64 { + var f float64 + for _, v := range c.Seconds { + f += v + } + return f +} + +// Operations lists the operation labels present, sorted, so a report renders +// the same column order every run. +func (c s3Counters) Operations() []string { + seen := map[string]bool{} + for k := range c.Requests { + seen[k] = true + } + for k := range c.Seconds { + seen[k] = true + } + + out := make([]string, 0, len(seen)) + for k := range seen { + out = append(out, k) + } + sort.Strings(out) + + return out +} + +// sub returns the counters accumulated between two readings. A counter can only +// rise while a process lives, and the harness restarts the daemon between +// measurements, so a negative difference means the reading came from a new +// process and the earlier value is dropped rather than wrapped. +func (c s3Counters) sub(earlier s3Counters) s3Counters { + out := newS3Counters() + + for k, v := range c.Requests { + if prev := earlier.Requests[k]; v >= prev { + out.Requests[k] = v - prev + } else { + out.Requests[k] = v + } + } + for k, v := range c.Errors { + if prev := earlier.Errors[k]; v >= prev { + if d := v - prev; d > 0 { + out.Errors[k] = d + } + } else { + out.Errors[k] = v + } + } + for k, v := range c.Seconds { + if prev := earlier.Seconds[k]; v >= prev { + out.Seconds[k] = v - prev + } else { + out.Seconds[k] = v + } + } + + return out +} + +// scrapeS3 reads the daemon's /metrics endpoint once. +func scrapeS3(ctx context.Context, metricsAddr string) (s3Counters, error) { + url := "http://" + metricsAddr + "/metrics" + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return s3Counters{}, fmt.Errorf("formatbench: can't build scrape request: %w", err) + } + + client := &http.Client{Timeout: 10 * time.Second} + resp, err := client.Do(req) + if err != nil { + return s3Counters{}, fmt.Errorf("formatbench: can't scrape %s: %w", url, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return s3Counters{}, fmt.Errorf("formatbench: %s answered %s", url, resp.Status) + } + + return parseS3Metrics(resp.Body) +} + +// parseS3Metrics reads the Prometheus text exposition format and keeps only the +// two S3 series. Every other line is skipped, including the histogram buckets +// and counts that share a prefix with the duration sum. +func parseS3Metrics(r io.Reader) (s3Counters, error) { + out := newS3Counters() + + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 0, 64*1024), 4<<20) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" || line[0] == '#' { + continue + } + + name, labels, value, err := parseSample(line) + if err != nil { + return out, err + } + + switch name { + case seriesRequests: + op := labels["operation"] + if op == "" { + continue + } + n := uint64(max(value, 0)) + out.Requests[op] += n + if labels["status"] != "ok" { + out.Errors[op] += n + } + case seriesSeconds: + op := labels["operation"] + if op == "" { + continue + } + out.Seconds[op] += max(value, 0) + } + } + if err := sc.Err(); err != nil { + return out, fmt.Errorf("formatbench: reading metrics: %w", err) + } + + return out, nil +} + +// parseSample splits one exposition line into its name, its labels, and its +// value. +// +// The line is read left to right, because a sample can carry a trailing +// timestamp: "name{labels} value [timestamp]". Reading from the right would +// take the timestamp as the value, and a timestamp parses as a number, so the +// mistake would be silent. +func parseSample(line string) (string, map[string]string, float64, error) { + name, labels, rest, err := splitHead(line) + if err != nil { + return "", nil, 0, err + } + + fields := strings.Fields(rest) + if len(fields) == 0 { + return "", nil, 0, fmt.Errorf("formatbench: metric line has no value: %q", line) + } + + value, err := strconv.ParseFloat(fields[0], 64) + if err != nil { + return "", nil, 0, fmt.Errorf("formatbench: metric %q has non-numeric value %q", name, fields[0]) + } + + return name, labels, value, nil +} + +// splitHead peels the metric name and its label set off the front of a line and +// returns whatever follows. The scan for the closing brace tracks quoting, so a +// label value holding a brace does not end the label set early. +func splitHead(line string) (string, map[string]string, string, error) { + open := strings.IndexByte(line, '{') + space := strings.IndexByte(line, ' ') + + if open < 0 || (space >= 0 && space < open) { + if space < 0 { + return "", nil, "", fmt.Errorf("formatbench: metric line has no value: %q", line) + } + return line[:space], nil, line[space+1:], nil + } + + inQuote := false + for i := open + 1; i < len(line); i++ { + switch { + case line[i] == '\\' && inQuote: + i++ + case line[i] == '"': + inQuote = !inQuote + case line[i] == '}' && !inQuote: + return line[:open], parseLabels(line[open+1 : i]), line[i+1:], nil + } + } + + return "", nil, "", fmt.Errorf("formatbench: metric line has unbalanced braces: %q", line) +} + +// parseLabels reads a comma-separated name="value" list. Backslash escapes +// inside a value are honored, so a label holding a quote or a comma parses. +func parseLabels(raw string) map[string]string { + out := map[string]string{} + + for i := 0; i < len(raw); { + eq := strings.IndexByte(raw[i:], '=') + if eq < 0 { + break + } + + key := strings.TrimSpace(raw[i : i+eq]) + i += eq + 1 + if i >= len(raw) || raw[i] != '"' { + break + } + i++ + + var val strings.Builder + for i < len(raw) && raw[i] != '"' { + if raw[i] == '\\' && i+1 < len(raw) { + i++ + switch raw[i] { + case 'n': + val.WriteByte('\n') + default: + val.WriteByte(raw[i]) + } + i++ + continue + } + val.WriteByte(raw[i]) + i++ + } + i++ // closing quote + + if key != "" { + out[key] = val.String() + } + + for i < len(raw) && (raw[i] == ',' || raw[i] == ' ') { + i++ + } + } + + return out +} diff --git a/cmd/formatbench/metrics_test.go b/cmd/formatbench/metrics_test.go new file mode 100644 index 0000000..4c37c9a --- /dev/null +++ b/cmd/formatbench/metrics_test.go @@ -0,0 +1,226 @@ +package main + +import ( + "strings" + "testing" +) + +// A trimmed /metrics body. The histogram bucket and count lines are here on +// purpose: they share a prefix with the duration sum, and a parser that keys on +// the prefix instead of the whole name silently triples the recorded time. +const sampleExposition = `# HELP objgit_s3_requests_total Tigris API calls. +# TYPE objgit_s3_requests_total counter +objgit_s3_requests_total{operation="GetObject",status="ok"} 8500 +objgit_s3_requests_total{operation="GetObject",status="error"} 3 +objgit_s3_requests_total{operation="PutObject",status="ok"} 20000 +objgit_s3_requests_total{operation="ListObjectsV2",status="ok"} 12 +# TYPE objgit_s3_request_duration_seconds histogram +objgit_s3_request_duration_seconds_bucket{operation="GetObject",le="0.1"} 4000 +objgit_s3_request_duration_seconds_sum{operation="GetObject"} 412.5 +objgit_s3_request_duration_seconds_count{operation="GetObject"} 8503 +objgit_s3_request_duration_seconds_sum{operation="PutObject"} 900.25 +go_goroutines 42 +` + +func TestParseS3Metrics(t *testing.T) { + t.Parallel() + + got, err := parseS3Metrics(strings.NewReader(sampleExposition)) + if err != nil { + t.Fatalf("parseS3Metrics: %v", err) + } + + for _, tt := range []struct { + name string + op string + want uint64 + }{ + {"both statuses sum into one operation", "GetObject", 8503}, + {"single status", "PutObject", 20000}, + {"listing", "ListObjectsV2", 12}, + {"an operation the daemon never made", "HeadObject", 0}, + } { + t.Run(tt.name, func(t *testing.T) { + if got.Requests[tt.op] != tt.want { + t.Logf("want: %d", tt.want) + t.Logf("got: %d", got.Requests[tt.op]) + t.Errorf("%s request count is wrong", tt.op) + } + }) + } + + if got.Errors["GetObject"] != 3 { + t.Errorf("want 3 GetObject errors, got %d", got.Errors["GetObject"]) + } + if got.Errors["PutObject"] != 0 { + t.Errorf("a status=ok sample was counted as an error: %d", got.Errors["PutObject"]) + } + + if got.Total() != 28515 { + t.Errorf("want 28515 total requests, got %d", got.Total()) + } + + // 412.5 + 900.25. If the bucket or count lines leaked in, this is larger. + if want := 1312.75; got.TotalSeconds() != want { + t.Logf("want: %v", want) + t.Logf("got: %v", got.TotalSeconds()) + t.Error("histogram bucket or count lines leaked into the duration sum") + } +} + +func TestParseSample(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + line string + wantName string + wantValue float64 + wantOp string + wantErr bool + }{ + { + name: "labelled counter", + line: `objgit_s3_requests_total{operation="GetObject",status="ok"} 8500`, + wantName: "objgit_s3_requests_total", + wantValue: 8500, + wantOp: "GetObject", + }, + { + name: "no labels", + line: `go_goroutines 42`, + wantName: "go_goroutines", + wantValue: 42, + }, + { + name: "value carries a timestamp", + line: `objgit_s3_requests_total{operation="PutObject",status="ok"} 7 1756651200000`, + wantName: "objgit_s3_requests_total", + wantValue: 7, + wantOp: "PutObject", + }, + { + name: "scientific notation", + line: `objgit_s3_request_duration_seconds_sum{operation="GetObject"} 1.5e+02`, + wantName: "objgit_s3_request_duration_seconds_sum", + wantValue: 150, + wantOp: "GetObject", + }, + { + name: "escaped quote in a label value", + line: `objgit_s3_requests_total{operation="Get\"Object",status="ok"} 1`, + wantName: "objgit_s3_requests_total", + wantValue: 1, + wantOp: `Get"Object`, + }, + { + name: "no value at all", + line: `objgit_s3_requests_total`, + wantErr: true, + }, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + name, labels, value, err := parseSample(tt.line) + if tt.wantErr { + if err == nil { + t.Fatal("wanted an error, got none") + } + return + } + if err != nil { + t.Fatalf("parseSample: %v", err) + } + + if name != tt.wantName { + t.Errorf("want name %q, got %q", tt.wantName, name) + } + if value != tt.wantValue { + t.Errorf("want value %v, got %v", tt.wantValue, value) + } + if labels["operation"] != tt.wantOp { + t.Errorf("want operation %q, got %q", tt.wantOp, labels["operation"]) + } + }) + } +} + +func TestS3CountersSub(t *testing.T) { + t.Parallel() + + before := newS3Counters() + before.Requests["GetObject"] = 100 + before.Requests["PutObject"] = 5 + before.Seconds["GetObject"] = 10 + + after := newS3Counters() + after.Requests["GetObject"] = 8600 + after.Requests["PutObject"] = 5 + after.Requests["HeadObject"] = 7 + after.Seconds["GetObject"] = 412.5 + + got := after.sub(before) + + for _, tt := range []struct { + name string + op string + want uint64 + }{ + {"counter that moved", "GetObject", 8500}, + {"counter that did not move", "PutObject", 0}, + {"operation absent from the earlier reading", "HeadObject", 7}, + } { + t.Run(tt.name, func(t *testing.T) { + if got.Requests[tt.op] != tt.want { + t.Logf("want: %d", tt.want) + t.Logf("got: %d", got.Requests[tt.op]) + t.Errorf("%s delta is wrong", tt.op) + } + }) + } + + if got.Seconds["GetObject"] != 402.5 { + t.Errorf("want 402.5 seconds, got %v", got.Seconds["GetObject"]) + } +} + +// A daemon restart resets every counter to zero. The later reading is then +// smaller than the earlier one, and the whole later value is the delta: it all +// belongs to the operation being measured. +func TestS3CountersSubAfterRestart(t *testing.T) { + t.Parallel() + + before := newS3Counters() + before.Requests["GetObject"] = 9000 + + after := newS3Counters() + after.Requests["GetObject"] = 12 + + if got := after.sub(before).Requests["GetObject"]; got != 12 { + t.Logf("want: 12") + t.Logf("got: %d", got) + t.Error("a counter reset was wrapped instead of taken at face value") + } +} + +func TestOperationsAreSorted(t *testing.T) { + t.Parallel() + + c := newS3Counters() + c.Requests["PutObject"] = 1 + c.Requests["GetObject"] = 1 + c.Seconds["ListObjectsV2"] = 1 + + got := c.Operations() + want := []string{"GetObject", "ListObjectsV2", "PutObject"} + + if len(got) != len(want) { + t.Fatalf("want %v, got %v", want, got) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("want %v, got %v", want, got) + } + } +} diff --git a/cmd/formatbench/report.go b/cmd/formatbench/report.go new file mode 100644 index 0000000..8eb70fc --- /dev/null +++ b/cmd/formatbench/report.go @@ -0,0 +1,386 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +// cell is one (build, repository, repetition) measurement: a push, a clone, the +// bucket it left behind, and whether that bucket gave the repository back +// unchanged. +type cell struct { + Build string `json:"build"` + Repo string `json:"repo"` + Rep int `json:"rep"` + Prefix string `json:"prefix"` + + Push opResult `json:"push"` + Clone opResult `json:"clone"` + + Usage bucketUsage `json:"usage"` + Verify verifyResult `json:"verify"` +} + +// valid reports whether this cell's timings can be believed. A push that did +// not finish, or a clone that came back different from what went in, is not a +// measurement of anything. +func (c cell) valid() bool { + return c.Push.ok() && (!c.Verify.Ran || c.Verify.OK) +} + +// runMeta records the conditions the numbers were taken under. Wall-clock +// figures from a laptop on wifi mean nothing without them. +type runMeta struct { + StartedAt time.Time `json:"started_at"` + FinishedAt time.Time `json:"finished_at"` + + Host string `json:"host"` + GoVersion string `json:"go_version"` + GitVersion string `json:"git_version"` + NumCPU int `json:"num_cpu"` + + Bucket string `json:"bucket"` + Org string `json:"org"` + RunDir string `json:"run_dir"` + + Builds []buildSpec `json:"builds"` + Repos []repoSpec `json:"repos"` +} + +// writeResults saves the raw records and the rendered tables side by side. The +// JSON is what you re-analyze; the Markdown is what goes in the post. +func writeResults(runDir string, meta runMeta, cells []cell) error { + payload := struct { + Meta runMeta `json:"meta"` + Cells []cell `json:"cells"` + }{Meta: meta, Cells: cells} + + raw, err := json.MarshalIndent(payload, "", " ") + if err != nil { + return fmt.Errorf("formatbench: can't encode results: %w", err) + } + + jsonPath := filepath.Join(runDir, "results.json") + if err := os.WriteFile(jsonPath, append(raw, '\n'), 0o644); err != nil { + return fmt.Errorf("formatbench: can't write %s: %w", jsonPath, err) + } + + mdPath := filepath.Join(runDir, "results.md") + if err := os.WriteFile(mdPath, []byte(renderReport(meta, cells)), 0o644); err != nil { + return fmt.Errorf("formatbench: can't write %s: %w", mdPath, err) + } + + return nil +} + +// appendRepoList records one prefix for later cleanup. It appends rather than +// rewrites, so an interrupted run still leaves a complete list. +func appendRepoList(path, prefix string) error { + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644) + if err != nil { + return fmt.Errorf("formatbench: can't open %s: %w", path, err) + } + defer f.Close() + + if _, err := fmt.Fprintln(f, prefix); err != nil { + return fmt.Errorf("formatbench: can't write %s: %w", path, err) + } + + return nil +} + +// renderReport writes the two tables the blog post needs, plus the conditions +// they were measured under. +func renderReport(meta runMeta, cells []cell) string { + var b strings.Builder + + fmt.Fprintf(&b, "# objgit storage format: before and after\n\n") + fmt.Fprintf(&b, "Every row is the median across repetitions. Ranges are min-max where\n") + fmt.Fprintf(&b, "more than one repetition finished.\n\n") + + writeConditions(&b, meta) + writePushTable(&b, meta, cells) + writeCloneTable(&b, meta, cells) + writeProblems(&b, cells) + + fmt.Fprintf(&b, "\nOh yeah, this was with my corporate laptop on wifi.\n") + + return b.String() +} + +func writeConditions(b *strings.Builder, meta runMeta) { + fmt.Fprintf(b, "## Run conditions\n\n") + fmt.Fprintf(b, "| Item | Value |\n| --- | --- |\n") + fmt.Fprintf(b, "| Started | %s |\n", meta.StartedAt.Format(time.RFC3339)) + fmt.Fprintf(b, "| Finished | %s |\n", meta.FinishedAt.Format(time.RFC3339)) + fmt.Fprintf(b, "| Host | %s, %d CPUs |\n", meta.Host, meta.NumCPU) + fmt.Fprintf(b, "| Go | %s |\n", meta.GoVersion) + fmt.Fprintf(b, "| Git | %s |\n", meta.GitVersion) + fmt.Fprintf(b, "| Bucket | %s |\n", meta.Bucket) + + for _, s := range meta.Builds { + ref := s.Ref + if ref == "" { + ref = "current checkout" + } + fmt.Fprintf(b, "| Build %q | %s (%s) |\n", s.Name, ref, orDash(s.Commit)) + } + + fmt.Fprintf(b, "\n### Corpus\n\n") + fmt.Fprintf(b, "| Repo | URL | Commits | Mirror pack |\n| --- | --- | --: | --: |\n") + for _, r := range meta.Repos { + fmt.Fprintf(b, "| %s | %s | %s | %s |\n", r.Name, r.URL, orDash(r.Commits), humanBytes(r.Pack)) + } + fmt.Fprintln(b) +} + +func writePushTable(b *strings.Builder, meta runMeta, cells []cell) { + fmt.Fprintf(b, "## Push\n\n") + fmt.Fprintf(b, "| Repo | Build | Wall | S3 requests | PUT | GET | HEAD | LIST | Keys | Bucket bytes |\n") + fmt.Fprintf(b, "| --- | --- | --: | --: | --: | --: | --: | --: | --: | --: |\n") + + for _, r := range meta.Repos { + for _, s := range meta.Builds { + got := pick(cells, s.Name, r.Name) + if len(got) == 0 { + continue + } + + fmt.Fprintf(b, "| %s | %s | %s | %s | %s | %s | %s | %s | %s | %s |\n", + r.Name, s.Name, + wallCell(got, func(c cell) opResult { return c.Push }), + countCell(got, func(c cell) opResult { return c.Push }, ""), + countCell(got, func(c cell) opResult { return c.Push }, "PutObject"), + countCell(got, func(c cell) opResult { return c.Push }, "GetObject"), + countCell(got, func(c cell) opResult { return c.Push }, "HeadObject"), + countCell(got, func(c cell) opResult { return c.Push }, "ListObjectsV2"), + usageCell(got, func(u bucketUsage) float64 { return float64(u.Keys) }, plainNumber), + usageCell(got, func(u bucketUsage) float64 { return float64(u.Bytes) }, humanFloatBytes), + ) + } + } + fmt.Fprintln(b) +} + +func writeCloneTable(b *strings.Builder, meta runMeta, cells []cell) { + fmt.Fprintf(b, "## Clone\n\n") + fmt.Fprintf(b, "| Repo | Build | Wall | S3 requests | GET | HEAD | LIST | Wire bytes |\n") + fmt.Fprintf(b, "| --- | --- | --: | --: | --: | --: | --: | --: |\n") + + for _, r := range meta.Repos { + for _, s := range meta.Builds { + got := pick(cells, s.Name, r.Name) + if len(got) == 0 { + continue + } + + clone := func(c cell) opResult { return c.Clone } + fmt.Fprintf(b, "| %s | %s | %s | %s | %s | %s | %s | %s |\n", + r.Name, s.Name, + wallCell(got, clone), + countCell(got, clone, ""), + countCell(got, clone, "GetObject"), + countCell(got, clone, "HeadObject"), + countCell(got, clone, "ListObjectsV2"), + wireCell(got), + ) + } + } + fmt.Fprintln(b) +} + +// writeProblems lists every cell that did not produce a usable number, with the +// reason. Nothing is silently dropped from the tables above. +func writeProblems(b *strings.Builder, cells []cell) { + var lines []string + + for _, c := range cells { + switch { + case c.Push.DNF: + lines = append(lines, fmt.Sprintf("- %s/%s rep %d: push DNF at %s, %d S3 requests and %s in the bucket by then", + c.Build, c.Repo, c.Rep, c.Push.WallStr, c.Push.S3.Total(), humanBytes(c.Usage.Bytes))) + case c.Push.Err != "": + lines = append(lines, fmt.Sprintf("- %s/%s rep %d: push failed: %s", c.Build, c.Repo, c.Rep, c.Push.Err)) + case c.Clone.DNF: + lines = append(lines, fmt.Sprintf("- %s/%s rep %d: clone DNF at %s, %d S3 requests by then", + c.Build, c.Repo, c.Rep, c.Clone.WallStr, c.Clone.S3.Total())) + case c.Clone.Err != "": + lines = append(lines, fmt.Sprintf("- %s/%s rep %d: clone failed: %s", c.Build, c.Repo, c.Rep, c.Clone.Err)) + case c.Verify.Ran && !c.Verify.OK: + lines = append(lines, fmt.Sprintf("- %s/%s rep %d: INVALID, the clone does not match the mirror: %s", + c.Build, c.Repo, c.Rep, strings.Join(c.Verify.Problems, "; "))) + } + } + + if len(lines) == 0 { + return + } + + fmt.Fprintf(b, "## Cells that produced no number\n\n") + fmt.Fprintf(b, "%s\n", strings.Join(lines, "\n")) +} + +// pick returns every cell for one build and repository, in repetition order. +func pick(cells []cell, build, repo string) []cell { + var out []cell + for _, c := range cells { + if c.Build == build && c.Repo == repo { + out = append(out, c) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Rep < out[j].Rep }) + return out +} + +// wallCell renders the median wall time with its range. A cell where nothing +// finished renders as DNF, never as an extrapolation. +func wallCell(cells []cell, get func(cell) opResult) string { + var secs []float64 + dnf := false + + for _, c := range cells { + r := get(c) + if r.DNF { + dnf = true + } + if !c.valid() || !r.ok() { + continue + } + secs = append(secs, r.Wall.Seconds()) + } + + if len(secs) == 0 { + if dnf { + return "DNF" + } + return "-" + } + + return withRange(secs, func(f float64) string { + return time.Duration(f * float64(time.Second)).Round(100 * time.Millisecond).String() + }) +} + +// countCell renders the median request count for one operation. An empty op +// means the total across every operation. +func countCell(cells []cell, get func(cell) opResult, op string) string { + var counts []float64 + + for _, c := range cells { + r := get(c) + if !c.valid() || !r.ok() { + continue + } + if op == "" { + counts = append(counts, float64(r.S3.Total())) + } else { + counts = append(counts, float64(r.S3.Requests[op])) + } + } + + if len(counts) == 0 { + return "-" + } + return withRange(counts, plainNumber) +} + +func usageCell(cells []cell, get func(bucketUsage) float64, render func(float64) string) string { + var vals []float64 + for _, c := range cells { + if !c.valid() { + continue + } + vals = append(vals, get(c.Usage)) + } + + if len(vals) == 0 { + return "-" + } + return withRange(vals, render) +} + +func wireCell(cells []cell) string { + var vals []float64 + for _, c := range cells { + if !c.valid() || !c.Clone.ok() { + continue + } + vals = append(vals, float64(c.Clone.WireBytes)) + } + + if len(vals) == 0 { + return "-" + } + return withRange(vals, humanFloatBytes) +} + +// withRange renders the median, and appends the span when the repetitions +// disagree. Wifi is noisy; hiding the spread would misrepresent the number. +func withRange(vals []float64, render func(float64) string) string { + sorted := append([]float64(nil), vals...) + sort.Float64s(sorted) + + mid := render(median(sorted)) + lo, hi := render(sorted[0]), render(sorted[len(sorted)-1]) + if len(sorted) < 2 || lo == hi { + return mid + } + + return fmt.Sprintf("%s (%s-%s)", mid, lo, hi) +} + +// median takes the lower of the two middle values on an even count, so the +// figure printed is always one that was actually measured. +func median(sorted []float64) float64 { + if len(sorted) == 0 { + return 0 + } + return sorted[(len(sorted)-1)/2] +} + +func plainNumber(f float64) string { + n := int64(f) + s := fmt.Sprintf("%d", n) + if n < 0 { + return s + } + + var out []byte + for i, c := range []byte(s) { + if i > 0 && (len(s)-i)%3 == 0 { + out = append(out, ',') + } + out = append(out, c) + } + return string(out) +} + +func humanBytes(n int64) string { return humanFloatBytes(float64(n)) } + +func humanFloatBytes(f float64) string { + const unit = 1024.0 + + if f < unit { + return fmt.Sprintf("%d B", int64(f)) + } + + div, exp := unit, 0 + for n := f / unit; n >= unit && exp < 3; n /= unit { + div *= unit + exp++ + } + + return fmt.Sprintf("%.2f %ciB", f/div, "KMGT"[exp]) +} + +func orDash(s string) string { + if s == "" { + return "-" + } + return s +} diff --git a/cmd/formatbench/report_test.go b/cmd/formatbench/report_test.go new file mode 100644 index 0000000..0ac8006 --- /dev/null +++ b/cmd/formatbench/report_test.go @@ -0,0 +1,234 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// mkCell builds a finished, verified cell so a test only has to state the parts +// it cares about. +func mkCell(build, repo string, rep int, pushSecs float64, gets, puts uint64) cell { + push := opResult{Wall: time.Duration(pushSecs * float64(time.Second)), S3: newS3Counters()} + push.S3.Requests["GetObject"] = gets + push.S3.Requests["PutObject"] = puts + + clone := opResult{Wall: time.Second, S3: newS3Counters(), WireBytes: 1 << 20} + clone.S3.Requests["GetObject"] = gets + + return cell{ + Build: build, + Repo: repo, + Rep: rep, + Push: push, + Clone: clone, + Usage: bucketUsage{Keys: 12, Bytes: 4096}, + Verify: verifyResult{Ran: true, OK: true}, + } +} + +func TestWallCellReportsMedianAndRange(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + cells []cell + want string + }{ + { + name: "one repetition has no range", + cells: []cell{mkCell("after", "objgit", 1, 4, 10, 2)}, + want: "4s", + }, + { + name: "three repetitions report the middle one", + cells: []cell{ + mkCell("after", "objgit", 1, 4, 10, 2), + mkCell("after", "objgit", 2, 9, 10, 2), + mkCell("after", "objgit", 3, 6, 10, 2), + }, + want: "6s (4s-9s)", + }, + { + name: "an even count takes the lower middle value", + cells: []cell{ + mkCell("after", "objgit", 1, 4, 10, 2), + mkCell("after", "objgit", 2, 8, 10, 2), + }, + want: "4s (4s-8s)", + }, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := wallCell(tt.cells, func(c cell) opResult { return c.Push }) + if got != tt.want { + t.Logf("want: %s", tt.want) + t.Logf("got: %s", got) + t.Error("wall time cell is wrong") + } + }) + } +} + +// A push that ran out of time is reported as DNF. It is never turned into an +// estimated finish, and it never averages with the repetitions that finished. +func TestWallCellReportsDNF(t *testing.T) { + t.Parallel() + + dnf := mkCell("before", "x", 1, 2700, 500, 40000) + dnf.Push.DNF = true + dnf.Push.Err = "git push stopped at the 45m0s limit" + + if got := wallCell([]cell{dnf}, func(c cell) opResult { return c.Push }); got != "DNF" { + t.Logf("want: DNF") + t.Logf("got: %s", got) + t.Error("a push that hit its time limit was reported as a measurement") + } + + ok := mkCell("before", "x", 2, 100, 500, 40000) + got := wallCell([]cell{dnf, ok}, func(c cell) opResult { return c.Push }) + if got != "1m40s" { + t.Logf("want: 1m40s") + t.Logf("got: %s", got) + t.Error("a DNF repetition polluted the median of the ones that finished") + } +} + +// A clone that does not match its mirror means the push was not a valid +// measurement of storing that repository, so none of the cell's numbers count. +func TestInvalidCellIsExcluded(t *testing.T) { + t.Parallel() + + bad := mkCell("after", "objgit", 1, 4, 10, 2) + bad.Verify = verifyResult{Ran: true, OK: false, Problems: []string{"clone is missing ref refs/heads/main"}} + + if bad.valid() { + t.Fatal("a cell whose clone does not match the mirror reported itself as valid") + } + + if got := wallCell([]cell{bad}, func(c cell) opResult { return c.Push }); got != "-" { + t.Logf("want: -") + t.Logf("got: %s", got) + t.Error("an INVALID cell contributed a timing to the table") + } +} + +func TestCountCell(t *testing.T) { + t.Parallel() + + cells := []cell{ + mkCell("before", "objgit", 1, 4, 8500, 20000), + mkCell("before", "objgit", 2, 4, 8500, 20000), + } + + for _, tt := range []struct { + name string + op string + want string + }{ + {"one operation", "PutObject", "20,000"}, + {"total across operations", "", "28,500"}, + {"an operation that never ran", "HeadObject", "0"}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := countCell(cells, func(c cell) opResult { return c.Push }, tt.op) + if got != tt.want { + t.Logf("want: %s", tt.want) + t.Logf("got: %s", got) + t.Error("request count cell is wrong") + } + }) + } +} + +func TestRenderReport(t *testing.T) { + t.Parallel() + + meta := runMeta{ + StartedAt: time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC), + FinishedAt: time.Date(2026, 9, 10, 14, 0, 0, 0, time.UTC), + Host: "laptop", + Builds: []buildSpec{ + {Name: "before", Ref: "v1.0.2", Commit: "a1b2c3d"}, + {Name: "after", Commit: "e4f5a6b-dirty"}, + }, + Repos: []repoSpec{{Name: "objgit", URL: "https://example.invalid/objgit", Commits: "1200", Pack: 15 << 20}}, + } + + dnf := mkCell("before", "objgit", 1, 2700, 400, 9000) + dnf.Push.DNF = true + dnf.Push.WallStr = "45m0s" + + got := renderReport(meta, []cell{dnf, mkCell("after", "objgit", 1, 12, 40, 3)}) + + for _, want := range []string{ + "## Push", + "## Clone", + // The commit is what makes a number checkable after the fact, and + // "-dirty" is the normal case for the side built from the live + // checkout, so both belong in the conditions table. + "| Build \"before\" | v1.0.2 (a1b2c3d) |", + "| Build \"after\" | current checkout (e4f5a6b-dirty) |", + "## Cells that produced no number", + "push DNF at 45m0s", + "Oh yeah, this was with my corporate laptop on wifi.", + } { + if !strings.Contains(got, want) { + t.Errorf("report is missing %q", want) + } + } +} + +func TestHumanFloatBytes(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + in float64 + want string + }{ + {0, "0 B"}, + {512, "512 B"}, + {1 << 10, "1.00 KiB"}, + {15 << 20, "15.00 MiB"}, + {1536 << 20, "1.50 GiB"}, + } { + t.Run(tt.want, func(t *testing.T) { + t.Parallel() + + if got := humanFloatBytes(tt.in); got != tt.want { + t.Logf("want: %s", tt.want) + t.Logf("got: %s", got) + t.Error("byte formatting is wrong") + } + }) + } +} + +func TestPlainNumber(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + in float64 + want string + }{ + {0, "0"}, + {12, "12"}, + {123, "123"}, + {1234, "1,234"}, + {20000, "20,000"}, + {2900000, "2,900,000"}, + } { + t.Run(tt.want, func(t *testing.T) { + t.Parallel() + + if got := plainNumber(tt.in); got != tt.want { + t.Logf("want: %s", tt.want) + t.Logf("got: %s", got) + t.Error("number formatting is wrong") + } + }) + } +} diff --git a/cmd/formatbench/uuid.go b/cmd/formatbench/uuid.go new file mode 100644 index 0000000..a6f7b5d --- /dev/null +++ b/cmd/formatbench/uuid.go @@ -0,0 +1,21 @@ +package main + +import ( + "crypto/rand" + "fmt" +) + +// newUUID returns a random version 4 UUID. The harness only needs repository +// names that never collide with an earlier run, so crypto/rand covers it and no +// dependency is needed. +func newUUID() (string, error) { + var b [16]byte + if _, err := rand.Read(b[:]); err != nil { + return "", fmt.Errorf("formatbench: can't read random bytes: %w", err) + } + + b[6] = (b[6] & 0x0f) | 0x40 // version 4 + b[8] = (b[8] & 0x3f) | 0x80 // variant 10 + + return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:16]), nil +} diff --git a/cmd/formatbench/verify.go b/cmd/formatbench/verify.go new file mode 100644 index 0000000..9bd0874 --- /dev/null +++ b/cmd/formatbench/verify.go @@ -0,0 +1,125 @@ +package main + +import ( + "context" + "fmt" + "sort" + "strings" + "time" +) + +// verifyLimit caps each of the three checks. They are local git operations +// against an on-disk clone, so they finish in seconds even for a large +// repository; the limit only stops a wedged subprocess. +const verifyLimit = 10 * time.Minute + +// verifyResult says whether the repository that came back out of the bucket is +// the repository that went in. A benchmark that measures a corrupt push is +// worthless, so a cell that fails here is reported as INVALID and its timings +// are withheld rather than printed with a footnote. +type verifyResult struct { + Ran bool `json:"ran"` + OK bool `json:"ok"` + + SourceCommits string `json:"source_commits,omitempty"` + CloneCommits string `json:"clone_commits,omitempty"` + + Problems []string `json:"problems,omitempty"` +} + +// verifyClone compares a clone against the mirror it came from. It checks three +// things: the same number of reachable commits, the same set of refs pointing +// at the same hashes, and an intact object graph. +func verifyClone(ctx context.Context, mirror, clone string) verifyResult { + out := verifyResult{Ran: true, OK: true} + + srcCount, _, err := runGit(ctx, verifyLimit, mirror, "rev-list", "--all", "--count") + if err != nil { + return out.fail("counting commits in the mirror: %v", err) + } + out.SourceCommits = strings.TrimSpace(srcCount) + + dstCount, _, err := runGit(ctx, verifyLimit, clone, "rev-list", "--all", "--count") + if err != nil { + return out.fail("counting commits in the clone: %v", err) + } + out.CloneCommits = strings.TrimSpace(dstCount) + + if out.SourceCommits != out.CloneCommits { + out = out.fail("mirror has %s reachable commits, clone has %s", out.SourceCommits, out.CloneCommits) + } + + srcRefs, err := benchRefs(ctx, mirror) + if err != nil { + return out.fail("reading refs from the mirror: %v", err) + } + + dstRefs, err := benchRefs(ctx, clone) + if err != nil { + return out.fail("reading refs from the clone: %v", err) + } + + for _, diff := range diffRefs(srcRefs, dstRefs) { + out = out.fail("%s", diff) + } + + if _, _, err := runGit(ctx, verifyLimit, clone, "fsck", "--full", "--no-progress"); err != nil { + out = out.fail("fsck on the clone: %v", err) + } + + return out +} + +func (r verifyResult) fail(format string, args ...any) verifyResult { + r.OK = false + r.Problems = append(r.Problems, fmt.Sprintf(format, args...)) + return r +} + +// benchRefs reads a repository's refs as name to hash. HEAD is left out: the +// mirror carries the upstream default branch and the daemon repoints a dangling +// HEAD on load, so the two disagree for reasons that have nothing to do with +// object storage. +func benchRefs(ctx context.Context, dir string) (map[string]string, error) { + out, _, err := runGit(ctx, verifyLimit, dir, "show-ref") + if err != nil { + return nil, err + } + + refs := map[string]string{} + for line := range strings.SplitSeq(strings.TrimSpace(out), "\n") { + hash, name, ok := strings.Cut(strings.TrimSpace(line), " ") + if !ok { + continue + } + refs[name] = hash + } + + return refs, nil +} + +// diffRefs reports every ref that is missing, extra, or pointing somewhere +// else. It returns the problems sorted, so two runs of the same fault read the +// same way. +func diffRefs(src, dst map[string]string) []string { + var problems []string + + for name, want := range src { + got, ok := dst[name] + switch { + case !ok: + problems = append(problems, fmt.Sprintf("clone is missing ref %s", name)) + case got != want: + problems = append(problems, fmt.Sprintf("ref %s is %s in the mirror and %s in the clone", name, want, got)) + } + } + + for name := range dst { + if _, ok := src[name]; !ok { + problems = append(problems, fmt.Sprintf("clone has an extra ref %s", name)) + } + } + + sort.Strings(problems) + return problems +} diff --git a/cmd/formatbench/verify_test.go b/cmd/formatbench/verify_test.go new file mode 100644 index 0000000..2ed465c --- /dev/null +++ b/cmd/formatbench/verify_test.go @@ -0,0 +1,73 @@ +package main + +import ( + "strings" + "testing" +) + +func TestDiffRefs(t *testing.T) { + t.Parallel() + + for _, tt := range []struct { + name string + src map[string]string + dst map[string]string + want []string + }{ + { + name: "identical", + src: map[string]string{"refs/heads/main": "aaa"}, + dst: map[string]string{"refs/heads/main": "aaa"}, + }, + { + name: "the clone lost a ref", + src: map[string]string{"refs/heads/main": "aaa", "refs/tags/v1": "bbb"}, + dst: map[string]string{"refs/heads/main": "aaa"}, + want: []string{"clone is missing ref refs/tags/v1"}, + }, + { + name: "the clone gained a ref", + src: map[string]string{"refs/heads/main": "aaa"}, + dst: map[string]string{"refs/heads/main": "aaa", "refs/heads/stray": "ccc"}, + want: []string{"clone has an extra ref refs/heads/stray"}, + }, + { + name: "a ref points somewhere else", + src: map[string]string{"refs/heads/main": "aaa"}, + dst: map[string]string{"refs/heads/main": "zzz"}, + want: []string{"ref refs/heads/main is aaa in the mirror and zzz in the clone"}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := diffRefs(tt.src, tt.dst) + if len(got) != len(tt.want) { + t.Fatalf("want %v, got %v", tt.want, got) + } + for i := range tt.want { + if got[i] != tt.want[i] { + t.Errorf("want %q, got %q", tt.want[i], got[i]) + } + } + }) + } +} + +func TestVerifyResultFailAccumulates(t *testing.T) { + t.Parallel() + + got := verifyResult{Ran: true, OK: true}. + fail("first problem"). + fail("second problem with %d", 2) + + if got.OK { + t.Error("a failed check left OK set") + } + if len(got.Problems) != 2 { + t.Fatalf("want 2 problems, got %v", got.Problems) + } + if !strings.Contains(got.Problems[1], "second problem with 2") { + t.Errorf("format arguments were dropped: %q", got.Problems[1]) + } +} diff --git a/cmd/objgitd/git_protocol.go b/cmd/objgitd/git_protocol.go index e9cdfed..1756bf7 100644 --- a/cmd/objgitd/git_protocol.go +++ b/cmd/objgitd/git_protocol.go @@ -67,6 +67,11 @@ type daemon struct { // very differently, and one semaphore over both would let a burst of clones // block pushes for reasons that have nothing to do with memory. pushes *pushLimiter + + // lfs is the Git LFS subsystem, nil when -allow-lfs is unset. A nil lfs + // registers no LFS route, so the feature reads as a 404 from outside + // instead of as a broken endpoint. + lfs *lfsService } // storerFor reports whether a repository already exists at st, returning st diff --git a/cmd/objgitd/http.go b/cmd/objgitd/http.go index ef496b2..a655480 100644 --- a/cmd/objgitd/http.go +++ b/cmd/objgitd/http.go @@ -33,6 +33,7 @@ func (d *daemon) httpHandler() http.Handler { mux.HandleFunc("POST /{orgID}/{repoName}/git-receive-pack", func(w http.ResponseWriter, r *http.Request) { d.handleRPC(w, r, transport.ReceivePackService) }) + d.registerLFS(mux) return mux } diff --git a/cmd/objgitd/lfs.go b/cmd/objgitd/lfs.go new file mode 100644 index 0000000..d71687b --- /dev/null +++ b/cmd/objgitd/lfs.go @@ -0,0 +1,691 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "slices" + "strconv" + "strings" + "time" + + ssh "github.com/gliderlabs/ssh" + "github.com/tigrisdata/objgit/internal/auth" + "github.com/tigrisdata/objgit/internal/lfs" + "github.com/tigrisdata/objgit/internal/metrics" + "github.com/tigrisdata/objgit/internal/repofs" +) + +// lfsService is the Git LFS subsystem. The daemon's field is nil when +// -allow-lfs is unset, and httpHandler then registers no LFS route at all, so +// the whole feature reads as a 404 from outside. That is the answer both +// git-lfs and the locking specification expect from a server without LFS. +type lfsService struct { + store *lfs.Store + // ttl answers how long a presigned URL minted right now may live. It is a + // function and not a value because the answer depends on the credential the + // daemon holds at that moment, which the SDK refreshes while the process + // runs. See lfsURLTTLFunc in main.go. + ttl func(context.Context) time.Duration + maxSize int64 + maxBatch int + // externalURL is the public base URL, used to build the verify action and + // the git-lfs-authenticate response. + externalURL string + allowLocks bool +} + +// urlTTL is the presigned-URL lifetime for one request. +func (s *lfsService) urlTTL(ctx context.Context) time.Duration { + return s.ttl(ctx) +} + +// maxBodyBytes is the largest LFS request body the daemon reads. +// +// It is derived from the batch cap rather than configured on its own: one +// object entry is an oid, a size, and punctuation, so 256 bytes each is +// generous. The floor keeps the small bodies (verify, locks) workable when +// -lfs-max-batch is tiny. +func (s *lfsService) maxBodyBytes() int64 { + const ( + perObject = 256 + floor = 64 << 10 + ) + if n := int64(s.maxBatch) * perObject; n > floor { + return n + } + return floor +} + +// Limits on what one repository's lock document may grow to hold. +// +// The document is read whole, decoded, and re-encoded on every lock call, and +// git-lfs calls locks/verify on every push. Nothing else in the LFS surface +// stores caller-supplied text, so these two bounds are what keep one anonymous +// request from permanently slowing every later push to that repository. +const ( + // maxLockPathBytes caps the path a lock may name. 4096 is PATH_MAX on + // Linux, so it is above anything git can put in a working tree, and no + // real lock is refused by it. + maxLockPathBytes = 4096 + // maxLocksPerRepo caps how many locks one repository holds at once. File + // locking coordinates a team on a handful of binary assets, so a thousand + // is far above real use while holding the document to a few megabytes even + // when every path is at the cap above. + maxLocksPerRepo = 1000 +) + +// registerLFS adds the LFS routes. It is a no-op when LFS is off. +// +// Every pattern has a fixed segment count and no trailing wildcard, so none can +// match another's requests and none shadows the git routes. +func (d *daemon) registerLFS(mux *http.ServeMux) { + if d.lfs == nil { + return + } + mux.HandleFunc("POST /{orgID}/{repoName}/info/lfs/objects/batch", d.handleLFSBatch) + mux.HandleFunc("POST /{orgID}/{repoName}/info/lfs/objects/verify", d.handleLFSVerify) + + if !d.lfs.allowLocks { + return + } + mux.HandleFunc("POST /{orgID}/{repoName}/info/lfs/locks", d.handleLFSLockCreate) + mux.HandleFunc("GET /{orgID}/{repoName}/info/lfs/locks", d.handleLFSLockList) + mux.HandleFunc("POST /{orgID}/{repoName}/info/lfs/locks/verify", d.handleLFSLockVerify) + mux.HandleFunc("POST /{orgID}/{repoName}/info/lfs/locks/{id}/unlock", d.handleLFSUnlock) +} + +// lfsBegin runs the checks every LFS route shares: content negotiation, path +// parsing, and authorization. It writes the error itself and returns ok=false +// when the request cannot proceed. +func (d *daemon) lfsBegin(w http.ResponseWriter, r *http.Request, op auth.Operation) (repofs.RepoRef, bool) { + // Answer as LFS from here on, including for failures. + w.Header().Set("Content-Type", lfs.MediaType) + + if !acceptsLFS(r) { + writeLFSError(w, http.StatusNotAcceptable, "this endpoint speaks "+lfs.MediaType) + return repofs.RepoRef{}, false + } + + ref, err := repofs.Parse(r.PathValue("orgID") + "/" + r.PathValue("repoName")) + if err != nil { + writeLFSError(w, http.StatusNotFound, err.Error()) + return repofs.RepoRef{}, false + } + + cred, _ := credFromRequest(r) + decision := d.authorize(r.Context(), auth.Request{ + Repo: ref.Path(), + Operation: op, + Cred: cred, + Transport: "http", + }) + switch decision { + case auth.Allow: + return ref, true + case auth.Unauthenticated: + // LFS names its own challenge header. git-lfs reads this one, not + // WWW-Authenticate, when it decides to ask for credentials. + w.Header().Set("LFS-Authenticate", `Basic realm="objgit"`) + w.Header().Set("WWW-Authenticate", `Basic realm="objgit"`) + writeLFSError(w, http.StatusUnauthorized, "authentication required") + return repofs.RepoRef{}, false + default: + // A denied write is 403 and never 401. A 401 tells git-lfs the + // credentials were wrong, so it asks the user and retries forever. + slog.Warn("lfs request denied by authorizer", + "repo", ref.Path(), "operation", op, "remote", r.RemoteAddr) + writeLFSError(w, http.StatusForbidden, "access denied") + return repofs.RepoRef{}, false + } +} + +// acceptsLFS reports whether the client will take an LFS answer. An absent or +// wildcard Accept is taken as yes, because curl and older clients send one. +func acceptsLFS(r *http.Request) bool { + accept := r.Header.Get("Accept") + if accept == "" { + return true + } + for _, part := range strings.Split(accept, ",") { + media, _, _ := strings.Cut(strings.TrimSpace(part), ";") + switch strings.TrimSpace(media) { + case lfs.MediaType, "*/*", "application/*": + return true + } + } + return false +} + +// writeLFSError renders a top-level failure in the shape git-lfs prints. +func writeLFSError(w http.ResponseWriter, status int, message string) { + w.Header().Set("Content-Type", lfs.MediaType) + w.WriteHeader(status) + if err := json.NewEncoder(w).Encode(lfs.ErrorBody{Message: message}); err != nil { + slog.Error("writing lfs error body", "err", err) + } +} + +// writeLFSJSON renders a successful body. +func writeLFSJSON(w http.ResponseWriter, status int, body any) { + w.Header().Set("Content-Type", lfs.MediaType) + w.WriteHeader(status) + if err := json.NewEncoder(w).Encode(body); err != nil { + slog.Error("writing lfs response body", "err", err) + } +} + +// capLFSBody bounds a request body before anything reads it. +// +// Every LFS route takes its body from an unauthenticated connection, so none of +// them may buffer an arbitrary amount of JSON. This runs on all of them, and +// not only on the routes that decode before authorizing. +func (d *daemon) capLFSBody(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, d.lfs.maxBodyBytes()) +} + +// decodeLFSBody reads a JSON request body, answering 422 on anything malformed +// and 413 on a body past the cap. +func decodeLFSBody(w http.ResponseWriter, r *http.Request, into any) bool { + err := json.NewDecoder(r.Body).Decode(into) + if err == nil { + return true + } + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + writeLFSError(w, http.StatusRequestEntityTooLarge, "request body is too large") + return false + } + writeLFSError(w, http.StatusUnprocessableEntity, "malformed request body") + return false +} + +// decodeOptionalLFSBody decodes a body the protocol lets a client leave empty. +// +// Only an oversized body is fatal here. Anything else decodes into the zero +// value, which is what an absent body means on these routes. +func decodeOptionalLFSBody(w http.ResponseWriter, r *http.Request, into any) bool { + err := json.NewDecoder(r.Body).Decode(into) + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + writeLFSError(w, http.StatusRequestEntityTooLarge, "request body is too large") + return false + } + return true +} + +// lfsOwner is the lock owner for a request: the Basic username, or anonymous. +func lfsOwner(r *http.Request) string { + if user, _, ok := r.BasicAuth(); ok && user != "" { + return user + } + return lfs.AnonymousOwner +} + +// verifyURL builds the absolute URL of this repository's verify endpoint, which +// the batch response hands to the client. +func (d *daemon) verifyURL(r *http.Request, ref repofs.RepoRef) string { + base := strings.TrimSuffix(d.lfs.externalURL, "/") + if base == "" { + // Fall back to the host the client reached, which is right for a + // direct connection and for most reverse proxies. + scheme := "http" + if r.TLS != nil { + scheme = "https" + } + if fwd := r.Header.Get("X-Forwarded-Proto"); fwd != "" { + scheme = fwd + } + base = scheme + "://" + r.Host + } + return fmt.Sprintf("%s/%s/%s.git/info/lfs/objects/verify", base, ref.OrgID, ref.Name) +} + +// handleLFSBatch serves POST .../info/lfs/objects/batch, the whole transfer +// protocol. It answers with presigned URLs; no object byte passes through here. +func (d *daemon) handleLFSBatch(w http.ResponseWriter, r *http.Request) { + start := time.Now() + + var req lfs.BatchRequest + // The operation decides the access needed, so the body is read before + // authorization rather than after. That puts an allocation ahead of the + // authorizer, so the cap matters most here: an unauthenticated caller must + // not be able to make the daemon hold an arbitrary amount of JSON before it + // is turned away. + d.capLFSBody(w, r) + if !decodeLFSBody(w, r, &req) { + metrics.ObserveLFSBatch("unknown", "invalid", start) + return + } + + op := auth.Read + if req.Operation == lfs.OperationUpload { + op = auth.Write + } + ref, ok := d.lfsBegin(w, r, op) + if !ok { + metrics.ObserveLFSBatch(req.Operation, "denied", start) + return + } + + resp, err := d.lfs.store.Batch(r.Context(), &req, lfs.BatchOptions{ + Repo: ref.Path(), + VerifyURL: d.verifyURL(r, ref), + TTL: d.lfs.urlTTL(r.Context()), + MaxSize: d.lfs.maxSize, + MaxBatch: d.lfs.maxBatch, + }) + if err != nil { + status, label := lfsBatchStatus(err) + if status == http.StatusInternalServerError { + slog.Error("lfs batch failed", "repo", ref.Path(), "operation", req.Operation, "err", err) + writeLFSError(w, status, "cannot serve batch request") + } else { + writeLFSError(w, status, err.Error()) + } + metrics.ObserveLFSBatch(req.Operation, label, start) + return + } + + for _, obj := range resp.Objects { + metrics.ObserveLFSObject(req.Operation, lfsObjectResult(obj)) + } + + slog.Info("serving lfs batch", + "repo", ref.Path(), + "operation", req.Operation, + "objects", len(resp.Objects), + "remote", r.RemoteAddr, + ) + writeLFSJSON(w, http.StatusOK, resp) + metrics.ObserveLFSBatch(req.Operation, "ok", start) +} + +// lfsBatchStatus maps a whole-request failure to its HTTP status. +func lfsBatchStatus(err error) (int, string) { + switch { + case errors.Is(err, lfs.ErrTooManyObjects): + return http.StatusRequestEntityTooLarge, "too_many_objects" + case errors.Is(err, lfs.ErrInvalidOID), + errors.Is(err, lfs.ErrInvalidRequest), + errors.Is(err, lfs.ErrObjectTooLarge), + errors.Is(err, lfs.ErrNoSupportedTransfer): + return http.StatusUnprocessableEntity, "invalid" + default: + return http.StatusInternalServerError, "error" + } +} + +// lfsObjectResult labels one object's outcome for metrics. +func lfsObjectResult(obj lfs.ObjectResult) string { + switch { + case obj.Error != nil && obj.Error.Code == http.StatusNotFound: + return "missing" + case obj.Error != nil: + return "error" + case obj.Actions == nil: + // No actions on an upload means the repository already holds it. + return "present" + default: + return "new" + } +} + +// handleLFSVerify serves POST .../info/lfs/objects/verify. +// +// This is the only integrity check that runs on the daemon's own credentials, +// and the only step that records that a repository holds an object. A +// presigned upload the client never verifies leaves no membership behind. +func (d *daemon) handleLFSVerify(w http.ResponseWriter, r *http.Request) { + ref, ok := d.lfsBegin(w, r, auth.Write) + if !ok { + return + } + + var p lfs.Pointer + d.capLFSBody(w, r) + if !decodeLFSBody(w, r, &p) { + return + } + + status, err := d.lfs.store.Verify(r.Context(), ref.Path(), p.OID, p.Size) + if err != nil { + if errors.Is(err, lfs.ErrInvalidOID) { + writeLFSError(w, http.StatusUnprocessableEntity, err.Error()) + metrics.ObserveLFSVerify("invalid") + return + } + slog.Error("lfs verify failed", "repo", ref.Path(), "oid", p.OID, "err", err) + writeLFSError(w, http.StatusInternalServerError, "cannot verify object") + metrics.ObserveLFSVerify("error") + return + } + + metrics.ObserveLFSVerify(status.String()) + switch status { + case lfs.VerifyOK: + writeLFSJSON(w, http.StatusOK, lfs.Pointer{OID: p.OID, Size: p.Size}) + case lfs.VerifyMissing: + writeLFSError(w, http.StatusNotFound, "object was not uploaded") + default: + writeLFSError(w, http.StatusUnprocessableEntity, "uploaded bytes do not match the object id") + } +} + +// lockRequest is the body of a lock create. +type lockRequest struct { + Path string `json:"path"` + Ref *lfs.Ref `json:"ref,omitempty"` +} + +// lockResponse wraps a single lock, which is how the protocol returns both a +// creation and a conflict. +type lockResponse struct { + Lock lfs.Lock `json:"lock"` + Message string `json:"message,omitempty"` +} + +// handleLFSLockCreate serves POST .../info/lfs/locks. +func (d *daemon) handleLFSLockCreate(w http.ResponseWriter, r *http.Request) { + ref, ok := d.lfsBegin(w, r, auth.Write) + if !ok { + return + } + + var req lockRequest + d.capLFSBody(w, r) + if !decodeLFSBody(w, r, &req) { + return + } + if len(req.Path) > maxLockPathBytes { + metrics.ObserveLFSLock("create", "invalid") + writeLFSError(w, http.StatusUnprocessableEntity, "lock path is too long") + return + } + if !d.lockRoomLeft(w, r, ref, req.Path) { + return + } + + lock, created, err := d.lfs.store.CreateLock(r.Context(), ref.Path(), req.Path, lfsOwner(r)) + if err != nil { + d.writeLockError(w, ref, "create", err) + return + } + if !created { + metrics.ObserveLFSLock("create", "conflict") + writeLFSJSON(w, http.StatusConflict, lockResponse{ + Lock: lock, + Message: "already created lock", + }) + return + } + metrics.ObserveLFSLock("create", "ok") + writeLFSJSON(w, http.StatusCreated, lockResponse{Lock: lock}) +} + +// lockRoomLeft reports whether repository ref may take another lock on path, +// writing the refusal itself when it may not. A lock that already holds path is +// always allowed through, so a repeated `git lfs lock` at the cap still gets the +// 409 that names the holder rather than a confusing refusal. +// +// This costs one extra read of the lock document per create, which is cheap +// next to how rare a lock is. It is a soft cap: two creates racing can both +// pass it. That is enough, because the point is to bound growth, not to make +// the thousandth lock exact. +func (d *daemon) lockRoomLeft(w http.ResponseWriter, r *http.Request, ref repofs.RepoRef, path string) bool { + locks, _, err := d.lfs.store.ListLocks(r.Context(), ref.Path(), lfs.LockFilter{}) + if err != nil { + d.writeLockError(w, ref, "create", err) + return false + } + if len(locks) < maxLocksPerRepo { + return true + } + if slices.ContainsFunc(locks, func(l lfs.Lock) bool { return l.Path == path }) { + return true + } + + slog.Warn("lfs lock refused: repository is at the lock cap", + "repo", ref.Path(), "locks", len(locks), "cap", maxLocksPerRepo) + metrics.ObserveLFSLock("create", "too_many") + writeLFSError(w, http.StatusRequestEntityTooLarge, + fmt.Sprintf("this repository already holds the most locks allowed (%d)", maxLocksPerRepo)) + return false +} + +// lockListResponse is the body of a lock listing. +type lockListResponse struct { + Locks []lfs.Lock `json:"locks"` + NextCursor string `json:"next_cursor,omitempty"` +} + +// handleLFSLockList serves GET .../info/lfs/locks. +func (d *daemon) handleLFSLockList(w http.ResponseWriter, r *http.Request) { + ref, ok := d.lfsBegin(w, r, auth.Read) + if !ok { + return + } + + q := r.URL.Query() + limit, _ := strconv.Atoi(q.Get("limit")) + locks, next, err := d.lfs.store.ListLocks(r.Context(), ref.Path(), lfs.LockFilter{ + Path: q.Get("path"), + ID: q.Get("id"), + Cursor: q.Get("cursor"), + Limit: limit, + }) + if err != nil { + d.writeLockError(w, ref, "list", err) + return + } + + metrics.ObserveLFSLock("list", "ok") + writeLFSJSON(w, http.StatusOK, lockListResponse{Locks: orEmpty(locks), NextCursor: next}) +} + +// lockVerifyRequest is the body git-lfs sends before a push. +type lockVerifyRequest struct { + Ref *lfs.Ref `json:"ref,omitempty"` + Cursor string `json:"cursor,omitempty"` + Limit int `json:"limit,omitempty"` +} + +// lockVerifyResponse splits the locks by holder. git-lfs halts a push when a +// file it is pushing appears in "theirs". +type lockVerifyResponse struct { + Ours []lfs.Lock `json:"ours"` + Theirs []lfs.Lock `json:"theirs"` + NextCursor string `json:"next_cursor,omitempty"` +} + +// handleLFSLockVerify serves POST .../info/lfs/locks/verify, which git-lfs +// calls on every push once this endpoint answers. +func (d *daemon) handleLFSLockVerify(w http.ResponseWriter, r *http.Request) { + ref, ok := d.lfsBegin(w, r, auth.Read) + if !ok { + return + } + + var req lockVerifyRequest + // An empty body is legal here, so a decode failure is not fatal. + d.capLFSBody(w, r) + if !decodeOptionalLFSBody(w, r, &req) { + return + } + + locks, next, err := d.lfs.store.ListLocks(r.Context(), ref.Path(), lfs.LockFilter{ + Cursor: req.Cursor, + Limit: req.Limit, + }) + if err != nil { + d.writeLockError(w, ref, "verify", err) + return + } + + ours, theirs := lfs.SplitLocks(locks, lfsOwner(r)) + metrics.ObserveLFSLock("verify", "ok") + writeLFSJSON(w, http.StatusOK, lockVerifyResponse{ + Ours: orEmpty(ours), + Theirs: orEmpty(theirs), + NextCursor: next, + }) +} + +// unlockRequest is the body of an unlock. +type unlockRequest struct { + Force bool `json:"force,omitempty"` +} + +// handleLFSUnlock serves POST .../info/lfs/locks/{id}/unlock. +func (d *daemon) handleLFSUnlock(w http.ResponseWriter, r *http.Request) { + ref, ok := d.lfsBegin(w, r, auth.Write) + if !ok { + return + } + + var req unlockRequest + // An empty body is legal here too: `force` is the only field. + d.capLFSBody(w, r) + if !decodeOptionalLFSBody(w, r, &req) { + return + } + + lock, err := d.lfs.store.DeleteLock(r.Context(), ref.Path(), r.PathValue("id"), lfsOwner(r), req.Force) + if err != nil { + d.writeLockError(w, ref, "unlock", err) + return + } + + metrics.ObserveLFSLock("unlock", "ok") + writeLFSJSON(w, http.StatusOK, lockResponse{Lock: lock}) +} + +// writeLockError renders a lock failure in HTTP terms. action is the short +// metric verb ("create", "list", "verify", "unlock"), so a failure lands under +// the same label as the success it failed to be. +func (d *daemon) writeLockError(w http.ResponseWriter, ref repofs.RepoRef, action string, err error) { + switch { + case errors.Is(err, lfs.ErrLockNotFound): + metrics.ObserveLFSLock(action, "not_found") + writeLFSError(w, http.StatusNotFound, "no such lock") + case errors.Is(err, lfs.ErrLockHeldByAnother): + metrics.ObserveLFSLock(action, "forbidden") + writeLFSError(w, http.StatusForbidden, err.Error()) + case errors.Is(err, lfs.ErrInvalidRequest): + metrics.ObserveLFSLock(action, "invalid") + writeLFSError(w, http.StatusUnprocessableEntity, err.Error()) + case errors.Is(err, lfs.ErrLockContention): + metrics.ObserveLFSLock(action, "contended") + writeLFSError(w, http.StatusConflict, "the lock list is busy, retry") + default: + slog.Error("lfs lock request failed", "action", action, "repo", ref.Path(), "err", err) + metrics.ObserveLFSLock(action, "error") + writeLFSError(w, http.StatusInternalServerError, "cannot serve lock request") + } +} + +// orEmpty renders a nil slice as [] rather than null, which some LFS clients +// refuse to decode. +func orEmpty(locks []lfs.Lock) []lfs.Lock { + if locks == nil { + return []lfs.Lock{} + } + return locks +} + +// lfsAuthenticateCommand is the SSH command git-lfs runs to discover where a +// repository's LFS HTTP API lives. An ssh:// remote cannot use LFS without it. +const lfsAuthenticateCommand = "git-lfs-authenticate" + +// commandNotFoundExit is the exit status git-lfs reads as "this server has no +// LFS", falling back to guessing the HTTP endpoint. +// +// Any other non-zero exit is reported to the user as a hard failure, so a +// server without LFS must answer 127 here. Exiting 1 would break every `git +// lfs` operation over ssh:// instead of leaving it alone. +const commandNotFoundExit = 127 + +// authenticateResponse tells git-lfs where to send its batch requests. +type authenticateResponse struct { + Href string `json:"href"` + Header map[string]string `json:"header"` + // ExpiresIn is how long the client may reuse this answer, in seconds. + ExpiresIn int `json:"expires_in"` +} + +// handleLFSAuthenticate serves `git-lfs-authenticate `. +// +// It writes the JSON answer to the session, which is the command's stdout. +// Diagnostics go to stderr, where they do not corrupt the answer. +func (d *daemon) handleLFSAuthenticate(s ssh.Session, cmd []string) { + // Without LFS, or without a public URL to point at, the honest answer is + // that the command does not exist. + if d.lfs == nil || d.lfs.externalURL == "" { + fmt.Fprintf(s.Stderr(), "%s: command not found\n", lfsAuthenticateCommand) + _ = s.Exit(commandNotFoundExit) + return + } + + if len(cmd) < 3 { + fmt.Fprintf(s.Stderr(), "objgitd: usage: %s \n", lfsAuthenticateCommand) + _ = s.Exit(1) + return + } + + ref, err := repofs.Parse(cmd[1]) + if err != nil { + fmt.Fprintf(s.Stderr(), "objgitd: %v\n", err) + _ = s.Exit(1) + return + } + + op := auth.Read + switch cmd[2] { + case lfs.OperationDownload: + case lfs.OperationUpload: + op = auth.Write + default: + fmt.Fprintf(s.Stderr(), "objgitd: unknown operation %q\n", cmd[2]) + _ = s.Exit(1) + return + } + + var cred auth.Credential = auth.Anonymous{} + if key := s.PublicKey(); key != nil { + cred = auth.PublicKey{Key: key} + } + if d.authorize(s.Context(), auth.Request{ + Repo: ref.Path(), + Operation: op, + Cred: cred, + Transport: "ssh", + }) != auth.Allow { + fmt.Fprintln(s.Stderr(), "objgitd: access denied") + _ = s.Exit(1) + return + } + + slog.Info("serving lfs authenticate", + "repo", ref.Path(), + "operation", cmd[2], + "remote", s.RemoteAddr().String(), + ) + + // The header map is empty because the HTTP side accepts anonymous + // requests. A deployment with real authentication mints a token here. + resp := authenticateResponse{ + Href: fmt.Sprintf("%s/%s/%s.git/info/lfs", + strings.TrimSuffix(d.lfs.externalURL, "/"), ref.OrgID, ref.Name), + Header: map[string]string{}, + ExpiresIn: int(d.lfs.urlTTL(s.Context()).Seconds()), + } + if err := json.NewEncoder(s).Encode(resp); err != nil { + slog.Error("writing lfs authenticate response", "repo", ref.Path(), "err", err) + _ = s.Exit(1) + return + } + _ = s.Exit(0) +} diff --git a/cmd/objgitd/lfs_ssh_test.go b/cmd/objgitd/lfs_ssh_test.go new file mode 100644 index 0000000..cab600e --- /dev/null +++ b/cmd/objgitd/lfs_ssh_test.go @@ -0,0 +1,175 @@ +package main + +import ( + "encoding/json" + "errors" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +// runSSHCommand runs one command over ssh against addr, returning stdout, +// stderr, and the exit status. It does not fail the test on a non-zero exit, +// because the exit code is what several of these cases assert on. +func runSSHCommand(t *testing.T, addr string, args ...string) (stdout, stderr string, code int) { + t.Helper() + + keyDir := t.TempDir() + key := filepath.Join(keyDir, "id_ed25519") + if out, err := exec.Command("ssh-keygen", "-t", "ed25519", "-N", "", "-f", key).CombinedOutput(); err != nil { + t.Fatalf("ssh-keygen: %v\n%s", err, out) + } + + host, port, ok := strings.Cut(addr, ":") + if !ok { + t.Fatalf("bad address %q", addr) + } + + sshArgs := []string{ + "-i", key, + "-o", "IdentitiesOnly=yes", + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-p", port, + "git@" + host, + } + cmd := exec.Command("ssh", append(sshArgs, args...)...) + + var outBuf, errBuf strings.Builder + cmd.Stdout = &outBuf + cmd.Stderr = &errBuf + err := cmd.Run() + + code = 0 + if err != nil { + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) { + t.Fatalf("running ssh: %v", err) + } + code = exitErr.ExitCode() + } + return outBuf.String(), errBuf.String(), code +} + +func withLFS(external string, allowLocks bool) func(*daemon) { + return func(d *daemon) { + d.lfs = &lfsService{ + ttl: staticLFSTTL(15 * time.Minute), + maxSize: 1 << 30, + maxBatch: 100, + externalURL: external, + allowLocks: allowLocks, + } + } +} + +// TestSSHLFSAuthenticate covers the handshake an ssh:// remote makes before it +// can use LFS at all: git-lfs asks the SSH server where the HTTP API lives. +func TestSSHLFSAuthenticate(t *testing.T) { + checkSSHBinaries(t) + + addr, _ := startSSHServer(t, true, false, withLFS("https://git.example.com", true)) + + stdout, stderr, code := runSSHCommand(t, addr, + "git-lfs-authenticate", "acme/test.git", "download") + if code != 0 { + t.Fatalf("exit %d, want 0\nstderr: %s", code, stderr) + } + + var got struct { + Href string `json:"href"` + Header map[string]string `json:"header"` + ExpiresIn int `json:"expires_in"` + } + if err := json.Unmarshal([]byte(stdout), &got); err != nil { + t.Fatalf("decoding %q: %v", stdout, err) + } + + const want = "https://git.example.com/acme/test.git/info/lfs" + if got.Href != want { + t.Errorf("href = %q, want %q", got.Href, want) + } + if got.ExpiresIn <= 0 { + t.Errorf("expires_in = %d, want a positive value", got.ExpiresIn) + } +} + +// TestSSHLFSAuthenticateExits127WhenUnavailable is the fallback contract. +// git-lfs only falls back to guessing the HTTP endpoint when the command exits +// 127 or says "command not found"; any other non-zero exit is reported to the +// user as a hard failure. Without this, turning LFS off would break every +// `git lfs` operation over ssh:// rather than leaving it alone. +func TestSSHLFSAuthenticateExits127WhenUnavailable(t *testing.T) { + checkSSHBinaries(t) + + for _, tt := range []struct { + name string + opts []func(*daemon) + }{ + { + name: "lfs is disabled", + opts: nil, + }, + { + name: "lfs is on but no external url is configured", + opts: []func(*daemon){withLFS("", true)}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + addr, _ := startSSHServer(t, true, false, tt.opts...) + + _, stderr, code := runSSHCommand(t, addr, + "git-lfs-authenticate", "acme/test.git", "download") + + if code != 127 { + t.Fatalf("exit %d, want 127\nstderr: %s", code, stderr) + } + if !strings.Contains(stderr, "command not found") { + t.Errorf("stderr = %q, want it to mention \"command not found\"", stderr) + } + }) + } +} + +func TestSSHLFSAuthenticateHonorsAuthorization(t *testing.T) { + checkSSHBinaries(t) + + addr, _ := startSSHServer(t, false, false, withLFS("https://git.example.com", true)) + + // Reads stay allowed with pushes disabled. + if _, stderr, code := runSSHCommand(t, addr, + "git-lfs-authenticate", "acme/test.git", "download"); code != 0 { + t.Fatalf("download exit %d, want 0\nstderr: %s", code, stderr) + } + + // An upload is a write, so the -allow-push gate applies. This must not be + // a 127: the command exists and deliberately refused. + _, stderr, code := runSSHCommand(t, addr, + "git-lfs-authenticate", "acme/test.git", "upload") + if code == 0 { + t.Fatal("upload was authorized with pushes disabled") + } + if code == 127 { + t.Fatal("a denied request must not look like a missing command") + } + if !strings.Contains(stderr, "denied") { + t.Errorf("stderr = %q, want it to mention the denial", stderr) + } +} + +func TestSSHLFSAuthenticateRejectsABadPath(t *testing.T) { + checkSSHBinaries(t) + + addr, _ := startSSHServer(t, true, false, withLFS("https://git.example.com", true)) + + _, _, code := runSSHCommand(t, addr, + "git-lfs-authenticate", "not-a-two-segment-path-because-it-has-one", "download") + if code == 0 { + t.Fatal("a malformed repository path was accepted") + } + if code == 127 { + t.Fatal("a bad path must not look like a missing command") + } +} diff --git a/cmd/objgitd/lfs_test.go b/cmd/objgitd/lfs_test.go new file mode 100644 index 0000000..99fcb56 --- /dev/null +++ b/cmd/objgitd/lfs_test.go @@ -0,0 +1,886 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + "github.com/go-git/go-billy/v6/memfs" + "github.com/tigrisdata/objgit/internal/auth" + "github.com/tigrisdata/objgit/internal/lfs" + "github.com/tigrisdata/objgit/internal/repofs" + tstorage "github.com/tigrisdata/storage-go" +) + +const ( + testOID = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + testRepo = "acme/test" +) + +// memBucket is an in-memory stand-in for the Tigris bucket. The handler tests +// need a bucket but CI has no credentials, so every LFS request lands here. +type memBucket struct { + mu sync.Mutex + objects map[string][]byte + meta map[string]map[string]string + etags map[string]string + seq int +} + +func newMemBucket() *memBucket { + return &memBucket{ + objects: map[string][]byte{}, + meta: map[string]map[string]string{}, + etags: map[string]string{}, + } +} + +func (m *memBucket) HeadObject(_ context.Context, in *s3.HeadObjectInput, _ ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { + m.mu.Lock() + defer m.mu.Unlock() + key := aws.ToString(in.Key) + body, ok := m.objects[key] + if !ok { + return nil, &types.NotFound{} + } + return &s3.HeadObjectOutput{ + ContentLength: aws.Int64(int64(len(body))), + Metadata: m.meta[key], + ETag: aws.String(m.etags[key]), + }, nil +} + +func (m *memBucket) GetObject(_ context.Context, in *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + m.mu.Lock() + defer m.mu.Unlock() + key := aws.ToString(in.Key) + body, ok := m.objects[key] + if !ok { + return nil, &types.NoSuchKey{} + } + return &s3.GetObjectOutput{ + Body: io.NopCloser(bytes.NewReader(body)), + ContentLength: aws.Int64(int64(len(body))), + ETag: aws.String(m.etags[key]), + }, nil +} + +func (m *memBucket) PutObject(_ context.Context, in *s3.PutObjectInput, _ ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + m.mu.Lock() + defer m.mu.Unlock() + key := aws.ToString(in.Key) + _, exists := m.objects[key] + if aws.ToString(in.IfNoneMatch) == "*" && exists { + return nil, errPreconditionFailed + } + if want := aws.ToString(in.IfMatch); want != "" && m.etags[key] != want { + return nil, errPreconditionFailed + } + var body []byte + if in.Body != nil { + body, _ = io.ReadAll(in.Body) + } + m.seq++ + m.objects[key] = body + m.meta[key] = in.Metadata + m.etags[key] = string(rune('a'+m.seq%26)) + "-etag" + return &s3.PutObjectOutput{}, nil +} + +// RenameObject models the Tigris extension: the source key is gone afterwards. +func (m *memBucket) RenameObject(_ context.Context, in *s3.CopyObjectInput, _ ...func(*s3.Options)) (*s3.CopyObjectOutput, error) { + m.mu.Lock() + defer m.mu.Unlock() + _, src, _ := strings.Cut(aws.ToString(in.CopySource), "/") + body, ok := m.objects[src] + if !ok { + return nil, &types.NoSuchKey{} + } + key := aws.ToString(in.Key) + m.seq++ + m.objects[key] = body + m.meta[key] = m.meta[src] + m.etags[key] = string(rune('a'+m.seq%26)) + "-etag" + delete(m.objects, src) + delete(m.meta, src) + delete(m.etags, src) + return &s3.CopyObjectOutput{}, nil +} + +func (m *memBucket) DeleteObject(_ context.Context, in *s3.DeleteObjectInput, _ ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) { + m.mu.Lock() + defer m.mu.Unlock() + key := aws.ToString(in.Key) + delete(m.objects, key) + delete(m.meta, key) + delete(m.etags, key) + return &s3.DeleteObjectOutput{}, nil +} + +// put seeds an object directly, bypassing conditional writes. +func (m *memBucket) put(key string, body []byte) { + m.mu.Lock() + defer m.mu.Unlock() + m.seq++ + m.objects[key] = body + m.etags[key] = string(rune('a'+m.seq%26)) + "-etag" +} + +// errPreconditionFailed carries the error code a refused conditional write +// reports, which internal/lfs matches to decide whether to retry. +var errPreconditionFailed = &smithy.GenericAPIError{ + Code: "PreconditionFailed", + Message: "At least one of the pre-conditions you specified did not hold", +} + +// staticLFSTTL is the per-request URL lifetime for a test: a fixed value, with +// no credential to inspect. The daemon builds this from the live credential +// instead (lfsURLTTLFunc). +func staticLFSTTL(d time.Duration) func(context.Context) time.Duration { + return func(context.Context) time.Duration { return d } +} + +// newLFSServer starts an httptest server whose daemon has LFS enabled over an +// in-memory bucket. +func newLFSServer(t *testing.T, allowPush, allowLocks bool) (*httptest.Server, *lfs.Store, *memBucket) { + t.Helper() + + bucket := newMemBucket() + client := s3.New(s3.Options{ + Region: "auto", + BaseEndpoint: aws.String("https://t3.storage.dev"), + Credentials: credentials.NewStaticCredentialsProvider( + "AKIAEXAMPLEEXAMPLE", "secretsecretsecret", ""), + }) + store := lfs.NewStore(bucket, lfs.NewPresigner(client, "test-bucket"), "test-bucket") + + d := &daemon{ + sysFS: memfs.New(), + resolver: repofs.BucketResolver{Base: newMemBase()}, + authz: auth.AllowAnonymous{AllowWrite: allowPush}, + lfs: &lfsService{ + store: store, + ttl: staticLFSTTL(15 * time.Minute), + maxSize: 1 << 30, + maxBatch: 100, + allowLocks: allowLocks, + }, + } + ts := httptest.NewServer(d.httpHandler()) + t.Cleanup(ts.Close) + return ts, store, bucket +} + +// postLFS sends an LFS request and returns the status and decoded body. +func postLFS(t *testing.T, ts *httptest.Server, path string, body any) (int, map[string]any) { + t.Helper() + return requestLFS(t, ts, http.MethodPost, path, body) +} + +func requestLFS(t *testing.T, ts *httptest.Server, method, path string, body any) (int, map[string]any) { + t.Helper() + + var r io.Reader + if body != nil { + raw, err := json.Marshal(body) + if err != nil { + t.Fatalf("encoding request: %v", err) + } + r = bytes.NewReader(raw) + } + req, err := http.NewRequest(method, ts.URL+path, r) + if err != nil { + t.Fatalf("building request: %v", err) + } + req.Header.Set("Accept", lfs.MediaType) + if body != nil { + req.Header.Set("Content-Type", lfs.MediaType) + } + + resp, err := ts.Client().Do(req) + if err != nil { + t.Fatalf("%s %s: %v", method, path, err) + } + defer resp.Body.Close() + + raw, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("reading response: %v", err) + } + out := map[string]any{} + if len(bytes.TrimSpace(raw)) > 0 { + if err := json.Unmarshal(raw, &out); err != nil { + t.Fatalf("decoding response %q: %v", raw, err) + } + } + // Every LFS answer has to be recognizable as one. + if ct := resp.Header.Get("Content-Type"); ct != lfs.MediaType { + t.Errorf("Content-Type = %q, want %q", ct, lfs.MediaType) + } + return resp.StatusCode, out +} + +func TestLFSBatchDownload(t *testing.T) { + ts, store, bucket := newLFSServer(t, true, true) + + // Seed membership through the real path: a verified upload. + seedObject(t, store, bucket) + + status, body := postLFS(t, ts, "/acme/test.git/info/lfs/objects/batch", map[string]any{ + "operation": "download", + "objects": []map[string]any{{"oid": testOID, "size": 4}}, + }) + + if status != http.StatusOK { + t.Fatalf("status = %d, want 200 (body %v)", status, body) + } + objects, _ := body["objects"].([]any) + if len(objects) != 1 { + t.Fatalf("got %d objects, want 1", len(objects)) + } + obj, _ := objects[0].(map[string]any) + actions, ok := obj["actions"].(map[string]any) + if !ok { + t.Fatalf("object has no actions: %v", obj) + } + if _, ok := actions["download"]; !ok { + t.Errorf("actions = %v, want a download entry", actions) + } +} + +// TestLFSBatchReportsAMissingObjectInsideA200 pins the protocol rule that a +// per-object failure is not a request failure. +func TestLFSBatchReportsAMissingObjectInsideA200(t *testing.T) { + ts, _, _ := newLFSServer(t, true, true) + + status, body := postLFS(t, ts, "/acme/test.git/info/lfs/objects/batch", map[string]any{ + "operation": "download", + "objects": []map[string]any{{"oid": testOID, "size": 4}}, + }) + + if status != http.StatusOK { + t.Fatalf("status = %d, want 200", status) + } + objects, _ := body["objects"].([]any) + obj, _ := objects[0].(map[string]any) + objErr, ok := obj["error"].(map[string]any) + if !ok { + t.Fatalf("object has no error: %v", obj) + } + if code, _ := objErr["code"].(float64); int(code) != http.StatusNotFound { + t.Errorf("object error code = %v, want 404", objErr["code"]) + } +} + +func TestLFSBatchUploadDeniedWithoutPush(t *testing.T) { + ts, _, _ := newLFSServer(t, false, true) + + status, _ := postLFS(t, ts, "/acme/test.git/info/lfs/objects/batch", map[string]any{ + "operation": "upload", + "objects": []map[string]any{{"oid": testOID, "size": 4}}, + }) + + // A denied write is 403. The git path answers 401 here, but an LFS client + // reads 401 as "retry with credentials" and loops. + if status != http.StatusForbidden { + t.Fatalf("status = %d, want 403", status) + } +} + +func TestLFSBatchDownloadAllowedWithoutPush(t *testing.T) { + ts, _, _ := newLFSServer(t, false, true) + + status, _ := postLFS(t, ts, "/acme/test.git/info/lfs/objects/batch", map[string]any{ + "operation": "download", + "objects": []map[string]any{{"oid": testOID, "size": 4}}, + }) + + if status != http.StatusOK { + t.Fatalf("status = %d, want 200", status) + } +} + +func TestLFSBatchRejects(t *testing.T) { + for _, tt := range []struct { + name string + body map[string]any + want int + }{ + { + name: "malformed oid", + body: map[string]any{ + "operation": "download", + "objects": []map[string]any{{"oid": "../../etc/passwd", "size": 4}}, + }, + want: http.StatusUnprocessableEntity, + }, + { + name: "unknown operation", + body: map[string]any{ + "operation": "sideways", + "objects": []map[string]any{{"oid": testOID, "size": 4}}, + }, + want: http.StatusUnprocessableEntity, + }, + { + name: "unsupported transfer", + body: map[string]any{ + "operation": "download", + "transfers": []string{"tus"}, + "objects": []map[string]any{{"oid": testOID, "size": 4}}, + }, + want: http.StatusUnprocessableEntity, + }, + { + name: "object over the size cap", + body: map[string]any{ + "operation": "upload", + "objects": []map[string]any{{"oid": testOID, "size": 1<<30 + 1}}, + }, + want: http.StatusUnprocessableEntity, + }, + } { + t.Run(tt.name, func(t *testing.T) { + ts, _, _ := newLFSServer(t, true, true) + status, _ := postLFS(t, ts, "/acme/test.git/info/lfs/objects/batch", tt.body) + if status != tt.want { + t.Fatalf("status = %d, want %d", status, tt.want) + } + }) + } +} + +func TestLFSBatchRejectsAnOversizedBatch(t *testing.T) { + ts, _, _ := newLFSServer(t, true, true) + + objects := make([]map[string]any, 101) + for i := range objects { + objects[i] = map[string]any{"oid": testOID, "size": 4} + } + + status, _ := postLFS(t, ts, "/acme/test.git/info/lfs/objects/batch", map[string]any{ + "operation": "download", + "objects": objects, + }) + + if status != http.StatusRequestEntityTooLarge { + t.Fatalf("status = %d, want 413", status) + } +} + +func TestLFSBatchRejectsAMalformedBody(t *testing.T) { + ts, _, _ := newLFSServer(t, true, true) + + req, err := http.NewRequest(http.MethodPost, + ts.URL+"/acme/test.git/info/lfs/objects/batch", strings.NewReader("{not json")) + if err != nil { + t.Fatalf("building request: %v", err) + } + req.Header.Set("Accept", lfs.MediaType) + + resp, err := ts.Client().Do(req) + if err != nil { + t.Fatalf("posting: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusUnprocessableEntity { + t.Fatalf("status = %d, want 422", resp.StatusCode) + } +} + +func TestLFSRejectsAnUnacceptableClient(t *testing.T) { + ts, _, _ := newLFSServer(t, true, true) + + req, err := http.NewRequest(http.MethodPost, + ts.URL+"/acme/test.git/info/lfs/objects/batch", strings.NewReader("{}")) + if err != nil { + t.Fatalf("building request: %v", err) + } + req.Header.Set("Accept", "text/plain") + + resp, err := ts.Client().Do(req) + if err != nil { + t.Fatalf("posting: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNotAcceptable { + t.Fatalf("status = %d, want 406", resp.StatusCode) + } +} + +// TestLFSRoutesAreAbsentWhenDisabled is what -allow-lfs off has to look like +// from outside: a plain 404, which both git-lfs and the locking spec read as +// "this server has no LFS". +func TestLFSRoutesAreAbsentWhenDisabled(t *testing.T) { + d := &daemon{ + sysFS: memfs.New(), + resolver: repofs.BucketResolver{Base: newMemBase()}, + authz: auth.AllowAnonymous{AllowWrite: true}, + } + ts := httptest.NewServer(d.httpHandler()) + t.Cleanup(ts.Close) + + for _, path := range []string{ + "/acme/test.git/info/lfs/objects/batch", + "/acme/test.git/info/lfs/objects/verify", + "/acme/test.git/info/lfs/locks", + "/acme/test.git/info/lfs/locks/verify", + "/acme/test.git/info/lfs/locks/abc/unlock", + } { + t.Run(path, func(t *testing.T) { + resp, err := ts.Client().Post(ts.URL+path, lfs.MediaType, strings.NewReader("{}")) + if err != nil { + t.Fatalf("posting: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + t.Errorf("status = %d, want 404", resp.StatusCode) + } + }) + } +} + +func TestLFSVerify(t *testing.T) { + for _, tt := range []struct { + name string + seed func(*testing.T, *memBucket) + size int + want int + }{ + { + name: "object is stored at the declared size", + seed: func(_ *testing.T, b *memBucket) { b.put(lfs.StagingKey(testRepo, testOID), []byte("test")) }, + size: 4, + want: http.StatusOK, + }, + { + name: "nothing was staged", + seed: func(*testing.T, *memBucket) {}, + size: 4, + want: http.StatusNotFound, + }, + { + name: "object is stored at another size", + seed: func(_ *testing.T, b *memBucket) { b.put(lfs.StagingKey(testRepo, testOID), []byte("longer")) }, + size: 4, + want: http.StatusUnprocessableEntity, + }, + } { + t.Run(tt.name, func(t *testing.T) { + ts, _, bucket := newLFSServer(t, true, true) + tt.seed(t, bucket) + + status, _ := postLFS(t, ts, "/acme/test.git/info/lfs/objects/verify", map[string]any{ + "oid": testOID, "size": tt.size, + }) + if status != tt.want { + t.Fatalf("status = %d, want %d", status, tt.want) + } + }) + } +} + +// TestLFSVerifyGrantsMembership is the join between upload and download: until +// verify runs, the repository does not hold the object and a download 404s. +func TestLFSVerifyGrantsMembership(t *testing.T) { + ts, _, bucket := newLFSServer(t, true, true) + // Stand in for the client's presigned upload, which lands in this + // repository's own staging area. + bucket.put(lfs.StagingKey(testRepo, testOID), []byte("test")) + + batch := map[string]any{ + "operation": "download", + "objects": []map[string]any{{"oid": testOID, "size": 4}}, + } + + _, before := postLFS(t, ts, "/acme/test.git/info/lfs/objects/batch", batch) + if !objectHasError(before) { + t.Fatal("an unverified object must not be downloadable") + } + + status, _ := postLFS(t, ts, "/acme/test.git/info/lfs/objects/verify", map[string]any{ + "oid": testOID, "size": 4, + }) + if status != http.StatusOK { + t.Fatalf("verify status = %d, want 200", status) + } + + _, after := postLFS(t, ts, "/acme/test.git/info/lfs/objects/batch", batch) + if objectHasError(after) { + t.Fatal("a verified object must be downloadable") + } +} + +func objectHasError(body map[string]any) bool { + objects, _ := body["objects"].([]any) + if len(objects) == 0 { + return true + } + obj, _ := objects[0].(map[string]any) + _, has := obj["error"] + return has +} + +func TestLFSLocks(t *testing.T) { + ts, _, _ := newLFSServer(t, true, true) + + status, body := postLFS(t, ts, "/acme/test.git/info/lfs/locks", map[string]any{ + "path": "big.bin", + }) + if status != http.StatusCreated { + t.Fatalf("create status = %d, want 201 (body %v)", status, body) + } + lock, _ := body["lock"].(map[string]any) + id, _ := lock["id"].(string) + if id == "" { + t.Fatalf("created lock has no id: %v", body) + } + + // A second lock on the same path is a conflict that names the holder. + status, body = postLFS(t, ts, "/acme/test.git/info/lfs/locks", map[string]any{ + "path": "big.bin", + }) + if status != http.StatusConflict { + t.Fatalf("conflict status = %d, want 409", status) + } + if held, _ := body["lock"].(map[string]any); held["id"] != id { + t.Errorf("conflict names lock %v, want %q", held["id"], id) + } + + status, body = requestLFS(t, ts, http.MethodGet, "/acme/test.git/info/lfs/locks", nil) + if status != http.StatusOK { + t.Fatalf("list status = %d, want 200", status) + } + if locks, _ := body["locks"].([]any); len(locks) != 1 { + t.Fatalf("listed %d locks, want 1", len(locks)) + } + + status, _ = postLFS(t, ts, "/acme/test.git/info/lfs/locks/"+id+"/unlock", map[string]any{}) + if status != http.StatusOK { + t.Fatalf("unlock status = %d, want 200", status) + } + + _, body = requestLFS(t, ts, http.MethodGet, "/acme/test.git/info/lfs/locks", nil) + if locks, _ := body["locks"].([]any); len(locks) != 0 { + t.Fatalf("listed %d locks after unlock, want 0", len(locks)) + } +} + +// TestLFSLocksVerifySplitsOwnership is the call git-lfs makes on every push. An +// anonymous deployment has one owner, so every lock is "ours" and no push is +// ever blocked. +func TestLFSLocksVerifySplitsOwnership(t *testing.T) { + ts, _, _ := newLFSServer(t, true, true) + + if status, _ := postLFS(t, ts, "/acme/test.git/info/lfs/locks", map[string]any{ + "path": "big.bin", + }); status != http.StatusCreated { + t.Fatalf("create status = %d, want 201", status) + } + + status, body := postLFS(t, ts, "/acme/test.git/info/lfs/locks/verify", map[string]any{}) + if status != http.StatusOK { + t.Fatalf("verify status = %d, want 200", status) + } + + ours, _ := body["ours"].([]any) + theirs, _ := body["theirs"].([]any) + if len(ours) != 1 { + t.Errorf("ours = %d locks, want 1", len(ours)) + } + if len(theirs) != 0 { + t.Errorf("theirs = %d locks, want 0; a same-owner lock must never block a push", len(theirs)) + } +} + +func TestLFSLocksAreAbsentWhenDisabled(t *testing.T) { + ts, _, _ := newLFSServer(t, true, false) + + for _, path := range []string{ + "/acme/test.git/info/lfs/locks", + "/acme/test.git/info/lfs/locks/verify", + "/acme/test.git/info/lfs/locks/abc/unlock", + } { + resp, err := ts.Client().Post(ts.URL+path, lfs.MediaType, strings.NewReader("{}")) + if err != nil { + t.Fatalf("posting %s: %v", path, err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + t.Errorf("%s status = %d, want 404", path, resp.StatusCode) + } + } +} + +func TestLFSLockCreateDeniedWithoutPush(t *testing.T) { + ts, _, _ := newLFSServer(t, false, true) + + status, _ := postLFS(t, ts, "/acme/test.git/info/lfs/locks", map[string]any{"path": "big.bin"}) + if status != http.StatusForbidden { + t.Fatalf("status = %d, want 403", status) + } +} + +// seedObject stores a blob and verifies it, which is the only way a repository +// comes to hold an object. +func seedObject(t *testing.T, store *lfs.Store, bucket *memBucket) { + t.Helper() + bucket.put(lfs.StagingKey(testRepo, testOID), []byte("test")) + + status, err := store.Verify(context.Background(), testRepo, testOID, 4) + if err != nil { + t.Fatalf("seeding object: %v", err) + } + if status != lfs.VerifyOK { + t.Fatalf("seeding object: verify status %v, want ok", status) + } +} + +// TestLFSVerifyCannotStealAnotherRepositorysObject is the end-to-end form of +// the read-oracle attack. An oid and its size sit side by side in every pointer +// file, so knowing both must not be enough to join a repository to an object. +func TestLFSVerifyCannotStealAnotherRepositorysObject(t *testing.T) { + ts, store, bucket := newLFSServer(t, true, true) + + // A victim repository uploads and verifies an object properly. + bucket.put(lfs.StagingKey("victim/private", testOID), []byte("test")) + if _, err := store.Verify(context.Background(), "victim/private", testOID, 4); err != nil { + t.Fatalf("seeding the victim: %v", err) + } + + // The attacker uploads nothing and claims the object in its own repository. + status, _ := postLFS(t, ts, "/attacker/scratch.git/info/lfs/objects/verify", map[string]any{ + "oid": testOID, "size": 4, + }) + if status != http.StatusNotFound { + t.Fatalf("verify status = %d, want 404; the attacker uploaded nothing", status) + } + + // And still cannot download it. + _, body := postLFS(t, ts, "/attacker/scratch.git/info/lfs/objects/batch", map[string]any{ + "operation": "download", + "objects": []map[string]any{{"oid": testOID, "size": 4}}, + }) + if !objectHasError(body) { + t.Fatal("the attacker obtained a download URL for another repository's object") + } +} + +// TestLFSBatchCapsTheRequestBody keeps an unauthenticated caller from making +// the daemon buffer an arbitrary body. The batch body is decoded before +// authorization, because the operation inside it decides which access is +// needed, so the cap has to come first. +func TestLFSBatchCapsTheRequestBody(t *testing.T) { + ts, _, _ := newLFSServer(t, false, true) + + // Far larger than maxBodyBytes for the configured batch cap. + huge := `{"operation":"download","objects":[` + + strings.Repeat(`{"oid":"`+testOID+`","size":1},`, 20000) + + `{"oid":"` + testOID + `","size":1}]}` + + req, err := http.NewRequest(http.MethodPost, + ts.URL+"/acme/test.git/info/lfs/objects/batch", strings.NewReader(huge)) + if err != nil { + t.Fatalf("building request: %v", err) + } + req.Header.Set("Accept", lfs.MediaType) + req.Header.Set("Content-Type", lfs.MediaType) + + resp, err := ts.Client().Do(req) + if err != nil { + t.Fatalf("posting: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + t.Fatal("an oversized body was accepted") + } +} + +// postRawLFS sends a body verbatim. The shared helpers encode a Go value, which +// cannot express a body that is deliberately too large to decode. +func postRawLFS(t *testing.T, ts *httptest.Server, path, body string) int { + t.Helper() + + req, err := http.NewRequest(http.MethodPost, ts.URL+path, strings.NewReader(body)) + if err != nil { + t.Fatalf("building request: %v", err) + } + req.Header.Set("Accept", lfs.MediaType) + req.Header.Set("Content-Type", lfs.MediaType) + + resp, err := ts.Client().Do(req) + if err != nil { + t.Fatalf("POST %s: %v", path, err) + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + return resp.StatusCode +} + +// TestLFSHandlersCapTheRequestBody covers every LFS route that reads a body. +// The batch route capped its own from the start; the rest read theirs whole, +// which let an anonymous caller hand the daemon as much JSON as it liked. +func TestLFSHandlersCapTheRequestBody(t *testing.T) { + ts, _, _ := newLFSServer(t, true, true) + + // Just past the cap for the batch size newLFSServer configures. Kept small + // on purpose: a body large enough to fill the socket buffer would race the + // server's early response against the client still writing. + svc := &lfsService{maxBatch: 100} + filler := strings.Repeat("a", int(svc.maxBodyBytes())+4096) + + for _, tt := range []struct { + name string + path string + body string + }{ + { + name: "batch", + path: "/acme/test.git/info/lfs/objects/batch", + body: `{"operation":"download","padding":"` + filler + `","objects":[]}`, + }, + { + name: "verify", + path: "/acme/test.git/info/lfs/objects/verify", + body: `{"oid":"` + testOID + `","size":4,"padding":"` + filler + `"}`, + }, + { + name: "lock create", + path: "/acme/test.git/info/lfs/locks", + body: `{"path":"` + filler + `"}`, + }, + { + name: "lock verify", + path: "/acme/test.git/info/lfs/locks/verify", + body: `{"cursor":"` + filler + `"}`, + }, + { + name: "unlock", + path: "/acme/test.git/info/lfs/locks/deadbeef/unlock", + body: `{"force":true,"padding":"` + filler + `"}`, + }, + } { + t.Run(tt.name, func(t *testing.T) { + if status := postRawLFS(t, ts, tt.path, tt.body); status != http.StatusRequestEntityTooLarge { + t.Fatalf("status = %d, want 413", status) + } + }) + } +} + +// TestLFSLockRejectsAnOversizedPath keeps an unbounded string out of the lock +// document. The path is stored verbatim and the document is read whole on every +// push, so one accepted giant path would slow every later push to the +// repository for good. +func TestLFSLockRejectsAnOversizedPath(t *testing.T) { + ts, _, _ := newLFSServer(t, true, true) + + status, _ := postLFS(t, ts, "/acme/test.git/info/lfs/locks", map[string]any{ + "path": strings.Repeat("a", maxLockPathBytes+1), + }) + if status != http.StatusUnprocessableEntity { + t.Fatalf("status = %d, want 422", status) + } + + // And nothing was written. + _, body := requestLFS(t, ts, http.MethodGet, "/acme/test.git/info/lfs/locks", nil) + if locks, _ := body["locks"].([]any); len(locks) != 0 { + t.Fatalf("listed %d locks, want 0", len(locks)) + } +} + +// TestLFSLockRejectsPastTheRepositoryCap bounds how many locks one repository +// can accumulate, for the same reason: every push reads the whole document. +func TestLFSLockRejectsPastTheRepositoryCap(t *testing.T) { + ts, _, bucket := newLFSServer(t, true, true) + seedLocks(t, bucket, maxLocksPerRepo) + + status, _ := postLFS(t, ts, "/acme/test.git/info/lfs/locks", map[string]any{ + "path": "one-too-many.bin", + }) + if status != http.StatusRequestEntityTooLarge { + t.Fatalf("status = %d, want 413", status) + } + + // A path that is already locked still answers with the conflict that names + // its holder, rather than the cap. + status, body := postLFS(t, ts, "/acme/test.git/info/lfs/locks", map[string]any{ + "path": "seeded/0.bin", + }) + if status != http.StatusConflict { + t.Fatalf("status = %d, want 409 (body %v)", status, body) + } +} + +// TestLFSURLTTLFollowsTheCredential pins that the presigned-URL lifetime is +// answered per request and not frozen at startup. A daemon that happens to +// start while a temporary credential is nearly expired would otherwise sign +// every URL for its whole life against the one-minute floor, long after the SDK +// replaced that credential. +func TestLFSURLTTLFollowsTheCredential(t *testing.T) { + const want = 15 * time.Minute + + expires := time.Now().Add(30 * time.Second) + client := &tstorage.Client{Client: s3.New(s3.Options{ + Region: "auto", + Credentials: aws.CredentialsProviderFunc(func(context.Context) (aws.Credentials, error) { + return aws.Credentials{ + AccessKeyID: "AKIAEXAMPLEEXAMPLE", + SecretAccessKey: "secretsecretsecret", + CanExpire: true, + Expires: expires, + }, nil + }), + })} + + ttl := lfsURLTTLFunc(client, want) + if got := ttl(t.Context()); got != time.Minute { + t.Fatalf("ttl on a nearly expired credential = %s, want the 1m floor", got) + } + + // The SDK refreshes the credential. The next request has to see that. + expires = time.Now().Add(time.Hour) + if got := ttl(t.Context()); got != want { + t.Fatalf("ttl after the credential was refreshed = %s, want %s", got, want) + } +} + +// seedLocks writes a lock document holding n locks straight into the bucket. +// Creating them one request at a time would re-encode a growing document a +// thousand times over. +func seedLocks(t *testing.T, bucket *memBucket, n int) { + t.Helper() + + doc := struct { + Locks []lfs.Lock `json:"locks"` + }{Locks: make([]lfs.Lock, n)} + for i := range doc.Locks { + doc.Locks[i] = lfs.Lock{ + ID: fmt.Sprintf("%032x", i), + Path: fmt.Sprintf("seeded/%d.bin", i), + LockedAt: time.Now().UTC(), + Owner: lfs.Owner{Name: lfs.AnonymousOwner}, + } + } + raw, err := json.Marshal(doc) + if err != nil { + t.Fatalf("encoding seeded lock document: %v", err) + } + bucket.put(lfs.LocksKey(testRepo), raw) +} diff --git a/cmd/objgitd/main.go b/cmd/objgitd/main.go index 3f54211..1f47dff 100644 --- a/cmd/objgitd/main.go +++ b/cmd/objgitd/main.go @@ -9,9 +9,12 @@ import ( "net" "net/http" "net/http/pprof" + "net/url" "os" "os/signal" "runtime" + "strings" + "sync/atomic" "syscall" "time" @@ -22,6 +25,7 @@ import ( "github.com/tigrisdata/objgit" "github.com/tigrisdata/objgit/internal" "github.com/tigrisdata/objgit/internal/auth" + "github.com/tigrisdata/objgit/internal/lfs" "github.com/tigrisdata/objgit/internal/metrics" "github.com/tigrisdata/objgit/internal/repofs" "github.com/tigrisdata/objgit/internal/s3fs" @@ -58,6 +62,13 @@ var ( maxConcurrentPushes = flag.Int("max-concurrent-pushes", 4, "pushes allowed to unpack a packfile at the same time; each one costs roughly 400 MiB of resident set for a large repository, so this is what bounds memory under concurrent pushes; 0 disables the limit") pushQueueTimeout = flag.Duration("push-queue-timeout", 2*time.Minute, "how long a push waits for a slot before it fails") + + allowLFS = flag.Bool("allow-lfs", false, "serve the Git LFS API; client transfers use presigned bucket URLs, and erofs snapshots can read verified LFS objects") + allowLFSLocks = flag.Bool("allow-lfs-locks", true, "serve the Git LFS file locking API; without a user store every anonymous caller is the same owner, so locking is advisory only") + externalURL = flag.String("external-url", "", "public base URL of this server, such as https://git.example.com; required for Git LFS over ssh:// because git-lfs-authenticate has to name the HTTP API") + lfsURLTTL = flag.Duration("lfs-url-ttl", 15*time.Minute, "how long a presigned Git LFS transfer URL stays valid") + lfsMaxSize = flag.Int64("lfs-max-size", 5<<30, "largest Git LFS object accepted, in bytes; one presigned PUT caps at 5 GiB") + lfsMaxBatch = flag.Int("lfs-max-batch", 500, "most objects accepted in one Git LFS batch request") ) // tigrisBase adapts *tigris.Storer to repofs.Base: Storer.Scoped returns the @@ -88,6 +99,23 @@ func main() { os.Exit(1) } + if *allowLFS { + if *httpBind == "" { + // Every LFS transfer is negotiated over HTTP, even for an ssh:// + // remote: git-lfs-authenticate exists only to name that endpoint. + slog.Error("-allow-lfs needs -http-bind; the Git LFS API is HTTP even for ssh:// remotes") + os.Exit(1) + } + if err := checkExternalURL(*externalURL); err != nil { + slog.Error("-external-url is not usable", "err", err) + os.Exit(1) + } + if *externalURL == "" && *sshBind != "" { + slog.Error("-allow-lfs with -ssh-bind needs -external-url; git-lfs-authenticate has to name a public HTTP URL") + os.Exit(1) + } + } + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer cancel() @@ -106,8 +134,9 @@ func main() { } // Harden the client's HTTP path so stale keep-alive connections to Tigris // fail fast and retry on a fresh connection instead of hanging the request - // forever (see internal/s3fs/resilient.go). Only sysFS (the SSH host key - // and webhook settings) uses this client; internal/storage/tigris dials its own. + // forever (see internal/s3fs/resilient.go). sysFS (the SSH host key + // and webhook settings) and the LFS store use this client; + // internal/storage/tigris dials its own. client := s3fs.Harden(rawClient) fsys, err := s3fs.NewS3FS(client, *bucket) @@ -169,6 +198,22 @@ func main() { snapshotTmpDir: *packCacheDir, } + if *allowLFS { + // The store talks to the bucket, so it gets the hardened client like + // every other request path. rawClient survives only for the presigner: + // s3.NewPresignClient needs the concrete *s3.Client, and hardening buys + // nothing there because presigning makes no network call. + d.lfs = &lfsService{ + store: lfs.NewStore(client, + lfs.NewPresigner(rawClient.Client, *bucket), *bucket), + ttl: lfsURLTTLFunc(rawClient, *lfsURLTTL), + maxSize: *lfsMaxSize, + maxBatch: *lfsMaxBatch, + externalURL: strings.TrimSuffix(*externalURL, "/"), + allowLocks: *allowLFSLocks, + } + } + slog.Info("objgitd listening", "version", objgit.Version, "http_bind", *httpBind, @@ -177,6 +222,8 @@ func main() { "bucket", *bucket, "allow_push", *allowPush, "allow_hooks", *allowHooks, + "allow_lfs", *allowLFS, + "external_url", *externalURL, "pack_cache_bytes", *packCacheBytes, "erofs_snapshots", *erofsSnapshots, "snapshot_cache_bytes", *snapshotCacheBytes, @@ -308,3 +355,94 @@ func handleHealthz(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "text/plain; charset=utf-8") fmt.Fprintln(w, "ok") } + +// checkExternalURL rejects an -external-url that cannot be used to build an +// absolute LFS endpoint. An empty value is allowed: the HTTP side then derives +// the base from the request, and only SSH needs the flag. +func checkExternalURL(raw string) error { + if raw == "" { + return nil + } + u, err := url.Parse(raw) + if err != nil { + return fmt.Errorf("parsing %q: %w", raw, err) + } + if u.Scheme != "http" && u.Scheme != "https" { + return fmt.Errorf("%q needs an http or https scheme", raw) + } + if u.Host == "" { + return fmt.Errorf("%q names no host", raw) + } + return nil +} + +// lfsURLTTLFunc builds the per-request presigned-URL lifetime. +// +// The answer is computed per request and never frozen at startup. A temporary +// credential is refreshed by the SDK while the process runs, so a daemon that +// happened to start two minutes before an expiry would otherwise clamp every +// URL it ever signs to the floor below, long after the credential behind it was +// replaced. Retrieve reads the SDK's credential cache and only reaches the +// provider when the credential has actually expired, so this is cheap enough to +// run on every LFS request. +// +// The returned function logs a clamp when it starts and when it stops, and not +// on every request: git-lfs asks for a batch on every fetch and every push. +func lfsURLTTLFunc(client *tstorage.Client, want time.Duration) func(context.Context) time.Duration { + var clamped, unreadable atomic.Bool + return func(ctx context.Context) time.Duration { + got, err := lfsTTL(ctx, client, want) + switch { + case err != nil && !unreadable.Swap(true): + slog.Warn("cannot inspect credentials for lfs url lifetime; using the configured value", + "lfs_url_ttl", want, "err", err) + case err == nil: + unreadable.Store(false) + } + switch { + case got < want && !clamped.Swap(true): + slog.Warn("credentials expire before the configured lfs url lifetime; clamping", + "lfs_url_ttl", want, "clamped_to", got) + case got >= want && clamped.Swap(false): + slog.Info("credentials now outlive the configured lfs url lifetime", + "lfs_url_ttl", want) + } + return got + } +} + +// lfsTTL clamps the presigned-URL lifetime to what the credentials can outlive. +// +// A presigned URL carries the session token of the credential that signed it, +// so a temporary credential (SSO, IMDS, assume-role) invalidates every URL it +// signed the moment it expires, whatever -lfs-url-ttl says. Static Tigris +// keypairs do not expire and keep the configured value. +// +// An error is reported rather than logged, because the caller runs this on every +// request and decides what is worth saying twice. +func lfsTTL(ctx context.Context, client *tstorage.Client, want time.Duration) (time.Duration, error) { + creds, err := client.Options().Credentials.Retrieve(ctx) + if err != nil { + return want, err + } + if !creds.CanExpire { + return want, nil + } + + // Leave a margin so a URL minted just before the clamp still outlives the + // round trip that uses it. + const margin = time.Minute + // A floor keeps a short-lived credential from producing a zero or negative + // expiry, which would sign URLs that are dead on arrival. Credentials this + // close to expiry are refreshed by the SDK well before the floor runs out. + const floor = time.Minute + + left := time.Until(creds.Expires) - margin + if left >= want { + return want, nil + } + if left < floor { + left = floor + } + return left, nil +} diff --git a/cmd/objgitd/snapshots.go b/cmd/objgitd/snapshots.go index c491a29..1ae500b 100644 --- a/cmd/objgitd/snapshots.go +++ b/cmd/objgitd/snapshots.go @@ -75,7 +75,13 @@ func (d *daemon) runSnapshots(repoPath string, st storage.Storer, updates []refU continue } - res, err := snapshot.Ensure(ctx, st, store, tree, d.snapshotTmpDir) + var openLFS snapshot.LFSOpener + if d.lfs != nil { + openLFS = func(ctx context.Context, oid string, size int64) (io.ReadCloser, error) { + return d.lfs.store.OpenVerified(ctx, repoPath, oid, size) + } + } + res, err := snapshot.EnsureWithLFS(ctx, st, store, tree, d.snapshotTmpDir, openLFS) if err != nil { log.Error("snapshot: build failed", "tree", tree.String(), "dur", res.Elapsed, "err", err) metrics.ObserveSnapshot("error", res.Elapsed, 0) diff --git a/cmd/objgitd/snapshots_test.go b/cmd/objgitd/snapshots_test.go index af07dca..efab04e 100644 --- a/cmd/objgitd/snapshots_test.go +++ b/cmd/objgitd/snapshots_test.go @@ -3,7 +3,10 @@ package main import ( "bytes" "context" + "crypto/sha256" + "encoding/hex" "errors" + "fmt" "io/fs" "log/slog" "net/http/httptest" @@ -21,6 +24,7 @@ import ( "github.com/go-git/go-git/v6/storage" "github.com/go-git/go-git/v6/storage/memory" "github.com/tigrisdata/objgit/internal/auth" + "github.com/tigrisdata/objgit/internal/lfs" "github.com/tigrisdata/objgit/internal/repofs" "github.com/tigrisdata/objgit/internal/snapshot" ) @@ -116,6 +120,51 @@ func TestSmartHTTPPushSnapshots(t *testing.T) { } } +func TestSmartHTTPPushSnapshotResolvesLFS(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + const repoPath = "acme/lfs-snap" + content := []byte("the full LFS payload\n") + sum := sha256.Sum256(content) + oid := hex.EncodeToString(sum[:]) + pointer := fmt.Sprintf("version https://git-lfs.github.com/spec/v1\noid sha256:%s\nsize %d\n", oid, len(content)) + + base := newSnapBase() + bucket := newMemBucket() + bucket.put(lfs.BlobKey(oid), content) + bucket.put(lfs.MarkerKey(repoPath, oid), nil) + bucket.meta[lfs.MarkerKey(repoPath, oid)] = map[string]string{"lfs-size": fmt.Sprint(len(content))} + d := &daemon{ + sysFS: memfs.New(), + resolver: repofs.BucketResolver{Base: base}, + authz: auth.AllowAnonymous{AllowWrite: true}, + snapshots: true, + snapshotTimeout: 30 * time.Second, + snapshotTmpDir: t.TempDir(), + lfs: &lfsService{store: lfs.NewStore(bucket, nil, "test-bucket")}, + } + ts := httptest.NewServer(d.httpHandler()) + t.Cleanup(ts.Close) + work := seedRepo(t) + writeFile(t, filepath.Join(work, "asset.bin"), pointer) + runGit(t, work, "add", "asset.bin") + runGit(t, work, "commit", "-m", "add LFS pointer") + if out, err := tryGit(work, "push", ts.URL+"/"+repoPath+".git", "main"); err != nil { + t.Fatalf("push: %v\n%s", err, out) + } + tree := plumbing.NewHash(strings.TrimSpace(runGit(t, work, "rev-parse", "HEAD^{tree}"))) + snap, err := snapshot.Open(context.Background(), base.repo(repoPath), tree) + if err != nil { + t.Fatal(err) + } + defer snap.Close() + got, err := fs.ReadFile(snap, "asset.bin") + if err != nil || !bytes.Equal(got, content) { + t.Errorf("asset.bin = %q, %v; want LFS payload", got, err) + } +} + func TestPushSnapshotsOff(t *testing.T) { if _, err := exec.LookPath("git"); err != nil { t.Skip("git not installed") diff --git a/cmd/objgitd/ssh.go b/cmd/objgitd/ssh.go index 0ef755a..9f837e1 100644 --- a/cmd/objgitd/ssh.go +++ b/cmd/objgitd/ssh.go @@ -145,6 +145,14 @@ func (d *daemon) handleSSH(s ssh.Session) { return } + // LFS is intercepted before gitServiceFor, which knows only the three git + // transport commands. The command arrives as three arguments: + // git-lfs-authenticate . + if cmd[0] == lfsAuthenticateCommand { + d.handleLFSAuthenticate(s, cmd) + return + } + service, ok := gitServiceFor(cmd[0]) if !ok { fmt.Fprintf(s.Stderr(), "objgitd: unsupported command %q\n", cmd[0]) diff --git a/cmd/objgitd/ssh_test.go b/cmd/objgitd/ssh_test.go index 8a2d2dd..bd05673 100644 --- a/cmd/objgitd/ssh_test.go +++ b/cmd/objgitd/ssh_test.go @@ -125,7 +125,7 @@ func checkSSHBinaries(t *testing.T) { // startSSHServer creates an in-memory daemon and starts the SSH server on an // ephemeral port. It returns the listening address and the backing resolver // store. -func startSSHServer(t *testing.T, allowPush, allowHooks bool) (string, *memBase) { +func startSSHServer(t *testing.T, allowPush, allowHooks bool, opts ...func(*daemon)) (string, *memBase) { t.Helper() mb := newMemBase() d := &daemon{ @@ -135,6 +135,9 @@ func startSSHServer(t *testing.T, allowPush, allowHooks bool) (string, *memBase) allowHooks: allowHooks, hookTimeout: 30 * time.Second, } + for _, opt := range opts { + opt(d) + } srv, err := newSSHServer(d, "") if err != nil { t.Fatalf("newSSHServer: %v", err) diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 0693a17..55c71c6 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -39,6 +39,7 @@ Three behaviors live on `*daemon`, so every transport acts the same way: | [snapshots.md](snapshots.md) | erofs images of pushed trees, and the snapshot cache. | | [tigris-storer.md](tigris-storer.md) | Repository storage: a `storage.Storer` on one Tigris bucket. | | [s3fs.md](s3fs.md) | Daemon-level state: a `billy.Filesystem` on Tigris. | +| [lfs.md](lfs.md) | Git LFS: presigned transfers, global blobs, per-repository markers. | Related documents outside this directory: @@ -46,4 +47,5 @@ Related documents outside this directory: binary layout, the failure modes, and the build order. - [../usage/hooks.md](../usage/hooks.md) — how to write a hook script. - [../usage/webhooks.md](../usage/webhooks.md) — how to configure and verify push webhooks. +- [../usage/lfs.md](../usage/lfs.md) — how to use Git LFS against objgitd. - `../plans/` — plans for work that is not yet complete. diff --git a/docs/architecture/lfs.md b/docs/architecture/lfs.md new file mode 100644 index 0000000..b4f11de --- /dev/null +++ b/docs/architecture/lfs.md @@ -0,0 +1,219 @@ +# Git LFS (`internal/lfs`) + +`objgitd` answers the Git LFS batch API with presigned Tigris URLs. The client +transfers to and from the bucket directly. The daemon reads LFS object bytes +only when it builds an EROFS snapshot of a tree with LFS pointers. That read +streams into the image builder and checks repository membership first. + +This matters because a large blob is the worst case for the git storer. +`EncodedObject` reads a whole body into memory, so one read of a large blob +costs its full size in resident set. An object over the 128 MiB pack cap also +gets a container to itself. LFS keeps both out of the repository: git holds a +small pointer file, and the bytes live under their own key. + +The feature is off by default. Without `-allow-lfs` the daemon registers no LFS +route, so every LFS path answers 404. + +## Layout in the bucket + +| Key | Contents | +| --- | --- | +| `lfs/blobs/` | The object bytes, stored once for the whole bucket | +| `//lfs/staging/` | An upload in flight, before it is verified | +| `//lfs/objects/` | A zero-byte marker: this repository holds that object | +| `//lfs/locks.json` | Every lock held in this repository | + +`` is the SHA-256 of the object, in lowercase hexadecimal. + +Bytes are global. Two repositories that hold the same file store it once. +Membership is per repository. + +## Membership is separate from storage + +Every batch decision reads the marker. No decision reads the blob. + +Without that split the server would be a read oracle. A caller who knows an oid +could ask any repository for it and receive a presigned URL for bytes it has no +access to. LFS oids are not secret. They sit in pointer files, in forks, and in +CI logs. + +## Uploads land in the repository, not in the shared key + +A client uploads to `//lfs/staging/`. `Verify` checks those +bytes, moves them to the shared key, and only then writes the marker. + +That indirection is what makes a marker mean something. A client that uploaded +straight to the shared key would leave verify able to check only that the bytes +exist and have the right length. A caller who merely knows an oid and a size can +already say both, and a pointer file contains both, side by side. Staging makes +the client prove it holds the content: to get a marker it must first write the +bytes, and Tigris checks them against the oid as it does. + +Staging also keeps a failed upload away from anything shared. A rejected object +is deleted from the caller's own staging key. Nothing in the verify path can +delete the blob other repositories are reading. + +The promotion uses `RenameObject`, a Tigris extension that moves an object in +place. It copies no data, so promoting a 5 GiB upload costs the same as +promoting an empty one, and it leaves no staged copy to collect. + +The cost is one upload per repository. A repository that does not yet hold an +object uploads it again, even when the bucket already has the bytes. The +promotion overwrites the same key, so storage stays at one copy. That is safe +because the key is the hash of the content. + +The marker is also the only reference count a collector can use. A blob whose +last marker is gone is unreachable. + +CAUTION: `repofs.Parse` rejects the organization ID `lfs`, and any path segment +that starts with a dot. A repository named `lfs/blobs` would otherwise take the +LFS prefix for itself. + +## The oid is a bucket key + +`ValidateOID` accepts 64 lowercase hexadecimal characters and nothing else. That +admits no separator, no dot, no escape, and no control byte. It runs before any +other work. + +Uppercase is rejected and never folded. git-lfs computes lowercase, so an +uppercase oid would store the same bytes under a second key. + +## Upload integrity + +The daemon never reads an upload body, so it cannot hash it. Two checks cover +that gap, and they run at different times. + +**At upload time**, the presigned PUT carries `ChecksumSHA256`. Tigris rejects a +body that does not hash to the key. A client cannot skip this: the checksum is a +signed header, so removing it makes the signature invalid. + +CAUTION: The stock AWS presigner moves any unrecognized `X-Amz-*` header into +the query string. `X-Amz-Checksum-Sha256` is one of those, and a hoisted +checksum is a checksum nobody sends and nothing enforces. `noHoistPresigner` +turns hoisting off for uploads. `TestPresignPutSignsTheChecksumHeader` pins it. + +**At verify time**, `Verify` heads the *staged* object with the checksum mode +enabled and compares both the length and the digest. This is the only check that +runs on the daemon's own credentials. It is also the only check on size, because +`Content-Length` is not signed on a presigned PUT and cannot be enforced there. + +A mismatch deletes the staged object, which belongs to the caller. Nothing here +can reach the shared blob, so a bad verify cannot destroy an object other +repositories depend on. + +`Verify` writes the marker, and nothing else does. An upload the client never +verifies leaves no membership behind. + +## Uploads and downloads sign differently + +A download uses the stock presigner. Turning hoisting off also signs +`x-amz-checksum-mode` on a GetObject, and a URL signed that way fails unless the +client sends that header back. A download URL has to stand on its own, so only +`Host` is signed. `TestPresignGetNeedsNoRequestHeaders` pins it. + +## Batch outcomes + +| Operation | Marker | Response for that object | +| --- | --- | --- | +| `download` | present | `actions.download`, a presigned GET | +| `download` | absent | `error`, code 404 | +| `upload` | present, size matches | no `actions` key, so the client skips it | +| `upload` | present, size differs | `error`, code 422 | +| `upload` | absent | `actions.upload` and `actions.verify` | + +Every object carries `authenticated: true`. Without it git-lfs runs credential +discovery against the storage host and can attach an `Authorization` header, +which a presigned request rejects. + +Every action carries `expires_in`, so a client can tell a stale URL from a +denied one. + +A per-object failure travels inside a 200. Only a whole-request failure gets a +non-200 status. + +## Request body size + +Every LFS handler caps the body it reads with `http.MaxBytesReader`. A body over +the cap gets a 413. These routes answer an anonymous caller when `-allow-lfs` is +set, so the daemon must not hold an arbitrary amount of JSON. + +The cap comes from `-lfs-max-batch`, and not from a flag of its own. One object +entry is an oid, a size, and punctuation, so the batch cap sets the size of the +largest legal body. + +The verify and lock routes accept an empty body, which the protocol permits. +Those handlers refuse an oversize body, and they tolerate every other decode +error. + +## Locks + +One JSON document per repository, written under the same compare-and-swap the +ref cache uses: `If-None-Match: *` for the first write, then `If-Match` with the +ETag. A refused write re-reads and re-applies, so a concurrent lock is never +overwritten. + +Lock IDs are generated by the server. The ID comes back as a path segment on +unlock, so it is never taken from the client. + +Two caps bound the document. One path is 4096 bytes at most, which is PATH_MAX +on Linux and more than git can check out. A repository holds 1000 locks at most, +which is far above the handful of binary assets that file locking coordinates. A +request over the path cap gets a 422. A request over the count cap gets a 413. + +The count cap is soft. Two creates that race can both pass it. The cap bounds +growth over time, and it is not an exact limit. A path that already holds a lock +passes the count cap, so a repeated `git lfs lock` still gets the 409 that names +the holder. + +CAUTION: The owner is the HTTP Basic username, or `anonymous`. With no user +store every anonymous caller is the same owner, so every lock lands in `ours` +and no push is ever blocked. Locking tells users apart only once callers send a +username. `-allow-lfs-locks=false` removes the four routes, and an unregistered +route is a 404, which the specification says does not halt a push. + +## SSH + +An `ssh://` remote still needs the HTTP listener. `git-lfs-authenticate` exists +only to name it, and it needs `-external-url`. + +CAUTION: When LFS is off, the command must exit **127** and say `command not +found`. git-lfs falls back to guessing the HTTP endpoint on 127 alone. Any other +non-zero exit is reported to the user as a hard failure, so exiting 1 would +break every `git lfs` operation over SSH rather than leaving it alone. + +## Credentials and URL lifetime + +A presigned URL carries the session token of the credential that signed it. A +temporary credential invalidates every URL it signed when it expires. At +startup `lfsTTL` checks whether the credential can expire and clamps +`-lfs-url-ttl` to the remaining session time. A static Tigris keypair keeps the +configured value. + +## Testing + +CI has no bucket credentials, so unit tests run against a fake bucket. The +presigner is the exception: presigning makes no network call, so the tests +exercise the real signer against static credentials and assert on the URL. A +fake presigner would hide exactly the hoisting bug these tests exist to catch. + +Three facts need a real bucket and live in `livebucket_test.go`, behind +`OBJGIT_TIGRIS_LIVE_BUCKET`: that Tigris enforces the checksum, that the header +cannot be dropped, and that a HEAD returns the stored digest. Run them before +trusting a change to `presign.go` or `store.go`. + +## Known gaps + +- Nothing deletes a blob. The marker gives a collector its reference count, but + no collector exists. +- `-lfs-max-size` caps the size a client *declares*, not the size it uploads. + Content-Length is not signed on a presigned PUT, so the real limit at upload + time is what the bucket accepts. `Verify` catches the difference, but a client + that never verifies leaves the staged object behind. Staged objects under + `//lfs/staging/` are the one place an abandoned upload + accumulates; a bucket TTL rule on that prefix is the simplest way to reap + them until a collector exists. +- One presigned PUT caps at 5 GiB. Larger objects need multipart upload. +- The `git-lfs-authenticate` header map is empty. Real authentication needs a + token issuer there. +- A presigned URL works for its full lifetime, whatever the Authorizer later + decides. diff --git a/docs/architecture/snapshots.md b/docs/architecture/snapshots.md index 21cafbe..b3d97cb 100644 --- a/docs/architecture/snapshots.md +++ b/docs/architecture/snapshots.md @@ -14,9 +14,9 @@ The design is in ## Object layout -| Key | Contents | -| --------------------------------- | ----------------------------------------- | -| `snapshots/erofs/v1/.erofs` | One EROFS image of the git tree ``. | +| Key | Contents | +| --------------------------------- | -------------------------------------------------------------------- | +| `snapshots/erofs/v2/.erofs` | One EROFS image of the git tree ``, with LFS pointers resolved. | The key uses the tree hash, not the commit hash. Many commits can point at one tree, such as a branch and a tag on one commit. These commits share one image. @@ -24,8 +24,9 @@ tree, such as a branch and a tag on one commit. These commits share one image. The builder pins every mtime to the Unix epoch. As a result, two builds of one tree give identical bytes, and a second build is never necessary. -`v1` names the mapping from git to EROFS, and the build options. A change to -either one uses `v2`. A `v2` build never overwrites a `v1` image. +`v2` names the mapping from git to EROFS, including LFS pointer resolution. +The earlier `v1` images may contain pointer text. A `v2` build never reuses or +overwrites a `v1` image. The key does not start with `objects/`, `packs/`, or `refs/`. The pack index, the loose object reads, and the loose ref listing therefore never see it. @@ -34,7 +35,7 @@ Each image carries user metadata: | Metadata key | Value | | -------------- | ---------------------------------------------------------------------------------- | -| `erofs-format` | `1` | +| `erofs-format` | `2` | | `erofs-sha256` | The hex SHA-256 of the image. The cache rejects a download that does not match it. | | `git-tree` | The hex tree hash. | | `erofs-files` | The count of regular files. | @@ -50,6 +51,17 @@ Each image carries user metadata: | `120000` symlink | Symlink. The target is the blob content. | | `160000` gitlink | Empty directory, mode `0755`. | +A canonical Git LFS v1 pointer in a regular blob becomes the referenced LFS +object's bytes. The builder uses the pointer's declared size and checks the +repository membership marker before it opens the shared blob. A missing +marker, size mismatch, missing blob, malformed pointer, or pointer with an +extension fails the build. Extensions need a client-side smudge filter to +reconstruct checkout bytes. A failed image build does not fail the push. + +Pointer resolution needs `-allow-lfs` so the daemon has an LFS store. If LFS +is disabled and a tree has a pointer, the image build fails instead of storing +pointer text. Non-LFS files build normally. + The gitlink rule gives the same result as `git checkout` of a submodule that is not initialized. @@ -81,10 +93,11 @@ Two pushes of one tree can run `Ensure` at the same time. Both build, and both put identical bytes. This is correct, so `Ensure` takes no lock. The temp file name is random for the same reason. -`Ensure` streams file content into the builder. The walk reads only the -size of each blob, through `EncodedObjectSize`. It gives the builder the size -and an open function with `AddFileFunc`. During `Build`, the builder opens -each blob one time, compresses it one pcluster at a time, and spools the +`Ensure` streams file content into the builder. The walk reads the size of +each blob through `EncodedObjectSize` and inspects blobs smaller than 1024 +bytes for LFS pointers. It gives the builder the file size and an open +function with `AddFileFunc`. During `Build`, the builder opens each Git blob +or LFS object one time, compresses it one pcluster at a time, and spools the compressed blocks to a temp file (`WithSpoolDir`, the same directory as the image). As a result, the memory for a build does not depend on the size of the tree. `TestEnsureMemory` builds 256 MiB of content with a live heap of @@ -116,7 +129,7 @@ A snapshot id is a tree hash, not the digest of the bytes. The cache therefore uses `GetChecked`, whose fetch returns the digest from the `erofs-sha256` metadata. A body that does not match is refused. -The cache id is `erofs-v1-`, with no repository prefix. Two repositories +The cache id is `erofs-v2-`, with no repository prefix. Two repositories that hold one tree share one local file. This is deduplication and not leakage. A caller gets to the id only through a tree that its own repository holds. diff --git a/docs/architecture/tigris-storer.md b/docs/architecture/tigris-storer.md index a6bfb2a..5906384 100644 --- a/docs/architecture/tigris-storer.md +++ b/docs/architecture/tigris-storer.md @@ -32,7 +32,7 @@ call has run. | `packs/.cue` | The record index for one container. | | `packed-refs` | Every reference in one object. See "References". | | `refs/` | One legacy loose reference. Read-only. | -| `snapshots/erofs/v1/.erofs` | One erofs image of a tree. See [snapshots.md](snapshots.md). | +| `snapshots/erofs/v2/.erofs` | One erofs image of a tree. See [snapshots.md](snapshots.md). | Shallow marks, the worktree index, and the repository configuration sit at root-level keys. They carry the same prefix as everything else. The `shallow` diff --git a/docs/plans/bin-cue-benchmark.md b/docs/plans/bin-cue-benchmark.md new file mode 100644 index 0000000..db254f4 --- /dev/null +++ b/docs/plans/bin-cue-benchmark.md @@ -0,0 +1,279 @@ +# Benchmark objgit before and after the bin/cue format change + +Status: Implemented as `cmd/formatbench`. See +[docs/usage/format-benchmark.md](../usage/format-benchmark.md) to run it. + +## Context + +`var/bin-cue-git.md:65-66` is the outline for the blog post "Object storage is +not a filesystem". One section is still a placeholder: + +``` +* Latency/storage improvements + * TODO(Xe): get before/after comparisons of a few repos +``` + +This plan fills that placeholder. It builds a harness that pushes three real +repositories into objgitd twice, once on each side of the storage format +change, and records numbers that go straight into the post. + +The change itself already shipped. Two commits made it, both on 2026-08-27: + +- `f4a1419` `feat(storage/tigris): bin/cue pack containers, async uploads, pack cache` +- `ebfe4e3` `refactor(objgitd)!: serve every repository from one bucket via the storer` + +Before those commits, objgitd stored git's native bare-repository layout as +bucket keys through `internal/s3fs`. After them, it stores `packs/.bin` +plus `packs/.cue` containers through `internal/storage/tigris`. + +## What "before" and "after" mean + +| Column | Ref | What it runs | +| --- | --- | --- | +| before | `v1.0.2` | `internal/s3fs` + go-git dotgit. One bucket. Listing cache and pack cache on. | +| after | `main` (`ce5a7ca`, v1.5.0) | `internal/storage/tigris`. bin/cue containers, zstd, cue v3. | + +`v1.0.2` is the last release where repositories live in the single `-bucket` +flag, which is also how HEAD works. That keeps the setup identical on both +sides. The releases after it (`v1.1.0`) moved repositories to per-keypair +buckets and turned repository-side caching off, which would move two variables +at once. + +One thing about this pair goes in the post as a caveat, not as a hidden fact: + +1. `v1.0.2` is missing two later `internal/s3fs` corrections, `bf81a1d` + (opt out of default request checksums) and `5bb834b` (directory markers). + RESOLVED: `v1.0.2` builds with Go 1.26.5 and completes a push and a clone + against Tigris today, so no cherry-pick is needed. The two corrections + stay unapplied, which keeps the "before" side exactly as it shipped. +2. HEAD is not only the format. It also carries zstd (`7bea058`), cue v3 with + delta reuse (`ea4d362`), pack prefetch (`eea720d`), and bounded pushes + (`877cd16`). The post must say the comparison is "the lazy thing" against + "today", not a controlled single-variable experiment. + +## Corpus + +Mirror-clone each repository once. Never write to the mirror. + +| Name | URL | Shape | +| --- | --- | --- | +| objgit | https://github.com/tigrisdata/objgit | Small. Mostly text. | +| x | https://github.com/Xe/x | Medium. Large object count. | +| tigris-blog | https://github.com/tigrisdata/tigris-blog | Blob-heavy. Images do not compress. | + +Confirm the `tigris-blog` clone URL before the first run. Record the resolved +commit, the object count from `git rev-list --all --count`, and the pack size +of the mirror for each one. + +## What gets measured + +Both builds report the same Prometheus counter with the same label values. +`internal/metrics/metrics.go:23` defines +`objgit_s3_requests_total{operation,status}`. `v1.0.2` feeds it from +`internal/s3fs/metrics.go` through `s3fs.SetMetricsObserver(metrics.ObserveS3)` +at `cmd/objgitd/main.go:81`. HEAD feeds it from `Storer.observe` +(`internal/storage/tigris/tigris.go:323`). Both use `GetObject`, `PutObject`, +`HeadObject`, `ListObjectsV2`, and `DeleteObject`. No proxy is needed. + +Per cell, that is per build and per repository and per workload: + +| Field | Source | +| --- | --- | +| Wall seconds | Driver timer around the `git` subprocess. | +| S3 requests by verb | Delta of `objgit_s3_requests_total{operation}`. | +| S3 time | Delta of `objgit_s3_request_duration_seconds_sum`. | +| Keys written | `ListObjectsV2` over the repository prefix after the push. | +| Bytes stored | Sum of the sizes from that same listing. | +| Bytes on the wire | Clone only. The `Receiving objects` line from git stderr. | + +Both eras key a repository under its `orgID/name` path. `v1.0.2` gets there +through the go-git loader (`cmd/objgitd/git_protocol.go:61`). HEAD gets there +through `repofs.BucketResolver.Resolve` (`internal/repofs/repofs.go:93-95`). + +CAUTION: they disagree about the `.git` suffix. HEAD parses the transport path +through `repofs.Parse`, which strips it (`internal/repofs/repofs.go:47`), so +its keys sit under `//`. `v1.0.2` hands the raw URL path to the +loader, which keeps it, so its keys sit under `/.git/`. A harness +that lists only one spelling reports an empty bucket for the other side. The +accounting and the cleanup both walk the two prefixes and sum them. + +## The harness + +Add `cmd/formatbench/`. It is a development tool, not part of the shipped +binary. `cmd/membench/` is the precedent, and `AGENTS.md:69` records that +membench is not shipped. + +Do not extend `cmd/membench` itself. It samples `/proc//status` +(`cmd/membench/proc.go:18`), so it drops every sample on macOS. This plan +needs wall time, request counts, and bucket size, and all three are portable. + +Reuse these patterns from membench rather than inventing new ones: + +- `moduleRoot()` (`cmd/membench/main.go:489`) to find the checkout root. +- `buildDaemon` (`main.go:335`) and `startDaemon` (`main.go:352`), which set + `cmd.Dir = root` so the daemon reads `BUCKET` and AWS credentials from `.env`. +- The `/metrics` scrape and parser in `cmd/membench/metrics.go:25-39`, widened + to read `CounterVec` series with labels instead of five bare gauges. +- The timestamped output directory and `repos.txt` cleanup list + (`report.go:342`). Bucket data is never deleted on its own. + +### Setup, once per session + +1. Run `git worktree add var/bench/src/before v1.0.2`. +2. Build `var/bench/bin/objgitd-before` from inside that worktree. Its + `go.mod` already declares `go 1.26.3`, so the current toolchain builds it. +3. Build `var/bench/bin/objgitd-after` from the checkout root. +4. Mirror-clone each corpus repository into `var/bench/mirrors/.git`. + +### One measurement cell + +1. Make a fresh repository path, `bench/--`. Never reuse + one. Every push is a cold push into an empty prefix. +2. Make a fresh empty directory for `-pack-cache-dir`. +3. Start the daemon and wait for it to answer. Flags for both builds: + `-bucket $BUCKET -http-bind 127.0.0.1:8080 -metrics-bind 127.0.0.1:9090 + -ssh-bind= -allow-push -pack-cache-dir `. Leave every other flag at + its default, so each build runs the way it shipped. +4. Scrape `/metrics` and record the counters. +5. Run `git push --mirror http://127.0.0.1:8080/bench/.git` from the + mirror. Record the wall time. A mirror push, and not a heads-and-tags + refspec: a mirror clone of a GitHub project also carries `refs/notes/*` and + `refs/pull/*`, whose commits a heads-and-tags push leaves behind. +6. Scrape `/metrics` again. Record the deltas. +7. List the bucket under the repository prefix. Record the key count and the + total bytes. +8. Stop the daemon. Delete the pack cache directory. +9. Start the daemon again with a new empty pack cache directory. This makes + the clone cold in the process, on disk, and in the pack index. +10. Scrape, run `git clone`, scrape again. Record the wall time and the + `Receiving objects` byte count from git stderr. +11. Stop the daemon. + +### Correctness gate + +A benchmark that measures a corrupt push is worthless. After the clone in +step 10, compare the clone against the mirror: + +- `git rev-list --all --count` must match. +- `git show-ref` must produce the same set of refs and hashes. +- `git fsck --full` must pass on the clone. + +If any of these fail, mark the cell `INVALID`. Do not report its timings. + +### Fairness controls + +- **Interleave the builds.** Run `(before, objgit)`, `(after, objgit)`, + `(before, x)`, `(after, x)`, and so on, then repeat the whole cycle. Wi-Fi + and Tigris both drift over an afternoon. If all the "before" runs happen + first, that drift looks like a result. +- Three repetitions for `objgit` and `tigris-blog`. Two for `x`. Report the + median with the minimum and maximum beside it. +- Stop and restart the daemon between every single measurement. +- Record the conditions once per session: `git --version`, `go version`, the + machine, the bucket region, and whether a VPN is on. Do not change any of + them mid-session. + +### Timeout policy + +Set a hard limit of 45 minutes for a push and 30 minutes for a clone. A prior +measurement in `docs/plans/pack-temp-file-cache.md:5-16` recorded 8,500 +`GetObject` calls to serve one clone of a 318-object repository under the old +layout, and that clone never finished. The `x` repository can do the same +thing on the "before" build. + +On a timeout, record `DNF` with the elapsed time and the request count +reached. Report the cell as `DNF` in the table. Do not extrapolate a finish +time from partial progress. + +### Output + +Two files in the run directory: + +- `results.json`, one record per cell, with every field above. +- `results.md`, the tables below already rendered, ready to paste. + +## The tables for the post + +Push: + +| repo | before wall | after wall | before PUTs | after PUTs | before keys | after keys | before bytes | after bytes | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | + +Clone: + +| repo | before wall | after wall | before GETs | after GETs | wire bytes | +| --- | --- | --- | --- | --- | --- | + +End the section with this line, as written: + +> Oh yeah, this was with my corporate laptop on wifi. + +## Files + +| Path | Change | +| --- | --- | +| `cmd/formatbench/main.go` | New. The driver: setup, cell loop, interleaving, timeouts. | +| `cmd/formatbench/metrics.go` | New. Scrape and diff `objgit_s3_requests_total` by label. | +| `cmd/formatbench/bucket.go` | New. List a prefix, sum key count and bytes. | +| `cmd/formatbench/git.go` | New. Timed git subprocesses and the wire-byte parser. | +| `cmd/formatbench/verify.go` | New. The correctness gate. | +| `cmd/formatbench/report.go` | New. `results.json` and `results.md`. | +| `cmd/formatbench/uuid.go` | New. Collision-free repository names. | +| `docs/plans/bin-cue-benchmark.md` | New. This plan, committed where `AGENTS.md` says plans live. | +| `docs/usage/format-benchmark.md` | New. How to run it. Follow `docs/usage/memory-benchmark.md` in shape. | +| `AGENTS.md` | One row in the "Where the code lives" table, next to `cmd/membench/`. | +| `var/bin-cue-git.md` | Replace the `TODO(Xe)` at line 66 with the finished tables. | + +## First results + +One repetition of `objgit` on each side, run on 2026-09-10 against a real +bucket. These are a smoke result, not the table for the post: one repetition +has no range, and one small repository does not show the loose-object cost the +old layout pays on a big one. + +Push: + +| Build | Wall | S3 requests | PUT | GET | LIST | Keys | Bucket bytes | +| --- | --: | --: | --: | --: | --: | --: | --: | +| before | 12.6s | 231 | 46 | 47 | 138 | 42 | 829.27 KiB | +| after | 2.1s | 19 | 5 | 11 | 3 | 4 | 902.98 KiB | + +Clone: + +| Build | Wall | S3 requests | GET | LIST | Wire bytes | +| --- | --: | --: | --: | --: | --: | +| before | 14.3s | 323 | 51 | 272 | 763.63 KiB | +| after | 13.4s | 53 | 49 | 4 | 767.96 KiB | + +Two things to read from this before the full run: + +1. The push win is already large on the smallest repository in the corpus: 231 + S3 calls against 19, and 42 keys against 4. +2. The clone difference is almost all `ListObjectsV2`, 272 against 4. The + `GetObject` counts are near equal because `ab60b2d` taught the old layout to + keep a pushed pack whole. A repository built by many pushes, or one with + loose objects, is where the old `GetObject` count grows. + +## Verification + +Done, with the results recorded above: + +1. `go build ./...` and `go vet ./...` pass. +2. `go test ./cmd/formatbench/` passes: 58 assertions over the metric parser, + the report writer, the wire-byte parser, the ref diff, and the prefix set. +3. Smoke run of `-repos objgit -builds after -reps 1` finishes in about a + minute and writes a complete `results.json`. +4. The `objgit_s3_requests_total` delta is non-zero on both sides, so the + observer is wired on both. A zero delta would mean the wiring broke, not + that the build is fast. +5. The correctness gate passes on both sides. It earned its place: it caught a + heads-and-tags refspec in the first draft that silently dropped + `refs/notes/*` and `refs/pull/*`. +6. `-cleanup` deletes both prefix spellings and leaves the bucket empty. + +Still to do: + +1. Full run: `go run ./cmd/formatbench`. +2. Paste the two tables into `var/bin-cue-git.md`, replacing the `TODO(Xe)`. +3. Delete the benchmark repositories with + `go run ./cmd/formatbench -cleanup var/bench/out//repos.txt`. diff --git a/docs/plans/erofs-lfs-resolution.md b/docs/plans/erofs-lfs-resolution.md new file mode 100644 index 0000000..40ab6a0 --- /dev/null +++ b/docs/plans/erofs-lfs-resolution.md @@ -0,0 +1,24 @@ +# Resolve LFS pointers in EROFS snapshots + +## Goal + +An EROFS image should contain the bytes a checkout reads for a regular Git +LFS file. The Git tree contains only a small pointer. The snapshot builder +must use the pointer's object size and stream the corresponding LFS blob. + +## Plan + +1. Rebase the LFS branch on `origin/main`, which introduced snapshots. +2. Recognize canonical v1 pointers in regular Git blobs smaller than 1024 + bytes. Leave ordinary blobs alone. Fail on malformed pointers and pointers + with extensions, since extensions require a client-side smudge filter. +3. Open the shared blob only after checking the repository's membership marker + and its size against the pointer. A missing or mismatched object fails the + image build. The push itself still succeeds. +4. Advance the image format to v2. The v1 images can contain pointer text, + so a builder must not treat one as a completed v2 image. +5. Test image contents, permissions, missing objects, marker isolation, and + the Smart HTTP push path with an in-memory bucket. + +The pointer size and canonical line format follow the +[Git LFS specification](https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md). diff --git a/docs/plans/lfs.md b/docs/plans/lfs.md new file mode 100644 index 0000000..49e3d96 --- /dev/null +++ b/docs/plans/lfs.md @@ -0,0 +1,508 @@ +# Plan: Git LFS support + +## Context + +`objgitd` stores every git object in a Tigris bucket. Large binary files are the +worst case for that design. `docs/reference/tigris-backend.md:466-474` records +one cost: `EncodedObject` reads a whole body into a `MemoryObject`, so one read +of a large blob costs its full size in resident heap. +`docs/architecture/tigris-storer.md:395-397` records a second: an object larger +than the 128 MiB pack cap gets a container to itself and exceeds the cap. + +Git LFS removes both. The repository holds a small pointer file. The large bytes +live under a separate key, and the client transfers them out of band. For a +server whose premise is object storage, the client can talk to the bucket +directly. + +There is no LFS code in the repository today. A search for `lfs` across `*.go` +and `*.md` returns nothing. + +Decisions from the user: + +- **Presign only.** `objgitd` never carries object bytes. The batch response + hands the client a presigned Tigris URL. The daemon is a control plane for + LFS, not a data plane. +- **Bucket-global deduplication.** Blob bytes live once at `lfs/blobs/`, + shared by every repository. +- **Full protocol surface.** The batch API, the `verify` endpoint, the file + locking API, and `git-lfs-authenticate` over SSH. + +Code and test conventions follow the `xe-go:xe-go-style` and +`xe-go:go-table-driven-tests` skills. Documentation follows `simple-english`. + +Three findings from the design review changed the shape of this plan. Each one +is marked CAUTION below, and each is verified against source, not assumed. + +## Files + +| Action | Path | Purpose | +| --- | --- | --- | +| Create | `internal/lfs/lfs.go` | Package doc, oid validation, key layout. | +| Create | `internal/lfs/batch.go` | Batch types and the batch resolver. | +| Create | `internal/lfs/store.go` | The bucket surface: `s3API` and the presigner. | +| Create | `internal/lfs/presign.go` | The custom `HTTPPresignerV4` that stops header hoisting. | +| Create | `internal/lfs/locks.go` | The lock document and its compare-and-swap writer. | +| Create | `internal/lfs/errors.go` | The `application/vnd.git-lfs+json` error body. | +| Create | `cmd/objgitd/lfs.go` | The HTTP handlers, on `*daemon`. | +| Edit | `cmd/objgitd/http.go` | New routes, registered only when LFS is on. | +| Edit | `cmd/objgitd/ssh.go` | Intercept `git-lfs-authenticate`. | +| Edit | `cmd/objgitd/git_protocol.go` | One new `daemon` field. | +| Edit | `internal/repofs/repofs.go` | Reject reserved organization names. | +| Edit | `cmd/objgitd/main.go` | Flags, the presign client, and the wiring. | +| Edit | `internal/metrics/metrics.go` | The `objgit_lfs_*` vectors and helpers. | +| Create | `internal/lfs/*_test.go`, `cmd/objgitd/lfs_test.go` | Tests. | +| Create | `docs/architecture/lfs.md`, `docs/usage/lfs.md`, `docs/plans/lfs.md` | Documentation. | +| Edit | `docs/architecture/README.md`, `AGENTS.md`, `README.md` | Index rows. | + +## Design + +### The package boundary + +`internal/lfs` owns the protocol and the bucket. It knows nothing about routing, +`*daemon`, or Prometheus. `cmd/objgitd/lfs.go` owns the handlers: parse the +path, call `d.authorize`, call into `internal/lfs`, render the result. This is +the split that `internal/auth` and `cmd/objgitd/http.go` already use. + +The package reaches the bucket through one narrow interface, in the style of +`s3API` in `internal/storage/tigris/tigris.go:86-93`, so tests can supply a fake +bucket. CI runs with no Tigris credentials. + +```go +type s3API interface { + HeadObject(context.Context, *s3.HeadObjectInput, ...func(*s3.Options)) (*s3.HeadObjectOutput, error) + PutObject(context.Context, *s3.PutObjectInput, ...func(*s3.Options)) (*s3.PutObjectOutput, error) + DeleteObject(context.Context, *s3.DeleteObjectInput, ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) +} +``` + +The presigner is **not** behind a fake. Presigning does no network I/O, so tests +run the real one against static credentials. See Testing. + +### The key layout + +``` +lfs/blobs/ the bytes, once per bucket +acme/test/lfs/objects/ a zero-byte membership marker +acme/test/lfs/locks.json one lock document per repository +acme/test/objects/ +acme/test/packs/.bin +``` + +Bytes are global. Membership is per repository. + +CAUTION: Global bytes alone are a cross-repository read oracle. Without the +marker, any caller can ask `/attacker/scratch/.../batch` for an oid belonging to +a private repository and receive a presigned GET for it. LFS oids leak in +pointer files, forks, and CI logs. Every batch decision therefore reads the +**marker**, never the global blob. An upload writes the marker after `verify` +succeeds. The marker is also the reference count that any future collector +needs. + +CAUTION: `repofs.Parse` accepts any two non-empty segments +(`internal/repofs/repofs.go:42-52`), so a repository named `lfs/blobs` takes the +LFS prefix for itself. `repofs.Parse` must reject the reserved organization name +`lfs`, and any segment that starts with `.` (which also protects the existing +`.objgit/ssh_host_ed25519_key`). Put this in `repofs`, so every transport gets +it, and pin it with a test. + +### Validation comes first + +The oid becomes part of a bucket key, and `repofs.Parse` does not sanitize +anything. Validation runs before any other work: + +- The oid is exactly 64 characters. +- Every character is `0-9` or `a-f`. Uppercase is rejected, never normalized. An + uppercase oid would create a second key for the same bytes and break dedup. +- The test is `hex.DecodeString` plus a length check. It is never a + `strings.Contains(oid, "..")` check. +- `size` is zero or more, and not larger than `-lfs-max-size`. +- `operation` is exactly `upload` or `download`. Anything else is an error, not + a default. + +A malformed request is 422 with an LFS error body. + +### The batch endpoint + +`POST /{orgID}/{repoName}/info/lfs/objects/batch` is the whole protocol. +`download` maps to `auth.Read`, `upload` to `auth.Write`. The handler calls +`d.authorize` the way `(*daemon).resolve` does (`cmd/objgitd/http.go:184-223`). + +For each object the handler does one `HeadObject` on the per-repository marker. +The calls run under an `errgroup` with `SetLimit`. A batch larger than +`-lfs-max-batch` is 413. + +| Operation | Marker present | Response for that object | +| --- | --- | --- | +| `download` | yes | `actions.download`, a presigned GET | +| `download` | no | `error`, code 404 | +| `upload` | yes, size matches | no `actions` key, so the client skips it | +| `upload` | yes, size differs | `error`, code 422 | +| `upload` | no | `actions.upload` and `actions.verify` | + +When the marker is absent but the global blob already exists, the upload action +still goes out. The client re-uploads bytes the bucket already holds. The +alternative, trusting a global blob the caller cannot prove it owns, is the read +oracle above. Correctness wins; the cost is one re-upload per new repository. + +Every object carries `"authenticated": true`. + +CAUTION: Without `authenticated`, git-lfs sends the transfer through +`DoWithAuthNoRetry`, which runs `git credential fill` against the Tigris host +and can attach an `Authorization` header. A SigV4 presigned request rejects a +request that also carries `Authorization`. The user then gets a credential +prompt for `t3.storage.dev`, or a 400 on every transfer. + +Every action carries `expires_in`, in seconds, so the client knows the href goes +stale. + +### Upload integrity + +A presigned PUT means the daemon never reads the body. Two checks are needed, +and only one of them works at PUT time. + +The presigned `PutObjectInput` sets `ChecksumSHA256` to the base64 encoding of +the oid's raw 32 bytes. The bucket then rejects a body whose sha256 does not +match the key. + +CAUTION: The stock presigner hoists that header into the query string, so it +never reaches `SignedHeader`. This is verified: +`aws-sdk-go-v2@v1.41.7/aws/signer/internal/v4/headers.go:67-70` hoists anything +matching `X-Amz-` that is not in `RequiredSignedHeaders`, and +`X-Amz-Checksum-Sha256` is not in that list. The fix is a custom +`HTTPPresignerV4` (`s3.PresignOptions.Presigner`) that wraps `v4.NewSigner()` +and sets `DisableHeaderHoisting = true`. The checksum then appears in +`SignedHeaders`, and `PresignedHTTPRequest.SignedHeader` carries it back. Put +that map, minus `Host`, into `actions.upload.header`; git-lfs sends every header +in it. + +CAUTION: `ContentLength` on a presigned PUT is a no-op. The SDK writes a +`Content-Length` header, but the signer skips it, and it is not hoisted. The +declared `size` is therefore **not** enforced at upload time. + +So `verify` is the real gate, not an optional extra. It is the only check that +runs on the daemon's own credentials. + +### The verify endpoint + +`POST /{orgID}/{repoName}/info/lfs/objects/verify` takes `{oid, size}`. It does +one `HeadObject` on the global blob with `ChecksumMode: ChecksumModeEnabled`, +then compares both the length and the returned sha256. + +- Both match: write the per-repository marker, return 200. +- Object missing: 404. +- Length or checksum wrong: delete the blob, return 422. + +The marker is written here, and only here. An upload that never verifies leaves +no membership behind. + +### Locks + +One JSON document per repository at `//lfs/locks.json`. Writes +use the compare-and-swap the ref cache already uses +(`internal/storage/tigris/refcache.go:200`): `If-None-Match: *` for the first +write, then `If-Match: `, re-read and retry on refusal, under a bounded +retry count. + +| Route | Operation | Behavior | +| --- | --- | --- | +| `POST .../info/lfs/locks` | Write | 201 with the lock, or 409 with the holder. | +| `GET .../info/lfs/locks` | Read | List, with `path`, `id`, `cursor`, `limit`. | +| `POST .../info/lfs/locks/verify` | Read | Split into `ours` and `theirs`. | +| `POST .../info/lfs/locks/{id}/unlock` | Write | Delete one. `force` takes another owner's lock. | + +Lock IDs are generated by the server as lowercase hex, and validated on read. A +client-supplied `{id}` is never used to build a key. + +The owner is the HTTP Basic username, or `anonymous` when there is none. The +`ours` and `theirs` split uses that name. + +CAUTION: git-lfs halts a push when a file matches a lock in `theirs`, and it +records `lfs..locksverify=true` after the first 200. With no user store, +every caller that sends no credential is the same owner `anonymous`. All locks +then land in `ours`, so a fully anonymous deployment never blocks a push. A +deployment where some users set a Basic username gets correct behavior. Say this +in `docs/usage/lfs.md`. `-allow-lfs-locks` turns the four routes off, and an +unregistered route is a 404, which the spec states does not halt a push. + +### git-lfs-authenticate over SSH + +`cmd/objgitd/ssh.go:123-136` rejects any command `gitServiceFor` does not know. +The interception goes before that call. The command arrives as three arguments: +`git-lfs-authenticate `. + +The handler parses the path with `repofs.Parse`, maps `download` to `auth.Read` +and `upload` to `auth.Write`, calls `d.authorize` with `Transport: "ssh"`, then +writes one JSON object to the session, not to stderr, and exits 0: + +```json +{"href":"https://git.example.com/acme/test.git/info/lfs","header":{},"expires_in":900} +``` + +CAUTION: When LFS is off, or `-external-url` is unset, the handler must write +`git-lfs-authenticate: command not found` to stderr and **exit 127**. git-lfs +treats any other non-zero exit as a real error and gives up. With exit 127 it +falls back to guessing the HTTP endpoint, which is the behavior every +non-LFS-aware git server produces today. Exit 1 stays for a bad path or a denied +request. + +The `header` map is empty, because the HTTP side accepts anonymous requests. + +### Flags + +| Flag | Default | Purpose | +| --- | --- | --- | +| `-allow-lfs` | `false` | Register the LFS routes. Opt-in, like `-allow-hooks`. | +| `-allow-lfs-locks` | `true` | Register the four lock routes. | +| `-external-url` | `""` | The public base URL, for `git-lfs-authenticate`. | +| `-lfs-url-ttl` | `15m` | How long a presigned URL stays valid. | +| `-lfs-max-size` | `5368709120` | The largest object. One PUT caps at 5 GiB. | +| `-lfs-max-batch` | `500` | Objects per batch, over which the answer is 413. | + +Startup checks, in the style of `main.go:76-84`: `-external-url` must be +absolute, carry a scheme, and end without a slash. It is required when +`-allow-lfs` and `-ssh-bind` are both set. `-allow-lfs` with an empty +`-http-bind` is fatal, because `git-lfs-authenticate` has nothing to point at. + +The presign client is built from `rawClient.Client` at `main.go:97`, the +embedded `*s3.Client` in `tstorage.Client`. It is **not** built from the +hardened client at `main.go:106`: `s3fs.Harden` returns an interface value, and +`s3.NewPresignClient` needs the concrete type. Hardening is irrelevant here, +because presigning performs no network I/O. + +CAUTION: `tstorage.New` falls back to the AWS credential chain when +`TIGRIS_STORAGE_ACCESS_KEY_ID` is unset, and `AGENTS.md` documents `AWS_PROFILE` +as the normal path. A chain that lands on SSO, IMDS, or assume-role yields a +credential that expires, and the presigned URL dies with it whatever +`-lfs-url-ttl` says. At startup, when `-allow-lfs` is set, retrieve once and +check `aws.Credentials.CanExpire`. If it can expire, clamp every URL's lifetime +to the remaining session time minus a margin, and log that clamp. Under a minute +left, answer the batch with 503 instead of handing out URLs that will fail. + +### HTTP status codes + +git-lfs reads these precisely. Both directions use +`Content-Type: application/vnd.git-lfs+json`. + +- 200 for every batch that parsed, even when individual objects carry an `error`. +- 401 with `LFS-Authenticate`, not only `WWW-Authenticate`. +- 403 when the authorizer denies a write. The git path returns 401 here + (`http.go:196`); LFS must not. +- 404 for a missing repository. 406 for a wrong `Accept`. 413 for an oversized + batch. 422 for a body that fails validation. +- Per-object errors are `objects[].error = {code, message}`. +- The top-level error body is `{message, request_id, documentation_url}`. + +### Route registration + +The patterns go in `httpHandler` inside an `if d.lfs != nil` guard. Existing +tests build bare `&daemon{...}` literals (`http_test.go:106`), so an +unconditional registration would nil-dereference. An unregistered route is a +ServeMux 404, which is exactly the "no LFS here" answer both git-lfs and the +locking spec expect. + +The nine patterns have fixed segment counts and no trailing wildcards, so no two +can match the same request and ServeMux cannot panic. Nothing shadows +`GET /{orgID}/{repoName}/info/refs`. + +### Metrics + +The package takes observer functions. `main` wires `metrics.*` into them, and +the package never imports Prometheus. New vectors under namespace `objgit`: + +- `lfs_batch_requests_total{operation,status}` +- `lfs_batch_duration_seconds{operation}` +- `lfs_objects_total{operation,result}`, where `result` is `new`, `present`, + `missing`, or `error`. The ratio of `present` to `new` is the dedup rate. +- `lfs_verify_total{status}` +- `lfs_lock_requests_total{action,status}` +- `lfs_presign_total{method}` + +### Testing + +CI runs `go vet ./...` and `go test ./...` with no bucket credentials +(`.github/workflows/go.yml`). Tests run against a fake bucket, in the style of +`fakeS3` in `internal/storage/tigris/client_test.go:43`. + +The presigner is tested for real, not faked. Build an `*s3.Client` with +`credentials.NewStaticCredentialsProvider`, a fixed `BaseEndpoint`, and a fixed +signing time, then assert the URL. This is the only test that catches a +regression in header hoisting, and a fake presigner would hide it forever. + +`internal/lfs`, table-driven with `tt`: + +- An oid of the wrong length, wrong case, or with non-hex bytes is rejected. +- An oid that encodes a path traversal is rejected. +- A negative size, and a size over the cap, are rejected. +- An unknown `operation` is rejected. +- Download of a present marker returns a download action. +- Download of a missing marker returns a per-object 404, and the request is + still 200. +- Download of an oid whose global blob exists but whose marker does not returns + 404. This is the read-oracle test. +- Upload of a present marker of the same size returns no actions. +- Upload of a present marker of a different size returns 422. +- Upload of a new oid returns an upload action and a verify action. +- The presigned upload URL's `X-Amz-SignedHeaders` contains + `x-amz-checksum-sha256`, and `SignedHeader` carries that header. +- Every object carries `authenticated`, and every action carries `expires_in`. +- A batch over `-lfs-max-batch` is 413. +- `verify` returns 200, 404, and 422, and writes the marker only on 200. +- `verify` deletes the blob on a checksum mismatch. +- A lock create on a free path returns the lock; on a held path, the holder. +- A refused compare-and-swap on the lock document retries, then succeeds. Use + the `putHook` pattern to land a competing write mid-loop. +- An unlock by another owner fails without `force`, and succeeds with it. + +`internal/repofs`: an organization named `lfs`, and any segment starting with +`.`, are rejected. + +`cmd/objgitd/lfs_test.go`, extending `newHTTPServer` with a fake store: + +- Every response carries `application/vnd.git-lfs+json`. +- A wrong `Accept` is 406. +- An upload batch with `-allow-push` unset is 403, not 401. +- Every LFS route is 404 when `d.lfs` is nil. +- A malformed body is 422, not 500. + +`cmd/objgitd/ssh_test.go`, gated on `exec.LookPath("ssh")` as the file already +does: + +- `git-lfs-authenticate acme/test.git download` prints JSON whose `href` + matches `-external-url`. +- With LFS off, the command exits 127 and prints `command not found`. + +One live test, gated on `OBJGIT_TIGRIS_LIVE_BUCKET` and +`exec.LookPath("git-lfs")`, in the style of +`internal/storage/tigris/livebucket_test.go:24`. + +CAUTION: Presign-only means `go test ./...` can never prove that LFS works. The +bytes never touch `objgitd`. Six facts are unknowable without a real bucket, and +each needs a named skipped test so the gap stays visible: whether Tigris +enforces `x-amz-checksum-sha256`; what it returns on a mismatch; whether +`HeadObject` with `ChecksumModeEnabled` returns the stored sha256; whether the +presigned host and region resolve and the signature is accepted; the +`If-Match` behavior under contention; and a real `git lfs push` and `pull`. +Treat the live test as the acceptance gate for this feature, not as follow-up +work. + +## Verification + +1. `go build ./...` and `go vet ./...` pass. +2. `go test ./...` passes with no bucket credentials. +3. Run the live round trip against a real bucket: + +```sh +./objgitd -bucket $BUCKET -http-bind :8080 -ssh-bind :2222 \ + -allow-push -allow-lfs -external-url http://localhost:8080 & + +git init lfsdemo && cd lfsdemo +git lfs install --local +git lfs track '*.bin' +head -c 50000000 /dev/urandom > big.bin +git add .gitattributes big.bin && git commit -m 'add big file' +git push http://localhost:8080/acme/lfs.git main + +tigris ls s3://$BUCKET/lfs/blobs/ # the bytes, once +tigris ls s3://$BUCKET/acme/lfs/lfs/objects/ # the marker + +cd .. && git clone http://localhost:8080/acme/lfs.git fresh +cmp lfsdemo/big.bin fresh/big.bin +``` + +4. Upload a corrupt body to a presigned URL with `curl`. The bucket rejects it, + or `verify` returns 422 and deletes the blob. Either outcome proves integrity; + both failing means the feature is not done. +5. Push the same file to a second repository. `objgit_lfs_objects_total` gains a + `present` count once the marker exists, and a later push of that repository + transfers nothing. +6. Ask for an oid from repository A through repository B's batch endpoint. The + answer is 404. +7. Repeat step 3 against `ssh://localhost:2222/acme/lfs.git`. Then run with + `-allow-lfs` unset and confirm that `git lfs pull` falls back instead of + failing. +8. Run `git lfs lock big.bin`, `git lfs locks`, and `git lfs unlock big.bin`. + +## Remaining gap + +- **Deletion.** Nothing deletes an LFS blob, the same gap packs have + (`docs/reference/tigris-backend.md:490-494`). The per-repository marker gives a + future collector the reference count it needs, but no collector exists. +- **Objects over 5 GiB.** One presigned PUT caps there. Multipart needs presigned + part URLs and a client that asks for them. +- **A token for `git-lfs-authenticate`.** The `header` map is empty. Real + authentication needs an issuer there. +- **The pure-SSH transfer protocol** (`git-lfs-transfer`). An `ssh://` remote + still needs the HTTP listener. +- **Authorization inside a presigned URL.** The URL works for its full lifetime, + whatever the Authorizer later decides. +- **One wasted upload per repository** for bytes the bucket already holds, which + is the price of closing the cross-repository read oracle. + +## What implementation changed + +The plan above is the design as approved. Three things came out differently, and +the code follows this section where the two disagree. + +**Downloads and uploads need different signers.** The plan turned off header +hoisting so an upload's checksum stays inside the signature. That part holds. +But the same setting also signs `x-amz-checksum-mode` on a GetObject, and a URL +signed that way is refused unless the client replays that header. A `git clone` +hid the fault, because git-lfs faithfully sends every header the batch response +names; a plain fetch of the same URL failed with `SignatureDoesNotMatch`. The +presigner now uses the no-hoist signer for uploads and the stock signer for +downloads. `TestPresignGetNeedsNoRequestHeaders` pins it. + +**The checksum question is settled.** Tigris does enforce +`x-amz-checksum-sha256` on a presigned PUT. Measured against a live bucket: +wrong bytes answer 400 `XAmzContentSHA256Mismatch`, matching bytes answer 200, +and omitting the header answers 403 `SignatureDoesNotMatch`. That last case is +the important one, because it means a client cannot opt out of the check. The +`-lfs-checksum` flag the plan proposed is therefore unnecessary, and does not +exist. + +**Blobs live at `lfs/blobs/`, not `lfs/objects/`.** The per-repository marker +took the `lfs/objects/` name, because that is the name a reader expects to mean +"objects this repository has". + +**Uploads go to a per-repository staging key, not straight to the shared blob.** +This is the largest change, and it came out of code review. The plan had the +client upload to `lfs/blobs/` and had verify grant membership when the blob +existed at the declared size. That check is not a check: an oid and its size sit +side by side in every pointer file, so anyone who could read a pointer could +claim the object in their own repository and then download it. The marker split +was supposed to prevent exactly that, and it did not. + +A client now uploads to `//lfs/staging/`. Verify reads that key, +not the shared one, and promotes it with `RenameObject`. Membership therefore +requires having written the bytes, which Tigris checked against the oid as it +accepted them. The same change fixes a second fault the plan had: verify used to +delete the *shared* blob on a size mismatch, so any writer could destroy an +object other repositories depend on. It now deletes only the caller's own staged +key. + +`TestVerifyNeedsProofOfPossession` and +`TestLFSVerifyCannotStealAnotherRepositorysObject` pin both. + +## Verification performed + +Run against a single-region Tigris bucket, with `git-lfs` 3.x: + +- A 5 MB file pushed and cloned back byte-for-byte identical. The git pack was + 716 bytes; the blob was the other 5 MB. +- The bucket held one blob under `lfs/blobs/` and one zero-byte marker under + `acme/lfsdemo/lfs/objects/`. +- The same file pushed to a second repository produced a second marker and no + second blob. +- An upload batch for an object the repository already held returned no actions, + so the client transferred nothing. +- A batch request from an unrelated repository for that object answered 404, + while the owning repository got a download action. +- `git lfs lock`, `git lfs locks`, and `git lfs unlock` all worked. +- The three upload-integrity cases above. +- After a push, the staging prefix was empty: `RenameObject` moved the object + rather than copying it. +- An unrelated repository calling verify with a known oid and size got 404, and + still could not download the object. + +`internal/lfs/livebucket_test.go` carries the parts of this that can be +automated, behind `OBJGIT_TIGRIS_LIVE_BUCKET`. diff --git a/docs/plans/pack-compaction-implementation.md b/docs/plans/pack-compaction-implementation.md new file mode 100644 index 0000000..04afac8 --- /dev/null +++ b/docs/plans/pack-compaction-implementation.md @@ -0,0 +1,3295 @@ +# Pack Compaction Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Merge small pack containers into larger ones and delete the sources after a grace period, so pack count and duplicate storage stop growing without bound. + +**Architecture:** A pass bin-packs *whole* source packs into an output container and copies every payload verbatim. Because the write-side containment rule already guarantees a delta's base sits in the same container as the delta, packing whole packs preserves containment for free — no decode, no delta resolution, no compressor. Deletion is deferred through a tombstone object with a deadline, so a request that built its pack index before the merge never loses the bytes under it. A background sweeper in the daemon runs one pass per repository. + +**Tech Stack:** Go 1.26, `github.com/aws/aws-sdk-go-v2/service/s3`, `github.com/go-git/go-git/v6`, `github.com/klauspost/compress/zstd`, `github.com/prometheus/client_golang`, `golang.org/x/sync/errgroup`. + +**Spec:** `docs/plans/pack-compaction.md` + +## Global Constraints + +- Module path `github.com/tigrisdata/objgit`. Go 1.26. +- Flags are **kebab-case**, paired with `flagenv` for the `UPPER_SNAKE` environment fallback. +- `slog` uses `"err"` as the error key, never `"error"`. +- Tests are **table-driven with `tt`**. Gate any test that shells out to git with `exec.LookPath("git")`. +- Reuse the shared test helpers `runGit`, `tryGit`, and `seedRepo` from `cmd/objgitd/git_protocol_test.go`. +- Every Prometheus vector lives in `internal/metrics`, registered with `promauto` under `namespace = "objgit"`. +- A pack is immutable. Never write to an existing `packs/.bin` or `packs/.cue`. +- Compaction never drops an object. Every hash in a source pack appears in the output. +- Architecture notes go in `docs/architecture/`, not in `AGENTS.md`. + +--- + +### Task 1: Teach the S3 fake about size, mtime, and delimiters + +The fake returns only `Key` from `ListObjectsV2`. Compaction needs the object size and the last-modified time to make grace-period and bin-packing decisions, and the sweeper needs `CommonPrefixes` to enumerate repositories. + +**Files:** +- Modify: `internal/storage/tigris/client_test.go` + +**Interfaces:** +- Consumes: nothing. +- Produces: `fakeObject.mtime time.Time`; `fakeS3.clock func() time.Time`; `(*fakeS3).setMtime(key string, t time.Time)`. `ListObjectsV2` now populates `types.Object.Size` and `types.Object.LastModified`, and honors `p.Delimiter` by returning `types.CommonPrefix` values. + +- [ ] **Step 1: Write the failing test** + +Add to `internal/storage/tigris/client_test.go`: + +```go +func TestFakeListReportsSizeMtimeAndPrefixes(t *testing.T) { + base := time.Date(2026, 8, 31, 12, 0, 0, 0, time.UTC) + f := newFakeS3(t) + f.clock = func() time.Time { return base } + f.put("org/repo/packs/a.bin", "hello", nil) + f.put("org/repo/packs/a.cue", "xy", nil) + f.put("org/other/HEAD", "z", nil) + + out, err := f.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{ + Bucket: sp("b"), Prefix: sp("org/repo/packs/"), + }) + if err != nil { + t.Fatalf("list: %v", err) + } + if len(out.Contents) != 2 { + t.Fatalf("want 2 objects, got %d", len(out.Contents)) + } + if got := *out.Contents[0].Size; got != 5 { + t.Errorf("size = %d, want 5", got) + } + if got := *out.Contents[0].LastModified; !got.Equal(base) { + t.Errorf("mtime = %v, want %v", got, base) + } + + out, err = f.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{ + Bucket: sp("b"), Prefix: sp("org/"), Delimiter: sp("/"), + }) + if err != nil { + t.Fatalf("list with delimiter: %v", err) + } + if len(out.Contents) != 0 { + t.Errorf("want no direct children, got %d", len(out.Contents)) + } + var got []string + for _, cp := range out.CommonPrefixes { + got = append(got, sv(cp.Prefix)) + } + want := []string{"org/other/", "org/repo/"} + if !slices.Equal(got, want) { + t.Errorf("common prefixes = %v, want %v", got, want) + } +} +``` + +Add `"slices"` to the test file's imports. + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `go test ./internal/storage/tigris/ -run TestFakeListReportsSizeMtimeAndPrefixes -v` +Expected: FAIL — `out.Contents[0].Size` is nil, so the test panics on the nil dereference. `f.clock` and `f.setMtime` do not compile. + +- [ ] **Step 3: Add the clock and mtime to the fake** + +In `internal/storage/tigris/client_test.go`, add `mtime` to `fakeObject`: + +```go +type fakeObject struct { + body []byte + meta map[string]string + etag string + mtime time.Time +} +``` + +Add the clock field to `fakeS3` next to the other knobs: + +```go + // clock stamps fakeObject.mtime. Tests that care about the grace period + // replace it so a deadline is deterministic instead of wall-clock bound. + clock func() time.Time +``` + +Add a helper beside the other fake helpers: + +```go +// now reads the fake's clock, defaulting to the wall clock. +func (f *fakeS3) now() time.Time { + if f.clock != nil { + return f.clock() + } + return time.Now() +} + +// setMtime backdates one key, so a test can age an object past a grace period +// without sleeping. +func (f *fakeS3) setMtime(key string, t time.Time) { + f.mu.Lock() + defer f.mu.Unlock() + o, ok := f.objs[key] + if !ok { + f.t.Fatalf("setMtime: %s is not in the fake bucket", key) + } + o.mtime = t + f.objs[key] = o +} +``` + +Two places construct a `fakeObject`, and both need the stamp. In `putLocked`, change the stored literal to: + +```go + f.objs[key] = fakeObject{body: []byte(body), meta: m, etag: f.nextETag(), mtime: f.now()} +``` + +Then find the `fakeObject` literal inside `(*fakeS3).PutObject` and add `mtime: f.now()` to it the same way. + +- [ ] **Step 4: Populate size, mtime, and common prefixes in the listing** + +Replace the body of `(*fakeS3).ListObjectsV2` after the `f.listErr` check: + +```go + prefix := sv(p.Prefix) + delim := sv(p.Delimiter) + + // With a delimiter, a key whose remainder past the prefix still contains + // the delimiter collapses into a common prefix instead of a result. + seenPrefix := map[string]struct{}{} + var matched []string + var prefixes []string + for k := range f.objs { + if !strings.HasPrefix(k, prefix) { + continue + } + if delim != "" { + rest := k[len(prefix):] + if i := strings.Index(rest, delim); i >= 0 { + cp := prefix + rest[:i+len(delim)] + if _, dup := seenPrefix[cp]; !dup { + seenPrefix[cp] = struct{}{} + prefixes = append(prefixes, cp) + } + continue + } + } + matched = append(matched, k) + } + sort.Strings(matched) + sort.Strings(prefixes) + + start := 0 + if tok := sv(p.ContinuationToken); tok != "" { + start, _ = strconv.Atoi(tok) + } + end := len(matched) + if f.listMax > 0 && start+int(f.listMax) < end { + end = start + int(f.listMax) + } + + out := &s3.ListObjectsV2Output{IsTruncated: bp(end < len(matched))} + for _, k := range matched[start:end] { + o := f.objs[k] + size := int64(len(o.body)) + mt := o.mtime + out.Contents = append(out.Contents, types.Object{ + Key: sp(k), + Size: &size, + LastModified: &mt, + }) + } + // Common prefixes ride along with the first page only, which is enough for + // every caller here and matches how S3 behaves for small listings. + if start == 0 { + for _, cp := range prefixes { + out.CommonPrefixes = append(out.CommonPrefixes, types.CommonPrefix{Prefix: sp(cp)}) + } + } + if bv(out.IsTruncated) { + out.NextContinuationToken = sp(strconv.Itoa(end)) + } + return out, nil +``` + +- [ ] **Step 5: Run the test to verify it passes** + +Run: `go test ./internal/storage/tigris/ -run TestFakeListReportsSizeMtimeAndPrefixes -v` +Expected: PASS + +- [ ] **Step 6: Run the whole package to catch regressions** + +Run: `go test ./internal/storage/tigris/` +Expected: PASS. The listing change is additive, so existing tests keep passing. + +- [ ] **Step 7: Commit** + +```bash +git add internal/storage/tigris/client_test.go +git commit -m "test(storage/tigris): fake reports object size, mtime, and common prefixes" +``` + +--- + +### Task 2: `listEntries` returns size and mtime alongside the key + +`listKeys` throws away the size and the last-modified time that `ListObjectsV2` already returned. Compaction needs both, and paying a second round trip for them would be waste. + +**Files:** +- Modify: `internal/storage/tigris/iter.go:17-47` +- Test: `internal/storage/tigris/iter_test.go` + +**Interfaces:** +- Consumes: Task 1's fake. +- Produces: `type objectEntry struct { key string; size int64; mtime time.Time }` and `func (s *Storer) listEntries(prefix string) ([]objectEntry, error)`. `listKeys` keeps its exact current signature and becomes a wrapper. + +- [ ] **Step 1: Write the failing test** + +Add to `internal/storage/tigris/iter_test.go`: + +```go +func TestListEntriesCarriesSizeAndMtime(t *testing.T) { + base := time.Date(2026, 8, 31, 12, 0, 0, 0, time.UTC) + f := newFakeS3(t) + f.clock = func() time.Time { return base } + f.put("packs/aa.bin", "0123456789", nil) + f.put("packs/aa.cue", "ab", nil) + s := newTestStorer(t, f) + + got, err := s.listEntries("packs/") + if err != nil { + t.Fatalf("listEntries: %v", err) + } + if len(got) != 2 { + t.Fatalf("want 2 entries, got %d", len(got)) + } + if got[0].key != "packs/aa.bin" || got[0].size != 10 { + t.Errorf("entry 0 = %+v, want packs/aa.bin size 10", got[0]) + } + if !got[0].mtime.Equal(base) { + t.Errorf("mtime = %v, want %v", got[0].mtime, base) + } +} +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `go test ./internal/storage/tigris/ -run TestListEntriesCarriesSizeAndMtime -v` +Expected: FAIL — `s.listEntries undefined`. + +- [ ] **Step 3: Implement `listEntries` and rewrite `listKeys` as a wrapper** + +In `internal/storage/tigris/iter.go`, replace `listKeys` with: + +```go +// objectEntry is one listed key plus the two facts ListObjectsV2 already +// returns with it. Compaction decides bin packing from size and grace-period +// eligibility from mtime, so throwing them away would cost a HEAD per key. +type objectEntry struct { + key string + size int64 + mtime time.Time +} + +// listEntries walks one prefix fully. S3 returns Contents lexicographically +// with monotone continuation tokens, so results come back sorted. +func (s *Storer) listEntries(prefix string) ([]objectEntry, error) { + var entries []objectEntry + token := "" + + for { + in := &s3.ListObjectsV2Input{ + Bucket: sp(s.bucket), + Prefix: sp(prefix), + } + if token != "" { + in.ContinuationToken = sp(token) + } + + start := time.Now() + page, err := s.client.ListObjectsV2(s.ctx, in) + s.observe("ListObjectsV2", start, err) + if err != nil { + return nil, fmt.Errorf("tigris: list %q: %w", prefix, err) + } + + for _, entry := range page.Contents { + k := sv(entry.Key) + if k == "" { + continue + } + e := objectEntry{key: k} + if entry.Size != nil { + e.size = *entry.Size + } + if entry.LastModified != nil { + e.mtime = *entry.LastModified + } + entries = append(entries, e) + } + if !bv(page.IsTruncated) || sv(page.NextContinuationToken) == "" { + break + } + token = sv(page.NextContinuationToken) + } + return entries, nil +} + +// listKeys is listEntries for the callers that want names only. +func (s *Storer) listKeys(prefix string) ([]string, error) { + entries, err := s.listEntries(prefix) + if err != nil { + return nil, err + } + keys := make([]string, 0, len(entries)) + for _, e := range entries { + keys = append(keys, e.key) + } + return keys, nil +} +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `go test ./internal/storage/tigris/ -run 'TestListEntries|TestIter' -v` +Expected: PASS + +- [ ] **Step 5: Run the whole package** + +Run: `go test ./internal/storage/tigris/` +Expected: PASS + +- [ ] **Step 6: Commit** + +```bash +git add internal/storage/tigris/iter.go internal/storage/tigris/iter_test.go +git commit -m "feat(storage/tigris): listEntries reports key size and mtime" +``` + +--- + +### Task 3: The tombstone binary format + +The tombstone object records which packs to delete and when. Its format copies `packed-refs` byte for byte, so a reader of one format can read the other by eye. + +**Files:** +- Create: `internal/storage/tigris/tombstones.go` +- Test: `internal/storage/tigris/tombstones_test.go` + +**Interfaces:** +- Consumes: `compressBlock` and the zstd decoder from `compress.go`, as `packedrefs.go` uses them. +- Produces: `const tombstonesKey = "pack-tombstones"`; `type tombstone struct { id string; deadline time.Time }`; `func encodeTombstones(ts []tombstone) []byte`; `func decodeTombstones(raw []byte) ([]tombstone, error)`; `var errBadTombstones error`. + +- [ ] **Step 1: Write the failing test** + +Create `internal/storage/tigris/tombstones_test.go`: + +```go +package tigris + +import ( + "errors" + "testing" + "time" +) + +func TestTombstoneRoundTrip(t *testing.T) { + for _, tt := range []struct { + name string + in []tombstone + }{ + {name: "empty", in: nil}, + { + name: "sorted on encode", + in: []tombstone{ + {id: "ff02", deadline: time.Unix(1756688400, 0).UTC()}, + {id: "0a1b", deadline: time.Unix(1756684800, 0).UTC()}, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + got, err := decodeTombstones(encodeTombstones(tt.in)) + if err != nil { + t.Fatalf("decode: %v", err) + } + if len(got) != len(tt.in) { + t.Fatalf("got %d entries, want %d", len(got), len(tt.in)) + } + for i := 1; i < len(got); i++ { + if got[i-1].id >= got[i].id { + t.Errorf("entries are not sorted by id: %v", got) + } + } + byID := map[string]time.Time{} + for _, e := range got { + byID[e.id] = e.deadline + } + for _, want := range tt.in { + if d, ok := byID[want.id]; !ok || !d.Equal(want.deadline) { + t.Errorf("id %s = %v (present %v), want %v", want.id, d, ok, want.deadline) + } + } + }) + } +} + +func TestTombstoneEncodingIsStable(t *testing.T) { + in := []tombstone{ + {id: "0a1b", deadline: time.Unix(1756684800, 0)}, + {id: "ff02", deadline: time.Unix(1756688400, 0)}, + } + a := encodeTombstones(in) + b := encodeTombstones([]tombstone{in[1], in[0]}) + if string(a) != string(b) { + t.Error("one set of tombstones must encode to identical bytes in any order") + } +} + +func TestTombstoneRejectsCorruption(t *testing.T) { + good := encodeTombstones([]tombstone{{id: "0a1b", deadline: time.Unix(1, 0)}}) + for _, tt := range []struct { + name string + in []byte + }{ + {name: "truncated header", in: good[:8]}, + {name: "bad magic", in: append([]byte{'X', 'X', 'X'}, good[3:]...)}, + {name: "unknown version", in: append(append([]byte{}, good[:3]...), append([]byte{9}, good[4:]...)...)}, + {name: "count disagrees with body", in: func() []byte { + b := append([]byte{}, good...) + b[tombCountOff+3] = 9 + return b + }()}, + } { + t.Run(tt.name, func(t *testing.T) { + if _, err := decodeTombstones(tt.in); !errors.Is(err, errBadTombstones) { + t.Errorf("err = %v, want errBadTombstones", err) + } + }) + } +} +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `go test ./internal/storage/tigris/ -run TestTombstone -v` +Expected: FAIL — `undefined: tombstone`, `undefined: encodeTombstones`. + +- [ ] **Step 3: Implement the format** + +Create `internal/storage/tigris/tombstones.go`: + +```go +package tigris + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "sort" + "strconv" + "strings" + "time" +) + +// tombstonesKey records packs whose contents were copied into a merged +// container, and the time after which deleting them is safe. It sits at the +// root beside packed-refs, shallow, index, and config. +// +// The key deliberately does not start with packs/, so a pack listing can never +// return it, for the same reason packed-refs does not start with refs/. +const tombstonesKey = "pack-tombstones" + +const ( + tombHeaderLen = 16 + tombVersion1 = 1 + + // tombCodecOff is header byte 4: the codec of the body block. + tombCodecOff = 4 + // tombCountOff is header byte 5: the entry count, big-endian uint32. It is + // a checksum on the body and nothing else, and it catches a body truncated + // on a line boundary, which no other check sees. + tombCountOff = 5 +) + +// tombMagic opens every version of the tombstone object. Header byte 3 carries +// the version itself. +var tombMagic = [3]byte{'O', 'G', 'T'} + +// errBadTombstones marks a malformed tombstone object. Corruption never +// masquerades as absence, the same posture errBadCue and errBadPackedRefs take. +// +// Reading an empty set out of a corrupt object is the dangerous failure: the +// pass would then believe no pack is pending deletion, write fresh tombstones +// for packs it just merged, and leak every source pack recorded before. +var errBadTombstones = errors.New("tigris: malformed pack-tombstones object") + +// tombstone is one pack awaiting deletion, and the time it becomes safe to +// delete. The deadline is absolute rather than a duration, so changing the +// grace flag never retroactively expires a record that was already written. +type tombstone struct { + id string + deadline time.Time +} + +// encodeTombstones serializes ts into the tombstone format: a 16-byte +// plaintext header, then a body that is raw or one zstd frame. +// +// The body is text, one entry per line, "\t\n", sorted +// by pack id. Sorting means one set of tombstones always encodes to identical +// bytes, so a test can assert bytes rather than compare sets, and a person who +// decompresses the body can read it. A pack id is hex, so it can hold no tab +// and the separator is never ambiguous. +func encodeTombstones(ts []tombstone) []byte { + sorted := make([]tombstone, len(ts)) + copy(sorted, ts) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].id < sorted[j].id }) + + var body bytes.Buffer + for _, e := range sorted { + body.WriteString(e.id) + body.WriteByte('\t') + body.WriteString(strconv.FormatInt(e.deadline.Unix(), 10)) + body.WriteByte('\n') + } + + stored, compressed := compressBlock(body.Bytes()) + + out := make([]byte, tombHeaderLen, tombHeaderLen+len(stored)) + copy(out, tombMagic[:]) + out[3] = tombVersion1 + if compressed { + out[tombCodecOff] = codecZstd + } + binary.BigEndian.PutUint32(out[tombCountOff:], uint32(len(sorted))) + return append(out, stored...) +} + +// decodeTombstones is encodeTombstones' inverse. +func decodeTombstones(raw []byte) ([]tombstone, error) { + if len(raw) < tombHeaderLen { + return nil, fmt.Errorf("%w: header truncated (%d bytes)", errBadTombstones, len(raw)) + } + if !bytes.Equal(raw[0:3], tombMagic[:]) { + return nil, fmt.Errorf("%w: bad magic", errBadTombstones) + } + if v := raw[3]; v != tombVersion1 { + return nil, fmt.Errorf("%w: unsupported format version %d", errBadTombstones, v) + } + + body := raw[tombHeaderLen:] + switch codec := raw[tombCodecOff]; codec { + case codecRaw: + case codecZstd: + out, err := cueDecoder().DecodeAll(body, nil) + if err != nil { + return nil, fmt.Errorf("%w: body does not decompress: %w", errBadTombstones, err) + } + body = out + default: + return nil, fmt.Errorf("%w: unknown body codec %d", errBadTombstones, codec) + } + + want := binary.BigEndian.Uint32(raw[tombCountOff:]) + + var out []tombstone + for line := range strings.SplitSeq(strings.TrimSuffix(string(body), "\n"), "\n") { + if line == "" { + continue + } + id, secs, ok := strings.Cut(line, "\t") + if !ok { + return nil, fmt.Errorf("%w: line %q has no separator", errBadTombstones, line) + } + unix, err := strconv.ParseInt(secs, 10, 64) + if err != nil { + return nil, fmt.Errorf("%w: line %q has an unreadable deadline: %w", errBadTombstones, line, err) + } + out = append(out, tombstone{id: id, deadline: time.Unix(unix, 0)}) + } + + if uint32(len(out)) != want { + return nil, fmt.Errorf("%w: header claims %d entries, body holds %d", errBadTombstones, want, len(out)) + } + return out, nil +} +``` + +NOTE: `cueDecoder`, `compressBlock`, `codecRaw`, and `codecZstd` all already exist in `compress.go`. If `strings.SplitSeq` is unavailable, use `for _, line := range strings.Split(...)`. + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `go test ./internal/storage/tigris/ -run TestTombstone -v` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add internal/storage/tigris/tombstones.go internal/storage/tigris/tombstones_test.go +git commit -m "feat(storage/tigris): add the pack-tombstones binary format" +``` + +--- + +### Task 4: Extract `casPut`, and read and write the tombstone object + +`commitRefs` inlines its conditional `PutObject`. The tombstone object needs exactly the same compare-and-swap, so factor the call out rather than writing it twice. + +**Files:** +- Modify: `internal/storage/tigris/refcache.go:196-215` +- Modify: `internal/storage/tigris/tombstones.go` +- Test: `internal/storage/tigris/tombstones_test.go` + +**Interfaces:** +- Consumes: `tombstonesKey`, `encodeTombstones`, `decodeTombstones` from Task 3; `fetchSmallETag` from `refs.go:343`; `isPreconditionFailed` from `tigris.go:358`. +- Produces: `var errCASConflict error`; `func (s *Storer) casPut(key string, body []byte, etag string) (string, error)`; `func (s *Storer) loadTombstones() ([]tombstone, string, error)`; `func (s *Storer) storeTombstones(ts []tombstone, etag string) (string, error)`. + +- [ ] **Step 1: Write the failing test** + +Add to `internal/storage/tigris/tombstones_test.go`: + +```go +func TestTombstoneStoreLoadRoundTrip(t *testing.T) { + f := newFakeS3(t) + s := newTestStorer(t, f) + + got, etag, err := s.loadTombstones() + if err != nil { + t.Fatalf("load on empty bucket: %v", err) + } + if len(got) != 0 || etag != "" { + t.Fatalf("empty bucket = %v/%q, want no entries and no etag", got, etag) + } + + in := []tombstone{{id: "0a1b", deadline: time.Unix(1756684800, 0)}} + etag, err = s.storeTombstones(in, "") + if err != nil { + t.Fatalf("store: %v", err) + } + if etag == "" { + t.Fatal("store returned no etag") + } + + got, got2, err := s.loadTombstones() + if err != nil { + t.Fatalf("load: %v", err) + } + if len(got) != 1 || got[0].id != "0a1b" { + t.Errorf("loaded %v, want one entry with id 0a1b", got) + } + if got2 != etag { + t.Errorf("etag = %q, want %q", got2, etag) + } +} + +func TestTombstoneStoreDetectsConflict(t *testing.T) { + f := newFakeS3(t) + s := newTestStorer(t, f) + + etag, err := s.storeTombstones([]tombstone{{id: "aa", deadline: time.Unix(1, 0)}}, "") + if err != nil { + t.Fatalf("first store: %v", err) + } + if _, err := s.storeTombstones([]tombstone{{id: "bb", deadline: time.Unix(2, 0)}}, ""); !errors.Is(err, errCASConflict) { + t.Errorf("store with no etag over an existing object: err = %v, want errCASConflict", err) + } + if _, err := s.storeTombstones([]tombstone{{id: "bb", deadline: time.Unix(2, 0)}}, "stale"); !errors.Is(err, errCASConflict) { + t.Errorf("store with a stale etag: err = %v, want errCASConflict", err) + } + if _, err := s.storeTombstones([]tombstone{{id: "bb", deadline: time.Unix(2, 0)}}, etag); err != nil { + t.Errorf("store with the current etag: %v", err) + } +} +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `go test ./internal/storage/tigris/ -run TestTombstoneStore -v` +Expected: FAIL — `undefined: errCASConflict`, `s.loadTombstones undefined`. + +- [ ] **Step 3: Add `casPut` and the tombstone accessors** + +Append to `internal/storage/tigris/tombstones.go`: + +```go +// errCASConflict marks a conditional PutObject that lost its race. The caller +// re-reads and retries, which is what makes it a distinct error rather than an +// opaque failure. +var errCASConflict = errors.New("tigris: conditional put lost its race") + +// casPut writes body to key under a compare-and-swap and reports the new ETag. +// +// An empty etag means "this object must not exist yet" and sends +// If-None-Match: *. Any other value sends If-Match. A refused precondition +// comes back as errCASConflict, never as an opaque error, so a retry loop can +// tell contention apart from a real failure. +func (s *Storer) casPut(key string, body []byte, etag string) (string, error) { + in := &s3.PutObjectInput{ + Bucket: sp(s.bucket), + Key: sp(key), + Body: bytes.NewReader(body), + } + if etag == "" { + in.IfNoneMatch = sp("*") + } else { + in.IfMatch = sp(etag) + } + + start := time.Now() + out, err := s.client.PutObject(s.ctx, in) + s.observe("PutObject", start, err) + switch { + case err == nil: + return sv(out.ETag), nil + case isPreconditionFailed(err): + return "", errCASConflict + default: + return "", fmt.Errorf("tigris: conditional put %s: %w", key, err) + } +} + +// loadTombstones reads the tombstone object and its compare-and-swap token. A +// missing object is an empty set with an empty token, which is what a +// repository that has never been compacted looks like. +func (s *Storer) loadTombstones() ([]tombstone, string, error) { + raw, etag, err := s.fetchSmallETag(s.prefix + tombstonesKey) + switch { + case err == nil: + case errors.Is(err, plumbing.ErrObjectNotFound): + return nil, "", nil + default: + return nil, "", fmt.Errorf("tigris: load tombstones: %w", err) + } + + ts, derr := decodeTombstones(raw) + if derr != nil { + return nil, "", derr + } + return ts, etag, nil +} + +// storeTombstones writes the set under a compare-and-swap against etag. +func (s *Storer) storeTombstones(ts []tombstone, etag string) (string, error) { + return s.casPut(s.prefix+tombstonesKey, encodeTombstones(ts), etag) +} +``` + +Add `"github.com/aws/aws-sdk-go-v2/service/s3"` and `"github.com/go-git/go-git/v6/plumbing"` to the file's imports. + +- [ ] **Step 4: Rewrite `commitRefs` to use `casPut`** + +In `internal/storage/tigris/refcache.go`, replace the block that builds `in`, calls `PutObject`, and switches on the error with: + +```go + newETag, err := s.casPut(s.prefix+packedRefsKey, encodePackedRefs(next), etag) + switch { + case err == nil: + case errors.Is(err, errCASConflict): + continue // somebody else landed first; re-read and re-apply + default: + return fmt.Errorf("tigris: commit refs: %w", err) + } +``` + +Then replace `s.refs.etag = sv(out.ETag)` with `s.refs.etag = newETag`. Remove any import that this leaves unused, and add `"errors"` if it is not already imported. + +- [ ] **Step 5: Run the tests to verify they pass** + +Run: `go test ./internal/storage/tigris/ -run 'TestTombstone|TestRef|TestCheckAndSet|TestPackedRefs' -v` +Expected: PASS. The existing ref compare-and-swap tests are the regression net for the refactor. + +- [ ] **Step 6: Run the whole package** + +Run: `go test ./internal/storage/tigris/` +Expected: PASS + +- [ ] **Step 7: Commit** + +```bash +git add internal/storage/tigris/tombstones.go internal/storage/tigris/tombstones_test.go internal/storage/tigris/refcache.go +git commit -m "refactor(storage/tigris): extract casPut, add tombstone load and store" +``` + +--- + +### Task 5: `surveyPacks` reads every pack's records, size, and mtime + +The pass and the cold index build both list `packs/` and parse every `.cue`. Share one function so the two can never disagree about what a pack is. + +**Files:** +- Modify: `internal/storage/tigris/packindex.go:553-595` +- Test: `internal/storage/tigris/compact_test.go` (create) + +**Interfaces:** +- Consumes: `listEntries` from Task 2; `parseCue` from `packindex.go:207`. +- Produces: `type packSurvey struct { id string; recs []cueRecord; size int64; mtime time.Time }` and `func (s *Storer) surveyPacks() (packs []packSurvey, orphanBins []objectEntry, err error)`. `ensurePacksBuilt` keeps its exact current signature and behavior. + +- [ ] **Step 1: Write the failing test** + +Create `internal/storage/tigris/compact_test.go`: + +```go +package tigris + +import ( + "testing" + "time" +) + +func TestSurveyPacksReportsPacksAndOrphans(t *testing.T) { + base := time.Date(2026, 8, 31, 12, 0, 0, 0, time.UTC) + f := newFakeS3(t) + f.clock = func() time.Time { return base } + s := newTestStorer(t, f) + + // One complete container, and one .bin whose .cue upload never landed. + writeFakePack(t, f, s, "aa", []testObj{{typ: plumbing.BlobObject, body: []byte("one")}}) + f.put("packs/orphan.bin", "dangling", nil) + + packs, orphans, err := s.surveyPacks() + if err != nil { + t.Fatalf("surveyPacks: %v", err) + } + if len(packs) != 1 || packs[0].id != "aa" { + t.Fatalf("packs = %+v, want one pack with id aa", packs) + } + if len(packs[0].recs) != 1 { + t.Errorf("pack aa holds %d records, want 1", len(packs[0].recs)) + } + if !packs[0].mtime.Equal(base) { + t.Errorf("mtime = %v, want %v", packs[0].mtime, base) + } + if len(orphans) != 1 || orphans[0].key != "packs/orphan.bin" { + t.Errorf("orphans = %+v, want packs/orphan.bin", orphans) + } +} +``` + +Add a shared test helper at the bottom of `compact_test.go`. It writes a real container through the package's own encoder, so tests never hand-roll `.cue` bytes: + +```go +type testObj struct { + typ plumbing.ObjectType + body []byte +} + +// hashOf computes an object id the same way the storer does. ObjectHasher.Compute +// returns an error, and threading that through every fixture would drown them. +func hashOf(t *testing.T, s *Storer, typ plumbing.ObjectType, body []byte) plumbing.Hash { + t.Helper() + h, err := s.oh.Compute(typ, body) + if err != nil { + t.Fatalf("hash %s object: %v", typ, err) + } + return h +} + +// writeFakePack stages a container with the given objects directly into the +// fake bucket under the given id, using the package's own cue encoder so the +// bytes are exactly what a real push would leave. +func writeFakePack(t *testing.T, f *fakeS3, s *Storer, id string, objs []testObj) { + t.Helper() + var bin []byte + recs := make([]cueRecord, 0, len(objs)) + for _, o := range objs { + h := hashOf(t, s, o.typ, o.body) + recs = append(recs, cueRecord{ + hash: h, + typ: o.typ, + codec: codecRaw, + offset: int64(len(bin)), + stored: int64(len(o.body)), + raw: int64(len(o.body)), + }) + bin = append(bin, o.body...) + } + sort.Slice(recs, func(i, j int) bool { + return bytes.Compare(recs[i].hash.Bytes(), recs[j].hash.Bytes()) < 0 + }) + f.put(s.prefix+packPrefix+id+binSuffix, string(bin), nil) + f.put(s.prefix+packPrefix+id+cueSuffix, string(encodeCue(s.oh.Size(), recs)), nil) +} +``` + +NOTE: sorting `recs` after computing offsets is safe, because each record carries its own offset. Add `"bytes"`, `"sort"`, and `"github.com/go-git/go-git/v6/plumbing"` to the imports. + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `go test ./internal/storage/tigris/ -run TestSurveyPacks -v` +Expected: FAIL — `s.surveyPacks undefined`. + +- [ ] **Step 3: Implement `surveyPacks` and rewire `ensurePacksBuilt`** + +In `internal/storage/tigris/packindex.go`, add above `ensurePacksBuilt`: + +```go +// packSurvey is one container as compaction sees it: its records, the size of +// its .bin, and when that .bin last changed. The pass decides bin packing from +// size and grace-period eligibility from mtime. +type packSurvey struct { + id string + recs []cueRecord + size int64 + mtime time.Time +} + +// surveyPacks lists packs/ once and parses every .cue under it. +// +// It also reports every .bin with no sibling .cue. A failed .cue upload leaves +// exactly that shape (packJob.run uploads the .bin first), and such a container +// is invisible to every reader, because the index is built from .cue files +// alone. Only compaction ever looks for them. +func (s *Storer) surveyPacks() ([]packSurvey, []objectEntry, error) { + entries, err := s.listEntries(s.prefix + packPrefix) + if err != nil { + return nil, nil, fmt.Errorf("tigris: list packs: %w", err) + } + + bins := make(map[string]objectEntry, len(entries)/2) + cues := make(map[string]objectEntry, len(entries)/2) + for _, e := range entries { + name := strings.TrimPrefix(e.key, s.prefix+packPrefix) + switch { + case strings.HasSuffix(name, binSuffix): + bins[strings.TrimSuffix(name, binSuffix)] = e + case strings.HasSuffix(name, cueSuffix): + cues[strings.TrimSuffix(name, cueSuffix)] = e + } + } + + hashLen := s.oh.Size() + packs := make([]packSurvey, 0, len(cues)) + for id, cue := range cues { + raw, err := s.fetchSmall(cue.key) + if err != nil { + return nil, nil, fmt.Errorf("tigris: fetch cue %s: %w", cue.key, err) + } + recs, err := parseCue(hashLen, raw) + if err != nil { + return nil, nil, fmt.Errorf("tigris: parse cue %s: %w", cue.key, err) + } + p := packSurvey{id: id, recs: recs} + if b, ok := bins[id]; ok { + p.size = b.size + p.mtime = b.mtime + } + packs = append(packs, p) + } + sort.Slice(packs, func(i, j int) bool { return packs[i].id < packs[j].id }) + + var orphans []objectEntry + for id, b := range bins { + if _, ok := cues[id]; !ok { + orphans = append(orphans, b) + } + } + sort.Slice(orphans, func(i, j int) bool { return orphans[i].key < orphans[j].key }) + + return packs, orphans, nil +} +``` + +Then replace the body of `ensurePacksBuilt` between the `built` check and the final lock with: + +```go + packs, _, err := s.surveyPacks() + if err != nil { + return err + } + + s.packs.mu.Lock() + defer s.packs.mu.Unlock() + if s.packs.built { + return nil // lost a race with a concurrent cold build on this instance + } + for _, p := range packs { + s.packs.sizes[p.id] = s.packs.indexRecords(p.id, p.recs) + } + s.packs.built = true + return nil +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `go test ./internal/storage/tigris/ -run 'TestSurveyPacks|TestPack' -v` +Expected: PASS + +- [ ] **Step 5: Run the whole package** + +Run: `go test ./internal/storage/tigris/` +Expected: PASS. The existing pack-index tests are the regression net for the `ensurePacksBuilt` rewrite. + +- [ ] **Step 6: Commit** + +```bash +git add internal/storage/tigris/packindex.go internal/storage/tigris/compact_test.go +git commit -m "refactor(storage/tigris): share one pack survey between the index build and compaction" +``` + +--- + +### Task 6: Selection and the treadmill guard + +Which packs a pass merges is pure arithmetic over the survey. Keep it a pure function so every edge case is a table row and no test needs a bucket. + +**Files:** +- Create: `internal/storage/tigris/compact.go` +- Test: `internal/storage/tigris/compact_test.go` + +**Interfaces:** +- Consumes: `packSurvey` from Task 5. +- Produces: `func selectForMerge(packs []packSurvey, maxPacks int, capBytes int64) [][]packSurvey`. + +- [ ] **Step 1: Write the failing test** + +Add to `internal/storage/tigris/compact_test.go`: + +```go +func TestSelectForMerge(t *testing.T) { + p := func(id string, size int64) packSurvey { return packSurvey{id: id, size: size} } + const cap = 1000 + + for _, tt := range []struct { + name string + packs []packSurvey + maxPacks int + want [][]string // ids per output group + }{ + { + name: "under the threshold does nothing", + packs: []packSurvey{p("a", 10), p("b", 10)}, + maxPacks: 2, + want: nil, + }, + { + name: "small packs group into one output", + packs: []packSurvey{p("a", 10), p("b", 20), p("c", 30)}, + maxPacks: 2, + want: [][]string{{"a", "b", "c"}}, + }, + { + name: "a group seals at the byte cap", + packs: []packSurvey{p("a", 400), p("b", 400), p("c", 400)}, + maxPacks: 2, + want: [][]string{{"a", "b"}, {"c"}}, + }, + { + name: "packs over half the cap are never sources", + packs: []packSurvey{p("a", 900), p("b", 900), p("c", 900)}, + maxPacks: 1, + want: nil, + }, + { + name: "no reduction means no pass", + packs: []packSurvey{p("a", 500), p("b", 500), p("c", 10)}, + maxPacks: 1, + want: nil, + }, + { + name: "ties break by id so the choice is deterministic", + packs: []packSurvey{p("z", 10), p("a", 10), p("m", 10)}, + maxPacks: 2, + want: [][]string{{"a", "m", "z"}}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + got := selectForMerge(tt.packs, tt.maxPacks, cap) + var ids [][]string + for _, g := range got { + var row []string + for _, s := range g { + row = append(row, s.id) + } + ids = append(ids, row) + } + if fmt.Sprint(ids) != fmt.Sprint(tt.want) { + t.Errorf("groups = %v, want %v", ids, tt.want) + } + }) + } +} +``` + +Add `"fmt"` to the test imports. + +NOTE on the "no reduction" row: only `c` is at or under half the cap, so there is one candidate and one output group. One output for one source reduces nothing, so the guard returns nil. + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `go test ./internal/storage/tigris/ -run TestSelectForMerge -v` +Expected: FAIL — `undefined: selectForMerge`. + +- [ ] **Step 3: Implement selection** + +Create `internal/storage/tigris/compact.go`: + +```go +package tigris + +import ( + "sort" +) + +// selectForMerge groups source packs into the output containers a pass will +// build. It returns nil when no pass is worth running. +// +// Two rules keep a pass from becoming a treadmill. A pack larger than half the +// byte cap is never a source, because merging containers that are already large +// rewrites the repository to gain almost nothing. And a selection that would +// produce at least as many outputs as it consumes is abandoned outright: it +// would rewrite the same bytes on every cycle, forever, and bill egress for no +// result. +// +// Candidates sort by size and break ties by id, so two daemons that survey the +// same repository choose the same grouping. That determinism is what makes a +// duplicated pass idempotent rather than merely harmless: identical grouping +// yields byte-identical outputs, which yields one content-addressed pack id. +func selectForMerge(packs []packSurvey, maxPacks int, capBytes int64) [][]packSurvey { + if len(packs) <= maxPacks { + return nil + } + + cand := make([]packSurvey, 0, len(packs)) + for _, p := range packs { + if p.size <= capBytes/2 { + cand = append(cand, p) + } + } + sort.Slice(cand, func(i, j int) bool { + if cand[i].size != cand[j].size { + return cand[i].size < cand[j].size + } + return cand[i].id < cand[j].id + }) + + var groups [][]packSurvey + var cur []packSurvey + var curBytes int64 + for _, p := range cand { + // Seal before the add, so a group never passes the cap. Deduplication + // only removes bytes, so the sum of the source sizes is a safe upper + // bound on what the output actually holds. + if len(cur) > 0 && curBytes+p.size > capBytes { + groups = append(groups, cur) + cur, curBytes = nil, 0 + } + cur = append(cur, p) + curBytes += p.size + } + if len(cur) > 0 { + groups = append(groups, cur) + } + + if len(groups) >= len(cand) { + return nil + } + return groups +} +``` + +- [ ] **Step 4: Run the test to verify it passes** + +Run: `go test ./internal/storage/tigris/ -run TestSelectForMerge -v` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add internal/storage/tigris/compact.go internal/storage/tigris/compact_test.go +git commit -m "feat(storage/tigris): select small packs for compaction with a treadmill guard" +``` + +--- + +### Task 7: `appendVerbatim` on a segment, and a `packWriter`-free seal + +The merge writes payloads that are already encoded. `packSegment.add` re-encodes through a compressor, which is exactly the cost the design avoids. And `seal` hangs off `packWriter`, which the merge does not have. + +**Files:** +- Modify: `internal/storage/tigris/packwriter.go:412-445` (add a method beside `add`) +- Modify: `internal/storage/tigris/packwriter.go:644-697` (`seal`) +- Test: `internal/storage/tigris/pack_test.go` + +**Interfaces:** +- Consumes: `packSegment`, `cueRecord`. +- Produces: `func (g *packSegment) appendVerbatim(rec cueRecord, r io.Reader) error`; `func sealSegment(s *Storer, seg *packSegment) (string, error)` returning the new pack id. `(*packWriter).seal` becomes a wrapper that discards the id. + +- [ ] **Step 1: Write the failing test** + +Add to `internal/storage/tigris/pack_test.go`: + +```go +func TestAppendVerbatimPreservesTheRecord(t *testing.T) { + f := newFakeS3(t) + s := newTestStorer(t, f) + + seg, err := newPackSegment(s) + if err != nil { + t.Fatalf("newPackSegment: %v", err) + } + defer seg.discard() + + body := []byte("already encoded bytes") + src := cueRecord{ + hash: hashOf(t, s, plumbing.BlobObject, body), + typ: plumbing.BlobObject, + codec: codecZstd, + offset: 4096, // the source offset, which must be rewritten + stored: int64(len(body)), + raw: 999, + } + + if err := seg.appendVerbatim(src, bytes.NewReader(body)); err != nil { + t.Fatalf("appendVerbatim: %v", err) + } + if len(seg.recs) != 1 { + t.Fatalf("segment holds %d records, want 1", len(seg.recs)) + } + got := seg.recs[0] + if got.offset != 0 { + t.Errorf("offset = %d, want 0 (rewritten to the segment position)", got.offset) + } + for _, c := range []struct { + name string + got, want any + }{ + {"hash", got.hash, src.hash}, + {"typ", got.typ, src.typ}, + {"codec", got.codec, src.codec}, + {"stored", got.stored, src.stored}, + {"raw", got.raw, src.raw}, + } { + if c.got != c.want { + t.Errorf("%s = %v, want %v", c.name, c.got, c.want) + } + } +} + +func TestAppendVerbatimRejectsAShortRead(t *testing.T) { + f := newFakeS3(t) + s := newTestStorer(t, f) + seg, err := newPackSegment(s) + if err != nil { + t.Fatalf("newPackSegment: %v", err) + } + defer seg.discard() + + src := cueRecord{stored: 100} + if err := seg.appendVerbatim(src, bytes.NewReader([]byte("short"))); err == nil { + t.Error("copying fewer bytes than the record claims must fail the merge") + } +} +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `go test ./internal/storage/tigris/ -run TestAppendVerbatim -v` +Expected: FAIL — `seg.appendVerbatim undefined`. + +- [ ] **Step 3: Implement `appendVerbatim`** + +In `internal/storage/tigris/packwriter.go`, add directly after `(*packSegment).add`: + +```go +// appendVerbatim copies one already-encoded payload into the segment without +// decoding it, and carries its record over with a rewritten offset. +// +// This is compaction's entire write path. A merge moves payloads between +// containers, so decoding and re-encoding them would burn CPU to produce the +// bytes it already has. Every field but offset survives: codec, so a zstd frame +// stays a frame; base, so a delta stays a delta; raw and typ, so the object +// still reads as itself. +// +// The caller guarantees containment, which is what makes carrying base over +// safe. See compact.go: a merge packs whole source packs into an output, so the +// base of every delta comes with it. +func (g *packSegment) appendVerbatim(rec cueRecord, r io.Reader) error { + n, err := io.Copy(g.file, r) + if err != nil { + return fmt.Errorf("copy %s into the merged container: %w", rec.hash, err) + } + if n != rec.stored { + return fmt.Errorf("%s: copied %d bytes, its record claims %d", rec.hash, n, rec.stored) + } + + rec.offset = g.offset + g.recs = append(g.recs, rec) + g.offset += n + return nil +} +``` + +- [ ] **Step 4: Free `seal` from `packWriter`** + +In the same file, rename the method and add the wrapper. Change the signature line of `func (w *packWriter) seal(seg *packSegment) error` to: + +```go +// sealSegment finishes one segment: it names the pack after its own checksum, +// stages the sibling .cue, and queues both for upload. It takes ownership of +// the segment's staging files, so every error path here removes them itself. +// +// It is a function rather than a packWriter method because compaction seals +// segments too, and it has no incoming packfile to write. +func sealSegment(s *Storer, seg *packSegment) (string, error) { +``` + +Inside the body, replace every `w.s` with `s`, and change each `return fmt.Errorf(...)` to `return "", fmt.Errorf(...)`. The final `return nil` becomes `return id, nil`. Then add the wrapper beside it: + +```go +func (w *packWriter) seal(seg *packSegment) error { + _, err := sealSegment(w.s, seg) + return err +} +``` + +- [ ] **Step 5: Run the tests to verify they pass** + +Run: `go test ./internal/storage/tigris/ -run 'TestAppendVerbatim|TestPack' -v` +Expected: PASS + +- [ ] **Step 6: Run the whole package** + +Run: `go test ./internal/storage/tigris/` +Expected: PASS + +- [ ] **Step 7: Commit** + +```bash +git add internal/storage/tigris/packwriter.go internal/storage/tigris/pack_test.go +git commit -m "feat(storage/tigris): add verbatim segment append and free seal from packWriter" +``` + +--- + +### Task 8: Merge one group of packs into one container + +This is the heart of the pass: choose the surviving record for each hash, then copy payloads verbatim into a new container. + +**Files:** +- Modify: `internal/storage/tigris/compact.go` +- Test: `internal/storage/tigris/compact_test.go` + +**Interfaces:** +- Consumes: `selectForMerge` (Task 6), `appendVerbatim` and `sealSegment` (Task 7), `streamPack` from `packindex.go:735`, `newPackSegment` from `packwriter.go:402`. +- Produces: `func chooseRecords(group []packSurvey) map[string][]cueRecord` and `func (s *Storer) mergeGroup(group []packSurvey) (string, int, error)` returning the new pack id and the number of duplicate records dropped. + +- [ ] **Step 1: Write the failing test** + +Add to `internal/storage/tigris/compact_test.go`: + +```go +func TestChooseRecordsDedupesAndPrefersWhole(t *testing.T) { + h1 := plumbing.NewHash("1111111111111111111111111111111111111111") + h2 := plumbing.NewHash("2222222222222222222222222222222222222222") + base := plumbing.NewHash("3333333333333333333333333333333333333333") + + group := []packSurvey{ + {id: "a", recs: []cueRecord{ + {hash: h1, stored: 5, base: base}, // delta form + {hash: h2, stored: 7}, + }}, + {id: "b", recs: []cueRecord{ + {hash: h1, stored: 50}, // whole form of the same object + }}, + } + + got := chooseRecords(group) + + var total int + for _, recs := range got { + total += len(recs) + } + if total != 2 { + t.Fatalf("kept %d records, want 2 (h1 once, h2 once)", total) + } + for _, r := range got["b"] { + if r.hash == h1 && r.base != plumbing.ZeroHash { + t.Error("kept the delta copy of h1, want the whole copy") + } + } + if len(got["b"]) != 1 { + t.Errorf("pack b contributes %d records, want 1 (the whole h1)", len(got["b"])) + } +} + +func TestMergeGroupProducesAReadableContainer(t *testing.T) { + f := newFakeS3(t) + s := newTestStorer(t, f) + + one := []byte("first object") + two := []byte("second object") + writeFakePack(t, f, s, "aa", []testObj{{typ: plumbing.BlobObject, body: one}}) + writeFakePack(t, f, s, "bb", []testObj{ + {typ: plumbing.BlobObject, body: two}, + {typ: plumbing.BlobObject, body: one}, // duplicated across the two packs + }) + + packs, _, err := s.surveyPacks() + if err != nil { + t.Fatalf("surveyPacks: %v", err) + } + + id, deduped, err := s.mergeGroup(packs) + if err != nil { + t.Fatalf("mergeGroup: %v", err) + } + if id == "" { + t.Fatal("mergeGroup returned no pack id") + } + if deduped != 1 { + t.Errorf("deduped %d records, want 1", deduped) + } + if err := s.up.flush(); err != nil { + t.Fatalf("flush: %v", err) + } + + // A fresh Storer sees only what is in the bucket, so this proves the merged + // container stands on its own. + fresh := newTestStorer(t, f) + for _, body := range [][]byte{one, two} { + h := hashOf(t, s, plumbing.BlobObject, body) + obj, err := fresh.EncodedObject(plumbing.AnyObject, h) + if err != nil { + t.Fatalf("read %s back: %v", h, err) + } + got, err := io.ReadAll(mustReader(obj)) + if err != nil { + t.Fatalf("read body: %v", err) + } + if !bytes.Equal(got, body) { + t.Errorf("object %s = %q, want %q", h, got, body) + } + } +} +``` + +Add `"io"` to the test imports. + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `go test ./internal/storage/tigris/ -run 'TestChooseRecords|TestMergeGroup' -v` +Expected: FAIL — `undefined: chooseRecords`, `s.mergeGroup undefined`. + +- [ ] **Step 3: Implement the merge** + +Append to `internal/storage/tigris/compact.go`: + +```go +// chooseRecords picks the surviving record for every hash in the group, keyed +// by the source pack that supplies its bytes. +// +// One hash can appear in several source packs, and only one copy goes into the +// output. Which copy is a real choice: when one is a delta and the other is +// whole, keep the whole one. Both are correct, but a kept delta can point at a +// base that is itself a delta from another source pack, so a chain grows one +// link on every compaction round. Keeping the whole form stops that growth, and +// it only ever applies to an object two source packs both hold. +// +// Deduplication never removes a hash from the output, only extra copies of one. +// That is what preserves delta containment: every base that was in a source +// pack is still somewhere in the output, so the delta that names it still +// resolves. +func chooseRecords(group []packSurvey) map[string][]cueRecord { + type pick struct { + packID string + rec cueRecord + } + best := make(map[plumbing.Hash]pick) + + for _, p := range group { + for _, r := range p.recs { + cur, seen := best[r.hash] + if !seen { + best[r.hash] = pick{packID: p.id, rec: r} + continue + } + // Replace a delta with a whole copy, and nothing else. + if cur.rec.base != plumbing.ZeroHash && r.base == plumbing.ZeroHash { + best[r.hash] = pick{packID: p.id, rec: r} + } + } + } + + out := make(map[string][]cueRecord, len(group)) + for _, p := range best { + out[p.packID] = append(out[p.packID], p.rec) + } + // Offset order turns each source read into one forward scan of its .bin. + for id := range out { + recs := out[id] + sort.Slice(recs, func(i, j int) bool { return recs[i].offset < recs[j].offset }) + } + return out +} + +// mergeGroup builds one output container from every pack in group and queues +// it for upload. It reports the new pack id and how many duplicate records it +// dropped. +// +// Each source .bin is downloaded whole, once. Sources are the smallest +// containers in the repository by construction (see selectForMerge), and one +// GET per source beats one ranged GET per object by orders of magnitude. +func (s *Storer) mergeGroup(group []packSurvey) (string, int, error) { + chosen := chooseRecords(group) + + var total int + for _, p := range group { + total += len(p.recs) + } + var kept int + for _, recs := range chosen { + kept += len(recs) + } + + seg, err := newPackSegment(s) + if err != nil { + return "", 0, fmt.Errorf("tigris: compact: %w", err) + } + sealed := false + defer func() { + if !sealed { + seg.discard() + } + }() + + for _, p := range group { + recs := chosen[p.id] + if len(recs) == 0 { + continue // every object this pack held is supplied by a sibling + } + + src, err := s.downloadPackToTemp(p.id) + if err != nil { + return "", 0, err + } + for _, r := range recs { + if err := seg.appendVerbatim(r, io.NewSectionReader(src, r.offset, r.stored)); err != nil { + src.Close() + return "", 0, fmt.Errorf("tigris: compact pack %s: %w", p.id, err) + } + } + src.Close() + } + + id, err := sealSegment(s, seg) + if err != nil { + return "", 0, err + } + sealed = true // sealSegment owns the staging files now + return id, total - kept, nil +} + +// downloadPackToTemp fetches one whole .bin into an unlinked temp file. The +// file dies with its last descriptor, so the caller only has to Close it. +// +// This deliberately bypasses the PackCache and the prefetch machinery in +// packindex.go. Those exist to make repeated reads cheap, and a source pack is +// about to be deleted, so caching it would evict something a client still +// wants. +func (s *Storer) downloadPackToTemp(id string) (*os.File, error) { + f, err := os.CreateTemp("", "objgit-tigris-compact-*") + if err != nil { + return nil, fmt.Errorf("tigris: create compaction temp file: %w", err) + } + os.Remove(f.Name()) // unlink now; the fd keeps the data alive + + if err := s.streamPack(id)(f); err != nil { + f.Close() + return nil, err + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + f.Close() + return nil, fmt.Errorf("tigris: rewind compaction temp file for %s: %w", id, err) + } + return f, nil +} +``` + +Extend the import block of `compact.go` to: + +```go +import ( + "fmt" + "io" + "os" + "sort" + + "github.com/go-git/go-git/v6/plumbing" +) +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `go test ./internal/storage/tigris/ -run 'TestChooseRecords|TestMergeGroup' -v` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add internal/storage/tigris/compact.go internal/storage/tigris/compact_test.go +git commit -m "feat(storage/tigris): merge whole packs into one container with verbatim copies" +``` + +--- + +### Task 9: The reap step deletes expired tombstones + +**Files:** +- Modify: `internal/storage/tigris/compact.go` +- Test: `internal/storage/tigris/compact_test.go` + +**Interfaces:** +- Consumes: `loadTombstones`, `storeTombstones`, `errCASConflict` (Task 4); `maxDeleteBatch` and the `DeleteObjects` idiom from `refcache.go:271`. +- Produces: `func (s *Storer) reapTombstones(now time.Time) (deleted int, err error)`. + +- [ ] **Step 1: Write the failing test** + +Add to `internal/storage/tigris/compact_test.go`: + +```go +func TestReapTombstones(t *testing.T) { + now := time.Date(2026, 8, 31, 12, 0, 0, 0, time.UTC) + f := newFakeS3(t) + s := newTestStorer(t, f) + + writeFakePack(t, f, s, "expired", []testObj{{typ: plumbing.BlobObject, body: []byte("x")}}) + writeFakePack(t, f, s, "live", []testObj{{typ: plumbing.BlobObject, body: []byte("y")}}) + + if _, err := s.storeTombstones([]tombstone{ + {id: "expired", deadline: now.Add(-time.Minute)}, + {id: "live", deadline: now.Add(time.Hour)}, + {id: "gone", deadline: now.Add(-time.Minute)}, // already deleted + }, ""); err != nil { + t.Fatalf("seed tombstones: %v", err) + } + + deleted, err := s.reapTombstones(now) + if err != nil { + t.Fatalf("reapTombstones: %v", err) + } + if deleted != 2 { + t.Errorf("deleted %d, want 2", deleted) + } + for _, k := range []string{"packs/expired.bin", "packs/expired.cue"} { + if f.has(k) { + t.Errorf("%s survived the reap", k) + } + } + for _, k := range []string{"packs/live.bin", "packs/live.cue"} { + if !f.has(k) { + t.Errorf("%s was deleted before its deadline", k) + } + } + + left, _, err := s.loadTombstones() + if err != nil { + t.Fatalf("load after reap: %v", err) + } + if len(left) != 1 || left[0].id != "live" { + t.Errorf("tombstones after reap = %v, want only live", left) + } +} + +func TestReapIsIdempotent(t *testing.T) { + now := time.Date(2026, 8, 31, 12, 0, 0, 0, time.UTC) + f := newFakeS3(t) + s := newTestStorer(t, f) + + if _, err := s.storeTombstones([]tombstone{{id: "gone", deadline: now.Add(-time.Hour)}}, ""); err != nil { + t.Fatalf("seed: %v", err) + } + if _, err := s.reapTombstones(now); err != nil { + t.Fatalf("first reap: %v", err) + } + if _, err := s.reapTombstones(now); err != nil { + t.Fatalf("second reap over an empty set: %v", err) + } +} +``` + +Add a `has` helper to the fake in `client_test.go` if one is not already present: + +```go +// has reports whether the fake bucket holds a key. +func (f *fakeS3) has(key string) bool { + f.mu.Lock() + defer f.mu.Unlock() + _, ok := f.objs[key] + return ok +} +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `go test ./internal/storage/tigris/ -run TestReap -v` +Expected: FAIL — `s.reapTombstones undefined`. + +- [ ] **Step 3: Implement the reap** + +Append to `internal/storage/tigris/compact.go`: + +```go +// reapTombstones deletes every pack whose deadline has passed, then rewrites +// the tombstone object without those entries. It reports how many packs it +// deleted. +// +// The order is delete first, tombstone second. If the process stops between +// the two, the tombstone names a pack that is already gone, and a DeleteObjects +// call against a missing key succeeds, so the next reap corrects the record. +// The opposite order loses the record of a pack that still exists, and nothing +// would ever delete it. +func (s *Storer) reapTombstones(now time.Time) (int, error) { + for attempt := 0; attempt < maxRefCASRetries; attempt++ { + ts, etag, err := s.loadTombstones() + if err != nil { + return 0, err + } + if len(ts) == 0 { + return 0, nil + } + + var expired, live []tombstone + for _, e := range ts { + if now.Before(e.deadline) { + live = append(live, e) + continue + } + expired = append(expired, e) + } + if len(expired) == 0 { + return 0, nil + } + + keys := make([]string, 0, len(expired)*2) + for _, e := range expired { + keys = append(keys, + s.prefix+packPrefix+e.id+binSuffix, + s.prefix+packPrefix+e.id+cueSuffix, + ) + } + if err := s.deleteKeys(keys); err != nil { + return 0, err + } + + switch _, err := s.storeTombstones(live, etag); { + case err == nil: + return len(expired), nil + case errors.Is(err, errCASConflict): + continue // another daemon rewrote the set; re-read and re-apply + default: + return 0, err + } + } + return 0, fmt.Errorf("tigris: pack-tombstones contention after %d attempts", maxRefCASRetries) +} + +// deleteKeys removes keys in batches of maxDeleteBatch. Deleting a key that is +// not there is not worth avoiding: S3 treats it as success, and checking first +// would cost a round trip to save one. +func (s *Storer) deleteKeys(keys []string) error { + for start := 0; start < len(keys); start += maxDeleteBatch { + batch := keys[start:min(start+maxDeleteBatch, len(keys))] + ids := make([]types.ObjectIdentifier, 0, len(batch)) + for _, k := range batch { + ids = append(ids, types.ObjectIdentifier{Key: sp(k)}) + } + + st := time.Now() + _, err := s.client.DeleteObjects(s.ctx, &s3.DeleteObjectsInput{ + Bucket: sp(s.bucket), + Delete: &types.Delete{Objects: ids, Quiet: bp(true)}, + }) + s.observe("DeleteObjects", st, err) + if err != nil { + return fmt.Errorf("tigris: delete %d pack keys: %w", len(batch), err) + } + } + return nil +} +``` + +Extend `compact.go`'s imports with `"errors"`, `"time"`, `"github.com/aws/aws-sdk-go-v2/service/s3"`, and `"github.com/aws/aws-sdk-go-v2/service/s3/types"`. + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `go test ./internal/storage/tigris/ -run TestReap -v` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add internal/storage/tigris/compact.go internal/storage/tigris/compact_test.go internal/storage/tigris/client_test.go +git commit -m "feat(storage/tigris): reap expired pack tombstones" +``` + +--- + +### Task 10: The backstop — subsumption and orphan cleanup + +Tombstones are state that can disagree with the bucket. Two cases leak packs, and both close with data the survey already holds. + +**Files:** +- Modify: `internal/storage/tigris/compact.go` +- Test: `internal/storage/tigris/compact_test.go` + +**Interfaces:** +- Consumes: `surveyPacks` (Task 5), `loadTombstones`/`storeTombstones` (Task 4), `deleteKeys` (Task 9). +- Produces: `func subsumedPacks(packs []packSurvey) []string`; `func (s *Storer) recordMissingTombstones(packs []packSurvey, grace time.Duration) (int, error)`; `func (s *Storer) deleteOrphanBins(orphans []objectEntry, now time.Time, grace time.Duration) (int, error)`. + +- [ ] **Step 1: Write the failing test** + +Add to `internal/storage/tigris/compact_test.go`: + +```go +func TestSubsumedPacks(t *testing.T) { + h1 := plumbing.NewHash("1111111111111111111111111111111111111111") + h2 := plumbing.NewHash("2222222222222222222222222222222222222222") + h3 := plumbing.NewHash("3333333333333333333333333333333333333333") + + for _, tt := range []struct { + name string + packs []packSurvey + want []string + }{ + { + name: "a merged pack subsumes both sources", + packs: []packSurvey{ + {id: "src1", recs: []cueRecord{{hash: h1}}}, + {id: "src2", recs: []cueRecord{{hash: h2}}}, + {id: "merged", recs: []cueRecord{{hash: h1}, {hash: h2}}}, + }, + want: []string{"src1", "src2"}, + }, + { + name: "a pack holding a unique object is never subsumed", + packs: []packSurvey{ + {id: "a", recs: []cueRecord{{hash: h1}, {hash: h3}}}, + {id: "b", recs: []cueRecord{{hash: h1}}}, + }, + want: []string{"b"}, + }, + { + name: "one pack alone is never subsumed", + packs: []packSurvey{{id: "only", recs: []cueRecord{{hash: h1}}}}, + want: nil, + }, + { + name: "an empty pack is not reported", + packs: []packSurvey{ + {id: "empty", recs: nil}, + {id: "a", recs: []cueRecord{{hash: h1}}}, + }, + want: nil, + }, + } { + t.Run(tt.name, func(t *testing.T) { + got := subsumedPacks(tt.packs) + if fmt.Sprint(got) != fmt.Sprint(tt.want) { + t.Errorf("subsumed = %v, want %v", got, tt.want) + } + }) + } +} + +func TestRecordMissingTombstonesUsesTheNewestPackClock(t *testing.T) { + old := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + recent := time.Date(2026, 8, 31, 12, 0, 0, 0, time.UTC) + h1 := plumbing.NewHash("1111111111111111111111111111111111111111") + + f := newFakeS3(t) + s := newTestStorer(t, f) + + packs := []packSurvey{ + {id: "ancient", recs: []cueRecord{{hash: h1}}, mtime: old}, + {id: "merged", recs: []cueRecord{{hash: h1}}, mtime: recent}, + } + + n, err := s.recordMissingTombstones(packs, time.Hour) + if err != nil { + t.Fatalf("recordMissingTombstones: %v", err) + } + if n != 1 { + t.Fatalf("wrote %d tombstones, want 1", n) + } + + ts, _, err := s.loadTombstones() + if err != nil { + t.Fatalf("load: %v", err) + } + if len(ts) != 1 || ts[0].id != "ancient" { + t.Fatalf("tombstones = %v, want one for ancient", ts) + } + // The deadline must come from the newest pack, not from the ancient one. + // Using the victim's own mtime would give it no grace at all. + if want := recent.Add(time.Hour); !ts[0].deadline.Equal(want) { + t.Errorf("deadline = %v, want %v", ts[0].deadline, want) + } +} + +func TestDeleteOrphanBinsRespectsGrace(t *testing.T) { + now := time.Date(2026, 8, 31, 12, 0, 0, 0, time.UTC) + f := newFakeS3(t) + s := newTestStorer(t, f) + f.put("packs/old.bin", "stale", nil) + f.put("packs/new.bin", "inflight", nil) + + orphans := []objectEntry{ + {key: "packs/old.bin", mtime: now.Add(-2 * time.Hour)}, + {key: "packs/new.bin", mtime: now.Add(-time.Minute)}, + } + + n, err := s.deleteOrphanBins(orphans, now, time.Hour) + if err != nil { + t.Fatalf("deleteOrphanBins: %v", err) + } + if n != 1 { + t.Errorf("deleted %d, want 1", n) + } + if f.has("packs/old.bin") { + t.Error("the aged orphan survived") + } + if !f.has("packs/new.bin") { + t.Error("an upload still in flight was deleted") + } +} +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `go test ./internal/storage/tigris/ -run 'TestSubsumed|TestRecordMissing|TestDeleteOrphan' -v` +Expected: FAIL — `undefined: subsumedPacks`. + +- [ ] **Step 3: Implement the backstop** + +Append to `internal/storage/tigris/compact.go`: + +```go +// subsumedPacks reports every pack whose every object also lives in some other +// pack. Such a pack can be deleted without losing anything. +// +// This is the backstop that makes the tombstone object an optimization rather +// than a correctness dependency. A crash between uploading a merged container +// and writing its tombstone leaves the sources subsumed and unrecorded, and +// without this nothing would ever delete them. +// +// A pack with no records is never reported. An empty pack is vacuously +// subsumed, and deleting one gains nothing while giving the rule a special case +// that is easy to get wrong. +func subsumedPacks(packs []packSurvey) []string { + // One entry per distinct object, capped at two: all this needs to know is + // whether a hash lives in more than one pack. + count := make(map[plumbing.Hash]uint8) + for _, p := range packs { + seen := make(map[plumbing.Hash]struct{}, len(p.recs)) + for _, r := range p.recs { + // Count each pack once per hash, so a pack holding a duplicate + // internally cannot make itself look covered. + if _, dup := seen[r.hash]; dup { + continue + } + seen[r.hash] = struct{}{} + if count[r.hash] < 2 { + count[r.hash]++ + } + } + } + + var out []string + for _, p := range packs { + if len(p.recs) == 0 { + continue + } + covered := true + for _, r := range p.recs { + if count[r.hash] < 2 { + covered = false + break + } + } + if covered { + out = append(out, p.id) + } + } + sort.Strings(out) + return out +} + +// recordMissingTombstones writes a tombstone for every subsumed pack that has +// none. It reports how many it added. +// +// CAUTION: The deadline is measured from the newest pack in the repository, +// never from the subsumed pack's own mtime. A source pack can be far older than +// the merge that subsumed it, so its own clock would give it no grace at all, +// and a reader holding an index built moments ago would lose the bytes under +// it. The newest pack's mtime is always at or after the merge that made this +// pack redundant, so it is a safe lower bound on when the replacement appeared. +func (s *Storer) recordMissingTombstones(packs []packSurvey, grace time.Duration) (int, error) { + subsumed := subsumedPacks(packs) + if len(subsumed) == 0 { + return 0, nil + } + + var newest time.Time + for _, p := range packs { + if p.mtime.After(newest) { + newest = p.mtime + } + } + deadline := newest.Add(grace) + + for attempt := 0; attempt < maxRefCASRetries; attempt++ { + ts, etag, err := s.loadTombstones() + if err != nil { + return 0, err + } + + have := make(map[string]struct{}, len(ts)) + for _, e := range ts { + have[e.id] = struct{}{} + } + + added := 0 + for _, id := range subsumed { + if _, ok := have[id]; ok { + continue + } + ts = append(ts, tombstone{id: id, deadline: deadline}) + added++ + } + if added == 0 { + return 0, nil + } + + switch _, err := s.storeTombstones(ts, etag); { + case err == nil: + return added, nil + case errors.Is(err, errCASConflict): + continue + default: + return 0, err + } + } + return 0, fmt.Errorf("tigris: pack-tombstones contention after %d attempts", maxRefCASRetries) +} + +// deleteOrphanBins removes a .bin that has no .cue, once it is older than the +// grace period. It reports how many it deleted. +// +// No tombstone is needed, because no reader can hold a reference to one: the +// pack index is built from .cue files alone, so a container with no .cue is +// invisible. The age test is the only thing that divides a true orphan from an +// upload that is still in flight, since packJob.run uploads the .bin first. +func (s *Storer) deleteOrphanBins(orphans []objectEntry, now time.Time, grace time.Duration) (int, error) { + var keys []string + for _, o := range orphans { + if now.Sub(o.mtime) > grace { + keys = append(keys, o.key) + } + } + if len(keys) == 0 { + return 0, nil + } + if err := s.deleteKeys(keys); err != nil { + return 0, err + } + return len(keys), nil +} +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `go test ./internal/storage/tigris/ -run 'TestSubsumed|TestRecordMissing|TestDeleteOrphan' -v` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add internal/storage/tigris/compact.go internal/storage/tigris/compact_test.go +git commit -m "feat(storage/tigris): back tombstones with subsumption and orphan cleanup" +``` + +--- + +### Task 11: `Compact` orchestration and its metrics + +**Files:** +- Modify: `internal/storage/tigris/compact.go` +- Modify: `internal/metrics/metrics.go` +- Test: `internal/storage/tigris/compact_test.go` + +**Interfaces:** +- Consumes: every function from Tasks 5 through 10. +- Produces: `type CompactOptions struct { MaxPacks int; Grace time.Duration; Now func() time.Time }`; `type CompactStats struct { PacksMerged, PacksWritten, PacksDeleted, ObjectsDeduped, OrphansDeleted int }`; `func (s *Storer) Compact(opt CompactOptions) (CompactStats, error)`; `func metrics.ObserveCompact(result string, dur time.Duration, st CompactCounts)`. + +- [ ] **Step 1: Write the failing test** + +Add to `internal/storage/tigris/compact_test.go`: + +```go +func TestCompactEndToEndOverTheFake(t *testing.T) { + now := time.Date(2026, 8, 31, 12, 0, 0, 0, time.UTC) + f := newFakeS3(t) + f.clock = func() time.Time { return now } + s := newTestStorer(t, f) + + bodies := [][]byte{} + for i := range 5 { + b := []byte(fmt.Sprintf("object number %d", i)) + bodies = append(bodies, b) + writeFakePack(t, f, s, fmt.Sprintf("p%d", i), []testObj{{typ: plumbing.BlobObject, body: b}}) + } + + opt := CompactOptions{MaxPacks: 2, Grace: time.Hour, Now: func() time.Time { return now }} + st, err := s.Compact(opt) + if err != nil { + t.Fatalf("Compact: %v", err) + } + if st.PacksMerged != 5 || st.PacksWritten != 1 { + t.Errorf("stats = %+v, want 5 merged into 1", st) + } + if err := s.up.flush(); err != nil { + t.Fatalf("flush: %v", err) + } + + // The sources are tombstoned, not yet deleted. + for i := range 5 { + if !f.has(fmt.Sprintf("packs/p%d.bin", i)) { + t.Errorf("source p%d was deleted before its grace period", i) + } + } + ts, _, err := s.loadTombstones() + if err != nil { + t.Fatalf("load tombstones: %v", err) + } + if len(ts) != 5 { + t.Errorf("wrote %d tombstones, want 5", len(ts)) + } + + // Every object still reads, from a Storer with no memory of the merge. + fresh := newTestStorer(t, f) + for _, b := range bodies { + h := hashOf(t, s, plumbing.BlobObject, b) + if _, err := fresh.EncodedObject(plumbing.AnyObject, h); err != nil { + t.Errorf("object %s is unreadable after compaction: %v", h, err) + } + } + + // After the grace period, a second pass deletes the sources. + later := now.Add(2 * time.Hour) + opt.Now = func() time.Time { return later } + if _, err := s.Compact(opt); err != nil { + t.Fatalf("second Compact: %v", err) + } + for i := range 5 { + if f.has(fmt.Sprintf("packs/p%d.bin", i)) { + t.Errorf("source p%d survived past its deadline", i) + } + } + after := newTestStorer(t, f) + for _, b := range bodies { + h := hashOf(t, s, plumbing.BlobObject, b) + if _, err := after.EncodedObject(plumbing.AnyObject, h); err != nil { + t.Errorf("object %s is unreadable after the reap: %v", h, err) + } + } +} + +func TestCompactUnderThresholdStillCleansOrphans(t *testing.T) { + now := time.Date(2026, 8, 31, 12, 0, 0, 0, time.UTC) + f := newFakeS3(t) + f.clock = func() time.Time { return now } + s := newTestStorer(t, f) + + writeFakePack(t, f, s, "only", []testObj{{typ: plumbing.BlobObject, body: []byte("x")}}) + f.put("packs/orphan.bin", "dangling", nil) + f.setMtime("packs/orphan.bin", now.Add(-2*time.Hour)) + + st, err := s.Compact(CompactOptions{MaxPacks: 16, Grace: time.Hour, Now: func() time.Time { return now }}) + if err != nil { + t.Fatalf("Compact: %v", err) + } + if st.PacksMerged != 0 { + t.Errorf("merged %d packs under the threshold, want 0", st.PacksMerged) + } + if st.OrphansDeleted != 1 { + t.Errorf("deleted %d orphans, want 1", st.OrphansDeleted) + } + if f.has("packs/orphan.bin") { + t.Error("the orphan survived a pass that did no merging") + } +} +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `go test ./internal/storage/tigris/ -run TestCompact -v` +Expected: FAIL — `undefined: CompactOptions`. + +- [ ] **Step 3: Implement `Compact`** + +Append to `internal/storage/tigris/compact.go`: + +```go +// CompactOptions configures one compaction pass. +type CompactOptions struct { + // MaxPacks is the pack count above which a pass merges. At or below it, a + // pass still reaps, cleans orphans, and runs the backstop. + MaxPacks int + // Grace is how long a merged-away pack survives before deletion. It must + // exceed the longest request: a Storer builds its pack index once per + // request, so a reader's view is never more stale than the request holding + // it, and a large clone runs for many minutes. + Grace time.Duration + // Now is the clock, overridable so a test can cross a deadline without + // sleeping. A nil value means time.Now. + Now func() time.Time +} + +func (o CompactOptions) now() time.Time { + if o.Now != nil { + return o.Now() + } + return time.Now() +} + +// CompactStats reports what one pass did. +type CompactStats struct { + PacksMerged int // source packs read + PacksWritten int // output containers produced + PacksDeleted int // packs whose grace period expired this pass + ObjectsDeduped int // duplicate records dropped during the merge + OrphansDeleted int // .bin files with no .cue +} + +// Compact runs one compaction pass over this Storer's repository. +// +// The order matters. The reap runs first, so a crash mid-pass still frees +// space on the next cycle. The survey then feeds three steps that run at every +// pack count — orphan cleanup, the subsumption backstop, and selection — +// because a failed upload leaves an orphan whatever the pack count is, and +// gating them behind the merge threshold would let a quiet repository leak +// forever. +// +// Compaction never writes a reference, so it cannot interact with the +// packed-refs compare-and-swap, and it needs no lock against a concurrent push: +// a pack that lands after the survey is simply not in this pass's input, and a +// pack that was surveyed cannot change, because packs are immutable. +func (s *Storer) Compact(opt CompactOptions) (CompactStats, error) { + var st CompactStats + now := opt.now() + + deleted, err := s.reapTombstones(now) + if err != nil { + return st, err + } + st.PacksDeleted = deleted + + packs, orphans, err := s.surveyPacks() + if err != nil { + return st, err + } + + dropped, err := s.deleteOrphanBins(orphans, now, opt.Grace) + if err != nil { + return st, err + } + st.OrphansDeleted = dropped + + if _, err := s.recordMissingTombstones(packs, opt.Grace); err != nil { + return st, err + } + + groups := selectForMerge(packs, opt.MaxPacks, s.mergeCapBytes()) + if len(groups) == 0 { + return st, nil + } + + var sources []tombstone + deadline := now.Add(opt.Grace) + for _, g := range groups { + _, deduped, err := s.mergeGroup(g) + if err != nil { + return st, err + } + st.PacksMerged += len(g) + st.PacksWritten++ + st.ObjectsDeduped += deduped + for _, p := range g { + sources = append(sources, tombstone{id: p.id, deadline: deadline}) + } + } + + // The merged containers must be durable before anything records their + // sources as deletable. flush surfaces the first upload failure, and a + // failure here leaves the sources untombstoned, which is the safe side. + if err := s.up.flush(); err != nil { + return st, fmt.Errorf("tigris: compact: %w", err) + } + + if err := s.addTombstones(sources); err != nil { + return st, err + } + return st, nil +} + +// mergeCapBytes is the byte cap one merged container may reach. +func (s *Storer) mergeCapBytes() int64 { + if s.maxPackBytes > 0 { + return s.maxPackBytes + } + return maxPackBytes +} + +// addTombstones records sources as pending deletion, under a compare-and-swap. +func (s *Storer) addTombstones(add []tombstone) error { + if len(add) == 0 { + return nil + } + for attempt := 0; attempt < maxRefCASRetries; attempt++ { + ts, etag, err := s.loadTombstones() + if err != nil { + return err + } + have := make(map[string]struct{}, len(ts)) + for _, e := range ts { + have[e.id] = struct{}{} + } + for _, e := range add { + if _, ok := have[e.id]; !ok { + ts = append(ts, e) + } + } + + switch _, err := s.storeTombstones(ts, etag); { + case err == nil: + return nil + case errors.Is(err, errCASConflict): + continue + default: + return err + } + } + return fmt.Errorf("tigris: pack-tombstones contention after %d attempts", maxRefCASRetries) +} +``` + +- [ ] **Step 4: Add the metrics** + +In `internal/metrics/metrics.go`, add to the `var` block: + +```go + compactRuns = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "compact", + Name: "runs_total", + Help: "Total pack compaction passes by outcome.", + }, []string{"result"}) + + compactDuration = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: "compact", + Name: "duration_seconds", + Help: "Wall-clock duration of one pack compaction pass.", + Buckets: prometheus.ExponentialBuckets(0.1, 3, 8), + }) + + compactPacks = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "compact", + Name: "packs_total", + Help: "Pack containers touched by compaction, by disposition.", + }, []string{"disposition"}) + + compactObjectsDeduped = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "compact", + Name: "objects_deduped_total", + Help: "Duplicate object records dropped while merging containers.", + }) +``` + +And add the helper beside the other exported helpers: + +```go +// CompactCounts mirrors tigris.CompactStats without importing it, so the +// storage package keeps its one-way dependency on metrics. +type CompactCounts struct { + Merged, Written, Deleted, Deduped, Orphans int +} + +// ObserveCompact records one compaction pass. result is "ok" or "error". +func ObserveCompact(result string, dur time.Duration, c CompactCounts) { + compactRuns.WithLabelValues(result).Inc() + compactDuration.Observe(dur.Seconds()) + compactObjectsDeduped.Add(float64(c.Deduped)) + for disposition, n := range map[string]int{ + "merged": c.Merged, + "written": c.Written, + "deleted": c.Deleted, + "orphan": c.Orphans, + } { + compactPacks.WithLabelValues(disposition).Add(float64(n)) + } +} +``` + +NOTE: `Compact` itself does not call this. The sweeper does, in Task 13, which keeps `internal/storage/tigris` free of a metrics import exactly as it is today. + +- [ ] **Step 5: Run the tests to verify they pass** + +Run: `go test ./internal/storage/tigris/ -run TestCompact -v && go build ./...` +Expected: PASS, and a clean build. + +- [ ] **Step 6: Run the whole suite** + +Run: `go test ./...` +Expected: PASS + +- [ ] **Step 7: Commit** + +```bash +git add internal/storage/tigris/compact.go internal/storage/tigris/compact_test.go internal/metrics/metrics.go +git commit -m "feat(storage/tigris): add the Compact pass and its metrics" +``` + +--- + +### Task 12: Repository enumeration and the compaction lease + +The sweeper needs the list of repositories in the bucket, and a way to avoid doing the same work on every daemon at once. + +**Files:** +- Modify: `internal/storage/tigris/compact.go` +- Test: `internal/storage/tigris/compact_test.go` + +**Interfaces:** +- Consumes: Task 1's delimiter support in the fake. +- Produces: `func (s *Storer) ListRepoPrefixes() ([]string, error)`; `func (s *Storer) TakeCompactLease(owner string, ttl time.Duration, now time.Time) (bool, error)`; `func (s *Storer) ReleaseCompactLease() error`. + +- [ ] **Step 1: Write the failing test** + +Add to `internal/storage/tigris/compact_test.go`: + +```go +func TestListRepoPrefixes(t *testing.T) { + f := newFakeS3(t) + s := newTestStorer(t, f) + f.put("acme/web.git/HEAD", "ref: refs/heads/main\n", nil) + f.put("acme/api.git/packed-refs", "x", nil) + f.put("other/tools.git/HEAD", "y", nil) + + got, err := s.ListRepoPrefixes() + if err != nil { + t.Fatalf("ListRepoPrefixes: %v", err) + } + want := []string{"acme/api.git", "acme/web.git", "other/tools.git"} + if fmt.Sprint(got) != fmt.Sprint(want) { + t.Errorf("prefixes = %v, want %v", got, want) + } +} + +func TestCompactLease(t *testing.T) { + now := time.Date(2026, 8, 31, 12, 0, 0, 0, time.UTC) + f := newFakeS3(t) + a := newTestStorer(t, f) + b := newTestStorer(t, f) + + ok, err := a.TakeCompactLease("daemon-a", time.Hour, now) + if err != nil || !ok { + t.Fatalf("first take = %v/%v, want true/nil", ok, err) + } + ok, err = b.TakeCompactLease("daemon-b", time.Hour, now) + if err != nil { + t.Fatalf("contended take: %v", err) + } + if ok { + t.Error("two daemons took the same lease at once") + } + + // An expired lease is available again, even though nobody released it. + ok, err = b.TakeCompactLease("daemon-b", time.Hour, now.Add(2*time.Hour)) + if err != nil { + t.Fatalf("take after expiry: %v", err) + } + if !ok { + t.Error("an expired lease must be available") + } + + if err := b.ReleaseCompactLease(); err != nil { + t.Fatalf("release: %v", err) + } + ok, err = a.TakeCompactLease("daemon-a", time.Hour, now) + if err != nil || !ok { + t.Fatalf("take after release = %v/%v, want true/nil", ok, err) + } +} +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `go test ./internal/storage/tigris/ -run 'TestListRepoPrefixes|TestCompactLease' -v` +Expected: FAIL — `s.ListRepoPrefixes undefined`. + +- [ ] **Step 3: Implement enumeration and the lease** + +Append to `internal/storage/tigris/compact.go`: + +```go +// compactLeaseKey stops two daemons from compacting one repository at the same +// time. It sits at the root of the repository prefix. +const compactLeaseKey = "compact-lease" + +// ListRepoPrefixes reports every repository path in the bucket, relative to +// this Storer's prefix and with no trailing slash. +// +// Two levels of delimiter listing, because RepoRef.Path is path.Join(OrgID, +// Name) and therefore always exactly two segments. One call returns the +// organization prefixes, and one call per organization returns its +// repositories. Listing every key instead would walk every object in the +// bucket to learn a handful of names. +func (s *Storer) ListRepoPrefixes() ([]string, error) { + orgs, err := s.listCommonPrefixes(s.prefix) + if err != nil { + return nil, err + } + + var repos []string + for _, org := range orgs { + found, err := s.listCommonPrefixes(org) + if err != nil { + return nil, err + } + for _, r := range found { + repos = append(repos, strings.TrimSuffix(strings.TrimPrefix(r, s.prefix), "/")) + } + } + sort.Strings(repos) + return repos, nil +} + +// listCommonPrefixes returns the immediate child "directories" of one prefix. +func (s *Storer) listCommonPrefixes(prefix string) ([]string, error) { + var out []string + token := "" + for { + in := &s3.ListObjectsV2Input{ + Bucket: sp(s.bucket), + Prefix: sp(prefix), + Delimiter: sp("/"), + } + if token != "" { + in.ContinuationToken = sp(token) + } + + start := time.Now() + page, err := s.client.ListObjectsV2(s.ctx, in) + s.observe("ListObjectsV2", start, err) + if err != nil { + return nil, fmt.Errorf("tigris: list prefixes under %q: %w", prefix, err) + } + for _, cp := range page.CommonPrefixes { + if p := sv(cp.Prefix); p != "" { + out = append(out, p) + } + } + if !bv(page.IsTruncated) || sv(page.NextContinuationToken) == "" { + break + } + token = sv(page.NextContinuationToken) + } + return out, nil +} + +// TakeCompactLease claims the right to compact this repository, and reports +// whether the claim succeeded. +// +// NOTE: The lease saves cost. It is not a correctness requirement. Selection is +// deterministic, so two daemons that compact one repository at the same time +// build byte-identical containers, which get one content-addressed id, which +// makes their PutObject calls idempotent. A fault here wastes work and cannot +// damage a repository. +func (s *Storer) TakeCompactLease(owner string, ttl time.Duration, now time.Time) (bool, error) { + key := s.prefix + compactLeaseKey + + raw, etag, err := s.fetchSmallETag(key) + switch { + case err == nil: + // Somebody holds it. Take it over only once it has expired. + if held, ok := parseLeaseExpiry(raw); ok && now.Before(held) { + return false, nil + } + case errors.Is(err, plumbing.ErrObjectNotFound): + etag = "" // free: claim it with If-None-Match: * + default: + return false, fmt.Errorf("tigris: read compaction lease: %w", err) + } + + body := fmt.Sprintf("%s\t%d\n", owner, now.Add(ttl).Unix()) + switch _, err := s.casPut(key, []byte(body), etag); { + case err == nil: + return true, nil + case errors.Is(err, errCASConflict): + return false, nil // another daemon claimed it between the read and the write + default: + return false, err + } +} + +// ReleaseCompactLease drops the lease. A missing key is success. +func (s *Storer) ReleaseCompactLease() error { + return s.removeSimple(s.prefix + compactLeaseKey) +} + +// parseLeaseExpiry reads the deadline out of a lease body. An unreadable body +// reports false, which makes the lease available: a lease nobody can parse is +// worse than no lease at all, because it would block compaction forever. +func parseLeaseExpiry(raw []byte) (time.Time, bool) { + _, secs, ok := strings.Cut(strings.TrimSpace(string(raw)), "\t") + if !ok { + return time.Time{}, false + } + unix, err := strconv.ParseInt(secs, 10, 64) + if err != nil { + return time.Time{}, false + } + return time.Unix(unix, 0), true +} +``` + +Add `"strconv"` and `"strings"` to `compact.go`'s imports. + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `go test ./internal/storage/tigris/ -run 'TestListRepoPrefixes|TestCompactLease' -v` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add internal/storage/tigris/compact.go internal/storage/tigris/compact_test.go +git commit -m "feat(storage/tigris): enumerate repositories and lease compaction" +``` + +--- + +### Task 13: The sweeper + +**Files:** +- Create: `internal/compactor/compactor.go` +- Test: `internal/compactor/compactor_test.go` + +**Interfaces:** +- Consumes: `(*tigris.Storer).ListRepoPrefixes`, `.Scoped`, `.Compact`, `.TakeCompactLease`, `.ReleaseCompactLease`, `tigris.CompactOptions`, `tigris.CompactStats`; `metrics.ObserveCompact` and `metrics.CompactCounts` from Task 11. +- Produces: `type Sweeper struct { Base *tigris.Storer; Owner string; Interval time.Duration; Opts tigris.CompactOptions; Rand *rand.Rand }`; `func (sw *Sweeper) Run(ctx context.Context) error`; `func (sw *Sweeper) SweepOnce(ctx context.Context) error`. + +- [ ] **Step 1: Write the failing test** + +Create `internal/compactor/compactor_test.go`: + +```go +package compactor + +import ( + "context" + "testing" + "time" +) + +func TestJitteredHoldsWithinBounds(t *testing.T) { + for _, tt := range []struct { + name string + interval time.Duration + }{ + {name: "hour", interval: time.Hour}, + {name: "minute", interval: time.Minute}, + } { + t.Run(tt.name, func(t *testing.T) { + lo := time.Duration(float64(tt.interval) * 0.75) + hi := time.Duration(float64(tt.interval) * 1.25) + for range 200 { + got := jittered(tt.interval) + if got < lo || got > hi { + t.Fatalf("jittered(%v) = %v, want within [%v, %v]", tt.interval, got, lo, hi) + } + } + }) + } +} + +func TestSweepOnceStopsOnContextCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + sw := &Sweeper{Interval: time.Hour} + if err := sw.Run(ctx); err != nil { + t.Errorf("Run after cancel = %v, want nil", err) + } +} +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `go test ./internal/compactor/ -v` +Expected: FAIL — the package does not exist. + +- [ ] **Step 3: Implement the sweeper** + +Create `internal/compactor/compactor.go`: + +```go +// Package compactor runs pack compaction over every repository in the bucket, +// on a timer, inside the daemon. +// +// The pass itself lives in internal/storage/tigris. This package owns only +// when it runs, over which repositories, and who is allowed to run it. +package compactor + +import ( + "context" + "log/slog" + "math/rand/v2" + "time" + + "github.com/tigrisdata/objgit/internal/metrics" + "github.com/tigrisdata/objgit/internal/storage/tigris" +) + +// leaseTTL bounds how long one daemon's claim on a repository survives. It is +// long enough for a large merge and short enough that a daemon killed mid-pass +// does not block the repository for a whole shift. +const leaseTTL = 30 * time.Minute + +// Sweeper compacts every repository in one bucket, on an interval. +type Sweeper struct { + // Base is the root Storer. The sweeper Scopes it per repository, exactly as + // repofs does per request. + Base *tigris.Storer + // Owner names this daemon in a lease body. Any stable string works. + Owner string + // Interval is the nominal time between sweeps. Each wait is jittered, so a + // fleet started at one moment does not stay in lockstep. + Interval time.Duration + // Opts configures each pass. + Opts tigris.CompactOptions +} + +// Run sweeps until ctx is done. It returns nil on cancellation, so an errgroup +// treats a clean shutdown as success. +func (sw *Sweeper) Run(ctx context.Context) error { + slog.Info("pack compaction sweeper started", + "interval", sw.Interval, "max_packs", sw.Opts.MaxPacks, "grace", sw.Opts.Grace) + + for { + select { + case <-ctx.Done(): + return nil + case <-time.After(jittered(sw.Interval)): + } + + if err := sw.SweepOnce(ctx); err != nil { + // A sweep failure is never fatal to the daemon. The next tick + // retries, and a repository that keeps failing shows up in the + // error counter rather than taking the process down. + slog.Error("pack compaction sweep failed", "err", err) + } + } +} + +// SweepOnce compacts every repository once. +func (sw *Sweeper) SweepOnce(ctx context.Context) error { + repos, err := sw.Base.ListRepoPrefixes() + if err != nil { + return err + } + + for _, repo := range repos { + if ctx.Err() != nil { + return nil + } + sw.sweepRepo(repo) + } + return nil +} + +// sweepRepo runs one pass under the repository's lease. One repository at a +// time: compaction is bulk I/O, and it must not compete with traffic serving a +// client. +func (sw *Sweeper) sweepRepo(repo string) { + s := sw.Base.Scoped(repo) + + taken, err := s.TakeCompactLease(sw.Owner, leaseTTL, time.Now()) + if err != nil { + slog.Warn("could not take the compaction lease", "repo", repo, "err", err) + return + } + if !taken { + slog.Debug("another daemon holds the compaction lease", "repo", repo) + return + } + defer func() { + if err := s.ReleaseCompactLease(); err != nil { + slog.Warn("could not release the compaction lease", "repo", repo, "err", err) + } + }() + + start := time.Now() + st, err := s.Compact(sw.Opts) + dur := time.Since(start) + + result := "ok" + if err != nil { + result = "error" + slog.Error("pack compaction failed", "repo", repo, "dur", dur, "err", err) + } + metrics.ObserveCompact(result, dur, metrics.CompactCounts{ + Merged: st.PacksMerged, + Written: st.PacksWritten, + Deleted: st.PacksDeleted, + Deduped: st.ObjectsDeduped, + Orphans: st.OrphansDeleted, + }) + + if err == nil && (st.PacksMerged > 0 || st.PacksDeleted > 0 || st.OrphansDeleted > 0) { + slog.Info("compacted packs", "repo", repo, "dur", dur, + "merged", st.PacksMerged, "written", st.PacksWritten, + "deleted", st.PacksDeleted, "deduped", st.ObjectsDeduped, + "orphans", st.OrphansDeleted) + } +} + +// jittered spreads a fleet's sweeps over plus or minus 25 percent of the +// interval, so N daemons started together do not all wake at one moment and +// contend for every lease in the bucket. +func jittered(d time.Duration) time.Duration { + if d <= 0 { + return d + } + spread := float64(d) * 0.25 + return time.Duration(float64(d) - spread + rand.Float64()*2*spread) +} +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `go test ./internal/compactor/ -v` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add internal/compactor/ +git commit -m "feat(compactor): add the background pack compaction sweeper" +``` + +--- + +### Task 14: Wire the sweeper into the daemon + +**Files:** +- Modify: `cmd/objgitd/main.go:36-50` (flags) and `cmd/objgitd/main.go:151` (errgroup) + +**Interfaces:** +- Consumes: `compactor.Sweeper` (Task 13), `tigris.CompactOptions` (Task 11). +- Produces: the flags `-compact-interval`, `-compact-max-packs`, `-compact-grace`. + +- [ ] **Step 1: Add the flags** + +In the `var` block in `cmd/objgitd/main.go`, after `packedRefs`: + +```go + compactInterval = flag.Duration("compact-interval", 0, "time between background pack compaction sweeps; 0 disables compaction entirely") + compactMaxPacks = flag.Int("compact-max-packs", 16, "pack count above which a repository's smallest containers are merged") + compactGrace = flag.Duration("compact-grace", time.Hour, "how long a merged-away pack survives before deletion; must exceed the longest clone") +``` + +CAUTION: The interval defaults to `0`, which turns compaction off. This flag is the only one in the daemon that deletes data, so it is opt-in for one release, in the same way `-allow-push` and `-allow-hooks` are. Turn it on by default only after a release has run it in production. + +- [ ] **Step 2: Start the sweeper on the errgroup** + +After the `-metrics-bind` block and before the `-http-bind` block in `main.go`, add: + +```go + if *compactInterval > 0 { + host, err := os.Hostname() + if err != nil { + host = "objgitd" + } + sweeper := &compactor.Sweeper{ + Base: base, + Owner: fmt.Sprintf("%s/%d", host, os.Getpid()), + Interval: *compactInterval, + Opts: tigris.CompactOptions{ + MaxPacks: *compactMaxPacks, + Grace: *compactGrace, + }, + } + g.Go(func() error { return sweeper.Run(gCtx) }) + } +``` + +NOTE: `base` is the root `*tigris.Storer` built at `main.go:126`. The sweeper takes it directly, not the `tigrisBase{s: base}` adapter the resolver wraps it in, because it needs `Compact` and `ListRepoPrefixes`, which that adapter does not expose. Add `"fmt"` and the `internal/compactor` import if they are not already present. + +- [ ] **Step 3: Add the flags to the startup log** + +In the `slog.Info` call that already logs `bucket`, `allow_push`, `allow_hooks`, and `pack_cache_bytes`, add: + +```go + "compact_interval", *compactInterval, +``` + +- [ ] **Step 4: Build and confirm the flags exist** + +Run: `go build ./... && go run ./cmd/objgitd -h 2>&1 | grep compact` +Expected: the three `-compact-*` flags are listed with their defaults. + +- [ ] **Step 5: Confirm the environment fallback works** + +Run: `COMPACT_MAX_PACKS=4 go run ./cmd/objgitd -h 2>&1 | grep compact-max-packs` +Expected: the flag is listed. `flagenv` maps `-compact-max-packs` to `COMPACT_MAX_PACKS` with no extra wiring, because it walks the whole flag set. + +- [ ] **Step 6: Run the whole suite** + +Run: `go test ./...` +Expected: PASS + +- [ ] **Step 7: Commit** + +```bash +git add cmd/objgitd/main.go +git commit -m "feat(objgitd): run the pack compaction sweeper behind -compact-interval" +``` + +--- + +### Task 15: The end-to-end regression test over a real git pack + +Every test so far uses hand-built containers. This one pushes a **real pack +built by real git, containing real deltas**, splits it across several +containers, compacts, and proves every object still reads back byte-identical. +It is the regression net for the containment invariant: if a merge ever +stranded a delta from its base, the read below fails outright. + +**Files:** +- Modify: `internal/storage/tigris/compact_test.go` +- Modify: `internal/storage/tigris/livebucket_test.go` + +**Interfaces:** +- Consumes: `buildPackFixture(t, numFiles int, withDeltaEdit bool) packFixture` and `writePack(t, s *Storer, fx packFixture)` from `pack_test.go:672,760`; `packFixture.hashes []plumbing.Hash`; `withMaxPackBytes` from `tigris.go:233`. + +- [ ] **Step 1: Write the test** + +Add to `internal/storage/tigris/compact_test.go`: + +```go +// TestCompactionPreservesARealPushWithDeltas is the containment invariant's +// regression net. buildPackFixture's withDeltaEdit arm makes git emit a real +// delta chain, and withMaxPackBytes splits the push across several containers, +// so the merge has to move deltas between containers to be wrong. +func TestCompactionPreservesARealPushWithDeltas(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + + now := time.Date(2026, 8, 31, 12, 0, 0, 0, time.UTC) + f := newFakeS3(t) + f.clock = func() time.Time { return now } + // A small cap turns one push into many containers, which is what gives + // compaction something to merge. + s := newTestStorer(t, f, withMaxPackBytes(8<<10)) + + fx := buildPackFixture(t, 40, true) + writePack(t, s, fx) + if err := s.up.flush(); err != nil { + t.Fatalf("flush the push: %v", err) + } + + before, _, err := s.surveyPacks() + if err != nil { + t.Fatalf("survey before: %v", err) + } + if len(before) < 3 { + t.Fatalf("the push made %d containers, want at least 3 for this test to mean anything", len(before)) + } + + // readAll proves every object in the fixture reads back and still hashes to + // its own name. A stranded delta fails the read; a corrupt merge fails the + // hash. + readAll := func(when string) { + t.Helper() + fresh := newTestStorer(t, f, withMaxPackBytes(8<<10)) + for _, h := range fx.hashes { + obj, err := fresh.EncodedObject(plumbing.AnyObject, h) + if err != nil { + t.Fatalf("%s: object %s is unreadable: %v", when, h, err) + } + body, err := io.ReadAll(mustReader(obj)) + if err != nil { + t.Fatalf("%s: read %s: %v", when, h, err) + } + if got := hashOf(t, fresh, obj.Type(), body); got != h { + t.Fatalf("%s: object %s rebuilt as %s", when, h, got) + } + } + } + readAll("before compaction") + + opt := CompactOptions{MaxPacks: 2, Grace: time.Hour, Now: func() time.Time { return now }} + st, err := s.Compact(opt) + if err != nil { + t.Fatalf("Compact: %v", err) + } + if st.PacksMerged == 0 { + t.Fatal("the pass merged nothing, so this test proves nothing") + } + if err := s.up.flush(); err != nil { + t.Fatalf("flush the merge: %v", err) + } + + // The overlap window: sources and merged containers both present. + readAll("during the overlap window") + + // Cross the grace period so the next pass reaps the sources. + later := now.Add(2 * time.Hour) + opt.Now = func() time.Time { return later } + if _, err := s.Compact(opt); err != nil { + t.Fatalf("second Compact: %v", err) + } + if err := s.up.flush(); err != nil { + t.Fatalf("flush the reap: %v", err) + } + + after, _, err := s.surveyPacks() + if err != nil { + t.Fatalf("survey after: %v", err) + } + if len(after) >= len(before) { + t.Errorf("container count went from %d to %d, want a reduction", len(before), len(after)) + } + readAll("after the reap") +} +``` + +Add `"os/exec"`, `"io"`, and `"time"` to `compact_test.go`'s imports if they are not already there. + +- [ ] **Step 2: Run the test** + +Run: `go test ./internal/storage/tigris/ -run TestCompactionPreservesARealPushWithDeltas -v` +Expected: PASS, and the log shows a container count that drops. + +- [ ] **Step 3: Add the live-bucket arm** + +Append to `internal/storage/tigris/livebucket_test.go`, following the skip idiom already at the top of that file: + +```go +// TestLiveBucketCompaction runs the same push-compact-read cycle against a +// real bucket. It exists because the fake cannot reproduce the two facts this +// design leans on hardest: that a conditional PutObject really is atomic, and +// that DeleteObjects against a missing key really does succeed. +func TestLiveBucketCompaction(t *testing.T) { + bucket := os.Getenv("OBJGIT_TIGRIS_LIVE_BUCKET") + if bucket == "" { + t.Skip("OBJGIT_TIGRIS_LIVE_BUCKET not set; skipping live-bucket verification") + } + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + + ctx := context.Background() + prefix := "compact-test-" + strconv.FormatInt(time.Now().UnixNano(), 36) + root, err := New(ctx, bucket, withMaxPackBytes(8<<10)) + if err != nil { + t.Fatalf("live construct: %v", err) + } + s := root.Scoped(prefix) + + fx := buildPackFixture(t, 40, true) + writePack(t, s, fx) + if err := s.up.flush(); err != nil { + t.Fatalf("flush: %v", err) + } + + // Grace of zero: the sources are deletable the moment they are tombstoned, + // so one extra pass reaps them and the test needs no wait. + opt := CompactOptions{MaxPacks: 2, Grace: 0} + if _, err := s.Compact(opt); err != nil { + t.Fatalf("Compact: %v", err) + } + if err := s.up.flush(); err != nil { + t.Fatalf("flush the merge: %v", err) + } + if _, err := s.Compact(opt); err != nil { + t.Fatalf("reap pass: %v", err) + } + + fresh := root.Scoped(prefix) + for _, h := range fx.hashes { + if _, err := fresh.EncodedObject(plumbing.AnyObject, h); err != nil { + t.Errorf("object %s is unreadable after live compaction: %v", h, err) + } + } + + t.Cleanup(func() { + keys, err := s.listKeys(s.prefix) + if err != nil { + t.Logf("cleanup list failed, %s may hold stray keys: %v", prefix, err) + return + } + if err := s.deleteKeys(keys); err != nil { + t.Logf("cleanup delete failed, %s may hold stray keys: %v", prefix, err) + } + }) +} +``` + +CAUTION: This test writes to a real bucket under its own timestamped prefix and deletes those keys in `t.Cleanup`. Point `OBJGIT_TIGRIS_LIVE_BUCKET` at a scratch bucket, never at one holding real repositories. + +- [ ] **Step 4: Run the live arm if a scratch bucket is available** + +Run: `OBJGIT_TIGRIS_LIVE_BUCKET= go test ./internal/storage/tigris/ -run TestLiveBucketCompaction -v` +Expected: PASS, or SKIP when the variable is unset. + +- [ ] **Step 5: Run the whole suite** + +Run: `go test ./...` +Expected: PASS + +- [ ] **Step 6: Commit** + +```bash +git add internal/storage/tigris/compact_test.go internal/storage/tigris/livebucket_test.go +git commit -m "test(storage/tigris): prove compaction preserves a real delta-bearing push" +``` + +--- + + +### Task 16: Documentation + +**Files:** +- Modify: `docs/architecture/tigris-storer.md` (the "Remaining gap" section at the end) +- Modify: `docs/reference/tigris-backend.md:476-503` (the build order) +- Modify: `AGENTS.md` (the "Where the code lives" table) + +- [ ] **Step 1: Replace the "Remaining gap" section in the architecture page** + +In `docs/architecture/tigris-storer.md`, replace the `## Remaining gap` section with: + +```markdown +## Compaction (`compact.go`, `tombstones.go`) + +A pass merges the smallest containers into larger ones, then deletes the +sources after a grace period. `internal/compactor` runs one pass per +repository on a timer. `-compact-interval` turns it on, and `0` turns it off. + +The rewrite merges **whole packs, not objects**. A merge bin-packs entire +source containers into an output and deduplicates hashes inside it, which +gives one invariant: the output holds the union of its sources' hashes. +Deduplication collapses copies and never removes a hash. + +That invariant buys three things. Delta containment holds with no check, +because the write-side rule already puts every delta's base in the same +container. Every payload is a verbatim byte copy, so compaction runs no +compressor and resolves no delta. And the sum of the source sizes bounds the +output, so bin packing decides from the survey alone. + +Two rules stop a pass becoming a treadmill. A pack above half the byte cap is +never a source, and a selection producing at least as many outputs as inputs +is abandoned. Without them a repository of near-cap containers is rewritten on +every cycle for no gain. + +CAUTION: A merged-away pack is not deleted at once. A request builds its pack +index once, so a clone that started before the merge still names the old +container. `pack-tombstones` records each source with a deadline, and a later +pass deletes it. `-compact-grace` must exceed the longest clone. + +The tombstone object is an optimization, not a correctness dependency. Every +pass recomputes which packs are subsumed, from records the survey already +loaded, and writes a tombstone for any subsumed pack that lacks one. A crash +between the upload and the tombstone write therefore heals on the next cycle. +The deadline comes from the **newest** pack's `LastModified`, never the +subsumed pack's own: a source can be far older than the merge that replaced +it, and its own clock would give it no grace at all. + +The same pass deletes a `.bin` that has no `.cue`, once it ages past the grace +period. A failed `.cue` upload leaves exactly that, and no reader can ever see +it, because the index is built from `.cue` files alone. + +Compaction needs no lock against a push. A pack landing after the survey is +not in that pass's input, a surveyed pack cannot change because packs are +immutable, and through the overlap window both copies of a duplicated object +are byte-identical, so whichever entry wins the index merge is correct. + +The `compact-lease` key stops two daemons doing one repository twice. It saves +cost and is not required for correctness: selection is deterministic, so two +daemons build byte-identical containers, which get one content-addressed id, +which makes their uploads idempotent. + +## Remaining gap + +There is no reachability-based garbage collection. A force-push or a deleted +branch leaves its objects in the bucket forever. Compaction copies every +object forward by design, so adding that filter means adding a second safety +mechanism: a push that has uploaded its objects but not yet committed its ref +looks unreachable. +``` + +- [ ] **Step 2: Update the build order** + +In `docs/reference/tigris-backend.md`, replace item 5 with: + +```markdown +5. **Done.** Add repacking. `internal/compactor` merges the smallest + containers and deletes the sources after a grace period, behind + `-compact-interval`. The same pass deletes a `.bin` that has no `.cue`, + which a failed upload leaves behind. Garbage collection by reachability is + still open. See + [the storer page](../architecture/tigris-storer.md#compaction-compactgo-tombstonesgo). +``` + +Change the first line of the section from `Items 1 to 4 and item 6 are complete.` to `Items 1 to 7 are complete.` + +- [ ] **Step 3: Add the new package to the index** + +In `AGENTS.md`, add a row to the "Where the code lives" table after `internal/bundler`: + +```markdown +| `internal/compactor` | The timer that runs pack compaction over every repository. | +``` + +- [ ] **Step 4: Check the prose** + +Read the three edits once against the `simple-english` rules: sentences at or under 25 words, no `should`, no semicolons, no contractions, and a condition before its command. + +- [ ] **Step 5: Commit** + +```bash +git add docs/architecture/tigris-storer.md docs/reference/tigris-backend.md AGENTS.md +git commit -m "docs: describe pack compaction and close build order item 5" +``` + +--- + +## Self-Review Notes + +Checked against `docs/plans/pack-compaction.md`: + +| Spec section | Task | +| --- | --- | +| Rewrite merges whole packs | 7, 8 | +| Prefer the whole copy over the delta copy | 8 (`chooseRecords`) | +| Selection by pack count, smallest first | 6 | +| Treadmill guard | 6 | +| No recompression | 7 (`appendVerbatim` carries `codec`) | +| Tombstone object and format | 3, 4 | +| Reap step, delete before tombstone rewrite | 9 | +| Backstop: subsumption, deadline from the newest pack | 10 | +| Backstop: orphan `.bin` | 10 | +| Sweeper, jitter, two-level enumeration | 12, 13 | +| Lease | 12, 13 | +| Concurrency with a live push | documented in 16; the overlap window is exercised in 15 | +| Flags | 14 | +| Metrics | 11 | +| Testing (every listed case) | 3, 6, 8, 9, 10, 11, 15 | + +Two items in the spec's testing list are folded rather than dropped. "A refused compare-and-swap on the tombstone retries" is covered by `TestTombstoneStoreDetectsConflict` in Task 4, plus the retry loops in Tasks 9 through 11. "A delta still reads after a merge" is covered in Task 15 by pushing a real git pack whose deltas git itself produced, then re-hashing every object after the merge and again after the reap. That is stronger than a synthetic delta, because a stranded base fails the read outright and a corrupt payload fails the hash. + +One deliberate simplification is worth recording. The spec says a subsumed pack's deadline comes from "the newest pack that subsumes it". Task 10 uses the newest pack in the whole repository instead. That value is always at or after the true newest subsuming pack, so the grace it grants is never shorter than required, and it costs one pass over the survey rather than a per-pack cover set. diff --git a/docs/plans/pack-compaction.md b/docs/plans/pack-compaction.md new file mode 100644 index 0000000..5ea5d59 --- /dev/null +++ b/docs/plans/pack-compaction.md @@ -0,0 +1,288 @@ +# Plan: pack compaction and tombstone reaping + +## Context + +Every push leaves at least one pack container. A push above `maxPackBytes` +leaves more. Nothing deletes a container, so packs collect forever. This is +item 5 of the build order in +[../reference/tigris-backend.md](../reference/tigris-backend.md). + +Two costs grow with the pack count: + +- A cold index build reads one `.cue` for each pack. `ensurePacksBuilt` runs + this for every repository on the first packed read of a request. +- Two clients that push the same objects store those objects twice. The pack + id is the SHA-256 of the `.bin`, so the two containers get different names + and both survive. Nothing deduplicates them. + +A failed upload adds a third cost. `packJob.run` uploads the `.bin` and then +the `.cue`. If the second upload fails, the bucket keeps a `.bin` that no +reader can see, because `ensurePacksBuilt` indexes `.cue` files only. + +Decisions from the user: + +- **Compaction only.** A pass copies every object forward. It never drops an + unreachable object. Deletion of a source pack is therefore lossless, and the + grace period is the only safety mechanism that the design needs. +- **A background sweeper in the daemon** runs the passes. There is no + subcommand and no post-push trigger. +- **An explicit tombstone object** records which packs to delete and when. +- **The pack count triggers a pass.** A pass merges the smallest packs. A + repository under the count threshold gets no work. +- **A pack is never edited.** A pass writes new containers, uploads them, and + deletes the old ones later. + +The last decision is not a preference. The pack id is the SHA-256 of its own +`.bin`, so an edit changes the name of the pack. Every pack in the bucket is +immutable by construction. + +## Design + +### The rewrite merges whole packs, not objects + +A merge bin-packs whole source packs into one output container. It +deduplicates hashes inside that output. It does not iterate objects across +pack boundaries. + +This gives one invariant: + +```text +hashes(output) = hashes(A) ∪ hashes(B) ∪ ... +``` + +for the source packs A, B, ... that the pass packs into that output. +Deduplication collapses copies of one hash. It never removes a hash. + +The invariant buys three properties at no cost. + +**Delta containment holds without a check.** The write-side rule in +`packwriter.go` already guarantees that the base of every delta in pack A is +also in pack A. Because `A` is a subset of the output, the base moves with the +delta. The pass needs no `inSeg` map and no call to `payloadFor`. + +**Every payload is a verbatim byte copy.** The pass copies the `stored` span +of each record from the source `.bin` into the output `.bin`. The new +`cueRecord` is the old record with a new `offset`. The fields `codec`, +`stored`, `raw`, `base`, and `typ` all carry over. Compaction never runs a +compressor and never resolves a delta. + +**Size accounting is a safe upper bound.** The output is never larger than the +sum of the source `.bin` sizes, because deduplication only removes bytes. The +pass therefore bin-packs against `maxPackBytes` with sizes from the survey, +before it reads one byte of payload. + +One refinement applies to a hash that two source packs in the same output both +hold. If one copy is a delta and the other is whole, keep the whole copy. Both +copies are correct. A kept delta can point at a delta from the other source +pack, which makes the chain one link longer for each pass. The whole copy +stops that growth. The pass has both records in memory already, so this costs +nothing. + +This also corrects the duplicate-push problem. Two concurrent pushes of the +same objects leave two packs of a similar size. Those two packs sort next to +each other under "merge the smallest", so one output usually holds both, and +deduplication collapses them. + +### Selection, and the guard against a treadmill + +`-compact-max-packs` sets the threshold. Its default is 16. + +The threshold gates the merge alone. The reap step, the survey, the +subsumption backstop, and the orphan cleanup run on every pass. A repository +under the threshold still needs them, because a failed upload leaves an orphan +`.bin` at any pack count. + +Above the threshold, the pass sorts the packs by `.bin` size, smallest first. +It takes the smallest packs and bin-packs them into outputs of at most +`maxPackBytes`. It takes packs until the projected count reaches the +threshold. The projected count is `total - taken + outputs`. + +CAUTION: A pass must not run when it cannot reduce the pack count. Packs that +are already near `maxPackBytes` merge into about as many outputs as inputs. A +pass then rewrites the repository on every cycle and bills egress for no +result. Two rules prevent this: + +1. Skip a source pack larger than `maxPackBytes / 2`. +2. If `outputs >= taken`, abandon the pass. + +Compaction does not recompress. A pack written with `-pack-compression=false` +keeps its raw payloads through a merge. Recompression needs a decode of every +payload, which is the cost that the verbatim copy exists to avoid. + +### The tombstone object + +The key is `pack-tombstones` at the root of the repository prefix. It sits +outside `packs/`, for the same reason that `packed-refs` sits outside `refs/`. +No pack listing can return it. + +The format copies `packed-refs` exactly, byte for byte. A 16-byte plaintext +header comes first. Bytes 0 to 2 hold the magic `OGT`. Byte 3 holds the +version. Byte 4 holds the body codec. Bytes 5 to 8 hold the entry count as a +big-endian `uint32`. The body is text, and it holds one entry for each line, +sorted by pack id: + +```text +3f2a... 1756684800 +9c81... 1756688400 +``` + +The separator is a tab. The second field is the deadline as a Unix time in +seconds. A hex id can hold no tab, so the field is never ambiguous. + +Writes use the same compare-and-swap loop as `commitRefs` in `refcache.go`. A +first write sends `If-None-Match: *`. Every later write sends +`If-Match: `. This plan factors that loop out of `refcache.go` into one +`casPut` helper, so the two callers cannot drift apart. + +### The reap step + +The reap step runs at the start of a pass, before the survey. A crash in the +middle of a pass therefore reaps on the next cycle. + +The step reads `pack-tombstones` and divides the entries by deadline. For each +expired entry it deletes the `.bin` and the `.cue` with `DeleteObjects`, 1000 +keys for each call. Then it writes the tombstone object again, without those +entries. + +The order is delete first, tombstone second. If the process stops between the +two, the tombstone names a pack that is already gone. A `DeleteObjects` call +against a missing key succeeds, so the next reap corrects the record. The +opposite order loses the record of a pack that still exists. + +### The backstop for tombstone drift + +A tombstone object is state that can disagree with the bucket. Two cases leak +packs, and both close with data that the survey already holds. + +**A crash between the upload and the tombstone write.** The merged pack +exists, the sources exist, and no tombstone names them. Nothing deletes them. + +**A `.bin` with no `.cue`.** A failed `.cue` upload leaves it. No reader can +see it and no tombstone names it. + +After the survey, the pass computes subsumption. A pack is subsumed when every +hash in its `.cue` is also in the `.cue` of some other pack. If a subsumed +pack has no tombstone, the pass writes one. + +CAUTION: Measure the deadline of that tombstone from the `LastModified` of the +newest pack that subsumes it. The `LastModified` of the subsumed pack is +wrong. An old source pack then gets no grace at all, and a live reader loses +the bytes under it. + +The pass deletes a `.bin` that has no `.cue` directly, with no tombstone, +after the `LastModified` of that `.bin` passes the grace period. No reader can +hold a reference to it, because no `.cue` ever named it. The age test is what +divides an orphan from an upload that is still in flight. + +Subsumption runs over records that the survey already loaded, so it costs CPU +and no extra round trips. The tombstone object is therefore an optimization +and not a correctness dependency. If it is lost, the next pass rebuilds it. + +### The sweeper + +A new package, `internal/compactor`, holds one goroutine. `main.go` starts it +on the errgroup, in the same shape as the other listeners. + +A ticker fires every `-compact-interval`. The interval carries a jitter of 25 +percent, so a fleet of daemons does not wake at one time. + +The sweeper enumerates repositories with two levels of `ListObjectsV2` and +`Delimiter: "/"`. One call at the root returns the organization prefixes. One +call for each organization returns the repository prefixes. Two levels match +`RepoRef.Path()`, which is `path.Join(OrgID, Name)`. + +For each repository the sweeper takes a lease, calls +`Base.Scoped(path).Compact(ctx, opts)`, and releases the lease. It runs one +repository at a time. Compaction is bulk I/O, and it must not compete with +traffic that serves a client. + +The lease is `compact-lease` at the root of the repository prefix. The sweeper +takes it with `If-None-Match: *`. The body holds an owner id and an expiry +time. The sweeper deletes the key to release it, and it ignores a lease that +is expired. + +NOTE: The lease saves cost. It is not a correctness requirement. Selection is +deterministic, because the pass sorts by size and breaks a tie by pack id. Two +daemons that compact one repository at the same time therefore build +byte-identical outputs. Identical bytes get one content-addressed id, so the +two PUT calls are idempotent. A fault in the lease cannot damage a repository. + +### Concurrency with a live push + +The pass needs no lock against a push, for four reasons: + +- A pack that lands after the survey is not in the input of that pass. +- A pack is immutable, and the sweeper is the only writer that deletes one. + Nothing can modify a pack that the survey read. +- Both the sources and the merged pack exist through the overlap window. A + cold index build merges both sets of `.cue` records, and the last write wins + for each hash. The two copies of a duplicated object are byte-identical, so + either entry is correct. This is the same property that makes two concurrent + duplicate pushes harmless today. +- Compaction never writes a reference, so it cannot interact with the + `packed-refs` compare-and-swap. + +### Flags + +| Flag | Default | Purpose | +| --------------------- | ------- | ------------------------------------------- | +| `-compact-interval` | `0` | Time between sweeps. `0` disables the sweeper. | +| `-compact-max-packs` | `16` | Pack count that triggers a pass. | +| `-compact-grace` | `1h` | Time between the merge and the delete. | + +Each flag is kebab-case with a `flagenv` fallback, such as `COMPACT_INTERVAL`. + +The interval defaults to `0`, which turns the subsystem off. This matches +`-allow-push` and `-allow-hooks`, which are opt-in because they carry +consequences. It does not match `-pack-compression` and `-packed-refs`, which +default to on. Those two flags delete nothing. Turn the interval on by default +in a later release, after the pass runs in production. + +The grace period must be longer than the longest request. A `Storer` builds +its pack index once for each request. The index of a reader is therefore never +more stale than the request that holds it. A large clone can run for many +minutes. + +### Metrics + +`internal/metrics` holds every vector, as it does today: + +- `objgit_compact_runs_total{result}` +- `objgit_compact_duration_seconds` +- `objgit_compact_packs_merged_total` +- `objgit_compact_packs_deleted_total` +- `objgit_compact_objects_deduped_total` +- `objgit_compact_bytes_reclaimed_total` +- `objgit_compact_orphan_bins_deleted_total` + +### Testing + +Unit tests are table-driven with `tt`, against the `s3API` fake. The pass uses +get, put, list, and delete, and the fake covers all of them. The cases: + +- A repository under the threshold gets no work. +- The treadmill guard abandons a pass of packs near the cap. +- Deduplication collapses one hash that two source packs hold. +- A delta still reads after a merge. This is the test for the containment + invariant. +- A duplicated hash keeps the whole copy over the delta copy. +- A tombstone deadline comes from the subsuming pack, not the subsumed pack. +- The reap deletes an expired entry and keeps a live one. +- The reap succeeds against a key that is already gone. +- The subsumption backstop writes a tombstone that no pass wrote. +- An orphan `.bin` older than the grace period is deleted. A younger one + survives. +- A refused compare-and-swap on the tombstone retries. + +One end-to-end test covers the whole pass. Seed a repository with `seedRepo`, +push several times to build many packs, run a pass, then clone. The clone must +match a clone taken before the pass. Gate the test with +`exec.LookPath("git")`, as `http_test.go` does. + +## Remaining gap + +This plan does not reclaim space from an object that no reference reaches. A +force-push or a deleted branch still leaves its objects in the bucket forever. +Reachability needs a second safety mechanism, because a push that uploaded its +objects but has not yet committed its reference looks unreachable. The rewrite +stage of this plan is where that filter goes later. diff --git a/docs/usage/format-benchmark.md b/docs/usage/format-benchmark.md new file mode 100644 index 0000000..7a333c2 --- /dev/null +++ b/docs/usage/format-benchmark.md @@ -0,0 +1,202 @@ +# Measuring the storage format change + +`cmd/formatbench` answers one question: what did the move from git's native +on-disk layout to the `.bin`/`.cue` containers cost, and what did it buy. + +It builds two daemons, pushes the same real repositories into both, and clones +each one back out. Nothing in `objgitd` had to change for this. Both builds +already report `objgit_s3_requests_total` on the metrics listener with the same +operation labels, so the request count each format costs is directly +comparable. + +## The two sides + +| Side | Ref | What it runs | +| --- | --- | --- | +| before | `v1.0.2` | `internal/s3fs` and go-git's dotgit layout. One bucket. Listing cache and pack cache on. | +| after | current checkout | `internal/storage/tigris`. `.bin`/`.cue` containers, zstd, cue v3. | + +The format landed in commits `f4a1419` and `ebfe4e3` on 2026-08-27, released as +v1.2.0. `v1.0.2` is the last release before it where repositories live in the +single `-bucket`, which is also how the current build works. The releases after +it moved repositories to per-keypair buckets and turned repository-side caching +off, so they would move two variables at once. + +CAUTION: The current build is not only the format. It also carries zstd +(`7bea058`), cue v3 with delta reuse (`ea4d362`), pack prefetch (`eea720d`), +and bounded pushes (`877cd16`). The comparison is "the first version" against +"today", and not a single-variable experiment. Say so wherever the numbers go. + +## Running it + +```text +go run ./cmd/formatbench +``` + +Run it from the objgit checkout. The harness finds `go.mod` and starts each +daemon with that directory as its working directory, so the daemon picks up +`BUCKET` and the AWS credentials from `.env` exactly as it would normally. + +Pushes go to a real Tigris bucket. Wall-clock numbers over a real network are +the point, so there is no local fake. + +A smoke run of one repository on one side takes about a minute: + +```text +go run ./cmd/formatbench -repos objgit -builds after -reps 1 +``` + +## What a run does + +1. **Build both sides.** The "before" daemon is compiled from a `git worktree` + checked out at `-before-ref` under `var/bench/src/`, because its `go.mod` + and its whole `internal` tree differ from the current checkout. The worktree + is made once and reused. +2. **Mirror the corpus.** `git clone --mirror` of each repository into + `var/bench/mirrors/`, once, and reused across runs. Every push reads from + that copy, so each repetition sends byte-identical history. Pass + `-refresh-mirrors` to fetch first. +3. **Measure each cell.** One cell is one build, one repository, one + repetition. Each cell gets a fresh repository prefix under a new UUID, so no + push is ever measured against a repository that already holds its objects. +4. **Write the tables.** `results.json` holds one record per cell. + `results.md` holds the rendered tables. + +Each cell runs two measurements, and each measurement gets its own daemon +process and its own empty pack cache directory: + +1. Start the daemon, scrape `/metrics`, `git push --mirror`, scrape again. +2. List the bucket prefix for the key count and the byte total. +3. Stop the daemon, delete its pack cache, start a new one. This makes the + clone cold in the process, on disk, and in the pack index that + `(*Storer).ensurePacksBuilt` rebuilds per request. +4. Scrape, `git clone --mirror`, scrape again. +5. Compare the clone against the mirror. + +The push is a mirror push, not a heads-and-tags push. A mirror clone of a +GitHub project also carries `refs/notes/*` and `refs/pull/*`, and those hold +commits that a heads-and-tags push leaves behind. + +## Fairness + +Builds are interleaved, not batched. The schedule is (repetition 1: every +repository on every build), then (repetition 2: the same). Network conditions +drift over an afternoon, and running every "before" cell first would let that +drift look like a result. + +Each row of the report is the median across repetitions, with the minimum and +maximum beside it when they differ. The median is used because two or three +samples over wifi have outliers and no useful mean. + +## The correctness gate + +A benchmark that measures a corrupt push is worthless, so every clone is +compared against the mirror it came from: + +- The same number of reachable commits, from `git rev-list --all --count`. +- The same set of refs pointing at the same hashes, from `git show-ref`. +- An intact object graph, from `git fsck --full`. + +A cell that fails any of these is reported as INVALID and its timings are +withheld from the tables. Pass `-verify=false` to skip the gate, which is worth +doing only when you already know the format is sound and you want the run to +finish faster. + +## Timeouts + +A push stops at `-push-timeout` (45 minutes by default) and a clone at +`-clone-timeout` (30 minutes). An operation that hits its limit is recorded as +DNF, together with the elapsed time, the S3 requests it made, and the bytes it +had written by then. + +A DNF is a result and it is reported as one. It is never turned into an +estimated finish time. Under the old layout, one clone of a 318-object +repository made more than 8,500 `GetObject` calls and never finished; see +`docs/plans/pack-temp-file-cache.md`. + +## What comes out + +`results.md` holds two tables. + +Push: + +| Repo | Build | Wall | S3 requests | PUT | GET | HEAD | LIST | Keys | Bucket bytes | + +Clone: + +| Repo | Build | Wall | S3 requests | GET | HEAD | LIST | Wire bytes | + +Then the run conditions (host, Go version, git version, bucket, corpus sizes) +and a list of every cell that produced no number, with the reason. + +## Cleanup + +The harness never deletes bucket data during a run. It appends every prefix it +creates to `repos.txt` after each cell, so an interrupted run still leaves a +complete list. Delete them afterwards: + +```text +go run ./cmd/formatbench -cleanup var/bench/out//repos.txt +``` + +NOTE: The two builds disagree about the `.git` suffix. The current build parses +the transport path through `repofs.Parse`, which strips it, so its keys sit +under `//`. The older build hands the raw URL path to go-git's +server loader over `internal/s3fs`, which keeps it, so its keys sit under +`/.git/`. Both the accounting and the cleanup look under both +spellings. + +## Flags + +| Flag | Default | What it does | +| --- | --- | --- | +| `-repos` | every entry | Comma-separated corpus entries to run. | +| `-builds` | `before,after` | Which columns to run. See below. | +| `-before-ref` | `v1.0.2` | The ref the "before" daemon is built from. | +| `-reps` | per entry | Override the repetition count for every repository. | +| `-out` | `var/bench/out` | Parent directory for run output. | +| `-org` | `formatbench` | Org segment every benchmark repository is created under. | +| `-bucket` | `BUCKET` | The bucket the daemon under test writes to. | +| `-push-timeout` | `45m` | Time limit for one push. | +| `-clone-timeout` | `30m` | Time limit for one clone. | +| `-verify` | `true` | Compare each clone against its mirror. | +| `-refresh-mirrors` | `false` | Fetch each mirror before the run. | +| `-cleanup` | none | Delete every prefix in a `repos.txt` and exit. | + +Every flag also reads from the environment through `flagenv`, which maps each +one to UPPER_SNAKE. `-before-ref` becomes `BEFORE_REF`. + +### More than two columns + +`-builds` takes three spellings, comma-separated: + +| Entry | What it builds | +| --- | --- | +| `before` | The ref in `-before-ref`, which defaults to `v1.0.2`. | +| `after` | The current checkout. | +| `=` | That ref, in a column labelled ``. An empty ref means the current checkout. | + +The third form is how you measure a candidate fix. Put the old build, the +current build, and the fix in one run: + +```text +go run ./cmd/formatbench -builds "before,main=v1.5.0,fixed=" -reps 2 +``` + +Run the candidate against a separate earlier run instead, and the difference +between the two numbers includes a day of network drift. One interleaved +session is the only honest way to read three points. + +Each ref is built once from its own `git worktree` under `var/bench/src/`. +Slashes in a ref name become dashes in that directory name. + +## The corpus + +The default corpus is three repositories, chosen for three shapes rather than +three names. Edit the `corpus` table in `cmd/formatbench/main.go` to change it. + +| Name | Shape | +| --- | --- | +| objgit | Small. Mostly text. | +| x | Medium. Large object count. | +| tigris-blog | Blob-heavy. Images do not compress. | diff --git a/docs/usage/lfs.md b/docs/usage/lfs.md new file mode 100644 index 0000000..3ba81cd --- /dev/null +++ b/docs/usage/lfs.md @@ -0,0 +1,135 @@ +# Git LFS + +`objgitd` serves the Git LFS API. Large files stay out of the git repository: +git keeps a small pointer file, and the bytes go straight to the Tigris bucket. + +The client transfers those bytes itself, over a presigned URL. They do not pass +through `objgitd`. + +## Turn it on + +LFS is off by default. Start the daemon with `-allow-lfs`: + +```sh +objgitd -bucket $BUCKET -http-bind :8080 -allow-push -allow-lfs +``` + +To use LFS from an `ssh://` remote, add `-external-url`: + +```sh +objgitd -bucket $BUCKET -http-bind :8080 -ssh-bind :2222 \ + -allow-push -allow-lfs -external-url https://git.example.com +``` + +The LFS API is HTTP, even for an `ssh://` remote. The SSH side only tells the +client where that API is, and it needs `-external-url` to do so. The daemon +refuses to start with `-allow-lfs` and an empty `-http-bind`. + +## Flags + +| Flag | Default | Purpose | +| --- | --- | --- | +| `-allow-lfs` | `false` | Serve the Git LFS API. | +| `-allow-lfs-locks` | `true` | Serve the file locking API. | +| `-external-url` | `""` | Public base URL, such as `https://git.example.com`. | +| `-lfs-url-ttl` | `15m` | How long a presigned transfer URL stays valid. | +| `-lfs-max-size` | `5368709120` | Largest object accepted, in bytes. | +| `-lfs-max-batch` | `500` | Most objects in one batch request. | + +Each flag also reads from an environment variable, in upper snake case. +`-allow-lfs` is `ALLOW_LFS`. + +## Push a large file + +```sh +git init myrepo && cd myrepo +git lfs install --local +git lfs track '*.bin' + +head -c 50000000 /dev/urandom > big.bin +git add .gitattributes big.bin +git commit -m 'add big file' + +git push http://localhost:8080/acme/myrepo.git main +``` + +The push reports the LFS upload separately from the git objects: + +```text +Uploading LFS objects: 100% (1/1), 50 MB | 0 B/s, done. +To http://localhost:8080/acme/myrepo.git + * [new branch] main -> main +``` + +A clone retrieves the bytes the same way: + +```sh +git clone http://localhost:8080/acme/myrepo.git fresh +``` + +## Where the bytes go + +```text +lfs/blobs/ the bytes, once for the whole bucket +acme/myrepo/lfs/staging/ an upload in flight, before it is verified +acme/myrepo/lfs/objects/ a marker: this repository holds it +acme/myrepo/packs/.bin the git objects, which stay small +``` + +A client uploads into its own repository's staging area. The server checks those +bytes and then moves them to the shared key. A repository has to prove it holds +an object before the server hands out a URL for it, so knowing an object ID is +not enough to read someone else's data. + +Object bytes are stored once per bucket. Two repositories that hold the same +file share one copy. + +A repository still uploads a file the first time it pushes it, even when another +repository already put those bytes in the bucket. That upload is what proves the +repository holds the object, and it ends at the same shared key, so the bucket +keeps one copy. + +## Watch the deduplication + +`objgit_lfs_objects_total` counts objects by outcome. On an upload batch, a +`present` result means the repository already held the object and the client +transferred nothing: + +```text +objgit_lfs_objects_total{operation="upload",result="new"} +objgit_lfs_objects_total{operation="upload",result="present"} +``` + +The ratio between those two is the deduplication rate. + +`objgit_lfs_verify_total{status="mismatch"}` is the series to alert on. It +counts uploads whose bytes did not match the object ID. + +## File locking + +```sh +git lfs lock big.bin +git lfs locks +git lfs unlock big.bin +``` + +CAUTION: Locking needs authentication to be useful. `objgitd` records the HTTP +Basic username as the lock owner. Without a username every caller is the same +owner `anonymous`, so every lock looks like the caller's own and no push is ever +blocked. Set a username, or turn locking off with `-allow-lfs-locks=false`. + +## Limits + +- One object caps at 5 GiB, because one presigned upload caps there. +- Nothing deletes an LFS object. A file removed from a repository keeps its + bytes in the bucket. +- An upload that is never verified stays under `//lfs/staging/`. Set + a bucket TTL rule on that prefix to reap abandoned uploads. +- `-lfs-max-size` limits the size a client declares, not the bytes it can send + to a presigned URL. The verify step catches an object that does not match what + was declared. +- A presigned URL stays valid for its full lifetime. Removing a user's access + does not cancel a URL they already hold. +- Temporary credentials shorten that lifetime. If the daemon runs with + credentials that expire, it clamps `-lfs-url-ttl` to the remaining session + time and logs the clamp. diff --git a/go.mod b/go.mod index 8a00fb8..4a74f2c 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect github.com/aws/aws-sdk-go-v2/config v1.32.17 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.19.16 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.16 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect diff --git a/internal/lfs/batch.go b/internal/lfs/batch.go new file mode 100644 index 0000000..92982ff --- /dev/null +++ b/internal/lfs/batch.go @@ -0,0 +1,272 @@ +package lfs + +import ( + "context" + "errors" + "fmt" + "slices" + "time" + + "golang.org/x/sync/errgroup" +) + +// The two batch operations. Any other value is a malformed request, never a +// default: guessing "download" for an unrecognized operation would answer a +// write request with read access. +const ( + OperationDownload = "download" + OperationUpload = "upload" +) + +// TransferBasic is the only transfer adapter this server implements. "basic" is +// a plain GET and PUT against the action's href, which is exactly what a +// presigned URL is. +const TransferBasic = "basic" + +// Action names in a batch response. +const ( + ActionDownload = "download" + ActionUpload = "upload" + ActionVerify = "verify" +) + +// hashAlgoSHA256 is the only hash Git LFS defines. An absent value means the +// same thing. +const hashAlgoSHA256 = "sha256" + +// defaultBatchConcurrency bounds the marker reads one batch makes. A batch +// carries up to a hundred objects by default, and each one costs a HEAD. +const defaultBatchConcurrency = 16 + +// Whole-request failures. These become an HTTP status, unlike a per-object +// error, which travels inside a 200. +var ( + // ErrInvalidRequest is a malformed batch body. + ErrInvalidRequest = errors.New("lfs: malformed batch request") + // ErrTooManyObjects is a batch larger than the configured cap. + ErrTooManyObjects = errors.New("lfs: too many objects in one batch") + // ErrObjectTooLarge is an object larger than the configured cap. + ErrObjectTooLarge = errors.New("lfs: object is larger than the limit") + // ErrNoSupportedTransfer means the client offered no transfer this server + // implements. + ErrNoSupportedTransfer = errors.New("lfs: no supported transfer adapter offered") +) + +// Pointer is one object in a batch request: its ID and its declared length. +type Pointer struct { + OID string `json:"oid"` + Size int64 `json:"size"` +} + +// Ref names the branch the client is working against. It is advisory, and this +// server does not use it. +type Ref struct { + Name string `json:"name"` +} + +// BatchRequest is the body of POST .../info/lfs/objects/batch. +type BatchRequest struct { + Operation string `json:"operation"` + Transfers []string `json:"transfers,omitempty"` + Ref *Ref `json:"ref,omitempty"` + Objects []Pointer `json:"objects"` + HashAlgo string `json:"hash_algo,omitempty"` +} + +// Action tells the client how to transfer one object. +type Action struct { + Href string `json:"href"` + Header map[string]string `json:"header,omitempty"` + // ExpiresIn is how many seconds the href stays valid. Without it a client + // cannot tell a stale URL from a denied one. + ExpiresIn int `json:"expires_in,omitempty"` +} + +// ObjectError is a per-object failure. It travels inside a 200 response. +type ObjectError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +// ObjectResult is one object's outcome in a batch response. +type ObjectResult struct { + OID string `json:"oid"` + Size int64 `json:"size"` + // Authenticated tells git-lfs the action href needs no further credentials. + // Without it the client runs credential discovery against the storage host + // and can attach an Authorization header, which a presigned request + // rejects outright. + Authenticated bool `json:"authenticated,omitempty"` + Actions map[string]*Action `json:"actions,omitempty"` + Error *ObjectError `json:"error,omitempty"` +} + +// BatchResponse is the body returned by the batch endpoint. +type BatchResponse struct { + Transfer string `json:"transfer"` + Objects []ObjectResult `json:"objects"` + HashAlgo string `json:"hash_algo,omitempty"` +} + +// BatchOptions is the per-request configuration the caller supplies. +type BatchOptions struct { + // Repo is the repository path, "orgID/name". + Repo string + // VerifyURL is the absolute URL of this repository's verify endpoint. + VerifyURL string + // TTL is how long a minted transfer URL stays valid. + TTL time.Duration + // MaxSize is the largest object accepted, in bytes. + MaxSize int64 + // MaxBatch is the most objects accepted in one request. + MaxBatch int + // Concurrency bounds the marker reads. Zero picks a default. + Concurrency int +} + +// Batch answers a Git LFS batch request. +// +// Every decision reads the per-repository marker, never the shared blob, so a +// caller who knows an oid cannot use one repository to reach another's data. +// The cost is that a repository which does not yet hold an object uploads it +// again even when the bytes are already in the bucket. +// +// A returned error is a whole-request failure. A per-object failure is not an +// error: it rides in the response as ObjectResult.Error, and the request still +// succeeds. +func (s *Store) Batch(ctx context.Context, req *BatchRequest, opt BatchOptions) (*BatchResponse, error) { + if err := validateBatch(req, opt); err != nil { + return nil, err + } + + results := make([]ObjectResult, len(req.Objects)) + g, gctx := errgroup.WithContext(ctx) + limit := opt.Concurrency + if limit <= 0 { + limit = defaultBatchConcurrency + } + g.SetLimit(limit) + + for i, p := range req.Objects { + // Results are indexed, not appended, so the response keeps the request + // order whatever order the reads finish in. + g.Go(func() error { + res, err := s.resolveObject(gctx, p, req.Operation, opt) + if err != nil { + return err + } + results[i] = res + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, err + } + + return &BatchResponse{ + Transfer: TransferBasic, + Objects: results, + HashAlgo: hashAlgoSHA256, + }, nil +} + +// validateBatch checks everything that makes the whole request unanswerable. +func validateBatch(req *BatchRequest, opt BatchOptions) error { + if req == nil { + return fmt.Errorf("%w: empty body", ErrInvalidRequest) + } + switch req.Operation { + case OperationDownload, OperationUpload: + case "": + return fmt.Errorf("%w: missing operation", ErrInvalidRequest) + default: + return fmt.Errorf("%w: unknown operation %q", ErrInvalidRequest, req.Operation) + } + if req.HashAlgo != "" && req.HashAlgo != hashAlgoSHA256 { + return fmt.Errorf("%w: unsupported hash algorithm %q", ErrInvalidRequest, req.HashAlgo) + } + // An empty list means the client accepts the default, which is "basic". + if len(req.Transfers) > 0 && !slices.Contains(req.Transfers, TransferBasic) { + return fmt.Errorf("%w: client offered %v", ErrNoSupportedTransfer, req.Transfers) + } + if opt.MaxBatch > 0 && len(req.Objects) > opt.MaxBatch { + return fmt.Errorf("%w: %d objects, limit is %d", ErrTooManyObjects, len(req.Objects), opt.MaxBatch) + } + for _, p := range req.Objects { + if err := ValidateOID(p.OID); err != nil { + return err + } + if p.Size < 0 { + return fmt.Errorf("%w: negative size for %s", ErrInvalidRequest, p.OID) + } + if opt.MaxSize > 0 && p.Size > opt.MaxSize { + return fmt.Errorf("%w: %s is %d bytes, limit is %d", ErrObjectTooLarge, p.OID, p.Size, opt.MaxSize) + } + } + return nil +} + +// resolveObject decides what one object's entry in the response says. +func (s *Store) resolveObject(ctx context.Context, p Pointer, operation string, opt BatchOptions) (ObjectResult, error) { + res := ObjectResult{OID: p.OID, Size: p.Size, Authenticated: true} + + size, held, err := s.Marker(ctx, opt.Repo, p.OID) + if err != nil { + return ObjectResult{}, err + } + + if operation == OperationDownload { + if !held { + res.Error = &ObjectError{Code: 404, Message: "object does not exist"} + return res, nil + } + req, err := s.pre.PresignGet(ctx, BlobKey(p.OID), opt.TTL) + if err != nil { + return ObjectResult{}, fmt.Errorf("presigning download for %s: %w", p.OID, err) + } + res.Actions = map[string]*Action{ + ActionDownload: { + Href: req.URL, + Header: TransferHeader(req), + ExpiresIn: int(opt.TTL.Seconds()), + }, + } + return res, nil + } + + if held { + if size != p.Size { + res.Error = &ObjectError{ + Code: 422, + Message: fmt.Sprintf("object is stored with size %d, not %d", size, p.Size), + } + return res, nil + } + // Already held. Omitting actions entirely is how a batch response says + // "nothing to do"; git-lfs skips the transfer. + return res, nil + } + + // The upload goes to this repository's staging key, not the shared blob. + // Verify promotes it. See the package doc for why membership needs that + // indirection. + req, err := s.pre.PresignPut(ctx, StagingKey(opt.Repo, p.OID), p.OID, opt.TTL) + if err != nil { + return ObjectResult{}, fmt.Errorf("presigning upload for %s: %w", p.OID, err) + } + res.Actions = map[string]*Action{ + ActionUpload: { + Href: req.URL, + Header: TransferHeader(req), + ExpiresIn: int(opt.TTL.Seconds()), + }, + // The daemon never sees the upload, so verify is the only step that can + // check the bytes and record that this repository holds them. + ActionVerify: { + Href: opt.VerifyURL, + Header: map[string]string{"Content-Type": MediaType}, + ExpiresIn: int(opt.TTL.Seconds()), + }, + } + return res, nil +} diff --git a/internal/lfs/batch_test.go b/internal/lfs/batch_test.go new file mode 100644 index 0000000..a0e76ef --- /dev/null +++ b/internal/lfs/batch_test.go @@ -0,0 +1,390 @@ +package lfs + +import ( + "context" + "errors" + "strings" + "testing" + "time" +) + +// headerValue looks a header up without caring how it is capitalized. +func headerValue(h map[string]string, name string) string { + for k, v := range h { + if strings.EqualFold(k, name) { + return v + } + } + return "" +} + +func testBatchOptions() BatchOptions { + return BatchOptions{ + Repo: testRepo, + VerifyURL: "https://git.example.com/acme/widgets.git/info/lfs/objects/verify", + TTL: 15 * time.Minute, + MaxSize: 1 << 30, + MaxBatch: 100, + } +} + +func runBatch(t *testing.T, f *fakeS3, req *BatchRequest) *BatchResponse { + t.Helper() + resp, err := newTestStore(t, f).Batch(context.Background(), req, testBatchOptions()) + if err != nil { + t.Fatalf("Batch: %v", err) + } + return resp +} + +func TestBatchDownload(t *testing.T) { + for _, tt := range []struct { + name string + seed func(*fakeS3) + wantAction bool + wantErrCode int + }{ + { + name: "repository holds the object", + seed: func(f *fakeS3) { + f.putMarker(testRepo, testOID, 4) + f.putBlob(testOID, 4, "") + }, + wantAction: true, + }, + { + name: "object is unknown", + seed: func(*fakeS3) {}, + wantErrCode: 404, + }, + { + // The read-oracle test. Another repository's upload put the bytes + // in the bucket, but this repository never proved it holds them, + // so it gets a 404 and never a URL. + name: "bytes exist but this repository has no marker", + seed: func(f *fakeS3) { + f.putBlob(testOID, 4, "") + f.putMarker("someone/else", testOID, 4) + }, + wantErrCode: 404, + }, + } { + t.Run(tt.name, func(t *testing.T) { + f := newFakeS3() + tt.seed(f) + + resp := runBatch(t, f, &BatchRequest{ + Operation: OperationDownload, + Objects: []Pointer{{OID: testOID, Size: 4}}, + }) + + if len(resp.Objects) != 1 { + t.Fatalf("got %d objects, want 1", len(resp.Objects)) + } + obj := resp.Objects[0] + + if tt.wantErrCode != 0 { + if obj.Error == nil { + t.Fatalf("object has no error, want code %d", tt.wantErrCode) + } + if obj.Error.Code != tt.wantErrCode { + t.Errorf("error code = %d, want %d", obj.Error.Code, tt.wantErrCode) + } + if obj.Actions != nil { + t.Error("a failed object must carry no actions") + } + return + } + + if obj.Error != nil { + t.Fatalf("unexpected object error: %+v", obj.Error) + } + if _, ok := obj.Actions[ActionDownload]; ok != tt.wantAction { + t.Fatalf("download action present = %v, want %v", ok, tt.wantAction) + } + }) + } +} + +func TestBatchUpload(t *testing.T) { + for _, tt := range []struct { + name string + seed func(*fakeS3) + size int64 + wantUpload bool + wantErrCode int + }{ + { + name: "new object gets an upload action", + seed: func(*fakeS3) {}, + size: 4, + wantUpload: true, + }, + { + // Deduplication: the repository already holds it, so git-lfs skips + // the transfer entirely. + name: "object already held is skipped", + seed: func(f *fakeS3) { + f.putMarker(testRepo, testOID, 4) + }, + size: 4, + wantUpload: false, + }, + { + name: "held at a different size is a conflict", + seed: func(f *fakeS3) { + f.putMarker(testRepo, testOID, 9) + }, + size: 4, + wantErrCode: 422, + }, + { + // A marker whose size cannot be read must leave the client a way + // out. Counting it as held at size 0 answers every later batch 422 + // and never offers an upload action, so nothing can repair it. + name: "marker with an unreadable size is uploaded again", + seed: func(f *fakeS3) { + f.set(MarkerKey(testRepo, testOID), fakeObject{}) + }, + size: 4, + wantUpload: true, + }, + { + // The bytes are in the bucket but this repository has no marker, so + // it must upload. Trusting the global blob here is the read oracle. + name: "bytes exist elsewhere so the client still uploads", + seed: func(f *fakeS3) { + f.putBlob(testOID, 4, "") + }, + size: 4, + wantUpload: true, + }, + } { + t.Run(tt.name, func(t *testing.T) { + f := newFakeS3() + tt.seed(f) + + resp := runBatch(t, f, &BatchRequest{ + Operation: OperationUpload, + Objects: []Pointer{{OID: testOID, Size: tt.size}}, + }) + obj := resp.Objects[0] + + if tt.wantErrCode != 0 { + if obj.Error == nil || obj.Error.Code != tt.wantErrCode { + t.Fatalf("error = %+v, want code %d", obj.Error, tt.wantErrCode) + } + return + } + if obj.Error != nil { + t.Fatalf("unexpected object error: %+v", obj.Error) + } + + _, hasUpload := obj.Actions[ActionUpload] + if hasUpload != tt.wantUpload { + t.Fatalf("upload action present = %v, want %v", hasUpload, tt.wantUpload) + } + if !tt.wantUpload { + if obj.Actions != nil { + t.Error("a skipped upload must carry no actions at all") + } + return + } + // An upload the daemon never sees needs a verify step, or nothing + // ever checks the bytes or records membership. + if _, ok := obj.Actions[ActionVerify]; !ok { + t.Error("an upload action must be paired with a verify action") + } + }) + } +} + +// TestBatchUploadCarriesTheChecksumHeader pins the end of the integrity chain: +// the signed checksum has to reach the client, or it never gets sent. +func TestBatchUploadCarriesTheChecksumHeader(t *testing.T) { + resp := runBatch(t, newFakeS3(), &BatchRequest{ + Operation: OperationUpload, + Objects: []Pointer{{OID: testOID, Size: 4}}, + }) + + up := resp.Objects[0].Actions[ActionUpload] + if up == nil { + t.Fatal("no upload action") + } + // Header names are case-insensitive over the wire, and SigV4 lowercases + // them before it checks a signature, so assert presence and not spelling. + if got := headerValue(up.Header, "x-amz-checksum-sha256"); got == "" { + t.Errorf("upload header = %v, want an x-amz-checksum-sha256 entry", up.Header) + } + if got := headerValue(up.Header, "host"); got != "" { + t.Error("upload header must not carry Host; the client sets it from the URL") + } +} + +// TestBatchMarksObjectsAuthenticated stops git-lfs from running credential +// discovery against the storage host, which would attach an Authorization +// header that a presigned request rejects. +func TestBatchMarksObjectsAuthenticated(t *testing.T) { + f := newFakeS3() + f.putMarker(testRepo, testOID, 4) + + resp := runBatch(t, f, &BatchRequest{ + Operation: OperationDownload, + Objects: []Pointer{{OID: testOID, Size: 4}}, + }) + + if !resp.Objects[0].Authenticated { + t.Error("objects must be marked authenticated") + } +} + +func TestBatchActionsCarryExpiry(t *testing.T) { + f := newFakeS3() + f.putMarker(testRepo, testOID, 4) + + resp := runBatch(t, f, &BatchRequest{ + Operation: OperationDownload, + Objects: []Pointer{{OID: testOID, Size: 4}}, + }) + + dl := resp.Objects[0].Actions[ActionDownload] + if dl.ExpiresIn != int(15*time.Minute/time.Second) { + t.Errorf("expires_in = %d, want %d", dl.ExpiresIn, int(15*time.Minute/time.Second)) + } +} + +func TestBatchReportsTheBasicTransfer(t *testing.T) { + resp := runBatch(t, newFakeS3(), &BatchRequest{ + Operation: OperationUpload, + Transfers: []string{"basic"}, + Objects: []Pointer{{OID: testOID, Size: 4}}, + }) + + if resp.Transfer != TransferBasic { + t.Errorf("transfer = %q, want %q", resp.Transfer, TransferBasic) + } +} + +// TestBatchRejects covers the whole-request failures, which are distinct from a +// per-object error: these produce an HTTP status, not a 200 with an error +// inside. +func TestBatchRejects(t *testing.T) { + for _, tt := range []struct { + name string + req *BatchRequest + wantErr error + }{ + { + name: "unknown operation", + req: &BatchRequest{ + Operation: "sideways", + Objects: []Pointer{{OID: testOID, Size: 4}}, + }, + wantErr: ErrInvalidRequest, + }, + { + name: "missing operation", + req: &BatchRequest{ + Objects: []Pointer{{OID: testOID, Size: 4}}, + }, + wantErr: ErrInvalidRequest, + }, + { + name: "malformed oid", + req: &BatchRequest{ + Operation: OperationDownload, + Objects: []Pointer{{OID: "../../etc/passwd", Size: 4}}, + }, + wantErr: ErrInvalidOID, + }, + { + name: "uppercase oid", + req: &BatchRequest{ + Operation: OperationDownload, + Objects: []Pointer{{OID: strings.ToUpper(testOID), Size: 4}}, + }, + wantErr: ErrInvalidOID, + }, + { + name: "negative size", + req: &BatchRequest{ + Operation: OperationUpload, + Objects: []Pointer{{OID: testOID, Size: -1}}, + }, + wantErr: ErrInvalidRequest, + }, + { + name: "size over the cap", + req: &BatchRequest{ + Operation: OperationUpload, + Objects: []Pointer{{OID: testOID, Size: 1<<30 + 1}}, + }, + wantErr: ErrObjectTooLarge, + }, + { + name: "unsupported hash algorithm", + req: &BatchRequest{ + Operation: OperationDownload, + HashAlgo: "sha512", + Objects: []Pointer{{OID: testOID, Size: 4}}, + }, + wantErr: ErrInvalidRequest, + }, + { + name: "no supported transfer offered", + req: &BatchRequest{ + Operation: OperationDownload, + Transfers: []string{"tus", "multipart"}, + Objects: []Pointer{{OID: testOID, Size: 4}}, + }, + wantErr: ErrNoSupportedTransfer, + }, + } { + t.Run(tt.name, func(t *testing.T) { + _, err := newTestStore(t, newFakeS3()).Batch(context.Background(), tt.req, testBatchOptions()) + if !errors.Is(err, tt.wantErr) { + t.Fatalf("Batch error = %v, want %v", err, tt.wantErr) + } + }) + } +} + +func TestBatchRejectsAnOversizedBatch(t *testing.T) { + objects := make([]Pointer, 101) + for i := range objects { + objects[i] = Pointer{OID: testOID, Size: 4} + } + + _, err := newTestStore(t, newFakeS3()).Batch(context.Background(), &BatchRequest{ + Operation: OperationDownload, + Objects: objects, + }, testBatchOptions()) + + if !errors.Is(err, ErrTooManyObjects) { + t.Fatalf("Batch error = %v, want ErrTooManyObjects", err) + } +} + +// TestBatchPreservesObjectOrder keeps the response aligned with the request, +// which the concurrent marker reads could otherwise scramble. +func TestBatchPreservesObjectOrder(t *testing.T) { + const otherOID = "ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb" + f := newFakeS3() + f.putMarker(testRepo, otherOID, 1) + + resp := runBatch(t, f, &BatchRequest{ + Operation: OperationDownload, + Objects: []Pointer{ + {OID: testOID, Size: 4}, + {OID: otherOID, Size: 1}, + }, + }) + + if len(resp.Objects) != 2 { + t.Fatalf("got %d objects, want 2", len(resp.Objects)) + } + if resp.Objects[0].OID != testOID || resp.Objects[1].OID != otherOID { + t.Errorf("object order = [%s %s], want [%s %s]", + resp.Objects[0].OID, resp.Objects[1].OID, testOID, otherOID) + } +} diff --git a/internal/lfs/errors.go b/internal/lfs/errors.go new file mode 100644 index 0000000..0151526 --- /dev/null +++ b/internal/lfs/errors.go @@ -0,0 +1,17 @@ +package lfs + +// MediaType is the content type every Git LFS API request and response carries. +// A client that does not accept it gets a 406, and a response that omits it is +// not recognized as an LFS answer. +const MediaType = "application/vnd.git-lfs+json" + +// ErrorBody is the top-level failure body. It is the shape git-lfs prints to +// the user, so Message is read by a person, not a program. +type ErrorBody struct { + Message string `json:"message"` + // RequestID lets an operator tie a client-side failure to a server log + // line. It is optional in the protocol. + RequestID string `json:"request_id,omitempty"` + // DocumentationURL points at an explanation of the failure. + DocumentationURL string `json:"documentation_url,omitempty"` +} diff --git a/internal/lfs/fake_test.go b/internal/lfs/fake_test.go new file mode 100644 index 0000000..9e1d19d --- /dev/null +++ b/internal/lfs/fake_test.go @@ -0,0 +1,239 @@ +package lfs + +import ( + "bytes" + "context" + "io" + "strconv" + "strings" + "sync" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" +) + +// fakeObject is one object in the fake bucket. +type fakeObject struct { + body []byte + size int64 + checksum string + meta map[string]string + etag string +} + +// fakeS3 is an in-memory bucket standing in for Tigris. CI runs with no +// credentials, so every store test goes through this. It mirrors the fakeS3 in +// internal/storage/tigris: an opaque counter ETag, and injectable failures per +// operation. +type fakeS3 struct { + mu sync.Mutex + objects map[string]fakeObject + etagSeq int + + headErr error + getErr error + renameErr error + putErr error + delErr error + + // putHook runs before a Put is applied, so a test can land a competing + // write in the middle of a compare-and-swap loop. + putHook func(key string) + + // headHook runs before a Head is answered. A returned error fails that one + // call, which is how a test makes the bucket fail for one key and answer + // another normally. + headHook func(key string) error + + // renameHook runs before a Rename is applied, so a test can land a + // competing verify between this call's head and its promotion. + renameHook func(src string) +} + +func newFakeS3() *fakeS3 { + return &fakeS3{objects: map[string]fakeObject{}} +} + +// errPreconditionFailed is what a bucket answers to a refused If-Match or +// If-None-Match, carrying the code isPreconditionFailed matches. +var errPreconditionFailed = &smithy.GenericAPIError{ + Code: "PreconditionFailed", + Message: "At least one of the pre-conditions you specified did not hold", +} + +// errBucketUnavailable is a transient bucket fault. It carries a code +// isNotFound does not match, because a fault is never absence. +var errBucketUnavailable = &smithy.GenericAPIError{ + Code: "InternalError", + Message: "We encountered an internal error. Please try again.", +} + +// readAll drains a request body, which is nil for a bodyless PUT. +func readAll(r io.Reader) []byte { + if r == nil { + return nil + } + b, err := io.ReadAll(r) + if err != nil { + return nil + } + return b +} + +func (f *fakeS3) nextETag() string { + f.etagSeq++ + return `"etag-` + strconv.Itoa(f.etagSeq) + `"` +} + +// set stores an object directly, bypassing the API. For seeding only. +func (f *fakeS3) set(key string, o fakeObject) { + f.mu.Lock() + defer f.mu.Unlock() + o.etag = f.nextETag() + f.objects[key] = o +} + +func (f *fakeS3) putBlob(oid string, size int64, checksum string) { + f.set(BlobKey(oid), fakeObject{size: size, checksum: checksum}) +} + +// putStaged seeds an upload that has landed but not yet been verified. +func (f *fakeS3) putStaged(repo, oid string, size int64, checksum string) { + f.set(StagingKey(repo, oid), fakeObject{size: size, checksum: checksum}) +} + +func (f *fakeS3) putMarker(repo, oid string, size int64) { + f.set(MarkerKey(repo, oid), fakeObject{ + meta: map[string]string{metaSize: strconv.FormatInt(size, 10)}, + }) +} + +// drop removes an object directly, bypassing the API. For seeding only. +func (f *fakeS3) drop(key string) { + f.mu.Lock() + defer f.mu.Unlock() + delete(f.objects, key) +} + +func (f *fakeS3) has(key string) bool { + f.mu.Lock() + defer f.mu.Unlock() + _, ok := f.objects[key] + return ok +} + +func (f *fakeS3) HeadObject(_ context.Context, in *s3.HeadObjectInput, _ ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { + if f.headErr != nil { + return nil, f.headErr + } + if f.headHook != nil { + if err := f.headHook(aws.ToString(in.Key)); err != nil { + return nil, err + } + } + f.mu.Lock() + defer f.mu.Unlock() + o, ok := f.objects[aws.ToString(in.Key)] + if !ok { + return nil, &types.NotFound{} + } + out := &s3.HeadObjectOutput{ + ContentLength: aws.Int64(o.size), + Metadata: o.meta, + ETag: aws.String(o.etag), + } + // Only report a checksum when the caller asked for one, the way S3 does. + if o.checksum != "" && in.ChecksumMode == types.ChecksumModeEnabled { + out.ChecksumSHA256 = aws.String(o.checksum) + } + return out, nil +} + +func (f *fakeS3) GetObject(_ context.Context, in *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + if f.getErr != nil { + return nil, f.getErr + } + f.mu.Lock() + defer f.mu.Unlock() + o, ok := f.objects[aws.ToString(in.Key)] + if !ok { + return nil, &types.NoSuchKey{} + } + return &s3.GetObjectOutput{ + Body: io.NopCloser(bytes.NewReader(o.body)), + ContentLength: aws.Int64(int64(len(o.body))), + ETag: aws.String(o.etag), + }, nil +} + +func (f *fakeS3) PutObject(_ context.Context, in *s3.PutObjectInput, _ ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + if f.putErr != nil { + return nil, f.putErr + } + key := aws.ToString(in.Key) + if f.putHook != nil { + f.putHook(key) + } + + f.mu.Lock() + defer f.mu.Unlock() + + cur, exists := f.objects[key] + // Conditional writes, the same compare-and-swap the ref cache uses. + if aws.ToString(in.IfNoneMatch) == "*" && exists { + return nil, errPreconditionFailed + } + if m := aws.ToString(in.IfMatch); m != "" && (!exists || cur.etag != m) { + return nil, errPreconditionFailed + } + + body := readAll(in.Body) + f.objects[key] = fakeObject{ + body: body, + size: int64(len(body)), + meta: in.Metadata, + etag: f.nextETag(), + } + return &s3.PutObjectOutput{ETag: aws.String(f.objects[key].etag)}, nil +} + +// RenameObject models the Tigris extension: the source key is gone afterwards. +func (f *fakeS3) RenameObject(_ context.Context, in *s3.CopyObjectInput, _ ...func(*s3.Options)) (*s3.CopyObjectOutput, error) { + if f.renameErr != nil { + return nil, f.renameErr + } + // CopySource is "/"; the fake has one bucket, so drop it. + _, src, _ := strings.Cut(aws.ToString(in.CopySource), "/") + if f.renameHook != nil { + f.renameHook(src) + } + + f.mu.Lock() + defer f.mu.Unlock() + o, ok := f.objects[src] + if !ok { + return nil, &types.NoSuchKey{} + } + o.etag = f.nextETag() + f.objects[aws.ToString(in.Key)] = o + delete(f.objects, src) + return &s3.CopyObjectOutput{}, nil +} + +func (f *fakeS3) DeleteObject(_ context.Context, in *s3.DeleteObjectInput, _ ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) { + if f.delErr != nil { + return nil, f.delErr + } + f.mu.Lock() + defer f.mu.Unlock() + delete(f.objects, aws.ToString(in.Key)) + return &s3.DeleteObjectOutput{}, nil +} + +func newTestStore(t *testing.T, f *fakeS3) *Store { + t.Helper() + return NewStore(f, newTestPresigner(t), "test-bucket") +} diff --git a/internal/lfs/lfs.go b/internal/lfs/lfs.go new file mode 100644 index 0000000..1f02122 --- /dev/null +++ b/internal/lfs/lfs.go @@ -0,0 +1,115 @@ +// Package lfs implements the Git LFS server protocol over a Tigris bucket. +// +// The batch endpoint answers with a presigned URL, and the client transfers +// to and from the bucket directly. EROFS snapshot builds also read LFS blobs +// from the bucket after checking repository membership. See +// docs/architecture/lfs.md. +// +// # Layout in the bucket +// +// lfs/blobs/ the bytes, stored once for the whole bucket +// //lfs/staging/ an upload in flight, before it is verified +// //lfs/objects/ a zero-byte marker: this repository has it +// //lfs/locks.json every lock held in this repository +// +// Bytes are global, so two repositories that hold the same file store it once. +// Membership is per repository, and every batch decision reads the marker, +// never the blob. Without that split, a caller who knows an oid could ask any +// repository for a presigned URL and read a blob belonging to a repository it +// cannot otherwise see. LFS oids are not secret: they sit in pointer files, +// forks, and CI logs. +// +// # Uploads land in the repository, not in the shared key +// +// A client uploads to its own staging key. Verify checks those bytes, copies +// them to the shared key, and only then writes the marker. +// +// The indirection is what makes membership mean something. If a client uploaded +// straight to the shared key, verify could only check that the bytes exist and +// have the right length, which a caller who merely knows an oid and a size can +// already say. An oid and its size sit next to each other in every pointer +// file. Staging makes the client prove it holds the content: to get a marker it +// must first write the bytes, and Tigris checks them against the oid as it +// does. +// +// Staging also keeps a failed upload from touching anything shared. A rejected +// object is deleted from the caller's own staging key, never from the blob +// other repositories are reading. +// +// The marker is also the only reference count a future collector can use. A +// blob whose last marker is gone is unreachable. +// +// This package imports no transport and no Prometheus client. It takes observer +// functions, and cmd/objgitd wires internal/metrics into them. +package lfs + +import ( + "errors" + "fmt" +) + +// ErrInvalidOID is returned for an object ID that is not exactly 64 lowercase +// hexadecimal characters. +var ErrInvalidOID = errors.New("lfs: object id must be 64 lowercase hex characters") + +// oidLen is the length of a hex-encoded SHA-256 digest. Git LFS uses SHA-256 +// for every object, and hash_algo has no other legal value. +const oidLen = 64 + +const ( + // blobPrefix holds object bytes, shared by every repository. + blobPrefix = "lfs/blobs/" + // markerSuffix holds one repository's membership markers, under its own + // "orgID/name" prefix. + markerSuffix = "/lfs/objects/" + // stagingSuffix holds one repository's uploads until they are verified. + stagingSuffix = "/lfs/staging/" + // locksSuffix holds one repository's lock document. + locksSuffix = "/lfs/locks.json" +) + +// ValidateOID reports whether oid is a well-formed Git LFS object ID. +// +// The oid becomes part of a bucket key, so this is the boundary that keeps +// caller-supplied text out of the key space. It accepts only 64 lowercase +// hexadecimal characters, which admits no separator, no dot, no escape, and no +// control byte. Uppercase is rejected rather than folded: git-lfs computes +// lowercase, so accepting uppercase would store the same bytes under a second +// key and break deduplication. +func ValidateOID(oid string) error { + if len(oid) != oidLen { + return fmt.Errorf("%w: got %d characters", ErrInvalidOID, len(oid)) + } + for i := range len(oid) { + c := oid[i] + if (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') { + continue + } + return fmt.Errorf("%w: illegal character %q at offset %d", ErrInvalidOID, c, i) + } + return nil +} + +// BlobKey is the bucket key holding the bytes for oid. It carries no repository +// prefix: the bytes are shared by the whole bucket. +// +// Callers must validate oid first. The key is only safe because ValidateOID +// admits nothing but lowercase hex. +func BlobKey(oid string) string { return blobPrefix + oid } + +// MarkerKey is the bucket key proving that the repository at repo holds oid. +// repo is a repofs.RepoRef.Path(), an "orgID/name" pair. +// +// Callers must validate oid first. +func MarkerKey(repo, oid string) string { return repo + markerSuffix + oid } + +// StagingKey is the bucket key a client uploads oid to, before it is verified. +// It is scoped to one repository, so writing it proves the caller holds the +// content rather than merely knowing its name. +// +// Callers must validate oid first. +func StagingKey(repo, oid string) string { return repo + stagingSuffix + oid } + +// LocksKey is the bucket key holding the lock document for the repository at +// repo. +func LocksKey(repo string) string { return repo + locksSuffix } diff --git a/internal/lfs/lfs_test.go b/internal/lfs/lfs_test.go new file mode 100644 index 0000000..c74fb10 --- /dev/null +++ b/internal/lfs/lfs_test.go @@ -0,0 +1,102 @@ +package lfs + +import ( + "errors" + "strings" + "testing" +) + +func TestValidateOID(t *testing.T) { + const valid = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + + for _, tt := range []struct { + name string + oid string + wantErr error + }{ + { + name: "64 lowercase hex characters", + oid: valid, + }, + { + name: "empty", + oid: "", + wantErr: ErrInvalidOID, + }, + { + name: "too short", + oid: valid[:63], + wantErr: ErrInvalidOID, + }, + { + name: "too long", + oid: valid + "a", + wantErr: ErrInvalidOID, + }, + { + // Uppercase is valid hex but must be rejected, not folded: git-lfs + // emits lowercase, so an uppercase oid would key the same bytes + // twice and break deduplication. + name: "uppercase hex", + oid: strings.ToUpper(valid), + wantErr: ErrInvalidOID, + }, + { + name: "non-hex character", + oid: "z" + valid[1:], + wantErr: ErrInvalidOID, + }, + { + name: "path traversal", + oid: "../../../etc/passwd", + wantErr: ErrInvalidOID, + }, + { + // 64 characters, so a length-only check would let this through. + name: "path traversal padded to the right length", + oid: "../.." + strings.Repeat("a", 59), + wantErr: ErrInvalidOID, + }, + { + name: "contains a slash", + oid: valid[:32] + "/" + valid[33:], + wantErr: ErrInvalidOID, + }, + } { + t.Run(tt.name, func(t *testing.T) { + err := ValidateOID(tt.oid) + if !errors.Is(err, tt.wantErr) { + t.Fatalf("ValidateOID(%q) error = %v, want %v", tt.oid, err, tt.wantErr) + } + }) + } +} + +func TestBlobKey(t *testing.T) { + const oid = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + + got := BlobKey(oid) + if want := "lfs/blobs/" + oid; got != want { + t.Errorf("BlobKey() = %q, want %q", got, want) + } +} + +func TestMarkerKey(t *testing.T) { + const oid = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + + got := MarkerKey("acme/widgets", oid) + if want := "acme/widgets/lfs/objects/" + oid; got != want { + t.Errorf("MarkerKey() = %q, want %q", got, want) + } +} + +// TestMarkerKeyIsNotBlobKey pins the property the whole access model rests on: +// the per-repository marker and the shared blob are different keys, so reading +// one never proves the other. +func TestMarkerKeyIsNotBlobKey(t *testing.T) { + const oid = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + + if MarkerKey("acme/widgets", oid) == BlobKey(oid) { + t.Error("marker key and blob key must differ") + } +} diff --git a/internal/lfs/livebucket_test.go b/internal/lfs/livebucket_test.go new file mode 100644 index 0000000..a45ba3a --- /dev/null +++ b/internal/lfs/livebucket_test.go @@ -0,0 +1,270 @@ +package lfs + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "os" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + tstorage "github.com/tigrisdata/storage-go" +) + +// The facts in this file cannot be learned from a fake bucket, and they are the +// ones the whole design rests on: +// +// 1. Tigris enforces x-amz-checksum-sha256 on a presigned PUT, so a client +// cannot write bytes that do not hash to the key. +// 2. The checksum is inside the signature, so a client cannot drop the header +// to escape that check. +// 3. HeadObject with ChecksumModeEnabled returns the stored digest, which is +// what Verify compares against. +// +// CI never runs these: it has no Tigris credentials. Run them by hand against a +// throw-away bucket before trusting a change to presign.go or store.go: +// +// AWS_PROFILE=tigris OBJGIT_TIGRIS_LIVE_BUCKET=my-test-bucket \ +// go test ./internal/lfs/ -run TestLive -v +// +// Use a Single-region or Multi-region bucket. A Global bucket reads eventually, +// which breaks the conditional writes the lock document depends on. + +// liveStore builds a Store against a real bucket, or skips the test. +func liveStore(t *testing.T) (*Store, string) { + t.Helper() + + bucket := os.Getenv("OBJGIT_TIGRIS_LIVE_BUCKET") + if bucket == "" { + t.Skip("OBJGIT_TIGRIS_LIVE_BUCKET not set") + } + + client, err := tstorage.New(context.Background()) + if err != nil { + t.Fatalf("dialing Tigris: %v", err) + } + return NewStore(client, NewPresigner(client.Client, bucket), bucket), bucket +} + +// liveOID mints an object unique to this run, so a failed test never leaves +// another run reading its bytes. +func liveOID(t *testing.T) (oid string, body []byte) { + t.Helper() + body = fmt.Appendf(nil, "objgit live test %d", time.Now().UnixNano()) + sum := sha256.Sum256(body) + return hex.EncodeToString(sum[:]), body +} + +// putPresigned performs the transfer a git-lfs client would, sending exactly +// the headers the batch response named. +func putPresigned(t *testing.T, href string, header map[string]string, body []byte) int { + t.Helper() + + req, err := http.NewRequest(http.MethodPut, href, bytes.NewReader(body)) + if err != nil { + t.Fatalf("building upload request: %v", err) + } + for k, v := range header { + req.Header.Set(k, v) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("uploading: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode >= 300 { + detail, _ := io.ReadAll(resp.Body) + t.Logf("upload answered %d: %s", resp.StatusCode, detail) + } + return resp.StatusCode +} + +// TestLiveUploadIntegrity is the acceptance gate for the presigned upload path. +func TestLiveUploadIntegrity(t *testing.T) { + s, bucket := liveStore(t) + ctx := context.Background() + + oid, body := liveOID(t) + t.Cleanup(func() { + _, _ = s.api.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(BlobKey(oid)), + }) + _, _ = s.api.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(MarkerKey(testRepo, oid)), + }) + _, _ = s.api.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(StagingKey(testRepo, oid)), + }) + }) + + // Uploads go to this repository's staging key, exactly as the batch + // endpoint directs a real client. + req, err := s.Presigner().PresignPut(ctx, StagingKey(testRepo, oid), oid, 15*time.Minute) + if err != nil { + t.Fatalf("PresignPut: %v", err) + } + header := TransferHeader(req) + + t.Run("bytes that do not match the oid are rejected", func(t *testing.T) { + if code := putPresigned(t, req.URL, header, []byte("not the right bytes")); code < 400 { + t.Fatalf("upload of wrong bytes answered %d; the bucket is not enforcing the checksum", code) + } + }) + + t.Run("dropping the checksum header is rejected", func(t *testing.T) { + // The checksum is a signed header, so removing it invalidates the + // signature. Without this the checksum would be advisory. + if code := putPresigned(t, req.URL, nil, body); code < 400 { + t.Fatalf("upload without the checksum header answered %d; the header is not signed", code) + } + }) + + t.Run("matching bytes are accepted", func(t *testing.T) { + if code := putPresigned(t, req.URL, header, body); code >= 300 { + t.Fatalf("upload of correct bytes answered %d, want 2xx", code) + } + }) + + t.Run("verify accepts the upload and records membership", func(t *testing.T) { + status, err := s.Verify(ctx, testRepo, oid, int64(len(body))) + if err != nil { + t.Fatalf("Verify: %v", err) + } + if status != VerifyOK { + t.Fatalf("Verify = %v, want ok", status) + } + + size, held, err := s.Marker(ctx, testRepo, oid) + if err != nil { + t.Fatalf("Marker: %v", err) + } + if !held { + t.Fatal("Verify did not record membership") + } + if size != int64(len(body)) { + t.Errorf("marker size = %d, want %d", size, len(body)) + } + }) + + t.Run("a second verify is idempotent", func(t *testing.T) { + // The staged object is gone after promotion, but the repository still + // holds the object, so a retried verify must not report a failure. + status, err := s.Verify(ctx, testRepo, oid, int64(len(body))) + if err != nil { + t.Fatalf("Verify: %v", err) + } + if status != VerifyOK { + t.Fatalf("Verify = %v, want ok", status) + } + }) +} + +// TestLiveDownloadRoundTrip proves a presigned GET returns the stored bytes. +func TestLiveDownloadRoundTrip(t *testing.T) { + s, bucket := liveStore(t) + ctx := context.Background() + + oid, body := liveOID(t) + t.Cleanup(func() { + for _, key := range []string{BlobKey(oid), MarkerKey(testRepo, oid), StagingKey(testRepo, oid)} { + _, _ = s.api.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + }) + } + }) + + up, err := s.Presigner().PresignPut(ctx, StagingKey(testRepo, oid), oid, 15*time.Minute) + if err != nil { + t.Fatalf("PresignPut: %v", err) + } + if code := putPresigned(t, up.URL, TransferHeader(up), body); code >= 300 { + t.Fatalf("seeding upload answered %d", code) + } + // Verify promotes the staged object to the shared key, which is where a + // download reads from. + if status, err := s.Verify(ctx, testRepo, oid, int64(len(body))); err != nil { + t.Fatalf("Verify: %v", err) + } else if status != VerifyOK { + t.Fatalf("Verify = %v, want ok", status) + } + + down, err := s.Presigner().PresignGet(ctx, BlobKey(oid), 15*time.Minute) + if err != nil { + t.Fatalf("PresignGet: %v", err) + } + + resp, err := http.Get(down.URL) + if err != nil { + t.Fatalf("downloading: %v", err) + } + defer resp.Body.Close() + + got, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("reading download: %v", err) + } + if !bytes.Equal(got, body) { + t.Errorf("downloaded %q, want %q", got, body) + } +} + +// TestLiveLockCAS drives the lock document's conditional writes against a real +// bucket, where the ETag semantics are the bucket's and not a fake's. +func TestLiveLockCAS(t *testing.T) { + s, bucket := liveStore(t) + ctx := context.Background() + + repo := fmt.Sprintf("livetest/locks-%d", time.Now().UnixNano()) + t.Cleanup(func() { + _, _ = s.api.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(LocksKey(repo)), + }) + }) + + first, created, err := s.CreateLock(ctx, repo, "big.bin", "alice") + if err != nil { + t.Fatalf("CreateLock: %v", err) + } + if !created { + t.Fatal("the first lock on a fresh repository must be created") + } + + // A second create on the same path must lose, not overwrite. + if _, created, err := s.CreateLock(ctx, repo, "big.bin", "bob"); err != nil { + t.Fatalf("CreateLock: %v", err) + } else if created { + t.Fatal("a held path was locked twice") + } + + // A second path must succeed against the now-existing document, which + // exercises the If-Match path rather than If-None-Match. + if _, created, err := s.CreateLock(ctx, repo, "other.bin", "alice"); err != nil { + t.Fatalf("CreateLock: %v", err) + } else if !created { + t.Fatal("a free path was not locked") + } + + locks, _, err := s.ListLocks(ctx, repo, LockFilter{}) + if err != nil { + t.Fatalf("ListLocks: %v", err) + } + if len(locks) != 2 { + t.Fatalf("got %d locks, want 2", len(locks)) + } + + if _, err := s.DeleteLock(ctx, repo, first.ID, "alice", false); err != nil { + t.Fatalf("DeleteLock: %v", err) + } +} diff --git a/internal/lfs/locks.go b/internal/lfs/locks.go new file mode 100644 index 0000000..72080d5 --- /dev/null +++ b/internal/lfs/locks.go @@ -0,0 +1,308 @@ +package lfs + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "slices" + "strconv" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +// lockCASRetries bounds the compare-and-swap loop. Contention on one +// repository's lock document is rare, so a handful of attempts is plenty, and a +// bound stops a permanently refused write from spinning. +const lockCASRetries = 8 + +// AnonymousOwner is the lock owner recorded when a caller presents no +// credential. +// +// With no user store, every anonymous caller is this one owner, so every lock +// looks like the caller's own and a push is never blocked. Locking only tells +// users apart once callers send a Basic username. See docs/usage/lfs.md. +const AnonymousOwner = "anonymous" + +// Lock failures. +var ( + // ErrLockNotFound is an unlock of an ID nobody holds. + ErrLockNotFound = errors.New("lfs: no such lock") + // ErrLockHeldByAnother is an unlock of someone else's lock without force. + ErrLockHeldByAnother = errors.New("lfs: lock is held by another user") + // ErrLockContention is a compare-and-swap that kept losing. + ErrLockContention = errors.New("lfs: lock document is too contended") +) + +// Owner names who holds a lock. +type Owner struct { + Name string `json:"name"` +} + +// Lock is one held path. +type Lock struct { + ID string `json:"id"` + Path string `json:"path"` + LockedAt time.Time `json:"locked_at"` + Owner Owner `json:"owner"` +} + +// lockDoc is the whole lock state of one repository, stored as one object. +// +// One document rather than one object per lock: a lock list has to be +// consistent, and a single object gives that through one conditional write. +// Listing many small objects would also cost a request per page. +type lockDoc struct { + Locks []Lock `json:"locks"` +} + +// LockFilter narrows a lock listing. +type LockFilter struct { + Path string + ID string + Cursor string + Limit int +} + +// newLockID mints an opaque lock ID. +// +// The ID travels back to the server as a path segment on unlock, so it is +// generated here and never taken from the client. Lowercase hex keeps it safe +// in a URL. +func newLockID() (string, error) { + var b [16]byte + if _, err := rand.Read(b[:]); err != nil { + return "", fmt.Errorf("generating lock id: %w", err) + } + return hex.EncodeToString(b[:]), nil +} + +// CreateLock locks path for owner. +// +// The bool reports whether a lock was created. When it is false the returned +// Lock is the one already holding the path, which the caller renders as a 409 +// so the client can name the holder. +func (s *Store) CreateLock(ctx context.Context, repo, path, owner string) (Lock, bool, error) { + if path == "" { + return Lock{}, false, fmt.Errorf("%w: empty lock path", ErrInvalidRequest) + } + if owner == "" { + owner = AnonymousOwner + } + + var held Lock + var created bool + err := s.mutateLocks(ctx, repo, func(doc *lockDoc) error { + // Reset per attempt: a retry re-reads the document, so a decision made + // against the previous read must not leak forward. + held, created = Lock{}, false + + if i := slices.IndexFunc(doc.Locks, func(l Lock) bool { return l.Path == path }); i >= 0 { + held = doc.Locks[i] + return errNoChange + } + + id, err := newLockID() + if err != nil { + return err + } + held = Lock{ + ID: id, + Path: path, + LockedAt: time.Now().UTC(), + Owner: Owner{Name: owner}, + } + created = true + doc.Locks = append(doc.Locks, held) + return nil + }) + if err != nil { + return Lock{}, false, err + } + return held, created, nil +} + +// DeleteLock releases a lock. +// +// Without force, only the owner may release it. force lets anyone break a lock, +// which is what `git lfs unlock --force` is for. +func (s *Store) DeleteLock(ctx context.Context, repo, id, owner string, force bool) (Lock, error) { + if owner == "" { + owner = AnonymousOwner + } + + var removed Lock + err := s.mutateLocks(ctx, repo, func(doc *lockDoc) error { + removed = Lock{} + + i := slices.IndexFunc(doc.Locks, func(l Lock) bool { return l.ID == id }) + if i < 0 { + return ErrLockNotFound + } + // EqualFold, matching SplitLocks. A username that round-trips through a + // credential helper can come back in a different case, and a user told + // a lock is theirs must be able to release it. + if !force && !strings.EqualFold(doc.Locks[i].Owner.Name, owner) { + return fmt.Errorf("%w: %s holds it", ErrLockHeldByAnother, doc.Locks[i].Owner.Name) + } + removed = doc.Locks[i] + doc.Locks = slices.Delete(doc.Locks, i, i+1) + return nil + }) + if err != nil { + return Lock{}, err + } + return removed, nil +} + +// ListLocks returns the locks held in repo, narrowed by filter. The second +// return is the next cursor, empty when the listing is complete. +func (s *Store) ListLocks(ctx context.Context, repo string, filter LockFilter) ([]Lock, string, error) { + doc, _, err := s.readLocks(ctx, repo) + if err != nil { + return nil, "", err + } + + out := make([]Lock, 0, len(doc.Locks)) + for _, l := range doc.Locks { + if filter.Path != "" && l.Path != filter.Path { + continue + } + if filter.ID != "" && l.ID != filter.ID { + continue + } + out = append(out, l) + } + + // The cursor is an index into the filtered list. The document is small and + // read whole, so there is nothing to gain from an opaque token. + start := 0 + if filter.Cursor != "" { + // strconv, not fmt.Sscanf: Sscanf reads "3junk" as 3 and reports no + // error, so a mangled cursor would silently skip entries. + n, err := strconv.Atoi(filter.Cursor) + if err != nil || n < 0 { + return nil, "", fmt.Errorf("%w: bad cursor %q", ErrInvalidRequest, filter.Cursor) + } + start = n + } + if start >= len(out) { + return nil, "", nil + } + out = out[start:] + + next := "" + if filter.Limit > 0 && len(out) > filter.Limit { + out = out[:filter.Limit] + next = strconv.Itoa(start + filter.Limit) + } + return out, next, nil +} + +// errNoChange tells mutateLocks the callback decided against writing. It never +// reaches the caller. +var errNoChange = errors.New("lfs: lock document unchanged") + +// mutateLocks applies fn to the repository's lock document under a +// compare-and-swap. +// +// This is the same conditional-write loop the ref cache uses: read the document +// with its ETag, apply the change, then write with If-Match (or If-None-Match +// for a document that does not exist yet). A refused write means another +// client committed first, so the loop re-reads and re-applies rather than +// overwriting that client's lock. +func (s *Store) mutateLocks(ctx context.Context, repo string, fn func(*lockDoc) error) error { + for range lockCASRetries { + doc, etag, err := s.readLocks(ctx, repo) + if err != nil { + return err + } + + if err := fn(doc); err != nil { + if errors.Is(err, errNoChange) { + return nil + } + return err + } + + body, err := json.Marshal(doc) + if err != nil { + return fmt.Errorf("encoding lock document: %w", err) + } + + in := &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(LocksKey(repo)), + Body: bytes.NewReader(body), + ContentType: aws.String("application/json"), + } + if etag == "" { + in.IfNoneMatch = aws.String("*") + } else { + in.IfMatch = aws.String(etag) + } + + _, err = s.api.PutObject(ctx, in) + if err == nil { + return nil + } + if !isPreconditionFailed(err) { + return fmt.Errorf("writing lock document for %s: %w", repo, err) + } + // Lost the race. Read the winner's document and try again. + } + return fmt.Errorf("%w: gave up after %d attempts", ErrLockContention, lockCASRetries) +} + +// readLocks loads a repository's lock document and the ETag to write against. +// A repository that has never been locked reads as an empty document with an +// empty ETag, which mutateLocks turns into an If-None-Match create. +func (s *Store) readLocks(ctx context.Context, repo string) (*lockDoc, string, error) { + out, err := s.api.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(LocksKey(repo)), + }) + if err != nil { + if isNotFound(err) { + return &lockDoc{}, "", nil + } + return nil, "", fmt.Errorf("reading lock document for %s: %w", repo, err) + } + defer out.Body.Close() + + body, err := io.ReadAll(out.Body) + if err != nil { + return nil, "", fmt.Errorf("reading lock document body for %s: %w", repo, err) + } + + doc := &lockDoc{} + if len(bytes.TrimSpace(body)) > 0 { + if err := json.Unmarshal(body, doc); err != nil { + return nil, "", fmt.Errorf("decoding lock document for %s: %w", repo, err) + } + } + return doc, aws.ToString(out.ETag), nil +} + +// SplitLocks divides locks into the caller's own and everyone else's, which is +// the shape `POST .../locks/verify` answers with. +func SplitLocks(locks []Lock, owner string) (ours, theirs []Lock) { + if owner == "" { + owner = AnonymousOwner + } + for _, l := range locks { + if strings.EqualFold(l.Owner.Name, owner) { + ours = append(ours, l) + continue + } + theirs = append(theirs, l) + } + return ours, theirs +} diff --git a/internal/lfs/locks_test.go b/internal/lfs/locks_test.go new file mode 100644 index 0000000..1b945de --- /dev/null +++ b/internal/lfs/locks_test.go @@ -0,0 +1,204 @@ +package lfs + +import ( + "context" + "errors" + "strings" + "testing" +) + +func mustLock(t *testing.T, s *Store, path, owner string) Lock { + t.Helper() + l, created, err := s.CreateLock(context.Background(), testRepo, path, owner) + if err != nil { + t.Fatalf("CreateLock(%q): %v", path, err) + } + if !created { + t.Fatalf("CreateLock(%q) did not create a lock", path) + } + return l +} + +func TestCreateLock(t *testing.T) { + s := newTestStore(t, newFakeS3()) + + l := mustLock(t, s, "big.bin", "alice") + + if l.Path != "big.bin" { + t.Errorf("Path = %q, want %q", l.Path, "big.bin") + } + if l.Owner.Name != "alice" { + t.Errorf("Owner = %q, want %q", l.Owner.Name, "alice") + } + if l.ID == "" { + t.Error("lock has no ID") + } + if l.LockedAt.IsZero() { + t.Error("lock has no timestamp") + } +} + +// TestLockIDsAreServerGenerated matters because the ID comes back to the server +// as a path segment on unlock. A client-chosen ID would be caller-controlled +// text in a route. +func TestLockIDsAreServerGenerated(t *testing.T) { + s := newTestStore(t, newFakeS3()) + + a := mustLock(t, s, "one.bin", "alice") + b := mustLock(t, s, "two.bin", "alice") + + if a.ID == b.ID { + t.Fatal("two locks share an ID") + } + for _, id := range []string{a.ID, b.ID} { + if strings.Trim(id, "0123456789abcdef") != "" { + t.Errorf("lock ID %q is not lowercase hex", id) + } + } +} + +func TestCreateLockOnAHeldPathConflicts(t *testing.T) { + s := newTestStore(t, newFakeS3()) + first := mustLock(t, s, "big.bin", "alice") + + got, created, err := s.CreateLock(context.Background(), testRepo, "big.bin", "bob") + if err != nil { + t.Fatalf("CreateLock: %v", err) + } + if created { + t.Fatal("a held path must not be locked twice") + } + // The conflict response has to name the existing holder, so the client can + // tell the user who to ask. + if got.ID != first.ID { + t.Errorf("conflict returned lock %q, want the holder %q", got.ID, first.ID) + } + if got.Owner.Name != "alice" { + t.Errorf("conflict owner = %q, want alice", got.Owner.Name) + } +} + +func TestListLocks(t *testing.T) { + s := newTestStore(t, newFakeS3()) + one := mustLock(t, s, "one.bin", "alice") + mustLock(t, s, "two.bin", "bob") + + for _, tt := range []struct { + name string + filter LockFilter + want int + }{ + {name: "no filter returns every lock", want: 2}, + {name: "by path", filter: LockFilter{Path: "one.bin"}, want: 1}, + {name: "by id", filter: LockFilter{ID: one.ID}, want: 1}, + {name: "by unknown path", filter: LockFilter{Path: "nope.bin"}, want: 0}, + {name: "limit", filter: LockFilter{Limit: 1}, want: 1}, + } { + t.Run(tt.name, func(t *testing.T) { + got, _, err := s.ListLocks(context.Background(), testRepo, tt.filter) + if err != nil { + t.Fatalf("ListLocks: %v", err) + } + if len(got) != tt.want { + t.Fatalf("got %d locks, want %d", len(got), tt.want) + } + }) + } +} + +func TestListLocksOnAnUnlockedRepository(t *testing.T) { + s := newTestStore(t, newFakeS3()) + + got, _, err := s.ListLocks(context.Background(), testRepo, LockFilter{}) + if err != nil { + t.Fatalf("ListLocks: %v", err) + } + if len(got) != 0 { + t.Fatalf("got %d locks, want 0", len(got)) + } +} + +func TestDeleteLock(t *testing.T) { + for _, tt := range []struct { + name string + owner string + force bool + wantErr error + }{ + {name: "the owner may unlock", owner: "alice"}, + {name: "another user may not", owner: "bob", wantErr: ErrLockHeldByAnother}, + {name: "another user may force", owner: "bob", force: true}, + } { + t.Run(tt.name, func(t *testing.T) { + s := newTestStore(t, newFakeS3()) + held := mustLock(t, s, "big.bin", "alice") + + got, err := s.DeleteLock(context.Background(), testRepo, held.ID, tt.owner, tt.force) + if !errors.Is(err, tt.wantErr) { + t.Fatalf("DeleteLock error = %v, want %v", err, tt.wantErr) + } + if tt.wantErr != nil { + return + } + if got.ID != held.ID { + t.Errorf("DeleteLock returned %q, want %q", got.ID, held.ID) + } + + remaining, _, err := s.ListLocks(context.Background(), testRepo, LockFilter{}) + if err != nil { + t.Fatalf("ListLocks: %v", err) + } + if len(remaining) != 0 { + t.Errorf("got %d locks after unlock, want 0", len(remaining)) + } + }) + } +} + +func TestDeleteUnknownLock(t *testing.T) { + s := newTestStore(t, newFakeS3()) + + _, err := s.DeleteLock(context.Background(), testRepo, "deadbeef", "alice", false) + if !errors.Is(err, ErrLockNotFound) { + t.Fatalf("DeleteLock error = %v, want ErrLockNotFound", err) + } +} + +// TestCreateLockRetriesALostRace drives the compare-and-swap the way two +// concurrent clients would. The hook lands a competing lock between this +// caller's read and its write, so the first conditional write is refused and +// the loop has to re-read and try again. +func TestCreateLockRetriesALostRace(t *testing.T) { + f := newFakeS3() + s := newTestStore(t, f) + + var raced bool + f.putHook = func(key string) { + if raced || key != LocksKey(testRepo) { + return + } + raced = true + // A competing writer commits first, invalidating the ETag this caller + // is about to write against. + f.set(key, fakeObject{body: []byte(`{"locks":[{"id":"aa","path":"other.bin","owner":{"name":"bob"}}]}`)}) + } + + l := mustLock(t, s, "big.bin", "alice") + + if !raced { + t.Fatal("the racing write never ran, so no retry was exercised") + } + + // Both locks must survive: the retry re-read the competitor's document + // rather than overwriting it. + got, _, err := s.ListLocks(context.Background(), testRepo, LockFilter{}) + if err != nil { + t.Fatalf("ListLocks: %v", err) + } + if len(got) != 2 { + t.Fatalf("got %d locks, want 2 (the retry dropped a concurrent write)", len(got)) + } + if _, _, err := s.ListLocks(context.Background(), testRepo, LockFilter{ID: l.ID}); err != nil { + t.Fatalf("locking own lock back: %v", err) + } +} diff --git a/internal/lfs/presign.go b/internal/lfs/presign.go new file mode 100644 index 0000000..ddd9e74 --- /dev/null +++ b/internal/lfs/presign.go @@ -0,0 +1,129 @@ +package lfs + +import ( + "context" + "encoding/base64" + "encoding/hex" + "fmt" + "net/http" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +// Presigner mints the URLs a git-lfs client transfers against. Every LFS byte +// moves over one of these, so the daemon carries none of them. +// +// Uploads and downloads are signed by different clients on purpose. See +// NewPresigner. +type Presigner struct { + // putClient signs uploads with header hoisting off, so the checksum stays + // a signed header. + putClient *s3.PresignClient + // getClient signs downloads with the stock settings, so the URL works on + // its own with no extra request headers. + getClient *s3.PresignClient + bucket string +} + +// noHoistPresigner signs a presigned request without moving headers into the +// query string. +// +// By default the SigV4 presigner hoists any "X-Amz-*" header that is not on its +// required-signed list into the query string (see AllowedQueryHoisting in +// aws/signer/internal/v4/headers.go). "X-Amz-Checksum-Sha256" is not on that +// list, so the stock presigner would strip it out of the signature and leave +// SignedHeader empty. Git LFS sends only the headers the batch response names, +// so a hoisted checksum is a checksum nobody sends and nothing enforces. +// +// Disabling hoisting affects nothing else here. The parameters that make a URL +// presigned (X-Amz-Algorithm, X-Amz-Credential, X-Amz-Signature and the rest) +// are built by the signer itself, not hoisted from headers. +type noHoistPresigner struct{ signer *v4.Signer } + +func (p noHoistPresigner) PresignHTTP( + ctx context.Context, creds aws.Credentials, r *http.Request, + payloadHash string, service string, region string, signingTime time.Time, + optFns ...func(*v4.SignerOptions), +) (string, http.Header, error) { + optFns = append(optFns, func(o *v4.SignerOptions) { o.DisableHeaderHoisting = true }) + return p.signer.PresignHTTP(ctx, creds, r, payloadHash, service, region, signingTime, optFns...) +} + +// NewPresigner builds a Presigner over an S3 client. +// +// Pass the concrete client, not a hardened wrapper: presigning makes no network +// call, so the hardened transport buys nothing, and s3.NewPresignClient needs +// the concrete type. In cmd/objgitd that is the *s3.Client embedded in the +// Tigris client. +// +// Uploads and downloads get different signers. Turning off header hoisting is +// what keeps an upload's checksum inside the signature, but it applies to every +// "X-Amz-*" header, and a GetObject carries "x-amz-checksum-mode". Signing that +// one would oblige every download client to send it back, and a plain fetch of +// the URL would fail with SignatureDoesNotMatch. A download needs nothing +// signed beyond the host, so it uses the stock signer and the URL stands alone. +func NewPresigner(c *s3.Client, bucket string) *Presigner { + return &Presigner{ + putClient: s3.NewPresignClient(c, func(o *s3.PresignOptions) { + o.Presigner = noHoistPresigner{signer: v4.NewSigner()} + }), + getClient: s3.NewPresignClient(c), + bucket: bucket, + } +} + +// PresignGet returns a URL the client reads an object from, valid for ttl. +func (p *Presigner) PresignGet(ctx context.Context, key string, ttl time.Duration) (*v4.PresignedHTTPRequest, error) { + return p.getClient.PresignGetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(p.bucket), + Key: aws.String(key), + }, s3.WithPresignExpires(ttl)) +} + +// PresignPut returns a URL the client writes an object to, valid for ttl. +// +// The request carries the object's own SHA-256 as ChecksumSHA256, so the bucket +// rejects a body that does not hash to the key it is being written to. The +// caller must copy the returned SignedHeader into the batch response, because +// the signature covers those headers and the client has to send them back. +// +// The declared size is deliberately absent: Content-Length is not signed on a +// presigned PUT, so it cannot be enforced here. Size is checked afterwards, by +// Verify. +func (p *Presigner) PresignPut(ctx context.Context, key, oid string, ttl time.Duration) (*v4.PresignedHTTPRequest, error) { + if err := ValidateOID(oid); err != nil { + return nil, err + } + raw, err := hex.DecodeString(oid) + if err != nil { + return nil, fmt.Errorf("decoding oid %q: %w", oid, err) + } + return p.putClient.PresignPutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(p.bucket), + Key: aws.String(key), + ChecksumSHA256: aws.String(base64.StdEncoding.EncodeToString(raw)), + }, s3.WithPresignExpires(ttl)) +} + +// TransferHeader renders a presigned request's signed headers as the "header" +// map of a batch action. Host is dropped because the client sets it from the +// URL, and sending it twice breaks the request. +func TransferHeader(req *v4.PresignedHTTPRequest) map[string]string { + if req == nil { + return nil + } + out := make(map[string]string, len(req.SignedHeader)) + for k := range req.SignedHeader { + if http.CanonicalHeaderKey(k) == "Host" { + continue + } + out[k] = req.SignedHeader.Get(k) + } + if len(out) == 0 { + return nil + } + return out +} diff --git a/internal/lfs/presign_test.go b/internal/lfs/presign_test.go new file mode 100644 index 0000000..727343f --- /dev/null +++ b/internal/lfs/presign_test.go @@ -0,0 +1,157 @@ +package lfs + +import ( + "context" + "net/http" + "net/url" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +const testOID = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + +// newTestPresigner builds a Presigner over static credentials and a fixed +// endpoint. Presigning performs no network I/O — the signing middleware +// short-circuits the request stack — so this exercises the real signer, not a +// fake. That is deliberate: a fake presigner cannot catch a change in which +// headers the SDK hoists into the query string, which is the one thing these +// tests exist to pin. +func newTestPresigner(t *testing.T) *Presigner { + t.Helper() + c := s3.New(s3.Options{ + Region: "auto", + BaseEndpoint: aws.String("https://t3.storage.dev"), + Credentials: credentials.NewStaticCredentialsProvider( + "AKIAEXAMPLEEXAMPLE", "secretsecretsecret", ""), + }) + return NewPresigner(c, "test-bucket") +} + +// TestPresignPutSignsTheChecksumHeader is the regression test for SigV4 header +// hoisting. The AWS SDK moves any unrecognized "X-Amz-*" header into the query +// string when it presigns, which would leave the checksum unsigned and absent +// from SignedHeader. Git LFS only sends headers the batch response names, so a +// hoisted checksum is an unenforced checksum. +func TestPresignPutSignsTheChecksumHeader(t *testing.T) { + p := newTestPresigner(t) + + req, err := p.PresignPut(context.Background(), BlobKey(testOID), testOID, 15*time.Minute) + if err != nil { + t.Fatalf("PresignPut: %v", err) + } + + u, err := url.Parse(req.URL) + if err != nil { + t.Fatalf("parsing presigned URL: %v", err) + } + q := u.Query() + + signed := q.Get("X-Amz-SignedHeaders") + if !strings.Contains(signed, "x-amz-checksum-sha256") { + t.Errorf("X-Amz-SignedHeaders = %q, want it to contain x-amz-checksum-sha256", signed) + } + + if got := q.Get("x-amz-checksum-sha256"); got != "" { + t.Errorf("checksum was hoisted into the query string as %q; it must stay a signed header", got) + } + + if got := req.SignedHeader.Get("x-amz-checksum-sha256"); got == "" { + t.Error("SignedHeader is missing x-amz-checksum-sha256") + } +} + +// TestPresignPutChecksumMatchesTheOID pins the encoding: S3 wants the raw +// digest in base64, not the hex text of the oid. +func TestPresignPutChecksumMatchesTheOID(t *testing.T) { + p := newTestPresigner(t) + + req, err := p.PresignPut(context.Background(), BlobKey(testOID), testOID, 15*time.Minute) + if err != nil { + t.Fatalf("PresignPut: %v", err) + } + + // base64(hex-decode(testOID)), computed independently of the implementation: + // printf 'test' | shasum -a 256 -b | cut -d' ' -f1 | xxd -r -p | base64 + const want = "n4bQgYhMfWWaL+qgxVrQFaO/TxsrC4Is0V1sFbDwCgg=" + if got := req.SignedHeader.Get("x-amz-checksum-sha256"); got != want { + t.Errorf("checksum header = %q, want %q", got, want) + } +} + +func TestPresignGet(t *testing.T) { + p := newTestPresigner(t) + + req, err := p.PresignGet(context.Background(), BlobKey(testOID), 15*time.Minute) + if err != nil { + t.Fatalf("PresignGet: %v", err) + } + + if req.Method != "GET" { + t.Errorf("Method = %q, want GET", req.Method) + } + if !strings.Contains(req.URL, BlobKey(testOID)) { + t.Errorf("URL %q does not contain the blob key", req.URL) + } + // A download needs no checksum: the client verifies the bytes it receives + // against the oid it asked for. + if got := req.SignedHeader.Get("x-amz-checksum-sha256"); got != "" { + t.Errorf("download carries an unexpected checksum header %q", got) + } +} + +// TestPresignGetNeedsNoRequestHeaders is a regression test for a download URL +// that only works when the caller replays extra headers. +// +// Turning off header hoisting to keep an upload's checksum signed also signs +// "x-amz-checksum-mode" on a GetObject. A URL signed that way is rejected with +// SignatureDoesNotMatch unless the client sends that header back, so a plain +// fetch of the href fails. Only Host may be signed on a download. +func TestPresignGetNeedsNoRequestHeaders(t *testing.T) { + p := newTestPresigner(t) + + req, err := p.PresignGet(context.Background(), BlobKey(testOID), 15*time.Minute) + if err != nil { + t.Fatalf("PresignGet: %v", err) + } + + for name := range req.SignedHeader { + if http.CanonicalHeaderKey(name) != "Host" { + t.Errorf("download signs %q; a download URL must stand on its own", name) + } + } + if got := TransferHeader(req); got != nil { + t.Errorf("download action would demand headers %v, want none", got) + } +} + +// TestPresignExpiryReachesTheURL keeps -lfs-url-ttl honest: the value has to +// land in the signature, not just in the batch response's expires_in. +func TestPresignExpiryReachesTheURL(t *testing.T) { + p := newTestPresigner(t) + + req, err := p.PresignGet(context.Background(), BlobKey(testOID), 42*time.Second) + if err != nil { + t.Fatalf("PresignGet: %v", err) + } + + u, err := url.Parse(req.URL) + if err != nil { + t.Fatalf("parsing presigned URL: %v", err) + } + if got := u.Query().Get("X-Amz-Expires"); got != "42" { + t.Errorf("X-Amz-Expires = %q, want %q", got, "42") + } +} + +func TestPresignPutRejectsABadOID(t *testing.T) { + p := newTestPresigner(t) + + if _, err := p.PresignPut(context.Background(), "lfs/blobs/nope", "nope", time.Minute); err == nil { + t.Fatal("PresignPut accepted an invalid oid") + } +} diff --git a/internal/lfs/snapshot_test.go b/internal/lfs/snapshot_test.go new file mode 100644 index 0000000..e37f561 --- /dev/null +++ b/internal/lfs/snapshot_test.go @@ -0,0 +1,56 @@ +package lfs + +import ( + "context" + "io" + "strings" + "testing" +) + +func TestOpenVerified(t *testing.T) { + const repo = "acme/owned" + const other = "acme/other" + const oid = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + tests := []struct { + name string + repo string + markerSize int64 + marker bool + blob string + wantErr string + }{ + {name: "held object", repo: repo, marker: true, markerSize: 4, blob: "test"}, + {name: "another repository cannot read", repo: other, marker: true, markerSize: 4, blob: "test", wantErr: "does not hold"}, + {name: "missing marker", repo: repo, blob: "test", wantErr: "does not hold"}, + {name: "wrong marker size", repo: repo, marker: true, markerSize: 5, blob: "test", wantErr: "marker size"}, + {name: "missing blob", repo: repo, marker: true, markerSize: 4, wantErr: "get object"}, + {name: "wrong blob size", repo: repo, marker: true, markerSize: 4, blob: "short", wantErr: "pointer says"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + bucket := newFakeS3() + if tt.marker { + bucket.putMarker(repo, oid, tt.markerSize) + } + if tt.blob != "" { + bucket.set(BlobKey(oid), fakeObject{body: []byte(tt.blob)}) + } + store := NewStore(bucket, nil, "test-bucket") + r, err := store.OpenVerified(context.Background(), tt.repo, oid, 4) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("OpenVerified error = %v, want %q", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatal(err) + } + defer r.Close() + got, err := io.ReadAll(r) + if err != nil || string(got) != "test" { + t.Errorf("body = %q, %v; want test", got, err) + } + }) + } +} diff --git a/internal/lfs/store.go b/internal/lfs/store.go new file mode 100644 index 0000000..2c2f8ef --- /dev/null +++ b/internal/lfs/store.go @@ -0,0 +1,307 @@ +package lfs + +import ( + "context" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "io" + "log/slog" + "strconv" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" +) + +// metaSize is the user-metadata key holding an object's declared length. A +// membership marker has no body, so its size has to live in metadata. +const metaSize = "lfs-size" + +// s3API is the bucket surface this package needs. It is an interface so tests +// can supply an in-memory bucket: CI runs with no Tigris credentials. +type s3API interface { + HeadObject(context.Context, *s3.HeadObjectInput, ...func(*s3.Options)) (*s3.HeadObjectOutput, error) + GetObject(context.Context, *s3.GetObjectInput, ...func(*s3.Options)) (*s3.GetObjectOutput, error) + PutObject(context.Context, *s3.PutObjectInput, ...func(*s3.Options)) (*s3.PutObjectOutput, error) + // RenameObject is a Tigris extension, not standard S3. It moves an object + // in place, with no data copy, which is how a staged upload is promoted to + // the shared key. objgit already depends on it elsewhere (internal/s3fs). + RenameObject(context.Context, *s3.CopyObjectInput, ...func(*s3.Options)) (*s3.CopyObjectOutput, error) + DeleteObject(context.Context, *s3.DeleteObjectInput, ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) +} + +// Store reads and writes the LFS key space in one bucket. +type Store struct { + api s3API + pre *Presigner + bucket string +} + +// NewStore builds a Store over a bucket and the presigner that mints its +// transfer URLs. +func NewStore(api s3API, pre *Presigner, bucket string) *Store { + return &Store{api: api, pre: pre, bucket: bucket} +} + +// Presigner returns the presigner this store mints transfer URLs with. +func (s *Store) Presigner() *Presigner { return s.pre } + +// VerifyStatus is the outcome of checking an uploaded object. +type VerifyStatus int + +const ( + // VerifyMissing means no blob is stored under that oid. + VerifyMissing VerifyStatus = iota + // VerifyMismatch means the stored bytes are not the object claimed. + VerifyMismatch + // VerifyOK means the object is stored and the repository now holds it. + VerifyOK +) + +// String renders the status for logs and metrics. +func (v VerifyStatus) String() string { + switch v { + case VerifyMissing: + return "missing" + case VerifyMismatch: + return "mismatch" + case VerifyOK: + return "ok" + default: + return "unknown" + } +} + +// Marker reports whether the repository at repo holds oid, and the size it was +// recorded with. +// +// This reads the per-repository marker, never the shared blob. A repository +// that never uploaded an object does not hold it, however many other +// repositories do. Reading the blob here instead would let any caller turn a +// known oid into a presigned URL for someone else's data. +func (s *Store) Marker(ctx context.Context, repo, oid string) (int64, bool, error) { + if err := ValidateOID(oid); err != nil { + return 0, false, err + } + out, err := s.api.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(MarkerKey(repo, oid)), + }) + if err != nil { + if isNotFound(err) { + return 0, false, nil + } + return 0, false, fmt.Errorf("heading lfs marker for %s: %w", oid, err) + } + // A marker whose size is absent or unreadable is not a zero-byte object. + // Reporting it as one holds the oid at size 0 forever: every later upload + // batch answers 422 "stored with size 0", and because the object reads as + // held, batch never offers an upload action, so the client has no way back. + // Treating the marker as unusable costs one re-upload, and that re-upload + // rewrites the marker with a good size. The repository loses no access it + // could otherwise keep, because absence only ever denies. + raw := out.Metadata[metaSize] + size, perr := strconv.ParseInt(raw, 10, 64) + if perr != nil { + slog.Warn("lfs marker has an unusable size, treating the object as not held so it can be uploaded again", + "repo", repo, "oid", oid, "value", raw, "err", perr) + return 0, false, nil + } + return size, true, nil +} + +// OpenVerified reads an LFS object only after checking that repo holds a +// marker with the expected size. Snapshot builds use this to replace pointers +// without making the shared blob key a cross-repository read oracle. +func (s *Store) OpenVerified(ctx context.Context, repo, oid string, size int64) (io.ReadCloser, error) { + storedSize, held, err := s.Marker(ctx, repo, oid) + if err != nil { + return nil, err + } + if !held { + return nil, fmt.Errorf("lfs: repository %s does not hold object %s", repo, oid) + } + if storedSize != size { + return nil, fmt.Errorf("lfs: object %s marker size %d differs from pointer size %d", oid, storedSize, size) + } + out, err := s.api.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(BlobKey(oid)), + }) + if err != nil { + return nil, fmt.Errorf("lfs: get object %s: %w", oid, err) + } + if got := aws.ToInt64(out.ContentLength); got != size { + out.Body.Close() + return nil, fmt.Errorf("lfs: object %s has size %d, pointer says %d", oid, got, size) + } + return out.Body, nil +} + +// Verify promotes a staged upload and records that repo holds it. +// +// It reads the caller's own staging key, never the shared blob. That is what +// makes a marker mean something: to reach this point the caller had to write +// the bytes, and Tigris checked them against the oid as it did. Reading the +// shared blob instead would grant membership to anyone who can name an oid and +// its size, which is exactly what a pointer file contains. +// +// This is also the only size check anywhere. A presigned PUT does not sign +// Content-Length, so the declared size cannot be enforced at upload time. +// +// A mismatch deletes the staged object, which belongs to this caller. Nothing +// here can touch the shared blob or another repository's data. +func (s *Store) Verify(ctx context.Context, repo, oid string, size int64) (VerifyStatus, error) { + if err := ValidateOID(oid); err != nil { + return VerifyMissing, err + } + staged := StagingKey(repo, oid) + + out, err := s.api.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(staged), + ChecksumMode: types.ChecksumModeEnabled, + }) + if err != nil { + if isNotFound(err) { + return s.verifyPromoted(ctx, repo, oid) + } + return VerifyMissing, fmt.Errorf("heading staged lfs object %s: %w", oid, err) + } + + if aws.ToInt64(out.ContentLength) != size { + return s.rejectStaged(ctx, staged, oid) + } + // Tigris does not have to report a stored checksum. The PUT-time checksum + // already covered these bytes, so an absent one falls back to the size + // check rather than failing every upload. + if got := aws.ToString(out.ChecksumSHA256); got != "" { + want, err := checksumOf(oid) + if err != nil { + return VerifyMissing, err + } + if got != want { + return s.rejectStaged(ctx, staged, oid) + } + } + + // Promote the staged bytes to the shared key. + // + // RenameObject is a Tigris extension that moves an object in place. It + // copies no data and leaves nothing behind, so promoting a 5 GiB upload + // costs the same as promoting an empty one, and there is no staged copy to + // clean up afterwards. + // + // Overwriting an existing blob is safe and expected: the key is the hash of + // the content, so a blob another repository already uploaded holds exactly + // these bytes. That is why two repositories uploading the same object need + // no coordination. + if _, err := s.api.RenameObject(ctx, &s3.CopyObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(BlobKey(oid)), + CopySource: aws.String(s.bucket + "/" + staged), + }); err != nil { + // The staging key went away between the head and the rename. Two + // verifies for the same object race in normal use: git-lfs retries a + // failed verify, and a client may re-POST. The first one consumes the + // staging key, so the second finds nothing to promote. That is the same + // state as an upload already promoted, and not a server fault. + if isNotFound(err) { + return s.verifyPromoted(ctx, repo, oid) + } + return VerifyMissing, fmt.Errorf("promoting staged lfs object %s: %w", oid, err) + } + + if err := s.putMarker(ctx, repo, oid, size); err != nil { + return VerifyMissing, err + } + return VerifyOK, nil +} + +// verifyPromoted answers a verify that found no staged bytes. Either the upload +// never happened, or an earlier verify already promoted it, so the marker +// decides. +// +// A bucket failure reading the marker is an error and never absence. Reporting +// it as VerifyMissing answers the client 404 "object was not uploaded", which +// sends it to re-upload an object that is already there, and records "missing" +// on the verify metric, which hides the fault from the alert. +func (s *Store) verifyPromoted(ctx context.Context, repo, oid string) (VerifyStatus, error) { + _, held, err := s.Marker(ctx, repo, oid) + if err != nil { + return VerifyMissing, err + } + if held { + return VerifyOK, nil + } + return VerifyMissing, nil +} + +// rejectStaged deletes a staged object whose bytes are not what they claim. +// The key is the caller's own, so this can never affect another repository. +func (s *Store) rejectStaged(ctx context.Context, staged, oid string) (VerifyStatus, error) { + _, err := s.api.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(staged), + }) + if err != nil { + return VerifyMismatch, fmt.Errorf("deleting mismatched staged lfs object %s: %w", oid, err) + } + return VerifyMismatch, nil +} + +// putMarker records that repo holds oid. The marker has no body; its size lives +// in user metadata so a later batch can answer an upload without reading the +// blob. +func (s *Store) putMarker(ctx context.Context, repo, oid string, size int64) error { + _, err := s.api.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(MarkerKey(repo, oid)), + Body: strings.NewReader(""), + Metadata: map[string]string{metaSize: strconv.FormatInt(size, 10)}, + }) + if err != nil { + return fmt.Errorf("writing lfs marker for %s: %w", oid, err) + } + return nil +} + +// checksumOf renders an oid as the base64 raw digest S3 compares against. +func checksumOf(oid string) (string, error) { + raw, err := hex.DecodeString(oid) + if err != nil { + return "", fmt.Errorf("decoding oid %q: %w", oid, err) + } + return base64.StdEncoding.EncodeToString(raw), nil +} + +// isNotFound reports whether err is a bucket miss. It matches the codes +// internal/storage/tigris matches: Tigris reports Head misses as NotFound and +// Get misses as NoSuchKey. +func isNotFound(err error) bool { + var apiErr smithy.APIError + if !errors.As(err, &apiErr) { + return false + } + switch apiErr.ErrorCode() { + case "NotFound", "NoSuchKey": + return true + default: + return false + } +} + +// isPreconditionFailed reports whether err is a rejected conditional write. +// Two writers racing on the lock document is the normal way to see this, so +// the lock writer retries rather than failing. Never map it to absence. +func isPreconditionFailed(err error) bool { + var apiErr smithy.APIError + if !errors.As(err, &apiErr) { + return false + } + return apiErr.ErrorCode() == "PreconditionFailed" +} diff --git a/internal/lfs/store_test.go b/internal/lfs/store_test.go new file mode 100644 index 0000000..1f26309 --- /dev/null +++ b/internal/lfs/store_test.go @@ -0,0 +1,413 @@ +package lfs + +import ( + "context" + "encoding/base64" + "encoding/hex" + "errors" + "testing" +) + +const testRepo = "acme/widgets" + +func testChecksum(t *testing.T, oid string) string { + t.Helper() + raw, err := hex.DecodeString(oid) + if err != nil { + t.Fatalf("decoding oid: %v", err) + } + return base64.StdEncoding.EncodeToString(raw) +} + +// TestMarkerReportsMembership covers the read the batch endpoint makes for +// every object. +func TestMarkerReportsMembership(t *testing.T) { + for _, tt := range []struct { + name string + seed func(*fakeS3) + wantOK bool + wantSize int64 + }{ + { + name: "no marker and no blob", + seed: func(*fakeS3) {}, + wantOK: false, + }, + { + name: "marker present", + seed: func(f *fakeS3) { + f.putMarker(testRepo, testOID, 1234) + }, + wantOK: true, + wantSize: 1234, + }, + { + // The access model rests on this case: the bytes exist, but this + // repository never proved it holds them, so the answer is no. + name: "blob present but marker absent", + seed: func(f *fakeS3) { + f.putBlob(testOID, 1234, "") + }, + wantOK: false, + }, + { + name: "marker belongs to another repository", + seed: func(f *fakeS3) { + f.putMarker("other/repo", testOID, 1234) + f.putBlob(testOID, 1234, "") + }, + wantOK: false, + }, + { + // An empty file is a real object, and its size parses, so it is + // held like any other. + name: "marker records a zero size", + seed: func(f *fakeS3) { + f.putMarker(testRepo, testOID, 0) + }, + wantOK: true, + wantSize: 0, + }, + { + // A marker with no readable size cannot answer an upload batch. If + // it read as size 0 the oid would be held at a size nothing + // matches, batch would answer 422 forever, and it would never offer + // an upload action to fix it. Reading it as absent costs one + // re-upload, which rewrites the marker. + name: "marker has no recorded size", + seed: func(f *fakeS3) { + f.set(MarkerKey(testRepo, testOID), fakeObject{}) + }, + wantOK: false, + }, + { + name: "marker has a malformed size", + seed: func(f *fakeS3) { + f.set(MarkerKey(testRepo, testOID), fakeObject{ + meta: map[string]string{metaSize: "four"}, + }) + }, + wantOK: false, + }, + } { + t.Run(tt.name, func(t *testing.T) { + f := newFakeS3() + tt.seed(f) + s := newTestStore(t, f) + + size, ok, err := s.Marker(context.Background(), testRepo, testOID) + if err != nil { + t.Fatalf("Marker: %v", err) + } + if ok != tt.wantOK { + t.Fatalf("Marker ok = %v, want %v", ok, tt.wantOK) + } + if ok && size != tt.wantSize { + t.Errorf("Marker size = %d, want %d", size, tt.wantSize) + } + }) + } +} + +// TestMarkerSeparatesAFaultFromAbsence keeps a bucket that is down from +// reading as "this repository does not hold the object". Every batch decision +// rests on this answer, so absence has to mean absence. +func TestMarkerSeparatesAFaultFromAbsence(t *testing.T) { + f := newFakeS3() + f.headErr = errBucketUnavailable + s := newTestStore(t, f) + + _, held, err := s.Marker(context.Background(), testRepo, testOID) + if !errors.Is(err, errBucketUnavailable) { + t.Fatalf("Marker error = %v, want errBucketUnavailable", err) + } + if held { + t.Error("Marker reported a held object from a failed read") + } +} + +func TestMarkerRejectsABadOID(t *testing.T) { + s := newTestStore(t, newFakeS3()) + + if _, _, err := s.Marker(context.Background(), testRepo, "../../etc/passwd"); !errors.Is(err, ErrInvalidOID) { + t.Fatalf("Marker error = %v, want ErrInvalidOID", err) + } +} + +// TestVerify covers the only integrity check that runs on the daemon's own +// credentials. A presigned PUT is not size-checked by the signature, so this is +// where a wrong upload is caught. +func TestVerify(t *testing.T) { + for _, tt := range []struct { + name string + seed func(*testing.T, *fakeS3) + size int64 + want VerifyStatus + wantMarker bool + wantBlob bool + }{ + { + name: "size and checksum match", + seed: func(t *testing.T, f *fakeS3) { + f.putStaged(testRepo, testOID, 4, testChecksum(t, testOID)) + }, + size: 4, + want: VerifyOK, + wantMarker: true, + wantBlob: true, + }, + { + name: "nothing was staged", + seed: func(*testing.T, *fakeS3) {}, + size: 4, + want: VerifyMissing, + wantBlob: false, + }, + { + name: "size differs", + seed: func(t *testing.T, f *fakeS3) { + f.putStaged(testRepo, testOID, 9, testChecksum(t, testOID)) + }, + size: 4, + want: VerifyMismatch, + // Wrong bytes are dropped from the caller's own staging area and + // never reach the shared key. + wantBlob: false, + }, + { + name: "checksum differs", + seed: func(t *testing.T, f *fakeS3) { + f.putStaged(testRepo, testOID, 4, base64.StdEncoding.EncodeToString(make([]byte, 32))) + }, + size: 4, + want: VerifyMismatch, + wantBlob: false, + }, + { + // Tigris may answer a HEAD without a stored checksum. The PUT-time + // checksum already covered the bytes, so size alone is accepted + // rather than failing every upload. + name: "no checksum reported falls back to the size check", + seed: func(t *testing.T, f *fakeS3) { + f.putStaged(testRepo, testOID, 4, "") + }, + size: 4, + want: VerifyOK, + wantMarker: true, + wantBlob: true, + }, + } { + t.Run(tt.name, func(t *testing.T) { + f := newFakeS3() + tt.seed(t, f) + s := newTestStore(t, f) + + got, err := s.Verify(context.Background(), testRepo, testOID, tt.size) + if err != nil { + t.Fatalf("Verify: %v", err) + } + if got != tt.want { + t.Fatalf("Verify = %v, want %v", got, tt.want) + } + if f.has(MarkerKey(testRepo, testOID)) != tt.wantMarker { + t.Errorf("marker present = %v, want %v", !tt.wantMarker, tt.wantMarker) + } + if f.has(BlobKey(testOID)) != tt.wantBlob { + t.Errorf("blob present = %v, want %v", !tt.wantBlob, tt.wantBlob) + } + // A promoted upload leaves no staged copy behind, and a rejected one + // is removed too, so staging is empty either way. + if f.has(StagingKey(testRepo, testOID)) { + t.Error("the staged object was left behind") + } + }) + } +} + +// TestVerifyRecordsTheSizeOnTheMarker keeps a later batch able to answer an +// upload of the same oid without reading the blob. +func TestVerifyRecordsTheSizeOnTheMarker(t *testing.T) { + f := newFakeS3() + f.putStaged(testRepo, testOID, 4, testChecksum(t, testOID)) + s := newTestStore(t, f) + + if _, err := s.Verify(context.Background(), testRepo, testOID, 4); err != nil { + t.Fatalf("Verify: %v", err) + } + + size, ok, err := s.Marker(context.Background(), testRepo, testOID) + if err != nil { + t.Fatalf("Marker: %v", err) + } + if !ok { + t.Fatal("Verify did not write the marker") + } + if size != 4 { + t.Errorf("marker size = %d, want 4", size) + } +} + +// TestVerifyToleratesADuplicate covers two verifies racing for the same +// object. This happens in normal use: git-lfs retries a failed verify, and a +// client may re-POST. The first call consumes the staging key, so the second +// finds nothing to promote. The marker answers it. None of these is a server +// fault, so none of them may return an error. +func TestVerifyToleratesADuplicate(t *testing.T) { + // promote is what the competing verify does: it moves the staged bytes to + // the shared key and records that the repository holds them. + promote := func(t *testing.T, f *fakeS3, staged string) { + f.putBlob(testOID, 4, testChecksum(t, testOID)) + f.drop(staged) + f.putMarker(testRepo, testOID, 4) + } + + for _, tt := range []struct { + name string + seed func(*testing.T, *fakeS3) + want VerifyStatus + }{ + { + // The whole race already finished before this call started. + name: "the earlier verify finished first", + seed: func(t *testing.T, f *fakeS3) { + promote(t, f, StagingKey(testRepo, testOID)) + }, + want: VerifyOK, + }, + { + // The staging key survives this call's head and is gone by its + // rename. Before the fix this was the 500. + name: "the rename loses the race after the marker is written", + seed: func(t *testing.T, f *fakeS3) { + f.putStaged(testRepo, testOID, 4, testChecksum(t, testOID)) + f.renameHook = func(src string) { promote(t, f, src) } + }, + want: VerifyOK, + }, + { + // The competing verify has promoted the bytes but has not written + // the marker yet. Nothing here can tell that apart from an upload + // that never happened, so the client is told to retry, which is a + // 404 and not a 500. + name: "the rename loses the race before the marker is written", + seed: func(t *testing.T, f *fakeS3) { + f.putStaged(testRepo, testOID, 4, testChecksum(t, testOID)) + f.renameHook = func(src string) { + f.putBlob(testOID, 4, testChecksum(t, testOID)) + f.drop(src) + } + }, + want: VerifyMissing, + }, + } { + t.Run(tt.name, func(t *testing.T) { + f := newFakeS3() + tt.seed(t, f) + s := newTestStore(t, f) + + got, err := s.Verify(context.Background(), testRepo, testOID, 4) + if err != nil { + t.Fatalf("Verify: %v", err) + } + if got != tt.want { + t.Fatalf("Verify = %v, want %v", got, tt.want) + } + // Whoever won the race, the bytes are under the shared key. + if !f.has(BlobKey(testOID)) { + t.Error("the promoted blob is gone") + } + }) + } +} + +// TestVerifyReportsAFailedMarkerRead keeps a bucket fault from being answered +// as a missing object. A 404 sends git-lfs to re-upload an object that is +// already there, and the verify metric records "missing" rather than "error", +// which is the signal an operator alerts on. +func TestVerifyReportsAFailedMarkerRead(t *testing.T) { + for _, tt := range []struct { + name string + seed func(*testing.T, *fakeS3) + }{ + { + name: "nothing is staged", + seed: func(*testing.T, *fakeS3) {}, + }, + { + name: "the rename found nothing to promote", + seed: func(t *testing.T, f *fakeS3) { + f.putStaged(testRepo, testOID, 4, testChecksum(t, testOID)) + f.renameHook = func(src string) { f.drop(src) } + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + f := newFakeS3() + tt.seed(t, f) + // The staging read answers normally; only the marker read fails. + f.headHook = func(key string) error { + if key == MarkerKey(testRepo, testOID) { + return errBucketUnavailable + } + return nil + } + s := newTestStore(t, f) + + status, err := s.Verify(context.Background(), testRepo, testOID, 4) + if !errors.Is(err, errBucketUnavailable) { + t.Fatalf("Verify = (%v, %v), want errBucketUnavailable", status, err) + } + }) + } +} + +// TestVerifyNeedsProofOfPossession is the security property the whole marker +// split exists for. Knowing an oid must not be enough to join a repository to +// an object: an oid and its size sit side by side in every pointer file, and +// pointer files travel through forks and CI logs. +// +// A caller who never uploaded anything must not be able to name someone else's +// object and come away holding it. +func TestVerifyNeedsProofOfPossession(t *testing.T) { + f := newFakeS3() + // Another repository already uploaded this object, so the bytes are in the + // bucket under the shared key. + f.putBlob(testOID, 4, testChecksum(t, testOID)) + f.putMarker("victim/private", testOID, 4) + + s := newTestStore(t, f) + + // The attacker uploaded nothing. It only knows the oid and the size. + status, err := s.Verify(context.Background(), "attacker/scratch", testOID, 4) + if err != nil { + t.Fatalf("Verify: %v", err) + } + if status == VerifyOK { + t.Fatal("verify granted membership to a caller that never uploaded the object") + } + + if _, held, err := s.Marker(context.Background(), "attacker/scratch", testOID); err != nil { + t.Fatalf("Marker: %v", err) + } else if held { + t.Fatal("the attacker now holds an object it never uploaded") + } +} + +// TestVerifyDoesNotDeleteTheSharedBlob covers the other half: a bad verify must +// not be able to destroy an object other repositories depend on. +func TestVerifyDoesNotDeleteTheSharedBlob(t *testing.T) { + f := newFakeS3() + f.putBlob(testOID, 4, testChecksum(t, testOID)) + f.putMarker("victim/private", testOID, 4) + + s := newTestStore(t, f) + + // A wrong declared size is the reject path. + if _, err := s.Verify(context.Background(), "attacker/scratch", testOID, 999); err != nil { + t.Fatalf("Verify: %v", err) + } + + if !f.has(BlobKey(testOID)) { + t.Fatal("a failed verify deleted the blob another repository holds") + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 7599762..1239353 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -202,6 +202,42 @@ var ( Name: "cas_retries_total", Help: "Packed-refs compare-and-swap attempts that lost a race and were retried.", }) + + lfsBatchRequests = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "lfs", + Name: "batch_requests_total", + Help: "Git LFS batch requests, by operation and outcome.", + }, []string{"operation", "status"}) + + lfsBatchDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: "lfs", + Name: "batch_duration_seconds", + Help: "Time to answer a Git LFS batch request.", + Buckets: prometheus.DefBuckets, + }, []string{"operation"}) + + lfsObjects = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "lfs", + Name: "objects_total", + Help: "Objects named in a Git LFS batch, by operation and outcome (new, present, missing, error).", + }, []string{"operation", "result"}) + + lfsVerify = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "lfs", + Name: "verify_total", + Help: "Git LFS verify calls, by outcome (ok, missing, mismatch, invalid, error).", + }, []string{"status"}) + + lfsLockRequests = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "lfs", + Name: "lock_requests_total", + Help: "Git LFS locking API calls, by action and outcome.", + }, []string{"action", "status"}) ) // ObserveS3 records one S3/Tigris API call. Its signature matches the observer @@ -389,3 +425,54 @@ func (c *listingCacheCollector) Collect(ch chan<- prometheus.Metric) { ch <- prometheus.MustNewConstMetric(lcItems, prometheus.GaugeValue, float64(s.SubtreeItems), "subtree") ch <- prometheus.MustNewConstMetric(lcItems, prometheus.GaugeValue, float64(s.HeadItems), "head") } + +// ObserveLFSBatch records a completed Git LFS batch request. operation is +// "upload" or "download", and status is "ok", "denied", "invalid", +// "too_many_objects", or "error". start is when the handler began. +func ObserveLFSBatch(operation, status string, start time.Time) { + operation = lfsOperationLabel(operation) + lfsBatchRequests.WithLabelValues(operation, status).Inc() + lfsBatchDuration.WithLabelValues(operation).Observe(time.Since(start).Seconds()) +} + +// ObserveLFSObject records one object's outcome inside a batch. +// +// On an upload batch, the ratio of result="present" to result="new" is the +// deduplication rate: "present" means the repository already held the object +// and the client transferred nothing. Judge the feature by that ratio, not by +// batch_requests_total. +func ObserveLFSObject(operation, result string) { + lfsObjects.WithLabelValues(lfsOperationLabel(operation), result).Inc() +} + +// lfsOperationLabel clamps a client-supplied operation to the two values the +// protocol defines. +// +// The batch body is read before authorization, so an unauthenticated caller +// chooses this string. Passing it through would let anyone mint a new label +// value per request and grow the metric registry without bound. +func lfsOperationLabel(operation string) string { + switch operation { + case "upload", "download": + return operation + default: + return "unknown" + } +} + +// ObserveLFSVerify records a Git LFS verify call. +// +// status="mismatch" is the one to alert on. A presigned upload is not +// size-checked by its signature, so verify is the only place wrong bytes are +// caught, and a rising mismatch rate means clients are writing objects that do +// not hash to the id they claim. +func ObserveLFSVerify(status string) { + lfsVerify.WithLabelValues(status).Inc() +} + +// ObserveLFSLock records a Git LFS locking API call. action is "create", +// "list", "verify", or "unlock". status is "ok", "invalid", "conflict", +// "too_many", "not_found", "forbidden", "contended", or "error". +func ObserveLFSLock(action, status string) { + lfsLockRequests.WithLabelValues(action, status).Inc() +} diff --git a/internal/repofs/repofs.go b/internal/repofs/repofs.go index 9a92d67..c46c7e6 100644 --- a/internal/repofs/repofs.go +++ b/internal/repofs/repofs.go @@ -25,6 +25,18 @@ var ErrInvalidPath = errors.New("repository path must be of the form {orgID}/{re // authentication challenge (HTTP 401) rather than a 404 or 500. var ErrUnauthenticated = errors.New("repofs: authentication required") +// ErrReservedPath is returned by Parse when a repository path would collide +// with a key prefix the daemon reserves for itself. +var ErrReservedPath = errors.New("repofs: repository path is reserved") + +// reservedOrgs are the org IDs a repository may not use, because the daemon +// stores its own data under those bucket prefixes. A repository is scoped to +// "orgID/name/", so an org of "lfs" would put a repository's keys underneath +// the LFS blob prefix and make any sweep of it eat a repository. +var reservedOrgs = map[string]struct{}{ + "lfs": {}, +} + // RepoRef identifies a repository. OrgID is an opaque reference a later API call // will validate; for now it is accepted as-is. Name has any trailing ".git" // stripped, so org/repo.git and org/repo denote the same repository. @@ -39,6 +51,12 @@ func (r RepoRef) Path() string { return path.Join(r.OrgID, r.Name) } // Parse converts a raw transport path into a RepoRef. It trims surrounding // slashes, requires exactly two non-empty segments, and strips a trailing // ".git" from the name. OrgID is not otherwise validated. +// +// Parse rejects the prefixes the daemon reserves for its own state with +// ErrReservedPath: a dot-prefixed segment (which holds the SSH host key under +// ".objgit/"), and a reserved org ID (which holds LFS blobs under "lfs/"). +// Both checks live here rather than in a transport, so every transport gets +// them. func Parse(raw string) (RepoRef, error) { parts := strings.Split(strings.Trim(raw, "/"), "/") if len(parts) != 2 || parts[0] == "" || parts[1] == "" { @@ -48,6 +66,12 @@ func Parse(raw string) (RepoRef, error) { if name == "" { return RepoRef{}, ErrInvalidPath } + if strings.HasPrefix(parts[0], ".") || strings.HasPrefix(name, ".") { + return RepoRef{}, ErrReservedPath + } + if _, ok := reservedOrgs[parts[0]]; ok { + return RepoRef{}, ErrReservedPath + } return RepoRef{OrgID: parts[0], Name: name}, nil } diff --git a/internal/repofs/repofs_test.go b/internal/repofs/repofs_test.go index b7e0e2a..fbf9ce1 100644 --- a/internal/repofs/repofs_test.go +++ b/internal/repofs/repofs_test.go @@ -63,6 +63,31 @@ func TestParse(t *testing.T) { input: "acme/.git", wantErr: ErrInvalidPath, }, + { + name: "reserved org lfs", + input: "lfs/blobs", + wantErr: ErrReservedPath, + }, + { + name: "reserved org lfs with .git", + input: "lfs/objects.git", + wantErr: ErrReservedPath, + }, + { + name: "dot-prefixed org", + input: ".objgit/widgets", + wantErr: ErrReservedPath, + }, + { + name: "dot-prefixed name", + input: "acme/.hidden", + wantErr: ErrReservedPath, + }, + { + name: "lfs is fine as a repository name", + input: "acme/lfs", + want: RepoRef{OrgID: "acme", Name: "lfs"}, + }, } for _, tt := range tests { diff --git a/internal/snapshot/ensure.go b/internal/snapshot/ensure.go index 9d8edd2..b230d30 100644 --- a/internal/snapshot/ensure.go +++ b/internal/snapshot/ensure.go @@ -11,6 +11,7 @@ import ( "os" "path" "strconv" + "strings" "time" "github.com/Xe/erofs" @@ -56,6 +57,16 @@ type Result struct { // Ensure is idempotent. Two concurrent calls for one tree both build and put // identical bytes, which is correct, so it takes no lock. func Ensure(ctx context.Context, objs storer.EncodedObjectStorer, store Store, tree plumbing.Hash, tmpDir string) (Result, error) { + return EnsureWithLFS(ctx, objs, store, tree, tmpDir, nil) +} + +// LFSOpener opens a verified LFS object belonging to the repository being +// snapshotted. The returned reader must yield exactly size bytes. +type LFSOpener func(ctx context.Context, oid string, size int64) (io.ReadCloser, error) + +// EnsureWithLFS builds a checkout view of tree, replacing Git LFS pointers +// with their object bytes. A pointer without an opener fails the build. +func EnsureWithLFS(ctx context.Context, objs storer.EncodedObjectStorer, store Store, tree plumbing.Hash, tmpDir string, openLFS LFSOpener) (Result, error) { start := time.Now() key := Key(tree) res := Result{Key: key} @@ -98,7 +109,7 @@ func Ensure(ctx context.Context, objs storer.EncodedObjectStorer, store Store, t opts = append(opts, erofs.WithSpoolDir(tmpDir)) } b := erofs.NewBuilder(f, opts...) - w := &walker{ctx: ctx, objs: objs, b: b} + w := &walker{ctx: ctx, objs: objs, b: b, openLFS: openLFS} if err := w.addTree("/", root); err != nil { return res, err } @@ -144,10 +155,11 @@ func hashFile(f *os.File) (string, int64, error) { } type walker struct { - ctx context.Context - objs storer.EncodedObjectStorer - b *erofs.Builder - files int + ctx context.Context + objs storer.EncodedObjectStorer + b *erofs.Builder + files int + openLFS LFSOpener } // addTree adds every entry of t under dir, depth first. It walks tree @@ -185,11 +197,29 @@ func (w *walker) addTree(dir string, t *object.Tree) error { if err != nil { return fmt.Errorf("snapshot: %s: blob size: %w", p, err) } + open := w.opener(e.Hash) + if size > 0 && size < 1024 { + data, err := w.blob(p, e.Hash) + if err != nil { + return err + } + oid, objectSize, pointer, err := parseLFSPointer(data) + if err != nil { + return fmt.Errorf("snapshot: %s: %w", p, err) + } + if pointer { + if w.openLFS == nil { + return fmt.Errorf("snapshot: %s: LFS object %s has no object store", p, oid) + } + size = objectSize + open = func() (io.ReadCloser, error) { return w.openLFS(w.ctx, oid, objectSize) } + } + } perm := fs.FileMode(0o644) if e.Mode == filemode.Executable { perm = 0o755 } - if err := w.b.AddFileFunc(p, info{e.Name, perm}, size, w.opener(e.Hash)); err != nil { + if err := w.b.AddFileFunc(p, info{e.Name, perm}, size, open); err != nil { return fmt.Errorf("snapshot: %s: %w", p, err) } w.files++ @@ -217,6 +247,35 @@ func (w *walker) addTree(dir string, t *object.Tree) error { return nil } +// parseLFSPointer accepts the canonical v1 pointer without extensions. +// Extensions require a client-side smudge filter, so building their raw +// stored bytes would silently make an incorrect checkout image. +func parseLFSPointer(data []byte) (oid string, size int64, pointer bool, err error) { + const version = "version https://git-lfs.github.com/spec/v1\n" + if !strings.HasPrefix(string(data), version) { + return "", 0, false, nil + } + lines := strings.Split(string(data), "\n") + if len(lines) != 4 || lines[3] != "" || !strings.HasPrefix(lines[1], "oid sha256:") || !strings.HasPrefix(lines[2], "size ") { + return "", 0, true, errors.New("unsupported or malformed LFS pointer") + } + oid = strings.TrimPrefix(lines[1], "oid sha256:") + if len(oid) != 64 { + return "", 0, true, errors.New("invalid LFS object id") + } + for _, c := range oid { + if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')) { + return "", 0, true, errors.New("invalid LFS object id") + } + } + rawSize := strings.TrimPrefix(lines[2], "size ") + size, err = strconv.ParseInt(rawSize, 10, 64) + if err != nil || size < 0 || strconv.FormatInt(size, 10) != rawSize { + return "", 0, true, errors.New("invalid LFS object size") + } + return oid, size, true, nil +} + // opener returns the source AddFileFunc calls during Build. It loads the blob // only then, so the object is garbage once its reader closes. func (w *walker) opener(h plumbing.Hash) func() (io.ReadCloser, error) { diff --git a/internal/snapshot/ensure_test.go b/internal/snapshot/ensure_test.go index 6960c06..22f0a1a 100644 --- a/internal/snapshot/ensure_test.go +++ b/internal/snapshot/ensure_test.go @@ -3,8 +3,11 @@ package snapshot import ( "bytes" "context" + "crypto/sha256" + "encoding/hex" "errors" "fmt" + "io" "io/fs" "os" "strings" @@ -191,6 +194,77 @@ func TestEnsureCompresses(t *testing.T) { } } +func TestEnsureResolvesLFSPointers(t *testing.T) { + content := []byte("actual LFS file contents\n") + digest := sha256.Sum256(content) + oid := hex.EncodeToString(digest[:]) + pointer := fmt.Sprintf("version https://git-lfs.github.com/spec/v1\noid sha256:%s\nsize %d\n", oid, len(content)) + tests := []struct { + name string + blob string + mode filemode.FileMode + opener bool + openErr error + want string + wantErr string + }{ + {name: "regular pointer", blob: pointer, mode: filemode.Regular, opener: true, want: string(content)}, + {name: "executable pointer", blob: pointer, mode: filemode.Executable, opener: true, want: string(content)}, + {name: "ordinary file", blob: "ordinary\n", mode: filemode.Regular, want: "ordinary\n"}, + {name: "missing object store", blob: pointer, mode: filemode.Regular, wantErr: "no object store"}, + {name: "missing LFS object", blob: pointer, mode: filemode.Regular, opener: true, openErr: fs.ErrNotExist, wantErr: "file does not exist"}, + {name: "malformed size", blob: strings.Replace(pointer, fmt.Sprintf("size %d", len(content)), "size -1", 1), mode: filemode.Regular, opener: true, wantErr: "invalid LFS object size"}, + {name: "extension", blob: strings.Replace(pointer, "oid sha256:", "ext-0-foo sha256:"+oid+"\noid sha256:", 1), mode: filemode.Regular, opener: true, wantErr: "unsupported or malformed LFS pointer"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + st := memory.NewStorage() + tree := buildTree(t, st, []fixture{{"file.bin", tt.mode, tt.blob}}) + store := NewMemStore() + var open LFSOpener + if tt.opener { + open = func(_ context.Context, gotOID string, gotSize int64) (io.ReadCloser, error) { + if gotOID != oid || gotSize != int64(len(content)) { + t.Errorf("open LFS %s, %d; want %s, %d", gotOID, gotSize, oid, len(content)) + } + if tt.openErr != nil { + return nil, tt.openErr + } + return io.NopCloser(bytes.NewReader(content)), nil + } + } + _, err := EnsureWithLFS(context.Background(), st, store, tree, t.TempDir(), open) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("EnsureWithLFS error = %v, want %q", err, tt.wantErr) + } + if store.PutCount() != 0 { + t.Error("stored an incomplete image") + } + return + } + if err != nil { + t.Fatalf("EnsureWithLFS: %v", err) + } + snap, err := Open(context.Background(), store, tree) + if err != nil { + t.Fatal(err) + } + defer snap.Close() + got, err := fs.ReadFile(snap, "file.bin") + if err != nil || string(got) != tt.want { + t.Errorf("file.bin = %q, %v; want %q", got, err, tt.want) + } + if tt.mode == filemode.Executable { + info, err := fs.Stat(snap, "file.bin") + if err != nil || info.Mode().Perm() != 0o755 { + t.Errorf("file.bin mode = %v, %v; want 0755", info, err) + } + } + }) + } +} + func TestEnsureDeterministic(t *testing.T) { entries := []fixture{ {"a.txt", filemode.Regular, strings.Repeat("a", 9000)}, @@ -233,7 +307,7 @@ func TestEnsureMetadata(t *testing.T) { if !ok { t.Fatal("no object at Key(tree)") } - want := map[string]string{MetaFormat: "1", MetaTree: tree.String(), MetaFiles: "2"} + want := map[string]string{MetaFormat: "2", MetaTree: tree.String(), MetaFiles: "2"} for k, v := range want { if meta[k] != v { t.Errorf("meta[%q] = %q, want %q", k, meta[k], v) diff --git a/internal/snapshot/snapshot.go b/internal/snapshot/snapshot.go index 91f869b..3b894e5 100644 --- a/internal/snapshot/snapshot.go +++ b/internal/snapshot/snapshot.go @@ -18,7 +18,7 @@ import ( // FormatVersion names the mapping from git to EROFS and the build options. // Changing either one is a new version, and a new key, so an image written // under one version is never overwritten by another. -const FormatVersion = 1 +const FormatVersion = 2 // Object metadata keys carried by every image. Lowercase, because S3 returns // user metadata keys lowercased. @@ -37,7 +37,7 @@ func Key(tree plumbing.Hash) string { return fmt.Sprintf("%sv%d/%s.erofs", keyPrefix, FormatVersion, tree) } -// CacheID returns the local cache id for key: "erofs-v1-". It holds no +// CacheID returns the local cache id for key: "erofs-v2-". It holds no // repository prefix, so two repositories with one tree share one cached file. func CacheID(key string) string { id := strings.TrimSuffix(strings.TrimPrefix(key, keyPrefix), ".erofs")