diff --git a/AGENTS.md b/AGENTS.md index 83b8dc8..6dc41e1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,6 +59,7 @@ Notes on the tests and on configuration: | `cmd/objgitd/shell.go` | The SSH `sh` command: an interactive shell in the hook sandbox. | | `cmd/objgitd/receivepack.go` | The go-git fork that streams hook output, plus `writePack`. | | `cmd/objgitd/hooks.go` | Ref diffing and the sandboxed hook run. | +| `cmd/objgitd/snapshots.go` | The erofs snapshot run after a push. | | `internal/auth` | The one authorization interface. | | `internal/repofs` | Maps a repository path to a `storage.Storer`. | | `internal/storage/tigris` | Repository storage. A `storage.Storer` on the bucket. | @@ -66,6 +67,7 @@ Notes on the tests and on configuration: | `internal/s3fs` | Daemon-level state only, which is the SSH host key. | | `internal/mountfs`, `internal/treefs`, `internal/kefkash` | The hook sandbox filesystem and shell wiring. | | `internal/metrics` | Every Prometheus vector, plus thin helpers. | +| `internal/snapshot` | erofs images of git trees: `Ensure`, `Open`, and the `Store` interface. | | `internal/slog.go` | JSON handler init. | | `cmd/membench/` | Push memory benchmark harness. Not shipped; see `docs/usage/memory-benchmark.md`. | @@ -80,6 +82,7 @@ describes the daemon and links to one page for each subsystem. | [auth.md](docs/architecture/auth.md) | Credentials, decisions, or a new `Authorizer`. | | [hooks.md](docs/architecture/hooks.md) | Push hooks, output streaming, or the sandbox. | | [metrics.md](docs/architecture/metrics.md) | Any metric or instrumentation seam. | +| [snapshots.md](docs/architecture/snapshots.md) | Snapshot images, the snapshot cache, or `runSnapshots`. | | [tigris-storer.md](docs/architecture/tigris-storer.md) | Object layout, refs, packs, the pack cache, or the upload path. | | [s3fs.md](docs/architecture/s3fs.md) | The `billy.Filesystem` over the bucket. | diff --git a/cmd/membench/main.go b/cmd/membench/main.go index 316edea..904c641 100644 --- a/cmd/membench/main.go +++ b/cmd/membench/main.go @@ -62,6 +62,7 @@ var ( daemonHTTPBind = flag.String("daemon-http-bind", "127.0.0.1:8080", "address the daemon under test serves smart HTTP on") daemonMetrics = flag.String("daemon-metrics-bind", "127.0.0.1:9090", "address the daemon under test serves /metrics and /debug/pprof on") daemonAllowHooks = flag.Bool("daemon-allow-hooks", false, "run push hooks in the daemon under test; off by default so hook cost is not mistaken for push cost") + daemonSnapshots = flag.Bool("daemon-erofs-snapshots", false, "build erofs snapshots in the daemon under test; off by default so snapshot cost is not mistaken for push cost") daemonReadyWait = flag.Duration("daemon-ready-wait", 60*time.Second, "how long to wait for the daemon to answer /metrics before giving up") slogLevel = flag.String("slog-level", "INFO", "log level (DEBUG, INFO, WARN, ERROR)") @@ -137,6 +138,7 @@ func run(ctx context.Context) error { "-ssh-bind=", "-allow-push", fmt.Sprintf("-allow-hooks=%t", *daemonAllowHooks), + fmt.Sprintf("-erofs-snapshots=%t", *daemonSnapshots), "-pack-cache-dir", packCache, } diff --git a/cmd/objgitd/git_protocol.go b/cmd/objgitd/git_protocol.go index 4f335e2..e9cdfed 100644 --- a/cmd/objgitd/git_protocol.go +++ b/cmd/objgitd/git_protocol.go @@ -54,6 +54,12 @@ type daemon struct { allowHooks bool hookTimeout time.Duration + // snapshots gates building an erofs image of each updated ref tip after + // a push; see snapshots.go. + snapshots bool + snapshotTimeout time.Duration + snapshotTmpDir string // parent directory for image temp files; "" is the OS temp dir + // pushes bounds how many pushes unpack a packfile at once, which is the // only thing that bounds the daemon's resident set under concurrent pushes. // A nil pushes is unlimited, which is what every test that does not care diff --git a/cmd/objgitd/hooks.go b/cmd/objgitd/hooks.go index 31d23a3..84e592f 100644 --- a/cmd/objgitd/hooks.go +++ b/cmd/objgitd/hooks.go @@ -45,8 +45,9 @@ type refUpdate struct { } // receivePack runs the receive-pack service and dispatches post-receive work -// for the commands whose ref updates succeeded. Hook output streams to the -// client before the response closes. Webhook delivery also finishes before the +// for the commands whose ref updates succeeded: erofs snapshots of the updated +// branch and tag tips (see runSnapshots), then hooks, then webhooks. Snapshot +// and hook output streams to the client before the response closes. Webhook delivery also finishes before the // response closes, but cannot reject an already accepted push. // // This is also the one place every push funnels through — smart HTTP and SSH @@ -57,7 +58,7 @@ func (d *daemon) receivePack(ctx context.Context, st storage.Storer, repoPath st if settingsErr != nil { slog.Error("webhook: load repository settings", "repo", repoPath, "err", settingsErr) } - if !d.allowHooks && (webhooks == nil || !webhooks.Enabled(repoPath)) { + if !d.allowHooks && !d.snapshots && (webhooks == nil || !webhooks.Enabled(repoPath)) { err := receivePackStreaming(ctx, st, r, w, req, d.pushes.admit, nil) d.healHEADAfterPush(err, st, repoPath) return err @@ -67,6 +68,9 @@ func (d *daemon) receivePack(ctx context.Context, st storage.Storer, repoPath st // updates are taken from this request's successful commands, so another // concurrent push cannot be attributed to this one. onUpdated := func(progress io.Writer, updates []refUpdate, acceptedAt time.Time) { + if d.snapshots { + d.runSnapshots(repoPath, st, updates, progress) + } if d.allowHooks { var branches []refUpdate for _, u := range updates { @@ -102,8 +106,8 @@ func (d *daemon) healHEADAfterPush(recvErr error, st storage.Storer, repoPath st // streaming each hook's output to progress (nil = log only). func (d *daemon) runHooks(repoPath, service string, st storage.Storer, updates []refUpdate, progress io.Writer) { for _, u := range updates { - if u.New.IsZero() { - continue // branch deletion: nothing to check out + if u.New.IsZero() || !u.Name.IsBranch() { + continue // a deletion, or a tag: hooks run for branches only } d.runHook(repoPath, service, st, u, progress) } diff --git a/cmd/objgitd/main.go b/cmd/objgitd/main.go index 975b40c..3f54211 100644 --- a/cmd/objgitd/main.go +++ b/cmd/objgitd/main.go @@ -46,6 +46,11 @@ var ( packCacheDir = flag.String("pack-cache-dir", "", "parent directory for the local pack cache; empty uses the OS temp directory") packCacheBytes = flag.Int64("pack-cache-bytes", 2<<30, "disk budget for the local pack cache, least-recently-used eviction; 0 disables caching") + erofsSnapshots = flag.Bool("erofs-snapshots", true, "build a zstd-compressed erofs image of the tree at each updated branch and tag tip after a push, and store it next to the repository; the push waits for the build") + snapshotTimeout = flag.Duration("snapshot-timeout", 2*time.Minute, "wall-clock limit for the erofs snapshots of one push") + snapshotCacheBytes = flag.Int64("snapshot-cache-bytes", 2<<30, "disk budget for the local erofs snapshot cache, least-recently-used eviction; 0 disables caching") + snapshotCacheMaxIdle = flag.Duration("snapshot-cache-max-idle", time.Hour, "evict a cached erofs snapshot that nobody opened for this long; 0 disables the idle sweep") + packCompression = flag.Bool("pack-compression", true, "store zstd-compressed payloads in newly written pack containers; reading compressed containers is always enabled, so this is safe to turn off for one release before a rollback") packedRefs = flag.Bool("packed-refs", true, "write every ref into one packed-refs object under a compare-and-swap, instead of one object per ref; reading packed refs is always enabled, so this is safe to turn off for one release before a rollback") @@ -131,6 +136,18 @@ func main() { storerOpts = append(storerOpts, tigris.WithPackCache(packCache)) } + // The snapshot cache sits next to the pack cache, under the same parent, + // with its own budget. snapshot.Open downloads whole images into it. + var snapCache *tigris.PackCache + if *snapshotCacheBytes > 0 { + snapCache, err = tigris.NewSnapshotCache(*packCacheDir, *snapshotCacheBytes, metrics.ObserveSnapshotCache) + if err != nil { + slog.Error("can't create snapshot cache", "pack_cache_dir", *packCacheDir, "err", err) + os.Exit(1) + } + storerOpts = append(storerOpts, tigris.WithSnapshotCache(snapCache)) + } + // Every repository lives in the same bucket as daemon system state, keyed // by an "orgID/name" prefix (repofs.BucketResolver via tigrisBase.Scoped). base, err := tigris.New(ctx, *bucket, storerOpts...) @@ -146,6 +163,10 @@ func main() { allowHooks: *allowHooks, hookTimeout: *hookTimeout, pushes: newPushLimiter(*maxConcurrentPushes, *pushQueueTimeout), + + snapshots: *erofsSnapshots, + snapshotTimeout: *snapshotTimeout, + snapshotTmpDir: *packCacheDir, } slog.Info("objgitd listening", @@ -157,12 +178,33 @@ func main() { "allow_push", *allowPush, "allow_hooks", *allowHooks, "pack_cache_bytes", *packCacheBytes, + "erofs_snapshots", *erofsSnapshots, + "snapshot_cache_bytes", *snapshotCacheBytes, "max_concurrent_pushes", *maxConcurrentPushes, "push_queue_timeout", *pushQueueTimeout, ) g, gCtx := errgroup.WithContext(ctx) + // The budget alone only evicts when a new image arrives. This sweep also + // cycles out images that nobody opened for a while. + if snapCache != nil && *snapshotCacheMaxIdle > 0 { + g.Go(func() error { + tick := time.NewTicker(*snapshotCacheMaxIdle / 4) + defer tick.Stop() + for { + select { + case <-gCtx.Done(): + return nil + case <-tick.C: + if n := snapCache.EvictIdle(*snapshotCacheMaxIdle); n > 0 { + slog.Debug("evicted idle snapshots", "count", n) + } + } + } + }) + } + if *metricsBind != "" { ln, err := net.Listen("tcp", *metricsBind) if err != nil { @@ -233,6 +275,9 @@ func main() { if cerr := packCache.Cleanup(); cerr != nil { slog.Warn("can't remove the pack cache directory", "err", cerr) } + if cerr := snapCache.Cleanup(); cerr != nil { + slog.Warn("can't remove the snapshot cache directory", "err", cerr) + } if err != nil { slog.Error("server stopped", "err", err) diff --git a/cmd/objgitd/snapshots.go b/cmd/objgitd/snapshots.go new file mode 100644 index 0000000..c491a29 --- /dev/null +++ b/cmd/objgitd/snapshots.go @@ -0,0 +1,148 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "slices" + "strings" + "time" + + "github.com/go-git/go-git/v6/plumbing" + "github.com/go-git/go-git/v6/plumbing/object" + "github.com/go-git/go-git/v6/plumbing/storer" + "github.com/go-git/go-git/v6/storage" + "github.com/tigrisdata/objgit/internal/metrics" + "github.com/tigrisdata/objgit/internal/snapshot" +) + +// maxTagDepth bounds how many annotated tags peelToTree follows. +const maxTagDepth = 16 + +// runSnapshots makes sure that an erofs image exists for the tree at the tip +// of each updated branch and tag. It runs synchronously inside onUpdated, so +// the push waits for it, and it writes one progress line per ref. A failure is logged +// and counted and never fails the push: the refs are already committed. +func (d *daemon) runSnapshots(repoPath string, st storage.Storer, updates []refUpdate, progress io.Writer) { + store, ok := st.(snapshot.Store) + if !ok { + slog.Debug("snapshot: storer holds no snapshot store, skipping", "repo", repoPath) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), d.snapshotTimeout) + defer cancel() + + // Branches go first, then tags, each by name, so the ref that builds a + // shared tree (and the progress output) is the same on every push, however + // the client ordered its commands. + ordered := slices.Clone(updates) + slices.SortFunc(ordered, func(a, b refUpdate) int { + if a.Name.IsBranch() != b.Name.IsBranch() { + if a.Name.IsBranch() { + return -1 + } + return 1 + } + return strings.Compare(a.Name.String(), b.Name.String()) + }) + + // A branch and a tag on one commit share one tree, and one build. + seen := map[plumbing.Hash]string{} + for _, u := range ordered { + if u.New.IsZero() || !(u.Name.IsBranch() || u.Name.IsTag()) { + continue // a deletion, or a ref such as refs/notes/* + } + log := slog.With("repo", repoPath, "ref", u.Name.String(), "sha", u.New.String()) + + tree, ok, err := peelToTree(st, u.New) + if err != nil { + log.Error("snapshot: resolve tree", "err", err) + metrics.ObserveSnapshot("error", 0, 0) + continue + } + if !ok { + log.Debug("snapshot: ref does not point at a commit, skipping") + continue + } + if status, dup := seen[tree]; dup { + if status == snapshot.StatusBuilt { + status = snapshot.StatusExists + } + writeSnapshotLine(progress, u.Name, tree, status) + continue + } + + res, err := snapshot.Ensure(ctx, st, store, tree, d.snapshotTmpDir) + if err != nil { + log.Error("snapshot: build failed", "tree", tree.String(), "dur", res.Elapsed, "err", err) + metrics.ObserveSnapshot("error", res.Elapsed, 0) + seen[tree] = "failed" + writeSnapshotLine(progress, u.Name, tree, "failed") + continue + } + metrics.ObserveSnapshot(res.Status, res.Elapsed, res.Bytes) + seen[tree] = res.Status + if res.Status == snapshot.StatusBuilt { + log.Info("snapshot: built", "key", res.Key, "files", res.Files, "bytes", res.Bytes, "dur", res.Elapsed) + writeSnapshotLine(progress, u.Name, tree, fmt.Sprintf("built, %d files, %s, %s", + res.Files, humanBytes(res.Bytes), res.Elapsed.Round(100*time.Millisecond))) + continue + } + writeSnapshotLine(progress, u.Name, tree, res.Status) + } +} + +// writeSnapshotLine writes one "remote: objgit: snapshot ..." line. It never +// carries error text, which can name bucket internals; the log has that. +func writeSnapshotLine(progress io.Writer, ref plumbing.ReferenceName, tree plumbing.Hash, status string) { + if progress == nil { + return + } + fmt.Fprintf(progress, "objgit: snapshot %s (tree %s): %s\n", ref, tree.String()[:7], status) +} + +// peelToTree returns the tree of the commit that h names, following annotated +// tags. ok is false, with no error, when h does not lead to a commit (a tag of +// a tree or a blob, or a tree itself), since there is then nothing to snapshot. +func peelToTree(st storer.EncodedObjectStorer, h plumbing.Hash) (plumbing.Hash, bool, error) { + for range maxTagDepth { + obj, err := st.EncodedObject(plumbing.AnyObject, h) + if err != nil { + return plumbing.ZeroHash, false, fmt.Errorf("load %s: %w", h, err) + } + switch obj.Type() { + case plumbing.CommitObject: + c, err := object.DecodeCommit(st, obj) + if err != nil { + return plumbing.ZeroHash, false, fmt.Errorf("decode commit %s: %w", h, err) + } + return c.TreeHash, true, nil + case plumbing.TagObject: + tag, err := object.DecodeTag(st, obj) + if err != nil { + return plumbing.ZeroHash, false, fmt.Errorf("decode tag %s: %w", h, err) + } + h = tag.Target + default: + return plumbing.ZeroHash, false, nil + } + } + return plumbing.ZeroHash, false, errors.New("tag chain is too deep") +} + +// humanBytes formats n in binary units with one decimal, such as "5.6 MiB". +func humanBytes(n int64) string { + const unit = 1024 + if n < unit { + return fmt.Sprintf("%d B", n) + } + div, exp := int64(unit), 0 + for m := n / unit; m >= unit; m /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp]) +} diff --git a/cmd/objgitd/snapshots_test.go b/cmd/objgitd/snapshots_test.go new file mode 100644 index 0000000..af07dca --- /dev/null +++ b/cmd/objgitd/snapshots_test.go @@ -0,0 +1,311 @@ +package main + +import ( + "bytes" + "context" + "errors" + "io/fs" + "log/slog" + "net/http/httptest" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/go-git/go-billy/v6/memfs" + "github.com/go-git/go-git/v6/plumbing" + "github.com/go-git/go-git/v6/plumbing/filemode" + "github.com/go-git/go-git/v6/plumbing/object" + "github.com/go-git/go-git/v6/storage" + "github.com/go-git/go-git/v6/storage/memory" + "github.com/tigrisdata/objgit/internal/auth" + "github.com/tigrisdata/objgit/internal/repofs" + "github.com/tigrisdata/objgit/internal/snapshot" +) + +// snapStorer is an in-memory storer that also holds snapshot images, the way +// *tigris.Storer does. +type snapStorer struct { + *memory.Storage + *snapshot.MemStore +} + +// snapBase is memBase with a snapshot store for each repository. +type snapBase struct { + mu sync.Mutex + repos map[string]*snapStorer +} + +func newSnapBase() *snapBase { return &snapBase{repos: map[string]*snapStorer{}} } + +func (b *snapBase) Scoped(prefix string) storage.Storer { + b.mu.Lock() + defer b.mu.Unlock() + st, ok := b.repos[prefix] + if !ok { + st = &snapStorer{Storage: memory.NewStorage(), MemStore: snapshot.NewMemStore()} + b.repos[prefix] = st + } + return st +} + +func (b *snapBase) repo(prefix string) *snapStorer { + b.mu.Lock() + defer b.mu.Unlock() + return b.repos[prefix] +} + +func newSnapshotDaemon(t *testing.T, base repofs.Base, on bool) *httptest.Server { + t.Helper() + d := &daemon{ + sysFS: memfs.New(), + resolver: repofs.BucketResolver{Base: base}, + authz: auth.AllowAnonymous{AllowWrite: true}, + snapshots: on, + snapshotTimeout: 30 * time.Second, + snapshotTmpDir: t.TempDir(), + } + ts := httptest.NewServer(d.httpHandler()) + t.Cleanup(ts.Close) + return ts +} + +func TestSmartHTTPPushSnapshots(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + base := newSnapBase() + ts := newSnapshotDaemon(t, base, true) + + work := seedRepo(t) + writeFile(t, filepath.Join(work, "README.md"), "hello snapshot\n") + writeFile(t, filepath.Join(work, "docs", "a.md"), "nested\n") + runGit(t, work, "add", ".") + runGit(t, work, "commit", "-m", "content") + runGit(t, work, "tag", "-a", "v1.0.0", "-m", "v1") + + out, err := tryGit(work, "push", ts.URL+"/acme/snap.git", "main", "v1.0.0") + if err != nil { + t.Fatalf("push: %v\n%s", err, out) + } + treeHex := strings.TrimSpace(runGit(t, work, "rev-parse", "HEAD^{tree}")) + tree := plumbing.NewHash(treeHex) + + repo := base.repo("acme/snap") + if keys := repo.Keys(); len(keys) != 1 || keys[0] != snapshot.Key(tree) { + t.Fatalf("snapshot keys = %v, want [%s]", keys, snapshot.Key(tree)) + } + snap, err := snapshot.Open(context.Background(), repo, tree) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer snap.Close() + got, err := fs.ReadFile(snap, "docs/a.md") + if err != nil || string(got) != "nested\n" { + t.Errorf("docs/a.md = %q, %v", got, err) + } + for _, want := range []string{ + "objgit: snapshot refs/heads/main (tree " + treeHex[:7] + "): built", + "objgit: snapshot refs/tags/v1.0.0 (tree " + treeHex[:7] + "): exists", + } { + if !strings.Contains(out, want) { + t.Errorf("push output missing %q:\n%s", want, out) + } + } +} + +func TestPushSnapshotsOff(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + base := newSnapBase() + ts := newSnapshotDaemon(t, base, false) + work := seedRepo(t) + if out, err := tryGit(work, "push", ts.URL+"/acme/off.git", "main"); err != nil { + t.Fatalf("push: %v\n%s", err, out) + } + if keys := base.repo("acme/off").Keys(); len(keys) != 0 { + t.Errorf("snapshot keys = %v, want none", keys) + } +} + +func TestRunSnapshotsPutFailure(t *testing.T) { + st := &snapStorer{Storage: memory.NewStorage(), MemStore: snapshot.NewMemStore()} + st.PutErr = errors.New("bucket down") + commit := writeCommit(t, st.Storage) + + d := &daemon{snapshots: true, snapshotTimeout: 10 * time.Second, snapshotTmpDir: t.TempDir()} + var progress bytes.Buffer + d.runSnapshots("acme/x", st, []refUpdate{{Name: plumbing.NewBranchReferenceName("main"), New: commit}}, &progress) + + if !strings.Contains(progress.String(), "): failed") { + t.Errorf("progress = %q, want a failed line", progress.String()) + } + if strings.Contains(progress.String(), "bucket down") { + t.Errorf("progress leaks the error text: %q", progress.String()) + } +} + +func TestRunSnapshotsSkipsStorerWithoutStore(t *testing.T) { + st := memory.NewStorage() + commit := writeCommit(t, st) + d := &daemon{snapshots: true, snapshotTimeout: 10 * time.Second} + var progress bytes.Buffer + d.runSnapshots("acme/x", st, []refUpdate{{Name: plumbing.NewBranchReferenceName("main"), New: commit}}, &progress) + if progress.Len() != 0 { + t.Errorf("progress = %q, want nothing", progress.String()) + } +} + +func TestPeelToTree(t *testing.T) { + st := memory.NewStorage() + commit := writeCommit(t, st) + c, err := object.GetCommit(st, commit) + if err != nil { + t.Fatal(err) + } + tagOfCommit := writeTag(t, st, "v1", commit, plumbing.CommitObject) + tagOfTag := writeTag(t, st, "v1-again", tagOfCommit, plumbing.TagObject) + tagOfTree := writeTag(t, st, "tree-tag", c.TreeHash, plumbing.TreeObject) + + tests := []struct { + name string + h plumbing.Hash + want plumbing.Hash + wantOK bool + }{ + {"commit", commit, c.TreeHash, true}, + {"tag of commit", tagOfCommit, c.TreeHash, true}, + {"tag of tag", tagOfTag, c.TreeHash, true}, + {"tag of tree", tagOfTree, plumbing.ZeroHash, false}, + {"tree", c.TreeHash, plumbing.ZeroHash, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok, err := peelToTree(st, tt.h) + if err != nil { + t.Fatalf("peelToTree: %v", err) + } + if ok != tt.wantOK || got != tt.want { + t.Errorf("peelToTree = %s, %v; want %s, %v", got, ok, tt.want, tt.wantOK) + } + }) + } +} + +func TestHumanBytes(t *testing.T) { + tests := []struct { + n int64 + want string + }{ + {0, "0 B"}, + {1023, "1023 B"}, + {1024, "1.0 KiB"}, + {5*1024*1024 + 600*1024, "5.6 MiB"}, + {3 << 30, "3.0 GiB"}, + } + for _, tt := range tests { + if got := humanBytes(tt.n); got != tt.want { + t.Errorf("humanBytes(%d) = %q, want %q", tt.n, got, tt.want) + } + } +} + +// TestHooksIgnoreTags pins that a pushed tag does not run the hook, now that +// every successful ref command, tags included, reaches onUpdated. +func TestHooksIgnoreTags(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not installed") + } + + var logBuf syncBuffer + prev := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelDebug}))) + defer slog.SetDefault(prev) + + ts := httptest.NewServer((&daemon{ + sysFS: memfs.New(), + resolver: repofs.BucketResolver{Base: newMemBase()}, + authz: auth.AllowAnonymous{AllowWrite: true}, + allowHooks: true, + hookTimeout: 30 * time.Second, + }).httpHandler()) + t.Cleanup(ts.Close) + + work := seedRepo(t) + writeFile(t, filepath.Join(work, ".objgit", "hooks", "receive-pack"), "echo hook_ran\n") + runGit(t, work, "add", ".") + runGit(t, work, "commit", "-m", "with hook") + if out, err := tryGit(work, "push", ts.URL+"/acme/tags.git", "main"); err != nil { + t.Fatalf("push main: %v\n%s", err, out) + } + if n := strings.Count(logBuf.String(), "hook: running"); n != 1 { + t.Fatalf("hook ran %d times after the branch push, want 1", n) + } + + runGit(t, work, "tag", "-a", "v1", "-m", "v1") + if out, err := tryGit(work, "push", ts.URL+"/acme/tags.git", "v1"); err != nil { + t.Fatalf("push tag: %v\n%s", err, out) + } + if n := strings.Count(logBuf.String(), "hook: running"); n != 1 { + t.Errorf("hook ran %d times after the tag push, want 1", n) + } +} + +// writeCommit stores a commit with a one-file tree and returns its hash. +func writeCommit(t *testing.T, st *memory.Storage) plumbing.Hash { + t.Helper() + blob := st.NewEncodedObject() + blob.SetType(plumbing.BlobObject) + w, err := blob.Writer() + if err != nil { + t.Fatal(err) + } + if _, err := w.Write([]byte("hi\n")); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + bh, err := st.SetEncodedObject(blob) + if err != nil { + t.Fatal(err) + } + tree := &object.Tree{Entries: []object.TreeEntry{{Name: "a", Mode: filemode.Regular, Hash: bh}}} + to := st.NewEncodedObject() + if err := tree.Encode(to); err != nil { + t.Fatal(err) + } + th, err := st.SetEncodedObject(to) + if err != nil { + t.Fatal(err) + } + sig := object.Signature{Name: "T", Email: "t@example.com", When: time.Unix(0, 0)} + c := &object.Commit{Author: sig, Committer: sig, Message: "m", TreeHash: th} + co := st.NewEncodedObject() + if err := c.Encode(co); err != nil { + t.Fatal(err) + } + ch, err := st.SetEncodedObject(co) + if err != nil { + t.Fatal(err) + } + return ch +} + +func writeTag(t *testing.T, st *memory.Storage, name string, target plumbing.Hash, typ plumbing.ObjectType) plumbing.Hash { + t.Helper() + sig := object.Signature{Name: "T", Email: "t@example.com", When: time.Unix(0, 0)} + tag := &object.Tag{Name: name, Tagger: sig, Message: name, TargetType: typ, Target: target} + o := st.NewEncodedObject() + if err := tag.Encode(o); err != nil { + t.Fatal(err) + } + h, err := st.SetEncodedObject(o) + if err != nil { + t.Fatal(err) + } + return h +} diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 504eb9e..0693a17 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -10,11 +10,11 @@ daemon through all three transports at the same time. The struct holds three fields, plus the hook configuration: -| Field | Type | Purpose | -| ---------- | -------------------- | ------------------------------------------- | -| `sysFS` | `billy.Filesystem` | Daemon-level state only, which is the SSH host key. Still `internal/s3fs`. | -| `resolver` | `repofs.Resolver` | Maps a repository path to its storage. | -| `authz` | `auth.Authorizer` | One decision point for every transport. | +| Field | Type | Purpose | +| ---------- | ------------------ | -------------------------------------------------------------------------- | +| `sysFS` | `billy.Filesystem` | Daemon-level state only, which is the SSH host key. Still `internal/s3fs`. | +| `resolver` | `repofs.Resolver` | Maps a repository path to its storage. | +| `authz` | `auth.Authorizer` | One decision point for every transport. | `resolver.Resolve` returns a go-git `storage.Storer` directly. No `billy.Filesystem` is involved in repository data. The production resolver is @@ -30,14 +30,15 @@ Three behaviors live on `*daemon`, so every transport acts the same way: ## Pages -| Page | Contents | -| ------------------------------------ | ---------------------------------------------------------------- | +| Page | Contents | +| ------------------------------------ | ---------------------------------------------------------------------------- | | [transports.md](transports.md) | Smart HTTP, git://, and SSH. Two protocol points that are easy to get wrong. | -| [auth.md](auth.md) | The one authorization interface that every transport calls. | -| [hooks.md](hooks.md) | Push hooks, webhook dispatch, live output streaming, and the kefka sandbox. | -| [metrics.md](metrics.md) | Prometheus vectors and the three instrumentation seams. | -| [tigris-storer.md](tigris-storer.md) | Repository storage: a `storage.Storer` on one Tigris bucket. | -| [s3fs.md](s3fs.md) | Daemon-level state: a `billy.Filesystem` on Tigris. | +| [auth.md](auth.md) | The one authorization interface that every transport calls. | +| [hooks.md](hooks.md) | Push hooks, webhook dispatch, live output streaming, and the kefka sandbox. | +| [metrics.md](metrics.md) | Prometheus vectors and the three instrumentation seams. | +| [snapshots.md](snapshots.md) | erofs images of pushed trees, and the snapshot cache. | +| [tigris-storer.md](tigris-storer.md) | Repository storage: a `storage.Storer` on one Tigris bucket. | +| [s3fs.md](s3fs.md) | Daemon-level state: a `billy.Filesystem` on Tigris. | Related documents outside this directory: diff --git a/docs/architecture/metrics.md b/docs/architecture/metrics.md index 49f14ce..1cb36a4 100644 --- a/docs/architecture/metrics.md +++ b/docs/architecture/metrics.md @@ -18,18 +18,20 @@ repository names have unbounded cardinality. Git operations are keyed by The package exposes thin helpers, so no call site carries label plumbing: -| Helper | Use | -| ----------------- | -------------------------------- | -| `ObserveS3` | The s3fs observer. | -| `ObserveGitOp` | One git operation. | -| `TrackInFlight` | Returns a deferred decrement. | -| `ObserveAuth` | Maps the `auth` enums to labels. | -| `ObserveHook` | One hook run. | -| `ObserveWebhook` | One push webhook event. | -| `ReposCreated` | A new repository. | -| `TrackPushWait` | Returns a deferred decrement. | -| `TrackPushSlot` | Returns a deferred decrement. | -| `ObservePushWait` | One wait for a push slot. | +| Helper | Use | +| ---------------------- | -------------------------------- | +| `ObserveS3` | The s3fs observer. | +| `ObserveGitOp` | One git operation. | +| `TrackInFlight` | Returns a deferred decrement. | +| `ObserveAuth` | Maps the `auth` enums to labels. | +| `ObserveHook` | One hook run. | +| `ObserveWebhook` | One push webhook event. | +| `ReposCreated` | A new repository. | +| `TrackPushWait` | Returns a deferred decrement. | +| `TrackPushSlot` | Returns a deferred decrement. | +| `ObservePushWait` | One wait for a push slot. | +| `ObserveSnapshot` | One snapshot Ensure call. | +| `ObserveSnapshotCache` | One snapshot cache event. | ## The push queue @@ -52,6 +54,20 @@ The outcome label separates the two ways a wait ends badly. `timeout` means the client waited out `-push-queue-timeout`, which is the signal that the cap is too low. `canceled` means the client hung up while queued, which is not. +## Snapshots + +Snapshot Ensure calls and cache events have five series: + +| Series | Type | Labels | Meaning | +| ---------------------------------------- | --------- | ------ | ------------------------------------ | +| `objgit_snapshot_builds_total` | counter | result | Ensure calls by result. | +| `objgit_snapshot_build_duration_seconds` | histogram | (none) | Time for builds. | +| `objgit_snapshot_image_bytes` | histogram | (none) | Built image size after compression. | +| `objgit_snapshot_cache_opens_total` | counter | result | Cache lookups by result (hit, miss). | +| `objgit_snapshot_cache_evictions_total` | counter | reason | Evictions by reason (budget, idle). | + +The size histogram is data for a later decision about a size limit. + ## Three instrumentation seams ### s3fs diff --git a/docs/architecture/snapshots.md b/docs/architecture/snapshots.md new file mode 100644 index 0000000..21cafbe --- /dev/null +++ b/docs/architecture/snapshots.md @@ -0,0 +1,180 @@ +# erofs snapshots (`internal/snapshot`) + +After a push, `objgitd` can build one EROFS image of the tree at the tip of +each updated branch and tag. It stores the image in the bucket, under the +prefix of the repository. A later reader, such as a web UI, opens the image +and reads files from it with no git object walk. + +The `-erofs-snapshots` flag controls this. It is on by default, and +`-erofs-snapshots=false` turns it off. The images come from +`github.com/Xe/erofs`, and they use Zstandard compression. + +The design is in +[../superpowers/specs/2026-09-23-erofs-snapshots-design.md](../superpowers/specs/2026-09-23-erofs-snapshots-design.md). + +## Object layout + +| Key | Contents | +| --------------------------------- | ----------------------------------------- | +| `snapshots/erofs/v1/.erofs` | One EROFS image of the git tree ``. | + +The key uses the tree hash, not the commit hash. Many commits can point at one +tree, such as a branch and a tag on one commit. These commits share one image. + +The builder pins every mtime to the Unix epoch. As a result, two builds of one +tree give identical bytes, and a second build is never necessary. + +`v1` names the mapping from git to EROFS, and the build options. A change to +either one uses `v2`. A `v2` build never overwrites a `v1` image. + +The key does not start with `objects/`, `packs/`, or `refs/`. The pack index, +the loose object reads, and the loose ref listing therefore never see it. + +Each image carries user metadata: + +| Metadata key | Value | +| -------------- | ---------------------------------------------------------------------------------- | +| `erofs-format` | `1` | +| `erofs-sha256` | The hex SHA-256 of the image. The cache rejects a download that does not match it. | +| `git-tree` | The hex tree hash. | +| `erofs-files` | The count of regular files. | + +## The mapping from git to EROFS + +| Git entry mode | EROFS entry | +| ---------------- | ---------------------------------------- | +| `040000` tree | Directory, mode `0755`. | +| `100644` blob | Regular file, mode `0644`. | +| `100664` blob | Regular file, mode `0644`. | +| `100755` blob | Regular file, mode `0755`. | +| `120000` symlink | Symlink. The target is the blob content. | +| `160000` gitlink | Empty directory, mode `0755`. | + +The gitlink rule gives the same result as `git checkout` of a submodule that +is not initialized. + +Two EROFS limits are stricter than git: + +| Limit | EROFS | Checked by | +| ------------------- | ---------- | ------------------------------------------------------------- | +| Entry name length | 255 bytes | `Ensure`. The builder does not check it. | +| Symlink target size | 1020 bytes | `Ensure`. The builder writes more, but the reader refuses it. | + +If an entry is over a limit, the build fails with an error that names the +path. `Ensure` does not skip the entry, because an image with a missing file +is worse than no image. + +## `Ensure` + +`snapshot.Ensure` is the one unit of work. The push path calls it now. A web +UI, a backfill, or a queue consumer can call it later with no change. + +1. It calls `StatSnapshot`. If the image exists, the result is `exists`. +2. It walks the tree entries, depth first, and adds each one to an + `erofs.Builder` on a temp file. +3. It builds the image, computes its SHA-256, and calls `PutSnapshot`. + +The walk uses tree entries and not `tree.Files()`. The `FileIter` of go-git +skips gitlinks. + +Two pushes of one tree can run `Ensure` at the same time. Both build, and both +put identical bytes. This is correct, so `Ensure` takes no lock. The temp file +name is random for the same reason. + +`Ensure` streams file content into the builder. The walk reads only the +size of each blob, through `EncodedObjectSize`. It gives the builder the size +and an open function with `AddFileFunc`. During `Build`, the builder opens +each blob one time, compresses it one pcluster at a time, and spools the +compressed blocks to a temp file (`WithSpoolDir`, the same directory as the +image). As a result, the memory for a build does not depend on the size of +the tree. `TestEnsureMemory` builds 256 MiB of content with a live heap of +less than 64 MiB. + +This needs `github.com/Xe/erofs` v0.8.0 or later. The image bytes are the same +as the bytes that v0.6.1 wrote for the same tree. + +## `Open` and the snapshot cache + +`snapshot.Open` downloads the whole image to a local file, then opens it with +`erofs.Open`. Every read after the download is local. The caller closes the +`Snapshot`. If the image is absent, `Open` returns `fs.ErrNotExist`. It does +not build the image. + +The local file comes from the snapshot cache. The cache is a second +`tigris.PackCache`, from `NewSnapshotCache`, in an `objgit-snapshots-*` +directory next to the pack cache. It has the properties of the pack cache: + +- One download for each image, however many callers ask at the same time. +- A byte budget, `-snapshot-cache-bytes`. When a new image passes the budget, + the cache deletes the least-recently-used images. +- An idle sweep. Every quarter of `-snapshot-cache-max-idle`, the cache + deletes the images that nobody opened for that long. +- Eviction is an unlink. An open `Snapshot` keeps working. +- A failed download is not cached. + +A snapshot id is a tree hash, not the digest of the bytes. The cache therefore +uses `GetChecked`, whose fetch returns the digest from the `erofs-sha256` +metadata. A body that does not match is refused. + +The cache id is `erofs-v1-`, with no repository prefix. Two repositories +that hold one tree share one local file. This is deduplication and not +leakage. A caller gets to the id only through a tree that its own repository +holds. + +If the storer has no snapshot cache, `OpenSnapshot` downloads the image to a +private temp file and unlinks it at once. + +## The push path + +`(*daemon).receivePack` calls `runSnapshots` in `onUpdated`, after the refs +are committed and report-status is sent. Snapshots run before hooks. + +`runSnapshots` handles branches first, then tags. It peels annotated tags to +their commit, and it skips a tag of a tree or a blob. A tree that one push +already handled is not built again. + +Each ref writes one progress line to the client: + +```text +remote: objgit: snapshot refs/heads/main (tree 3d4e5f6): built, 1234 files, 5.6 MiB, 1.2s +remote: objgit: snapshot refs/tags/v1.0.0 (tree 3d4e5f6): exists +remote: objgit: snapshot refs/heads/wip (tree 0a1b2c3): failed +``` + +A snapshot failure never fails the push. The daemon logs the error and counts +it in `objgit_snapshot_builds_total{result="error"}`. The failure line does +not show the error text, because the text can contain bucket details. + +The push slot from `-max-concurrent-pushes` stays held while snapshots run. +The push cap therefore also limits the count of concurrent builds, and so the +temp disk that builds use at one time. + +The daemon gets the store with a type assertion, `st.(snapshot.Store)`. A +storer without the methods skips snapshots. + +## Flags + +| Flag | Default | Meaning | +| -------------------------- | ------- | ------------------------------------------------------------------------------------ | +| `-erofs-snapshots` | `true` | Build an image for each updated ref tip after a push. | +| `-snapshot-timeout` | `2m` | Wall-clock limit for the snapshots of one push. | +| `-snapshot-cache-bytes` | `2 GiB` | Disk budget for the snapshot cache. `0` disables the cache. | +| `-snapshot-cache-max-idle` | `1h` | The cache deletes an image that nobody opened for this long. `0` disables the sweep. | + +## Known risks + +- **Temp disk.** A build writes the image and a spool of compressed blocks + to the temp directory. Each one is at most the size of the image. +- **Push latency.** The client waits for the build. +- **The single PUT limit.** An image larger than 5 GiB fails, because + `PutSnapshot` sends one `PutObject`. +- **A crash during a build** leaves no image and no partial object. The next + `Ensure` for that tree builds it. +- **An image larger than the cache budget** is admitted, and it evicts every + other image. + +## The path to a message queue + +The queue changes only the caller. The message is `{repoPath, tree}`. The +consumer resolves the storer for `repoPath` and calls `Ensure`. `Ensure` is +idempotent, so a queue that delivers a message twice is correct. diff --git a/docs/architecture/tigris-storer.md b/docs/architecture/tigris-storer.md index efe2ff2..a6bfb2a 100644 --- a/docs/architecture/tigris-storer.md +++ b/docs/architecture/tigris-storer.md @@ -25,13 +25,14 @@ call has run. ## Layout -| Key | Contents | -| ------------------ | --------------------------------------------------- | -| `objects/` | One loose git object. | -| `packs/.bin` | One flat container of full objects. | -| `packs/.cue` | The record index for one container. | -| `packed-refs` | Every reference in one object. See "References". | -| `refs/` | One legacy loose reference. Read-only. | +| Key | Contents | +| --------------------------------- | ------------------------------------------------------------ | +| `objects/` | One loose git object. | +| `packs/.bin` | One flat container of full objects. | +| `packs/.cue` | The record index for one container. | +| `packed-refs` | Every reference in one object. See "References". | +| `refs/` | One legacy loose reference. Read-only. | +| `snapshots/erofs/v1/.erofs` | One erofs image of a tree. See [snapshots.md](snapshots.md). | Shallow marks, the worktree index, and the repository configuration sit at root-level keys. They carry the same prefix as everything else. The `shallow` @@ -62,10 +63,10 @@ listing gave only key names. Two keys hold reference state: -| Key | Role | -| -------------- | --------------------------------------------------------- | -| `packed-refs` | Every reference. Its ETag is the compare-and-swap token. | -| `refs/` | One legacy loose reference. Read-only. The fold deletes it. | +| Key | Role | +| ------------- | ----------------------------------------------------------- | +| `packed-refs` | Every reference. Its ETag is the compare-and-swap token. | +| `refs/` | One legacy loose reference. Read-only. The fold deletes it. | `packed-refs` does not start with `refs/`, so the loose listing cannot return it. @@ -248,11 +249,11 @@ references therefore cost N+1 `GetObject` calls on a key that is not there. These numbers come from a live bucket, before the memo: -| Repository | References | Failed GETs | Time in GetObject | Clone | -| ----------- | ---------: | ----------: | ----------------: | --------- | -| objgit | 37 | 38 | 11.1 s | 11.8 s | -| tigris-blog | 506 | 507 | 144 s | 3 m 47 s | -| Xe/x | 1,077 | 1,078 | 338 s | 6 m 11 s | +| Repository | References | Failed GETs | Time in GetObject | Clone | +| ----------- | ---------: | ----------: | ----------------: | -------- | +| objgit | 37 | 38 | 11.1 s | 11.8 s | +| tigris-blog | 506 | 507 | 144 s | 3 m 47 s | +| Xe/x | 1,077 | 1,078 | 338 s | 6 m 11 s | For `Xe/x` that is 338 of 412 seconds, or 73% of the clone. @@ -308,7 +309,7 @@ of it, which leaves the pack walk no room to run ahead. against `BufferedByteLimit`. `sizeHint` clamps the size to that limit. CAUTION: The clamp is not a nicety. The bundler takes the size from a semaphore -whose capacity *is* `BufferedByteLimit`, and `x/sync/semaphore` parks a size +whose capacity _is_ `BufferedByteLimit`, and `x/sync/semaphore` parks a size larger than the whole capacity until the context is done. An unclamped oversized job therefore hangs its push. The byte cap keeps every ordinary container far under the limit, but a lone object above the cap can still make a @@ -319,7 +320,7 @@ the same `Storer`. The delta-base resolution of go-git reads back objects that the same push wrote moments earlier, long before any flush. Without the pending overlay in `objects.go`, a real push fails with `apply delta patch: object not found`. -This is why `registerPending` runs *before* `enqueue`, and why +This is why `registerPending` runs _before_ `enqueue`, and why `EncodedObject` falls through to S3 only on `os.ErrNotExist`. CAUTION: Local staging paths must never come from content hashes. Two @@ -382,7 +383,7 @@ make a few kilobytes. A thousand large blobs make tens of gigabytes. Container size is what sets the size of one PUT, of one prefetched `.bin` download, and of one pack cache eviction, so the byte cap bounds all three. -The walk checks the cap *before* it adds an object. A container at 127 MiB must +The walk checks the cap _before_ it adds an object. A container at 127 MiB must not swallow a 500 MiB blob and land at 627 MiB. CAUTION: One container can exceed the cap. An object larger than the whole cap @@ -466,7 +467,7 @@ concurrent use. `metrics.ObserveS3` already is. `WithPackCache` decides where the downloaded copy lands. Without the option, each `Storer` gets a private temp file that is unlinked -but still open. The file dies with the `Storer`, so the *next* clone downloads +but still open. The file dies with the `Storer`, so the _next_ clone downloads the pack again. With the option, one process-wide `*PackCache` keeps whole `.bin` files in a @@ -502,6 +503,26 @@ Three design notes are worth not deriving a second time: take one ranged GET in that window, which is correct. The alternative serves bytes out of a body that failed its checksum. +### The second instance: the snapshot cache + +The same type also caches erofs snapshot images. `NewSnapshotCache` makes a +second instance in an `objgit-snapshots-*` directory, with its own budget. +`WithSnapshotCache` installs it, and `Scoped` shares it. + +Three functions exist for this instance. The pack cache does not use them. + +- `GetChecked` is `Get` for an id that is not a content digest. Its fetch + returns the lower-case hex SHA-256 that the body must have. `Get` is + `GetChecked` with a fetch that returns the id. +- `EvictIdle` deletes every settled entry that nobody claimed during the idle + limit. It never touches a download that has not settled. +- The observer gets `miss` for each download that one call runs, and `hit` + when a call returns a file with no download. It also gets `evict_budget` + and `evict_idle`. The observer always runs outside the cache lock, and + after the waiters of a download are released. + +See [snapshots.md](snapshots.md). + ## Testing seams Tests can use a fake S3 client, through the `s3API` interface. That interface diff --git a/docs/superpowers/plans/2026-09-23-erofs-snapshots.md b/docs/superpowers/plans/2026-09-23-erofs-snapshots.md new file mode 100644 index 0000000..e3d338d --- /dev/null +++ b/docs/superpowers/plans/2026-09-23-erofs-snapshots.md @@ -0,0 +1,2557 @@ +# erofs Snapshots Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** After a push, build one Zstandard-compressed EROFS image of the tree at each updated ref tip, store it in the bucket, and give a local-cache-backed `Open` for later readers. + +**Architecture:** A new `internal/snapshot` package holds the build (`Ensure`) and the open (`Open`) over a small `Store` interface. `*tigris.Storer` implements `Store`, with a second `PackCache` instance as the local LRU cache for downloads. The daemon calls `Ensure` synchronously in the existing `onUpdated` seam of receive-pack. + +**Tech Stack:** Go 1.26, go-git v6, `github.com/Xe/erofs` v0.6.1, AWS SDK v2 S3 client via `internal/storage/tigris`, Prometheus `promauto`. + +**Spec:** `docs/superpowers/specs/2026-09-23-erofs-snapshots-design.md`. Read it before any task. + +## Global Constraints + +- Object key: `snapshots/erofs/v1/.erofs`, relative to the repository prefix. +- Metadata keys (lowercase, exact): `erofs-format` = `1`, `erofs-sha256` = hex SHA-256 of the image, `git-tree` = hex tree hash, `erofs-files` = count of regular files. +- Builder options (exact): `erofs.WithBlockSize(12)`, `erofs.WithEpoch(time.Unix(0, 0))`, `erofs.WithCompression(erofs.CompressionZstd)`. +- Cache id: `erofs-v1-`. Cache directory pattern: `objgit-snapshots-*`, in the same parent as the pack cache. +- Flags: `-erofs-snapshots` (false), `-snapshot-timeout` (2m), `-snapshot-cache-bytes` (2 GiB, 0 disables), `-snapshot-cache-max-idle` (1h, 0 disables the sweep). Each flag gets a `flagenv` fallback automatically. +- A snapshot failure never fails a push. +- `slog` uses `"err"` as the error key, never `"error"`. +- Tests are table-driven with `tt`. Tests that shell out to `git` are gated with `exec.LookPath("git")`. +- No `repo` label on any metric. +- Every commit ends with the line `Signed-off-by: Xe Iaso `. Use Conventional Commits. +- Documentation follows the `simple-english` skill (ASD-STE100 pragmatic mode). +- Run only the tests of the package you change until the final task. Other agents work in the same tree at the same time. +- Commit only the files your task names: `git add `, never `git add -A`. If `git commit` fails on `index.lock`, wait two seconds and retry. + +## Review Focus + +1. **Unusual names in a tree** (spaces, UTF-8, a leading dot, a 255-byte name). A person expects each one to round-trip. Pinned in Task 1, `TestEnsureMapsModes`. +2. **A directory with many entries** (more dirents than one 4096-byte block holds). A person expects every entry listed. Pinned in Task 1, `TestEnsureMapsModes` case "large directory". +3. **The snapshot timeout expires during a build.** A person expects no partial object, and the push still succeeds. Pinned in Task 1, `TestEnsureCanceled`, and Task 5, `TestRunSnapshotsPutFailure`. +4. **A bucket error on `PutSnapshot`.** A person expects the push to succeed and the client to see `failed`. Pinned in Task 5, `TestRunSnapshotsPutFailure`. +5. **A tag of a tag, and a tag of a tree.** A person expects the first to resolve to the commit tree, and the second to be skipped with no error. Pinned in Task 5, `TestPeelToTree`. + +--- + +## File Structure + +| Path | Task | Responsibility | +| ------------------------------------------------------------------------------ | ---- | ------------------------------------------------------------------------- | +| `internal/snapshot/snapshot.go` | 0 | Package doc, `Store`, `File`, `Key`, `CacheID`, metadata constants. | +| `internal/snapshot/ensure.go` | 1 | `Ensure`, `Result`, the tree walk. | +| `internal/snapshot/open.go` | 1 | `Snapshot`, `Open`. | +| `internal/snapshot/memstore.go` | 1 | `MemStore`, a map-backed `Store` for tests in any package. | +| `internal/snapshot/ensure_test.go`, `open_test.go`, `fixture_test.go` | 1 | Tests and the git fixture helper. | +| `internal/storage/tigris/packcache.go` (+ `_test.go`) | 2 | `GetChecked`, `EvictIdle`, observer, suffix, `NewSnapshotCache`. | +| `internal/metrics/metrics.go` | 3 | Five series, two helpers. | +| `docs/architecture/metrics.md` | 3 | The new series. | +| `internal/storage/tigris/snapshot.go` (+ `_test.go`) | 4 | `*Storer` implements `snapshot.Store`. | +| `internal/storage/tigris/tigris.go` | 4 | `snapCache` field and `WithSnapshotCache`. | +| `cmd/objgitd/snapshots.go` (+ `_test.go`) | 5 | `runSnapshots`, `peelToTree`, progress formatting. | +| `cmd/objgitd/hooks.go`, `git_protocol.go`, `hooks_test.go` | 5 | Gate, tags in `snapshotRefs`, branch filter in `runHooks`, daemon fields. | +| `cmd/objgitd/main.go` | 6 | Flags, cache, sweep goroutine, cleanup. | +| `docs/architecture/snapshots.md`, `tigris-storer.md`, `README.md`, `AGENTS.md` | 7 | Documentation. | + +Dependency order: Task 0 → (Tasks 1, 2, 3 in parallel) → (Task 4 after 2; Task 5 after 1 and 3) → (Tasks 6 and 7 after 4 and 5). + +--- + +### Task 0: Scaffold (the controller does this before any fan-out) + +**Files:** + +- Modify: `go.mod`, `go.sum` +- Create: `internal/snapshot/snapshot.go` + +**Interfaces:** + +- Produces: `snapshot.Store`, `snapshot.File`, `snapshot.Key(plumbing.Hash) string`, `snapshot.CacheID(key string) string`, `snapshot.FormatVersion`, `snapshot.MetaFormat`, `snapshot.MetaSHA256`, `snapshot.MetaTree`, `snapshot.MetaFiles`. + +- [ ] **Step 1: Add the dependency** + +Run: `go get github.com/Xe/erofs@v0.6.1` + +- [ ] **Step 2: Write `internal/snapshot/snapshot.go`** + +```go +// Package snapshot builds and opens EROFS images of git trees. An image is +// stored next to the repository it came from, under Key(tree), and a later +// reader (such as a web UI) opens it with Open instead of walking git objects. +// +// The package never imports internal/storage/tigris. Storage reaches it +// through Store, which *tigris.Storer implements. +package snapshot + +import ( + "context" + "fmt" + "io" + "strings" + + "github.com/go-git/go-git/v6/plumbing" +) + +// FormatVersion names the mapping from git to EROFS and the build options. +// Changing either one is a new version, and a new key, so an image written +// under one version is never overwritten by another. +const FormatVersion = 1 + +// Object metadata keys carried by every image. Lowercase, because S3 returns +// user metadata keys lowercased. +const ( + MetaFormat = "erofs-format" + MetaSHA256 = "erofs-sha256" + MetaTree = "git-tree" + MetaFiles = "erofs-files" +) + +const keyPrefix = "snapshots/erofs/" + +// Key returns the object key of the image of tree, relative to the +// repository prefix. +func Key(tree plumbing.Hash) string { + return fmt.Sprintf("%sv%d/%s.erofs", keyPrefix, FormatVersion, tree) +} + +// CacheID returns the local cache id for key: "erofs-v1-". It holds no +// repository prefix, so two repositories with one tree share one cached file. +func CacheID(key string) string { + id := strings.TrimSuffix(strings.TrimPrefix(key, keyPrefix), ".erofs") + return "erofs-" + strings.ReplaceAll(id, "/", "-") +} + +// File is a local copy of an image. *os.File satisfies it. +type File interface { + io.ReaderAt + io.Closer +} + +// Store holds snapshot images for one repository. Keys are relative to the +// repository prefix. +type Store interface { + // StatSnapshot returns the size of the image at key, or an error that + // matches fs.ErrNotExist when key is absent. + StatSnapshot(ctx context.Context, key string) (int64, error) + // PutSnapshot stores size bytes from body at key, with meta as user + // metadata. + PutSnapshot(ctx context.Context, key string, body io.ReadSeeker, size int64, meta map[string]string) error + // OpenSnapshot returns a local copy of the image at key. The caller must + // close it. It returns an error that matches fs.ErrNotExist when key is + // absent. + OpenSnapshot(ctx context.Context, key string) (File, error) +} +``` + +- [ ] **Step 3: Build** + +Run: `go build ./... && go mod tidy && go build ./...` +Expected: success. `go mod tidy` can drop the erofs requirement because nothing imports it yet. If it does, keep it with `go get github.com/Xe/erofs@v0.6.1` again after Task 1 imports it, and do not run `go mod tidy` here. + +- [ ] **Step 4: Commit** + +```bash +git add go.mod go.sum internal/snapshot/snapshot.go +git commit -m "feat(snapshot): add the Store interface and key layout" -m "Signed-off-by: Xe Iaso " +``` + +--- + +### Task 1: `Ensure`, `Open`, and `MemStore` + +**Files:** + +- Create: `internal/snapshot/ensure.go`, `internal/snapshot/open.go`, `internal/snapshot/memstore.go` +- Test: `internal/snapshot/fixture_test.go`, `internal/snapshot/ensure_test.go`, `internal/snapshot/open_test.go` + +**Interfaces:** + +- Consumes: everything from Task 0. +- Produces: + - `type Result struct { Key string; Status string; Files int; Bytes int64; Elapsed time.Duration }` + - `const StatusBuilt = "built"`, `const StatusExists = "exists"` + - `func Ensure(ctx context.Context, objs storer.EncodedObjectStorer, store Store, tree plumbing.Hash, tmpDir string) (Result, error)` + - `type Snapshot struct { *erofs.FS; f File }`, `func (s *Snapshot) Close() error` + - `func Open(ctx context.Context, store Store, tree plumbing.Hash) (*Snapshot, error)` + - `type MemStore struct { PutErr error; ... }`, `func NewMemStore() *MemStore`, `func (m *MemStore) Keys() []string`, `func (m *MemStore) Object(key string) (body []byte, meta map[string]string, ok bool)`, `func (m *MemStore) PutCount() int` + +- [ ] **Step 1: Write the fixture helper `fixture_test.go`** + +```go +package snapshot + +import ( + "path" + "sort" + "strings" + "testing" + + "github.com/go-git/go-git/v6/plumbing" + "github.com/go-git/go-git/v6/plumbing/filemode" + "github.com/go-git/go-git/v6/plumbing/object" + "github.com/go-git/go-git/v6/storage/memory" +) + +// fixture is one entry of a test tree. For a gitlink, content is ignored and +// hash is the submodule commit. Directories come from the paths. +type fixture struct { + path string + mode filemode.FileMode + content string +} + +// buildTree writes the blobs and trees for entries into st and returns the +// root tree hash. It sorts entries the way git does, so the hash is stable. +func buildTree(t *testing.T, st *memory.Storage, entries []fixture) plumbing.Hash { + t.Helper() + type dir struct { + entries []object.TreeEntry + subdirs map[string]*dir + } + root := &dir{subdirs: map[string]*dir{}} + for _, e := range entries { + parts := strings.Split(e.path, "/") + d := root + for _, p := range parts[:len(parts)-1] { + sub, ok := d.subdirs[p] + if !ok { + sub = &dir{subdirs: map[string]*dir{}} + d.subdirs[p] = sub + } + d = sub + } + var h plumbing.Hash + if e.mode == filemode.Submodule { + h = plumbing.NewHash("1111111111111111111111111111111111111111") + } else { + h = writeBlob(t, st, e.content) + } + d.entries = append(d.entries, object.TreeEntry{Name: path.Base(e.path), Mode: e.mode, Hash: h}) + } + var write func(d *dir) plumbing.Hash + write = func(d *dir) plumbing.Hash { + for name, sub := range d.subdirs { + d.entries = append(d.entries, object.TreeEntry{Name: name, Mode: filemode.Dir, Hash: write(sub)}) + } + sort.Slice(d.entries, func(i, j int) bool { + return sortName(d.entries[i]) < sortName(d.entries[j]) + }) + tree := &object.Tree{Entries: d.entries} + obj := st.NewEncodedObject() + if err := tree.Encode(obj); err != nil { + t.Fatalf("encode tree: %v", err) + } + h, err := st.SetEncodedObject(obj) + if err != nil { + t.Fatalf("store tree: %v", err) + } + return h + } + return write(root) +} + +// sortName is the key git sorts tree entries by: a directory sorts as if its +// name ended in "/". +func sortName(e object.TreeEntry) string { + if e.Mode == filemode.Dir { + return e.Name + "/" + } + return e.Name +} + +func writeBlob(t *testing.T, st *memory.Storage, content string) plumbing.Hash { + t.Helper() + obj := st.NewEncodedObject() + obj.SetType(plumbing.BlobObject) + w, err := obj.Writer() + if err != nil { + t.Fatalf("blob writer: %v", err) + } + if _, err := w.Write([]byte(content)); err != nil { + t.Fatalf("write blob: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("close blob: %v", err) + } + h, err := st.SetEncodedObject(obj) + if err != nil { + t.Fatalf("store blob: %v", err) + } + return h +} +``` + +- [ ] **Step 2: Write the failing tests `ensure_test.go`** + +The test file holds these tests. Write all of them now. + +```go +package snapshot + +import ( + "bytes" + "context" + "errors" + "fmt" + "io/fs" + "strings" + "testing" + + "github.com/Xe/erofs" + "github.com/go-git/go-git/v6/plumbing" + "github.com/go-git/go-git/v6/plumbing/filemode" + "github.com/go-git/go-git/v6/storage/memory" +) + +func TestEnsureMapsModes(t *testing.T) { + long := strings.Repeat("n", 255) + var many []fixture + for i := range 500 { + many = append(many, fixture{fmt.Sprintf("big/file-%04d.txt", i), filemode.Regular, fmt.Sprintf("%d\n", i)}) + } + + tests := []struct { + name string + entries []fixture + // want maps an image path to its mode, and to its content or + // symlink target. A directory has no content. + want map[string]struct { + mode fs.FileMode + content string + } + }{ + { + name: "every mode", + entries: []fixture{ + {"README.md", filemode.Regular, "hello\n"}, + {"bin/run.sh", filemode.Executable, "#!/bin/sh\necho hi\n"}, + {"link", filemode.Symlink, "README.md"}, + {"old", filemode.Deprecated, "group writable\n"}, + {"vendor/lib", filemode.Submodule, ""}, + }, + want: map[string]struct { + mode fs.FileMode + content string + }{ + "README.md": {0o644, "hello\n"}, + "bin": {fs.ModeDir | 0o755, ""}, + "bin/run.sh": {0o755, "#!/bin/sh\necho hi\n"}, + "link": {fs.ModeSymlink | 0o777, "README.md"}, + "old": {0o644, "group writable\n"}, + "vendor": {fs.ModeDir | 0o755, ""}, + "vendor/lib": {fs.ModeDir | 0o755, ""}, + }, + }, + { + name: "unusual names", + entries: []fixture{ + {"with space.txt", filemode.Regular, "a"}, + {"ünïcødé.txt", filemode.Regular, "b"}, + {".hidden", filemode.Regular, "c"}, + {long, filemode.Regular, "d"}, + }, + want: map[string]struct { + mode fs.FileMode + content string + }{ + "with space.txt": {0o644, "a"}, + "ünïcødé.txt": {0o644, "b"}, + ".hidden": {0o644, "c"}, + long: {0o644, "d"}, + }, + }, + { + name: "large directory", + entries: many, + want: map[string]struct { + mode fs.FileMode + content string + }{ + "big/file-0000.txt": {0o644, "0\n"}, + "big/file-0499.txt": {0o644, "499\n"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + st := memory.NewStorage() + tree := buildTree(t, st, tt.entries) + store := NewMemStore() + + res, err := Ensure(context.Background(), st, store, tree, t.TempDir()) + if err != nil { + t.Fatalf("Ensure: %v", err) + } + if res.Status != StatusBuilt { + t.Fatalf("Status = %q, want %q", res.Status, StatusBuilt) + } + + snap, err := Open(context.Background(), store, tree) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer snap.Close() + + for p, w := range tt.want { + info, err := snap.Lstat(p) + if err != nil { + t.Errorf("Lstat(%q): %v", p, err) + continue + } + if info.Mode() != w.mode { + t.Errorf("%q mode = %v, want %v", p, info.Mode(), w.mode) + } + switch { + case w.mode&fs.ModeSymlink != 0: + got, err := snap.ReadLink(p) + if err != nil || got != w.content { + t.Errorf("ReadLink(%q) = %q, %v; want %q", p, got, err, w.content) + } + case w.mode.IsRegular(): + got, err := fs.ReadFile(snap, p) + if err != nil || string(got) != w.content { + t.Errorf("ReadFile(%q) = %q, %v; want %q", p, got, err, w.content) + } + } + } + + if tt.name == "large directory" { + ents, err := fs.ReadDir(snap, "big") + if err != nil || len(ents) != 500 { + t.Errorf("ReadDir(big) = %d entries, %v; want 500", len(ents), err) + } + } + }) + } +} + +func TestEnsureValidates(t *testing.T) { + tests := []struct { + name string + entries []fixture + }{ + {"flat", []fixture{{"a", filemode.Regular, "a"}}}, + {"nested", []fixture{{"x/y/z", filemode.Regular, strings.Repeat("z", 10000)}}}, + {"gitlink", []fixture{{"sub", filemode.Submodule, ""}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + st := memory.NewStorage() + tree := buildTree(t, st, tt.entries) + store := NewMemStore() + if _, err := Ensure(context.Background(), st, store, tree, t.TempDir()); err != nil { + t.Fatalf("Ensure: %v", err) + } + body, _, _ := store.Object(Key(tree)) + res, err := erofs.Validate(bytes.NewReader(body), int64(len(body))) + if err != nil { + t.Fatalf("Validate: %v", err) + } + if len(res.Errors) != 0 { + t.Errorf("Validate errors: %v", res.Errors) + } + }) + } +} + +func TestEnsureCompresses(t *testing.T) { + text := strings.Repeat("all work and no play makes jack a dull boy\n", 8000) // ~350 KiB + st := memory.NewStorage() + tree := buildTree(t, st, []fixture{{"story.txt", filemode.Regular, text}}) + store := NewMemStore() + res, err := Ensure(context.Background(), st, store, tree, t.TempDir()) + if err != nil { + t.Fatalf("Ensure: %v", err) + } + if res.Bytes >= int64(len(text)) { + t.Errorf("image is %d bytes, want less than the %d-byte file", res.Bytes, len(text)) + } + snap, err := Open(context.Background(), store, tree) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer snap.Close() + got, err := fs.ReadFile(snap, "story.txt") + if err != nil || string(got) != text { + t.Errorf("story.txt did not read back unchanged: %v", err) + } +} + +func TestEnsureDeterministic(t *testing.T) { + entries := []fixture{ + {"a.txt", filemode.Regular, strings.Repeat("a", 9000)}, + {"d/b.txt", filemode.Executable, "b"}, + {"l", filemode.Symlink, "a.txt"}, + } + var bodies [2][]byte + var sums [2]string + for i := range 2 { + st := memory.NewStorage() + tree := buildTree(t, st, entries) + store := NewMemStore() + if _, err := Ensure(context.Background(), st, store, tree, t.TempDir()); err != nil { + t.Fatalf("Ensure %d: %v", i, err) + } + body, meta, _ := store.Object(Key(tree)) + bodies[i], sums[i] = body, meta[MetaSHA256] + } + if !bytes.Equal(bodies[0], bodies[1]) { + t.Error("two builds of one tree gave different bytes") + } + if sums[0] == "" || sums[0] != sums[1] { + t.Errorf("sha256 metadata = %q and %q, want equal and set", sums[0], sums[1]) + } +} + +func TestEnsureMetadata(t *testing.T) { + st := memory.NewStorage() + tree := buildTree(t, st, []fixture{ + {"a", filemode.Regular, "a"}, + {"b", filemode.Regular, "b"}, + {"l", filemode.Symlink, "a"}, + }) + store := NewMemStore() + res, err := Ensure(context.Background(), st, store, tree, t.TempDir()) + if err != nil { + t.Fatalf("Ensure: %v", err) + } + _, meta, ok := store.Object(Key(tree)) + if !ok { + t.Fatal("no object at Key(tree)") + } + want := map[string]string{MetaFormat: "1", MetaTree: tree.String(), MetaFiles: "2"} + for k, v := range want { + if meta[k] != v { + t.Errorf("meta[%q] = %q, want %q", k, meta[k], v) + } + } + if res.Files != 2 { + t.Errorf("Files = %d, want 2", res.Files) + } +} + +func TestEnsureExists(t *testing.T) { + st := memory.NewStorage() + tree := buildTree(t, st, []fixture{{"a", filemode.Regular, "a"}}) + store := NewMemStore() + if _, err := Ensure(context.Background(), st, store, tree, t.TempDir()); err != nil { + t.Fatalf("first Ensure: %v", err) + } + res, err := Ensure(context.Background(), st, store, tree, t.TempDir()) + if err != nil { + t.Fatalf("second Ensure: %v", err) + } + if res.Status != StatusExists { + t.Errorf("Status = %q, want %q", res.Status, StatusExists) + } + if n := store.PutCount(); n != 1 { + t.Errorf("PutCount = %d, want 1", n) + } +} + +func TestEnsureLimits(t *testing.T) { + tests := []struct { + name string + entries []fixture + path string // must appear in the error + }{ + {"name too long", []fixture{{strings.Repeat("n", 256), filemode.Regular, "x"}}, strings.Repeat("n", 256)}, + {"symlink target too long", []fixture{{"l", filemode.Symlink, strings.Repeat("t", 1021)}}, "l"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + st := memory.NewStorage() + tree := buildTree(t, st, tt.entries) + store := NewMemStore() + _, err := Ensure(context.Background(), st, store, tree, t.TempDir()) + if err == nil || !strings.Contains(err.Error(), tt.path) { + t.Fatalf("Ensure error = %v, want one that names %q", err, tt.path) + } + if n := store.PutCount(); n != 0 { + t.Errorf("PutCount = %d, want 0", n) + } + }) + } +} + +func TestEnsureEmptyTree(t *testing.T) { + st := memory.NewStorage() + tree := buildTree(t, st, nil) + store := NewMemStore() + if _, err := Ensure(context.Background(), st, store, tree, t.TempDir()); err != nil { + t.Fatalf("Ensure: %v", err) + } + snap, err := Open(context.Background(), store, tree) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer snap.Close() + ents, err := fs.ReadDir(snap, ".") + if err != nil || len(ents) != 0 { + t.Errorf("ReadDir(.) = %v, %v; want empty", ents, err) + } +} + +func TestEnsureCanceled(t *testing.T) { + st := memory.NewStorage() + tree := buildTree(t, st, []fixture{{"a", filemode.Regular, "a"}}) + store := NewMemStore() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := Ensure(ctx, st, store, tree, t.TempDir()) + if !errors.Is(err, context.Canceled) { + t.Fatalf("Ensure error = %v, want context.Canceled", err) + } + if n := store.PutCount(); n != 0 { + t.Errorf("PutCount = %d, want 0", n) + } +} + +func TestEnsureRemovesTempFile(t *testing.T) { + st := memory.NewStorage() + tree := buildTree(t, st, []fixture{{"a", filemode.Regular, "a"}}) + tmp := t.TempDir() + for _, putErr := range []error{nil, errors.New("bucket down")} { + store := NewMemStore() + store.PutErr = putErr + _, _ = Ensure(context.Background(), st, store, tree, tmp) + ents, _ := os.ReadDir(tmp) + if len(ents) != 0 { + t.Errorf("putErr=%v: %d files left in tmpDir", putErr, len(ents)) + } + } +} + +var _ = plumbing.ZeroHash +``` + +Add `"os"` to the imports for `TestEnsureRemovesTempFile`. + +`TestEnsureCanceled` requires `Ensure` to check `ctx.Err()` before `StatSnapshot`, and between entries. + +- [ ] **Step 3: Write `open_test.go`** + +```go +package snapshot + +import ( + "context" + "errors" + "io/fs" + "testing" + + "github.com/go-git/go-git/v6/plumbing" +) + +func TestOpenMissing(t *testing.T) { + _, err := Open(context.Background(), NewMemStore(), plumbing.NewHash("2222222222222222222222222222222222222222")) + if !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("Open error = %v, want fs.ErrNotExist", err) + } +} +``` + +- [ ] **Step 4: Run the tests to make sure they fail** + +Run: `go test ./internal/snapshot/` +Expected: FAIL, with `undefined: Ensure`, `undefined: NewMemStore`, and `undefined: Open`. + +- [ ] **Step 5: Write `memstore.go`** + +```go +package snapshot + +import ( + "bytes" + "context" + "fmt" + "io" + "io/fs" + "maps" + "slices" + "sync" +) + +// MemStore is a Store in memory. It exists for tests, in this package and in +// others that need a Store without a bucket. +type MemStore struct { + // PutErr, when set, is returned by every PutSnapshot. + PutErr error + + mu sync.Mutex + objs map[string]memObject + puts int +} + +type memObject struct { + body []byte + meta map[string]string +} + +// NewMemStore returns an empty MemStore. +func NewMemStore() *MemStore { return &MemStore{objs: map[string]memObject{}} } + +func (m *MemStore) StatSnapshot(_ context.Context, key string) (int64, error) { + m.mu.Lock() + defer m.mu.Unlock() + o, ok := m.objs[key] + if !ok { + return 0, fmt.Errorf("snapshot: %s: %w", key, fs.ErrNotExist) + } + return int64(len(o.body)), nil +} + +func (m *MemStore) PutSnapshot(_ context.Context, key string, body io.ReadSeeker, size int64, meta map[string]string) error { + m.mu.Lock() + m.puts++ + putErr := m.PutErr + m.mu.Unlock() + if putErr != nil { + return putErr + } + b, err := io.ReadAll(body) + if err != nil { + return err + } + if int64(len(b)) != size { + return fmt.Errorf("snapshot: put %s: read %d bytes, want %d", key, len(b), size) + } + m.mu.Lock() + defer m.mu.Unlock() + m.objs[key] = memObject{body: b, meta: maps.Clone(meta)} + return nil +} + +func (m *MemStore) OpenSnapshot(_ context.Context, key string) (File, error) { + m.mu.Lock() + defer m.mu.Unlock() + o, ok := m.objs[key] + if !ok { + return nil, fmt.Errorf("snapshot: %s: %w", key, fs.ErrNotExist) + } + return nopCloser{bytes.NewReader(o.body)}, nil +} + +// Keys returns every stored key, sorted. +func (m *MemStore) Keys() []string { + m.mu.Lock() + defer m.mu.Unlock() + return slices.Sorted(maps.Keys(m.objs)) +} + +// Object returns the body and metadata stored at key. +func (m *MemStore) Object(key string) ([]byte, map[string]string, bool) { + m.mu.Lock() + defer m.mu.Unlock() + o, ok := m.objs[key] + return o.body, o.meta, ok +} + +// PutCount returns how many times PutSnapshot ran, failed calls included. +func (m *MemStore) PutCount() int { + m.mu.Lock() + defer m.mu.Unlock() + return m.puts +} + +type nopCloser struct{ *bytes.Reader } + +func (nopCloser) Close() error { return nil } +``` + +- [ ] **Step 6: Write `open.go`** + +```go +package snapshot + +import ( + "context" + "fmt" + + "github.com/Xe/erofs" + "github.com/go-git/go-git/v6/plumbing" +) + +// Snapshot is an open image. Every read is local. Close releases the file. +type Snapshot struct { + *erofs.FS + f File +} + +// Close releases the local file behind the image. +func (s *Snapshot) Close() error { return s.f.Close() } + +// Open returns the image of tree. It does not build a missing image: it +// returns an error that matches fs.ErrNotExist instead. Call Ensure first when +// the image must exist. +func Open(ctx context.Context, store Store, tree plumbing.Hash) (*Snapshot, error) { + key := Key(tree) + f, err := store.OpenSnapshot(ctx, key) + if err != nil { + return nil, fmt.Errorf("snapshot: open %s: %w", key, err) + } + fsys, err := erofs.Open(f) + if err != nil { + f.Close() + return nil, fmt.Errorf("snapshot: read %s: %w", key, err) + } + return &Snapshot{FS: fsys, f: f}, nil +} +``` + +- [ ] **Step 7: Write `ensure.go`** + +```go +package snapshot + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "strconv" + "time" + + "github.com/Xe/erofs" + "github.com/go-git/go-git/v6/plumbing" + "github.com/go-git/go-git/v6/plumbing/filemode" + "github.com/go-git/go-git/v6/plumbing/object" + "github.com/go-git/go-git/v6/plumbing/storer" +) + +// Result statuses. +const ( + StatusBuilt = "built" + StatusExists = "exists" +) + +// Two EROFS limits that git does not have. The builder checks neither one, +// and the reader refuses a symlink target longer than maxSymlink. +const ( + maxName = 255 + maxSymlink = 255 * 4 +) + +// epoch pins every mtime, so one tree always gives identical bytes. +var epoch = time.Unix(0, 0) + +// Result describes one Ensure call. +type Result struct { + Key string + Status string // StatusBuilt or StatusExists + Files int // regular files in the image; 0 when Status is StatusExists + Bytes int64 // image size; 0 when Status is StatusExists + Elapsed time.Duration +} + +// Ensure makes sure that the image of tree exists in store. If the image is +// absent, it builds the image from the git objects in objs, in a temp file in +// tmpDir (the OS temp directory when tmpDir is empty), and puts it. +// +// Ensure is idempotent. Two concurrent calls for one tree both build and put +// identical bytes, which is correct, so it takes no lock. +func Ensure(ctx context.Context, objs storer.EncodedObjectStorer, store Store, tree plumbing.Hash, tmpDir string) (Result, error) { + start := time.Now() + key := Key(tree) + res := Result{Key: key} + done := func() { res.Elapsed = time.Since(start) } + + if err := ctx.Err(); err != nil { + return res, err + } + if _, err := store.StatSnapshot(ctx, key); err == nil { + res.Status = StatusExists + done() + return res, nil + } else if !errors.Is(err, fs.ErrNotExist) { + return res, fmt.Errorf("snapshot: stat %s: %w", key, err) + } + + root, err := object.GetTree(objs, tree) + if err != nil { + return res, fmt.Errorf("snapshot: load tree %s: %w", tree, err) + } + + // The name is random, never the tree hash: two pushes of one tree can + // build at the same time. + f, err := os.CreateTemp(tmpDir, "objgit-snapshot-*.erofs") + if err != nil { + return res, fmt.Errorf("snapshot: create temp file: %w", err) + } + defer func() { + f.Close() + os.Remove(f.Name()) + }() + + b := erofs.NewBuilder(f, + erofs.WithBlockSize(12), + erofs.WithEpoch(epoch), + erofs.WithCompression(erofs.CompressionZstd), + ) + w := &walker{ctx: ctx, objs: objs, b: b} + if err := w.addTree("/", root); err != nil { + return res, err + } + if err := b.Build(); err != nil { + return res, fmt.Errorf("snapshot: build %s: %w", key, err) + } + + sum, size, err := hashFile(f) + if err != nil { + return res, err + } + meta := map[string]string{ + MetaFormat: strconv.Itoa(FormatVersion), + MetaSHA256: sum, + MetaTree: tree.String(), + MetaFiles: strconv.Itoa(w.files), + } + if err := store.PutSnapshot(ctx, key, f, size, meta); err != nil { + return res, fmt.Errorf("snapshot: put %s: %w", key, err) + } + + res.Status = StatusBuilt + res.Files = w.files + res.Bytes = size + done() + return res, nil +} + +// hashFile returns the hex SHA-256 and size of f, and leaves f at offset 0. +func hashFile(f *os.File) (string, int64, error) { + if _, err := f.Seek(0, io.SeekStart); err != nil { + return "", 0, fmt.Errorf("snapshot: rewind image: %w", err) + } + h := sha256.New() + size, err := io.Copy(h, f) + if err != nil { + return "", 0, fmt.Errorf("snapshot: hash image: %w", err) + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + return "", 0, fmt.Errorf("snapshot: rewind image: %w", err) + } + return hex.EncodeToString(h.Sum(nil)), size, nil +} + +type walker struct { + ctx context.Context + objs storer.EncodedObjectStorer + b *erofs.Builder + files int +} + +// addTree adds every entry of t under dir, depth first. It walks tree +// entries and not t.Files(), because go-git's FileIter skips gitlinks. +func (w *walker) addTree(dir string, t *object.Tree) error { + for _, e := range t.Entries { + if err := w.ctx.Err(); err != nil { + return err + } + p := path.Join(dir, e.Name) + if len(e.Name) > maxName { + return fmt.Errorf("snapshot: %s: name is %d bytes, EROFS allows %d", p, len(e.Name), maxName) + } + switch e.Mode { + case filemode.Dir: + sub, err := object.GetTree(w.objs, e.Hash) + if err != nil { + return fmt.Errorf("snapshot: %s: load tree: %w", p, err) + } + if err := w.b.AddDir(p, info{e.Name, fs.ModeDir | 0o755}); err != nil { + return fmt.Errorf("snapshot: %s: %w", p, err) + } + if err := w.addTree(p, sub); err != nil { + return err + } + case filemode.Submodule: + // Same as git checkout of a submodule that is not initialized. + if err := w.b.AddDir(p, info{e.Name, fs.ModeDir | 0o755}); err != nil { + return fmt.Errorf("snapshot: %s: %w", p, err) + } + case filemode.Regular, filemode.Deprecated, filemode.Executable: + data, err := w.blob(p, e.Hash) + if err != nil { + return err + } + perm := fs.FileMode(0o644) + if e.Mode == filemode.Executable { + perm = 0o755 + } + if err := w.b.AddFile(p, info{e.Name, perm}, data); err != nil { + return fmt.Errorf("snapshot: %s: %w", p, err) + } + w.files++ + case filemode.Symlink: + data, err := w.blob(p, e.Hash) + if err != nil { + return err + } + if len(data) > maxSymlink { + return fmt.Errorf("snapshot: %s: symlink target is %d bytes, EROFS allows %d", p, len(data), maxSymlink) + } + if err := w.b.AddSymlink(p, string(data), info{e.Name, fs.ModeSymlink | 0o777}); err != nil { + return fmt.Errorf("snapshot: %s: %w", p, err) + } + default: + return fmt.Errorf("snapshot: %s: unknown git mode %o", p, uint32(e.Mode)) + } + } + return nil +} + +func (w *walker) blob(p string, h plumbing.Hash) ([]byte, error) { + blob, err := object.GetBlob(w.objs, h) + if err != nil { + return nil, fmt.Errorf("snapshot: %s: load blob: %w", p, err) + } + r, err := blob.Reader() + if err != nil { + return nil, fmt.Errorf("snapshot: %s: read blob: %w", p, err) + } + defer r.Close() + data, err := io.ReadAll(r) + if err != nil { + return nil, fmt.Errorf("snapshot: %s: read blob: %w", p, err) + } + return data, nil +} + +// info is the fs.FileInfo the builder reads the mode and mtime from. +type info struct { + name string + mode fs.FileMode +} + +func (i info) Name() string { return i.name } +func (i info) Size() int64 { return 0 } +func (i info) Mode() fs.FileMode { return i.mode } +func (i info) ModTime() time.Time { return epoch } +func (i info) IsDir() bool { return i.mode.IsDir() } +func (i info) Sys() any { return nil } +``` + +- [ ] **Step 8: Run the tests** + +Run: `go test ./internal/snapshot/ -v` +Expected: PASS for every test. + +If `TestEnsureMapsModes` fails on the symlink mode, print `info.Mode()` for `link` and set the expected mode in the table to what the reader reports for a symlink. Do not change `ensure.go` for this. The spec does not fix the permission bits of a symlink. + +- [ ] **Step 9: Vet and commit** + +```bash +go vet ./internal/snapshot/ +git add internal/snapshot/ +git commit -m "feat(snapshot): build and open erofs images of git trees" -m "Signed-off-by: Xe Iaso " +``` + +--- + +### Task 2: `PackCache` additions + +**Files:** + +- Modify: `internal/storage/tigris/packcache.go`, `internal/storage/tigris/packindex.go` (only the `verifiedCopy` call in `openWholePack`) +- Test: `internal/storage/tigris/packcache_test.go` + +**Interfaces:** + +- Produces: + - `func NewSnapshotCache(parent string, maxBytes int64, observe func(event string)) (*PackCache, error)` + - `func (c *PackCache) GetChecked(id string, fetch func(io.Writer) (wantSHA256 string, err error)) (*os.File, error)` + - `func (c *PackCache) EvictIdle(maxIdle time.Duration) int` + - Event strings: `"hit"`, `"miss"`, `"evict_budget"`, `"evict_idle"`. +- `Get` and `NewPackCache` keep their signatures and behavior. + +- [ ] **Step 1: Write the failing tests** + +Read the existing helpers at the top of `packcache_test.go` (`cachePayload`, `newTestPackCache`, `readAll`, `countCacheFiles`) and reuse them. `countCacheFiles` counts files by suffix. If it hardcodes `.bin`, change it to use `c.suffix`. Add: + +```go +func TestPackCacheGetChecked(t *testing.T) { + body := []byte("snapshot bytes") + sum := sha256.Sum256(body) + good := hex.EncodeToString(sum[:]) + + tests := []struct { + name string + want string + wantErr bool + }{ + {"digest matches", good, false}, + {"digest differs", strings.Repeat("0", 64), true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c, err := NewSnapshotCache(t.TempDir(), 1<<20, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { c.Cleanup() }) + f, err := c.GetChecked("erofs-v1-abc", func(w io.Writer) (string, error) { + _, err := w.Write(body) + return tt.want, err + }) + if tt.wantErr { + if err == nil { + f.Close() + t.Fatal("GetChecked accepted bytes with the wrong digest") + } + if n := countCacheFiles(t, c); n != 0 { + t.Errorf("cache kept %d files after a bad digest", n) + } + return + } + if err != nil { + t.Fatalf("GetChecked: %v", err) + } + defer f.Close() + if got := readAll(t, f); !bytes.Equal(got, body) { + t.Errorf("body = %q, want %q", got, body) + } + if !strings.HasSuffix(f.Name(), ".erofs") { + t.Errorf("cached file %q does not end in .erofs", f.Name()) + } + }) + } +} + +func TestPackCacheGetStillChecksID(t *testing.T) { + c := newTestPackCache(t, 1<<20) + _, err := c.Get(strings.Repeat("0", 64), func(w io.Writer) error { + _, err := w.Write([]byte("not the digest")) + return err + }) + if err == nil { + t.Fatal("Get accepted bytes whose SHA-256 is not the id") + } +} + +func TestPackCacheEvictIdle(t *testing.T) { + now := time.Unix(1000, 0) + c, err := NewSnapshotCache(t.TempDir(), 0, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { c.Cleanup() }) + c.now = func() time.Time { return now } + + put := func(id string) *os.File { + body := []byte(id) + sum := sha256.Sum256(body) + f, err := c.GetChecked(id, func(w io.Writer) (string, error) { + _, err := w.Write(body) + return hex.EncodeToString(sum[:]), err + }) + if err != nil { + t.Fatalf("GetChecked(%s): %v", id, err) + } + return f + } + + old := put("old") // used at t=1000 + now = now.Add(time.Hour) // t=4600 + put("fresh").Close() // used at t=4600 + now = now.Add(time.Minute) + + if n := c.EvictIdle(30 * time.Minute); n != 1 { + t.Fatalf("EvictIdle = %d, want 1", n) + } + if n := countCacheFiles(t, c); n != 1 { + t.Errorf("%d files left, want 1", n) + } + // The evicted entry's open descriptor still reads. + if got := readAll(t, old); string(got) != "old" { + t.Errorf("evicted file read %q, want %q", got, "old") + } + old.Close() +} + +func TestPackCacheObserver(t *testing.T) { + var mu sync.Mutex + var events []string + observe := func(e string) { mu.Lock(); events = append(events, e); mu.Unlock() } + + now := time.Unix(1000, 0) + c, err := NewSnapshotCache(t.TempDir(), 10, observe) // 10-byte budget + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { c.Cleanup() }) + c.now = func() time.Time { return now } + + get := func(id string, body []byte) { + sum := sha256.Sum256(body) + f, err := c.GetChecked(id, func(w io.Writer) (string, error) { + _, err := w.Write(body) + return hex.EncodeToString(sum[:]), err + }) + if err != nil { + t.Fatalf("GetChecked(%s): %v", id, err) + } + f.Close() + } + + get("a", []byte("123456")) // miss + get("a", []byte("123456")) // hit + get("b", []byte("abcdef")) // miss, evicts a: evict_budget + now = now.Add(time.Hour) + c.EvictIdle(time.Minute) // evicts b: evict_idle + + want := []string{"miss", "hit", "miss", "evict_budget", "evict_idle"} + mu.Lock() + defer mu.Unlock() + if !slices.Equal(events, want) { + t.Errorf("events = %v, want %v", events, want) + } +} +``` + +Add imports as needed: `bytes`, `crypto/sha256`, `encoding/hex`, `slices`, `strings`, `sync`, `time`. + +- [ ] **Step 2: Run the tests to make sure they fail** + +Run: `go test ./internal/storage/tigris/ -run 'TestPackCache'` +Expected: FAIL with `undefined: NewSnapshotCache` and `c.now undefined`. + +- [ ] **Step 3: Implement** + +In `packcache.go`: + +1. Add fields to `PackCache`: + + ```go + suffix string // file name suffix of a cached entry, binSuffix for packs + observe func(event string) // optional; see NewSnapshotCache + now func() time.Time // wall clock for EvictIdle; a field so tests can move it + ``` + +2. Add `usedAt time.Time` to `cacheEntry`, next to `used`. Comment: `// wall-clock time of the last claim; EvictIdle reads it`. + +3. Replace `NewPackCache` with a shared constructor: + + ```go + func NewPackCache(parent string, maxBytes int64) (*PackCache, error) { + return newFileCache(parent, "objgit-packs-*", binSuffix, maxBytes, nil) + } + + // NewSnapshotCache creates a cache for snapshot images, in a fresh + // objgit-snapshots-* directory under parent. It is the same cache as + // NewPackCache, with its own directory and budget. Snapshot ids are not + // content digests, so callers use GetChecked, not Get. observe, when not + // nil, receives "hit", "miss", "evict_budget", and "evict_idle". + func NewSnapshotCache(parent string, maxBytes int64, observe func(event string)) (*PackCache, error) { + return newFileCache(parent, "objgit-snapshots-*", ".erofs", maxBytes, observe) + } + + func newFileCache(parent, pattern, suffix string, maxBytes int64, observe func(string)) (*PackCache, error) { + dir, err := os.MkdirTemp(parent, pattern) + if err != nil { + return nil, fmt.Errorf("tigris: create cache directory: %w", err) + } + return &PackCache{ + dir: dir, + maxBytes: maxBytes, + entries: make(map[string]*cacheEntry), + suffix: suffix, + observe: observe, + now: time.Now, + }, nil + } + + func (c *PackCache) emit(event string, n int) { + if c.observe == nil { + return + } + for range n { + c.observe(event) + } + } + ``` + +4. `claim`: use `c.suffix` in the path, and set `e.usedAt = c.now()` on both branches. + +5. Make `Get` a wrapper, and move its body to `GetChecked`: + + ```go + func (c *PackCache) Get(id string, fetch func(io.Writer) error) (*os.File, error) { + return c.GetChecked(id, expectID(id, fetch)) + } + + // expectID adapts a pack fetch, whose digest is its id, to GetChecked. + func expectID(id string, fetch func(io.Writer) error) func(io.Writer) (string, error) { + return func(w io.Writer) (string, error) { return id, fetch(w) } + } + + // GetChecked is Get for ids that are not content digests: fetch writes the + // body and returns the SHA-256 the body must have. A body that disagrees + // is refused and not cached. + func (c *PackCache) GetChecked(id string, fetch func(io.Writer) (wantSHA256 string, err error)) (*os.File, error) { + // the old Get body, with fill(e, fetch) unchanged, plus: + // after claim: if mine { c.emit("miss", 1) } else { c.emit("hit", 1) } + } + ``` + + `fill` takes `fetch func(io.Writer) (string, error)` and passes it to `verifiedCopy`. + +6. `verifiedCopy(w io.Writer, id string, fetch func(io.Writer) (string, error), progress *atomic.Int64)`: compare the computed digest with the string that `fetch` returns, not with `id`. Keep `id` for the error message. In `packindex.go` `openWholePack`, change the call to `verifiedCopy(f, id, expectID(id, s.streamPack(id)), &ps.n)`. + +7. `evictLocked` returns the count of entries it removed. In `fill`, keep the count, unlock, then call `c.emit("evict_budget", n)`. The callback must run outside the lock. + +8. Add `EvictIdle`: + + ```go + // EvictIdle unlinks every settled entry that no caller claimed in the last + // maxIdle, and reports how many it removed. It never touches a download + // that has not settled. Open descriptors keep working, as with budget + // eviction. + func (c *PackCache) EvictIdle(maxIdle time.Duration) int { + c.mu.Lock() + cutoff := c.now().Add(-maxIdle) + n := 0 + for id, e := range c.entries { + if e.size == 0 || !e.usedAt.Before(cutoff) { + continue + } + delete(c.entries, id) + c.cur -= e.size + os.Remove(e.path) + n++ + slog.Debug("cache entry idle, uncached", "id", id, "bytes", e.size, "idle_since", e.usedAt) + } + c.mu.Unlock() + c.emit("evict_idle", n) + return n + } + ``` + +9. Update the `PackCache` doc comment. Add one paragraph: the type also serves snapshot images, through `NewSnapshotCache` and `GetChecked`. + +- [ ] **Step 4: Run the tests** + +Run: `go test ./internal/storage/tigris/ -run 'TestPackCache|Pack' -count=1` +Expected: PASS, including every test that existed before. + +Then run the whole package once: `go test ./internal/storage/tigris/ -count=1` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add internal/storage/tigris/packcache.go internal/storage/tigris/packcache_test.go internal/storage/tigris/packindex.go +git commit -m "feat(storage/tigris): let the pack cache serve snapshot images" -m "Adds GetChecked for ids that are not digests, EvictIdle, and an event observer." -m "Signed-off-by: Xe Iaso " +``` + +--- + +### Task 3: Metrics + +**Files:** + +- Modify: `internal/metrics/metrics.go`, `docs/architecture/metrics.md` +- Test: `internal/metrics/metrics_test.go` (create it if it does not exist) + +**Interfaces:** + +- Produces: + - `func ObserveSnapshot(result string, dur time.Duration, bytes int64)`: `result` is `"built"`, `"exists"`, or `"error"`. The two histograms observe only for `"built"`. + - `func ObserveSnapshotCache(event string)`: `"hit"` and `"miss"` go to `objgit_snapshot_cache_opens_total{result}`. `"evict_budget"` and `"evict_idle"` go to `objgit_snapshot_cache_evictions_total{reason}` with `budget` and `idle`. Other events are ignored. + +- [ ] **Step 1: Write the failing test** + +```go +package metrics + +import ( + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestObserveSnapshot(t *testing.T) { + tests := []struct { + result string + wantHist bool + }{ + {"built", true}, + {"exists", false}, + {"error", false}, + } + for _, tt := range tests { + t.Run(tt.result, func(t *testing.T) { + before := testutil.ToFloat64(snapshotBuilds.WithLabelValues(tt.result)) + histBefore := testutil.CollectAndCount(snapshotImageBytes) + ObserveSnapshot(tt.result, time.Second, 1<<20) + if got := testutil.ToFloat64(snapshotBuilds.WithLabelValues(tt.result)); got != before+1 { + t.Errorf("builds_total{%s} = %v, want %v", tt.result, got, before+1) + } + _ = histBefore + }) + } +} + +func TestObserveSnapshotCache(t *testing.T) { + tests := []struct { + event string + vec string + label string + }{ + {"hit", "opens", "hit"}, + {"miss", "opens", "miss"}, + {"evict_budget", "evictions", "budget"}, + {"evict_idle", "evictions", "idle"}, + } + for _, tt := range tests { + t.Run(tt.event, func(t *testing.T) { + c := snapshotCacheOpens.WithLabelValues(tt.label) + if tt.vec == "evictions" { + c = snapshotCacheEvictions.WithLabelValues(tt.label) + } + before := testutil.ToFloat64(c) + ObserveSnapshotCache(tt.event) + if got := testutil.ToFloat64(c); got != before+1 { + t.Errorf("%s{%s} = %v, want %v", tt.vec, tt.label, got, before+1) + } + }) + } +} +``` + +Remove the unused `histBefore` lines if `go vet` complains. The histogram gating is simple enough that the code review covers it. + +- [ ] **Step 2: Run the test to make sure it fails** + +Run: `go test ./internal/metrics/` +Expected: FAIL with `undefined: snapshotBuilds`. + +- [ ] **Step 3: Implement** + +Add to the `var (...)` block in `metrics.go`, after the hook series: + +```go + snapshotBuilds = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "snapshot", + Name: "builds_total", + Help: "erofs snapshot Ensure calls by result (built, exists, error).", + }, []string{"result"}) + + snapshotDuration = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: "snapshot", + Name: "build_duration_seconds", + Help: "Time for one erofs snapshot Ensure call that built an image.", + Buckets: prometheus.ExponentialBuckets(0.1, 2, 12), // 0.1s .. ~205s + }) + + snapshotImageBytes = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: "snapshot", + Name: "image_bytes", + Help: "Size of one built erofs snapshot image, after compression.", + Buckets: prometheus.ExponentialBuckets(64<<10, 4, 10), // 64 KiB .. 16 GiB + }) + + snapshotCacheOpens = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "snapshot", + Name: "cache_opens_total", + Help: "Snapshot cache lookups by result (hit, miss).", + }, []string{"result"}) + + snapshotCacheEvictions = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "snapshot", + Name: "cache_evictions_total", + Help: "Snapshot cache evictions by reason (budget, idle).", + }, []string{"reason"}) +``` + +Add the helpers after `ObserveHook`: + +```go +// ObserveSnapshot records one snapshot Ensure call. The duration and size +// histograms only describe builds, so they observe only when result is +// "built". +func ObserveSnapshot(result string, dur time.Duration, bytes int64) { + snapshotBuilds.WithLabelValues(result).Inc() + if result == "built" { + snapshotDuration.Observe(dur.Seconds()) + snapshotImageBytes.Observe(float64(bytes)) + } +} + +// ObserveSnapshotCache records one snapshot cache event. Its signature +// matches the observer tigris.NewSnapshotCache takes. +func ObserveSnapshotCache(event string) { + switch event { + case "hit", "miss": + snapshotCacheOpens.WithLabelValues(event).Inc() + case "evict_budget": + snapshotCacheEvictions.WithLabelValues("budget").Inc() + case "evict_idle": + snapshotCacheEvictions.WithLabelValues("idle").Inc() + } +} +``` + +- [ ] **Step 4: Run the test** + +Run: `go test ./internal/metrics/ -count=1` +Expected: PASS. + +- [ ] **Step 5: Document** + +Invoke the `simple-english` skill. In `docs/architecture/metrics.md`: + +- Add `ObserveSnapshot` and `ObserveSnapshotCache` to the helper table. +- Add a section `## Snapshots` with a table of the five series (type, labels, meaning), copied from the spec's Metrics table. +- Add one sentence: the size histogram is the data for a later decision about a size limit. + +- [ ] **Step 6: Commit** + +```bash +git add internal/metrics/ docs/architecture/metrics.md +git commit -m "feat(metrics): add erofs snapshot series" -m "Signed-off-by: Xe Iaso " +``` + +--- + +### Task 4: `*tigris.Storer` implements `snapshot.Store` + +**Files:** + +- Create: `internal/storage/tigris/snapshot.go`, `internal/storage/tigris/snapshot_test.go` +- Modify: `internal/storage/tigris/tigris.go` (one field, one option, one line in the `Scoped` doc comment) + +**Interfaces:** + +- Consumes: `snapshot.Store`, `snapshot.File`, `snapshot.CacheID`, `snapshot.MetaSHA256` (Task 0); `(*PackCache).GetChecked`, `NewSnapshotCache` (Task 2). +- Produces: `func WithSnapshotCache(c *PackCache) Option`; `*Storer` satisfies `snapshot.Store`. + +- [ ] **Step 1: Write the failing tests `snapshot_test.go`** + +Read `client_test.go` first. Reuse `newFakeS3`, `newTestStorer`, and `countingObserver` (read its doc comment for the return shape). Tests: + +```go +package tigris + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "io/fs" + "testing" + + "github.com/tigrisdata/objgit/internal/snapshot" +) + +func putTestSnapshot(t *testing.T, s *Storer, key string, body []byte, sum string) { + t.Helper() + meta := map[string]string{snapshot.MetaFormat: "1"} + if sum != "" { + meta[snapshot.MetaSHA256] = sum + } + if err := s.PutSnapshot(context.Background(), key, bytes.NewReader(body), int64(len(body)), meta); err != nil { + t.Fatalf("PutSnapshot: %v", err) + } +} + +func digest(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) } + +func TestPutSnapshotKeyAndMetadata(t *testing.T) { + f := newFakeS3(t) + s := newTestStorer(t, f).Scoped("acme/widgets") + body := []byte("image") + putTestSnapshot(t, s, "snapshots/erofs/v1/abc.erofs", body, digest(body)) + + o, ok := f.objs["acme/widgets/snapshots/erofs/v1/abc.erofs"] + if !ok { + t.Fatalf("no object under the scoped key; have %v", keysOf(f)) + } + if !bytes.Equal(o.body, body) || o.meta[snapshot.MetaSHA256] != digest(body) { + t.Errorf("stored %q with meta %v", o.body, o.meta) + } +} + +func TestStatSnapshot(t *testing.T) { + tests := []struct { + name string + put bool + wantErr error + }{ + {"present", true, nil}, + {"absent", false, fs.ErrNotExist}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := newTestStorer(t, newFakeS3(t)) + if tt.put { + putTestSnapshot(t, s, "k.erofs", []byte("12345"), digest([]byte("12345"))) + } + size, err := s.StatSnapshot(context.Background(), "k.erofs") + if tt.wantErr != nil { + if !errors.Is(err, tt.wantErr) { + t.Fatalf("err = %v, want %v", err, tt.wantErr) + } + return + } + if err != nil || size != 5 { + t.Fatalf("StatSnapshot = %d, %v; want 5, nil", size, err) + } + }) + } +} + +func TestOpenSnapshot(t *testing.T) { + body := []byte("an erofs image, pretend") + tests := []struct { + name string + cache bool + sum string + put bool + wantErr error // nil means success; errAny means any error + }{ + {"cached", true, digest(body), true, nil}, + {"no cache", false, digest(body), true, nil}, + {"bad digest", true, digest([]byte("other")), true, errAny}, + {"bad digest no cache", false, digest([]byte("other")), true, errAny}, + {"missing digest", true, "", true, errAny}, + {"absent", true, "", false, fs.ErrNotExist}, + {"absent no cache", false, "", false, fs.ErrNotExist}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var opts []Option + var c *PackCache + if tt.cache { + var err error + c, err = NewSnapshotCache(t.TempDir(), 1<<20, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { c.Cleanup() }) + opts = append(opts, WithSnapshotCache(c)) + } + s := newTestStorer(t, newFakeS3(t), opts...) + if tt.put { + putTestSnapshot(t, s, "snapshots/erofs/v1/abc.erofs", body, tt.sum) + } + f, err := s.OpenSnapshot(context.Background(), "snapshots/erofs/v1/abc.erofs") + switch { + case tt.wantErr == errAny: + if err == nil { + f.Close() + t.Fatal("OpenSnapshot succeeded, want an error") + } + if c != nil && countCacheFiles(t, c) != 0 { + t.Error("the cache kept a file after a failed download") + } + return + case tt.wantErr != nil: + if !errors.Is(err, tt.wantErr) { + t.Fatalf("err = %v, want %v", err, tt.wantErr) + } + return + case err != nil: + t.Fatalf("OpenSnapshot: %v", err) + } + defer f.Close() + got := make([]byte, len(body)) + if _, err := f.ReadAt(got, 0); err != nil && !errors.Is(err, io.EOF) { + t.Fatalf("ReadAt: %v", err) + } + if !bytes.Equal(got, body) { + t.Errorf("read %q, want %q", got, body) + } + }) + } +} + +var errAny = errors.New("any error") + +func TestOpenSnapshotCachesDownload(t *testing.T) { + c, err := NewSnapshotCache(t.TempDir(), 1<<20, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { c.Cleanup() }) + f := newFakeS3(t) + obs, counts := countingObserver() + base := newTestStorer(t, f, WithSnapshotCache(c), obs) + widgets, gadgets := base.Scoped("acme/widgets"), base.Scoped("acme/gadgets") + + body := []byte("shared tree image") + putTestSnapshot(t, widgets, "snapshots/erofs/v1/abc.erofs", body, digest(body)) + putTestSnapshot(t, gadgets, "snapshots/erofs/v1/abc.erofs", body, digest(body)) + + for _, s := range []*Storer{widgets, widgets, gadgets} { + sf, err := s.OpenSnapshot(context.Background(), "snapshots/erofs/v1/abc.erofs") + if err != nil { + t.Fatalf("OpenSnapshot: %v", err) + } + sf.Close() + } + if got := counts()["GetObject"]; got != 1 { + t.Errorf("GetObject calls = %d, want 1", got) + } +} + +func TestSnapshotKeysInvisible(t *testing.T) { + s := newTestStorer(t, newFakeS3(t)) + body := []byte("x") + putTestSnapshot(t, s, "snapshots/erofs/v1/abc.erofs", body, digest(body)) + + refs, err := s.IterReferences() + if err != nil { + t.Fatal(err) + } + n := 0 + _ = refs.ForEach(func(*plumbing.Reference) error { n++; return nil }) + if n != 0 { + t.Errorf("IterReferences returned %d refs, want 0", n) + } + objs, err := s.IterEncodedObjects(plumbing.AnyObject) + if err != nil { + t.Fatal(err) + } + n = 0 + _ = objs.ForEach(func(plumbing.EncodedObject) error { n++; return nil }) + if n != 0 { + t.Errorf("IterEncodedObjects returned %d objects, want 0", n) + } +} +``` + +Adapt `countingObserver` usage to its real return shape. If no `keysOf` helper exists, build the key list inline for the error message. Add the `plumbing` import. + +- [ ] **Step 2: Run the tests to make sure they fail** + +Run: `go test ./internal/storage/tigris/ -run 'Snapshot'` +Expected: FAIL with `s.PutSnapshot undefined` and `undefined: WithSnapshotCache`. + +- [ ] **Step 3: Implement** + +In `tigris.go`, add a field to `Storer` after `cache`: + +```go + snapCache *PackCache // optional process-wide snapshot image cache; see snapshot.go +``` + +Add the option after `WithPackCache`: + +```go +// WithSnapshotCache installs the local cache that OpenSnapshot downloads +// images into. Like WithPackCache, pass one cache built once per process — +// see NewSnapshotCache. Scoped shares it. +func WithSnapshotCache(c *PackCache) Option { + return func(s *Storer) { s.snapCache = c } +} +``` + +In the `Scoped` doc comment, change "Two things are shared on purpose" to "Three things are shared on purpose", and add the snapshot cache, for the same reason as the pack cache. + +Create `snapshot.go`: + +```go +package tigris + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "io/fs" + "log/slog" + "os" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/tigrisdata/objgit/internal/snapshot" +) + +// Storer holds the snapshot images of its repository, next to the git data. +var _ snapshot.Store = (*Storer)(nil) + +// StatSnapshot sends one HeadObject for key under this Storer's prefix. +func (s *Storer) StatSnapshot(ctx context.Context, key string) (int64, error) { + start := time.Now() + out, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: sp(s.bucket), + Key: sp(s.prefix + key), + }) + s.observe("HeadObject", start, err) + switch { + case err == nil: + case isNotFound(err): + return 0, fmt.Errorf("tigris: snapshot %s: %w", key, fs.ErrNotExist) + default: + return 0, fmt.Errorf("tigris: head snapshot %s: %w", key, err) + } + if out.ContentLength == nil { + return 0, nil + } + return *out.ContentLength, nil +} + +// PutSnapshot sends one PutObject. It does not use the upload queue: no ref +// waits for an image, so there is nothing for a flush to order. +func (s *Storer) PutSnapshot(ctx context.Context, key string, body io.ReadSeeker, size int64, meta map[string]string) error { + start := time.Now() + _, err := s.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: sp(s.bucket), + Key: sp(s.prefix + key), + Body: body, + ContentLength: &size, + ContentType: sp("application/octet-stream"), + Metadata: meta, + }) + s.observe("PutObject", start, err) + if err != nil { + return fmt.Errorf("tigris: put snapshot %s: %w", key, err) + } + return nil +} + +// OpenSnapshot returns a local copy of the image at key. With a snapshot +// cache, the copy lives in the cache and later calls, from any repository +// that holds the same tree, open it with no request. Without one, the copy is +// a private unlinked temp file, as openWholePack does for packs. +// +// The body must match its erofs-sha256 metadata, so a bad download is never +// served and never cached. +func (s *Storer) OpenSnapshot(ctx context.Context, key string) (snapshot.File, error) { + fetch := s.snapshotFetch(ctx, key) + if s.snapCache != nil { + f, err := s.snapCache.GetChecked(snapshot.CacheID(key), fetch) + if err != nil { + return nil, err + } + return f, nil + } + + slog.Debug("no snapshot cache installed, downloading to a private temp file", "key", key) + f, err := os.CreateTemp("", "objgit-snapshot-*") + if err != nil { + return nil, fmt.Errorf("tigris: create snapshot temp file: %w", err) + } + os.Remove(f.Name()) // the descriptor keeps the data until Close + + h := sha256.New() + want, err := fetch(io.MultiWriter(f, h)) + if err == nil { + if got := hex.EncodeToString(h.Sum(nil)); got != want { + err = fmt.Errorf("tigris: snapshot %s failed checksum (got %s, want %s)", key, got, want) + } + } + if err != nil { + f.Close() + return nil, err + } + return f, nil +} + +// snapshotFetch returns the GetChecked fetch for key: it copies the body and +// returns the digest from the object's erofs-sha256 metadata. +func (s *Storer) snapshotFetch(ctx context.Context, key string) func(io.Writer) (string, error) { + return func(w io.Writer) (string, error) { + start := time.Now() + out, err := s.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: sp(s.bucket), + Key: sp(s.prefix + key), + }) + s.observe("GetObject", start, err) + switch { + case err == nil: + case isNotFound(err): + return "", fmt.Errorf("tigris: snapshot %s: %w", key, fs.ErrNotExist) + default: + return "", fmt.Errorf("tigris: get snapshot %s: %w", key, err) + } + defer out.Body.Close() + want := out.Metadata[snapshot.MetaSHA256] + if want == "" { + return "", fmt.Errorf("tigris: snapshot %s has no %s metadata", key, snapshot.MetaSHA256) + } + if _, err := io.Copy(w, out.Body); err != nil { + return "", fmt.Errorf("tigris: download snapshot %s: %w", key, err) + } + return want, nil + } +} +``` + +`sp` is in `tigris.go`. `ip` exists only in `client_test.go`, so production code takes the address of `size` directly. + +- [ ] **Step 4: Run the tests** + +Run: `go test ./internal/storage/tigris/ -count=1` +Expected: PASS for the whole package. + +- [ ] **Step 5: Commit** + +```bash +git add internal/storage/tigris/snapshot.go internal/storage/tigris/snapshot_test.go internal/storage/tigris/tigris.go +git commit -m "feat(storage/tigris): store erofs snapshots next to the repository" -m "Signed-off-by: Xe Iaso " +``` + +--- + +### Task 5: The push path + +**Files:** + +- Create: `cmd/objgitd/snapshots.go`, `cmd/objgitd/snapshots_test.go` +- Modify: `cmd/objgitd/hooks.go` (`snapshotRefs`, `receivePack`, `runHooks`), `cmd/objgitd/git_protocol.go` (daemon fields), `cmd/objgitd/hooks_test.go` (one new test) + +**Interfaces:** + +- Consumes: `snapshot.Ensure`, `snapshot.Open`, `snapshot.Store`, `snapshot.NewMemStore`, `snapshot.Key`, `snapshot.StatusBuilt`, `snapshot.StatusExists` (Task 1); `metrics.ObserveSnapshot` (Task 3). +- Produces: daemon fields `snapshots bool`, `snapshotTimeout time.Duration`, `snapshotTmpDir string`; `func (d *daemon) runSnapshots(repoPath string, st storage.Storer, updates []refUpdate, progress io.Writer)`; `func peelToTree(st storer.EncodedObjectStorer, h plumbing.Hash) (plumbing.Hash, bool, error)`. + +- [ ] **Step 1: Write the failing tests `snapshots_test.go`** + +Read `hooks_test.go` and `http_test.go` first: `runGit`, `tryGit`, `writeFile`, `memBase`, and how the hook tests capture logs (`syncBuffer`). Then write: + +```go +package main + +import ( + "bytes" + "context" + "errors" + "io/fs" + "net/http/httptest" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/go-git/go-billy/v6/memfs" + "github.com/go-git/go-git/v6/plumbing" + "github.com/go-git/go-git/v6/plumbing/object" + "github.com/go-git/go-git/v6/storage" + "github.com/go-git/go-git/v6/storage/memory" + "github.com/tigrisdata/objgit/internal/auth" + "github.com/tigrisdata/objgit/internal/repofs" + "github.com/tigrisdata/objgit/internal/snapshot" +) + +// snapStorer is an in-memory storer that also holds snapshot images, the +// way *tigris.Storer does. +type snapStorer struct { + *memory.Storage + *snapshot.MemStore +} + +// snapBase is memBase with a snapshot store per repository. +type snapBase struct { + mu sync.Mutex + repos map[string]*snapStorer +} + +func newSnapBase() *snapBase { return &snapBase{repos: map[string]*snapStorer{}} } + +func (b *snapBase) Scoped(prefix string) storage.Storer { + b.mu.Lock() + defer b.mu.Unlock() + st, ok := b.repos[prefix] + if !ok { + st = &snapStorer{Storage: memory.NewStorage(), MemStore: snapshot.NewMemStore()} + b.repos[prefix] = st + } + return st +} + +func (b *snapBase) repo(prefix string) *snapStorer { + b.mu.Lock() + defer b.mu.Unlock() + return b.repos[prefix] +} + +func newSnapshotDaemon(t *testing.T, base repofs.Base, on bool) *httptest.Server { + t.Helper() + d := &daemon{ + sysFS: memfs.New(), + resolver: repofs.BucketResolver{Base: base}, + authz: auth.AllowAnonymous{AllowWrite: true}, + snapshots: on, + snapshotTimeout: 30 * time.Second, + snapshotTmpDir: t.TempDir(), + } + ts := httptest.NewServer(d.httpHandler()) + t.Cleanup(ts.Close) + return ts +} + +func TestSmartHTTPPushSnapshots(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not on PATH") + } + base := newSnapBase() + ts := newSnapshotDaemon(t, base, true) + + work := seedRepo(t) + writeFile(t, filepath.Join(work, "README.md"), "hello snapshot\n") + writeFile(t, filepath.Join(work, "docs", "a.md"), "nested\n") + runGit(t, work, "add", ".") + runGit(t, work, "commit", "-m", "content") + runGit(t, work, "tag", "-a", "v1.0.0", "-m", "v1") + + out, err := tryGit(work, "push", ts.URL+"/acme/snap.git", "main", "v1.0.0") + if err != nil { + t.Fatalf("push: %v\n%s", err, out) + } + treeHex := strings.TrimSpace(runGit(t, work, "rev-parse", "HEAD^{tree}")) + tree := plumbing.NewHash(treeHex) + + repo := base.repo("acme/snap") + if keys := repo.Keys(); len(keys) != 1 || keys[0] != snapshot.Key(tree) { + t.Fatalf("snapshot keys = %v, want [%s]", keys, snapshot.Key(tree)) + } + snap, err := snapshot.Open(context.Background(), repo, tree) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer snap.Close() + got, err := fs.ReadFile(snap, "docs/a.md") + if err != nil || string(got) != "nested\n" { + t.Errorf("docs/a.md = %q, %v", got, err) + } + for _, want := range []string{ + "objgit: snapshot refs/heads/main (tree " + treeHex[:7] + "): built", + "objgit: snapshot refs/tags/v1.0.0 (tree " + treeHex[:7] + "): exists", + } { + if !strings.Contains(out, want) { + t.Errorf("push output missing %q:\n%s", want, out) + } + } +} + +func TestPushSnapshotsOffByDefault(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not on PATH") + } + base := newSnapBase() + ts := newSnapshotDaemon(t, base, false) + work := seedRepo(t) + if out, err := tryGit(work, "push", ts.URL+"/acme/off.git", "main"); err != nil { + t.Fatalf("push: %v\n%s", err, out) + } + if keys := base.repo("acme/off").Keys(); len(keys) != 0 { + t.Errorf("snapshot keys = %v, want none", keys) + } +} + +func TestRunSnapshotsPutFailure(t *testing.T) { + st := &snapStorer{Storage: memory.NewStorage(), MemStore: snapshot.NewMemStore()} + st.PutErr = errors.New("bucket down") + commit := writeCommit(t, st.Storage) + + d := &daemon{snapshots: true, snapshotTimeout: 10 * time.Second, snapshotTmpDir: t.TempDir()} + var progress bytes.Buffer + d.runSnapshots("acme/x", st, []refUpdate{{Name: plumbing.NewBranchReferenceName("main"), New: commit}}, &progress) + + if !strings.Contains(progress.String(), "): failed") { + t.Errorf("progress = %q, want a failed line", progress.String()) + } + if strings.Contains(progress.String(), "bucket down") { + t.Errorf("progress leaks the error text: %q", progress.String()) + } +} + +func TestRunSnapshotsSkipsStorerWithoutStore(t *testing.T) { + st := memory.NewStorage() + commit := writeCommit(t, st) + d := &daemon{snapshots: true, snapshotTimeout: 10 * time.Second} + var progress bytes.Buffer + d.runSnapshots("acme/x", st, []refUpdate{{Name: plumbing.NewBranchReferenceName("main"), New: commit}}, &progress) + if progress.Len() != 0 { + t.Errorf("progress = %q, want nothing", progress.String()) + } +} + +func TestPeelToTree(t *testing.T) { + st := memory.NewStorage() + commit := writeCommit(t, st) + c, err := object.GetCommit(st, commit) + if err != nil { + t.Fatal(err) + } + tagOfCommit := writeTag(t, st, "v1", commit, plumbing.CommitObject) + tagOfTag := writeTag(t, st, "v1-again", tagOfCommit, plumbing.TagObject) + tagOfTree := writeTag(t, st, "tree-tag", c.TreeHash, plumbing.TreeObject) + + tests := []struct { + name string + h plumbing.Hash + want plumbing.Hash + wantOK bool + }{ + {"commit", commit, c.TreeHash, true}, + {"tag of commit", tagOfCommit, c.TreeHash, true}, + {"tag of tag", tagOfTag, c.TreeHash, true}, + {"tag of tree", tagOfTree, plumbing.ZeroHash, false}, + {"tree", c.TreeHash, plumbing.ZeroHash, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok, err := peelToTree(st, tt.h) + if err != nil { + t.Fatalf("peelToTree: %v", err) + } + if ok != tt.wantOK || got != tt.want { + t.Errorf("peelToTree = %s, %v; want %s, %v", got, ok, tt.want, tt.wantOK) + } + }) + } +} + +// writeCommit stores a commit with a one-file tree and returns its hash. +func writeCommit(t *testing.T, st *memory.Storage) plumbing.Hash { + t.Helper() + blob := st.NewEncodedObject() + blob.SetType(plumbing.BlobObject) + w, _ := blob.Writer() + w.Write([]byte("hi\n")) + w.Close() + bh, err := st.SetEncodedObject(blob) + if err != nil { + t.Fatal(err) + } + tree := &object.Tree{Entries: []object.TreeEntry{{Name: "a", Mode: 0o100644, Hash: bh}}} + to := st.NewEncodedObject() + if err := tree.Encode(to); err != nil { + t.Fatal(err) + } + th, err := st.SetEncodedObject(to) + if err != nil { + t.Fatal(err) + } + sig := object.Signature{Name: "T", Email: "t@example.com", When: time.Unix(0, 0)} + c := &object.Commit{Author: sig, Committer: sig, Message: "m", TreeHash: th} + co := st.NewEncodedObject() + if err := c.Encode(co); err != nil { + t.Fatal(err) + } + ch, err := st.SetEncodedObject(co) + if err != nil { + t.Fatal(err) + } + return ch +} + +func writeTag(t *testing.T, st *memory.Storage, name string, target plumbing.Hash, typ plumbing.ObjectType) plumbing.Hash { + t.Helper() + sig := object.Signature{Name: "T", Email: "t@example.com", When: time.Unix(0, 0)} + tag := &object.Tag{Name: name, Tagger: sig, Message: name, TargetType: typ, Target: target} + o := st.NewEncodedObject() + if err := tag.Encode(o); err != nil { + t.Fatal(err) + } + h, err := st.SetEncodedObject(o) + if err != nil { + t.Fatal(err) + } + return h +} +``` + +Use the real `filemode.Regular` constant in `writeCommit` instead of `0o100644` if the literal does not compile. If `runGit` does not return the output, use `exec.Command("git", "-C", work, "rev-parse", "HEAD^{tree}").Output()`. + +Add `TestHooksIgnoreTags` to `hooks_test.go`, next to the existing HTTP hook test. Copy its setup (the log capture, the daemon with `allowHooks: true`, and the hook script). Then: + +1. Push `main` and make sure that the logs contain `hook: running` once. +2. Run `git tag -a v1 -m v1`, and push `v1` only. +3. Make sure that the count of `hook: running` in the logs is still 1. + +- [ ] **Step 2: Run the tests to make sure they fail** + +Run: `go test ./cmd/objgitd/ -run 'Snapshot|PeelToTree|HooksIgnoreTags'` +Expected: FAIL to compile, with `unknown field snapshots` and `undefined: peelToTree`. + +- [ ] **Step 3: Add the daemon fields in `git_protocol.go`** + +After `hookTimeout`: + +```go + // snapshots gates building an erofs image of each updated ref tip after + // a push; see snapshots.go. + snapshots bool + snapshotTimeout time.Duration + snapshotTmpDir string // parent directory for image temp files; "" is the OS temp dir +``` + +- [ ] **Step 4: Write `snapshots.go`** + +```go +package main + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "slices" + "strings" + "time" + + "github.com/go-git/go-git/v6/plumbing" + "github.com/go-git/go-git/v6/plumbing/object" + "github.com/go-git/go-git/v6/plumbing/storer" + "github.com/go-git/go-git/v6/storage" + "github.com/tigrisdata/objgit/internal/metrics" + "github.com/tigrisdata/objgit/internal/snapshot" +) + +// maxTagDepth bounds how many annotated tags peelToTree follows. +const maxTagDepth = 16 + +// runSnapshots makes sure that an erofs image exists for the tree at the tip +// of each updated ref. It runs synchronously inside onUpdated, so the push +// waits for it, and it writes one progress line per ref. A failure is logged +// and counted and never fails the push: the refs are already committed. +func (d *daemon) runSnapshots(repoPath string, st storage.Storer, updates []refUpdate, progress io.Writer) { + store, ok := st.(snapshot.Store) + if !ok { + slog.Debug("snapshot: storer holds no snapshot store, skipping", "repo", repoPath) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), d.snapshotTimeout) + defer cancel() + + // diffRefs ranges over a map, so its order is random. Branches go first, + // then tags, each by name, so the ref that builds a shared tree (and the + // progress output) is the same on every push. + ordered := slices.Clone(updates) + slices.SortFunc(ordered, func(a, b refUpdate) int { + if a.Name.IsBranch() != b.Name.IsBranch() { + if a.Name.IsBranch() { + return -1 + } + return 1 + } + return strings.Compare(a.Name.String(), b.Name.String()) + }) + + // A branch and a tag on one commit share one tree, and one build. + seen := map[plumbing.Hash]string{} + for _, u := range ordered { + if u.New.IsZero() { + continue + } + log := slog.With("repo", repoPath, "ref", u.Name.String(), "sha", u.New.String()) + + tree, ok, err := peelToTree(st, u.New) + if err != nil { + log.Error("snapshot: resolve tree", "err", err) + metrics.ObserveSnapshot("error", 0, 0) + continue + } + if !ok { + log.Debug("snapshot: ref does not point at a commit, skipping") + continue + } + if status, dup := seen[tree]; dup { + if status == snapshot.StatusBuilt { + status = snapshot.StatusExists + } + writeSnapshotLine(progress, u.Name, tree, status) + continue + } + + res, err := snapshot.Ensure(ctx, st, store, tree, d.snapshotTmpDir) + if err != nil { + log.Error("snapshot: build failed", "tree", tree.String(), "dur", res.Elapsed, "err", err) + metrics.ObserveSnapshot("error", res.Elapsed, 0) + seen[tree] = "failed" + writeSnapshotLine(progress, u.Name, tree, "failed") + continue + } + metrics.ObserveSnapshot(res.Status, res.Elapsed, res.Bytes) + seen[tree] = res.Status + if res.Status == snapshot.StatusBuilt { + log.Info("snapshot: built", "key", res.Key, "files", res.Files, "bytes", res.Bytes, "dur", res.Elapsed) + writeSnapshotLine(progress, u.Name, tree, fmt.Sprintf("built, %d files, %s, %s", + res.Files, humanBytes(res.Bytes), res.Elapsed.Round(100*time.Millisecond))) + continue + } + writeSnapshotLine(progress, u.Name, tree, res.Status) + } +} + +// writeSnapshotLine writes one "remote: objgit: snapshot ..." line. It never +// carries error text, which can name bucket internals; the log has that. +func writeSnapshotLine(progress io.Writer, ref plumbing.ReferenceName, tree plumbing.Hash, status string) { + if progress == nil { + return + } + fmt.Fprintf(progress, "objgit: snapshot %s (tree %s): %s\n", ref, tree.String()[:7], status) +} + +// peelToTree returns the tree of the commit that h names, following annotated +// tags. ok is false, with no error, when h does not lead to a commit (a tag of +// a tree or a blob, or a tree itself), since there is then nothing to snapshot. +func peelToTree(st storer.EncodedObjectStorer, h plumbing.Hash) (plumbing.Hash, bool, error) { + for range maxTagDepth { + obj, err := st.EncodedObject(plumbing.AnyObject, h) + if err != nil { + return plumbing.ZeroHash, false, fmt.Errorf("load %s: %w", h, err) + } + switch obj.Type() { + case plumbing.CommitObject: + c, err := object.DecodeCommit(st, obj) + if err != nil { + return plumbing.ZeroHash, false, fmt.Errorf("decode commit %s: %w", h, err) + } + return c.TreeHash, true, nil + case plumbing.TagObject: + tag, err := object.DecodeTag(st, obj) + if err != nil { + return plumbing.ZeroHash, false, fmt.Errorf("decode tag %s: %w", h, err) + } + h = tag.Target + default: + return plumbing.ZeroHash, false, nil + } + } + return plumbing.ZeroHash, false, errors.New("tag chain is too deep") +} + +// humanBytes formats n in binary units with one decimal, e.g. "5.6 MiB". +func humanBytes(n int64) string { + const unit = 1024 + if n < unit { + return fmt.Sprintf("%d B", n) + } + div, exp := int64(unit), 0 + for m := n / unit; m >= unit; m /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp]) +} +``` + +- [ ] **Step 5: Change `hooks.go`** + +1. `snapshotRefs`: keep branches **and** tags: + + ```go + if r.Type() == plumbing.HashReference && (r.Name().IsBranch() || r.Name().IsTag()) { + ``` + + Update its doc comment: it returns every branch and tag ref. `runHooks` filters to branches, and snapshots use both. + +2. `receivePack`: take the before snapshot, and set `onUpdated`, when hooks or snapshots are on: + + ```go + if !d.allowHooks && !d.snapshots { + err := receivePackStreaming(ctx, st, r, w, req, d.pushes.admit, nil) + d.healHEADAfterPush(err, st, repoPath) + return err + } + ``` + + Change the log messages that start with `hook: ref snapshot` to start with `push: ref snapshot`, because they now serve both features. In `onUpdated`, replace the `d.runHooks(...)` call with: + + ```go + if d.snapshots { + d.runSnapshots(repoPath, st, updates, progress) + } + if d.allowHooks { + d.runHooks(repoPath, "receive-pack", st, updates, progress) + } + ``` + + Update the doc comment of `receivePack` to name both features. + +3. `runHooks`: skip every update that is not a branch: + + ```go + if u.New.IsZero() || !u.Name.IsBranch() { + continue // deletion, or a tag: hooks run for branches only + } + ``` + +- [ ] **Step 6: Run the tests** + +Run: `go test ./cmd/objgitd/ -count=1` +Expected: PASS for the whole package, including the hook tests that existed before. + +- [ ] **Step 7: Commit** + +```bash +git add cmd/objgitd/snapshots.go cmd/objgitd/snapshots_test.go cmd/objgitd/hooks.go cmd/objgitd/hooks_test.go cmd/objgitd/git_protocol.go +git commit -m "feat(objgitd): build erofs snapshots of pushed ref tips" -m "Signed-off-by: Xe Iaso " +``` + +--- + +### Task 6: `main.go` wiring + +**Files:** + +- Modify: `cmd/objgitd/main.go` + +**Interfaces:** + +- Consumes: `tigris.NewSnapshotCache`, `tigris.WithSnapshotCache`, `(*PackCache).EvictIdle`, `(*PackCache).Cleanup` (Tasks 2, 4); `metrics.ObserveSnapshotCache` (Task 3); daemon fields (Task 5). + +- [ ] **Step 1: Add the flags**, after `packCacheBytes`: + +```go + erofsSnapshots = flag.Bool("erofs-snapshots", false, "build a zstd-compressed erofs image of the tree at each updated branch and tag tip after a push, and store it next to the repository") + snapshotTimeout = flag.Duration("snapshot-timeout", 2*time.Minute, "wall-clock limit for the erofs snapshots of one push") + snapshotCacheBytes = flag.Int64("snapshot-cache-bytes", 2<<30, "disk budget for the local erofs snapshot cache, least-recently-used eviction; 0 disables caching") + snapshotCacheMaxIdle = flag.Duration("snapshot-cache-max-idle", time.Hour, "evict a cached erofs snapshot that nobody opened for this long; 0 disables the idle sweep") +``` + +- [ ] **Step 2: Build the cache**, after the pack cache block: + +```go + // The snapshot cache sits next to the pack cache, under the same parent, + // with its own budget. Open downloads whole images into it. + var snapCache *tigris.PackCache + if *snapshotCacheBytes > 0 { + snapCache, err = tigris.NewSnapshotCache(*packCacheDir, *snapshotCacheBytes, metrics.ObserveSnapshotCache) + if err != nil { + slog.Error("can't create snapshot cache", "pack_cache_dir", *packCacheDir, "err", err) + os.Exit(1) + } + storerOpts = append(storerOpts, tigris.WithSnapshotCache(snapCache)) + } +``` + +- [ ] **Step 3: Set the daemon fields**: + +```go + snapshots: *erofsSnapshots, + snapshotTimeout: *snapshotTimeout, + snapshotTmpDir: *packCacheDir, +``` + +Add `"erofs_snapshots", *erofsSnapshots` and `"snapshot_cache_bytes", *snapshotCacheBytes` to the `objgitd listening` log line. + +- [ ] **Step 4: Start the idle sweep**, after `g, gCtx := errgroup.WithContext(ctx)`: + +```go + if snapCache != nil && *snapshotCacheMaxIdle > 0 { + g.Go(func() error { + tick := time.NewTicker(*snapshotCacheMaxIdle / 4) + defer tick.Stop() + for { + select { + case <-gCtx.Done(): + return nil + case <-tick.C: + if n := snapCache.EvictIdle(*snapshotCacheMaxIdle); n > 0 { + slog.Debug("evicted idle snapshots", "count", n) + } + } + } + }) + } +``` + +- [ ] **Step 5: Clean up at shutdown**, next to the pack cache cleanup: + +```go + if cerr := snapCache.Cleanup(); cerr != nil { + slog.Warn("can't remove the snapshot cache directory", "err", cerr) + } +``` + +`Cleanup` accepts a nil receiver, so this is safe when the cache is off. + +- [ ] **Step 6: Build and vet** + +Run: `go build ./... && go vet ./cmd/objgitd/` +Expected: success. + +Run: `go run ./cmd/objgitd -h 2>&1 | grep -E 'erofs-snapshots|snapshot-'` +Expected: the four new flags are listed. + +- [ ] **Step 7: Commit** + +```bash +git add cmd/objgitd/main.go +git commit -m "feat(objgitd): wire erofs snapshot flags and cache" -m "Signed-off-by: Xe Iaso " +``` + +--- + +### Task 7: Documentation + +**Files:** + +- Create: `docs/architecture/snapshots.md` +- Modify: `docs/architecture/tigris-storer.md`, `docs/architecture/README.md`, `AGENTS.md` + +Invoke the `simple-english` skill before you write. Copy facts from the spec. Do not add facts that are not in the spec or the code. + +- [ ] **Step 1: Write `docs/architecture/snapshots.md`** + +Sections, in this order. Each one is descriptive text, with tables where the spec has tables: + +1. `# erofs snapshots (internal/snapshot)`: what the feature does, in three sentences. Name the flag `-erofs-snapshots`, which is off by default. +2. `## Object layout`: the key, why it is the tree hash, the `v1` rule, and the metadata table. +3. `## The mapping from git to EROFS`: the mode table, the gitlink rule, and the two limits table. +4. `## Ensure`: the steps, and why it takes no lock. +5. `## Open and the snapshot cache`: the local download, `GetChecked`, the budget, the idle sweep, the fallback without a cache, and why a shared cache id is not leakage. +6. `## The push path`: `runSnapshots`, the order (snapshots, then hooks), the progress lines, and the rule that a failure never fails a push. +7. `## Flags`: the flag table from the spec. +8. `## Known risks`: copy the list from the spec. +9. `## The path to a message queue`: copy from the spec. + +- [ ] **Step 2: Update the other pages** + +- `docs/architecture/tigris-storer.md`: + - Add `snapshots/erofs/v1/.erofs` to the layout table. + - In "The pack cache", add a paragraph about the second instance: `NewSnapshotCache`, `GetChecked`, `EvictIdle`, and the observer. Link to `snapshots.md`. +- `docs/architecture/README.md`: add a `snapshots.md` row to the pages table. +- `AGENTS.md`: + - Add `internal/snapshot` to "Where the code lives". + - Add `cmd/objgitd/snapshots.go` to the same table. + - Add a `snapshots.md` row to the architecture table: "Read it before you change snapshot images, the snapshot cache, or `runSnapshots`." + +- [ ] **Step 3: Self-check** + +Run the self-check of the `simple-english` skill on every changed paragraph: sentence length, no `should`, no semicolons, and conditions before commands. + +- [ ] **Step 4: Commit** + +```bash +git add docs/architecture/snapshots.md docs/architecture/tigris-storer.md docs/architecture/README.md AGENTS.md +git commit -m "docs: describe erofs snapshots and the snapshot cache" -m "Signed-off-by: Xe Iaso " +``` + +--- + +### Task 8: Whole-branch check (the controller does this) + +- [ ] **Step 1: Full test suite** + +Run: `go vet ./... && go test ./... -count=1` +Expected: PASS. + +- [ ] **Step 2: Whole-branch review** + +Dispatch one reviewer on `git diff origin/main...HEAD` against the spec. Fix the findings, then run Step 1 again. diff --git a/docs/superpowers/specs/2026-09-23-erofs-snapshots-design.md b/docs/superpowers/specs/2026-09-23-erofs-snapshots-design.md new file mode 100644 index 0000000..cdaa5e9 --- /dev/null +++ b/docs/superpowers/specs/2026-09-23-erofs-snapshots-design.md @@ -0,0 +1,473 @@ +# Design: erofs snapshots of pushed trees + +## Goal + +After a push, `objgitd` builds one EROFS image of the tree at the tip of each +updated ref. It stores the image in the bucket, under the prefix of the +repository. A later web UI opens the image and reads directories and files +from it, with no git object walk. + +The package is `github.com/Xe/erofs`. The design started on v0.6.1. The code +now needs v0.8.0, for the streaming builder. + +## Decisions + +These decisions come from the design discussion on 2026-09-23. + +| Question | Decision | +| ------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Which commits get images | The new tip of each updated branch and tag, on push. Other commits get an image on demand, through the same `Ensure` function, when a later feature asks for one. | +| When the build runs | Synchronously, inside the push, after report-status. A message queue replaces this later. | +| Size limit | None. Every tree gets an image. We measure first, then decide on a limit. | +| Compression | Zstandard, at the default level of the builder. | +| How a reader gets bytes | `Open` downloads the whole image into a local cache directory, next to the pack cache. The cache evicts least-recently-used images when it passes its byte budget, and on a timer when an image is idle too long. | +| Builder memory | Bounded. `Ensure` uses `AddFileFunc` from erofs v0.8.0, which implements [2026-09-23-erofs-streaming-builder-design.md](2026-09-23-erofs-streaming-builder-design.md). | + +## Non-goals + +- A web UI. This design only produces images and gives a function to open one. +- Garbage collection of images in the bucket. Packs have no GC either. +- A backfill command for repositories that exist before this change. `Ensure` + makes this simple to add later. +- Images of commits that are not ref tips, at push time. +- Putting a new image into the local cache at build time. The first `Open` + after a push downloads it. + +## Object layout + +One new key family goes into the layout table in +[../../architecture/tigris-storer.md](../../architecture/tigris-storer.md): + +| Key | Contents | +| --------------------------------- | ----------------------------------------- | +| `snapshots/erofs/v1/.erofs` | One EROFS image of the git tree ``. | + +The key uses the **tree** hash, not the commit hash. Many commits can point at +one tree: a branch and a tag on one commit, a revert, or a merge with no +change. These commits share one image. + +For one version of the builder, the image bytes depend only on the tree and on +the `v1` mapping below. The builder pins every mtime with +`erofs.WithEpoch(time.Unix(0, 0))`. As a result, two builds of one tree give +identical bytes, and a second build is never necessary. + +`v1` names the mapping from git to EROFS, and the build options. A change to +either (for example, a different compression algorithm, or a new gitlink rule) +uses `v2`. A `v2` build never overwrites a `v1` image, and a reader asks for +the version that it knows. + +The key does not start with `objects/`, `packs/`, or `refs/`. As a result, the +pack index listing, the loose object reads, and the loose ref listing never +see it. + +Each image carries user metadata: + +| Metadata key | Value | +| -------------- | ------------------------------------------------------------------------------------- | +| `erofs-format` | `1` | +| `erofs-sha256` | The hex SHA-256 of the image bytes. The local cache uses it to reject a bad download. | +| `git-tree` | The hex tree hash. | +| `erofs-files` | The count of regular files. | + +The web UI finds an image in two steps. It resolves the commit to its tree +through the git storer. Then it opens the key for that tree. + +## The mapping from git to EROFS + +| Git entry mode | EROFS entry | +| ---------------- | ---------------------------------------- | +| `040000` tree | Directory, mode `0755`. | +| `100644` blob | Regular file, mode `0644`. | +| `100755` blob | Regular file, mode `0755`. | +| `120000` symlink | Symlink. The target is the blob content. | +| `160000` gitlink | Empty directory, mode `0755`. | + +The gitlink rule is the same result that `git checkout` gives for a submodule +that is not initialized. The image does not record the submodule commit. A +reader that needs it reads the tree entry from git. + +The owner and the group of every entry are 0. The root directory has mode +`0755`. + +Two EROFS limits are stricter than git: + +| Limit | EROFS | Git | Where the limit is | +| ------------------- | ------------------------ | -------------- | ------------------------------------------------------------------------------------------- | +| Entry name length | 255 bytes | No fixed limit | `ondisk.NameLen`. The builder does not check it. `Validate` does. | +| Symlink target size | 1020 bytes (`NameLen*4`) | No fixed limit | `readSymlink` in `inode.go`. The builder writes a longer target, but the reader refuses it. | + +`Ensure` checks both limits before it adds the entry. If an entry is over a +limit, the build fails with an error that names the path. It does not skip the +entry, because an image with a missing file is worse than no image. + +## The `internal/snapshot` package + +One new package holds the build and the open. It depends on go-git and on +`github.com/Xe/erofs`. It does not import `internal/storage/tigris`. + +### `Store` + +```go +// Store holds snapshot images for one repository. Keys are relative to the +// repository prefix. +type Store interface { + // StatSnapshot returns fs.ErrNotExist when key is absent. + StatSnapshot(ctx context.Context, key string) (size int64, err error) + PutSnapshot(ctx context.Context, key string, body io.ReadSeeker, size int64, meta map[string]string) error + // OpenSnapshot returns a local copy of the image at key. The caller + // must close it. It returns fs.ErrNotExist when key is absent. + OpenSnapshot(ctx context.Context, key string) (File, error) +} + +// File is a local image. *os.File satisfies it. +type File interface { + io.ReaderAt + io.Closer +} +``` + +`*tigris.Storer` implements `Store` on its scoped prefix, in a new file +`internal/storage/tigris/snapshot.go`: + +- `StatSnapshot` sends one `HeadObject`. +- `PutSnapshot` sends one `PutObject` directly. It does not use the upload + queue, because no ref waits for the image. +- `OpenSnapshot` gets the image through the snapshot cache. See "The snapshot + cache". + +The daemon gets the store with a type assertion, `st.(snapshot.Store)`. This is +the same optional-method pattern that `UpdateReferences` uses. If the storer +does not implement `Store`, the daemon skips snapshots and writes one debug +log line. The protocol tests use `memory.NewStorage()`, so they wrap it in a +test storer that adds a map-backed `Store`. + +### `Ensure` + +```go +type Result struct { + Key string + Status string // "built" or "exists" + Files int + Bytes int64 + Elapsed time.Duration +} + +func Ensure(ctx context.Context, objs storer.EncodedObjectStorer, store Store, tree plumbing.Hash, tmpDir string) (Result, error) +``` + +`Ensure` is the one unit of work. The push path calls it now. A web UI, a +backfill, or a queue consumer calls it later, with no change to it. + +The steps: + +1. Call `StatSnapshot` on the key. If the key exists, return `exists`. +2. Create a temp file in `tmpDir` with `os.CreateTemp`. Do not name the file + after the tree hash. Two pushes of one tree can run at the same time. The + CAUTION about content-addressed temp paths in + [../../architecture/tigris-storer.md](../../architecture/tigris-storer.md) + applies here too. +3. Make an `erofs.NewBuilder` on the file, with these options: + - `WithBlockSize(12)`, for 4096-byte blocks + - `WithEpoch(time.Unix(0, 0))` + - `WithCompression(erofs.CompressionZstd)` +4. Walk the tree with `object.GetTree` and the tree entries, depth first. + Add each entry through `AddDir`, `AddFileFunc`, or `AddSymlink`, as the + mapping gives. For a file, read only the size with `EncodedObjectSize`. + The open function loads the blob with `object.GetBlob` and returns + `blob.Reader()`, so the builder reads the content during `Build`. +5. Call `Build`. +6. Rewind the file and compute its SHA-256. +7. Rewind the file again. Call `PutSnapshot` with the file, its size, and the + metadata. +8. Remove the temp file. + +The builder keeps the compressed form of a file only when that form is +smaller. It stores files of one block or less, and files with a known +compressed extension (for example `.png` or `.gz`), without compression. The +builder decides this internally, and `Ensure` does not change it. + +Two pushes of one tree at the same time both build, and both put identical +bytes. The second `PutObject` replaces the first with the same content. This +is correct, so `Ensure` uses no lock and no conditional write. + +The walk uses tree entries and not `tree.Files()`. The `FileIter` of go-git +skips gitlinks (`plumbing/object/file.go`), so an image built from it has no +submodule directories. + +### `Open` + +```go +// Snapshot is an open image. Close releases the local file. +type Snapshot struct { + *erofs.FS + f File +} + +func (s *Snapshot) Close() error + +func Open(ctx context.Context, store Store, tree plumbing.Hash) (*Snapshot, error) +``` + +`Open` calls `OpenSnapshot` for the key of `tree`, then calls `erofs.Open` on +the local file. Every read after that is a local read. A directory listing +does many small reads, so a local file is necessary for a usable web UI. + +If the key is absent, `Open` returns `fs.ErrNotExist`. It does not build the +image. A caller that wants the image to exist calls `Ensure` first. + +## The snapshot cache + +The snapshot cache is a second `*tigris.PackCache`, with its own directory +and its own budget. It uses the same code as the pack cache, so it has the +same properties: + +- One download for each image, however many callers ask at the same time. +- A byte budget. When a new image makes the total pass the budget, the cache + deletes the least-recently-used images until the total is in the budget + again. +- Eviction is an unlink. A `Snapshot` that is open keeps its descriptor, so a + reader never breaks. The disk space returns when the last reader closes. +- A failed download is not cached. +- Each `objgitd` process starts with an empty cache directory and removes it + at shutdown. + +The directory is `objgit-snapshots-*`, in the same parent as the pack cache +(`-pack-cache-dir`, or the OS temp directory). + +### Changes to `PackCache` + +`PackCache` needs four small changes. The pack cache does not use the new +functions, so its behavior does not change. + +1. **A name for the directory.** `NewPackCache` makes `objgit-packs-*` today. + Add an unexported constructor that takes the directory pattern. + `NewPackCache` calls it with `objgit-packs-*`. A new `NewSnapshotCache` + calls it with `objgit-snapshots-*`. + +2. **A digest from the fetch.** `Get(id, fetch)` requires `id` to be the + SHA-256 of the bytes. A snapshot id is the tree hash, so this is not true. + Add this function: + + ```go + // GetChecked is Get, but fetch returns the SHA-256 that the bytes must have. + func (c *PackCache) GetChecked(id string, fetch func(io.Writer) (wantSHA256 string, err error)) (*os.File, error) + ``` + + `verifiedCopy` compares against the returned digest instead of `id`. `Get` + becomes `GetChecked` with a fetch that returns `id`. The snapshot fetch + sends `GetObject`, copies the body, and returns the `erofs-sha256` + metadata value. If the metadata is absent, the fetch returns an error, and + the cache keeps nothing. + +3. **An idle sweep.** Add a wall-clock stamp, `usedAt`, next to the `used` + sequence number. `claim` sets both. Then add this function: + + ```go + // EvictIdle unlinks every settled entry that no caller has asked for + // since maxIdle ago. It never touches a download that has not settled. + func (c *PackCache) EvictIdle(maxIdle time.Duration) int + ``` + +4. **An observer.** `internal/storage/tigris` does not import + `internal/metrics`. It takes callbacks, as `WithObserver` does. Add an + optional `observe func(event string)` field. `NewSnapshotCache` takes it as + a parameter, and `NewPackCache` leaves it nil. The cache calls it with one + of these events: + + | Event | When | + | -------------- | -------------------------------------------------- | + | `hit` | `GetChecked` returns a file with no download. | + | `miss` | `GetChecked` starts a download. | + | `evict_budget` | `evictLocked` unlinks an entry to meet the budget. | + | `evict_idle` | `EvictIdle` unlinks an entry. | + + The callback runs outside the cache lock. `main.go` wires it to + `metrics.ObserveSnapshotCache`. + +The snapshot cache id is `erofs-v1-`. It does not contain the +repository prefix, so two repositories that hold one tree share one local +file. This is deduplication and not leakage, for the same reason as for +packs: a caller reaches the id only through a tree that its own repository +holds, and the bytes are an image of that tree. + +### Wiring + +`tigris.WithSnapshotCache(*PackCache)` is a new `Option`, like +`WithPackCache`. `Scoped` shares the snapshot cache, as it shares the pack +cache. + +If the storer has no snapshot cache, `OpenSnapshot` downloads the image to a +private temp file and unlinks it at once. The returned descriptor is the only +reference. This is the same fallback that `fetchWholePack` uses. + +`main.go` makes the snapshot cache when `-snapshot-cache-bytes` is more than 0. It starts one goroutine in the errgroup. The goroutine calls +`EvictIdle(*snapshotCacheMaxIdle)` every `*snapshotCacheMaxIdle / 4`, and +stops when the context is done. At shutdown, `main.go` calls `Cleanup` on the +snapshot cache, next to the call for the pack cache. + +## Daemon wiring + +### Flags + +| Flag | Default | Environment | Meaning | +| -------------------------- | ------- | ------------------------- | ----------------------------------------------------------------------------------------- | +| `-erofs-snapshots` | `true` | `EROFS_SNAPSHOTS` | Build an image for each updated ref tip after a push. | +| `-snapshot-timeout` | `2m` | `SNAPSHOT_TIMEOUT` | Wall-clock limit for the snapshots of one push. | +| `-snapshot-cache-bytes` | `2 GiB` | `SNAPSHOT_CACHE_BYTES` | Disk budget for the local snapshot cache. `0` disables the cache. | +| `-snapshot-cache-max-idle` | `1h` | `SNAPSHOT_CACHE_MAX_IDLE` | The cache deletes an image that nobody opened for this long. `0` disables the idle sweep. | + +`*daemon` gets two fields, `snapshots bool` and `snapshotTimeout +time.Duration`. The temp directory for builds is `-pack-cache-dir`, or the OS +temp directory when that flag is empty. + +### The push path + +`(*daemon).receivePack` in `cmd/objgitd/hooks.go` changes in four places: + +1. It takes the ref snapshot before the push, and it sets `onUpdated`, when + `d.allowHooks || d.snapshots`. Today it does this only for hooks. +2. `snapshotRefs` returns branches **and** tags. `runHooks` skips every update + that is not a branch, so the hook behavior does not change. +3. `onUpdated` calls `d.runSnapshots` first, then `d.runHooks`. +4. `d.runSnapshots` does these steps for each update that is not a deletion: + 1. Peel the new hash to a commit. An annotated tag goes through + `object.GetTag` and `Tag.Commit`. A tag that points at a tree or a blob + is skipped. + 2. Get the tree hash of the commit. + 3. Skip a tree that this push already handled. + 4. Call `snapshot.Ensure`. + +All snapshots of one push share one context, from +`context.WithTimeout(context.Background(), d.snapshotTimeout)`. This is the +same pattern that `runHook` uses. + +Snapshots run before hooks, so a hook can print a link to an image that +already exists. + +The push slot from `-max-concurrent-pushes` stays held while `onUpdated` +runs. The push cap therefore also bounds the count of concurrent builds. + +### Progress output + +If the client negotiated sideband, each ref writes one line to `progress`: + +```text +remote: objgit: snapshot refs/heads/main (tree 3d4e5f6): built, 1234 files, 5.6 MiB, 1.2s +remote: objgit: snapshot refs/tags/v1.0.0 (tree 3d4e5f6): exists +remote: objgit: snapshot refs/heads/wip (tree 0a1b2c3): failed +``` + +The size in the `built` line is the size of the compressed image. The failure +line does not show the error text, because the error can contain bucket +details. The log carries the full error. + +### Failures + +A snapshot failure never fails the push. The refs are committed before +`onUpdated` runs, and report-status has gone to the client. The daemon logs +the failure with `slog.Error` and the `"err"` key, and it counts the failure in +a metric. + +`onUpdated` does not run when the client does not negotiate report-status. +Hooks have the same limit today. Every current git client sends report-status. + +## Metrics + +New series go in `internal/metrics`, with two helpers, +`ObserveSnapshot(result string, dur time.Duration, bytes int64)` and +`ObserveSnapshotCache(event string)`. The second helper maps each cache event +to one of the two cache counters: + +| Series | Type | Labels | Meaning | +| ---------------------------------------- | --------- | -------- | ------------------------------------------- | +| `objgit_snapshot_builds_total` | counter | `result` | `built`, `exists`, or `error`. | +| `objgit_snapshot_build_duration_seconds` | histogram | none | Time for one `Ensure` that built an image. | +| `objgit_snapshot_image_bytes` | histogram | none | Size of one built image, after compression. | +| `objgit_snapshot_cache_opens_total` | counter | `result` | `hit` or `miss` for each `OpenSnapshot`. | +| `objgit_snapshot_cache_evictions_total` | counter | `reason` | `budget` or `idle`. | + +The size histogram is the data for the later decision about a size limit. Use +exponential buckets from 64 KiB to 16 GiB. The cache counters are the data for +the choice of `-snapshot-cache-bytes` and `-snapshot-cache-max-idle`. + +There is no `repo` label, per the rule in +[../../architecture/metrics.md](../../architecture/metrics.md). + +## Known risks + +- **Memory.** Resolved by erofs v0.8.0. Before it, `erofs.Builder` kept the + bytes and the compressed blocks of every file until `Build` returned, up to + approximately twice the size of the tree. +- **Push latency.** The client waits for the build. For a large tree the build + reads every blob once and compresses it. The pack prefetch makes the reads + local after the first read of each container. +- **The single PUT limit.** `PutSnapshot` sends one `PutObject`. An image + larger than 5 GiB fails. The repository has no multipart upload path today. + If the size histogram shows images near this limit, add one. +- **A crash during a build** leaves no image and no partial object. `PutObject` + is atomic. The next `Ensure` for that tree builds it. +- **An image larger than the cache budget.** The cache admits it, then evicts + every other settled image. The pack cache has the same behavior for a large + container. The size histogram shows if this happens. + +## Testing + +All tests are table-driven with `tt`. + +`internal/snapshot` unit tests use `memory.NewStorage()` for git objects and a +map-backed `Store`: + +| Test | What it proves | +| ------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | +| `TestEnsureMapsModes` | Every row of the mapping table. It opens the image with `Open` and compares the mode, the content, and the symlink target of each entry. | +| `TestEnsureValidates` | `erofs.Validate` returns no errors for each fixture tree. | +| `TestEnsureCompresses` | A tree with one large text file gives an image smaller than the file. The file reads back unchanged. | +| `TestEnsureDeterministic` | Two builds of one tree give identical bytes and the same `erofs-sha256`. | +| `TestEnsureExists` | A second `Ensure` returns `exists` and calls `PutSnapshot` zero times. | +| `TestEnsureLimits` | A 256-byte name, and a 1021-byte symlink target, each give an error that names the path, and no `PutSnapshot` call. | +| `TestEnsureEmptyTree` | The empty tree gives a valid image with a root directory only. | +| `TestOpenMissing` | `Open` of a tree with no image returns `fs.ErrNotExist`. | + +`internal/storage/tigris` tests use the existing `fakeS3`: + +| Test | What it proves | +| ---------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| `TestPutSnapshotKeyAndMetadata` | The key is under the scoped prefix, and the metadata is present. | +| `TestStatSnapshotNotFound` | A 404 becomes `fs.ErrNotExist`. | +| `TestOpenSnapshotCachesDownload` | Two `OpenSnapshot` calls send one `GetObject`. Two scoped storers with the same tree also send one. | +| `TestOpenSnapshotRejectsBadDigest` | A body that does not match `erofs-sha256` gives an error, and the cache keeps nothing. | +| `TestOpenSnapshotMissingDigest` | An object with no `erofs-sha256` gives an error. | +| `TestOpenSnapshotNoCache` | With no snapshot cache, `OpenSnapshot` still returns a readable file. | +| `TestPackCacheEvictIdle` | `EvictIdle` removes only settled entries older than the limit. An open descriptor stays readable after its entry is evicted. | +| `TestPackCacheGetUnchanged` | `Get` still refuses bytes whose SHA-256 is not `id`. | +| `TestPackCacheObserver` | The observer gets `miss`, then `hit`, then `evict_budget` and `evict_idle`, each in the correct case. | +| `TestSnapshotKeysInvisible` | A snapshot key does not appear in the pack index or in `IterReferences`. | + +`cmd/objgitd` protocol tests shell out to `git`, gated with +`exec.LookPath("git")`: + +- `TestSmartHTTPPushSnapshots`: push a branch and an annotated tag on one + commit, with `-erofs-snapshots` on. Exactly one image exists. A file reads + back through `snapshot.Open`. The client output contains the `remote: +objgit: snapshot` lines. +- `TestPushSnapshotsOff`: no image exists after a push with the flag off. +- `TestHooksIgnoreTags`: a pushed tag does not run the hook, now that + `snapshotRefs` returns tags. + +## Documentation changes + +- `docs/architecture/tigris-storer.md`: add the new key to the layout table. + In the pack cache section, add `GetChecked`, `EvictIdle`, and the second + instance. +- A new page `docs/architecture/snapshots.md`, linked from + `docs/architecture/README.md` and from the table in `AGENTS.md`. +- `docs/architecture/metrics.md`: add the new series and helpers. +- `AGENTS.md`: add `internal/snapshot` to the table of code locations. + +## The path to a message queue + +The queue changes only the caller. The message is `{repoPath, tree}`. The +consumer resolves the storer for `repoPath` and calls `Ensure`. `Ensure` is +idempotent, so a queue that delivers a message twice is correct. After the +move, `runSnapshots` sends messages and does not build, and the push no longer +waits for the build. diff --git a/docs/superpowers/specs/2026-09-23-erofs-streaming-builder-design.md b/docs/superpowers/specs/2026-09-23-erofs-streaming-builder-design.md new file mode 100644 index 0000000..0a3c65a --- /dev/null +++ b/docs/superpowers/specs/2026-09-23-erofs-streaming-builder-design.md @@ -0,0 +1,247 @@ +# Design: a streaming file source for `erofs.Builder` + +This spec is for work in the `github.com/Xe/erofs` repository, not in objgit. +Copy it into `docs/superpowers/specs/` of that repository before you start. +Read the `CLAUDE.md` of that repository first. + +The code references are to `github.com/Xe/erofs` v0.6.1. + +## Problem + +`Builder.AddFile(p, info, data []byte)` takes the whole content of a file. +The builder keeps each slice on `buildInode.data` until `Build` returns. With +compression on, `tryCompressInodes` also keeps the compressed blocks of every +file, on `compressedFileData.blocks`, until `Build` returns. The memory for +one build is therefore up to twice the total size of every file in the image. + +objgit builds one Zstandard-compressed image of a git tree after each push +(see [2026-09-23-erofs-snapshots-design.md](2026-09-23-erofs-snapshots-design.md)). +A 2 GiB tree costs up to 4 GiB of heap for each build. objgit already reads +each blob through an `io.Reader`, so it does not need the bytes in memory at +all. + +`AddFromFS` has the same problem. It calls `fs.ReadFile` for each file, so +`mkfs.erofs --dir` holds the whole directory tree in memory. + +## Goal + +Add a way to give the builder a file as a size and an open function. The +memory for a build must not depend on file content, with compression on or +off. It must depend only on the count of inodes, the count of directory +entries, and the count of lclusters. + +## Non-goals + +- A change to the on-disk format. The output must stay bytewise compatible + with the kernel driver and with `mkfs.erofs`. +- A change to the reader. +- Parallel compression. The builder compresses one group at a time, as it does + today. + +## API + +```go +// AddFileFunc adds a regular file whose content comes from open. The builder +// calls open once, during Build, and reads exactly size bytes from it. It +// closes the reader before it calls open for the next file. +func (b *Builder) AddFileFunc(p string, info fs.FileInfo, size int64, open func() (io.ReadCloser, error)) error + +// WithSpoolDir sets the directory for the temp file that holds compressed +// blocks during Build. The default is os.TempDir(). Build removes the file +// before it returns. +func WithSpoolDir(dir string) BuildOption +``` + +The rules for `AddFileFunc`: + +- The builder calls `open` during `Build`, not during `AddFileFunc`. +- The builder calls `open` exactly once for each file. If `size` is 0, it + does not call `open`. +- If the reader returns fewer or more than `size` bytes, `Build` returns an + error that names the path, the declared size, and the count of bytes read. +- If `open` or a read returns an error, `Build` returns it, wrapped with the + path. +- Only one reader is open at a time. + +`AddFile` does not change. Internally it can become `AddFileFunc` with an +open function over `bytes.NewReader(data)`. The implementation decides this. + +## The invariant + +**An image built with `AddFileFunc` is byte-identical to the same image built +with `AddFile`.** This is true with no compression, with LZ4, and with +Zstandard. Every test below compares the bytes of the two builds. + +## How the build works today + +`Build` (`builder.go:251`) runs these steps in this order: + +| Step | Function | Needs the file bytes? | +| ---- | -------------------------------------- | ---------------------------------------------------------- | +| 3 | `computeLayouts` (`builder.go:550`) | No. It uses only the length. | +| 3b | `tryCompressInodes` (`builder.go:533`) | Yes. It compresses each file to decide if it stays flat. | +| 4 | `assignNIDs` | No. It uses the metadata sizes from steps 3 and 3b. | +| 5 | `layoutDataBlocks` (`builder.go:688`) | No. It uses the length, or the count of compressed blocks. | +| 6 | `writeMetadata` (`builder.go:747`) | Yes, for the inline tail of a flat file. | +| 7 | `writeDataBlocks` (`builder.go:849`) | Yes. | + +Two facts make streaming possible: + +1. For a flat file, steps 3, 4, and 5 use only the length. After step 5, the + builder knows where the full blocks go (`ino.startBlk * blockSize`) and + where the inline tail goes (`ino.metaOff + 64`). One open of the file at + step 7 can write both parts. +2. For a compressed file, `tryCompressFile` (`builder_compress.go:66`) + compresses one pcluster group at a time. The group boundaries come from the + size alone. The metadata size comes from the count of lclusters, which also + comes from the size alone (`computeCompressedMetaSize`). Only two results + need the bytes: the count of physical blocks for the file, and the decision + to keep the file flat (`!anyBig`). + +## The design + +There are two kinds of lazy file. The builder decides the kind from the size +and the path alone, with the same test that `tryCompressFile` uses at its +start: + +| Kind | Test | When the builder opens it | +| --------- | ------------------------------------------------------------------------------------------------------------------------ | ------------------------- | +| Flat-only | Compression is off, or the algorithm is not LZ4 or Zstandard, or `size <= blockSize`, or `isIncompressible(path, size)`. | At step 7. | +| Candidate | All other files. | At step 3b. | + +### Flat-only files + +1. **Store the source on the inode.** Add `open func() (io.ReadCloser, error)` + to `buildInode`. `ino.size` already holds the size. +2. **Use `ino.size`, not `len(ino.data)`, for layout.** Replace each + `len(ino.data)` that means "the file size" in `computeLayouts`, + `layoutDataBlocks`, `writeInode` (`builder.go:773`), and + `writeDataBlocks`. Symlinks keep the target in `ino.data`, and `ino.size` + already equals its length, so this change does not affect them. +3. **Write the inode header without the tail.** In `writeInode`, if + `ino.open != nil`, write the 64-byte inode and skip the inline tail. +4. **Write the data at step 7.** For each flat-only lazy inode with + `ino.size > 0`: + 1. Call `open`. + 2. Copy the full blocks to `ino.startBlk * blockSize`, through one + `blockSize` buffer. For `InodeFlatPlain`, pad the last block with zeros, + as `writeDataBlocks` does today. + 3. For `InodeFlatInline`, copy the tail bytes to `ino.metaOff + 64`. + 4. Read one more byte. If the read does not return `io.EOF`, return the + size error. + 5. Close the reader. + +### Candidate files + +1. **Open a spool at the start of step 3b.** The spool is one temp file, from + `os.CreateTemp` in the spool directory. `Build` removes it before it + returns, on success and on error. +2. **Compress from the reader.** Change `tryCompressFile` so that it reads + each group from an `io.Reader` into one reused buffer, with + `io.ReadFull`. The group loop and the index entries do not change. Each + physical block that the loop makes today goes to the end of the spool, and + not into `blocks`. This is true for both kinds of group: + - A compressed group writes its right-aligned compressed blocks. + - An incompressible group writes each lcluster as one raw block, padded + with zeros. +3. **Replace `blocks [][]byte`.** In `compressedFileData`, replace `blocks` + with the spool offset of the first block and the count of blocks. + `layoutCompressedBlocks` (`builder_compress.go:335`) uses the count. + `writeCompressedBlocks` (`builder_compress.go:348`) copies the blocks from + the spool to `ino.startBlk * blockSize`, through one `blockSize` buffer. +4. **Keep the spool for a flat fallback.** If no group of the file benefits + (`!anyBig`), the file stays flat, as today. In this case every group was + incompressible, so the spool already holds the raw file data, padded to + blocks. Keep the spool offset on the inode. At step 7, copy the file from + the spool, and not from `open`. The tail for `InodeFlatInline` is the first + `size % blockSize` bytes of the last spool block. As a result, the builder + still calls `open` once. +5. **Check the size.** After the last group, read one more byte. If the read + does not return `io.EOF`, return the size error. + +The spool is at most the size of the data area of the image. Memory for +compression is one group buffer, `(K + 1) * blockSize` bytes, and the output +of one `compressGroup` call. Pass a reused destination slice to +`EncodeAll` and to `lz4.CompressBlock`, so that each group does not allocate a +new one. + +### `AddFromFS` + +Move `AddFromFS` to the lazy path. Use `info.Size()` and `fsys.Open(p)`. After +this change, a file that changes size between the walk and `Build` gives the +size error. Put this fact in the doc comment of `AddFromFS`. + +### Documentation + +- `README.md`: add a short example of `AddFileFunc`. +- `CHANGELOG.md`: add entries for `AddFileFunc`, `WithSpoolDir`, and the new + behavior of `AddFromFS`. + +## Testing + +Use table-driven tests, as the repository does today. + +`TestAddFileFuncMatchesAddFile` builds each case twice, once with `AddFile` +and once with `AddFileFunc`, and compares the bytes. It also runs `Validate` +on the result and reads each file back through `Open`. Use a block size of +4096, and use these files: + +| Case | Content | Size | What it covers | +| --------------- | ------------------------------------------------------ | ------------- | --------------------------------------------------- | +| empty | none | 0 | `open` is not called. | +| tiny | text | 1 | Flat inline, tail only. | +| max inline | text | 4096 − 64 | The largest tail that fits in the inode block. | +| over inline | text | 4096 − 63 | The tail does not fit. Flat plain. | +| one block | text | 4096 | Flat plain, no tail. Not a candidate. | +| block plus one | text | 4097 | The smallest candidate. | +| multi block | text | 3 × 4096 + 17 | Many groups and a tail. | +| random | random bytes, `.bin` name | 1 MiB + 5 | A candidate that falls back to flat from the spool. | +| mixed groups | 64 KiB text, then 64 KiB random | 128 KiB + 9 | Compressed and PLAIN groups in one file. | +| known extension | text, `.png` name | 64 KiB | Flat-only because of `isIncompressible`. | +| large | text | 10 MiB + 5 | The copy loops over many buffers. | +| mixed tree | All of the above, in nested directories, with symlinks | — | The inode order and the offsets between files. | + +Run the table three times: with no compression, with +`CompressionAutoLZ4`, and with `CompressionZstd`. + +Other tests: + +| Test | What it proves | +| ------------------------------ | --------------------------------------------------------------------------------------------------------------------------- | +| `TestAddFileFuncShortRead` | A reader that returns `size − 1` bytes gives an error that names the path. Run it for a flat-only file and for a candidate. | +| `TestAddFileFuncLongRead` | A reader that returns `size + 1` bytes gives the same error. Run it for both kinds. | +| `TestAddFileFuncOpenError` | An error from `open` comes out of `Build`, wrapped with the path. | +| `TestAddFileFuncOpensOnce` | Each `open` runs exactly once, for both kinds and for the flat fallback. Zero-size files do not call `open`. | +| `TestAddFileFuncOneReaderOpen` | No more than one reader is open at a time. | +| `TestSpoolRemoved` | The spool directory is empty after `Build`, on success and after a read error. | +| `TestAddFileFuncMemory` | See below. | +| `TestAddFromFSLazy` | `AddFromFS` over an `fstest.MapFS` gives the same bytes as v0.6.1 gave for the same tree. Keep a golden file from v0.6.1. | + +`TestAddFileFuncMemory` is the acceptance test for this spec. It builds 1 GiB +of content, in 1024 files of 1 MiB each, from a reader that generates +compressible text. Each `open` call runs `runtime.GC`, then reads +`runtime.MemStats.HeapAlloc`, and keeps the largest value. The largest value +must stay below 64 MiB. Run it with no compression and with Zstandard. If it +is slow, gate it with `testing.Short`. + +`TotalAlloc` is not a correct measure here. It counts every allocation, also +the ones that the collector already freed. The test measures live heap, which +is what the goal limits. + +## Release + +Release this change as v0.7.0. The API only grows, but `AddFromFS` has a new +failure mode and now writes a temp file, so a minor version is correct. + +## The consumer change in objgit + +After v0.7.0, objgit changes `snapshot.Ensure`: + +- It calls `AddFileFunc` for each blob, with `blob.Size` and a function that + returns `blob.Reader()`. +- It passes `WithSpoolDir` with the same temp directory that it uses for the + image. +- It no longer reads blob content during the tree walk. + +The objgit snapshot tests stay the same. `TestEnsureDeterministic` also +proves that the image bytes do not change when objgit moves to the new API. diff --git a/docs/usage/memory-benchmark.md b/docs/usage/memory-benchmark.md index b725271..6f9b992 100644 --- a/docs/usage/memory-benchmark.md +++ b/docs/usage/memory-benchmark.md @@ -60,17 +60,17 @@ repository it created, and cleanup is yours to run. Each run gets its own timestamped directory under `-out`: -| File | What it holds | -| ------------------------ | -------------------------------------------------------------------- | -| `report.md` | The tables. Start here. | -| `samples.csv` | One row per sample, tagged with phase and repository. Plot-ready. | -| `heap-baseline.pb.gz` | Settled heap before any push. | -| `heap-peak-*.pb.gz` | Heap at each new resident-set high-water mark. | -| `heap-final.pb.gz` | Settled heap after every push. | -| `allocs-final.pb.gz` | Total allocation over the run. | -| `daemon.log` | The daemon's own JSON log. | -| `repos.txt` | Every repository created, for cleanup. | -| `source.git`, `objgitd` | The mirror and the binary under test. | +| File | What it holds | +| ----------------------- | ----------------------------------------------------------------- | +| `report.md` | The tables. Start here. | +| `samples.csv` | One row per sample, tagged with phase and repository. Plot-ready. | +| `heap-baseline.pb.gz` | Settled heap before any push. | +| `heap-peak-*.pb.gz` | Heap at each new resident-set high-water mark. | +| `heap-final.pb.gz` | Settled heap after every push. | +| `allocs-final.pb.gz` | Total allocation over the run. | +| `daemon.log` | The daemon's own JSON log. | +| `repos.txt` | Every repository created, for cleanup. | +| `source.git`, `objgitd` | The mirror and the binary under test. | The two commands that get the most out of a run: @@ -99,19 +99,20 @@ go tool pprof -http=: /heap-peak-.pb.gz ## Flags worth knowing -| Flag | Default | Meaning | -| ----------------------- | -------------------- | ------------------------------------------------------------------------ | -| `-repo` | `$HOME/Code/Xe/x` | Repository to push. Mirror-cloned once, never written to. | -| `-org` | `benchtest` | Org segment every benchmark repository is created under. | -| `-seq-pushes` | `5` | Sequential pushes, each to a fresh repository. | -| `-conc-steps` | `1,2,4,8` | Concurrency levels to sweep. Empty skips the sweep. | -| `-sample-interval` | `250ms` | How often `/proc` and `/metrics` are read. | -| `-idle-gap` | `5s` | Idle time between pushes, so memory can settle. | -| `-window-slack` | `0` (uses idle gap) | How far outside a push's wall clock memory is still attributed to it. | -| `-peak-growth` | `0.05` | Fractional rise in resident set that triggers a heap capture. | -| `-peak-cooldown` | `2s` | Minimum time between two peak captures. | -| `-daemon-binary` | build it | Prebuilt `objgitd` to test instead of building `./cmd/objgitd`. | -| `-daemon-allow-hooks` | `false` | Off so hook cost is not mistaken for push cost. | +| Flag | Default | Meaning | +| ------------------------- | ------------------- | --------------------------------------------------------------------- | +| `-repo` | `$HOME/Code/Xe/x` | Repository to push. Mirror-cloned once, never written to. | +| `-org` | `benchtest` | Org segment every benchmark repository is created under. | +| `-seq-pushes` | `5` | Sequential pushes, each to a fresh repository. | +| `-conc-steps` | `1,2,4,8` | Concurrency levels to sweep. Empty skips the sweep. | +| `-sample-interval` | `250ms` | How often `/proc` and `/metrics` are read. | +| `-idle-gap` | `5s` | Idle time between pushes, so memory can settle. | +| `-window-slack` | `0` (uses idle gap) | How far outside a push's wall clock memory is still attributed to it. | +| `-peak-growth` | `0.05` | Fractional rise in resident set that triggers a heap capture. | +| `-peak-cooldown` | `2s` | Minimum time between two peak captures. | +| `-daemon-binary` | build it | Prebuilt `objgitd` to test instead of building `./cmd/objgitd`. | +| `-daemon-allow-hooks` | `false` | Off so hook cost is not mistaken for push cost. | +| `-daemon-erofs-snapshots` | `false` | Off so snapshot cost is not mistaken for push cost. | All flags take an environment fallback through `flagenv`, in UPPER_SNAKE. The daemon-facing flags are prefixed `-daemon-` so they do not collide with diff --git a/go.mod b/go.mod index d6e8466..8a00fb8 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/tigrisdata/objgit go 1.26.3 require ( + github.com/Xe/erofs v0.8.0 github.com/aws/aws-sdk-go-v2 v1.41.7 github.com/aws/aws-sdk-go-v2/service/s3 v1.102.0 github.com/aws/smithy-go v1.26.0 @@ -21,6 +22,11 @@ require ( tangled.org/xeiaso.net/kefka v0.0.6-0.20260528192045-e0a84e40ceb8 ) +require ( + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/pierrec/lz4/v4 v4.1.26 // indirect +) + require ( github.com/BurntSushi/toml v1.4.1-0.20240526193622-a339e1f7089c // indirect github.com/Microsoft/go-winio v0.6.2 // indirect @@ -50,7 +56,7 @@ require ( github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect github.com/go-git/gcfg/v2 v2.0.2 // indirect github.com/kevinburke/ssh_config v1.6.0 // indirect - github.com/klauspost/compress v1.18.0 + github.com/klauspost/compress v1.18.5 github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pborman/getopt/v2 v2.1.0 // indirect diff --git a/go.sum b/go.sum index 3bee081..96821b4 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM= github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo= +github.com/Xe/erofs v0.8.0 h1:OSD900oSKFP91cM+RyCTXW0FyamjtpwA5jTGLEeyxuw= +github.com/Xe/erofs v0.8.0/go.mod h1:GGxvJkCKU4yKfJ+Trap2xC8iN0HGuTUEsD5oz+olLJI= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= @@ -83,8 +85,8 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/kevinburke/ssh_config v1.6.0 h1:J1FBfmuVosPHf5GRdltRLhPJtJpTlMdKTBjRgTaQBFY= github.com/kevinburke/ssh_config v1.6.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= +github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -100,6 +102,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pborman/getopt/v2 v2.1.0 h1:eNfR+r+dWLdWmV8g5OlpyrTYHkhVNxHBdN2cCrJmOEA= github.com/pborman/getopt/v2 v2.1.0/go.mod h1:4NtW75ny4eBw9fO1bhtNdYTlZKYX5/tBLtsOpwKIKd0= +github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= +github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU= github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 0158fd3..7599762 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -87,6 +87,43 @@ var ( Buckets: prometheus.DefBuckets, }) + snapshotBuilds = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "snapshot", + Name: "builds_total", + Help: "erofs snapshot Ensure calls by result (built, exists, error).", + }, []string{"result"}) + + snapshotDuration = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: "snapshot", + Name: "build_duration_seconds", + Help: "Time for one erofs snapshot Ensure call that built an image.", + Buckets: prometheus.ExponentialBuckets(0.1, 2, 12), // 0.1s .. ~205s + }) + + snapshotImageBytes = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: "snapshot", + Name: "image_bytes", + Help: "Size of one built erofs snapshot image, after compression.", + Buckets: prometheus.ExponentialBuckets(64<<10, 4, 10), // 64 KiB .. 16 GiB + }) + + snapshotCacheOpens = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "snapshot", + Name: "cache_opens_total", + Help: "Snapshot cache lookups by result (hit, miss).", + }, []string{"result"}) + + snapshotCacheEvictions = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "snapshot", + Name: "cache_evictions_total", + Help: "Snapshot cache evictions by reason (budget, idle).", + }, []string{"reason"}) + webhookDeliveries = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: "webhook", @@ -251,6 +288,30 @@ func ObserveHook(status string, dur time.Duration) { hookDuration.Observe(dur.Seconds()) } +// ObserveSnapshot records one snapshot Ensure call. The duration and size +// histograms only describe builds, so they observe only when result is +// "built". +func ObserveSnapshot(result string, dur time.Duration, bytes int64) { + snapshotBuilds.WithLabelValues(result).Inc() + if result == "built" { + snapshotDuration.Observe(dur.Seconds()) + snapshotImageBytes.Observe(float64(bytes)) + } +} + +// ObserveSnapshotCache records one snapshot cache event. Its signature +// matches the observer tigris.NewSnapshotCache takes. +func ObserveSnapshotCache(event string) { + switch event { + case "hit", "miss": + snapshotCacheOpens.WithLabelValues(event).Inc() + case "evict_budget": + snapshotCacheEvictions.WithLabelValues("budget").Inc() + case "evict_idle": + snapshotCacheEvictions.WithLabelValues("idle").Inc() + } +} + // ObserveWebhook records the final outcome and elapsed time of one event. func ObserveWebhook(status string, dur time.Duration) { webhookDeliveries.WithLabelValues(status).Inc() diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go new file mode 100644 index 0000000..371caa2 --- /dev/null +++ b/internal/metrics/metrics_test.go @@ -0,0 +1,53 @@ +package metrics + +import ( + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestObserveSnapshot(t *testing.T) { + tests := []struct { + result string + }{ + {"built"}, + {"exists"}, + {"error"}, + } + for _, tt := range tests { + t.Run(tt.result, func(t *testing.T) { + before := testutil.ToFloat64(snapshotBuilds.WithLabelValues(tt.result)) + ObserveSnapshot(tt.result, time.Second, 1<<20) + if got := testutil.ToFloat64(snapshotBuilds.WithLabelValues(tt.result)); got != before+1 { + t.Errorf("builds_total{%s} = %v, want %v", tt.result, got, before+1) + } + }) + } +} + +func TestObserveSnapshotCache(t *testing.T) { + tests := []struct { + event string + vec string + label string + }{ + {"hit", "opens", "hit"}, + {"miss", "opens", "miss"}, + {"evict_budget", "evictions", "budget"}, + {"evict_idle", "evictions", "idle"}, + } + for _, tt := range tests { + t.Run(tt.event, func(t *testing.T) { + c := snapshotCacheOpens.WithLabelValues(tt.label) + if tt.vec == "evictions" { + c = snapshotCacheEvictions.WithLabelValues(tt.label) + } + before := testutil.ToFloat64(c) + ObserveSnapshotCache(tt.event) + if got := testutil.ToFloat64(c); got != before+1 { + t.Errorf("%s{%s} = %v, want %v", tt.vec, tt.label, got, before+1) + } + }) + } +} diff --git a/internal/snapshot/ensure.go b/internal/snapshot/ensure.go new file mode 100644 index 0000000..9d8edd2 --- /dev/null +++ b/internal/snapshot/ensure.go @@ -0,0 +1,262 @@ +package snapshot + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "strconv" + "time" + + "github.com/Xe/erofs" + "github.com/go-git/go-git/v6/plumbing" + "github.com/go-git/go-git/v6/plumbing/filemode" + "github.com/go-git/go-git/v6/plumbing/object" + "github.com/go-git/go-git/v6/plumbing/storer" +) + +// Result statuses. +const ( + StatusBuilt = "built" + StatusExists = "exists" +) + +// Two EROFS limits that git does not have. The builder checks neither one, +// and the reader refuses a symlink target longer than maxSymlink. +const ( + maxName = 255 + maxSymlink = 255 * 4 +) + +// blockSizeBits is the image block size as a power of two: 1<<12 = 4096 +// bytes. It is the argument to erofs.WithBlockSize. +const blockSizeBits = 12 + +// epoch pins every mtime, so one tree always gives identical bytes. +var epoch = time.Unix(0, 0) + +// Result describes one Ensure call. +type Result struct { + Key string + Status string // StatusBuilt or StatusExists + Files int // regular files in the image; 0 when Status is StatusExists + Bytes int64 // image size; 0 when Status is StatusExists + Elapsed time.Duration +} + +// Ensure makes sure that the image of tree exists in store. If the image is +// absent, it builds the image from the git objects in objs, in a temp file in +// tmpDir (the OS temp directory when tmpDir is empty), and puts it. +// +// Ensure is idempotent. Two concurrent calls for one tree both build and put +// identical bytes, which is correct, so it takes no lock. +func Ensure(ctx context.Context, objs storer.EncodedObjectStorer, store Store, tree plumbing.Hash, tmpDir string) (Result, error) { + start := time.Now() + key := Key(tree) + res := Result{Key: key} + done := func() { res.Elapsed = time.Since(start) } + + if err := ctx.Err(); err != nil { + return res, err + } + if _, err := store.StatSnapshot(ctx, key); err == nil { + res.Status = StatusExists + done() + return res, nil + } else if !errors.Is(err, fs.ErrNotExist) { + return res, fmt.Errorf("snapshot: stat %s: %w", key, err) + } + + root, err := object.GetTree(objs, tree) + if err != nil { + return res, fmt.Errorf("snapshot: load tree %s: %w", tree, err) + } + + // The name is random, never the tree hash: two pushes of one tree can + // build at the same time. + f, err := os.CreateTemp(tmpDir, "objgit-snapshot-*.erofs") + if err != nil { + return res, fmt.Errorf("snapshot: create temp file: %w", err) + } + defer func() { + f.Close() + os.Remove(f.Name()) + }() + opts := []erofs.BuildOption{ + erofs.WithBlockSize(blockSizeBits), + erofs.WithEpoch(epoch), + erofs.WithCompression(erofs.CompressionZstd), + } + if tmpDir != "" { + // Build spools compressed blocks to a temp file of its own; keep it + // next to the image. + opts = append(opts, erofs.WithSpoolDir(tmpDir)) + } + b := erofs.NewBuilder(f, opts...) + w := &walker{ctx: ctx, objs: objs, b: b} + if err := w.addTree("/", root); err != nil { + return res, err + } + if err := b.Build(); err != nil { + return res, fmt.Errorf("snapshot: build %s: %w", key, err) + } + + sum, size, err := hashFile(f) + if err != nil { + return res, err + } + meta := map[string]string{ + MetaFormat: strconv.Itoa(FormatVersion), + MetaSHA256: sum, + MetaTree: tree.String(), + MetaFiles: strconv.Itoa(w.files), + } + if err := store.PutSnapshot(ctx, key, f, size, meta); err != nil { + return res, fmt.Errorf("snapshot: put %s: %w", key, err) + } + + res.Status = StatusBuilt + res.Files = w.files + res.Bytes = size + done() + return res, nil +} + +// hashFile returns the hex SHA-256 and size of f, and leaves f at offset 0. +func hashFile(f *os.File) (string, int64, error) { + if _, err := f.Seek(0, io.SeekStart); err != nil { + return "", 0, fmt.Errorf("snapshot: rewind image: %w", err) + } + h := sha256.New() + size, err := io.Copy(h, f) + if err != nil { + return "", 0, fmt.Errorf("snapshot: hash image: %w", err) + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + return "", 0, fmt.Errorf("snapshot: rewind image: %w", err) + } + return hex.EncodeToString(h.Sum(nil)), size, nil +} + +type walker struct { + ctx context.Context + objs storer.EncodedObjectStorer + b *erofs.Builder + files int +} + +// addTree adds every entry of t under dir, depth first. It walks tree +// entries and not t.Files(), because go-git's FileIter skips gitlinks. +func (w *walker) addTree(dir string, t *object.Tree) error { + for _, e := range t.Entries { + if err := w.ctx.Err(); err != nil { + return err + } + p := path.Join(dir, e.Name) + if len(e.Name) > maxName { + return fmt.Errorf("snapshot: %s: name is %d bytes, EROFS allows %d", p, len(e.Name), maxName) + } + switch e.Mode { + case filemode.Dir: + sub, err := object.GetTree(w.objs, e.Hash) + if err != nil { + return fmt.Errorf("snapshot: %s: load tree: %w", p, err) + } + if err := w.b.AddDir(p, info{e.Name, fs.ModeDir | 0o755}); err != nil { + return fmt.Errorf("snapshot: %s: %w", p, err) + } + if err := w.addTree(p, sub); err != nil { + return err + } + case filemode.Submodule: + // Same as git checkout of a submodule that is not initialized. + if err := w.b.AddDir(p, info{e.Name, fs.ModeDir | 0o755}); err != nil { + return fmt.Errorf("snapshot: %s: %w", p, err) + } + case filemode.Regular, filemode.Deprecated, filemode.Executable: + // Only the size here. The builder opens the blob during Build, + // one file at a time, so no blob content stays in memory. + size, err := w.objs.EncodedObjectSize(e.Hash) + if err != nil { + return fmt.Errorf("snapshot: %s: blob size: %w", p, err) + } + perm := fs.FileMode(0o644) + if e.Mode == filemode.Executable { + perm = 0o755 + } + if err := w.b.AddFileFunc(p, info{e.Name, perm}, size, w.opener(e.Hash)); err != nil { + return fmt.Errorf("snapshot: %s: %w", p, err) + } + w.files++ + case filemode.Symlink: + // Check the size before the read, so a huge blob in symlink + // mode is refused without loading it. + size, err := w.objs.EncodedObjectSize(e.Hash) + if err != nil { + return fmt.Errorf("snapshot: %s: blob size: %w", p, err) + } + if size > maxSymlink { + return fmt.Errorf("snapshot: %s: symlink target is %d bytes, EROFS allows %d", p, size, maxSymlink) + } + data, err := w.blob(p, e.Hash) + if err != nil { + return err + } + if err := w.b.AddSymlink(p, string(data), info{e.Name, fs.ModeSymlink | 0o777}); err != nil { + return fmt.Errorf("snapshot: %s: %w", p, err) + } + default: + return fmt.Errorf("snapshot: %s: unknown git mode %o", p, uint32(e.Mode)) + } + } + return nil +} + +// opener returns the source AddFileFunc calls during Build. It loads the blob +// only then, so the object is garbage once its reader closes. +func (w *walker) opener(h plumbing.Hash) func() (io.ReadCloser, error) { + return func() (io.ReadCloser, error) { + blob, err := object.GetBlob(w.objs, h) + if err != nil { + return nil, fmt.Errorf("load blob %s: %w", h, err) + } + return blob.Reader() + } +} + +// blob reads a whole blob. Only symlink targets use it; they are at most +// maxSymlink bytes. +func (w *walker) blob(p string, h plumbing.Hash) ([]byte, error) { + blob, err := object.GetBlob(w.objs, h) + if err != nil { + return nil, fmt.Errorf("snapshot: %s: load blob: %w", p, err) + } + r, err := blob.Reader() + if err != nil { + return nil, fmt.Errorf("snapshot: %s: read blob: %w", p, err) + } + defer r.Close() + data, err := io.ReadAll(r) + if err != nil { + return nil, fmt.Errorf("snapshot: %s: read blob: %w", p, err) + } + return data, nil +} + +// info is the fs.FileInfo the builder reads the mode and mtime from. +type info struct { + name string + mode fs.FileMode +} + +func (i info) Name() string { return i.name } +func (i info) Size() int64 { return 0 } +func (i info) Mode() fs.FileMode { return i.mode } +func (i info) ModTime() time.Time { return epoch } +func (i info) IsDir() bool { return i.mode.IsDir() } +func (i info) Sys() any { return nil } diff --git a/internal/snapshot/ensure_test.go b/internal/snapshot/ensure_test.go new file mode 100644 index 0000000..6960c06 --- /dev/null +++ b/internal/snapshot/ensure_test.go @@ -0,0 +1,339 @@ +package snapshot + +import ( + "bytes" + "context" + "errors" + "fmt" + "io/fs" + "os" + "strings" + "testing" + + "github.com/Xe/erofs" + "github.com/go-git/go-git/v6/plumbing" + "github.com/go-git/go-git/v6/plumbing/filemode" + "github.com/go-git/go-git/v6/storage/memory" +) + +func TestEnsureMapsModes(t *testing.T) { + long := strings.Repeat("n", 255) + var many []fixture + for i := range 500 { + many = append(many, fixture{fmt.Sprintf("big/file-%04d.txt", i), filemode.Regular, fmt.Sprintf("%d\n", i)}) + } + + tests := []struct { + name string + entries []fixture + // want maps an image path to its mode, and to its content or + // symlink target. A directory has no content. + want map[string]struct { + mode fs.FileMode + content string + } + }{ + { + name: "every mode", + entries: []fixture{ + {"README.md", filemode.Regular, "hello\n"}, + {"bin/run.sh", filemode.Executable, "#!/bin/sh\necho hi\n"}, + {"link", filemode.Symlink, "README.md"}, + {"old", filemode.Deprecated, "group writable\n"}, + {"vendor/lib", filemode.Submodule, ""}, + }, + want: map[string]struct { + mode fs.FileMode + content string + }{ + "README.md": {0o644, "hello\n"}, + "bin": {fs.ModeDir | 0o755, ""}, + "bin/run.sh": {0o755, "#!/bin/sh\necho hi\n"}, + "link": {fs.ModeSymlink | 0o777, "README.md"}, + "old": {0o644, "group writable\n"}, + "vendor": {fs.ModeDir | 0o755, ""}, + "vendor/lib": {fs.ModeDir | 0o755, ""}, + }, + }, + { + name: "unusual names", + entries: []fixture{ + {"with space.txt", filemode.Regular, "a"}, + {"ünïcødé.txt", filemode.Regular, "b"}, + {".hidden", filemode.Regular, "c"}, + {long, filemode.Regular, "d"}, + }, + want: map[string]struct { + mode fs.FileMode + content string + }{ + "with space.txt": {0o644, "a"}, + "ünïcødé.txt": {0o644, "b"}, + ".hidden": {0o644, "c"}, + long: {0o644, "d"}, + }, + }, + { + name: "large directory", + entries: many, + want: map[string]struct { + mode fs.FileMode + content string + }{ + "big/file-0000.txt": {0o644, "0\n"}, + "big/file-0499.txt": {0o644, "499\n"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + st := memory.NewStorage() + tree := buildTree(t, st, tt.entries) + store := NewMemStore() + + res, err := Ensure(context.Background(), st, store, tree, t.TempDir()) + if err != nil { + t.Fatalf("Ensure: %v", err) + } + if res.Status != StatusBuilt { + t.Fatalf("Status = %q, want %q", res.Status, StatusBuilt) + } + + snap, err := Open(context.Background(), store, tree) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer snap.Close() + + for p, w := range tt.want { + info, err := snap.Lstat(p) + if err != nil { + t.Errorf("Lstat(%q): %v", p, err) + continue + } + if info.Mode() != w.mode { + t.Errorf("%q mode = %v, want %v", p, info.Mode(), w.mode) + } + switch { + case w.mode&fs.ModeSymlink != 0: + got, err := snap.ReadLink(p) + if err != nil || got != w.content { + t.Errorf("ReadLink(%q) = %q, %v; want %q", p, got, err, w.content) + } + case w.mode.IsRegular(): + got, err := fs.ReadFile(snap, p) + if err != nil || string(got) != w.content { + t.Errorf("ReadFile(%q) = %q, %v; want %q", p, got, err, w.content) + } + } + } + + if tt.name == "large directory" { + ents, err := fs.ReadDir(snap, "big") + if err != nil || len(ents) != 500 { + t.Errorf("ReadDir(big) = %d entries, %v; want 500", len(ents), err) + } + } + }) + } +} + +func TestEnsureValidates(t *testing.T) { + tests := []struct { + name string + entries []fixture + }{ + {"flat", []fixture{{"a", filemode.Regular, "a"}}}, + {"nested", []fixture{{"x/y/z", filemode.Regular, strings.Repeat("z", 10000)}}}, + {"gitlink", []fixture{{"sub", filemode.Submodule, ""}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + st := memory.NewStorage() + tree := buildTree(t, st, tt.entries) + store := NewMemStore() + if _, err := Ensure(context.Background(), st, store, tree, t.TempDir()); err != nil { + t.Fatalf("Ensure: %v", err) + } + body, _, _ := store.Object(Key(tree)) + res, err := erofs.Validate(bytes.NewReader(body), int64(len(body))) + if err != nil { + t.Fatalf("Validate: %v", err) + } + if len(res.Errors) != 0 { + t.Errorf("Validate errors: %v", res.Errors) + } + }) + } +} + +func TestEnsureCompresses(t *testing.T) { + text := strings.Repeat("all work and no play makes jack a dull boy\n", 8000) // ~350 KiB + st := memory.NewStorage() + tree := buildTree(t, st, []fixture{{"story.txt", filemode.Regular, text}}) + store := NewMemStore() + res, err := Ensure(context.Background(), st, store, tree, t.TempDir()) + if err != nil { + t.Fatalf("Ensure: %v", err) + } + if res.Bytes >= int64(len(text)) { + t.Errorf("image is %d bytes, want less than the %d-byte file", res.Bytes, len(text)) + } + snap, err := Open(context.Background(), store, tree) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer snap.Close() + got, err := fs.ReadFile(snap, "story.txt") + if err != nil || string(got) != text { + t.Errorf("story.txt did not read back unchanged: %v", err) + } +} + +func TestEnsureDeterministic(t *testing.T) { + entries := []fixture{ + {"a.txt", filemode.Regular, strings.Repeat("a", 9000)}, + {"d/b.txt", filemode.Executable, "b"}, + {"l", filemode.Symlink, "a.txt"}, + } + var bodies [2][]byte + var sums [2]string + for i := range 2 { + st := memory.NewStorage() + tree := buildTree(t, st, entries) + store := NewMemStore() + if _, err := Ensure(context.Background(), st, store, tree, t.TempDir()); err != nil { + t.Fatalf("Ensure %d: %v", i, err) + } + body, meta, _ := store.Object(Key(tree)) + bodies[i], sums[i] = body, meta[MetaSHA256] + } + if !bytes.Equal(bodies[0], bodies[1]) { + t.Error("two builds of one tree gave different bytes") + } + if sums[0] == "" || sums[0] != sums[1] { + t.Errorf("sha256 metadata = %q and %q, want equal and set", sums[0], sums[1]) + } +} + +func TestEnsureMetadata(t *testing.T) { + st := memory.NewStorage() + tree := buildTree(t, st, []fixture{ + {"a", filemode.Regular, "a"}, + {"b", filemode.Regular, "b"}, + {"l", filemode.Symlink, "a"}, + }) + store := NewMemStore() + res, err := Ensure(context.Background(), st, store, tree, t.TempDir()) + if err != nil { + t.Fatalf("Ensure: %v", err) + } + _, meta, ok := store.Object(Key(tree)) + if !ok { + t.Fatal("no object at Key(tree)") + } + want := map[string]string{MetaFormat: "1", MetaTree: tree.String(), MetaFiles: "2"} + for k, v := range want { + if meta[k] != v { + t.Errorf("meta[%q] = %q, want %q", k, meta[k], v) + } + } + if res.Files != 2 { + t.Errorf("Files = %d, want 2", res.Files) + } +} + +func TestEnsureExists(t *testing.T) { + st := memory.NewStorage() + tree := buildTree(t, st, []fixture{{"a", filemode.Regular, "a"}}) + store := NewMemStore() + if _, err := Ensure(context.Background(), st, store, tree, t.TempDir()); err != nil { + t.Fatalf("first Ensure: %v", err) + } + res, err := Ensure(context.Background(), st, store, tree, t.TempDir()) + if err != nil { + t.Fatalf("second Ensure: %v", err) + } + if res.Status != StatusExists { + t.Errorf("Status = %q, want %q", res.Status, StatusExists) + } + if n := store.PutCount(); n != 1 { + t.Errorf("PutCount = %d, want 1", n) + } +} + +func TestEnsureLimits(t *testing.T) { + tests := []struct { + name string + entries []fixture + path string // must appear in the error + }{ + {"name too long", []fixture{{strings.Repeat("n", 256), filemode.Regular, "x"}}, strings.Repeat("n", 256)}, + {"symlink target too long", []fixture{{"l", filemode.Symlink, strings.Repeat("t", 1021)}}, "l"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + st := memory.NewStorage() + tree := buildTree(t, st, tt.entries) + store := NewMemStore() + _, err := Ensure(context.Background(), st, store, tree, t.TempDir()) + if err == nil || !strings.Contains(err.Error(), tt.path) { + t.Fatalf("Ensure error = %v, want one that names %q", err, tt.path) + } + if n := store.PutCount(); n != 0 { + t.Errorf("PutCount = %d, want 0", n) + } + }) + } +} + +func TestEnsureEmptyTree(t *testing.T) { + st := memory.NewStorage() + tree := buildTree(t, st, nil) + store := NewMemStore() + if _, err := Ensure(context.Background(), st, store, tree, t.TempDir()); err != nil { + t.Fatalf("Ensure: %v", err) + } + snap, err := Open(context.Background(), store, tree) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer snap.Close() + ents, err := fs.ReadDir(snap, ".") + if err != nil || len(ents) != 0 { + t.Errorf("ReadDir(.) = %v, %v; want empty", ents, err) + } +} + +func TestEnsureCanceled(t *testing.T) { + st := memory.NewStorage() + tree := buildTree(t, st, []fixture{{"a", filemode.Regular, "a"}}) + store := NewMemStore() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := Ensure(ctx, st, store, tree, t.TempDir()) + if !errors.Is(err, context.Canceled) { + t.Fatalf("Ensure error = %v, want context.Canceled", err) + } + if n := store.PutCount(); n != 0 { + t.Errorf("PutCount = %d, want 0", n) + } +} + +func TestEnsureRemovesTempFile(t *testing.T) { + st := memory.NewStorage() + tree := buildTree(t, st, []fixture{{"a", filemode.Regular, "a"}}) + tmp := t.TempDir() + for _, putErr := range []error{nil, errors.New("bucket down")} { + store := NewMemStore() + store.PutErr = putErr + _, _ = Ensure(context.Background(), st, store, tree, tmp) + ents, _ := os.ReadDir(tmp) + if len(ents) != 0 { + t.Errorf("putErr=%v: %d files left in tmpDir", putErr, len(ents)) + } + } +} + +var _ = plumbing.ZeroHash diff --git a/internal/snapshot/fixture_test.go b/internal/snapshot/fixture_test.go new file mode 100644 index 0000000..44310ba --- /dev/null +++ b/internal/snapshot/fixture_test.go @@ -0,0 +1,101 @@ +package snapshot + +import ( + "path" + "sort" + "strings" + "testing" + + "github.com/go-git/go-git/v6/plumbing" + "github.com/go-git/go-git/v6/plumbing/filemode" + "github.com/go-git/go-git/v6/plumbing/object" + "github.com/go-git/go-git/v6/storage/memory" +) + +// fixture is one entry of a test tree. For a gitlink, content is ignored and +// hash is the submodule commit. Directories come from the paths. +type fixture struct { + path string + mode filemode.FileMode + content string +} + +// buildTree writes the blobs and trees for entries into st and returns the +// root tree hash. It sorts entries the way git does, so the hash is stable. +func buildTree(t *testing.T, st *memory.Storage, entries []fixture) plumbing.Hash { + t.Helper() + type dir struct { + entries []object.TreeEntry + subdirs map[string]*dir + } + root := &dir{subdirs: map[string]*dir{}} + for _, e := range entries { + parts := strings.Split(e.path, "/") + d := root + for _, p := range parts[:len(parts)-1] { + sub, ok := d.subdirs[p] + if !ok { + sub = &dir{subdirs: map[string]*dir{}} + d.subdirs[p] = sub + } + d = sub + } + var h plumbing.Hash + if e.mode == filemode.Submodule { + h = plumbing.NewHash("1111111111111111111111111111111111111111") + } else { + h = writeBlob(t, st, e.content) + } + d.entries = append(d.entries, object.TreeEntry{Name: path.Base(e.path), Mode: e.mode, Hash: h}) + } + var write func(d *dir) plumbing.Hash + write = func(d *dir) plumbing.Hash { + for name, sub := range d.subdirs { + d.entries = append(d.entries, object.TreeEntry{Name: name, Mode: filemode.Dir, Hash: write(sub)}) + } + sort.Slice(d.entries, func(i, j int) bool { + return sortName(d.entries[i]) < sortName(d.entries[j]) + }) + tree := &object.Tree{Entries: d.entries} + obj := st.NewEncodedObject() + if err := tree.Encode(obj); err != nil { + t.Fatalf("encode tree: %v", err) + } + h, err := st.SetEncodedObject(obj) + if err != nil { + t.Fatalf("store tree: %v", err) + } + return h + } + return write(root) +} + +// sortName is the key git sorts tree entries by: a directory sorts as if its +// name ended in "/". +func sortName(e object.TreeEntry) string { + if e.Mode == filemode.Dir { + return e.Name + "/" + } + return e.Name +} + +func writeBlob(t *testing.T, st *memory.Storage, content string) plumbing.Hash { + t.Helper() + obj := st.NewEncodedObject() + obj.SetType(plumbing.BlobObject) + w, err := obj.Writer() + if err != nil { + t.Fatalf("blob writer: %v", err) + } + if _, err := w.Write([]byte(content)); err != nil { + t.Fatalf("write blob: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("close blob: %v", err) + } + h, err := st.SetEncodedObject(obj) + if err != nil { + t.Fatalf("store blob: %v", err) + } + return h +} diff --git a/internal/snapshot/memory_test.go b/internal/snapshot/memory_test.go new file mode 100644 index 0000000..f9ff2e0 --- /dev/null +++ b/internal/snapshot/memory_test.go @@ -0,0 +1,97 @@ +package snapshot + +import ( + "bytes" + "context" + "fmt" + "runtime" + "testing" + + "github.com/go-git/go-git/v6/plumbing" + "github.com/go-git/go-git/v6/plumbing/filemode" + "github.com/go-git/go-git/v6/plumbing/object" + "github.com/go-git/go-git/v6/storage/memory" +) + +// synthStorer serves trees from a memory.Storage and makes each blob on +// demand, so the test's own heap does not hold the tree's content. Every +// blob load samples the live heap, the same way the upstream erofs memory +// test does. +type synthStorer struct { + *memory.Storage + blobs map[plumbing.Hash]int // hash -> index, for the content generator + size int + peak uint64 + loads int +} + +func (s *synthStorer) content(i int) []byte { + line := fmt.Appendf(nil, "file %d: all work and no play makes jack a dull boy\n", i) + return bytes.Repeat(line, s.size/len(line)+1)[:s.size] +} + +func (s *synthStorer) EncodedObject(t plumbing.ObjectType, h plumbing.Hash) (plumbing.EncodedObject, error) { + i, ok := s.blobs[h] + if !ok { + return s.Storage.EncodedObject(t, h) + } + runtime.GC() + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + s.peak = max(s.peak, ms.HeapAlloc) + s.loads++ + + obj := &plumbing.MemoryObject{} + obj.SetType(plumbing.BlobObject) + if _, err := obj.Write(s.content(i)); err != nil { + return nil, err + } + return obj, nil +} + +func (s *synthStorer) EncodedObjectSize(h plumbing.Hash) (int64, error) { + if _, ok := s.blobs[h]; ok { + return int64(s.size), nil + } + return s.Storage.EncodedObjectSize(h) +} + +// TestEnsureMemory pins that Ensure streams blobs into the builder: a +// 256 MiB tree must build with a small live heap. Reading every blob during +// the walk, as Ensure did before erofs v0.8.0, holds all of it. +func TestEnsureMemory(t *testing.T) { + if testing.Short() { + t.Skip("builds 256 MiB of content") + } + const files, size = 256, 1 << 20 + st := &synthStorer{Storage: memory.NewStorage(), blobs: map[plumbing.Hash]int{}, size: size} + + var entries []object.TreeEntry + for i := range files { + h := plumbing.NewHash(fmt.Sprintf("%040x", i+1)) + st.blobs[h] = i + entries = append(entries, object.TreeEntry{Name: fmt.Sprintf("f%04d.txt", i), Mode: filemode.Regular, Hash: h}) + } + obj := st.NewEncodedObject() + if err := (&object.Tree{Entries: entries}).Encode(obj); err != nil { + t.Fatal(err) + } + tree, err := st.SetEncodedObject(obj) + if err != nil { + t.Fatal(err) + } + + store := NewMemStore() + res, err := Ensure(context.Background(), st, store, tree, t.TempDir()) + if err != nil { + t.Fatalf("Ensure: %v", err) + } + if st.loads != files { + t.Errorf("blob loads = %d, want %d (one for each file)", st.loads, files) + } + t.Logf("peak live heap: %.1f MiB for %d MiB of content; image %d bytes", + float64(st.peak)/(1<<20), files*size>>20, res.Bytes) + if st.peak >= 64<<20 { + t.Errorf("peak live heap %d bytes, want less than 64 MiB", st.peak) + } +} diff --git a/internal/snapshot/memstore.go b/internal/snapshot/memstore.go new file mode 100644 index 0000000..c3f90a8 --- /dev/null +++ b/internal/snapshot/memstore.go @@ -0,0 +1,98 @@ +package snapshot + +import ( + "bytes" + "context" + "fmt" + "io" + "io/fs" + "maps" + "slices" + "sync" +) + +// MemStore is a Store in memory. It exists for tests, in this package and in +// others that need a Store without a bucket. +type MemStore struct { + // PutErr, when set, is returned by every PutSnapshot. + PutErr error + + mu sync.Mutex + objs map[string]memObject + puts int +} + +type memObject struct { + body []byte + meta map[string]string +} + +// NewMemStore returns an empty MemStore. +func NewMemStore() *MemStore { return &MemStore{objs: map[string]memObject{}} } + +func (m *MemStore) StatSnapshot(_ context.Context, key string) (int64, error) { + m.mu.Lock() + defer m.mu.Unlock() + o, ok := m.objs[key] + if !ok { + return 0, fmt.Errorf("snapshot: %s: %w", key, fs.ErrNotExist) + } + return int64(len(o.body)), nil +} + +func (m *MemStore) PutSnapshot(_ context.Context, key string, body io.ReadSeeker, size int64, meta map[string]string) error { + m.mu.Lock() + m.puts++ + putErr := m.PutErr + m.mu.Unlock() + if putErr != nil { + return putErr + } + b, err := io.ReadAll(body) + if err != nil { + return err + } + if int64(len(b)) != size { + return fmt.Errorf("snapshot: put %s: read %d bytes, want %d", key, len(b), size) + } + m.mu.Lock() + defer m.mu.Unlock() + m.objs[key] = memObject{body: b, meta: maps.Clone(meta)} + return nil +} + +func (m *MemStore) OpenSnapshot(_ context.Context, key string) (File, error) { + m.mu.Lock() + defer m.mu.Unlock() + o, ok := m.objs[key] + if !ok { + return nil, fmt.Errorf("snapshot: %s: %w", key, fs.ErrNotExist) + } + return nopCloser{bytes.NewReader(o.body)}, nil +} + +// Keys returns every stored key, sorted. +func (m *MemStore) Keys() []string { + m.mu.Lock() + defer m.mu.Unlock() + return slices.Sorted(maps.Keys(m.objs)) +} + +// Object returns the body and metadata stored at key. +func (m *MemStore) Object(key string) ([]byte, map[string]string, bool) { + m.mu.Lock() + defer m.mu.Unlock() + o, ok := m.objs[key] + return o.body, o.meta, ok +} + +// PutCount returns how many times PutSnapshot ran, failed calls included. +func (m *MemStore) PutCount() int { + m.mu.Lock() + defer m.mu.Unlock() + return m.puts +} + +type nopCloser struct{ *bytes.Reader } + +func (nopCloser) Close() error { return nil } diff --git a/internal/snapshot/open.go b/internal/snapshot/open.go new file mode 100644 index 0000000..4d8bf1d --- /dev/null +++ b/internal/snapshot/open.go @@ -0,0 +1,35 @@ +package snapshot + +import ( + "context" + "fmt" + + "github.com/Xe/erofs" + "github.com/go-git/go-git/v6/plumbing" +) + +// Snapshot is an open image. Every read is local. Close releases the file. +type Snapshot struct { + *erofs.FS + f File +} + +// Close releases the local file behind the image. +func (s *Snapshot) Close() error { return s.f.Close() } + +// Open returns the image of tree. It does not build a missing image: it +// returns an error that matches fs.ErrNotExist instead. Call Ensure first when +// the image must exist. +func Open(ctx context.Context, store Store, tree plumbing.Hash) (*Snapshot, error) { + key := Key(tree) + f, err := store.OpenSnapshot(ctx, key) + if err != nil { + return nil, fmt.Errorf("snapshot: open %s: %w", key, err) + } + fsys, err := erofs.Open(f) + if err != nil { + f.Close() + return nil, fmt.Errorf("snapshot: read %s: %w", key, err) + } + return &Snapshot{FS: fsys, f: f}, nil +} diff --git a/internal/snapshot/open_test.go b/internal/snapshot/open_test.go new file mode 100644 index 0000000..03f428f --- /dev/null +++ b/internal/snapshot/open_test.go @@ -0,0 +1,17 @@ +package snapshot + +import ( + "context" + "errors" + "io/fs" + "testing" + + "github.com/go-git/go-git/v6/plumbing" +) + +func TestOpenMissing(t *testing.T) { + _, err := Open(context.Background(), NewMemStore(), plumbing.NewHash("2222222222222222222222222222222222222222")) + if !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("Open error = %v, want fs.ErrNotExist", err) + } +} diff --git a/internal/snapshot/snapshot.go b/internal/snapshot/snapshot.go new file mode 100644 index 0000000..91f869b --- /dev/null +++ b/internal/snapshot/snapshot.go @@ -0,0 +1,66 @@ +// Package snapshot builds and opens EROFS images of git trees. An image is +// stored next to the repository it came from, under Key(tree), and a later +// reader (such as a web UI) opens it with Open instead of walking git objects. +// +// The package never imports internal/storage/tigris. Storage reaches it +// through Store, which *tigris.Storer implements. +package snapshot + +import ( + "context" + "fmt" + "io" + "strings" + + "github.com/go-git/go-git/v6/plumbing" +) + +// FormatVersion names the mapping from git to EROFS and the build options. +// Changing either one is a new version, and a new key, so an image written +// under one version is never overwritten by another. +const FormatVersion = 1 + +// Object metadata keys carried by every image. Lowercase, because S3 returns +// user metadata keys lowercased. +const ( + MetaFormat = "erofs-format" + MetaSHA256 = "erofs-sha256" + MetaTree = "git-tree" + MetaFiles = "erofs-files" +) + +const keyPrefix = "snapshots/erofs/" + +// Key returns the object key of the image of tree, relative to the +// repository prefix. +func Key(tree plumbing.Hash) string { + return fmt.Sprintf("%sv%d/%s.erofs", keyPrefix, FormatVersion, tree) +} + +// CacheID returns the local cache id for key: "erofs-v1-". It holds no +// repository prefix, so two repositories with one tree share one cached file. +func CacheID(key string) string { + id := strings.TrimSuffix(strings.TrimPrefix(key, keyPrefix), ".erofs") + return "erofs-" + strings.ReplaceAll(id, "/", "-") +} + +// File is a local copy of an image. *os.File satisfies it. +type File interface { + io.ReaderAt + io.Closer +} + +// Store holds snapshot images for one repository. Keys are relative to the +// repository prefix. +type Store interface { + // StatSnapshot returns the size of the image at key, or an error that + // matches fs.ErrNotExist when key is absent. + StatSnapshot(ctx context.Context, key string) (int64, error) + // PutSnapshot stores size bytes from body at key, with meta as user + // metadata. + PutSnapshot(ctx context.Context, key string, body io.ReadSeeker, size int64, meta map[string]string) error + // OpenSnapshot returns a local copy of the image at key. The caller must + // close it. It returns an error that matches fs.ErrNotExist when key is + // absent. + OpenSnapshot(ctx context.Context, key string) (File, error) +} diff --git a/internal/storage/tigris/packcache.go b/internal/storage/tigris/packcache.go index 2415989..514ec58 100644 --- a/internal/storage/tigris/packcache.go +++ b/internal/storage/tigris/packcache.go @@ -44,9 +44,16 @@ import ( // // A nil *PackCache is not usable; Storers without one fall back to a private // unlinked temp file per instance. See Storer.fetchWholePack. +// +// The same type also serves erofs snapshot images, through NewSnapshotCache +// and GetChecked: a snapshot id is not a content digest, so those callers +// supply the digest a download must have instead of reusing the id. type PackCache struct { dir string maxBytes int64 + suffix string // file name suffix of a cached entry; binSuffix for packs + observe func(event string) // optional; see NewSnapshotCache + now func() time.Time // wall clock for EvictIdle; a field so tests can move it mu sync.Mutex entries map[string]*cacheEntry @@ -58,12 +65,13 @@ type PackCache struct { // download runs, so a second caller for the same id waits on ready instead of // starting a duplicate download. type cacheEntry struct { - id string - path string - size int64 // 0 until the download settles; also marks an entry unevictable - used uint64 // LRU stamp, higher is more recent - ready chan struct{} - err error // download outcome; read only after ready closes + id string + path string + size int64 // 0 until the download settles; also marks an entry unevictable + used uint64 // LRU stamp, higher is more recent + usedAt time.Time // wall-clock time of the last claim; EvictIdle reads it + ready chan struct{} + err error // download outcome; read only after ready closes // stream is the in-progress view of this entry's download: set once the // staging file exists, cleared when the download settles either way. It @@ -78,9 +86,26 @@ type cacheEntry struct { // maxBytes of zero or less means no budget and therefore no eviction, which // only makes sense for tests. func NewPackCache(parent string, maxBytes int64) (*PackCache, error) { - dir, err := os.MkdirTemp(parent, "objgit-packs-*") + return newFileCache(parent, "objgit-packs-*", binSuffix, maxBytes, nil) +} + +// NewSnapshotCache creates a cache for snapshot images, in a fresh +// objgit-snapshots-* directory under parent. It is the same cache as +// NewPackCache, with its own directory and budget. Snapshot ids are not +// content digests, so callers use GetChecked, not Get. observe, when not +// nil, receives "hit", "miss", "evict_budget", and "evict_idle". +func NewSnapshotCache(parent string, maxBytes int64, observe func(event string)) (*PackCache, error) { + return newFileCache(parent, "objgit-snapshots-*", ".erofs", maxBytes, observe) +} + +// newFileCache is the shared constructor behind NewPackCache and +// NewSnapshotCache. pattern names the temp directory (os.MkdirTemp's +// pattern), and suffix names the file extension a settled entry gets on +// disk. +func newFileCache(parent, pattern, suffix string, maxBytes int64, observe func(string)) (*PackCache, error) { + dir, err := os.MkdirTemp(parent, pattern) if err != nil { - return nil, fmt.Errorf("tigris: create pack cache directory: %w", err) + return nil, fmt.Errorf("tigris: create cache directory: %w", err) } // A run that died without Cleanup leaves its cache directory behind. On // the OS temp directory that is harmless, but on a persistent parent @@ -89,25 +114,43 @@ func NewPackCache(parent string, maxBytes int64) (*PackCache, error) { // daemon owns the parent in every supported deployment, so the sweep is // safe; a reader from an older process keeps its descriptors through the // unlink, exactly like an eviction. - sweepOrphanedCaches(parent, dir) + sweepOrphanedCaches(parent, strings.TrimSuffix(pattern, "*"), dir) return &PackCache{ dir: dir, maxBytes: maxBytes, entries: make(map[string]*cacheEntry), + suffix: suffix, + observe: observe, + now: time.Now, }, nil } -// sweepOrphanedCaches removes the objgit-packs-* directories under parent -// that earlier runs left behind, and keeps keep, the directory this run just -// created. It is best effort: a ReadDir failure leaves everything alone, and +// emit reports event to the observer n times, when one is installed. Callers +// must call it outside c.mu: the observer is arbitrary caller code, and +// running it under the lock would let a slow or reentrant callback stall +// every other claim, forget, or fill in the process. +func (c *PackCache) emit(event string, n int) { + if c.observe == nil { + return + } + for range n { + c.observe(event) + } +} + +// sweepOrphanedCaches removes the directories under parent whose names start +// with prefix, which earlier runs left behind, and keeps keep, the directory +// this run just created. Each cache sweeps only its own prefix: the pack cache +// and the snapshot cache share a parent, and the one made second must not +// delete the directory of the first. It is best effort: a ReadDir failure leaves everything alone, and // a directory that cannot be removed stays for the next start to try again. -func sweepOrphanedCaches(parent, keep string) { +func sweepOrphanedCaches(parent, prefix, keep string) { entries, err := os.ReadDir(parent) if err != nil { return } for _, e := range entries { - if !e.IsDir() || !strings.HasPrefix(e.Name(), "objgit-packs-") { + if !e.IsDir() || !strings.HasPrefix(e.Name(), prefix) { continue } dir := filepath.Join(parent, e.Name()) @@ -135,18 +178,43 @@ func (c *PackCache) Cleanup() error { // // The caller owns the returned descriptor and must close it. func (c *PackCache) Get(id string, fetch func(io.Writer) error) (*os.File, error) { - // Bounded, because a pack can in principle be evicted by competing + return c.GetChecked(id, expectID(id, fetch)) +} + +// expectID adapts a pack fetch, whose digest is its own id, to the shape +// GetChecked wants. +func expectID(id string, fetch func(io.Writer) error) func(io.Writer) (string, error) { + return func(w io.Writer) (string, error) { return id, fetch(w) } +} + +// GetChecked is Get for ids that are not content digests, such as a snapshot +// id (see NewSnapshotCache): fetch writes the body and reports the lower-case +// hex SHA-256 it must have. A body whose digest disagrees is refused and not +// cached. +// +// The observer sees "miss" once for each download this call runs, and "hit" +// only when this call returns a file without running one. Joining another +// caller's download counts as a hit, because this call downloaded nothing. +// +// The caller owns the returned descriptor and must close it. +func (c *PackCache) GetChecked(id string, fetch func(w io.Writer) (wantSHA256 string, err error)) (*os.File, error) { + ranFill := false + // Bounded, because an entry can in principle be evicted by competing // downloads between the fill and the open. Exhausting the budget returns an // error, and packindex.go's bulk tier degrades to ranged GETs on one. for try := 0; try < 3; try++ { e, mine := c.claim(id) if mine { - slog.Debug("pack not cached, downloading", "pack", id, "try", try) - c.fill(e, fetch) + ranFill = true + c.emit("miss", 1) + slog.Debug("cache entry not cached, downloading", "id", id, "try", try) + // fill has closed e.ready by the time it returns, so the observer + // below never holds up a waiter. + c.emit("evict_budget", c.fill(e, fetch)) } else { // Either a settled entry or a download another caller is already // running; both mean this caller downloads nothing. - slog.Debug("pack served from cache", "pack", id, "path", e.path) + slog.Debug("cache entry served from cache", "id", id, "path", e.path) } <-e.ready if e.err != nil { @@ -156,17 +224,20 @@ func (c *PackCache) Get(id string, fetch func(io.Writer) error) (*os.File, error f, err := os.Open(e.path) switch { case err == nil: + if !ranFill { + c.emit("hit", 1) + } return f, nil case errors.Is(err, os.ErrNotExist): // Evicted between the lookup and the open. Forget this entry and // go round again, which re-downloads it. - slog.Debug("cached pack vanished before it could be opened, retrying", "pack", id, "try", try) + slog.Debug("cached entry vanished before it could be opened, retrying", "id", id, "try", try) c.forget(e) default: - return nil, fmt.Errorf("tigris: open cached pack %s: %w", id, err) + return nil, fmt.Errorf("tigris: open cached entry %s: %w", id, err) } } - return nil, fmt.Errorf("tigris: cached pack %s was evicted faster than it could be read", id) + return nil, fmt.Errorf("tigris: cached entry %s was evicted faster than it could be read", id) } // partial reports the in-progress view of id's download, or nil when no @@ -197,13 +268,15 @@ func (c *PackCache) claim(id string) (*cacheEntry, bool) { c.seq++ if e, ok := c.entries[id]; ok { e.used = c.seq + e.usedAt = c.now() return e, false } e := &cacheEntry{ - id: id, - path: filepath.Join(c.dir, id+binSuffix), - used: c.seq, - ready: make(chan struct{}), + id: id, + path: filepath.Join(c.dir, id+c.suffix), + used: c.seq, + usedAt: c.now(), + ready: make(chan struct{}), } c.entries[id] = e return e, true @@ -222,7 +295,9 @@ func (c *PackCache) forget(e *cacheEntry) { } // fill runs the download for an entry this caller claimed, then releases every -// waiter. A failed download un-registers the entry, so the failure is not +// waiter, and reports how many entries the admission evicted to meet the +// budget. It emits nothing itself: the caller reports the count after fill +// returns, which is after ready closes. A failed download un-registers the entry, so the failure is not // cached: the per-Storer packAccess.start guard in packindex.go is what stops // a retry storm, and a genuinely transient error stays retryable by a later // request. @@ -231,7 +306,7 @@ func (c *PackCache) forget(e *cacheEntry) { // e.stream, so readers of this pack never have to wait for the whole container // — see packindex.go's watermark tier. The stream is torn down before ready // closes, so nothing can be served out of a body that failed its checksum. -func (c *PackCache) fill(e *cacheEntry, fetch func(io.Writer) error) { +func (c *PackCache) fill(e *cacheEntry, fetch func(io.Writer) (string, error)) int { defer close(e.ready) start := time.Now() @@ -240,10 +315,10 @@ func (c *PackCache) fill(e *cacheEntry, fetch func(io.Writer) error) { // filesystem boundary. tmp, err := os.CreateTemp(c.dir, "download-*") if err != nil { - e.err = fmt.Errorf("tigris: stage pack %s: %w", e.id, err) - slog.Debug("pack download failed", "pack", e.id, "err", e.err) + e.err = fmt.Errorf("tigris: stage cache entry %s: %w", e.id, err) + slog.Debug("cache download failed", "id", e.id, "err", e.err) c.forget(e) - return + return 0 } // A second, read-only descriptor on the same inode. It has to be its own @@ -260,13 +335,13 @@ func (c *PackCache) fill(e *cacheEntry, fetch func(io.Writer) error) { // references the stream at all. runtime.AddCleanup(ps, func(f *os.File) { f.Close() }, rd) } else { - slog.Debug("cannot open the staging file for watermark reads", "pack", e.id, "err", oerr) + slog.Debug("cannot open the staging file for watermark reads", "id", e.id, "err", oerr) } size, err := verifiedCopy(tmp, e.id, fetch, progress) e.stream.Store(nil) if cerr := tmp.Close(); err == nil && cerr != nil { - err = fmt.Errorf("tigris: stage pack %s: %w", e.id, cerr) + err = fmt.Errorf("tigris: stage cache entry %s: %w", e.id, cerr) } if err == nil { err = os.Rename(tmp.Name(), e.path) @@ -274,36 +349,39 @@ func (c *PackCache) fill(e *cacheEntry, fetch func(io.Writer) error) { if err != nil { os.Remove(tmp.Name()) e.err = err - slog.Debug("pack download failed", "pack", e.id, "dur", time.Since(start), "err", err) + slog.Debug("cache download failed", "id", e.id, "dur", time.Since(start), "err", err) c.forget(e) - return + return 0 } c.mu.Lock() e.size = size c.cur += size - slog.Debug("pack cached", - "pack", e.id, + slog.Debug("cache entry filled", + "id", e.id, "bytes", size, "dur", time.Since(start), "cache_bytes", c.cur, "cache_max_bytes", c.maxBytes, - "cache_packs", len(c.entries)) - c.evictLocked(e.id) + "cache_entries", len(c.entries)) + n := c.evictLocked(e.id) c.mu.Unlock() + return n } -// evictLocked unlinks least-recently-used packs until the budget is met, +// evictLocked unlinks least-recently-used entries until the budget is met, // never touching keep (the entry being admitted) or an entry whose download -// has not settled. An in-flight download's bytes are therefore uncounted, so -// several concurrent downloads can overshoot the budget until they land. +// has not settled, and reports how many it removed. An in-flight download's +// bytes are therefore uncounted, so several concurrent downloads can overshoot +// the budget until they land. // -// A single pack larger than the whole budget is admitted anyway — the read has -// to work — and simply evicts everything else. -func (c *PackCache) evictLocked(keep string) { +// A single entry larger than the whole budget is admitted anyway — the read +// has to work — and simply evicts everything else. +func (c *PackCache) evictLocked(keep string) int { if c.maxBytes <= 0 { - return + return 0 } + n := 0 for c.cur > c.maxBytes { var victim *cacheEntry for _, e := range c.entries { @@ -316,38 +394,68 @@ func (c *PackCache) evictLocked(keep string) { } if victim == nil { // nothing evictable left; the survivors are all in use - slog.Debug("pack cache over budget with nothing evictable", - "cache_bytes", c.cur, "cache_max_bytes", c.maxBytes, "cache_packs", len(c.entries)) - return + slog.Debug("cache over budget with nothing evictable", + "cache_bytes", c.cur, "cache_max_bytes", c.maxBytes, "cache_entries", len(c.entries)) + return n } delete(c.entries, victim.id) c.cur -= victim.size // Open descriptors survive the unlink and keep serving reads; the disk // space comes back when the last one closes. os.Remove(victim.path) - slog.Debug("pack uncached", - "pack", victim.id, + n++ + slog.Debug("cache entry over budget, uncached", + "id", victim.id, "bytes", victim.size, "admitted", keep, "cache_bytes", c.cur, "cache_max_bytes", c.maxBytes, - "cache_packs", len(c.entries)) + "cache_entries", len(c.entries)) } + return n +} + +// EvictIdle unlinks every settled entry that no caller claimed in the last +// maxIdle, and reports how many it removed. It never touches a download that +// has not settled. Open descriptors keep working, as with budget eviction. +func (c *PackCache) EvictIdle(maxIdle time.Duration) int { + c.mu.Lock() + cutoff := c.now().Add(-maxIdle) + n := 0 + for id, e := range c.entries { + if e.size == 0 || !e.usedAt.Before(cutoff) { + continue + } + delete(c.entries, id) + c.cur -= e.size + os.Remove(e.path) + n++ + slog.Debug("cache entry idle, uncached", "id", id, "bytes", e.size, "idle_since", e.usedAt) + } + c.mu.Unlock() + c.emit("evict_idle", n) + return n } // verifiedCopy runs fetch into w, hashing as it goes, and rejects a body whose -// SHA-256 disagrees with the pack id. It reports how many bytes landed. +// SHA-256 disagrees with the digest fetch itself reports. It reports how many +// bytes landed. +// +// id is not the value being verified against — it names the cache entry, for +// the error message only. For a pack, whose id is its own digest, expectID +// makes the two the same string. // // progress, when non-nil, is the watermark a concurrent reader polls: it is // bumped after every write, so a byte counted there is already in w. -func verifiedCopy(w io.Writer, id string, fetch func(io.Writer) error, progress *atomic.Int64) (int64, error) { +func verifiedCopy(w io.Writer, id string, fetch func(io.Writer) (string, error), progress *atomic.Int64) (int64, error) { h := sha256.New() cnt := &countingWriter{w: io.MultiWriter(w, h), progress: progress} - if err := fetch(cnt); err != nil { + want, err := fetch(cnt) + if err != nil { return 0, err } - if got := hex.EncodeToString(h.Sum(nil)); got != id { - return 0, fmt.Errorf("tigris: downloaded pack %s failed checksum (got %s)", id, got) + if got := hex.EncodeToString(h.Sum(nil)); got != want { + return 0, fmt.Errorf("tigris: downloaded entry %s failed checksum (got %s, want %s)", id, got, want) } return cnt.n, nil } diff --git a/internal/storage/tigris/packcache_test.go b/internal/storage/tigris/packcache_test.go index 4b435f0..1c9e4b5 100644 --- a/internal/storage/tigris/packcache_test.go +++ b/internal/storage/tigris/packcache_test.go @@ -4,15 +4,18 @@ import ( "bytes" "crypto/sha256" "encoding/hex" + "errors" "fmt" "io" "os" "os/exec" "path/filepath" + "slices" "strings" "sync" "sync/atomic" "testing" + "time" "github.com/go-git/go-git/v6/plumbing" ) @@ -115,6 +118,51 @@ func TestPackCacheSweepsOrphans(t *testing.T) { } } +// TestSnapshotCacheSweepKeepsPackCache pins that each cache sweeps only its +// own prefix. main.go makes the pack cache first and the snapshot cache second, +// under one parent. A sweep keyed on the pack prefix would delete the live pack +// cache the moment the snapshot cache starts. +func TestSnapshotCacheSweepKeepsPackCache(t *testing.T) { + t.Parallel() + + parent := t.TempDir() + for _, name := range []string{"objgit-packs-stale", "objgit-snapshots-stale"} { + if err := os.MkdirAll(filepath.Join(parent, name), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", name, err) + } + } + + packs, err := NewPackCache(parent, 0) + if err != nil { + t.Fatalf("NewPackCache: %v", err) + } + t.Cleanup(func() { _ = packs.Cleanup() }) + snaps, err := NewSnapshotCache(parent, 0, nil) + if err != nil { + t.Fatalf("NewSnapshotCache: %v", err) + } + t.Cleanup(func() { _ = snaps.Cleanup() }) + + for _, tt := range []struct { + path string + want bool // want the path to still exist after both sweeps + }{ + {path: packs.dir, want: true}, + {path: snaps.dir, want: true}, + {path: filepath.Join(parent, "objgit-packs-stale"), want: false}, + {path: filepath.Join(parent, "objgit-snapshots-stale"), want: false}, + } { + _, err := os.Stat(tt.path) + exists := err == nil + if err != nil && !os.IsNotExist(err) { + t.Fatalf("stat %s: %v", tt.path, err) + } + if exists != tt.want { + t.Errorf("%s exists = %v, want %v", tt.path, exists, tt.want) + } + } +} + // TestPackCacheDownloadsOnce is the whole point of the cache: a second request // for a pack already on disk must not go back to the network. func TestPackCacheDownloadsOnce(t *testing.T) { @@ -424,3 +472,239 @@ func TestPackCacheSharedAcrossStorers(t *testing.T) { t.Errorf("without a cache two readers caused %d whole-pack GETs, want 2", got) } } + +// TestPackCacheGetChecked pins GetChecked's contract for ids that are not +// content digests: fetch reports the SHA-256 the body must have, and a body +// that disagrees is refused and leaves nothing behind. +func TestPackCacheGetChecked(t *testing.T) { + body := []byte("snapshot bytes") + sum := sha256.Sum256(body) + good := hex.EncodeToString(sum[:]) + + tests := []struct { + name string + want string + wantErr bool + }{ + {"digest matches", good, false}, + {"digest differs", strings.Repeat("0", 64), true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c, err := NewSnapshotCache(t.TempDir(), 1<<20, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { c.Cleanup() }) + f, err := c.GetChecked("erofs-v1-abc", func(w io.Writer) (string, error) { + _, err := w.Write(body) + return tt.want, err + }) + if tt.wantErr { + if err == nil { + f.Close() + t.Fatal("GetChecked accepted bytes with the wrong digest") + } + if n := countCacheFiles(t, c); n != 0 { + t.Errorf("cache kept %d files after a bad digest", n) + } + return + } + if err != nil { + t.Fatalf("GetChecked: %v", err) + } + defer f.Close() + if got := readAll(t, f); !bytes.Equal(got, body) { + t.Errorf("body = %q, want %q", got, body) + } + if !strings.HasSuffix(f.Name(), ".erofs") { + t.Errorf("cached file %q does not end in .erofs", f.Name()) + } + }) + } +} + +// TestPackCacheGetStillChecksID pins that Get, unlike GetChecked, still treats +// id itself as the digest fetch's bytes must hash to. +func TestPackCacheGetStillChecksID(t *testing.T) { + c := newTestPackCache(t, 1<<20) + _, err := c.Get(strings.Repeat("0", 64), func(w io.Writer) error { + _, err := w.Write([]byte("not the digest")) + return err + }) + if err == nil { + t.Fatal("Get accepted bytes whose SHA-256 is not the id") + } +} + +// TestPackCacheEvictIdle pins the idle sweep: an entry nobody has claimed +// since maxIdle ago is unlinked, an entry claimed more recently survives, and +// an open descriptor on the evicted entry keeps reading. +func TestPackCacheEvictIdle(t *testing.T) { + now := time.Unix(1000, 0) + c, err := NewSnapshotCache(t.TempDir(), 0, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { c.Cleanup() }) + c.now = func() time.Time { return now } + + put := func(id string) *os.File { + body := []byte(id) + sum := sha256.Sum256(body) + f, err := c.GetChecked(id, func(w io.Writer) (string, error) { + _, err := w.Write(body) + return hex.EncodeToString(sum[:]), err + }) + if err != nil { + t.Fatalf("GetChecked(%s): %v", id, err) + } + return f + } + + old := put("old") // used at t=1000 + now = now.Add(time.Hour) // t=4600 + put("fresh").Close() // used at t=4600 + now = now.Add(time.Minute) + + if n := c.EvictIdle(30 * time.Minute); n != 1 { + t.Fatalf("EvictIdle = %d, want 1", n) + } + if n := countCacheFiles(t, c); n != 1 { + t.Errorf("%d files left, want 1", n) + } + // The evicted entry's open descriptor still reads. + if got := readAll(t, old); string(got) != "old" { + t.Errorf("evicted file read %q, want %q", got, "old") + } + old.Close() +} + +// TestPackCacheObserver pins the event sequence an observer sees across a +// miss, a hit, a budget eviction, and an idle eviction. The observer must run +// outside c.mu, so a slow or reentrant callback can never deadlock the cache. +func TestPackCacheObserver(t *testing.T) { + var mu sync.Mutex + var events []string + var c *PackCache + observe := func(e string) { + // Taking c.mu from inside the callback deadlocks if any emit ever + // runs under the lock. The long idle limit means it evicts nothing. + c.EvictIdle(24 * time.Hour) + mu.Lock() + events = append(events, e) + mu.Unlock() + } + + now := time.Unix(1000, 0) + c, err := NewSnapshotCache(t.TempDir(), 10, observe) // 10-byte budget + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { c.Cleanup() }) + c.now = func() time.Time { return now } + + get := func(id string, body []byte, fetchErr error) { + sum := sha256.Sum256(body) + f, err := c.GetChecked(id, func(w io.Writer) (string, error) { + if fetchErr != nil { + return "", fetchErr + } + _, err := w.Write(body) + return hex.EncodeToString(sum[:]), err + }) + if (err != nil) != (fetchErr != nil) { + t.Errorf("GetChecked(%s) err = %v, want %v", id, err, fetchErr) + } + if f != nil { + f.Close() + } + } + + done := make(chan struct{}) + go func() { + defer close(done) + get("a", []byte("123456"), nil) // miss + get("a", []byte("123456"), nil) // hit + get("x", nil, errors.New("bucket down")) // miss, and no hit for the failure + get("b", []byte("abcdef"), nil) // miss, evicts a: evict_budget + now = now.Add(time.Hour) + c.EvictIdle(time.Minute) // evicts b: evict_idle + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("observer deadlocked: an emit runs under c.mu") + } + + want := []string{"miss", "hit", "miss", "miss", "evict_budget", "evict_idle"} + mu.Lock() + defer mu.Unlock() + if !slices.Equal(events, want) { + t.Errorf("events = %v, want %v", events, want) + } +} + +// TestPackCacheEvictIdleSkipsInFlight pins two EvictIdle invariants: it never +// touches a download that has not settled, and an idle eviction gives its +// bytes back to the budget. +func TestPackCacheEvictIdleSkipsInFlight(t *testing.T) { + var events []string + var mu sync.Mutex + now := time.Unix(1000, 0) + c, err := NewSnapshotCache(t.TempDir(), 10, func(e string) { mu.Lock(); events = append(events, e); mu.Unlock() }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { c.Cleanup() }) + c.now = func() time.Time { return now } + + body := []byte("12345678") // 8 of the 10-byte budget + sum := sha256.Sum256(body) + release := make(chan struct{}) + started := make(chan struct{}) + got := make(chan error, 1) + go func() { + f, err := c.GetChecked("slow", func(w io.Writer) (string, error) { + close(started) + <-release + _, err := w.Write(body) + return hex.EncodeToString(sum[:]), err + }) + if f != nil { + f.Close() + } + got <- err + }() + <-started + + now = now.Add(time.Hour) + if n := c.EvictIdle(0); n != 0 { + t.Fatalf("EvictIdle evicted %d entries while a download was in flight, want 0", n) + } + close(release) + if err := <-got; err != nil { + t.Fatalf("GetChecked after EvictIdle: %v", err) + } + + // Settled now: an idle sweep takes it, and its bytes leave the budget, so + // a second 8-byte entry fits without a budget eviction. + now = now.Add(time.Hour) + if n := c.EvictIdle(time.Minute); n != 1 { + t.Fatalf("EvictIdle = %d, want 1", n) + } + sum2 := sha256.Sum256([]byte("abcdefgh")) + f, err := c.GetChecked("next", func(w io.Writer) (string, error) { + _, err := w.Write([]byte("abcdefgh")) + return hex.EncodeToString(sum2[:]), err + }) + if err != nil { + t.Fatal(err) + } + f.Close() + mu.Lock() + defer mu.Unlock() + if slices.Contains(events, "evict_budget") { + t.Errorf("events = %v: EvictIdle did not return the evicted bytes to the budget", events) + } +} diff --git a/internal/storage/tigris/packindex.go b/internal/storage/tigris/packindex.go index 268f03c..d88b5da 100644 --- a/internal/storage/tigris/packindex.go +++ b/internal/storage/tigris/packindex.go @@ -712,7 +712,7 @@ func (s *Storer) openWholePack(id string, a *packAccess) (*os.File, error) { // nothing is ever served out of a body that failed its checksum. ps := &packStream{f: f} a.stream.Store(ps) - _, err = verifiedCopy(f, id, s.streamPack(id), &ps.n) + _, err = verifiedCopy(f, id, expectID(id, s.streamPack(id)), &ps.n) a.stream.Store(nil) if err != nil { f.Close() diff --git a/internal/storage/tigris/snapshot.go b/internal/storage/tigris/snapshot.go new file mode 100644 index 0000000..dd0bceb --- /dev/null +++ b/internal/storage/tigris/snapshot.go @@ -0,0 +1,126 @@ +package tigris + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "io/fs" + "log/slog" + "os" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/tigrisdata/objgit/internal/snapshot" +) + +// Storer holds the snapshot images of its repository, next to the git data. +var _ snapshot.Store = (*Storer)(nil) + +// StatSnapshot sends one HeadObject for key under this Storer's prefix. +func (s *Storer) StatSnapshot(ctx context.Context, key string) (int64, error) { + start := time.Now() + out, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: sp(s.bucket), + Key: sp(s.prefix + key), + }) + s.observe("HeadObject", start, err) + switch { + case err == nil: + case isNotFound(err): + return 0, fmt.Errorf("tigris: snapshot %s: %w", key, fs.ErrNotExist) + default: + return 0, fmt.Errorf("tigris: head snapshot %s: %w", key, err) + } + if out.ContentLength == nil { + return 0, nil + } + return *out.ContentLength, nil +} + +// PutSnapshot sends one PutObject. It does not use the upload queue: no ref +// waits for an image, so there is nothing for a flush to order. +func (s *Storer) PutSnapshot(ctx context.Context, key string, body io.ReadSeeker, size int64, meta map[string]string) error { + start := time.Now() + _, err := s.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: sp(s.bucket), + Key: sp(s.prefix + key), + Body: body, + ContentLength: &size, + ContentType: sp("application/octet-stream"), + Metadata: meta, + }) + s.observe("PutObject", start, err) + if err != nil { + return fmt.Errorf("tigris: put snapshot %s: %w", key, err) + } + return nil +} + +// OpenSnapshot returns a local copy of the image at key. With a snapshot +// cache, the copy lives in the cache, and later calls from any repository +// that holds the same tree open it with no request. Without one, the copy is +// a private unlinked temp file, as openWholePack does for packs. +// +// The body must match its erofs-sha256 metadata, so a bad download is never +// served and never cached. +func (s *Storer) OpenSnapshot(ctx context.Context, key string) (snapshot.File, error) { + fetch := s.snapshotFetch(ctx, key) + if s.snapCache != nil { + f, err := s.snapCache.GetChecked(snapshot.CacheID(key), fetch) + if err != nil { + return nil, err + } + return f, nil + } + + slog.Debug("no snapshot cache installed, downloading to a private temp file", "key", key) + f, err := os.CreateTemp("", "objgit-snapshot-*") + if err != nil { + return nil, fmt.Errorf("tigris: create snapshot temp file: %w", err) + } + os.Remove(f.Name()) // the descriptor keeps the data until Close + + h := sha256.New() + want, err := fetch(io.MultiWriter(f, h)) + if err == nil { + if got := hex.EncodeToString(h.Sum(nil)); got != want { + err = fmt.Errorf("tigris: snapshot %s failed checksum (got %s, want %s)", key, got, want) + } + } + if err != nil { + f.Close() + return nil, err + } + return f, nil +} + +// snapshotFetch returns the GetChecked fetch for key: it copies the body and +// returns the digest from the object's erofs-sha256 metadata. +func (s *Storer) snapshotFetch(ctx context.Context, key string) func(io.Writer) (string, error) { + return func(w io.Writer) (string, error) { + start := time.Now() + out, err := s.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: sp(s.bucket), + Key: sp(s.prefix + key), + }) + s.observe("GetObject", start, err) + switch { + case err == nil: + case isNotFound(err): + return "", fmt.Errorf("tigris: snapshot %s: %w", key, fs.ErrNotExist) + default: + return "", fmt.Errorf("tigris: get snapshot %s: %w", key, err) + } + defer out.Body.Close() + want := out.Metadata[snapshot.MetaSHA256] + if want == "" { + return "", fmt.Errorf("tigris: snapshot %s has no %s metadata", key, snapshot.MetaSHA256) + } + if _, err := io.Copy(w, out.Body); err != nil { + return "", fmt.Errorf("tigris: download snapshot %s: %w", key, err) + } + return want, nil + } +} diff --git a/internal/storage/tigris/snapshot_test.go b/internal/storage/tigris/snapshot_test.go new file mode 100644 index 0000000..0127d4b --- /dev/null +++ b/internal/storage/tigris/snapshot_test.go @@ -0,0 +1,214 @@ +package tigris + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "io/fs" + "testing" + + "github.com/go-git/go-git/v6/plumbing" + "github.com/tigrisdata/objgit/internal/snapshot" +) + +func putTestSnapshot(t *testing.T, s *Storer, key string, body []byte, sum string) { + t.Helper() + meta := map[string]string{snapshot.MetaFormat: "1"} + if sum != "" { + meta[snapshot.MetaSHA256] = sum + } + if err := s.PutSnapshot(context.Background(), key, bytes.NewReader(body), int64(len(body)), meta); err != nil { + t.Fatalf("PutSnapshot: %v", err) + } +} + +func digest(b []byte) string { + s := sha256.Sum256(b) + return hex.EncodeToString(s[:]) +} + +const testSnapshotKey = "snapshots/erofs/v1/abc.erofs" + +func TestPutSnapshotKeyAndMetadata(t *testing.T) { + f := newFakeS3(t) + s := newTestStorer(t, f).Scoped("acme/widgets") + body := []byte("image") + putTestSnapshot(t, s, testSnapshotKey, body, digest(body)) + + f.mu.Lock() + defer f.mu.Unlock() + o, ok := f.objs["acme/widgets/"+testSnapshotKey] + if !ok { + var keys []string + for k := range f.objs { + keys = append(keys, k) + } + t.Fatalf("no object under the scoped key; have %v", keys) + } + if !bytes.Equal(o.body, body) { + t.Errorf("stored body %q, want %q", o.body, body) + } + if got := o.meta[snapshot.MetaSHA256]; got != digest(body) { + t.Errorf("stored %s = %q, want %q", snapshot.MetaSHA256, got, digest(body)) + } +} + +func TestStatSnapshot(t *testing.T) { + tests := []struct { + name string + put bool + wantErr error + }{ + {"present", true, nil}, + {"absent", false, fs.ErrNotExist}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := newTestStorer(t, newFakeS3(t)) + if tt.put { + putTestSnapshot(t, s, testSnapshotKey, []byte("12345"), digest([]byte("12345"))) + } + size, err := s.StatSnapshot(context.Background(), testSnapshotKey) + if tt.wantErr != nil { + if !errors.Is(err, tt.wantErr) { + t.Fatalf("err = %v, want %v", err, tt.wantErr) + } + return + } + if err != nil || size != 5 { + t.Fatalf("StatSnapshot = %d, %v; want 5, nil", size, err) + } + }) + } +} + +// errAny marks a test case that expects some error, of no particular kind. +var errAny = errors.New("any error") + +func TestOpenSnapshot(t *testing.T) { + body := []byte("an erofs image, pretend") + tests := []struct { + name string + cache bool + sum string + put bool + wantErr error // nil is success; errAny is any error + }{ + {"cached", true, digest(body), true, nil}, + {"no cache", false, digest(body), true, nil}, + {"bad digest", true, digest([]byte("other")), true, errAny}, + {"bad digest no cache", false, digest([]byte("other")), true, errAny}, + {"missing digest", true, "", true, errAny}, + {"missing digest no cache", false, "", true, errAny}, + {"absent", true, "", false, fs.ErrNotExist}, + {"absent no cache", false, "", false, fs.ErrNotExist}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var opts []Option + var c *PackCache + if tt.cache { + var err error + c, err = NewSnapshotCache(t.TempDir(), 1<<20, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { c.Cleanup() }) + opts = append(opts, WithSnapshotCache(c)) + } + s := newTestStorer(t, newFakeS3(t), opts...) + if tt.put { + putTestSnapshot(t, s, testSnapshotKey, body, tt.sum) + } + f, err := s.OpenSnapshot(context.Background(), testSnapshotKey) + switch { + case tt.wantErr == errAny: + if err == nil { + f.Close() + t.Fatal("OpenSnapshot succeeded, want an error") + } + if c != nil && countCacheFiles(t, c) != 0 { + t.Error("the cache kept a file after a failed download") + } + return + case tt.wantErr != nil: + if !errors.Is(err, tt.wantErr) { + t.Fatalf("err = %v, want %v", err, tt.wantErr) + } + return + case err != nil: + t.Fatalf("OpenSnapshot: %v", err) + } + defer f.Close() + got := make([]byte, len(body)) + if _, err := f.ReadAt(got, 0); err != nil && !errors.Is(err, io.EOF) { + t.Fatalf("ReadAt: %v", err) + } + if !bytes.Equal(got, body) { + t.Errorf("read %q, want %q", got, body) + } + }) + } +} + +// TestOpenSnapshotCachesDownload pins that the cache is shared: two opens in +// one repository, and one in another repository with the same tree, send one +// GetObject between them. +func TestOpenSnapshotCachesDownload(t *testing.T) { + c, err := NewSnapshotCache(t.TempDir(), 1<<20, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { c.Cleanup() }) + obs, counts := countingObserver() + base := newTestStorer(t, newFakeS3(t), WithSnapshotCache(c), obs) + widgets, gadgets := base.Scoped("acme/widgets"), base.Scoped("acme/gadgets") + + body := []byte("shared tree image") + putTestSnapshot(t, widgets, testSnapshotKey, body, digest(body)) + putTestSnapshot(t, gadgets, testSnapshotKey, body, digest(body)) + + for _, s := range []*Storer{widgets, widgets, gadgets} { + sf, err := s.OpenSnapshot(context.Background(), testSnapshotKey) + if err != nil { + t.Fatalf("OpenSnapshot: %v", err) + } + sf.Close() + } + if got := counts()["GetObject"]; got != 1 { + t.Errorf("GetObject calls = %d, want 1", got) + } +} + +func TestSnapshotKeysInvisible(t *testing.T) { + s := newTestStorer(t, newFakeS3(t)) + body := []byte("x") + putTestSnapshot(t, s, testSnapshotKey, body, digest(body)) + + refs, err := s.IterReferences() + if err != nil { + t.Fatal(err) + } + n := 0 + if err := refs.ForEach(func(*plumbing.Reference) error { n++; return nil }); err != nil { + t.Fatal(err) + } + if n != 0 { + t.Errorf("IterReferences returned %d refs, want 0", n) + } + + objs, err := s.IterEncodedObjects(plumbing.AnyObject) + if err != nil { + t.Fatal(err) + } + n = 0 + if err := objs.ForEach(func(plumbing.EncodedObject) error { n++; return nil }); err != nil { + t.Fatal(err) + } + if n != 0 { + t.Errorf("IterEncodedObjects returned %d objects, want 0", n) + } +} diff --git a/internal/storage/tigris/tigris.go b/internal/storage/tigris/tigris.go index ef6d608..d39fe38 100644 --- a/internal/storage/tigris/tigris.go +++ b/internal/storage/tigris/tigris.go @@ -120,6 +120,7 @@ type Storer struct { // Reading it is unconditional — see WithPackedRefs for why the two differ. packedRefs bool cache *PackCache // optional process-wide local pack cache; see packcache.go + snapCache *PackCache // optional process-wide snapshot image cache; see snapshot.go // fetchSem bounds whole-pack downloads in flight; see maxLivePackFetches. // Scoped shares it rather than replacing it, so one root Storer's // descendants — every repository, in production — share one budget. @@ -172,6 +173,13 @@ func WithPackCache(c *PackCache) Option { return func(s *Storer) { s.cache = c } } +// WithSnapshotCache installs the local cache that OpenSnapshot downloads +// images into. Like WithPackCache, pass one cache built once per process — +// see NewSnapshotCache. Scoped shares it. +func WithSnapshotCache(c *PackCache) Option { + return func(s *Storer) { s.snapCache = c } +} + // WithPackCompression controls whether newly written pack containers store // zstd payloads. Reading them is never gated: a Storer resolves compressed and // raw containers alike no matter how this is set. @@ -308,10 +316,13 @@ var _ storer.PackfileWriter = (*Storer)(nil) // Every one of those is a pointer field, so the struct copy above shares it // until it is replaced. Adding a cache to Storer means adding a line here. // -// Two things are shared on purpose. The pack cache, because sharing downloaded -// packs across requests is the whole point of it, and its keys are content -// hashes, so a hit is always the exact bytes the caller named. And fetchSem, -// because the bandwidth it rations is the process's, not one repository's. +// Three things are shared on purpose. The pack cache, because sharing +// downloaded packs across requests is the whole point of it, and its keys are +// content hashes, so a hit is always the exact bytes the caller named. The +// snapshot cache, for the same reason: its ids are tree hashes, and an image +// of one tree is the same tree in every repository that holds it. And +// fetchSem, because the bandwidth it rations is the process's, not one +// repository's. func (s *Storer) Scoped(prefix string) *Storer { cp := *s prefix = strings.Trim(prefix, "/")